Make calibre timezone aware

src/calibre/ebooks/conversion/plumber.py

@@ -12,6 +12,7 @@ from calibre.customize.ui import input_profiles, output_profiles, \
         run_plugins_on_preprocess, run_plugins_on_postprocess
 from calibre.ebooks.conversion.preprocess import HTMLPreProcessor
 from calibre.ptempfile import PersistentTemporaryDirectory
+from calibre.utils.date import parse_date
 from calibre import extract, walk
 
 DEBUG_README=u'''
@@ -65,7 +66,7 @@ class Plumber(object):
     metadata_option_names = [
         'title', 'authors', 'title_sort', 'author_sort', 'cover', 'comments',
         'publisher', 'series', 'series_index', 'rating', 'isbn',
-        'tags', 'book_producer', 'language'
+        'tags', 'book_producer', 'language', 'pubdate', 'timestamp'
         ]
 
     def __init__(self, input, output, log, report_progress=DummyReporter(),
@@ -461,6 +462,14 @@ OptionRecommendation(name='language',
     recommended_value=None, level=OptionRecommendation.LOW,
     help=_('Set the language.')),
 
+OptionRecommendation(name='pubdate',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('Set the publication date.')),
+
+OptionRecommendation(name='timestamp',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('Set the book timestamp (used by the date column in calibre).')),
+
 ]
 
         input_fmt = os.path.splitext(self.input)[1]
@@ -619,6 +628,14 @@ OptionRecommendation(name='language',
                     except ValueError:
                         self.log.warn(_('Values of series index and rating must'
                         ' be numbers. Ignoring'), val)
+                        continue
+                elif x in ('timestamp', 'pubdate'):
+                    try:
+                        val = parse_date(val, assume_utc=x=='pubdate')
+                    except:
+                        self.log.exception(_('Failed to parse date/time') + ' ' +
+                                unicode(val))
+                        continue
                 setattr(mi, x, val)
 
 

src/calibre/ebooks/lrf/pylrs/pylrs.py

@@ -50,6 +50,7 @@ from pylrf import (LrfWriter, LrfObject, LrfTag, LrfToc,
         STREAM_COMPRESSED, LrfTagStream, LrfStreamBase, IMAGE_TYPE_ENCODING,
         BINDING_DIRECTION_ENCODING, LINE_TYPE_ENCODING, LrfFileStream,
         STREAM_FORCE_COMPRESSED)
+from calibre.utils.date import isoformat
 
 DEFAULT_SOURCE_ENCODING = "cp1252"      # defualt is us-windows character set
 DEFAULT_GENREADING      = "fs"          # default is yes to both lrf and lrs
@@ -852,7 +853,7 @@ class DocInfo(object):
         self.thumbnail = None
         self.language = "en"
         self.creator  = None
-        self.creationdate = date.today().isoformat()
+        self.creationdate = str(isoformat(date.today()))
         self.producer = "%s v%s"%(__appname__, __version__)
         self.numberofpages = "0"
 

src/calibre/ebooks/metadata/__init__.py

@@ -13,6 +13,7 @@ from urlparse import urlparse
 from calibre import relpath
 
 from calibre.utils.config import tweaks
+from calibre.utils.date import isoformat
 
 _author_pat = re.compile(',?\s+(and|with)\s+', re.IGNORECASE)
 def string_to_authors(raw):
@@ -344,9 +345,9 @@ class MetaInformation(object):
         if self.rating is not None:
             fmt('Rating', self.rating)
         if self.timestamp is not None:
-            fmt('Timestamp', self.timestamp.isoformat(' '))
+            fmt('Timestamp', isoformat(self.timestamp))
         if self.pubdate is not None:
-            fmt('Published', self.pubdate.isoformat(' '))
+            fmt('Published', isoformat(self.pubdate))
         if self.rights is not None:
             fmt('Rights', unicode(self.rights))
         if self.lccn:

src/calibre/ebooks/metadata/amazon.py

@@ -10,9 +10,9 @@ import sys, re
 from datetime import datetime
 
 from lxml import etree
-from dateutil import parser
 
 from calibre import browser
+from calibre.utils.date import parse_date
 from calibre.ebooks.metadata import MetaInformation, string_to_authors
 
 AWS_NS = 'http://webservices.amazon.com/AWSECommerceService/2005-10-05'
@@ -44,9 +44,8 @@ def get_social_metadata(title, authors, publisher, isbn):
         try:
             d = root.findtext('.//'+AWS('PublicationDate'))
             if d:
-                default = datetime.utcnow()
+                default = datetime.utcnow().replace(day=15)
-                default = datetime(default.year, default.month, 15)
+                d = parse_date(d[0].text, assume_utc=True, default=default)
-                d = parser.parse(d[0].text, default=default)
                 mi.pubdate = d
         except:
             pass

src/calibre/ebooks/metadata/google_books.py

@@ -9,11 +9,11 @@ from functools import partial
 from datetime import datetime
 
 from lxml import etree
-from dateutil import parser
 
 from calibre import browser, preferred_encoding
 from calibre.ebooks.metadata import MetaInformation
 from calibre.utils.config import OptionParser
+from calibre.utils.date import parse_date
 
 NAMESPACES = {
               'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
@@ -156,9 +156,8 @@ class ResultList(list):
         try:
             d = date(entry)
             if d:
-                default = datetime.utcnow()
+                default = datetime.utcnow().replace(day=15)
-                default = datetime(default.year, default.month, 15)
+                d = parse_date(d[0].text, assume_utc=True, default=default)
-                d = parser.parse(d[0].text, default=default)
             else:
                 d = None
         except:

src/calibre/ebooks/metadata/opf2.py

@@ -12,12 +12,12 @@ from urllib import unquote
 from urlparse import urlparse
 
 from lxml import etree
-from dateutil import parser
 
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.constants import __appname__, __version__, filesystem_encoding
 from calibre.ebooks.metadata.toc import TOC
 from calibre.ebooks.metadata import MetaInformation, string_to_authors
+from calibre.utils.date import parse_date, isoformat
 
 
 class Resource(object):
@@ -449,9 +449,10 @@ class OPF(object):
     series          = MetadataField('series', is_dc=False)
     series_index    = MetadataField('series_index', is_dc=False, formatter=float, none_is=1)
     rating          = MetadataField('rating', is_dc=False, formatter=int)
-    pubdate         = MetadataField('date', formatter=parser.parse)
+    pubdate         = MetadataField('date', formatter=parse_date)
     publication_type = MetadataField('publication_type', is_dc=False)
-    timestamp       = MetadataField('timestamp', is_dc=False, formatter=parser.parse)
+    timestamp       = MetadataField('timestamp', is_dc=False,
+                                    formatter=parse_date)
 
 
     def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True):
@@ -1046,7 +1047,7 @@ def metadata_to_opf(mi, as_string=True):
         factory(DC('creator'), au, mi.author_sort, 'aut')
     factory(DC('contributor'), mi.book_producer, __appname__, 'bkp')
     if hasattr(mi.pubdate, 'isoformat'):
-        factory(DC('date'), mi.pubdate.isoformat())
+        factory(DC('date'), isoformat(mi.pubdate))
     factory(DC('language'), mi.language)
     if mi.category:
         factory(DC('type'), mi.category)
@@ -1069,7 +1070,7 @@ def metadata_to_opf(mi, as_string=True):
     if mi.rating is not None:
         meta('rating', str(mi.rating))
     if hasattr(mi.timestamp, 'isoformat'):
-        meta('timestamp', mi.timestamp.isoformat())
+        meta('timestamp', isoformat(mi.timestamp))
     if mi.publication_type:
         meta('publication_type', mi.publication_type)
 

src/calibre/ebooks/oeb/transforms/metadata.py

@@ -7,7 +7,7 @@ __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import os
-from datetime import datetime
+from calibre.utils.date import isoformat, now
 
 def meta_info_to_oeb_metadata(mi, m, log):
     from calibre.ebooks.oeb.base import OPF
@@ -60,10 +60,10 @@ def meta_info_to_oeb_metadata(mi, m, log):
             m.add('subject', t)
     if mi.pubdate is not None:
         m.clear('date')
-        m.add('date', mi.pubdate.isoformat())
+        m.add('date', isoformat(mi.pubdate))
     if mi.timestamp is not None:
         m.clear('timestamp')
-        m.add('timestamp', mi.timestamp.isoformat())
+        m.add('timestamp', isoformat(mi.timestamp))
     if mi.rights is not None:
         m.clear('rights')
         m.add('rights', mi.rights)
@@ -71,7 +71,7 @@ def meta_info_to_oeb_metadata(mi, m, log):
         m.clear('publication_type')
         m.add('publication_type', mi.publication_type)
     if not m.timestamp:
-        m.add('timestamp', datetime.now().isoformat())
+        m.add('timestamp', isoformat(now()))
 
 
 class MergeMetadata(object):

src/calibre/gui2/dialogs/scheduler.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 Scheduler for automated recipe downloads
 '''
 
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from PyQt4.Qt import QDialog, SIGNAL, Qt, QTime, QObject, QMenu, \
         QAction, QIcon, QMutex, QTimer
@@ -17,6 +17,7 @@ from calibre.gui2.search_box import SearchBox2
 from calibre.gui2 import config as gconf, error_dialog
 from calibre.web.feeds.recipes.model import RecipeModel
 from calibre.ptempfile import PersistentTemporaryFile
+from calibre.utils.date import utcnow
 
 class SchedulerDialog(QDialog, Ui_Dialog):
 
@@ -185,7 +186,7 @@ class SchedulerDialog(QDialog, Ui_Dialog):
                 self.day.setCurrentIndex(day+1)
                 self.time.setTime(QTime(hour, minute))
 
-            d = datetime.utcnow() - last_downloaded
+            d = utcnow() - last_downloaded
             def hm(x): return (x-x%3600)//3600, (x%3600 - (x%3600)%60)//60
             hours, minutes = hm(d.seconds)
             tm = _('%d days, %d hours and %d minutes ago')%(d.days, hours, minutes)

src/calibre/library/catalog.py

@@ -10,6 +10,7 @@ from calibre.customize.conversion import OptionRecommendation, DummyReporter
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString
 from calibre.ptempfile import PersistentTemporaryDirectory
 from calibre.utils.logging import Log
+from calibre.utils.date import isoformat
 
 FIELDS = ['all', 'author_sort', 'authors', 'comments',
           'cover', 'formats', 'id', 'isbn', 'pubdate', 'publisher', 'rating',
@@ -102,7 +103,9 @@ class CSV_XML(CatalogPlugin):
                         item = ', '.join(item)
                     elif field == 'isbn':
                         # Could be 9, 10 or 13 digits
-                        field = u'%s' % re.sub(r'[\D]','',field)
+                        item = u'%s' % re.sub(r'[\D]', '', item)
+                    elif field in ['pubdate', 'timestamp']:
+                        item = isoformat(item)
 
                     if x < len(fields) - 1:
                         if item is not None:
@@ -163,12 +166,12 @@ class CSV_XML(CatalogPlugin):
             if 'date' in fields:
                 record_child = etree.SubElement(record, 'date')
                 record_child.set(PY + "if", "record['date']")
-                record_child.text = "${record['date']}"
+                record_child.text = "${record['date'].isoformat()}"
 
             if 'pubdate' in fields:
                 record_child = etree.SubElement(record, 'pubdate')
                 record_child.set(PY + "if", "record['pubdate']")
-                record_child.text = "${record['pubdate']}"
+                record_child.text = "${record['pubdate'].isoformat()}"
 
             if 'size' in fields:
                 record_child = etree.SubElement(record, 'size')

src/calibre/library/cli.py

@@ -17,6 +17,7 @@ from calibre.ebooks.metadata.meta import get_metadata
 from calibre.library.database2 import LibraryDatabase2
 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
 from calibre.utils.genshi.template import MarkupTemplate
+from calibre.utils.date import isoformat
 
 FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
     'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
@@ -37,8 +38,8 @@ XML_TEMPLATE = '''\
         </authors>
         <publisher>${record['publisher']}</publisher>
         <rating>${record['rating']}</rating>
-        <date>${record['timestamp']}</date>
+        <date>${record['timestamp'].isoformat()}</date>
-        <pubdate>${record['pubdate']}</pubdate>
+        <pubdate>${record['pubdate'].isoformat()}</pubdate>
         <size>${record['size']}</size>
         <tags py:if="record['tags']">
         <py:for each="tag in record['tags']">
@@ -68,7 +69,7 @@ STANZA_TEMPLATE='''\
     <uri>http://calibre-ebook.com</uri>
   </author>
   <id>$id</id>
-  <updated>${updated.strftime('%Y-%m-%dT%H:%M:%SZ')}</updated>
+  <updated>${updated.isoformat()}</updated>
   <subtitle>
         ${subtitle}
   </subtitle>
@@ -77,7 +78,7 @@ STANZA_TEMPLATE='''\
       <title>${record['title']}</title>
       <id>urn:calibre:${record['uuid']}</id>
       <author><name>${record['author_sort']}</name></author>
-      <updated>${record['timestamp'].strftime('%Y-%m-%dT%H:%M:%SZ')}</updated>
+      <updated>${record['timestamp'].isoformat()}</updated>
       <link type="application/epub+zip" href="${quote(record['fmt_epub'].replace(sep, '/'))}"/>
       <link py:if="record['cover']" rel="x-stanza-cover-image" type="image/png" href="${quote(record['cover'].replace(sep, '/'))}"/>
       <link py:if="record['cover']" rel="x-stanza-cover-image-thumbnail" type="image/png" href="${quote(record['cover'].replace(sep, '/'))}"/>
@@ -144,6 +145,9 @@ def do_list(db, fields, sort_by, ascending, search_text, line_width, separator,
         widths = list(map(lambda x : 0, fields))
         for record in data:
             for f in record.keys():
+                if hasattr(record[f], 'isoformat'):
+                    record[f] = isoformat(record[f], as_utc=False)
+                else:
                     record[f] = unicode(record[f])
                 record[f] = record[f].replace('\n', ' ')
         for i in data:

src/calibre/library/database2.py

@@ -9,7 +9,6 @@ The database used to store ebook metadata
 import os, re, sys, shutil, cStringIO, glob, collections, textwrap, \
        itertools, functools, traceback
 from itertools import repeat
-from datetime import datetime
 from math import floor
 
 from PyQt4.QtCore import QThread, QReadWriteLock
@@ -34,6 +33,7 @@ from calibre.ptempfile import PersistentTemporaryFile
 from calibre.customize.ui import run_plugins_on_import
 
 from calibre.utils.filenames import ascii_filename
+from calibre.utils.date import utcnow, now as nowf, utcfromtimestamp
 from calibre.ebooks import BOOK_EXTENSIONS
 
 if iswindows:
@@ -715,12 +715,12 @@ class LibraryDatabase2(LibraryDatabase):
 
     def last_modified(self):
         ''' Return last modified time as a UTC datetime object'''
-        return datetime.utcfromtimestamp(os.stat(self.dbpath).st_mtime)
+        return utcfromtimestamp(os.stat(self.dbpath).st_mtime)
 
     def check_if_modified(self):
         if self.last_modified() > self.last_update_check:
             self.refresh()
-        self.last_update_check = datetime.utcnow()
+        self.last_update_check = utcnow()
 
     def path(self, index, index_is_id=False):
         'Return the relative path to the directory containing this books files as a unicode string.'
@@ -1123,7 +1123,7 @@ class LibraryDatabase2(LibraryDatabase):
 
     def tags_older_than(self, tag, delta):
         tag = tag.lower().strip()
-        now = datetime.now()
+        now = nowf()
         for r in self.data._data:
             if r is not None:
                 if (now - r[FIELD_MAP['timestamp']]) > delta:
@@ -1484,7 +1484,7 @@ class LibraryDatabase2(LibraryDatabase):
             stream.close()
         self.conn.commit()
         if existing:
-            t = datetime.utcnow()
+            t = utcnow()
             self.set_timestamp(db_id, t, notify=False)
             self.set_pubdate(db_id, t, notify=False)
         self.data.refresh_ids(self, [db_id]) # Needed to update format list and size

src/calibre/library/sqlite.py

@@ -12,46 +12,18 @@ from sqlite3 import IntegrityError, OperationalError
 from threading import Thread
 from Queue import Queue
 from threading import RLock
-from datetime import tzinfo, datetime, timedelta
+from datetime import datetime
 
 from calibre.ebooks.metadata import title_sort
+from calibre.utils.date import parse_date, isoformat
 
 global_lock = RLock()
 
 def convert_timestamp(val):
-    datepart, timepart = val.split(' ')
+    return parse_date(val, as_utc=False)
-    tz, mult = None, 1
-    x = timepart.split('+')
-    if len(x) > 1:
-        timepart, tz = x
-    else:
-        x = timepart.split('-')
-        if len(x) > 1:
-            timepart, tz = x
-            mult = -1
-
-    year, month, day = map(int, datepart.split("-"))
-    timepart_full = timepart.split(".")
-    hours, minutes, seconds = map(int, timepart_full[0].split(":"))
-    if len(timepart_full) == 2:
-        microseconds = int(timepart_full[1])
-    else:
-        microseconds = 0
-    if tz is not None:
-        h, m = map(int, tz.split(':'))
-        delta = timedelta(minutes=mult*(60*h + m))
-        tz = type('CustomTZ', (tzinfo,), {'utcoffset':lambda self, dt:delta,
-                                          'dst':lambda self,dt:timedelta(0)})()
-
-    val = datetime(year, month, day, hours, minutes, seconds, microseconds,
-                   tzinfo=tz)
-    if tz is not None:
-        val = datetime(*(val.utctimetuple()[:6]))
-    return val
 
 def adapt_datetime(dt):
-    dt = datetime(*(dt.utctimetuple()[:6]))
+    return isoformat(dt)
-    return dt.isoformat(' ')
 
 sqlite.register_adapter(datetime, adapt_datetime)
 sqlite.register_converter('timestamp', convert_timestamp)

src/calibre/utils/date.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from datetime import datetime
+
+from dateutil.parser import parse
+from dateutil.tz import tzlocal, tzutc
+
+_utc_tz = tzutc()
+_local_tz = tzlocal()
+
+def parse_date(date_string, assume_utc=False, as_utc=True, default=None):
+    '''
+    Parse a date/time string into a timezone aware datetime object. The timezone
+    is always either UTC or the local timezone.
+
+    :param assume_utc: If True and date_string does not specify a timezone,
+    assume UTC, otherwise assume local timezone.
+
+    :param as_utc: If True, return a UTC datetime
+
+    :param default: Missing fields are filled in from default. If None, the
+    current date is used.
+    '''
+    if default is None:
+        func = datetime.utcnow if assume_utc else datetime.now
+        default = func().replace(hour=0, minute=0, second=0, microsecond=0,
+                tzinfo=_utc_tz if assume_utc else _local_tz)
+    dt = parse(date_string, default=default)
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=_utc_tz if assume_utc else _local_tz)
+    dt = dt.astimezone(_utc_tz if as_utc else _local_tz)
+    return dt
+
+def isoformat(date_time, assume_utc=False, as_utc=True):
+    if not hasattr(date_time, 'tzinfo'):
+        return unicode(date_time.isoformat())
+    if date_time.tzinfo is None:
+        date_time = date_time.replace(tzinfo=_utc_tz if assume_utc else
+                _local_tz)
+    date_time = date_time.astimezone(_utc_tz if as_utc else _local_tz)
+    return unicode(date_time.isoformat())
+
+def now():
+    return datetime.now().replace(tzinfo=_local_tz)
+
+def utcnow():
+    return datetime.utcnow().replace(tzinfo=_utc_tz)
+
+def utcfromtimestamp(stamp):
+    return datetime.utcfromtimestamp(stamp).replace(tzinfo=_utc_tz)

src/calibre/web/feeds/recipes/collection.py

@@ -12,9 +12,10 @@ from datetime import datetime, timedelta
 
 from lxml import etree
 from lxml.builder import ElementMaker
-from dateutil import parser
 
 from calibre import browser
+from calibre.utils.date import parse_date, now as nowf, utcnow, tzlocal, \
+        isoformat
 
 NS = 'http://calibre-ebook.com/recipe_collection'
 E = ElementMaker(namespace=NS, nsmap={None:NS})
@@ -125,7 +126,12 @@ class SchedulerConfig(object):
         self.lock = RLock()
         if os.access(self.conf_path, os.R_OK):
             with ExclusiveFile(self.conf_path) as f:
+                try:
                     self.root = etree.fromstring(f.read())
+                except:
+                    print 'Failed to read recipe scheduler config'
+                    import traceback
+                    traceback.print_exc()
         elif os.path.exists(old_conf_path):
             self.migrate_old_conf(old_conf_path)
 
@@ -151,7 +157,7 @@ class SchedulerConfig(object):
                     ld = x.get('last_downloaded', None)
                     if ld and last_downloaded is None:
                         try:
-                            last_downloaded = parser.parse(ld)
+                            last_downloaded = parse_date(ld)
                         except:
                             pass
                     self.root.remove(x)
@@ -161,7 +167,7 @@ class SchedulerConfig(object):
             sr = E.scheduled_recipe({
                 'id' : recipe.get('id'),
                 'title': recipe.get('title'),
-                'last_downloaded':last_downloaded.isoformat(),
+                'last_downloaded':isoformat(last_downloaded),
                 }, self.serialize_schedule(schedule_type, schedule))
             self.root.append(sr)
             self.write_scheduler_file()
@@ -189,7 +195,7 @@ class SchedulerConfig(object):
 
     def update_last_downloaded(self, recipe_id):
         with self.lock:
-            now = datetime.utcnow()
+            now = utcnow()
             for x in self.iter_recipes():
                 if x.get('id', False) == recipe_id:
                     typ, sch, last_downloaded = self.un_serialize_schedule(x)
@@ -199,7 +205,7 @@ class SchedulerConfig(object):
                         if abs(actual_interval - nominal_interval) < \
                                 timedelta(hours=1):
                             now = last_downloaded + nominal_interval
-                    x.set('last_downloaded', now.isoformat())
+                    x.set('last_downloaded', isoformat(now))
                     break
             self.write_scheduler_file()
 
@@ -243,20 +249,18 @@ class SchedulerConfig(object):
                     sch = float(sch)
                 elif typ == 'day/time':
                     sch = list(map(int, sch.split(':')))
-                return typ, sch, parser.parse(recipe.get('last_downloaded'))
+                return typ, sch, parse_date(recipe.get('last_downloaded'))
 
     def recipe_needs_to_be_downloaded(self, recipe):
         try:
             typ, sch, ld = self.un_serialize_schedule(recipe)
         except:
             return False
-        utcnow = datetime.utcnow()
         if typ == 'interval':
-            return utcnow - ld > timedelta(sch)
+            return utcnow() - ld > timedelta(sch)
         elif typ == 'day/time':
-            now = datetime.now()
+            now = nowf()
-            offset = now - utcnow
+            ld_local = ld.astimezone(tzlocal())
-            ld_local = ld + offset
             day, hour, minute = sch
 
             is_today = day < 0 or day > 6 or \