More timezone aware fixes

src/calibre/ebooks/metadata/amazon.py

@@ -7,12 +7,11 @@ __docformat__ = 'restructuredtext en'
 Fetch metadata using Amazon AWS
 '''
 import sys, re
-from datetime import datetime
 
 from lxml import etree
 
 from calibre import browser
-from calibre.utils.date import parse_date
+from calibre.utils.date import parse_date, utcnow
 from calibre.ebooks.metadata import MetaInformation, string_to_authors
 
 AWS_NS = 'http://webservices.amazon.com/AWSECommerceService/2005-10-05'
@@ -44,7 +43,7 @@ def get_social_metadata(title, authors, publisher, isbn):
         try:
             d = root.findtext('.//'+AWS('PublicationDate'))
             if d:
-                default = datetime.utcnow().replace(day=15)
+                default = utcnow().replace(day=15)
                 d = parse_date(d[0].text, assume_utc=True, default=default)
                 mi.pubdate = d
         except:

src/calibre/ebooks/metadata/google_books.py

@@ -6,14 +6,13 @@ __docformat__ = 'restructuredtext en'
 import sys, textwrap
 from urllib import urlencode
 from functools import partial
-from datetime import datetime
 
 from lxml import etree
 
 from calibre import browser, preferred_encoding
 from calibre.ebooks.metadata import MetaInformation
 from calibre.utils.config import OptionParser
-from calibre.utils.date import parse_date
+from calibre.utils.date import parse_date, utcnow
 
 NAMESPACES = {
               'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
@@ -156,7 +155,7 @@ class ResultList(list):
         try:
             d = date(entry)
             if d:
-                default = datetime.utcnow().replace(day=15)
+                default = utcnow().replace(day=15)
                 d = parse_date(d[0].text, assume_utc=True, default=default)
             else:
                 d = None

src/calibre/ebooks/metadata/mobi.py

@@ -11,11 +11,11 @@ __docformat__ = 'restructuredtext en'
 
 from struct import pack, unpack
 from cStringIO import StringIO
-from datetime import datetime
 
 from calibre.ebooks.mobi import MobiError
 from calibre.ebooks.mobi.writer import rescale_image, MAX_THUMB_DIMEN
 from calibre.ebooks.mobi.langcodes import iana2mobi
+from calibre.utils.date import now as nowf
 
 class StreamSlicer(object):
 
@@ -331,7 +331,7 @@ class MetadataUpdater(object):
             recs.append((106, self.timestamp))
             pop_exth_record(106)
         else:
-            recs.append((106, str(datetime.now()).encode(self.codec, 'replace')))
+            recs.append((106, nowf().isoformat().encode(self.codec, 'replace')))
             pop_exth_record(106)
         if self.cover_record is not None:
             recs.append((201, pack('>I', self.cover_rindex)))

src/calibre/ebooks/mobi/reader.py

@@ -4,13 +4,11 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Read data from .mobi files
 '''
 
-import datetime
 import functools
 import os
 import re
 import struct
 import textwrap
-
 import cStringIO
 
 try:
@@ -23,6 +21,7 @@ from lxml import html, etree
 
 from calibre import entity_to_unicode, CurrentDir
 from calibre.utils.filenames import ascii_filename
+from calibre.utils.date import parse_date
 from calibre.ptempfile import TemporaryDirectory
 from calibre.ebooks import DRMError
 from calibre.ebooks.chardet import ENCODING_PATS
@@ -96,8 +95,7 @@ class EXTHHeader(object):
             self.mi.tags = list(set(self.mi.tags))
         elif id == 106:
             try:
-                self.mi.publish_date = datetime.datetime.strptime(
+                self.mi.pubdate = parse_date(content, as_utc=False)
-                    content, '%Y-%m-%d', ).date()
             except:
                 pass
         elif id == 108:

src/calibre/gui2/dialogs/metadata_single.py

@@ -10,7 +10,6 @@ import os
 import re
 import time
 import traceback
-from datetime import datetime, timedelta
 
 from PyQt4.Qt import SIGNAL, QObject, QCoreApplication, Qt, QTimer, QThread, QDate, \
     QPixmap, QListWidgetItem, QDialog
@@ -29,6 +28,7 @@ from calibre.ebooks.metadata.library_thing import cover_from_isbn
 from calibre import islinux
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.utils.config import prefs, tweaks
+from calibre.utils.date import qt_to_dt
 from calibre.customize.ui import run_plugins_on_import, get_isbndb_key
 from calibre.gui2.dialogs.config.social import SocialMetadata
 
@@ -354,12 +354,9 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         self.comments.setPlainText(comments if comments else '')
         cover = self.db.cover(row)
         pubdate = db.pubdate(self.id, index_is_id=True)
-        self.local_timezone_offset = timedelta(seconds=time.timezone) - timedelta(hours=time.daylight)
-        pubdate = pubdate - self.local_timezone_offset
         self.pubdate.setDate(QDate(pubdate.year, pubdate.month,
             pubdate.day))
         timestamp = db.timestamp(self.id, index_is_id=True)
-        timestamp = timestamp - self.local_timezone_offset
         self.date.setDate(QDate(timestamp.year, timestamp.month,
             timestamp.day))
 
@@ -583,7 +580,6 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                         if book.isbn: self.isbn.setText(book.isbn)
                         if book.pubdate:
                             d = book.pubdate
-                            d = d - self.local_timezone_offset
                             self.pubdate.setDate(QDate(d.year, d.month, d.day))
                         summ = book.comments
                         if summ:
@@ -656,12 +652,10 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
             self.db.set_series_index(self.id, self.series_index.value(), notify=False)
             self.db.set_comment(self.id, qstring_to_unicode(self.comments.toPlainText()), notify=False)
             d = self.pubdate.date()
-            d = datetime(d.year(), d.month(), d.day())
+            d = qt_to_dt(d)
-            d = d + self.local_timezone_offset
             self.db.set_pubdate(self.id, d)
             d = self.date.date()
-            d = datetime(d.year(), d.month(), d.day())
+            d = qt_to_dt(d)
-            d = d + self.local_timezone_offset
             self.db.set_timestamp(self.id, d)
 
             if self.cover_changed:

src/calibre/gui2/library.py

@@ -1,8 +1,7 @@
 from calibre.ebooks.metadata import authors_to_string
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import os, textwrap, traceback, time, re
+import os, textwrap, traceback, re
-from datetime import timedelta, datetime
 from operator import attrgetter
 
 from math import cos, sin, pi
@@ -25,6 +24,7 @@ from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.ebooks.metadata import string_to_authors, fmt_sidx
 from calibre.utils.config import tweaks
+from calibre.utils.date import dt_factory, qt_to_dt, isoformat
 
 class LibraryDelegate(QItemDelegate):
     COLOR    = QColor("blue")
@@ -567,13 +567,11 @@ class BooksModel(QAbstractTableModel):
         def timestamp(r):
             dt = self.db.data[r][tmdx]
             if dt:
-                dt = dt - timedelta(seconds=time.timezone) + timedelta(hours=time.daylight)
                 return QDate(dt.year, dt.month, dt.day)
 
         def pubdate(r):
             dt = self.db.data[r][pddx]
             if dt:
-                dt = dt - timedelta(seconds=time.timezone) + timedelta(hours=time.daylight)
                 return QDate(dt.year, dt.month, dt.day)
 
         def rating(r):
@@ -670,13 +668,11 @@ class BooksModel(QAbstractTableModel):
             elif column == 'timestamp':
                 if val.isNull() or not val.isValid():
                     return False
-                dt = datetime(val.year(), val.month(), val.day()) + timedelta(seconds=time.timezone) - timedelta(hours=time.daylight)
+                self.db.set_timestamp(id, qt_to_dt(val, as_utc=False))
-                self.db.set_timestamp(id, dt)
             elif column == 'pubdate':
                 if val.isNull() or not val.isValid():
                     return False
-                dt = datetime(val.year(), val.month(), val.day()) + timedelta(seconds=time.timezone) - timedelta(hours=time.daylight)
+                self.db.set_pubdate(id, qt_to_dt(val, as_utc=False))
-                self.db.set_pubdate(id, dt)
             else:
                 self.db.set(row, column, val)
             self.emit(SIGNAL("dataChanged(QModelIndex, QModelIndex)"), \
@@ -1032,7 +1028,8 @@ class DeviceBooksModel(BooksModel):
         def datecmp(x, y):
             x = self.db[x].datetime
             y = self.db[y].datetime
-            return cmp(datetime(*x[0:6]), datetime(*y[0:6]))
+            return cmp(dt_factory(x, assume_utc=True), dt_factory(y,
+                assume_utc=True))
         def sizecmp(x, y):
             x, y = int(self.db[x].size), int(self.db[y].size)
             return cmp(x, y)
@@ -1081,10 +1078,8 @@ class DeviceBooksModel(BooksModel):
             type = ext[1:].lower()
         data[_('Format')] = type
         data[_('Path')] = item.path
-        dt = item.datetime
+        dt = dt_factory(item.datetime, assume_utc=True)
-        dt = datetime(*dt[0:6])
+        data[_('Timestamp')] = isoformat(dt, sep=' ', as_utc=False)
-        dt = dt - timedelta(seconds=time.timezone) + timedelta(hours=time.daylight)
-        data[_('Timestamp')] = strftime('%a %b %d %H:%M:%S %Y', dt.timetuple())
         data[_('Tags')] = ', '.join(item.tags)
         self.emit(SIGNAL('new_bookdisplay_data(PyQt_PyObject)'), data)
 
@@ -1119,8 +1114,7 @@ class DeviceBooksModel(BooksModel):
                 return QVariant(BooksView.human_readable(size))
             elif col == 3:
                 dt = self.db[self.map[row]].datetime
-                dt = datetime(*dt[0:6])
+                dt = dt_factory(dt, assume_utc=True, as_utc=False)
-                dt = dt - timedelta(seconds=time.timezone) + timedelta(hours=time.daylight)
                 return QVariant(strftime(BooksView.TIME_FMT, dt.timetuple()))
             elif col == 4:
                 tags = self.db[self.map[row]].tags

src/calibre/library/server.py

@@ -11,7 +11,6 @@ import sys, textwrap, operator, os, re, logging, cStringIO
 import __builtin__
 from itertools import repeat
 from logging.handlers import RotatingFileHandler
-from datetime import datetime
 from threading import Thread
 
 import cherrypy
@@ -31,15 +30,16 @@ from calibre.utils.config import config_dir
 from calibre.utils.mdns import publish as publish_zeroconf, \
                                stop_server as stop_zeroconf
 from calibre.ebooks.metadata import fmt_sidx, title_sort
+from calibre.utils.date import now as nowf, fromtimestamp
 
 def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None):
     if not hasattr(dt, 'timetuple'):
-        dt = datetime.now()
+        dt = nowf()
     dt = dt.timetuple()
     try:
         return _strftime(fmt, dt)
     except:
-        return _strftime(fmt, datetime.now().timetuple())
+        return _strftime(fmt, nowf().timetuple())
 
 def expose(func):
 
@@ -351,7 +351,7 @@ class LibraryServer(object):
                         map(int, self.opts.max_cover.split('x'))
         self.max_stanza_items = opts.max_opds_items
         path = P('content_server')
-        self.build_time = datetime.fromtimestamp(os.stat(path).st_mtime)
+        self.build_time = fromtimestamp(os.stat(path).st_mtime)
         self.default_cover =  open(P('content_server/default_cover.jpg'), 'rb').read()
 
         cherrypy.config.update({
@@ -429,7 +429,7 @@ class LibraryServer(object):
         cherrypy.response.headers['Content-Type'] = 'image/jpeg'
         cherrypy.response.timeout = 3600
         path = getattr(cover, 'name', False)
-        updated = datetime.utcfromtimestamp(os.stat(path).st_mtime) if path and \
+        updated = fromtimestamp(os.stat(path).st_mtime) if path and \
             os.access(path, os.R_OK) else self.build_time
         cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
         try:
@@ -476,7 +476,7 @@ class LibraryServer(object):
         cherrypy.response.timeout = 3600
         path = getattr(fmt, 'name', None)
         if path and os.path.exists(path):
-            updated = datetime.utcfromtimestamp(os.stat(path).st_mtime)
+            updated = fromtimestamp(os.stat(path).st_mtime)
             cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
         return fmt.read()
 
@@ -841,7 +841,7 @@ class LibraryServer(object):
         if not os.path.exists(path):
             raise cherrypy.HTTPError(404, '%s not found'%name)
         if self.opts.develop:
-            lm = datetime.fromtimestamp(os.stat(path).st_mtime)
+            lm = fromtimestamp(os.stat(path).st_mtime)
             cherrypy.response.headers['Last-Modified'] = self.last_modified(lm)
         return open(path, 'rb').read()
 

src/calibre/utils/date.py

@@ -11,8 +11,8 @@ from datetime import datetime
 from dateutil.parser import parse
 from dateutil.tz import tzlocal, tzutc
 
-_utc_tz = tzutc()
+utc_tz = _utc_tz = tzutc()
-_local_tz = tzlocal()
+local_tz = _local_tz = tzlocal()
 
 def parse_date(date_string, assume_utc=False, as_utc=True, default=None):
     '''
@@ -34,17 +34,48 @@ def parse_date(date_string, assume_utc=False, as_utc=True, default=None):
     dt = parse(date_string, default=default)
     if dt.tzinfo is None:
         dt = dt.replace(tzinfo=_utc_tz if assume_utc else _local_tz)
-    dt = dt.astimezone(_utc_tz if as_utc else _local_tz)
+    return dt.astimezone(_utc_tz if as_utc else _local_tz)
+
+def strptime(val, fmt, assume_utc=False, as_utc=True):
+    dt = datetime.strptime(val, fmt)
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=_utc_tz if assume_utc else _local_tz)
+    return dt.astimezone(_utc_tz if as_utc else _local_tz)
+
+def dt_factory(time_t, assume_utc=False, as_utc=True):
+    dt = datetime(time_t[0:6])
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=_utc_tz if assume_utc else _local_tz)
+    return dt.astimezone(_utc_tz if as_utc else _local_tz)
+
+def qt_to_dt(qdate_or_qdatetime, as_utc=True):
+    from PyQt4.Qt import Qt
+    o = qdate_or_qdatetime
+    if hasattr(o, 'toUTC'):
+        # QDateTime
+        o = unicode(o.toUTC().toString(Qt.ISODate))
+        return parse_date(o, assume_utc=True, as_utc=as_utc)
+    dt = datetime(o.year(), o.month(), o.day()).replace(tzinfo=_local_tz)
+    return dt.astimezone(_utc_tz if as_utc else _local_tz)
+
+def fromtimestamp(ctime, as_utc=True):
+    dt = datetime.utcfromtimestamp().replace(tzinfo=_utc_tz)
+    if not as_utc:
+        dt = dt.astimezone(_local_tz)
     return dt
 
-def isoformat(date_time, assume_utc=False, as_utc=True):
+def fromordinal(day, as_utc=True):
+    return datetime.fromordinal(day).replace(
+            tzinfo=_utc_tz if as_utc else _local_tz)
+
+def isoformat(date_time, assume_utc=False, as_utc=True, sep='T'):
     if not hasattr(date_time, 'tzinfo'):
         return unicode(date_time.isoformat())
     if date_time.tzinfo is None:
         date_time = date_time.replace(tzinfo=_utc_tz if assume_utc else
                 _local_tz)
     date_time = date_time.astimezone(_utc_tz if as_utc else _local_tz)
-    return unicode(date_time.isoformat())
+    return unicode(date_time.isoformat(sep))
 
 def now():
     return datetime.now().replace(tzinfo=_local_tz)

src/calibre/web/feeds/__init__.py

@@ -6,17 +6,16 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Contains the logic for parsing feeds.
 '''
 import time, traceback, copy, re
-from datetime import datetime
+
+from lxml import html
 
 from calibre.web.feeds.feedparser import parse
 from calibre.utils.logging import default_log
 from calibre import entity_to_unicode
-from lxml import html
+from calibre.utils.date import dt_factory, utcnow, local_tz
 
 class Article(object):
 
-    time_offset = datetime.now() - datetime.utcnow()
-
     def __init__(self, id, title, url, author, summary, published, content):
         self.downloaded = False
         self.id = id
@@ -48,8 +47,8 @@ class Article(object):
         self.author = author
         self.content = content
         self.date = published
-        self.utctime = datetime(*self.date[:6])
+        self.utctime = dt_factory(self.date, assume_utc=True, as_utc=True)
-        self.localtime = self.utctime + self.time_offset
+        self.localtime = self.utctime.astimezone(local_tz)
 
     @dynamic_property
     def title(self):
@@ -146,7 +145,7 @@ class Feed(object):
             content     = item.get('content', '')
             author      = item.get('author', '')
             article = Article(id, title, link, author, description, published, content)
-            delta = datetime.utcnow() - article.utctime
+            delta = utcnow() - article.utctime
             if delta.days*24*3600 + delta.seconds <= 24*3600*self.oldest_article:
                 self.articles.append(article)
             else:
@@ -183,7 +182,7 @@ class Feed(object):
         if not link and not content:
             return
         article = Article(id, title, link, author, description, published, content)
-        delta = datetime.utcnow() - article.utctime
+        delta = utcnow() - article.utctime
         if delta.days*24*3600 + delta.seconds <= 24*3600*self.oldest_article:
             self.articles.append(article)
         else:

src/calibre/web/feeds/news.py

@@ -11,7 +11,6 @@ import os, time, traceback, re, urlparse, sys
 from collections import defaultdict
 from functools import partial
 from contextlib import nested, closing
-from datetime import datetime
 
 
 from calibre import browser, __appname__, iswindows, \
@@ -29,7 +28,7 @@ from calibre.web.fetch.simple import RecursiveFetcher
 from calibre.utils.threadpool import WorkRequest, ThreadPool, NoResultsPending
 from calibre.ptempfile import PersistentTemporaryFile, \
                               PersistentTemporaryDirectory
-
+from calibre.utils.date import now as nowf
 
 class BasicNewsRecipe(Recipe):
     '''
@@ -1080,11 +1079,11 @@ class BasicNewsRecipe(Recipe):
         mi.publisher = __appname__
         mi.author_sort = __appname__
         mi.publication_type = 'periodical:'+self.publication_type
-        mi.timestamp = datetime.now()
+        mi.timestamp = nowf()
         mi.comments = self.description
         if not isinstance(mi.comments, unicode):
             mi.comments = mi.comments.decode('utf-8', 'replace')
-        mi.pubdate = datetime.now()
+        mi.pubdate = nowf()
         opf_path = os.path.join(dir, 'index.opf')
         ncx_path = os.path.join(dir, 'index.ncx')
 

src/calibre/web/feeds/recipes/collection.py

@@ -8,14 +8,14 @@ __docformat__ = 'restructuredtext en'
 
 import os, calendar
 from threading import RLock
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from lxml import etree
 from lxml.builder import ElementMaker
 
 from calibre import browser
 from calibre.utils.date import parse_date, now as nowf, utcnow, tzlocal, \
-        isoformat
+        isoformat, fromordinal
 
 NS = 'http://calibre-ebook.com/recipe_collection'
 E = ElementMaker(namespace=NS, nsmap={None:NS})
@@ -163,7 +163,7 @@ class SchedulerConfig(object):
                     self.root.remove(x)
                     break
             if last_downloaded is None:
-                last_downloaded = datetime.fromordinal(1)
+                last_downloaded = fromordinal(1)
             sr = E.scheduled_recipe({
                 'id' : recipe.get('id'),
                 'title': recipe.get('title'),