Speedup startup times for large libraries with a faster string-to-date converter

setup/extensions.py

@@ -68,6 +68,10 @@ if isosx:
 
 extensions = [
 
+    Extension('speedup',
+        ['calibre/utils/speedup.c'],
+        ),
+
     Extension('icu',
         ['calibre/utils/icu.c'],
         libraries=icu_libs,

src/calibre/constants.py

@@ -69,6 +69,7 @@ if plugins is None:
                 'chmlib',
                 'chm_extra',
                 'icu',
+                'speedup',
             ] + \
                     (['winutil'] if iswindows else []) + \
                     (['usbobserver'] if isosx else []):

src/calibre/library/sqlite.py

@@ -17,19 +17,54 @@ from datetime import datetime
 from functools import partial
 
 from calibre.ebooks.metadata import title_sort, author_to_author_sort
-from calibre.utils.date import parse_date, isoformat
+from calibre.utils.date import parse_date, isoformat, local_tz
 from calibre import isbytestring, force_unicode
-from calibre.constants import iswindows, DEBUG
+from calibre.constants import iswindows, DEBUG, plugins
 from calibre.utils.icu import strcmp
 from calibre import prints
 
+from dateutil.tz import tzoffset
+
 global_lock = RLock()
 
-def convert_timestamp(val):
+_c_speedup = plugins['speedup'][0]
+
+def _c_convert_timestamp(val):
+    if not val:
+        return None
+    try:
+        ret = _c_speedup.parse_date(val.strip())
+    except:
+        ret = None
+    if ret is None:
+        return parse_date(val, as_utc=False)
+    year, month, day, hour, minutes, seconds, tzmins = ret
+    return datetime(year, month, day, hour, minutes, seconds,
+                tzinfo=tzoffset(None, tzmins)).astimezone(local_tz)
+
+def _py_convert_timestamp(val):
     if val:
+        tzmins = 0
+        try:
+            sign = {'+':1, '-':-1}.get(val[-6], None)
+            if sign is not None:
+                tzmins = (int(val[-5:-3])*60 + int(val[-2:])) * sign
+            year = int(val[0:4])
+            month = int(val[5:7])
+            day = int(val[8:10])
+            hour = int(val[11:13])
+            min = int(val[14:16])
+            sec = int(val[17:19])
+            return datetime(year, month, day, hour, min, sec,
+                    tzinfo=tzoffset(None, tzmins)).astimezone(local_tz)
+        except:
+            pass
         return parse_date(val, as_utc=False)
     return None
 
+convert_timestamp = _py_convert_timestamp if _c_speedup is None else \
+                    _c_convert_timestamp
+
 def adapt_datetime(dt):
     return isoformat(dt, sep=' ')
 

src/calibre/utils/speedup.c

@@ -0,0 +1,82 @@
+#define UNICODE
+#include <Python.h>
+
+#include <stdlib.h>
+
+static PyObject *
+speedup_parse_date(PyObject *self, PyObject *args) {
+    const char *raw, *orig, *tz;
+    char *end;
+    long year, month, day, hour, minute, second, tzh = 0, tzm = 0, sign = 0;
+    size_t len;
+    if(!PyArg_ParseTuple(args, "s", &raw)) return NULL;
+    len = strlen(raw);
+    if (len < 19) Py_RETURN_NONE;
+
+    orig = raw;
+
+
+    year = strtol(raw, &end, 10);
+    if ((end - raw) != 4) Py_RETURN_NONE;
+    raw += 5;
+    
+
+    month = strtol(raw, &end, 10);
+    if ((end - raw) != 2) Py_RETURN_NONE;
+    raw += 3;
+    
+
+    day = strtol(raw, &end, 10);
+    if ((end - raw) != 2) Py_RETURN_NONE;
+    raw += 3;
+    
+    hour = strtol(raw, &end, 10);
+    if ((end - raw) != 2) Py_RETURN_NONE;
+    raw += 3;
+
+    minute = strtol(raw, &end, 10);
+    if ((end - raw) != 2) Py_RETURN_NONE;
+    raw += 3;
+
+    second = strtol(raw, &end, 10);
+    if ((end - raw) != 2) Py_RETURN_NONE;
+    raw += 3;
+
+
+    tz = orig + len - 6;
+
+    if (*tz == '+') sign = 1;
+    if (*tz == '-') sign = -1;
+    if (sign != 0) {
+        // We have TZ info
+        tz += 1;
+
+        tzh = strtol(tz, &end, 10);
+        if ((end - tz) != 2) Py_RETURN_NONE;
+        tz += 3;
+
+        tzm = strtol(tz, &end, 10);
+        if ((end - tz) != 2) Py_RETURN_NONE;
+    }
+
+    return Py_BuildValue("lllllll", year, month, day, hour, minute, second,
+            (tzh*60 + tzm)*sign);
+}
+
+static PyMethodDef speedup_methods[] = {
+    {"parse_date", speedup_parse_date, METH_VARARGS,
+        "parse_date()\n\nParse ISO dates faster."
+    },
+
+    {NULL, NULL, 0, NULL}
+};
+
+
+PyMODINIT_FUNC
+initspeedup(void) {
+    PyObject *m;
+    m = Py_InitModule3("speedup", speedup_methods,
+    "Implementation of methods in C for speed."
+    );
+    if (m == NULL) return;
+}