Increased speed of runtime_thermostat_summary population

api/index.php

@@ -85,3 +85,37 @@ function array_standard_deviation($array) {
 
   return round(sqrt($variance / $count), 1);
 }
+
+/**
+ * Convert a local datetime string to a UTC datetime string.
+ *
+ * @param string $local_datetime Local datetime string.
+ * @param string $local_time_zone The local time zone to convert from.
+ *
+ * @return string The UTC datetime string.
+ */
+function get_utc_datetime($local_datetime, $local_time_zone, $format = 'Y-m-d H:i:s') {
+  $local_time_zone = new DateTimeZone($local_time_zone);
+  $utc_time_zone = new DateTimeZone('UTC');
+  $date_time = new DateTime($local_datetime, $local_time_zone);
+  $date_time->setTimezone($utc_time_zone);
+
+  return $date_time->format($format);
+}
+
+/**
+ * Convert a UTC datetime string to a UTC datetime string.
+ *
+ * @param string $utc_datetime Local datetime string.
+ * @param string $local_time_zone The local time zone to convert from.
+ *
+ * @return string The UTC datetime string.
+ */
+function get_local_datetime($utc_datetime, $local_time_zone, $format = 'Y-m-d H:i:s') {
+  $local_time_zone = new DateTimeZone($local_time_zone);
+  $utc_time_zone = new DateTimeZone('UTC');
+  $date_time = new DateTime($utc_datetime, $utc_time_zone);
+  $date_time->setTimezone($local_time_zone);
+
+  return $date_time->format($format);
+}

api/runtime.php

@@ -229,7 +229,7 @@ class runtime extends cora\api {
       // Populate on the fly.
       $this->api(
         'runtime_thermostat_summary',
-        'populate',
+        'populate_backwards',
         $thermostat_id
       );
 
@@ -308,7 +308,7 @@ class runtime extends cora\api {
     // Populate at the end of a full sync forwards.
     $this->api(
       'runtime_thermostat_summary',
-      'populate',
+      'populate_forwards',
       $thermostat_id
     );
   }
@@ -435,14 +435,14 @@ class runtime extends cora\api {
         'thermostat_id' => $thermostat['thermostat_id'],
         'timestamp' => [
           'value' => [
-            $this->get_utc_datetime(
+            get_utc_datetime(
               date(
                 'Y-m-d H:i:s',
                 strtotime($columns_begin['date'] . ' ' . $columns_begin['time'] . ' -1 hour')
               ),
               $thermostat['time_zone']
             ),
-            $this->get_utc_datetime(
+            get_utc_datetime(
               date(
                 'Y-m-d H:i:s',
                 strtotime($columns_end['date'] . ' ' . $columns_end['time'] . ' +1 hour')
@@ -515,7 +515,7 @@ class runtime extends cora\api {
 
       // Date and time are first two columns of the returned data. It is
       // returned in thermostat time, so convert it to UTC first.
-      $timestamp = $this->get_utc_datetime(
+      $timestamp = get_utc_datetime(
         $columns['date'] . ' ' . $columns['time'],
         $thermostat['time_zone']
       );
@@ -680,14 +680,14 @@ class runtime extends cora\api {
           'sensor_id' => array_column($sensors, 'sensor_id'),
           'timestamp' => [
             'value' => [
-              $this->get_utc_datetime(
+              get_utc_datetime(
                 date(
                   'Y-m-d H:i:s',
                   strtotime($columns_begin['date'] . ' ' . $columns_begin['time'] . ' -1 hour')
                 ),
                 $thermostat['time_zone']
               ),
-              $this->get_utc_datetime(
+              get_utc_datetime(
                 date(
                   'Y-m-d H:i:s',
                   strtotime($columns_end['date'] . ' ' . $columns_end['time'] . ' +1 hour')
@@ -747,7 +747,7 @@ class runtime extends cora\api {
             if (isset($datas[$sensor['sensor_id']]) === false) {
               $datas[$sensor['sensor_id']] = [
                 'sensor_id' => $sensor['sensor_id'],
-                'timestamp' => $this->get_utc_datetime(
+                'timestamp' => get_utc_datetime(
                   $columns['date'] . ' ' . $columns['time'],
                   $thermostat['time_zone']
                 )
@@ -821,40 +821,6 @@ class runtime extends cora\api {
     return $return;
   }
 
-  /**
-   * Convert a local datetime string to a UTC datetime string.
-   *
-   * @param string $local_datetime Local datetime string.
-   * @param string $local_time_zone The local time zone to convert from.
-   *
-   * @return string The UTC datetime string.
-   */
-  private function get_utc_datetime($local_datetime, $local_time_zone, $format = 'Y-m-d H:i:s') {
-    $local_time_zone = new DateTimeZone($local_time_zone);
-    $utc_time_zone = new DateTimeZone('UTC');
-    $date_time = new DateTime($local_datetime, $local_time_zone);
-    $date_time->setTimezone($utc_time_zone);
-
-    return $date_time->format($format);
-  }
-
-  /**
-   * Convert a UTC datetime string to a UTC datetime string.
-   *
-   * @param string $utc_datetime Local datetime string.
-   * @param string $local_time_zone The local time zone to convert from.
-   *
-   * @return string The UTC datetime string.
-   */
-  private function get_local_datetime($utc_datetime, $local_time_zone, $format = 'Y-m-d H:i:s') {
-    $local_time_zone = new DateTimeZone($local_time_zone);
-    $utc_time_zone = new DateTimeZone('UTC');
-    $date_time = new DateTime($utc_datetime, $utc_time_zone);
-    $date_time->setTimezone($local_time_zone);
-
-    return $date_time->format($format);
-  }
-
   /**
    * Download all data that exists for a specific thermostat.
    *
@@ -1046,7 +1012,7 @@ class runtime extends cora\api {
 
       $current_timestamp = $chunk_begin;
       while($current_timestamp <= $chunk_end) {
-        $local_datetime = $this->get_local_datetime(
+        $local_datetime = get_local_datetime(
           date('Y-m-d H:i:s', $current_timestamp),
           $thermostat['time_zone']
         );

api/runtime_thermostat.php

@@ -79,7 +79,7 @@ class runtime_thermostat extends cora\crud {
         strtotime($attributes['timestamp']['value']) - min(strtotime($thermostat['first_connected']), strtotime($thermostat['sync_begin'])) > $max_range
       )
     ) {
-      throw new \Exception('Max range is 31 days. ' . (time() - strtotime($attributes['timestamp']['value'])), 10205);
+      throw new \Exception('Max range is 31 days.', 10205);
     }
 
     // Accept timestamps in roughly any format; always convert back to something nice and in UTC

api/runtime_thermostat_summary.php

@@ -94,17 +94,17 @@ class runtime_thermostat_summary extends cora\crud {
   }
 
   /**
-   * Populate the runtime_thermostat_summary table.
+   * Populate from the max populated date until now.
    *
    * @param int $thermostat_id
    */
-  public function populate($thermostat_id) {
+  public function populate_forwards($thermostat_id) {
     $thermostat = $this->api('thermostat', 'get', $thermostat_id);
 
     $query = '
       select
-        min(`date`) `min_date`,
         max(`date`) `max_date`
+        #"2021-01-20" `max_date`
       from
         `runtime_thermostat_summary`
       where
@@ -114,80 +114,195 @@ class runtime_thermostat_summary extends cora\crud {
     $result = $this->database->query($query);
     $row = $result->fetch_assoc();
 
-    if($row['min_date'] === null || $row['max_date'] === null) {
+    if($row['max_date'] === null) {
-      $start = 'now() - interval 10 year'; // Just grab everything
+      $populate_begin = strtotime($thermostat['data_begin']); // Just grab everything
     } else {
-      if(strtotime($row['min_date']) > strtotime($thermostat['sync_begin'])) {
+      $populate_begin = strtotime($row['max_date']);
-        $start = '"' . date('Y-m-d 00:00:00', strtotime($thermostat['sync_begin'])) . '"';
-      } else {
-        $start = '"' . date('Y-m-d 00:00:00', strtotime($row['max_date'] . ' - 1 day')) . '"';
-      }
     }
+    $populate_end = time();
 
-    // TODO
+    $populate_begin = date('Y-m-d', $populate_begin);
-    // Query takes a full second to run for my data which would add some amount of time for the sync...
+    $populate_end = date('Y-m-d', $populate_end);
-    // Going to need to add a stop as well so only adding in relevant data points as the backwards sync runs
+
-    //
+    return $this->populate($thermostat_id, $populate_begin, $populate_end);
-    // TODO
+  }
-    // timezone convert!
+
+  /**
+   * Populate from the beginning of time until the min populated date.
+   *
+   * @param int $thermostat_id
+   */
+  public function populate_backwards($thermostat_id) {
+    $thermostat = $this->api('thermostat', 'get', $thermostat_id);
 
     $query = '
-      insert into
-        `runtime_thermostat_summary`
       select
-        null `runtime_summary_id`,
+        min(`date`) `min_date`
-        `thermostat`.`user_id` `user_id`,
+        #"2020-09-25" `min_date`
-        `thermostat_id` `thermostat_id`,
-        date(convert_tz(`timestamp`, "UTC", "' . $thermostat['time_zone'] . '")) `date`,
-        count(*) `count`,
-        sum(case when `compressor_mode` = "cool" then `compressor_1` else 0 end) `sum_compressor_cool_1`,
-        sum(case when `compressor_mode` = "cool" then `compressor_2` else 0 end) `sum_compressor_cool_2`,
-        sum(case when `compressor_mode` = "heat" then `compressor_1` else 0 end) `sum_compressor_heat_1`,
-        sum(case when `compressor_mode` = "heat" then `compressor_2` else 0 end) `sum_compressor_heat_2`,
-        sum(`auxiliary_heat_1`) `sum_auxiliary_heat_1`,
-        sum(`auxiliary_heat_2`) `sum_auxiliary_heat_2`,
-        sum(`fan`) `sum_fan`,
-        sum(case when `accessory_type` = "humidifier" then `accessory` else 0 end) `sum_humidifier`,
-        sum(case when `accessory_type` = "dehumidifier" then `accessory` else 0 end) `sum_dehumidifier`,
-        sum(case when `accessory_type` = "ventilator" then `accessory` else 0 end) `sum_ventilator`,
-        sum(case when `accessory_type` = "economizer" then `accessory` else 0 end) `sum_economizer`,
-        round(avg(`outdoor_temperature`)) `avg_outdoor_temperature`,
-        round(avg(`outdoor_humidity`)) `avg_outdoor_humidity`,
-        min(`outdoor_temperature`) `min_outdoor_temperature`,
-        max(`outdoor_temperature`) `max_outdoor_temperature`,
-        round(avg(`indoor_temperature`)) `avg_indoor_temperature`,
-        round(avg(`indoor_humidity`)) `avg_indoor_humidity`,
-        0 `deleted`
       from
-        `runtime_thermostat`
+        `runtime_thermostat_summary`
-      join
-        `thermostat` using(`thermostat_id`)
       where
-            convert_tz(`timestamp`, "UTC", "' . $thermostat['time_zone'] . '") > ' . $start . '
+            `user_id` = ' . $this->database->escape($this->session->get_user_id()) . '
-        and thermostat_id = ' . $thermostat['thermostat_id'] . '
+        and `thermostat_id` = ' . $this->database->escape($thermostat_id) . '
-      group by
-        `thermostat_id`,
-        date(convert_tz(`timestamp`, "UTC", "' . $thermostat['time_zone'] . '"))
-      on duplicate key update
-        `count` = values(`count`),
-        `sum_compressor_cool_1` = values(`sum_compressor_cool_1`),
-        `sum_compressor_cool_2` = values(`sum_compressor_cool_2`),
-        `sum_compressor_heat_1` = values(`sum_compressor_heat_1`),
-        `sum_compressor_heat_2` = values(`sum_compressor_heat_2`),
-        `sum_auxiliary_heat_1` = values(`sum_auxiliary_heat_1`),
-        `sum_auxiliary_heat_2` = values(`sum_auxiliary_heat_2`),
-        `sum_fan` = values(`sum_fan`),
-        `sum_humidifier` = values(`sum_humidifier`),
-        `sum_dehumidifier` = values(`sum_dehumidifier`),
-        `sum_ventilator` = values(`sum_ventilator`),
-        `sum_economizer` = values(`sum_economizer`),
-        `avg_outdoor_temperature` = values(`avg_outdoor_temperature`),
-        `avg_outdoor_humidity` = values(`avg_outdoor_humidity`),
-        `min_outdoor_temperature` = values(`min_outdoor_temperature`),
-        `max_outdoor_temperature` = values(`max_outdoor_temperature`),
-        `avg_indoor_temperature` = values(`avg_indoor_temperature`),
-        `avg_indoor_humidity` = values(`avg_indoor_humidity`)
     ';
-    $this->database->query($query);
+    $result = $this->database->query($query);
+    $row = $result->fetch_assoc();
+
+    if($row['min_date'] === null) {
+      $populate_end = time();
+    } else {
+      // Include
+      $populate_end = strtotime($row['min_date']);
+    }
+    $populate_begin = strtotime($thermostat['data_begin']);
+
+    $populate_begin = date('Y-m-d', $populate_begin);
+    $populate_end = date('Y-m-d', $populate_end);
+
+    return $this->populate($thermostat_id, $populate_begin, $populate_end);
+  }
+
+  /**
+   * Populate the runtime_thermostat_summary table.
+   *
+   * @param int $thermostat_id
+   * @param string $populate_begin Local date to begin populating, inclusive.
+   * @param string $populate_end Local date to end populating, inclusive.
+   */
+  private function populate($thermostat_id, $populate_begin, $populate_end) {
+    $thermostat = $this->api('thermostat', 'get', $thermostat_id);
+
+    // Convert date strings to timestamps to make them easier to work with.
+    $populate_begin = strtotime($populate_begin . ' 00:00:00');
+    $populate_end = strtotime($populate_end . ' 23:59:59');
+
+    $chunk_begin = $populate_begin;
+    $chunk_end = $populate_begin;
+
+    $data = [];
+    do {
+      $chunk_end = strtotime('+1 week', $chunk_begin);
+
+      // MySQL "between" is inclusive so go back 5 minutes to avoid
+      // double-counting rows.
+      $chunk_end = strtotime('-5 minute', $chunk_end);
+
+      // Don't overshoot into data that's already populated
+      $chunk_end = min($chunk_end, $populate_end);
+
+      $chunk_begin_datetime = get_utc_datetime(
+        date('Y-m-d H:i:s', $chunk_begin),
+        $thermostat['time_zone']
+      );
+
+      $chunk_end_datetime = get_utc_datetime(
+        date('Y-m-d H:i:s', $chunk_end),
+        $thermostat['time_zone']
+      );
+
+      $runtime_thermostats = $this->api(
+        'runtime_thermostat',
+        'read',
+        [
+          'attributes' => [
+            'thermostat_id' => $thermostat['thermostat_id'],
+            'timestamp' => [
+              'operator' => 'between',
+              'value' => [$chunk_begin_datetime, $chunk_end_datetime]
+            ]
+          ]
+        ]
+      );
+
+      foreach($runtime_thermostats as $runtime_thermostat) {
+        $date = get_local_datetime(
+          $runtime_thermostat['timestamp'],
+          $thermostat['time_zone'],
+          'Y-m-d'
+        );
+
+        if(isset($data[$date]) === false) {
+          $data[$date] = [
+            'count' => 0,
+            'sum_fan' => 0,
+            'min_outdoor_temperature' => INF,
+            'max_outdoor_temperature' => -INF,
+            'sum_auxiliary_heat_1' => 0,
+            'sum_auxiliary_heat_2' => 0,
+            'sum_compressor_cool_1' => 0,
+            'sum_compressor_cool_2' => 0,
+            'sum_compressor_heat_1' => 0,
+            'sum_compressor_heat_2' => 0,
+            'sum_humidifier' => 0,
+            'sum_dehumidifier' => 0,
+            'sum_ventilator' => 0,
+            'sum_economizer' => 0,
+            'avg_outdoor_temperature' => [],
+            'avg_outdoor_humidity' => [],
+            'avg_indoor_temperature' => [],
+            'avg_indoor_humidity' => []
+          ];
+        }
+
+        $runtime_thermostat['outdoor_temperature'] *= 10;
+        $runtime_thermostat['indoor_temperature'] *= 10;
+
+        $data[$date]['count']++;
+        $data[$date]['sum_fan'] += $runtime_thermostat['fan'];
+        $data[$date]['min_outdoor_temperature'] = min($runtime_thermostat['outdoor_temperature'], $data[$date]['min_outdoor_temperature']);
+        $data[$date]['max_outdoor_temperature'] = max($runtime_thermostat['outdoor_temperature'], $data[$date]['max_outdoor_temperature']);
+        $data[$date]['sum_auxiliary_heat_1'] += $runtime_thermostat['auxiliary_heat_1'];
+        $data[$date]['sum_auxiliary_heat_2'] += $runtime_thermostat['auxiliary_heat_2'];
+
+        if($runtime_thermostat['compressor_mode'] === 'cool') {
+          $data[$date]['sum_compressor_cool_1'] += $runtime_thermostat['compressor_1'];
+          $data[$date]['sum_compressor_cool_2'] += $runtime_thermostat['compressor_2'];
+        } else if($runtime_thermostat['compressor_mode'] === 'heat') {
+          $data[$date]['sum_compressor_heat_1'] += $runtime_thermostat['compressor_1'];
+          $data[$date]['sum_compressor_heat_2'] += $runtime_thermostat['compressor_2'];
+        }
+
+        if($runtime_thermostat['accessory_type'] === 'humidifier') {
+          $data[$date]['sum_humidifier'] += $runtime_thermostat['accessory'];
+        } else if($runtime_thermostat['compressor_mode'] === 'dehumidifier') {
+          $data[$date]['sum_dehumidifier'] += $runtime_thermostat['accessory'];
+        } else if($runtime_thermostat['compressor_mode'] === 'ventilator') {
+          $data[$date]['sum_ventilator'] += $runtime_thermostat['accessory'];
+        } else if($runtime_thermostat['compressor_mode'] === 'economizer') {
+          $data[$date]['sum_economizer'] += $runtime_thermostat['accessory'];
+        }
+
+        $data[$date]['avg_outdoor_temperature'][] = $runtime_thermostat['outdoor_temperature'];
+        $data[$date]['avg_outdoor_humidity'][] = $runtime_thermostat['outdoor_humidity'];
+        $data[$date]['avg_indoor_temperature'][] = $runtime_thermostat['indoor_temperature'];
+        $data[$date]['avg_indoor_humidity'][] = $runtime_thermostat['indoor_humidity'];
+      }
+
+      $chunk_begin = strtotime('+5 minute', $chunk_end);
+    } while ($chunk_end < $populate_end);
+
+    // Write to the database.
+    foreach($data as $date => &$row) {
+      $row['avg_outdoor_temperature'] = round(array_sum($row['avg_outdoor_temperature']) / count($row['avg_outdoor_temperature']));
+      $row['avg_outdoor_humidity'] = round(array_sum($row['avg_outdoor_humidity']) / count($row['avg_outdoor_humidity']));
+      $row['avg_indoor_temperature'] = round(array_sum($row['avg_indoor_temperature']) / count($row['avg_indoor_temperature']));
+      $row['avg_indoor_humidity'] = round(array_sum($row['avg_indoor_humidity']) / count($row['avg_indoor_humidity']));
+
+      $row['date'] = $date;
+      $row['user_id'] = $thermostat['user_id'];
+      $row['thermostat_id'] = $thermostat['thermostat_id'];
+
+      $existing_runtime_thermostat_summary = $this->get([
+        'thermostat_id' => $thermostat['thermostat_id'],
+        'date' => $date
+      ]);
+
+      if($existing_runtime_thermostat_summary === null) {
+        $this->create($row);
+      } else {
+        $row['runtime_thermostat_summary_id'] = $existing_runtime_thermostat_summary['runtime_thermostat_summary_id'];
+        $this->update($row);
+      }
+    }
   }
 }