From 1c65cd6fcf1b18e9c3c59faa04c2a96cfd3e0232 Mon Sep 17 00:00:00 2001 From: Jon Ziebell Date: Wed, 24 Feb 2021 19:30:42 -0500 Subject: [PATCH] Increased speed of runtime_thermostat_summary population --- api/index.php | 34 ++++ api/runtime.php | 52 +----- api/runtime_thermostat.php | 2 +- api/runtime_thermostat_summary.php | 253 +++++++++++++++++++++-------- 4 files changed, 228 insertions(+), 113 deletions(-) diff --git a/api/index.php b/api/index.php index a51c165..2c154a8 100644 --- a/api/index.php +++ b/api/index.php @@ -85,3 +85,37 @@ function array_standard_deviation($array) { return round(sqrt($variance / $count), 1); } + +/** + * Convert a local datetime string to a UTC datetime string. + * + * @param string $local_datetime Local datetime string. + * @param string $local_time_zone The local time zone to convert from. + * + * @return string The UTC datetime string. + */ +function get_utc_datetime($local_datetime, $local_time_zone, $format = 'Y-m-d H:i:s') { + $local_time_zone = new DateTimeZone($local_time_zone); + $utc_time_zone = new DateTimeZone('UTC'); + $date_time = new DateTime($local_datetime, $local_time_zone); + $date_time->setTimezone($utc_time_zone); + + return $date_time->format($format); +} + +/** + * Convert a UTC datetime string to a UTC datetime string. + * + * @param string $utc_datetime Local datetime string. + * @param string $local_time_zone The local time zone to convert from. + * + * @return string The UTC datetime string. + */ +function get_local_datetime($utc_datetime, $local_time_zone, $format = 'Y-m-d H:i:s') { + $local_time_zone = new DateTimeZone($local_time_zone); + $utc_time_zone = new DateTimeZone('UTC'); + $date_time = new DateTime($utc_datetime, $utc_time_zone); + $date_time->setTimezone($local_time_zone); + + return $date_time->format($format); +} diff --git a/api/runtime.php b/api/runtime.php index 8efe3c7..83282fc 100644 --- a/api/runtime.php +++ b/api/runtime.php @@ -229,7 +229,7 @@ class runtime extends cora\api { // Populate on the fly. $this->api( 'runtime_thermostat_summary', - 'populate', + 'populate_backwards', $thermostat_id ); @@ -308,7 +308,7 @@ class runtime extends cora\api { // Populate at the end of a full sync forwards. $this->api( 'runtime_thermostat_summary', - 'populate', + 'populate_forwards', $thermostat_id ); } @@ -435,14 +435,14 @@ class runtime extends cora\api { 'thermostat_id' => $thermostat['thermostat_id'], 'timestamp' => [ 'value' => [ - $this->get_utc_datetime( + get_utc_datetime( date( 'Y-m-d H:i:s', strtotime($columns_begin['date'] . ' ' . $columns_begin['time'] . ' -1 hour') ), $thermostat['time_zone'] ), - $this->get_utc_datetime( + get_utc_datetime( date( 'Y-m-d H:i:s', strtotime($columns_end['date'] . ' ' . $columns_end['time'] . ' +1 hour') @@ -515,7 +515,7 @@ class runtime extends cora\api { // Date and time are first two columns of the returned data. It is // returned in thermostat time, so convert it to UTC first. - $timestamp = $this->get_utc_datetime( + $timestamp = get_utc_datetime( $columns['date'] . ' ' . $columns['time'], $thermostat['time_zone'] ); @@ -680,14 +680,14 @@ class runtime extends cora\api { 'sensor_id' => array_column($sensors, 'sensor_id'), 'timestamp' => [ 'value' => [ - $this->get_utc_datetime( + get_utc_datetime( date( 'Y-m-d H:i:s', strtotime($columns_begin['date'] . ' ' . $columns_begin['time'] . ' -1 hour') ), $thermostat['time_zone'] ), - $this->get_utc_datetime( + get_utc_datetime( date( 'Y-m-d H:i:s', strtotime($columns_end['date'] . ' ' . $columns_end['time'] . ' +1 hour') @@ -747,7 +747,7 @@ class runtime extends cora\api { if (isset($datas[$sensor['sensor_id']]) === false) { $datas[$sensor['sensor_id']] = [ 'sensor_id' => $sensor['sensor_id'], - 'timestamp' => $this->get_utc_datetime( + 'timestamp' => get_utc_datetime( $columns['date'] . ' ' . $columns['time'], $thermostat['time_zone'] ) @@ -821,40 +821,6 @@ class runtime extends cora\api { return $return; } - /** - * Convert a local datetime string to a UTC datetime string. - * - * @param string $local_datetime Local datetime string. - * @param string $local_time_zone The local time zone to convert from. - * - * @return string The UTC datetime string. - */ - private function get_utc_datetime($local_datetime, $local_time_zone, $format = 'Y-m-d H:i:s') { - $local_time_zone = new DateTimeZone($local_time_zone); - $utc_time_zone = new DateTimeZone('UTC'); - $date_time = new DateTime($local_datetime, $local_time_zone); - $date_time->setTimezone($utc_time_zone); - - return $date_time->format($format); - } - - /** - * Convert a UTC datetime string to a UTC datetime string. - * - * @param string $utc_datetime Local datetime string. - * @param string $local_time_zone The local time zone to convert from. - * - * @return string The UTC datetime string. - */ - private function get_local_datetime($utc_datetime, $local_time_zone, $format = 'Y-m-d H:i:s') { - $local_time_zone = new DateTimeZone($local_time_zone); - $utc_time_zone = new DateTimeZone('UTC'); - $date_time = new DateTime($utc_datetime, $utc_time_zone); - $date_time->setTimezone($local_time_zone); - - return $date_time->format($format); - } - /** * Download all data that exists for a specific thermostat. * @@ -1046,7 +1012,7 @@ class runtime extends cora\api { $current_timestamp = $chunk_begin; while($current_timestamp <= $chunk_end) { - $local_datetime = $this->get_local_datetime( + $local_datetime = get_local_datetime( date('Y-m-d H:i:s', $current_timestamp), $thermostat['time_zone'] ); diff --git a/api/runtime_thermostat.php b/api/runtime_thermostat.php index 3287f49..7630f28 100755 --- a/api/runtime_thermostat.php +++ b/api/runtime_thermostat.php @@ -79,7 +79,7 @@ class runtime_thermostat extends cora\crud { strtotime($attributes['timestamp']['value']) - min(strtotime($thermostat['first_connected']), strtotime($thermostat['sync_begin'])) > $max_range ) ) { - throw new \Exception('Max range is 31 days. ' . (time() - strtotime($attributes['timestamp']['value'])), 10205); + throw new \Exception('Max range is 31 days.', 10205); } // Accept timestamps in roughly any format; always convert back to something nice and in UTC diff --git a/api/runtime_thermostat_summary.php b/api/runtime_thermostat_summary.php index 839b2cf..6892e4e 100755 --- a/api/runtime_thermostat_summary.php +++ b/api/runtime_thermostat_summary.php @@ -94,17 +94,17 @@ class runtime_thermostat_summary extends cora\crud { } /** - * Populate the runtime_thermostat_summary table. + * Populate from the max populated date until now. * * @param int $thermostat_id */ - public function populate($thermostat_id) { + public function populate_forwards($thermostat_id) { $thermostat = $this->api('thermostat', 'get', $thermostat_id); $query = ' select - min(`date`) `min_date`, max(`date`) `max_date` + #"2021-01-20" `max_date` from `runtime_thermostat_summary` where @@ -114,80 +114,195 @@ class runtime_thermostat_summary extends cora\crud { $result = $this->database->query($query); $row = $result->fetch_assoc(); - if($row['min_date'] === null || $row['max_date'] === null) { - $start = 'now() - interval 10 year'; // Just grab everything + if($row['max_date'] === null) { + $populate_begin = strtotime($thermostat['data_begin']); // Just grab everything } else { - if(strtotime($row['min_date']) > strtotime($thermostat['sync_begin'])) { - $start = '"' . date('Y-m-d 00:00:00', strtotime($thermostat['sync_begin'])) . '"'; - } else { - $start = '"' . date('Y-m-d 00:00:00', strtotime($row['max_date'] . ' - 1 day')) . '"'; - } + $populate_begin = strtotime($row['max_date']); } + $populate_end = time(); - // TODO - // Query takes a full second to run for my data which would add some amount of time for the sync... - // Going to need to add a stop as well so only adding in relevant data points as the backwards sync runs - // - // TODO - // timezone convert! + $populate_begin = date('Y-m-d', $populate_begin); + $populate_end = date('Y-m-d', $populate_end); + + return $this->populate($thermostat_id, $populate_begin, $populate_end); + } + + /** + * Populate from the beginning of time until the min populated date. + * + * @param int $thermostat_id + */ + public function populate_backwards($thermostat_id) { + $thermostat = $this->api('thermostat', 'get', $thermostat_id); $query = ' - insert into - `runtime_thermostat_summary` select - null `runtime_summary_id`, - `thermostat`.`user_id` `user_id`, - `thermostat_id` `thermostat_id`, - date(convert_tz(`timestamp`, "UTC", "' . $thermostat['time_zone'] . '")) `date`, - count(*) `count`, - sum(case when `compressor_mode` = "cool" then `compressor_1` else 0 end) `sum_compressor_cool_1`, - sum(case when `compressor_mode` = "cool" then `compressor_2` else 0 end) `sum_compressor_cool_2`, - sum(case when `compressor_mode` = "heat" then `compressor_1` else 0 end) `sum_compressor_heat_1`, - sum(case when `compressor_mode` = "heat" then `compressor_2` else 0 end) `sum_compressor_heat_2`, - sum(`auxiliary_heat_1`) `sum_auxiliary_heat_1`, - sum(`auxiliary_heat_2`) `sum_auxiliary_heat_2`, - sum(`fan`) `sum_fan`, - sum(case when `accessory_type` = "humidifier" then `accessory` else 0 end) `sum_humidifier`, - sum(case when `accessory_type` = "dehumidifier" then `accessory` else 0 end) `sum_dehumidifier`, - sum(case when `accessory_type` = "ventilator" then `accessory` else 0 end) `sum_ventilator`, - sum(case when `accessory_type` = "economizer" then `accessory` else 0 end) `sum_economizer`, - round(avg(`outdoor_temperature`)) `avg_outdoor_temperature`, - round(avg(`outdoor_humidity`)) `avg_outdoor_humidity`, - min(`outdoor_temperature`) `min_outdoor_temperature`, - max(`outdoor_temperature`) `max_outdoor_temperature`, - round(avg(`indoor_temperature`)) `avg_indoor_temperature`, - round(avg(`indoor_humidity`)) `avg_indoor_humidity`, - 0 `deleted` + min(`date`) `min_date` + #"2020-09-25" `min_date` from - `runtime_thermostat` - join - `thermostat` using(`thermostat_id`) + `runtime_thermostat_summary` where - convert_tz(`timestamp`, "UTC", "' . $thermostat['time_zone'] . '") > ' . $start . ' - and thermostat_id = ' . $thermostat['thermostat_id'] . ' - group by - `thermostat_id`, - date(convert_tz(`timestamp`, "UTC", "' . $thermostat['time_zone'] . '")) - on duplicate key update - `count` = values(`count`), - `sum_compressor_cool_1` = values(`sum_compressor_cool_1`), - `sum_compressor_cool_2` = values(`sum_compressor_cool_2`), - `sum_compressor_heat_1` = values(`sum_compressor_heat_1`), - `sum_compressor_heat_2` = values(`sum_compressor_heat_2`), - `sum_auxiliary_heat_1` = values(`sum_auxiliary_heat_1`), - `sum_auxiliary_heat_2` = values(`sum_auxiliary_heat_2`), - `sum_fan` = values(`sum_fan`), - `sum_humidifier` = values(`sum_humidifier`), - `sum_dehumidifier` = values(`sum_dehumidifier`), - `sum_ventilator` = values(`sum_ventilator`), - `sum_economizer` = values(`sum_economizer`), - `avg_outdoor_temperature` = values(`avg_outdoor_temperature`), - `avg_outdoor_humidity` = values(`avg_outdoor_humidity`), - `min_outdoor_temperature` = values(`min_outdoor_temperature`), - `max_outdoor_temperature` = values(`max_outdoor_temperature`), - `avg_indoor_temperature` = values(`avg_indoor_temperature`), - `avg_indoor_humidity` = values(`avg_indoor_humidity`) + `user_id` = ' . $this->database->escape($this->session->get_user_id()) . ' + and `thermostat_id` = ' . $this->database->escape($thermostat_id) . ' '; - $this->database->query($query); + $result = $this->database->query($query); + $row = $result->fetch_assoc(); + + if($row['min_date'] === null) { + $populate_end = time(); + } else { + // Include + $populate_end = strtotime($row['min_date']); + } + $populate_begin = strtotime($thermostat['data_begin']); + + $populate_begin = date('Y-m-d', $populate_begin); + $populate_end = date('Y-m-d', $populate_end); + + return $this->populate($thermostat_id, $populate_begin, $populate_end); + } + + /** + * Populate the runtime_thermostat_summary table. + * + * @param int $thermostat_id + * @param string $populate_begin Local date to begin populating, inclusive. + * @param string $populate_end Local date to end populating, inclusive. + */ + private function populate($thermostat_id, $populate_begin, $populate_end) { + $thermostat = $this->api('thermostat', 'get', $thermostat_id); + + // Convert date strings to timestamps to make them easier to work with. + $populate_begin = strtotime($populate_begin . ' 00:00:00'); + $populate_end = strtotime($populate_end . ' 23:59:59'); + + $chunk_begin = $populate_begin; + $chunk_end = $populate_begin; + + $data = []; + do { + $chunk_end = strtotime('+1 week', $chunk_begin); + + // MySQL "between" is inclusive so go back 5 minutes to avoid + // double-counting rows. + $chunk_end = strtotime('-5 minute', $chunk_end); + + // Don't overshoot into data that's already populated + $chunk_end = min($chunk_end, $populate_end); + + $chunk_begin_datetime = get_utc_datetime( + date('Y-m-d H:i:s', $chunk_begin), + $thermostat['time_zone'] + ); + + $chunk_end_datetime = get_utc_datetime( + date('Y-m-d H:i:s', $chunk_end), + $thermostat['time_zone'] + ); + + $runtime_thermostats = $this->api( + 'runtime_thermostat', + 'read', + [ + 'attributes' => [ + 'thermostat_id' => $thermostat['thermostat_id'], + 'timestamp' => [ + 'operator' => 'between', + 'value' => [$chunk_begin_datetime, $chunk_end_datetime] + ] + ] + ] + ); + + foreach($runtime_thermostats as $runtime_thermostat) { + $date = get_local_datetime( + $runtime_thermostat['timestamp'], + $thermostat['time_zone'], + 'Y-m-d' + ); + + if(isset($data[$date]) === false) { + $data[$date] = [ + 'count' => 0, + 'sum_fan' => 0, + 'min_outdoor_temperature' => INF, + 'max_outdoor_temperature' => -INF, + 'sum_auxiliary_heat_1' => 0, + 'sum_auxiliary_heat_2' => 0, + 'sum_compressor_cool_1' => 0, + 'sum_compressor_cool_2' => 0, + 'sum_compressor_heat_1' => 0, + 'sum_compressor_heat_2' => 0, + 'sum_humidifier' => 0, + 'sum_dehumidifier' => 0, + 'sum_ventilator' => 0, + 'sum_economizer' => 0, + 'avg_outdoor_temperature' => [], + 'avg_outdoor_humidity' => [], + 'avg_indoor_temperature' => [], + 'avg_indoor_humidity' => [] + ]; + } + + $runtime_thermostat['outdoor_temperature'] *= 10; + $runtime_thermostat['indoor_temperature'] *= 10; + + $data[$date]['count']++; + $data[$date]['sum_fan'] += $runtime_thermostat['fan']; + $data[$date]['min_outdoor_temperature'] = min($runtime_thermostat['outdoor_temperature'], $data[$date]['min_outdoor_temperature']); + $data[$date]['max_outdoor_temperature'] = max($runtime_thermostat['outdoor_temperature'], $data[$date]['max_outdoor_temperature']); + $data[$date]['sum_auxiliary_heat_1'] += $runtime_thermostat['auxiliary_heat_1']; + $data[$date]['sum_auxiliary_heat_2'] += $runtime_thermostat['auxiliary_heat_2']; + + if($runtime_thermostat['compressor_mode'] === 'cool') { + $data[$date]['sum_compressor_cool_1'] += $runtime_thermostat['compressor_1']; + $data[$date]['sum_compressor_cool_2'] += $runtime_thermostat['compressor_2']; + } else if($runtime_thermostat['compressor_mode'] === 'heat') { + $data[$date]['sum_compressor_heat_1'] += $runtime_thermostat['compressor_1']; + $data[$date]['sum_compressor_heat_2'] += $runtime_thermostat['compressor_2']; + } + + if($runtime_thermostat['accessory_type'] === 'humidifier') { + $data[$date]['sum_humidifier'] += $runtime_thermostat['accessory']; + } else if($runtime_thermostat['compressor_mode'] === 'dehumidifier') { + $data[$date]['sum_dehumidifier'] += $runtime_thermostat['accessory']; + } else if($runtime_thermostat['compressor_mode'] === 'ventilator') { + $data[$date]['sum_ventilator'] += $runtime_thermostat['accessory']; + } else if($runtime_thermostat['compressor_mode'] === 'economizer') { + $data[$date]['sum_economizer'] += $runtime_thermostat['accessory']; + } + + $data[$date]['avg_outdoor_temperature'][] = $runtime_thermostat['outdoor_temperature']; + $data[$date]['avg_outdoor_humidity'][] = $runtime_thermostat['outdoor_humidity']; + $data[$date]['avg_indoor_temperature'][] = $runtime_thermostat['indoor_temperature']; + $data[$date]['avg_indoor_humidity'][] = $runtime_thermostat['indoor_humidity']; + } + + $chunk_begin = strtotime('+5 minute', $chunk_end); + } while ($chunk_end < $populate_end); + + // Write to the database. + foreach($data as $date => &$row) { + $row['avg_outdoor_temperature'] = round(array_sum($row['avg_outdoor_temperature']) / count($row['avg_outdoor_temperature'])); + $row['avg_outdoor_humidity'] = round(array_sum($row['avg_outdoor_humidity']) / count($row['avg_outdoor_humidity'])); + $row['avg_indoor_temperature'] = round(array_sum($row['avg_indoor_temperature']) / count($row['avg_indoor_temperature'])); + $row['avg_indoor_humidity'] = round(array_sum($row['avg_indoor_humidity']) / count($row['avg_indoor_humidity'])); + + $row['date'] = $date; + $row['user_id'] = $thermostat['user_id']; + $row['thermostat_id'] = $thermostat['thermostat_id']; + + $existing_runtime_thermostat_summary = $this->get([ + 'thermostat_id' => $thermostat['thermostat_id'], + 'date' => $date + ]); + + if($existing_runtime_thermostat_summary === null) { + $this->create($row); + } else { + $row['runtime_thermostat_summary_id'] = $existing_runtime_thermostat_summary['runtime_thermostat_summary_id']; + $this->update($row); + } + } } }