Cutlery/beestat
1
0
Fork 0
mirror of https://github.com/beestat/app.git synced 2025-07-09 03:04:07 -04:00
Code Actions Packages Releases Activity

Fixed #212 - Server running out of memory when doing global/all comparisons

Browse Source 
Added chunking to the thermostat group select to reduce memory usage.
This commit is contained in:
Jon Ziebell 2020-01-08 20:43:05 -05:00
parent b2239e4bbb
commit e5b1f5538f
2 changed files with 81 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
api/cora/database.php
View File

@ -413,11 +413,13 @@ final class database extends \mysqli {
* @param array $columns The columns to return. If not specified, all * @param array $columns The columns to return. If not specified, all
* columns are returned. * columns are returned.
* @param mixed $order_by String or array of order_bys. * @param mixed $order_by String or array of order_bys.
* @param mixed $limit Number or array of numbers as arguments to the MySQL
* limit clause.
* *
* @return array An array of the database rows with the specified columns. * @return array An array of the database rows with the specified columns.
* Even a single result will still be returned in an array of size 1. * Even a single result will still be returned in an array of size 1.
*/ */
public function read($resource, $attributes = [], $columns = [], $order_by = []) { public function read($resource, $attributes = [], $columns = [], $order_by = [], $limit = []) {
$table = $this->get_table($resource); $table = $this->get_table($resource);
// Build the column listing. // Build the column listing.
@ -454,6 +456,7 @@ final class database extends \mysqli {
); );
} }
// Order by
if (is_array($order_by) === false) { if (is_array($order_by) === false) {
$order_by = [$order_by]; $order_by = [$order_by];
} }
@ -471,9 +474,27 @@ final class database extends \mysqli {
); );
} }
// Limit
if (is_array($limit) === false) {
$limit = [$limit];
}
if (count($limit) === 0) {
$limit = '';
} else {
$limit = ' limit ' .
implode(
',',
array_map(
[$this, 'escape'],
$limit
)
);
}
// Put everything together and return the result. // Put everything together and return the result.
$query = 'select ' . $columns . ' from ' . $query = 'select ' . $columns . ' from ' .
$this->escape_identifier($table) . $where . $order_by; $this->escape_identifier($table) . $where . $order_by . $limit;
$result = $this->query($query); $result = $this->query($query);
/** /**

102
api/thermostat_group.php
View File

@ -204,56 +204,70 @@ class thermostat_group extends cora\crud {
unset($attributes['address_radius']); unset($attributes['address_radius']);
} }
// Get all matching thermostat groups.
$other_thermostat_groups = $this->database->read(
'thermostat_group',
$attributes
);
// Get all the scores from the other thermostat groups
$scores = []; $scores = [];
foreach($other_thermostat_groups as $other_thermostat_group) { $limit_start = 0;
if( $limit_count = 1000;
isset($other_thermostat_group['temperature_profile'][$type]) === true &&
isset($other_thermostat_group['temperature_profile'][$type]['score']) === true && /**
$other_thermostat_group['temperature_profile'][$type]['score'] !== null && * Selecting lots of rows can eventually run PHP out of memory, so chunk
isset($other_thermostat_group['temperature_profile'][$type]['metadata']) === true && * this up into several queries to avoid that.
isset($other_thermostat_group['temperature_profile'][$type]['metadata']['generated_at']) === true && */
strtotime($other_thermostat_group['temperature_profile'][$type]['metadata']['generated_at']) > strtotime('-1 month') do {
) { // Get all matching thermostat groups.
// Skip thermostat_groups that are too far away. $other_thermostat_groups = $this->database->read(
'thermostat_group',
$attributes,
[], // columns
[], // order_by
[$limit_start, $limit_count] // limit
);
// Get all the scores from the other thermostat groups
foreach($other_thermostat_groups as $other_thermostat_group) {
if( if(
isset($address_radius) === true && isset($other_thermostat_group['temperature_profile'][$type]) === true &&
$this->haversine_great_circle_distance( isset($other_thermostat_group['temperature_profile'][$type]['score']) === true &&
$address_latitude, $other_thermostat_group['temperature_profile'][$type]['score'] !== null &&
$address_longitude, isset($other_thermostat_group['temperature_profile'][$type]['metadata']) === true &&
$other_thermostat_group['address_latitude'], isset($other_thermostat_group['temperature_profile'][$type]['metadata']['generated_at']) === true &&
$other_thermostat_group['address_longitude'] strtotime($other_thermostat_group['temperature_profile'][$type]['metadata']['generated_at']) > strtotime('-1 month')
) > $address_radius
) { ) {
continue; // Skip thermostat_groups that are too far away.
} if(
isset($address_radius) === true &&
$this->haversine_great_circle_distance(
$address_latitude,
$address_longitude,
$other_thermostat_group['address_latitude'],
$other_thermostat_group['address_longitude']
) > $address_radius
) {
continue;
}
// Ignore profiles with too few datapoints. Ideally this would be time- // Ignore profiles with too few datapoints. Ideally this would be time-
// based...so don't use a profile if it hasn't experienced a full year // based...so don't use a profile if it hasn't experienced a full year
// or heating/cooling system, but that isn't stored presently. A good // or heating/cooling system, but that isn't stored presently. A good
// approximation is to make sure there is a solid set of data driving // approximation is to make sure there is a solid set of data driving
// the profile. // the profile.
$required_delta_count = (($type === 'resist') ? 40 : 20); $required_delta_count = (($type === 'resist') ? 40 : 20);
if(count($other_thermostat_group['temperature_profile'][$type]['deltas']) < $required_delta_count) { if(count($other_thermostat_group['temperature_profile'][$type]['deltas']) < $required_delta_count) {
continue; continue;
} }
// Round the scores so they can be better displayed on a histogram or // Round the scores so they can be better displayed on a histogram or
// bell curve. // bell curve.
// TODO: Might be able to get rid of this? I don't think new scores are calculated at this level of detail anymore... // TODO: Might be able to get rid of this? I don't think new scores are calculated at this level of detail anymore...
// $scores[] = round( // $scores[] = round(
// $other_thermostat_group['temperature_profile'][$type]['score'], // $other_thermostat_group['temperature_profile'][$type]['score'],
// 1 // 1
// ); // );
$scores[] = $other_thermostat_group['temperature_profile'][$type]['score']; $scores[] = $other_thermostat_group['temperature_profile'][$type]['score'];
}
} }
}
$limit_start += $limit_count;
} while (count($other_thermostat_groups) === $limit_count);
sort($scores); sort($scores);