Cutlery/beestat
1
0
Fork 0
mirror of https://github.com/beestat/app.git synced 2025-07-09 03:04:07 -04:00
Code Actions Packages Releases Activity

Fixed #265 - Change logging from push to pull

Browse Source 
Deceptively simple commit message. This is a massive change that completely reworks the most fundamental part of the API. Not only does it remove the push logging, it also restructures logging to give better insight into what happens during an API call.
This commit is contained in:
Jon Ziebell 2020-03-15 08:12:28 -04:00
parent 9224d95dc9
commit 790aaf354e
19 changed files with 1131 additions and 1531 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
api/cora/api.php
View File

@ -38,11 +38,11 @@ abstract class api {
protected $setting;
/**
* Cora object.
* Request object.
*
* @var cora
* @var request
*/
protected $cora;
protected $request;
/**
* Construct and set the variables. The namespace is stripped from the
@ -57,7 +57,7 @@ abstract class api {
$class_parts = explode('\\', $this->resource);
$this->table = end($class_parts);
$this->database = database::get_instance();
$this->cora = cora::get_instance();
$this->request = request::get_instance();
$this->setting = setting::get_instance();
$this->session = session::get_instance();
}

17
api/cora/api_cache.php
View File

@ -42,15 +42,6 @@ class api_cache extends crud {
$attributes['key'] = $key;
$attributes['expires_at'] = date('Y-m-d H:i:s', time() + $duration);
$attributes['response_data'] = $response_data;
$attributes['request_resource'] = $api_call['resource'];
$attributes['request_method'] = $api_call['method'];
if(isset($api_call['arguments']) === true) {
$attributes['request_arguments'] = $api_call['arguments'];
}
else {
$attributes['request_arguments'] = null;
}
return $this->create($attributes);
}
@ -95,11 +86,11 @@ class api_cache extends crud {
*/
private function generate_key($api_call) {
return sha1(
'resource=' . $api_call['resource'] .
'method=' . $api_call['method'] .
'resource=' . $api_call->get_resource() .
'method=' . $api_call->get_method() .
'arguments=' . (
isset($api_call['arguments']) === true ?
json_encode($api_call['arguments']) : ''
$api_call->get_arguments() !== null ?
json_encode($api_call->get_arguments()) : ''
) .
'user_id=' . (
$this->session->get_user_id() !== null ?

277
api/cora/api_call.php Normal file
View File

@ -0,0 +1,277 @@
<?php
namespace cora;
/**
* Process a single API call
*
* @author Jon Ziebell
*/
final class api_call {
/**
* The API call resource.
*
* @var string
*/
private $resource;
/**
* The API call method.
*
* @var string
*/
private $method;
/**
* The API call arguments.
*
* @var array
*/
private $arguments;
/**
* The API call alias.
*
* @var string
*/
private $alias;
/**
* The current auto-alias. If an alias is not provided, an auto-alias is
* assigned.
*
* @var integer
*/
private static $auto_alias = 0;
/**
* The response of this API call.
*
* @var mixed
*/
private $response;
/**
* When this API call is cached until.
*
* @var string
*/
private $cached_until;
/**
* Construct a new API call.
*
* @throws exception if the resource is not provided.
* @throws exception if the method is not provided.
*
* @param array $api_call
*/
public function __construct($api_call) {
if(isset($api_call['resource']) === false) {
throw new \exception('Resource is required.', 1501);
}
if(isset($api_call['method']) === false) {
throw new \exception('Method is required.', 1502);
}
if(isset($api_call['arguments']) === false) {
$api_call['arguments'] = '{}';
}
$this->resource = $api_call['resource'];
$this->method = $api_call['method'];
$this->arguments = $this->parse_arguments($api_call['arguments']);
if(isset($api_call['alias']) === true) {
$this->alias = $api_call['alias'];
} else {
$this->alias = $this->get_auto_alias();
}
}
/**
* Process the API call.
*
* @throws exception If the method does not exist.
*/
public function process() {
$this->restrict_private();
$resource_instance = new $this->resource();
if(method_exists($resource_instance, $this->method) === false) {
throw new \exception('Method does not exist.', 1503);
}
// Caching! If this API call is configured for caching,
// $cache_config = $this->setting->get('cache');
if( // Is cacheable
isset($this->resource::$cache) === true &&
isset($this->resource::$cache[$this->method]) === true
) {
$api_cache_instance = new api_cache();
$api_cache = $api_cache_instance->retrieve($this);
if($api_cache !== null) {
// If there was a cache entry available, use that.
$this->response = $api_cache['response_data'];
$this->cached_until = date('Y-m-d H:i:s', strtotime($api_cache['expires_at']));
} else {
// Else just run the API call, then cache it.
$this->response = call_user_func_array(
[$resource_instance, $this->method],
$this->arguments
);
$api_cache = $api_cache_instance->cache(
$this,
$this->response,
$this->resource::$cache[$this->method]
);
$this->cached_until = date('Y-m-d H:i:s', strtotime($api_cache['expires_at']));
}
}
else { // Not cacheable
$this->response = call_user_func_array(
[$resource_instance, $this->method],
$this->arguments
);
}
}
/**
* Restrict private API calls.
*
* @throws exception If the method does not exist in the resource's
* public/private maps.
* @throws exception If the resource/method is private and the session is
* not valid.
*/
private function restrict_private() {
if(in_array($this->method, $this->resource::$exposed['private'])) {
$type = 'private';
} else if(in_array($this->method, $this->resource::$exposed['public'])) {
$type = 'public';
} else {
throw new exception('Method is not mapped.', 1504);
}
$session = session::get_instance();
if(
$type === 'private' &&
$session->is_valid() === false
) {
throw new exception('Session is expired.', 1505, false);
}
}
/**
* Gets an array of arguments in the correct order for the method being
* called.
*
* @param string $json The arguments JSON.
*
* @throws exception If the arguments in the api_call were not valid JSON.
*
* @return array The requested arguments.
*/
private function parse_arguments($json) {
$arguments = [];
// Arguments are not strictly required. If a method requires them then you
// will still get an error, but they are not required by the API.
if($json !== null) {
// All arguments are sent in the "arguments" key as JSON.
$decoded = json_decode($json, true);
if($decoded === null) {
throw new exception('Arguments are not valid JSON.', 1506);
}
$reflection_method = new \ReflectionMethod(
$this->resource,
$this->method
);
$parameters = $reflection_method->getParameters();
foreach($parameters as $parameter) {
if(isset($decoded[$parameter->getName()]) === true) {
$argument = $decoded[$parameter->getName()];
}
else {
if($parameter->isOptional() === true) {
$argument = $parameter->getDefaultValue();
} else {
$argument = null;
}
}
$arguments[] = $argument;
}
}
return $arguments;
}
/**
* Get the resource.
*
* @return string
*/
public function get_resource() {
return $this->resource;
}
/**
* Get the method.
*
* @return string
*/
public function get_method() {
return $this->method;
}
/**
* Get the arguments.
*
* @return string
*/
public function get_arguments() {
return $this->arguments;
}
/**
* Get the alias.
*
* @return string
*/
public function get_alias() {
return $this->alias;
}
/**
* Get the response.
*
* @return mixed
*/
public function get_response() {
return $this->response;
}
/**
* Get cached_until property.
*
* @return number
*/
public function get_cached_until() {
return $this->cached_until;
}
/**
* Get the next auto-alias.
*
* @return number
*/
private function get_auto_alias() {
return api_call::$auto_alias++;
}
}

58
api/cora/api_log2.php Normal file
View File

@ -0,0 +1,58 @@
<?php
namespace cora;
/**
* Stores a log of API requests and responses. Intended usage is to process
* the request to the end (exception or not) and then log it.
*
* @author Jon Ziebell
*/
final class api_log2 extends crud {
/**
* Insert an item into the api_log resource. Force the IP to the request IP
* and disallow overriding the timestamp.
*
* @param array $attributes The attributes to insert.
*
* @return int The ID of the inserted row.
*/
public function create($attributes) {
// Insert using the transactionless connection.
$database = database::get_transactionless_instance();
return $database->create($this->resource, $attributes);
}
/**
* Get the number of requests since a given timestamp for a given IP
* address. Handy for rate limiting.
*
* @param string $ip_address The IP to look at.
* @param int $timestamp The timestamp to check from.
*
* @return int The number of requests on or after $timestamp.
*/
public function get_number_requests_since($ip_address, $timestamp) {
$ip_address_escaped = $this->database->escape(ip2long($ip_address));
$timestamp_escaped = $this->database->escape(
date('Y-m-d H:i:s', $timestamp)
);
$query = '
select
count(*) `number_requests_since`
from
`api_log2`
where
`ip_address` = ' . $ip_address_escaped . '
and `timestamp` >= ' . $timestamp_escaped . '
';
$result = $this->database->query($query);
$row = $result->fetch_assoc();
return $row['number_requests_since'];
}
}

1115
api/cora/cora.php
View File

File diff suppressed because it is too large Load Diff

110
api/cora/database.php
View File

@ -2,15 +2,6 @@
namespace cora;
/**
* This exception is thrown by database->query if the query failed due to a
* duplicate entry in the database.
*
* @author Jon Ziebell
*/
final class DuplicateEntryException extends \Exception {
};
/**
* This is a MySQLi database wrapper. It provides access to some basic
* functions like select, insert, and update. Those functions automatically
@ -61,25 +52,18 @@ final class database extends \mysqli {
private $transaction_started = false;
/**
* The total number of queries executed.
* The executed queries.
*
* @var int
*/
private $query_count = 0;
private $queries = [];
/**
* The total time all queries have taken to execute.
* The request object.
*
* @var float
* @var request
*/
private $query_time = 0;
/**
* The cora object.
*
* @var cora
*/
private $cora;
private $request;
/**
* The setting object.
@ -100,7 +84,7 @@ final class database extends \mysqli {
* @throws \Exception If failing to connect to the database.
*/
public function __construct() {
$this->cora = cora::get_instance();
$this->request = request::get_instance();
$this->setting = setting::get_instance();
parent::__construct(
@ -113,26 +97,30 @@ final class database extends \mysqli {
// does not have a native type for decimals so that gets left behind.
parent::options(MYSQLI_OPT_INT_AND_FLOAT_NATIVE, true);
// $this->connect_error = 'this is broken';
if($this->connect_error !== null) {
$this->cora->set_error_extra_info(
throw new exception(
'Could not connect to database.',
1200,
true,
[
'database_error' => $this->connect_error
]
);
throw new \Exception('Could not connect to database.', 1200);
}
$database_name = $this->setting->get('database_name');
if($database_name !== null) {
$success = $this->select_db($database_name);
if($success === false) {
$this->cora->set_error_extra_info(
throw new exception(
'Could not select database.',
1208,
true,
[
'database_error' => $this->error
]
);
throw new \Exception('Could not select database.', 1208);
}
}
}
@ -188,7 +176,7 @@ final class database extends \mysqli {
if($this->transaction_started === false) {
$result = $this->query('start transaction');
if($result === false) {
throw new \Exception('Failed to start database transaction.', 1201);
throw new exception('Failed to start database transaction.', 1201);
}
$this->transaction_started = true;
}
@ -204,7 +192,7 @@ final class database extends \mysqli {
$this->transaction_started = false;
$result = $this->query('commit');
if($result === false) {
throw new \Exception('Failed to commit database transaction.', 1202);
throw new exception('Failed to commit database transaction.', 1202);
}
}
}
@ -220,7 +208,7 @@ final class database extends \mysqli {
$this->transaction_started = false;
$result = $this->query('rollback');
if($result === false) {
throw new \Exception('Failed to rollback database transaction.', 1203);
throw new exception('Failed to rollback database transaction.', 1203);
}
}
}
@ -279,12 +267,14 @@ final class database extends \mysqli {
return '`' . $identifier . '`';
}
else {
$this->cora->set_error_extra_info(
throw new exception(
'Query identifier is invalid.',
1204,
true,
[
'identifier' => $identifier
]
);
throw new \Exception('Query identifier is invalid.', 1204);
}
}
@ -309,7 +299,7 @@ final class database extends \mysqli {
else if(is_array($value) === true) {
if(isset($value['operator']) === true) {
if(in_array($value['operator'], ['>', '<', '=', '>=', '<=', 'between']) === false) {
throw new \Exception('Invalid operator', 1213);
throw new exception('Invalid operator', 1213);
}
if($value['operator'] === 'between') {
return $this->escape_identifier($column) . ' between ' . $this->escape($value['value'][0]) . ' and ' . $this->escape($value['value'][1]);
@ -378,28 +368,23 @@ final class database extends \mysqli {
$result = parent::query($query);
$stop = microtime(true);
if($result === false) {
$database_error = $this->error;
$this->rollback_transaction();
$this->queries[] = [
'query' => $query,
'time' => (($stop - $start) * 1000)
];
$this->cora->set_error_extra_info(
if($result === false) {
$this->rollback_transaction();
throw new exception(
'Database query failed.',
1206,
true,
[
'database_error' => $database_error,
'database_error' => $this->error,
'query' => $query
]
);
if(stripos($database_error, 'duplicate entry') !== false) {
throw new DuplicateEntryException('Duplicate database entry.', 1205);
}
else {
throw new \Exception('Database query failed.', 1206);
}
}
// Don't log info about transactions...they're a wash
$this->query_count++;
$this->query_time += ($stop - $start);
return $result;
}
@ -541,7 +526,7 @@ final class database extends \mysqli {
) {
foreach($resource::$converged as $column => $column_properties) {
if(isset($row[$column]) === true) {
throw new \Exception('Column `' . $column . '` exists; cannot be overwritten by converged column.', 1212);
throw new exception('Column `' . $column . '` exists; cannot be overwritten by converged column.', 1212);
}
$row[$column] = (isset($row['converged'][$column]) === true) ? $row['converged'][$column] : null;
}
@ -641,7 +626,7 @@ final class database extends \mysqli {
// Check for errors
if(isset($attributes[$table . '_id']) === false) {
throw new \Exception('ID is required for update.', 1214);
throw new exception('ID is required for update.', 1214);
}
// Extract the ID.
@ -650,7 +635,7 @@ final class database extends \mysqli {
// Check for errors
if(count($attributes) === 0) {
throw new \Exception('Updates require at least one attribute.', 1207);
throw new exception('Updates require at least one attribute.', 1207);
}
// Converge the diverged attributes.
@ -775,7 +760,7 @@ final class database extends \mysqli {
* @return int The query count.
*/
public function get_query_count() {
return $this->query_count;
return count($this->queries);
}
/**
@ -784,7 +769,20 @@ final class database extends \mysqli {
* @return float The total execution time.
*/
public function get_query_time() {
return $this->query_time;
$query_time = 0;
foreach ($this->queries as $query) {
$query_time += $query['time'];
}
return $query_time;
}
/**
* Gets the time taken to execute all of the queries.
*
* @return float The total execution time.
*/
public function get_queries() {
return $this->queries;
}
/**
@ -878,10 +876,10 @@ final class database extends \mysqli {
');
$row = $result->fetch_assoc();
if($row['lock'] === 0) {
throw new \Exception('Lock not established by this thread.', 1210);
throw new exception('Lock not established by this thread.', 1210);
}
else if($row['lock'] === null) {
throw new \Exception('Lock does not exist.', 1211);
throw new exception('Lock does not exist.', 1211);
}
}

7
api/cora/exception.php
View File

@ -14,12 +14,17 @@ namespace cora;
* @author Jon Ziebell
*/
final class exception extends \Exception {
public function __construct($message, $code, $reportable = true) {
public function __construct($message, $code, $reportable = true, $extra = null) {
$this->reportable = $reportable;
$this->extra = $extra;
return parent::__construct($message, $code, null);
}
public function getReportable() {
return $this->reportable;
}
public function getExtraInfo() {
return $this->extra;
}
}

68
api/cora/json_path.php
View File

@ -1,68 +0,0 @@
<?php
namespace cora;
/**
* For lack of a better name, I'm calling this the json_path class. It does
* not provide a full implementation of JSONPath and may never do so as a lot
* of the functionality is overkill or not appropriate. This provides super
* basic functionality only.
*
* @author Jon Ziebell
*/
final class json_path {
/**
* Given a data array and a path, return the requested data. Example paths:
* foo.bar
* foo.0.bar
* foo.bar.baz
* foo.bar.baz[] To get all baz inside bar array.
*
* @param mixed $data The data referenced by $json_path.
* @param string $json_path The path.
*
* @return mixed The requested data.
*/
public function evaluate($data, $json_path) {
// Split up the path into an array.
$key_array = explode('.', $json_path);
return $this->extract_key($data, $key_array);
}
/**
* Recursively extract keys from the data array. Basically, $data.foo.bar is
* represented by sending $data, ['foo', 'bar']. If you end a key with
* [] it will get every instance of that value inside the current array.
*
* @param mixed $data The data to traverse. You can send anything here but
* don't expect to not get an exception if you try to traverse things like
* non-existent indices.
* @param array $key_array The array keys to use to traverse $data.
*
* @throws \Exception If any of the provided keys do not exist in the data.
*
* @return mixed The requested data.
*/
private function extract_key($data, $key_array) {
$key = array_shift($key_array);
if($key === null) {
return $data;
}
else {
if(substr($key, -2) === '[]') {
return array_column($data, substr($key, 0, strlen($key) - 2));
}
else {
if(array_key_exists($key, $data) === false) {
throw new \Exception('Invalid path string.', 1500);
}
else {
return $this->extract_key($data[$key], $key_array);
}
}
}
}
}

659
api/cora/request.php Normal file
View File

@ -0,0 +1,659 @@
<?php
namespace cora;
/**
* Workhorse for processing an API request. This has all of the core
* functionality.
*
* @author Jon Ziebell
*/
final class request {
/**
* The singleton.
*/
private static $instance;
/**
* The timestamp when processing of the API request started.
*
* @var int
*/
private $begin_timestamp;
/**
* The original request passed to this object, usually $_REQUEST.
*
* @var array
*/
private $request;
/**
* A list of all of the API calls extracted from the request.
*
* @var array
*/
private $api_calls;
/**
* The current API user.
*
* @var array
*/
private $api_user;
/**
* The actual response in array form.
*
* @var array
*/
private $response;
/**
* Detailed error information for use in debugging.
*
* @var array
*/
private $error_detail = [];
/**
* How long the API call is cached for. Used when setting the
* beestat-cached-until header.
*
* @var array
*/
private $cached_until = [];
/**
* This stores the currently executing API call. If that API call were to
* fail, I need to know which one I was running in order to propery log the
* error.
*
* @var array
*/
private $current_api_call = null;
/**
* The headers to output in the shutdown handler.
*
* @var array
*/
private $headers;
/**
* This is necessary because of the shutdown handler. According to the PHP
* documentation and various bug reports, when the shutdown function
* executes the current working directory changes back to root.
* https://bugs.php.net/bug.php?id=36529. This is cool and all but it breaks
* the autoloader. My solution for this is to just change the working
* directory back to what it was when the script originally ran.
*
* Obviously I could hardcode this but then users would have to configure
* the cwd when installing Cora. This handles it automatically and seems to
* work just fine. Note that if the class that the autoloader needs is
* already loaded, the shutdown handler won't break. So it's usually not a
* problem but this is a good thing to fix.
*
* @var string
*/
private $current_working_directory;
/**
* Save the request variables for use later on. If unset, they are defaulted
* to null. Any of these values being null will throw an exception as soon
* as you try to process the request. The reason that doesn't happen here is
* so that I can store exactly what was sent to me for logging purposes.
*/
private function __construct() {
$this->begin_timestamp = microtime(true);
// See class variable documentation for reasoning.
$this->current_working_directory = getcwd();
}
/**
* Use this function to instantiate this class instead of calling new
* request() (which isn't allowed anyways).
*
* @return cora A new cora object or the already created one.
*/
public static function get_instance() {
if(isset(self::$instance) === false) {
self::$instance = new self();
}
return self::$instance;
}
/**
* Execute the request. It is run through the rate limiter, checked for
* errors, etc, then processed.
*
* @param array $request Basically just $_REQUEST or a slight mashup of it
* for batch requests.
*/
public function process($request) {
$this->request = $request;
$this->rate_limit();
$this->force_ssl();
$this->set_api_user();
$this->set_api_calls();
$this->validate_aliases();
$this->set_default_headers();
// Touch the session, if there is one. If the API user does not have a
// session key set it will pull from the cookie.
$session = session::get_instance();
$session->touch($this->api_user['session_key']);
// Process each request.
foreach($this->api_calls as $api_call) {
$api_call->process();
}
$this->set_cached_until_header();
}
/**
* Build a list of API calls from the request. For a single request, it's
* just the request. For batch requests, add each item in the batch
* parameter to this array.
*
* @throws exception If this is a batch request and the batch data is not
* valid JSON
* @throws exception If this is a batch request and it exceeds the maximum
* number of api calls allowed in one batch.
*/
private function set_api_calls() {
$setting = setting::get_instance();
$this->api_calls = [];
if(isset($this->request['batch']) === true) {
$batch = json_decode($this->request['batch'], true);
if($batch === null) {
throw new exception('Batch is not valid JSON.', 1012);
}
$batch_limit = $setting->get('batch_limit');
if($batch_limit !== null && count($batch) > $batch_limit) {
throw new exception('Batch limit exceeded.', 1013);
}
foreach($batch as $api_call) {
$this->api_calls[] = new api_call($api_call);
}
}
else {
$this->api_calls[] = new api_call($this->request);
}
}
/**
* Check for any issues with the aliases.
*
* @throws exception If any duplicate aliases are used.
*/
private function validate_aliases() {
$aliases = [];
foreach($this->api_calls as $api_call) {
$aliases[] = $api_call->get_alias();
}
$number_aliases = count($aliases);
$number_unique_aliases = count(array_unique($aliases));
// Check for duplicates.
if($number_aliases !== $number_unique_aliases) {
throw new exception('Duplicate alias.', 1018);
}
}
/**
* Check to see if the request from the current IP address needs to be rate
* limited. If $requests_per_minute is null then there is no rate limiting.
*
* @throws exception If over the rate limit.
*/
private function rate_limit() {
$setting = setting::get_instance();
$requests_per_minute = $setting->get('requests_per_minute');
if($requests_per_minute === null) {
return false;
}
$api_log_resource = new api_log2();
$requests_this_minute = $api_log_resource->get_number_requests_since(
$_SERVER['REMOTE_ADDR'],
(time() - 60)
);
// A couple quick error checks
if($requests_this_minute > $requests_per_minute) {
throw new exception('Rate limit reached.', 1005);
}
}
/**
* Force secure connections.
*
* @throws exception if not secure.
*/
private function force_ssl() {
$setting = setting::get_instance();
if($setting->get('force_ssl') === true && empty($_SERVER['HTTPS']) === true) {
throw new exception('Request must be sent over HTTPS.', 1006);
}
}
/**
* Set the current API user based on the request API key.
*
* @throws exception if the API key is not set.
* @throws exception if the API key is not valid.
*/
private function set_api_user() {
// Make sure the API key that was sent is present and valid.
if(isset($this->request['api_key']) === false) {
throw new exception('API Key is required.', 1000);
}
$api_user_resource = new api_user();
$api_users = $api_user_resource->read(['api_key' => $this->request['api_key']]);
if(count($api_users) !== 1) {
throw new exception('API key is invalid.', 1003);
} else {
$this->api_user = $api_users[0];
}
}
/**
* Get the current API user.
*
* @return array
*/
public function get_api_user() {
return $this->api_user;
}
/**
* Log the request and response to the database. The logged response is
* truncated to 16kb for sanity.
*/
private function log() {
$database = database::get_instance();
$session = session::get_instance();
$setting = setting::get_instance();
$api_log_resource = new api_log2();
// If exception.
if(isset($this->response['data']['error_code']) === true) {
$api_log_resource->create([
'user_id' => $session->get_user_id(),
'api_user_id' => $this->api_user['api_user_id'],
'ip_address' => ip2long($_SERVER['REMOTE_ADDR']),
'timestamp' => date('Y-m-d H:i:s', $this->begin_timestamp),
'request' => $this->request,
'response' => $this->response,
'error_code' => $this->response['data']['error_code'],
'error_detail' => $this->error_detail,
'total_time' => $this->total_time,
'query_count' => $database->get_query_count(),
'query_time' => $database->get_query_time(),
]);
}
else {
$user_resource = new \user();
$user = $user_resource->get($session->get_user_id());
$api_log_resource->create([
'user_id' => $session->get_user_id(),
'api_user_id' => $this->api_user['api_user_id'],
'ip_address' => ip2long($_SERVER['REMOTE_ADDR']),
'timestamp' => date('Y-m-d H:i:s', $this->begin_timestamp),
'request' => $this->request,
'response' => ($user['debug'] === true) ? $this->response : null,
'total_time' => $this->total_time,
'query_count' => $database->get_query_count(),
'query_time' => $database->get_query_time(),
]);
}
}
/**
* Sets the headers that should be used for this API call. This is useful
* for doing things like returning files from the API where the content-type
* is no longer application/json. This replaces all headers; headers are not
* outputted to the browser until all API calls have completed, so the last
* call to this function will win.
*
* @param array $headers The headers to output.
* @param bool $custom_response Whether or not to wrap the response with the
* Cora data or just output the API call's return value.
*
* @throws exception If this is a batch request and a custom response was
* requested.
* @throws exception If this is a batch request and the content type was
* altered from application/json
* @throws exception If this is not a batch request and the content type was
* altered from application/json without a custom response.
*/
public function set_headers($headers, $custom_response = false) {
if(isset($this->request['batch']) === true) {
if($custom_response === true) {
throw new exception('Batch API requests can not use a custom response.', 1015);
}
if($this->content_type_is_json($headers) === false) {
throw new exception('Batch API requests must return JSON.', 1014);
}
}
else {
// Not a batch request
if($custom_response === false && $this->content_type_is_json($headers) === false) {
throw new exception('Non-custom responses must return JSON.', 1016);
}
}
$this->headers = $headers;
}
/**
* Return whether or not the current output headers indicate that the
* content type is JSON. This is mostly just used to make sure that batch
* API calls output JSON.
*
* @param array $headers The headers to look at.
*
* @return bool Whether or not the output has a content type of
* application/json
*/
private function content_type_is_json($headers) {
return isset($headers['Content-type']) === true
&& stristr($headers['Content-type'], 'application/json') !== false;
}
/**
* Output whatever the headers are currently set to.
*/
private function output_headers() {
foreach($this->headers as $key => $value) {
header($key . ': ' . $value);
}
}
/**
* Resets the headers to default. Have to do this in case one of the API
* calls changes them and there was an error to handle.
*/
private function set_default_headers() {
$this->headers['Content-type'] = 'application/json; charset=UTF-8';
}
/**
* Set the beestat-cached-until header.
*/
private function set_cached_until_header() {
$beestat_cached_until = [];
foreach($this->api_calls as $api_call) {
$cached_until = $api_call->get_cached_until();
if($cached_until !== null) {
$beestat_cached_until[$api_call->get_alias()] = $api_call->get_cached_until();
}
}
if(count($beestat_cached_until) > 0) {
if(isset($this->request['batch']) === true) {
$this->headers['beestat-cached-until'] = json_encode($beestat_cached_until);
} else {
$this->headers['beestat-cached-until'] = reset($beestat_cached_until);
}
}
}
/**
* Override of the default PHP error handler. Sets the error response then
* dies and lets the shutdown handler take over.
*
* @param int $error_code The error number from PHP.
* @param string $error_message The error message.
* @param string $error_file The file the error happend in.
* @param int $error_line The line of the file the error happened on.
*
* @return string The JSON response with the error details.
*/
public function error_handler($error_code, $error_message, $error_file, $error_line) {
$this->set_error_response(
$error_message,
$error_code,
true
);
$this->error_detail['file'] = $error_file;
$this->error_detail['line'] = $error_line;
$this->error_detail['trace'] = debug_backtrace(false);
try {
$database = database::get_instance();
$this->error_detail['queries'] = $database->get_queries();
} catch(Exception $e) {}
die(); // Do not continue execution; shutdown handler will now run.
}
/**
* Override of the default PHP exception handler. Sets the error response
* then dies and lets the shutdown handler take over.
*
* @param Exception $e The exception.
*/
public function exception_handler($e) {
$this->set_error_response(
$e->getMessage(),
$e->getCode(),
(method_exists($e, 'getReportable') === true ? $e->getReportable() : true)
);
$this->error_detail['file'] = $e->getFile();
$this->error_detail['line'] = $e->getLine();
$this->error_detail['trace'] = $e->getTrace();
$this->error_detail['extra'] = (method_exists($e, 'getExtraInfo') === true ? $e->getExtraInfo() : null);
try {
$database = database::get_instance();
$this->error_detail['queries'] = $database->get_queries();
} catch(Exception $e) {}
die(); // Do not continue execution; shutdown handler will now run.
}
/**
* Handle all exceptions by generating a JSON response with the error
* details. If debugging is enabled, a bunch of other information is sent
* back to help out.
*
* There are a few places that call this function to set an error response,
* so this can't just be done in the exception handler alone. If an error
* occurs, rollback the current transaction.
*
* @param string $error_message The error message.
* @param mixed $error_code The supplied error code.
* @param array $reportable Whether or not the error is reportable.
*/
public function set_error_response($error_message, $error_code, $reportable) {
$setting = setting::get_instance();
$session = session::get_instance();
// I guess if this fails then things are really bad, but let's at least
// protect against additional exceptions if the database connection or
// similar fails.
try {
$database = database::get_instance();
$database->rollback_transaction();
} catch(\Exception $e) {}
$this->response = [
'success' => false,
'data' => [
'error_message' => $error_message,
'error_code' => $error_code
]
];
// Send data to Sentry for error logging.
// https://docs.sentry.io/development/sdk-dev/event-payloads/
$api_user_id = $this->api_user['api_user_id'];
if (
$reportable === true &&
$setting->get('sentry_key') !== null &&
$setting->get('sentry_project_id') !== null &&
$api_user_id === 1 &&
false // Temporarily disabling; over rate limit anyways
) {
$data = [
'event_id' => str_replace('-', '', exec('uuidgen -r')),
'timestamp' => date('c'),
'logger' => 'cora',
'platform' => 'php',
'level' => 'error',
'tags' => [
'error_code' => $error_code,
'api_user_id' => $api_user_id
],
'extra' => [
'error_file' => $this->error_detail['file'],
'error_line' => $this->error_detail['line'],
'error_trace' => $this->error_detail['trace'],
'error_extra' => $this->error_detail['extra']
],
'exception' => [
'type' => 'Exception',
'value' => $error_message,
'handled' => false
],
'user' => [
'id' => $session->get_user_id(),
'ip_address' => $_SERVER['REMOTE_ADDR']
]
];
exec(
'curl ' .
'-H "Content-Type: application/json" ' .
'-H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=' . $setting->get('sentry_key') . '" ' .
'--silent ' . // silent; keeps logs out of stderr
'--show-error ' . // override silent on failure
'--max-time 10 ' .
'--connect-timeout 5 ' .
'--data \'' . json_encode($data) . '\' ' .
'"https://sentry.io/api/' . $setting->get('sentry_project_id') . '/store/" > /dev/null &'
);
}
}
/**
* Executes when the script finishes. If there was an error that somehow
* didn't get caught, then this will find it with error_get_last and return
* appropriately. Note that error_get_last() will only get something when an
* error wasn't caught by my error/exception handlers. The default PHP error
* handler fills this in. Doesn't do anything if an exception was thrown due
* to the rate limit.
*
* @throws \Exception If a this was a batch request but one of the api calls
* changed the content-type to anything but the default.
*/
public function shutdown_handler() {
// Since the shutdown handler is rather verbose in what it has to check for
// and do, it's possible it will fail or detect an error that needs to be
// handled. For example, someone could return binary data from an API call
// which will fail a json_encode, or someone could change the headers in a
// batch API call, which isn't allowed. I can't throw an exception since I'm
// already in the shutdown handler...it will be caught but it won't execute
// a new shutdown handler and no output will be sent to the client. I just
// have to handle all problems manually.
try {
$this->total_time = round((microtime(true) - $this->begin_timestamp) * 1000);
// Fix the current working directory. See documentation on this class
// variable for details.
chdir($this->current_working_directory);
// If I didn't catch an error/exception with my handlers, look here...this
// will catch fatal errors that I can't.
$error = error_get_last();
if($error !== null) {
$this->set_error_response(
$error['message'],
$error['type'],
true
);
$this->error_detail['file'] = $error['file'];
$this->error_detail['line'] = $error['line'];
$this->error_detail['trace'] = debug_backtrace(false);
try {
$database = database::get_instance();
$this->error_detail['queries'] = $database->get_queries();
} catch(Exception $e) {}
}
// If the response has already been set by one of the error handlers, end
// execution here and just log & output the response.
if(isset($this->response) === true) {
// Don't log anything for rate limit breaches.
if($this->response['data']['error_code'] !== 1005) {
$this->log();
}
// Override whatever headers might have already been set.
$this->set_default_headers();
$this->output_headers();
die(json_encode($this->response));
}
else {
// If we got here, no errors have occurred.
// For non-custom responses, build the response, log it, and output it.
$this->response = ['success' => true];
if(isset($this->request['batch']) === true) {
$this->response['data'] = [];
foreach($this->api_calls as $api_call) {
$this->response['data'][$api_call->get_alias()] = $api_call->get_response();
}
}
else {
$this->response['data'] = $this->api_calls[0]->get_response();
}
// Log all of the API calls that were made.
$this->log();
// Output the response
$this->output_headers();
die(json_encode($this->response));
}
}
catch(\Exception $e) {
$this->set_error_response(
$e->getMessage(),
$e->getCode(),
(method_exists($e, 'getReportable') === true ? $e->getReportable() : true)
);
$this->error_detail['file'] = $e->getFile();
$this->error_detail['line'] = $e->getLine();
$this->error_detail['trace'] = $e->getTrace();
$this->error_detail['extra'] = (method_exists($e, 'getExtraInfo') === true ? $e->getExtraInfo() : null);
try {
$database = database::get_instance();
$this->error_detail['queries'] = $database->get_queries();
} catch(Exception $e) {}
$this->set_default_headers();
$this->output_headers();
die(json_encode($this->response));
}
}
}

49
api/cora/session.php
View File

@ -10,18 +10,11 @@ namespace cora;
final class session {
/**
* The session_key for this session.
* The current session.
*
* @var string
* @var array
*/
private $session_key = null;
/**
* The user_id for this session.
*
* @var int
*/
private $user_id = null;
private $session;
/**
* The singleton.
@ -91,7 +84,7 @@ final class session {
$database = database::get_instance();
$session_key = $this->generate_session_key();
$database->create(
$this->session = $database->create(
'cora\session',
[
'session_key' => $session_key,
@ -126,9 +119,6 @@ final class session {
}
}
$this->session_key = $session_key;
$this->user_id = $user_id;
return $session_key;
}
@ -182,7 +172,7 @@ final class session {
return false;
}
$database->update(
$this->session = $database->update(
'cora\session',
[
'session_id' => $session['session_id'],
@ -191,8 +181,7 @@ final class session {
]
);
$this->session_key = $session['session_key'];
$this->user_id = $session['user_id'];
return true;
}
else {
$this->delete_cookie('session_key');
@ -220,16 +209,15 @@ final class session {
public function delete($session_key = null) {
$database = database::get_instance();
if($session_key === null) {
$session_key = $this->session_key;
$session_key = $this->session['session_key'];
}
$sessions = $database->read('cora\session', ['session_key' => $session_key]);
if(count($sessions) === 1) {
$database->delete('cora\session', $sessions[0]['session_id']);
// Remove these if the current session got logged out.
if($session_key === $this->session_key) {
$this->session_key = null;
$this->user_id = null;
if($session_key === $this->session['session_key']) {
$this->session = null;
}
return true;
}
@ -244,11 +232,16 @@ final class session {
* @return int The current user_id.
*/
public function get_user_id() {
return $this->user_id;
return $this->session['user_id'];
}
public function delete_user_id() {
$this->user_id = null;
/**
* Get the session.
*
* @return array The session.
*/
public function get() {
return $this->session;
}
/**
@ -309,4 +302,12 @@ final class session {
$this->set_cookie($name, '', time() - 86400);
}
/**
* Check if the session is valid.
*
* @return bool Whether or not the session is valid.
*/
public function is_valid() {
return $this->session !== null;
}
}

10
api/cora/setting.example.php
View File

@ -153,16 +153,6 @@ final class setting {
'database_password' => '',
'database_name' => '',
/**
* Influx database connection information. This is where most logging is
* sent to.
*/
'influx_database_host' => '',
'influx_database_port' => 8086,
'influx_database_name' => '',
'influx_database_username' => '',
'influx_database_password' => '',
/**
* Key and project id obtained from the Sentry DSN. See sentry.io.
*/

12
api/ecobee.php
View File

@ -15,10 +15,8 @@ class ecobee extends external_api {
]
];
protected static $log_influx = true;
protected static $log_mysql = 'error';
protected static $influx_retention_policy = '30d';
protected static $log_mysql = 'all';
protected static $log_mysql_verbose = false;
protected static $cache = false;
protected static $cache_for = null;
@ -215,7 +213,7 @@ class ecobee extends external_api {
if ($response === null) {
// If this hasn't already been logged, log the error.
if($this::$log_mysql !== 'all') {
$this->log_mysql($curl_response);
$this->log_mysql($curl_response, true);
}
throw new Exception('Invalid JSON');
}
@ -239,7 +237,7 @@ class ecobee extends external_api {
else if (isset($response['status']) === true && $response['status']['code'] === 16) {
// Token has been deauthorized by user. You must re-request authorization.
if($this::$log_mysql !== 'all') {
$this->log_mysql($curl_response);
$this->log_mysql($curl_response, true);
}
$this->api('ecobee_token', 'delete', $ecobee_token['ecobee_token_id']);
$this->api('user', 'log_out', ['all' => true]);
@ -248,7 +246,7 @@ class ecobee extends external_api {
else if (isset($response['status']) === true && $response['status']['code'] !== 0) {
// Any other error
if($this::$log_mysql !== 'all') {
$this->log_mysql($curl_response);
$this->log_mysql($curl_response, true);
}
throw new Exception($response['status']['message']);
}

106
api/external_api.php
View File

@ -9,12 +9,6 @@
*/
class external_api extends cora\api {
/**
* Whether or not to log the API call to Influx. This will only log the
* event and basic timing information; no detail.
*/
protected static $log_influx = true;
/**
* Whether or not to log the API call to MySQL. This will log the entire
* request and response in full detail. Valid values are "error", "all", and
@ -23,10 +17,9 @@ class external_api extends cora\api {
protected static $log_mysql = 'error';
/**
* Default retention policy when inserting data. Autogen is the default
* infinite one; also available is 30d.
* Whether or not to include the request and response in non-errored logs.
*/
protected static $influx_retention_policy = 'autogen';
protected static $log_mysql_verbose = true;
/**
* Whether or not to cache API calls. This will store a hash of the request
@ -59,7 +52,6 @@ class external_api extends cora\api {
}
$this->request_timestamp = time();
$this->request_timestamp_microtime = $this->microtime();
$curl_handle = curl_init();
curl_setopt($curl_handle, CURLOPT_URL, $arguments['url']);
@ -79,7 +71,7 @@ class external_api extends cora\api {
curl_setopt($curl_handle, CURLOPT_POSTFIELDS, $arguments['post_fields']);
}
if($this::$log_influx !== false || $this::$log_mysql !== false) {
if($this::$log_mysql !== false) {
curl_setopt($curl_handle, CURLINFO_HEADER_OUT, true);
}
@ -98,34 +90,28 @@ class external_api extends cora\api {
if($cache_entry === null) {
$curl_response = curl_exec($curl_handle);
$this->curl_info = curl_getinfo($curl_handle);
if($curl_response === false || curl_errno($curl_handle) !== 0) {
$this->cora->set_error_extra_info([
'curl_error' => curl_error($curl_handle)
]);
// Error logging
if($this::$log_influx === true) {
$this->log_influx(
$this->resource . '_api_log',
true
);
}
if($this::$log_mysql === 'all' || $this::$log_mysql === 'error') {
$this->log_mysql($curl_response);
$this->log_mysql($curl_response, true);
}
throw new cora\exception('Could not connect to ' . $this->resource . '.', 10600, false);
throw new cora\exception(
'Could not connect to external API.',
10600,
false,
[
'resource' => $this->resource,
'curl_error' => curl_error($curl_handle)
]
);
}
// General (success) logging
if($this::$log_influx === true) {
$this->log_influx(
$this->resource . '_api_log',
false
);
}
if($this::$log_mysql === 'all') {
$this->log_mysql($curl_response);
}
@ -218,63 +204,27 @@ class external_api extends cora\api {
);
}
/**
* Log to InfluxDB/Grafana.
*
* @param array $measurement Which measurement to log as.
* @param boolean $exception Whether or not this was an exception (failure
* to connect, etc).
*/
private function log_influx($measurement, $exception) {
$this->api(
'logger',
'log_influx',
[
'measurement' => $measurement,
'tags' => [
'user_id' => $this->session->get_user_id(),
'api_user_id' => $this->cora->get_api_user()['api_user_id'],
'exception' => $exception === true ? '1' : '0'
],
'fields' => [
'http_code' => (int) $this->curl_info['http_code'],
'connect_time' => round($this->curl_info['connect_time'], 4)
],
'timestamp' => $this->request_timestamp_microtime,
'retention_policy' => $this::$influx_retention_policy
]
);
}
/**
* Log to MySQL with the complete details.
*
* @param array $curl_response The response of the cURL request.
* @param boolean $force_verbose Whether or not to force verbose logging.
*/
protected function log_mysql($curl_response) {
protected function log_mysql($curl_response, $force_verbose = false) {
$attributes = [
'api_user_id' => $this->request->get_api_user()['api_user_id'],
'request_timestamp' => date('Y-m-d H:i:s', $this->request_timestamp)
];
if($this::$log_mysql_verbose === true || $force_verbose === true) {
$attributes['request'] = $this->curl_info;
$attributes['response'] = $curl_response;
}
$this->api(
($this->resource . '_api_log'),
'create',
[
'attributes' => [
'api_user_id' => $this->cora->get_api_user()['api_user_id'],
'request_timestamp' => date('Y-m-d H:i:s', $this->request_timestamp),
'request' => $this->curl_info,
'response' => $curl_response,
]
]
['attributes' => $attributes]
);
}
/**
* Get microtime for influx.
*
* @link https://github.com/influxdata/influxdb-php
*
* @return string
*/
private function microtime() {
list($usec, $sec) = explode(' ', microtime());
return sprintf('%d%06d', $sec, $usec * 1000000);
}
}

16
api/index.php
View File

@ -28,13 +28,16 @@ spl_autoload_register(function($class) {
include str_replace('\\', '/', $class) . '.php';
});
// Construct cora and set up error handlers.
$cora = cora\cora::get_instance();
set_error_handler([$cora, 'error_handler']);
set_exception_handler([$cora, 'exception_handler']);
// Construct request and set up error handlers.
$request = cora\request::get_instance();
set_error_handler([$request, 'error_handler']);
set_exception_handler([$request, 'exception_handler']);
// The shutdown handler will output the response.
register_shutdown_function([$cora, 'shutdown_handler']);
register_shutdown_function([$request, 'shutdown_handler']);
// Go!
$request->process($_REQUEST);
// Useful function
function array_median($array) {
@ -57,6 +60,3 @@ function array_mean($array) {
return array_sum($array) / count($array);
}
// Go!
$cora->process_request($_REQUEST);

141
api/logger.php
View File

@ -1,141 +0,0 @@
<?php
/**
* Log stuff to an Influx database. Does all the hard work so you don't have
* to.
*
* @author Jon Ziebell
*/
class logger extends cora\api {
/**
* Log something to InfluxDB. This will fire off the insert query as a
* background process and allow PHP to continue without blocking.
*
* @param string $measurement The measurement.
* @param array $tags Zero to many tags. These are indexed and searchable.
* Use these for things that need where clauses outside of timestamp.
* @param array $fields At least one field. These are not indexed.
* @param string $timestamp The timestamp in microseconds.
* @param string $retention_policy The retention policy to write to.
* Defaults to "autogen" which is infinite.
*/
public function log_influx($measurement, $tags, $fields, $timestamp, $retention_policy = null) {
// If this is not configured, do not log.
if(
$this->setting->get('influx_database_host') === null ||
$this->setting->get('influx_database_port') === null ||
$this->setting->get('influx_database_name') === null ||
$this->setting->get('influx_database_username') === null ||
$this->setting->get('influx_database_password') === null
) {
return;
}
$tag_string = $this->get_tag_string($tags);
$field_string = $this->get_field_string($fields);
$data_binary =
$measurement .
($tag_string !== '' ? ',' : '') .
$tag_string . ' ' .
$field_string . ' ' .
$timestamp;
$url =
$this->setting->get('influx_database_host') .
':' .
$this->setting->get('influx_database_port') .
'/write' .
'?db=' . $this->setting->get('influx_database_name') .
($retention_policy !== null ? ('&rp=' . $retention_policy) : '') .
'&precision=u';
exec(
'curl ' .
'-u ' . $this->setting->get('influx_database_username') . ':' . $this->setting->get('influx_database_password') . ' ' .
'-POST "' . $url . '" ' .
'--silent ' . // silent; keeps logs out of stderr
'--show-error ' . // override silent on failure
'--max-time 10 ' .
'--connect-timeout 5 ' .
'--data-binary \'' . $data_binary . '\' > /dev/null &'
);
}
/**
* Convert an array into a key/value string.
*
* @param array $array The input array. Null values are removed.
*
* @return string A string like "k1=v1,k2=v2". If no non-null values are
* present this will be an empty string.
*/
private function get_field_string($fields) {
$parts = [];
foreach($fields as $key => $value) {
if($value === null) {
continue;
} else if(is_bool($value) === true) {
$value = ($value === true) ? 'true' : 'false';
} else if(is_int($value) === true) {
$value = $value . 'i';
} else if(is_float($value) === true) {
$value = $value;
} else {
$value = $this->escape_field_value($value);
}
$parts[] = $key . '=' . $value;
}
return implode(',', $parts);
}
/**
* Convert a tag array into a key/value string. Tags are always strings in
* Influx.
*
* @param array $array The input array. Null values are removed.
*
* @return string A string like "k1=v1,k2=v2". If no non-null values are
* present this will be an empty string.
*/
private function get_tag_string($tags) {
$parts = [];
foreach($tags as $key => $value) {
if($value === null) {
continue;
} else {
$parts[] = $this->escape_tag_key_value($key) . '=' . $this->escape_tag_key_value($value);
}
}
return implode(',', $parts);
}
/**
* Add slashes where necessary to prevent injection attacks. Tag values just
* sit there unquoted (you can't quote them or the quote gets included as
* part of the value) so we have to escape other special characters in that
* context.
*
* @param string $value The value to escape.
*/
private function escape_tag_key_value($value) {
return str_replace([' ', ',', '='], ['\ ', '\,', '\='], $value);
}
/**
* Add slashes where necessary to prevent injection attacks. Field values
* sit inside of "", so escape any " characters. At a higher level they sit
* inside of a ' from the cURL body. Escape these as well.
*
* @param string $value The value to escape.
*/
private function escape_field_value($value) {
return '"' . str_replace(['"', "'"], ['\"', "'\''"], $value) . '"';
}
}

1
api/mailchimp.php
View File

@ -7,7 +7,6 @@
*/
class mailchimp extends external_api {
protected static $log_influx = true;
protected static $log_mysql = 'all';
protected static $cache = false;

1
api/patreon.php
View File

@ -15,7 +15,6 @@ class patreon extends external_api {
'public' => []
];
protected static $log_influx = true;
protected static $log_mysql = 'all';
protected static $cache = false;

1
api/smarty_streets.php
View File

@ -15,7 +15,6 @@
*/
class smarty_streets extends external_api {
protected static $log_influx = true;
protected static $log_mysql = 'all';
protected static $cache = true;

6
js/beestat/api.js
View File

@ -168,7 +168,7 @@ beestat.api.prototype.load_ = function(response_text) {
if (
response.data &&
(
response.data.error_code === 1004 || // Session is expired.
response.data.error_code === 1505 || // Session is expired.
response.data.error_code === 10000 || // Could not get first token.
response.data.error_code === 10001 || // Could not refresh ecobee token; no token found.
response.data.error_code === 10002 || // Could not refresh ecobee token; ecobee returned no token.
@ -186,7 +186,7 @@ beestat.api.prototype.load_ = function(response_text) {
return;
}
// Cach responses
// Cache responses
var cached_until_header = this.xhr_.getResponseHeader('beestat-cached-until');
if (this.is_batch_() === true) {
@ -244,7 +244,7 @@ beestat.api.prototype.is_batch_ = function() {
*/
beestat.api.prototype.cache_ = function(api_call, data, until) {
var server_date = moment(this.xhr_.getResponseHeader('date'));
var duration = moment.duration(moment(until).diff(server_date));
var duration = moment.duration(moment.utc(until).diff(server_date));
beestat.api.cache[this.get_key_(api_call)] = {
'data': data,