From 790aaf354e5873785846599abd740012e096e7c4 Mon Sep 17 00:00:00 2001 From: Jon Ziebell Date: Sun, 15 Mar 2020 08:12:28 -0400 Subject: [PATCH] Fixed #265 - Change logging from push to pull Deceptively simple commit message. This is a massive change that completely reworks the most fundamental part of the API. Not only does it remove the push logging, it also restructures logging to give better insight into what happens during an API call. --- api/cora/api.php | 8 +- api/cora/api_cache.php | 17 +- api/cora/api_call.php | 277 +++++++++ api/cora/api_log2.php | 58 ++ api/cora/cora.php | 1115 ---------------------------------- api/cora/database.php | 110 ++-- api/cora/exception.php | 7 +- api/cora/json_path.php | 68 --- api/cora/request.php | 659 ++++++++++++++++++++ api/cora/session.php | 49 +- api/cora/setting.example.php | 10 - api/ecobee.php | 12 +- api/external_api.php | 106 +--- api/index.php | 16 +- api/logger.php | 141 ----- api/mailchimp.php | 1 - api/patreon.php | 1 - api/smarty_streets.php | 1 - js/beestat/api.js | 6 +- 19 files changed, 1131 insertions(+), 1531 deletions(-) create mode 100644 api/cora/api_call.php create mode 100644 api/cora/api_log2.php delete mode 100644 api/cora/cora.php delete mode 100644 api/cora/json_path.php create mode 100644 api/cora/request.php delete mode 100644 api/logger.php diff --git a/api/cora/api.php b/api/cora/api.php index e980190..cf3ec54 100644 --- a/api/cora/api.php +++ b/api/cora/api.php @@ -38,11 +38,11 @@ abstract class api { protected $setting; /** - * Cora object. + * Request object. * - * @var cora + * @var request */ - protected $cora; + protected $request; /** * Construct and set the variables. The namespace is stripped from the @@ -57,7 +57,7 @@ abstract class api { $class_parts = explode('\\', $this->resource); $this->table = end($class_parts); $this->database = database::get_instance(); - $this->cora = cora::get_instance(); + $this->request = request::get_instance(); $this->setting = setting::get_instance(); $this->session = session::get_instance(); } diff --git a/api/cora/api_cache.php b/api/cora/api_cache.php index 4c0c34d..634cc76 100644 --- a/api/cora/api_cache.php +++ b/api/cora/api_cache.php @@ -42,15 +42,6 @@ class api_cache extends crud { $attributes['key'] = $key; $attributes['expires_at'] = date('Y-m-d H:i:s', time() + $duration); $attributes['response_data'] = $response_data; - $attributes['request_resource'] = $api_call['resource']; - $attributes['request_method'] = $api_call['method']; - - if(isset($api_call['arguments']) === true) { - $attributes['request_arguments'] = $api_call['arguments']; - } - else { - $attributes['request_arguments'] = null; - } return $this->create($attributes); } @@ -95,11 +86,11 @@ class api_cache extends crud { */ private function generate_key($api_call) { return sha1( - 'resource=' . $api_call['resource'] . - 'method=' . $api_call['method'] . + 'resource=' . $api_call->get_resource() . + 'method=' . $api_call->get_method() . 'arguments=' . ( - isset($api_call['arguments']) === true ? - json_encode($api_call['arguments']) : '' + $api_call->get_arguments() !== null ? + json_encode($api_call->get_arguments()) : '' ) . 'user_id=' . ( $this->session->get_user_id() !== null ? diff --git a/api/cora/api_call.php b/api/cora/api_call.php new file mode 100644 index 0000000..ece9ab0 --- /dev/null +++ b/api/cora/api_call.php @@ -0,0 +1,277 @@ +resource = $api_call['resource']; + $this->method = $api_call['method']; + $this->arguments = $this->parse_arguments($api_call['arguments']); + + if(isset($api_call['alias']) === true) { + $this->alias = $api_call['alias']; + } else { + $this->alias = $this->get_auto_alias(); + } + } + + /** + * Process the API call. + * + * @throws exception If the method does not exist. + */ + public function process() { + $this->restrict_private(); + + $resource_instance = new $this->resource(); + if(method_exists($resource_instance, $this->method) === false) { + throw new \exception('Method does not exist.', 1503); + } + + // Caching! If this API call is configured for caching, + // $cache_config = $this->setting->get('cache'); + if( // Is cacheable + isset($this->resource::$cache) === true && + isset($this->resource::$cache[$this->method]) === true + ) { + $api_cache_instance = new api_cache(); + $api_cache = $api_cache_instance->retrieve($this); + + if($api_cache !== null) { + // If there was a cache entry available, use that. + $this->response = $api_cache['response_data']; + $this->cached_until = date('Y-m-d H:i:s', strtotime($api_cache['expires_at'])); + } else { + // Else just run the API call, then cache it. + $this->response = call_user_func_array( + [$resource_instance, $this->method], + $this->arguments + ); + + $api_cache = $api_cache_instance->cache( + $this, + $this->response, + $this->resource::$cache[$this->method] + ); + $this->cached_until = date('Y-m-d H:i:s', strtotime($api_cache['expires_at'])); + } + } + else { // Not cacheable + $this->response = call_user_func_array( + [$resource_instance, $this->method], + $this->arguments + ); + } + } + + /** + * Restrict private API calls. + * + * @throws exception If the method does not exist in the resource's + * public/private maps. + * @throws exception If the resource/method is private and the session is + * not valid. + */ + private function restrict_private() { + if(in_array($this->method, $this->resource::$exposed['private'])) { + $type = 'private'; + } else if(in_array($this->method, $this->resource::$exposed['public'])) { + $type = 'public'; + } else { + throw new exception('Method is not mapped.', 1504); + } + + $session = session::get_instance(); + + if( + $type === 'private' && + $session->is_valid() === false + ) { + throw new exception('Session is expired.', 1505, false); + } + } + + /** + * Gets an array of arguments in the correct order for the method being + * called. + * + * @param string $json The arguments JSON. + * + * @throws exception If the arguments in the api_call were not valid JSON. + * + * @return array The requested arguments. + */ + private function parse_arguments($json) { + $arguments = []; + + // Arguments are not strictly required. If a method requires them then you + // will still get an error, but they are not required by the API. + if($json !== null) { + // All arguments are sent in the "arguments" key as JSON. + $decoded = json_decode($json, true); + + if($decoded === null) { + throw new exception('Arguments are not valid JSON.', 1506); + } + + $reflection_method = new \ReflectionMethod( + $this->resource, + $this->method + ); + $parameters = $reflection_method->getParameters(); + + foreach($parameters as $parameter) { + if(isset($decoded[$parameter->getName()]) === true) { + $argument = $decoded[$parameter->getName()]; + } + else { + if($parameter->isOptional() === true) { + $argument = $parameter->getDefaultValue(); + } else { + $argument = null; + } + } + $arguments[] = $argument; + } + } + + return $arguments; + } + + /** + * Get the resource. + * + * @return string + */ + public function get_resource() { + return $this->resource; + } + + /** + * Get the method. + * + * @return string + */ + public function get_method() { + return $this->method; + } + + /** + * Get the arguments. + * + * @return string + */ + public function get_arguments() { + return $this->arguments; + } + + /** + * Get the alias. + * + * @return string + */ + public function get_alias() { + return $this->alias; + } + + /** + * Get the response. + * + * @return mixed + */ + public function get_response() { + return $this->response; + } + + /** + * Get cached_until property. + * + * @return number + */ + public function get_cached_until() { + return $this->cached_until; + } + + /** + * Get the next auto-alias. + * + * @return number + */ + private function get_auto_alias() { + return api_call::$auto_alias++; + } +} diff --git a/api/cora/api_log2.php b/api/cora/api_log2.php new file mode 100644 index 0000000..d00264d --- /dev/null +++ b/api/cora/api_log2.php @@ -0,0 +1,58 @@ +create($this->resource, $attributes); + } + + /** + * Get the number of requests since a given timestamp for a given IP + * address. Handy for rate limiting. + * + * @param string $ip_address The IP to look at. + * @param int $timestamp The timestamp to check from. + * + * @return int The number of requests on or after $timestamp. + */ + public function get_number_requests_since($ip_address, $timestamp) { + $ip_address_escaped = $this->database->escape(ip2long($ip_address)); + $timestamp_escaped = $this->database->escape( + date('Y-m-d H:i:s', $timestamp) + ); + + $query = ' + select + count(*) `number_requests_since` + from + `api_log2` + where + `ip_address` = ' . $ip_address_escaped . ' + and `timestamp` >= ' . $timestamp_escaped . ' + '; + + $result = $this->database->query($query); + $row = $result->fetch_assoc(); + + return $row['number_requests_since']; + } + +} diff --git a/api/cora/cora.php b/api/cora/cora.php deleted file mode 100644 index 32c2c66..0000000 --- a/api/cora/cora.php +++ /dev/null @@ -1,1115 +0,0 @@ -start_timestamp = microtime(true); - $this->start_timestamp_microtime = $this->microtime(); - - // See class variable documentation for reasoning. - $this->current_working_directory = getcwd(); - } - - /** - * Use this function to instantiate this class instead of calling new cora() - * (which isn't allowed anyways). This is necessary so that the API class - * can have access to Cora. - * - * @return cora A new cora object or the already created one. - */ - public static function get_instance() { - if(isset(self::$instance) === false) { - self::$instance = new self(); - } - return self::$instance; - } - - /** - * Execute the request. It is run through the rate limiter, checked for - * errors, then processed. Requests sent after the rate limit is reached are - * not logged. - * - * @param array $request Basically just $_REQUEST or a slight mashup of it - * for batch requests. - * - * @throws \Exception If the rate limit threshhold is reached. - * @throws \Exception If SSL is required but not used. - * @throws \Exception If the API key is not provided. - * @throws \Exception If the API key is invalid. - * @throws \Exception If the session is expired. - * @throws \Exception If a resource is not provided. - * @throws \Exception If a method is not provided. - * @throws \Exception If the requested method does not exist. - */ - public function process_request($request) { - // This is necessary in order for the shutdown handler/log function to have - // access to this data, but it's not used anywhere else. - $this->request = $request; - - // Setting class for getting settings. Anything that extends cora\api gets - // this automatically. - $this->setting = setting::get_instance(); - - // Used to have this in the constructor, but the database uses this class - // which causes a dependency loop in the constructors. - $this->database = database::get_instance(); - - // A couple quick error checks - if($this->is_over_rate_limit() === true) { - throw new \Exception('Rate limit reached.', 1005); - } - if($this->setting->get('force_ssl') === true && empty($_SERVER['HTTPS']) === true) { - throw new \Exception('Request must be sent over HTTPS.', 1006); - } - - // Make sure the API key that was sent is present and valid. - if(isset($request['api_key']) === false) { - throw new \Exception('API Key is required.', 1000); - } - - $api_user_resource = new api_user(); - $api_users = $api_user_resource->read(['api_key' => $request['api_key']]); - if(count($api_users) !== 1) { - throw new \Exception('Invalid API key.', 1003); - } else { - $this->api_user = $api_users[0]; - } - - // Build a list of API calls. - $this->build_api_call_list($request); - - // Check the API request for errors. - $this->check_api_request_for_errors(); - - // Set the default headers as a catch-all. Most API calls won't touch these, - // but it is possible for them to override headers as desired. - $this->set_default_headers(); - - // Get this every time. It's only used for session API calls. Non-session - // API calls don't bother with this. - $session = session::get_instance(); - $session_is_valid = $session->touch($this->api_user['session_key']); - - // Make sure the updated session timestamp doesn't get rolled back on - // exception. - $this->database->commit_transaction(); - - // Process each request. - foreach($this->api_calls as $api_call) { - // Store the currently running API call for tracking if an error occurs. - $this->current_api_call = $api_call; - - // These are required before we can move on with any more processing or - // error checking. - if(isset($api_call['resource']) === false) { - throw new \Exception('Resource is required.', 1001); - } - if(isset($api_call['method']) === false) { - throw new \Exception('Method is required.', 1002); - } - - // Sets $call_type to 'public' or 'private' - $call_type = $this->get_api_call_type($api_call); - - // If the request requires a session, make sure it's valid. - if($call_type === 'private') { - if($session_is_valid === false) { - throw new exception('Session is expired.', 1004, false); - } - } - - // If the resource doesn't exist, spl_autoload_register() will throw a - // fatal error. The shutdown handler will "catch" it. It is not possible - // to catch exceptions directly from the autoloader using try/catch. - $resource_instance = new $api_call['resource'](); - - // If the method doesn't exist - if(method_exists($resource_instance, $api_call['method']) === false) { - throw new \Exception('Method does not exist.', 1009); - } - - $arguments = $this->get_arguments($api_call); - - // Process the request and save some statistics. - $start_time = microtime(true); - $start_query_count = $this->database->get_query_count(); - $start_query_time = $this->database->get_query_time(); - - if(isset($api_call['alias']) === true) { - $index = $api_call['alias']; - } - else { - $index = count($this->response_data); - } - - // Caching! If this API call is configured for caching, - // $cache_config = $this->setting->get('cache'); - if( // Is cacheable - isset($api_call['resource']::$cache) === true && - isset($api_call['resource']::$cache[$api_call['method']]) === true - ) { - $api_cache_instance = new api_cache(); - $api_cache = $api_cache_instance->retrieve($api_call); - - if($api_cache !== null) { - // If there was a cache entry available, use that. - $this->response_data[$index] = $api_cache['response_data']; - $this->from_cache[$index] = true; - $this->cached_until[$index] = date('c', strtotime($api_cache['expires_at'])); - } else { - // Else just run the API call, then cache it. - $this->response_data[$index] = call_user_func_array( - [$resource_instance, $api_call['method']], - $arguments - ); - $this->from_cache[$index] = false; - - $api_cache = $api_cache_instance->cache( - $api_call, - $this->response_data[$index], - $api_call['resource']::$cache[$api_call['method']] - ); - $this->cached_until[$index] = date('c', strtotime($api_cache['expires_at'])); - } - } - else { // Not cacheable - $this->response_data[$index] = call_user_func_array( - [$resource_instance, $api_call['method']], - $arguments - ); - $this->from_cache[$index] = false; - } - - $this->response_times[$index] = (microtime(true) - $start_time); - $this->response_query_counts[$index] = $this->database->get_query_count() - $start_query_count; - $this->response_query_times[$index] = $this->database->get_query_time() - $start_query_time; - } - - $this->set_cached_until_header(); - } - - /** - * Build a list of API calls from the request. For a single request, it's - * just the request. For batch requests, add each item in the batch - * parameter to this array. - * - * @param array $request The original request. - * - * @throws \Exception If this is a batch request and the batch data is not - * valid JSON - * @throws \Exception If this is a batch request and it exceeds the maximum - * number of api calls allowed in one batch. - */ - private function build_api_call_list($request) { - $this->api_calls = []; - if(isset($request['batch']) === true) { - $batch = json_decode($request['batch'], true); - if($batch === null) { - throw new \Exception('Batch is not valid JSON.', 1012); - } - $batch_limit = $this->setting->get('batch_limit'); - if($batch_limit !== null && count($batch) > $batch_limit) { - throw new \Exception('Batch limit exceeded.', 1013); - } - foreach($batch as $api_call) { - // Put this on each API call for logging. - $api_call['api_key'] = $request['api_key']; - $this->api_calls[] = $api_call; - } - } - else { - $this->api_calls[] = $request; - } - } - - /** - * Check the API request for various errors. - * - * @throws \Exception If something other than ALL or NO aliases are set. - * @throws \Exception If Any duplicate aliases are used. - */ - private function check_api_request_for_errors() { - $aliases = []; - foreach($this->api_calls as $api_call) { - if(isset($api_call['alias']) === true) { - $aliases[] = $api_call['alias']; - } - } - - // Check to make sure either all or none are set. - $number_aliases = count($aliases); - if(count($this->api_calls) !== $number_aliases && $number_aliases !== 0) { - throw new \Exception('All API calls must have an alias if at least one is set.', 1017); - } - - // Check for duplicates. - $number_unique_aliases = count(array_unique($aliases)); - if($number_aliases !== $number_unique_aliases) { - throw new \Exception('Duplicate alias on API call.', 1018); - } - } - - /** - * Returns 'session' or 'non_session' depending on where the API method is - * located at. Session methods require a valid session in order to execute. - * - * @param array $api_call The API call to get the type for. - * - * @throws \Exception If the method was not found in the map. - * - * @return string The type. - */ - private function get_api_call_type($api_call) { - if(in_array($api_call['method'], $api_call['resource']::$exposed['private'])) { - return 'private'; - } - else if(in_array($api_call['method'], $api_call['resource']::$exposed['public'])) { - return 'public'; - } - else { - throw new \Exception('Requested method is not mapped.', 1008); - } - } - - /** - * Check to see if the request from the current IP address needs to be rate - * limited. If $requests_per_minute is null then there is no rate limiting. - * - * @return bool If this request puts us over the rate threshold. - */ - private function is_over_rate_limit() { - $requests_per_minute = $this->setting->get('requests_per_minute'); - - if($requests_per_minute === null) { - return false; - } - - $api_log_resource = new api_log(); - $requests_this_minute = $api_log_resource->get_number_requests_since( - $_SERVER['REMOTE_ADDR'], - time() - 60 - ); - - return ($requests_this_minute > $requests_per_minute); - } - - /** - * Fetches a list of arguments when passed an array of keys. Since the - * arguments are passed from JS to PHP in JSON, I don't need to cast any of - * the values as the data types are preserved. Since the argument order from - * the client doesn't matter, this makes sure that the arguments are placed - * in the correct order for calling the function. - * - * @param array $api_call The API call. - * - * @throws \Exception If the arguments in the api_call were not valid JSON. - * - * @return array The requested arguments. - */ - private function get_arguments($api_call) { - $arguments = []; - - // Arguments are not strictly required. If a method requires them then you - // will still get an error, but they are not required by the API. - if(isset($api_call['arguments']) === true) { - // All arguments are sent in the "arguments" key as JSON. - $api_call_arguments = json_decode($api_call['arguments'], true); - - if($api_call_arguments === null) { - throw new \Exception('Arguments are not valid JSON.', 1011); - } - - $reflection_method = new \ReflectionMethod( - $api_call['resource'], - $api_call['method'] - ); - $parameters = $reflection_method->getParameters(); - - foreach($parameters as $parameter) { - if(isset($api_call_arguments[$parameter->getName()]) === true) { - $argument = $api_call_arguments[$parameter->getName()]; - - // If this is a batch request, look for JSONPath arguments. - if(isset($this->request['batch']) === true) { - $argument = $this->evaluate_json_path_argument($argument); - } - } - else { - $argument = null; - } - $arguments[] = $argument; - } - } - return $arguments; - } - - /** - * Recursively check all values in an argument. If any of them are JSON - * path, evaluate them. - * - * @param mixed $argument The argument to check. - * - * @return mixed The argument with the evaluated path. - */ - private function evaluate_json_path_argument($argument) { - if(is_array($argument) === true) { - foreach($argument as $key => $value) { - $argument[$key] = $this->evaluate_json_path_argument($value); - } - } - else if(preg_match('/^{=(.*)}$/', $argument, $matches) === 1) { - $json_path_resource = new json_path(); - $json_path = $matches[1]; - $argument = $json_path_resource->evaluate($this->response_data, $json_path); - } - return $argument; - } - - /** - * Sets error_extra_info. - * - * @param mixed $error_extra_info Whatever you want the extra info to be. - */ - public function set_error_extra_info($error_extra_info) { - $this->error_extra_info = $error_extra_info; - } - - /** - * Get error_extra_info. - * - * @return mixed - */ - public function get_error_extra_info() { - return $this->error_extra_info; - } - - /** - * Get the current API user. - * - * @return array - */ - public function get_api_user() { - return $this->api_user; - } - - /** - * Sets the headers that should be used for this API call. This is useful - * for doing things like returning files from the API where the content-type - * is no longer application/json. This replaces all headers; headers are not - * outputted to the browser until all API calls have completed, so the last - * call to this function will win. - * - * @param array $headers The headers to output. - * @param bool $custom_response Whether or not to wrap the response with the - * Cora data or just output the API call's return value. - * - * @throws \Exception If this is a batch request and a custom response was - * requested. - * @throws \Exception If this is a batch request and the content type was - * altered from application/json - * @throws \Exception If this is not a batch request and the content type - * was altered from application/json without a custom response. - */ - public function set_headers($headers, $custom_response = false) { - if(isset($this->request['batch']) === true) { - if($custom_response === true) { - throw new \Exception('Batch API requests can not use a custom response.', 1015); - } - if($this->content_type_is_json($headers) === false) { - throw new \Exception('Batch API requests must return JSON.', 1014); - } - } - else { - // Not a batch request - if($custom_response === false && $this->content_type_is_json($headers) === false) { - throw new \Exception('Non-custom responses must return JSON.', 1016); - } - } - $this->headers = $headers; - $this->custom_response = $custom_response; - } - - /** - * Return whether or not the current output headers indicate that the - * content type is JSON. This is mostly just used to make sure that batch - * API calls output JSON. - * - * @param array $headers The headers to look at. - * - * @return bool Whether or not the output has a content type of - * application/json - */ - private function content_type_is_json($headers) { - return isset($headers['Content-type']) === true - && stristr($headers['Content-type'], 'application/json') !== false; - } - - /** - * Override of the default PHP error handler. Grabs the error info and sends - * it to the exception handler which returns a JSON response. - * - * @param int $error_code The error number from PHP. - * @param string $error_message The error message. - * @param string $error_file The file the error happend in. - * @param int $error_line The line of the file the error happened on. - * - * @return string The JSON response with the error details. - */ - public function error_handler($error_code, $error_message, $error_file, $error_line) { - $this->set_error_response( - $error_message, - $error_code, - $error_file, - $error_line, - debug_backtrace(false), - true - ); - die(); // Do not continue execution; shutdown handler will now run. - } - - /** - * Override of the default PHP exception handler. All unhandled exceptions - * go here. - * - * @param Exception $e The exception. - */ - public function exception_handler($e) { - $this->set_error_response( - $e->getMessage(), - $e->getCode(), - $e->getFile(), - $e->getLine(), - $e->getTrace(), - (method_exists($e, 'getReportable') === true ? $e->getReportable() : true) - ); - die(); // Do not continue execution; shutdown handler will now run. - } - - /** - * Handle all exceptions by generating a JSON response with the error - * details. If debugging is enabled, a bunch of other information is sent - * back to help out. - * - * @param string $error_message The error message. - * @param mixed $error_code The supplied error code. - * @param string $error_file The file the error happened in. - * @param int $error_line The line of the file the error happened on. - * @param array $error_trace The stack trace for the error. - */ - public function set_error_response($error_message, $error_code, $error_file, $error_line, $error_trace, $reportable) { - // There are a few places that call this function to set an error response, - // so this can't just be done in the exception handler alone. If an error - // occurs, rollback the current transaction. Also only attempt to roll back - // the transaction if the database was successfully created/connected to. - if($this->database !== null) { - $this->database->rollback_transaction(); - } - - $this->response = [ - 'success' => false, - 'data' => [ - 'error_message' => $error_message, - 'error_code' => $error_code, - 'error_file' => $error_file, - 'error_line' => $error_line, - 'error_trace' => $error_trace, - 'error_extra_info' => $this->error_extra_info - ] - ]; - - $session = session::get_instance(); - $user_id = $session->get_user_id(); - - if(isset($this->request['api_key']) === true) { - $api_user_resource = new api_user(); - $api_users = $api_user_resource->read(['api_key' => $this->request['api_key']]); - $api_user_id = $api_users[0]['api_user_id']; - } - else { - $api_user_id = null; - } - - // Send data to Sentry for error logging. - // https://docs.sentry.io/development/sdk-dev/event-payloads/ - if ( - $reportable === true && - $this->setting->get('sentry_key') !== null && - $this->setting->get('sentry_project_id') !== null && - $api_user_id === 1 - ) { - $data = [ - 'event_id' => str_replace('-', '', exec('uuidgen -r')), - 'timestamp' => date('c'), - 'logger' => 'cora', - 'platform' => 'php', - 'level' => 'error', - 'tags' => [ - 'error_code' => $error_code, - 'api_user_id' => $api_user_id - ], - 'extra' => [ - 'error_file' => $error_file, - 'error_line' => $error_line, - 'error_trace' => $error_trace, - 'error_extra_info' => $this->error_extra_info - ], - 'exception' => [ - 'type' => 'Exception', - 'value' => $error_message, - 'handled' => false - ], - 'user' => [ - 'id' => $user_id, - 'ip_address' => $_SERVER['REMOTE_ADDR'] - ] - ]; - - exec( - 'curl ' . - '-H "Content-Type: application/json" ' . - '-H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=' . $this->setting->get('sentry_key') . '" ' . - '--silent ' . // silent; keeps logs out of stderr - '--show-error ' . // override silent on failure - '--max-time 10 ' . - '--connect-timeout 5 ' . - '--data \'' . json_encode($data) . '\' ' . - '"https://sentry.io/api/' . $this->setting->get('sentry_project_id') . '/store/" > /dev/null &' - ); - } - } - - /** - * Executes when the script finishes. If there was an error that somehow - * didn't get caught, then this will find it with error_get_last and return - * appropriately. Note that error_get_last() will only get something when an - * error wasn't caught by my error/exception handlers. The default PHP error - * handler fills this in. Doesn't do anything if an exception was thrown due - * to the rate limit. - * - * @throws \Exception If a this was a batch request but one of the api calls - * changed the content-type to anything but the default. - */ - public function shutdown_handler() { - // Since the shutdown handler is rather verbose in what it has to check for - // and do, it's possible it will fail or detect an error that needs to be - // handled. For example, someone could return binary data from an API call - // which will fail a json_encode, or someone could change the headers in a - // batch API call, which isn't allowed. I can't throw an exception since I'm - // already in the shutdown handler...it will be caught but it won't execute - // a new shutdown handler and no output will be sent to the client. I just - // have to handle all problems manually. - try { - // Fix the current working directory. See documentation on this class - // variable for details. - chdir($this->current_working_directory); - - // If I didn't catch an error/exception with my handlers, look here...this - // will catch fatal errors that I can't. - $error = error_get_last(); - if($error !== null) { - $this->set_error_response( - $error['message'], - $error['type'], - $error['file'], - $error['line'], - debug_backtrace(false), - true - ); - } - - // If the response has already been set by one of the error handlers, end - // execution here and just log & output the response. - if(isset($this->response) === true) { - // Don't log anything for rate limit breaches. - if($this->response['data']['error_code'] !== 1005) { - $this->log(); - } - - // Override whatever headers might have already been set. - $this->set_default_headers(); - $this->output_headers(); - die($this->get_json_response()); - } - else { - // If we got here, no errors have occurred. - - // For non-custom responses, build the response, log it, and output it. - $this->response = ['success' => true]; - - if(isset($this->request['batch']) === true) { - $this->response['data'] = $this->response_data; - } - else { - // $this->response['data'] = $this->response_data[0]; - $this->response['data'] = reset($this->response_data); - } - - // Log all of the API calls that were made. - $this->log(); - - // Output the response - $this->output_headers(); - die($this->get_json_response()); - } - } - catch(\Exception $e) { - $this->set_error_response( - $e->getMessage(), - $e->getCode(), - $e->getFile(), - $e->getLine(), - $e->getTrace(), - (method_exists($e, 'getReportable') === true ? $e->getReportable() : true) - ); - $this->set_default_headers(); - $this->output_headers(); - die($this->get_json_response()); - } - } - - /** - * Gets the json_encoded response. This is called from the shutdown handler - * and removes debug information if debugging is disabled and then - * json_encodes the data. - * - * @return string The JSON encoded response. - */ - private function get_json_response() { - $response = $this->response; - if($this->setting->get('debug') === false && $response['success'] === false) { - unset($response['data']['error_file']); - unset($response['data']['error_line']); - unset($response['data']['error_trace']); - unset($response['data']['error_extra_info']); - } - return json_encode($response); - } - - /** - * Output whatever the headers are currently set to. - */ - private function output_headers() { - foreach($this->headers as $key => $value) { - header($key . ': ' . $value); - } - } - - /** - * Resets the headers to default. Have to do this in case one of the API - * calls changes them and there was an error to handle. - */ - private function set_default_headers() { - $this->headers['Content-type'] = 'application/json; charset=UTF-8'; - } - - /** - * Set the beestat-cached-until header. - */ - private function set_cached_until_header() { - if(isset($this->request['batch']) === true) { // Batch - $beestat_cached_until = []; - foreach($this->cached_until as $index => $cached_until) { - $beestat_cached_until[$index] = $cached_until; - } - if(count($beestat_cached_until) > 0) { - $this->headers['beestat-cached-until'] = json_encode($beestat_cached_until); - } - } else { // Single - if(count($this->cached_until) === 1) { - $this->headers['beestat-cached-until'] = reset($this->cached_until); - } - } - } - - /** - * Returns true for all loggable content types. Mostly JSON, XML, and other - * text-based types. - * - * @return bool Whether or not the output has a content type that can be - * logged. - */ - private function content_type_is_loggable() { - if(isset($this->headers['Content-type']) === false) { - return false; - } - else { - $loggable_content_types = [ - 'application/json', - 'application/xml', - 'application/javascript', - 'text/html', - 'text/xml', - 'text/plain', - 'text/css' - ]; - foreach($loggable_content_types as $loggable_content_type) { - if(stristr($this->headers['Content-type'], $loggable_content_type) !== false) { - return true; - } - } - } - } - - /** - * Log the request and response to the database. The logged response is - * truncated to 16kb for sanity. - */ - private function log() { - $api_log_resource = new api_log(); - $session = session::get_instance(); - - $user_id = $session->get_user_id(); - - // If exception. This is lenghty because I have to check to make sure - // everything was set or else use null. - if(isset($this->response['data']['error_code']) === true) { - if(isset($this->request['api_key']) === true) { - $api_user_resource = new api_user(); - $api_users = $api_user_resource->read(['api_key' => $this->request['api_key']]); - $request_api_user_id = $api_users[0]['api_user_id']; - } - else { - $request_api_user_id = null; - } - - $request_resource = null; - $request_method = null; - $request_arguments = null; - if($this->current_api_call !== null) { - if(isset($this->current_api_call['resource']) === true) { - $request_resource = $this->current_api_call['resource']; - } - if(isset($this->current_api_call['method']) === true) { - $request_method = $this->current_api_call['method']; - } - if(isset($this->current_api_call['arguments']) === true) { - $request_arguments = $this->current_api_call['arguments']; - } - } - $response_error_code = $this->response['data']['error_code']; - $response_time = null; - $response_query_count = null; - $response_query_time = null; - $response_data = substr(json_encode($this->response['data']), 0, 16384); - $from_cache = null; - - $api_log_resource->create( - [ - 'user_id' => $user_id, - 'request_api_user_id' => $request_api_user_id, - 'request_resource' => $request_resource, - 'request_method' => $request_method, - 'request_arguments' => preg_replace('/"(password)":".*"/', '"$1":"[removed]"', $request_arguments), - 'response_error_code' => $response_error_code, - 'response_data' => preg_replace('/"(password)":".*"/', '"$1":"[removed]"', $response_data), - 'response_time' => $response_time, - 'response_query_count' => $response_query_count, - 'response_query_time' => $response_query_time, - 'from_cache' => $from_cache - ] - ); - - $this->log_influx( - [ - 'user_id' => $user_id, - 'request_api_user_id' => $request_api_user_id, - 'request_resource' => $request_resource, - 'request_method' => $request_method, - 'request_timestamp' => $this->start_timestamp_microtime, - 'response_error_code' => $response_error_code, - 'response_time' => $response_time, - 'response_query_count' => $response_query_count, - 'response_query_time' => $response_query_time, - 'from_cache' => $from_cache - ] - ); - - } - else { - $response_error_code = null; - $count_api_calls = count($this->api_calls); - for($i = 0; $i < $count_api_calls; $i++) { - $api_call = $this->api_calls[$i]; - - $api_user_resource = new api_user(); - $api_users = $api_user_resource->read(['api_key' => $api_call['api_key']]); - $request_api_user_id = $api_users[0]['api_user_id']; - - $request_resource = $api_call['resource']; - $request_method = $api_call['method']; - if(isset($api_call['arguments']) === true) { - $request_arguments = $api_call['arguments']; - } - else { - $request_arguments = null; - } - - if(isset($api_call['alias']) === true) { - $index = $api_call['alias']; - } - else { - $index = $i; - } - - $response_time = $this->response_times[$index]; - $response_query_count = $this->response_query_counts[$index]; - $response_query_time = $this->response_query_times[$index]; - - $from_cache = $this->from_cache[$index]; - - $api_log_resource->create( - [ - 'user_id' => $user_id, - 'request_api_user_id' => $request_api_user_id, - 'request_resource' => $request_resource, - 'request_method' => $request_method, - 'request_arguments' => preg_replace('/"(password)":".*"/', '"$1":"[removed]"', $request_arguments), - 'response_error_code' => $response_error_code, - 'response_data' => null, // Can't store this; uses too much disk. - 'response_time' => $response_time, - 'response_query_count' => $response_query_count, - 'response_query_time' => $response_query_time, - 'from_cache' => $from_cache - ] - ); - - $this->log_influx( - [ - 'user_id' => $user_id, - 'request_api_user_id' => $request_api_user_id, - 'request_resource' => $request_resource, - 'request_method' => $request_method, - 'request_timestamp' => $this->start_timestamp_microtime, - 'response_error_code' => $response_error_code, - 'response_time' => $response_time, - 'response_query_count' => $response_query_count, - 'response_query_time' => $response_query_time, - 'from_cache' => $from_cache, - ] - ); - } - } - } - - /** - * Log to InfluxDB/Grafana. - * - * @param array $data - */ - private function log_influx($data) { - $logger_resource = new \logger(); - $logger_resource->log_influx( - 'api_log', - [ - 'request_api_user_id' => (string) $data['request_api_user_id'], - 'exception' => $data['response_error_code'] === null ? '0' : '1', - 'from_cache' => $data['from_cache'] === false ? '0' : '1' - ], - [ - 'user_id' => (int) $data['user_id'], - 'request_resource' => (string) $data['request_resource'], - 'request_method' => (string) $data['request_method'], - 'response_time' => round($data['response_time'], 4), - 'response_query_count' => (int) $data['response_query_count'], - 'response_error_code' => $data['response_error_code'] === null ? null : (int) $data['response_error_code'], - 'response_query_time' => round($data['response_query_time'], 4) - ], - $data['request_timestamp'] - ); - } - - /** - * Get microtime for influx. - * - * @link https://github.com/influxdata/influxdb-php - * - * @return string - */ - private function microtime() { - list($usec, $sec) = explode(' ', microtime()); - return sprintf('%d%06d', $sec, $usec * 1000000); - } - - -} diff --git a/api/cora/database.php b/api/cora/database.php index a5823c5..1b01b25 100644 --- a/api/cora/database.php +++ b/api/cora/database.php @@ -2,15 +2,6 @@ namespace cora; -/** - * This exception is thrown by database->query if the query failed due to a - * duplicate entry in the database. - * - * @author Jon Ziebell - */ -final class DuplicateEntryException extends \Exception { -}; - /** * This is a MySQLi database wrapper. It provides access to some basic * functions like select, insert, and update. Those functions automatically @@ -61,25 +52,18 @@ final class database extends \mysqli { private $transaction_started = false; /** - * The total number of queries executed. + * The executed queries. * * @var int */ - private $query_count = 0; + private $queries = []; /** - * The total time all queries have taken to execute. + * The request object. * - * @var float + * @var request */ - private $query_time = 0; - - /** - * The cora object. - * - * @var cora - */ - private $cora; + private $request; /** * The setting object. @@ -100,7 +84,7 @@ final class database extends \mysqli { * @throws \Exception If failing to connect to the database. */ public function __construct() { - $this->cora = cora::get_instance(); + $this->request = request::get_instance(); $this->setting = setting::get_instance(); parent::__construct( @@ -113,26 +97,30 @@ final class database extends \mysqli { // does not have a native type for decimals so that gets left behind. parent::options(MYSQLI_OPT_INT_AND_FLOAT_NATIVE, true); + // $this->connect_error = 'this is broken'; if($this->connect_error !== null) { - $this->cora->set_error_extra_info( + throw new exception( + 'Could not connect to database.', + 1200, + true, [ 'database_error' => $this->connect_error ] ); - throw new \Exception('Could not connect to database.', 1200); } $database_name = $this->setting->get('database_name'); if($database_name !== null) { $success = $this->select_db($database_name); if($success === false) { - - $this->cora->set_error_extra_info( + throw new exception( + 'Could not select database.', + 1208, + true, [ 'database_error' => $this->error ] ); - throw new \Exception('Could not select database.', 1208); } } } @@ -188,7 +176,7 @@ final class database extends \mysqli { if($this->transaction_started === false) { $result = $this->query('start transaction'); if($result === false) { - throw new \Exception('Failed to start database transaction.', 1201); + throw new exception('Failed to start database transaction.', 1201); } $this->transaction_started = true; } @@ -204,7 +192,7 @@ final class database extends \mysqli { $this->transaction_started = false; $result = $this->query('commit'); if($result === false) { - throw new \Exception('Failed to commit database transaction.', 1202); + throw new exception('Failed to commit database transaction.', 1202); } } } @@ -220,7 +208,7 @@ final class database extends \mysqli { $this->transaction_started = false; $result = $this->query('rollback'); if($result === false) { - throw new \Exception('Failed to rollback database transaction.', 1203); + throw new exception('Failed to rollback database transaction.', 1203); } } } @@ -279,12 +267,14 @@ final class database extends \mysqli { return '`' . $identifier . '`'; } else { - $this->cora->set_error_extra_info( + throw new exception( + 'Query identifier is invalid.', + 1204, + true, [ 'identifier' => $identifier ] ); - throw new \Exception('Query identifier is invalid.', 1204); } } @@ -309,7 +299,7 @@ final class database extends \mysqli { else if(is_array($value) === true) { if(isset($value['operator']) === true) { if(in_array($value['operator'], ['>', '<', '=', '>=', '<=', 'between']) === false) { - throw new \Exception('Invalid operator', 1213); + throw new exception('Invalid operator', 1213); } if($value['operator'] === 'between') { return $this->escape_identifier($column) . ' between ' . $this->escape($value['value'][0]) . ' and ' . $this->escape($value['value'][1]); @@ -378,29 +368,24 @@ final class database extends \mysqli { $result = parent::query($query); $stop = microtime(true); - if($result === false) { - $database_error = $this->error; - $this->rollback_transaction(); + $this->queries[] = [ + 'query' => $query, + 'time' => (($stop - $start) * 1000) + ]; - $this->cora->set_error_extra_info( + if($result === false) { + $this->rollback_transaction(); + throw new exception( + 'Database query failed.', + 1206, + true, [ - 'database_error' => $database_error, + 'database_error' => $this->error, 'query' => $query ] ); - - if(stripos($database_error, 'duplicate entry') !== false) { - throw new DuplicateEntryException('Duplicate database entry.', 1205); - } - else { - throw new \Exception('Database query failed.', 1206); - } } - // Don't log info about transactions...they're a wash - $this->query_count++; - $this->query_time += ($stop - $start); - return $result; } @@ -541,7 +526,7 @@ final class database extends \mysqli { ) { foreach($resource::$converged as $column => $column_properties) { if(isset($row[$column]) === true) { - throw new \Exception('Column `' . $column . '` exists; cannot be overwritten by converged column.', 1212); + throw new exception('Column `' . $column . '` exists; cannot be overwritten by converged column.', 1212); } $row[$column] = (isset($row['converged'][$column]) === true) ? $row['converged'][$column] : null; } @@ -641,7 +626,7 @@ final class database extends \mysqli { // Check for errors if(isset($attributes[$table . '_id']) === false) { - throw new \Exception('ID is required for update.', 1214); + throw new exception('ID is required for update.', 1214); } // Extract the ID. @@ -650,7 +635,7 @@ final class database extends \mysqli { // Check for errors if(count($attributes) === 0) { - throw new \Exception('Updates require at least one attribute.', 1207); + throw new exception('Updates require at least one attribute.', 1207); } // Converge the diverged attributes. @@ -775,7 +760,7 @@ final class database extends \mysqli { * @return int The query count. */ public function get_query_count() { - return $this->query_count; + return count($this->queries); } /** @@ -784,7 +769,20 @@ final class database extends \mysqli { * @return float The total execution time. */ public function get_query_time() { - return $this->query_time; + $query_time = 0; + foreach ($this->queries as $query) { + $query_time += $query['time']; + } + return $query_time; + } + + /** + * Gets the time taken to execute all of the queries. + * + * @return float The total execution time. + */ + public function get_queries() { + return $this->queries; } /** @@ -878,10 +876,10 @@ final class database extends \mysqli { '); $row = $result->fetch_assoc(); if($row['lock'] === 0) { - throw new \Exception('Lock not established by this thread.', 1210); + throw new exception('Lock not established by this thread.', 1210); } else if($row['lock'] === null) { - throw new \Exception('Lock does not exist.', 1211); + throw new exception('Lock does not exist.', 1211); } } diff --git a/api/cora/exception.php b/api/cora/exception.php index 2250a10..22a7926 100644 --- a/api/cora/exception.php +++ b/api/cora/exception.php @@ -14,12 +14,17 @@ namespace cora; * @author Jon Ziebell */ final class exception extends \Exception { - public function __construct($message, $code, $reportable = true) { + public function __construct($message, $code, $reportable = true, $extra = null) { $this->reportable = $reportable; + $this->extra = $extra; return parent::__construct($message, $code, null); } public function getReportable() { return $this->reportable; } + + public function getExtraInfo() { + return $this->extra; + } } diff --git a/api/cora/json_path.php b/api/cora/json_path.php deleted file mode 100644 index 96c200d..0000000 --- a/api/cora/json_path.php +++ /dev/null @@ -1,68 +0,0 @@ -extract_key($data, $key_array); - } - - /** - * Recursively extract keys from the data array. Basically, $data.foo.bar is - * represented by sending $data, ['foo', 'bar']. If you end a key with - * [] it will get every instance of that value inside the current array. - * - * @param mixed $data The data to traverse. You can send anything here but - * don't expect to not get an exception if you try to traverse things like - * non-existent indices. - * @param array $key_array The array keys to use to traverse $data. - * - * @throws \Exception If any of the provided keys do not exist in the data. - * - * @return mixed The requested data. - */ - private function extract_key($data, $key_array) { - $key = array_shift($key_array); - if($key === null) { - return $data; - } - else { - if(substr($key, -2) === '[]') { - return array_column($data, substr($key, 0, strlen($key) - 2)); - } - else { - if(array_key_exists($key, $data) === false) { - throw new \Exception('Invalid path string.', 1500); - } - else { - return $this->extract_key($data[$key], $key_array); - } - } - } - } - -} diff --git a/api/cora/request.php b/api/cora/request.php new file mode 100644 index 0000000..180bd33 --- /dev/null +++ b/api/cora/request.php @@ -0,0 +1,659 @@ +begin_timestamp = microtime(true); + + // See class variable documentation for reasoning. + $this->current_working_directory = getcwd(); + } + + /** + * Use this function to instantiate this class instead of calling new + * request() (which isn't allowed anyways). + * + * @return cora A new cora object or the already created one. + */ + public static function get_instance() { + if(isset(self::$instance) === false) { + self::$instance = new self(); + } + return self::$instance; + } + + /** + * Execute the request. It is run through the rate limiter, checked for + * errors, etc, then processed. + * + * @param array $request Basically just $_REQUEST or a slight mashup of it + * for batch requests. + */ + public function process($request) { + $this->request = $request; + + $this->rate_limit(); + $this->force_ssl(); + + $this->set_api_user(); + $this->set_api_calls(); + $this->validate_aliases(); + $this->set_default_headers(); + + // Touch the session, if there is one. If the API user does not have a + // session key set it will pull from the cookie. + $session = session::get_instance(); + $session->touch($this->api_user['session_key']); + + // Process each request. + foreach($this->api_calls as $api_call) { + $api_call->process(); + } + + $this->set_cached_until_header(); + } + + /** + * Build a list of API calls from the request. For a single request, it's + * just the request. For batch requests, add each item in the batch + * parameter to this array. + * + * @throws exception If this is a batch request and the batch data is not + * valid JSON + * @throws exception If this is a batch request and it exceeds the maximum + * number of api calls allowed in one batch. + */ + private function set_api_calls() { + $setting = setting::get_instance(); + + $this->api_calls = []; + + if(isset($this->request['batch']) === true) { + $batch = json_decode($this->request['batch'], true); + if($batch === null) { + throw new exception('Batch is not valid JSON.', 1012); + } + $batch_limit = $setting->get('batch_limit'); + if($batch_limit !== null && count($batch) > $batch_limit) { + throw new exception('Batch limit exceeded.', 1013); + } + foreach($batch as $api_call) { + $this->api_calls[] = new api_call($api_call); + } + } + else { + $this->api_calls[] = new api_call($this->request); + } + } + + /** + * Check for any issues with the aliases. + * + * @throws exception If any duplicate aliases are used. + */ + private function validate_aliases() { + $aliases = []; + foreach($this->api_calls as $api_call) { + $aliases[] = $api_call->get_alias(); + } + + $number_aliases = count($aliases); + $number_unique_aliases = count(array_unique($aliases)); + + // Check for duplicates. + if($number_aliases !== $number_unique_aliases) { + throw new exception('Duplicate alias.', 1018); + } + } + + /** + * Check to see if the request from the current IP address needs to be rate + * limited. If $requests_per_minute is null then there is no rate limiting. + * + * @throws exception If over the rate limit. + */ + private function rate_limit() { + $setting = setting::get_instance(); + + $requests_per_minute = $setting->get('requests_per_minute'); + + if($requests_per_minute === null) { + return false; + } + + $api_log_resource = new api_log2(); + $requests_this_minute = $api_log_resource->get_number_requests_since( + $_SERVER['REMOTE_ADDR'], + (time() - 60) + ); + + // A couple quick error checks + if($requests_this_minute > $requests_per_minute) { + throw new exception('Rate limit reached.', 1005); + } + } + + /** + * Force secure connections. + * + * @throws exception if not secure. + */ + private function force_ssl() { + $setting = setting::get_instance(); + + if($setting->get('force_ssl') === true && empty($_SERVER['HTTPS']) === true) { + throw new exception('Request must be sent over HTTPS.', 1006); + } + } + + /** + * Set the current API user based on the request API key. + * + * @throws exception if the API key is not set. + * @throws exception if the API key is not valid. + */ + private function set_api_user() { + // Make sure the API key that was sent is present and valid. + if(isset($this->request['api_key']) === false) { + throw new exception('API Key is required.', 1000); + } + + $api_user_resource = new api_user(); + $api_users = $api_user_resource->read(['api_key' => $this->request['api_key']]); + if(count($api_users) !== 1) { + throw new exception('API key is invalid.', 1003); + } else { + $this->api_user = $api_users[0]; + } + } + + /** + * Get the current API user. + * + * @return array + */ + public function get_api_user() { + return $this->api_user; + } + + /** + * Log the request and response to the database. The logged response is + * truncated to 16kb for sanity. + */ + private function log() { + $database = database::get_instance(); + $session = session::get_instance(); + $setting = setting::get_instance(); + $api_log_resource = new api_log2(); + + // If exception. + if(isset($this->response['data']['error_code']) === true) { + $api_log_resource->create([ + 'user_id' => $session->get_user_id(), + 'api_user_id' => $this->api_user['api_user_id'], + 'ip_address' => ip2long($_SERVER['REMOTE_ADDR']), + 'timestamp' => date('Y-m-d H:i:s', $this->begin_timestamp), + 'request' => $this->request, + 'response' => $this->response, + 'error_code' => $this->response['data']['error_code'], + 'error_detail' => $this->error_detail, + 'total_time' => $this->total_time, + 'query_count' => $database->get_query_count(), + 'query_time' => $database->get_query_time(), + ]); + } + else { + $user_resource = new \user(); + $user = $user_resource->get($session->get_user_id()); + + $api_log_resource->create([ + 'user_id' => $session->get_user_id(), + 'api_user_id' => $this->api_user['api_user_id'], + 'ip_address' => ip2long($_SERVER['REMOTE_ADDR']), + 'timestamp' => date('Y-m-d H:i:s', $this->begin_timestamp), + 'request' => $this->request, + 'response' => ($user['debug'] === true) ? $this->response : null, + 'total_time' => $this->total_time, + 'query_count' => $database->get_query_count(), + 'query_time' => $database->get_query_time(), + ]); + } + } + + /** + * Sets the headers that should be used for this API call. This is useful + * for doing things like returning files from the API where the content-type + * is no longer application/json. This replaces all headers; headers are not + * outputted to the browser until all API calls have completed, so the last + * call to this function will win. + * + * @param array $headers The headers to output. + * @param bool $custom_response Whether or not to wrap the response with the + * Cora data or just output the API call's return value. + * + * @throws exception If this is a batch request and a custom response was + * requested. + * @throws exception If this is a batch request and the content type was + * altered from application/json + * @throws exception If this is not a batch request and the content type was + * altered from application/json without a custom response. + */ + public function set_headers($headers, $custom_response = false) { + if(isset($this->request['batch']) === true) { + if($custom_response === true) { + throw new exception('Batch API requests can not use a custom response.', 1015); + } + if($this->content_type_is_json($headers) === false) { + throw new exception('Batch API requests must return JSON.', 1014); + } + } + else { + // Not a batch request + if($custom_response === false && $this->content_type_is_json($headers) === false) { + throw new exception('Non-custom responses must return JSON.', 1016); + } + } + $this->headers = $headers; + } + + /** + * Return whether or not the current output headers indicate that the + * content type is JSON. This is mostly just used to make sure that batch + * API calls output JSON. + * + * @param array $headers The headers to look at. + * + * @return bool Whether or not the output has a content type of + * application/json + */ + private function content_type_is_json($headers) { + return isset($headers['Content-type']) === true + && stristr($headers['Content-type'], 'application/json') !== false; + } + + /** + * Output whatever the headers are currently set to. + */ + private function output_headers() { + foreach($this->headers as $key => $value) { + header($key . ': ' . $value); + } + } + + /** + * Resets the headers to default. Have to do this in case one of the API + * calls changes them and there was an error to handle. + */ + private function set_default_headers() { + $this->headers['Content-type'] = 'application/json; charset=UTF-8'; + } + + /** + * Set the beestat-cached-until header. + */ + private function set_cached_until_header() { + $beestat_cached_until = []; + foreach($this->api_calls as $api_call) { + $cached_until = $api_call->get_cached_until(); + if($cached_until !== null) { + $beestat_cached_until[$api_call->get_alias()] = $api_call->get_cached_until(); + } + } + + if(count($beestat_cached_until) > 0) { + if(isset($this->request['batch']) === true) { + $this->headers['beestat-cached-until'] = json_encode($beestat_cached_until); + } else { + $this->headers['beestat-cached-until'] = reset($beestat_cached_until); + } + } + } + + /** + * Override of the default PHP error handler. Sets the error response then + * dies and lets the shutdown handler take over. + * + * @param int $error_code The error number from PHP. + * @param string $error_message The error message. + * @param string $error_file The file the error happend in. + * @param int $error_line The line of the file the error happened on. + * + * @return string The JSON response with the error details. + */ + public function error_handler($error_code, $error_message, $error_file, $error_line) { + $this->set_error_response( + $error_message, + $error_code, + true + ); + + $this->error_detail['file'] = $error_file; + $this->error_detail['line'] = $error_line; + $this->error_detail['trace'] = debug_backtrace(false); + try { + $database = database::get_instance(); + $this->error_detail['queries'] = $database->get_queries(); + } catch(Exception $e) {} + + die(); // Do not continue execution; shutdown handler will now run. + } + + /** + * Override of the default PHP exception handler. Sets the error response + * then dies and lets the shutdown handler take over. + * + * @param Exception $e The exception. + */ + public function exception_handler($e) { + $this->set_error_response( + $e->getMessage(), + $e->getCode(), + (method_exists($e, 'getReportable') === true ? $e->getReportable() : true) + ); + + $this->error_detail['file'] = $e->getFile(); + $this->error_detail['line'] = $e->getLine(); + $this->error_detail['trace'] = $e->getTrace(); + $this->error_detail['extra'] = (method_exists($e, 'getExtraInfo') === true ? $e->getExtraInfo() : null); + try { + $database = database::get_instance(); + $this->error_detail['queries'] = $database->get_queries(); + } catch(Exception $e) {} + + die(); // Do not continue execution; shutdown handler will now run. + } + + /** + * Handle all exceptions by generating a JSON response with the error + * details. If debugging is enabled, a bunch of other information is sent + * back to help out. + * + * There are a few places that call this function to set an error response, + * so this can't just be done in the exception handler alone. If an error + * occurs, rollback the current transaction. + * + * @param string $error_message The error message. + * @param mixed $error_code The supplied error code. + * @param array $reportable Whether or not the error is reportable. + */ + public function set_error_response($error_message, $error_code, $reportable) { + $setting = setting::get_instance(); + $session = session::get_instance(); + + // I guess if this fails then things are really bad, but let's at least + // protect against additional exceptions if the database connection or + // similar fails. + try { + $database = database::get_instance(); + $database->rollback_transaction(); + } catch(\Exception $e) {} + + $this->response = [ + 'success' => false, + 'data' => [ + 'error_message' => $error_message, + 'error_code' => $error_code + ] + ]; + + // Send data to Sentry for error logging. + // https://docs.sentry.io/development/sdk-dev/event-payloads/ + $api_user_id = $this->api_user['api_user_id']; + if ( + $reportable === true && + $setting->get('sentry_key') !== null && + $setting->get('sentry_project_id') !== null && + $api_user_id === 1 && + false // Temporarily disabling; over rate limit anyways + ) { + $data = [ + 'event_id' => str_replace('-', '', exec('uuidgen -r')), + 'timestamp' => date('c'), + 'logger' => 'cora', + 'platform' => 'php', + 'level' => 'error', + 'tags' => [ + 'error_code' => $error_code, + 'api_user_id' => $api_user_id + ], + 'extra' => [ + 'error_file' => $this->error_detail['file'], + 'error_line' => $this->error_detail['line'], + 'error_trace' => $this->error_detail['trace'], + 'error_extra' => $this->error_detail['extra'] + ], + 'exception' => [ + 'type' => 'Exception', + 'value' => $error_message, + 'handled' => false + ], + 'user' => [ + 'id' => $session->get_user_id(), + 'ip_address' => $_SERVER['REMOTE_ADDR'] + ] + ]; + + exec( + 'curl ' . + '-H "Content-Type: application/json" ' . + '-H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=' . $setting->get('sentry_key') . '" ' . + '--silent ' . // silent; keeps logs out of stderr + '--show-error ' . // override silent on failure + '--max-time 10 ' . + '--connect-timeout 5 ' . + '--data \'' . json_encode($data) . '\' ' . + '"https://sentry.io/api/' . $setting->get('sentry_project_id') . '/store/" > /dev/null &' + ); + } + } + + /** + * Executes when the script finishes. If there was an error that somehow + * didn't get caught, then this will find it with error_get_last and return + * appropriately. Note that error_get_last() will only get something when an + * error wasn't caught by my error/exception handlers. The default PHP error + * handler fills this in. Doesn't do anything if an exception was thrown due + * to the rate limit. + * + * @throws \Exception If a this was a batch request but one of the api calls + * changed the content-type to anything but the default. + */ + public function shutdown_handler() { + // Since the shutdown handler is rather verbose in what it has to check for + // and do, it's possible it will fail or detect an error that needs to be + // handled. For example, someone could return binary data from an API call + // which will fail a json_encode, or someone could change the headers in a + // batch API call, which isn't allowed. I can't throw an exception since I'm + // already in the shutdown handler...it will be caught but it won't execute + // a new shutdown handler and no output will be sent to the client. I just + // have to handle all problems manually. + try { + $this->total_time = round((microtime(true) - $this->begin_timestamp) * 1000); + + // Fix the current working directory. See documentation on this class + // variable for details. + chdir($this->current_working_directory); + + // If I didn't catch an error/exception with my handlers, look here...this + // will catch fatal errors that I can't. + $error = error_get_last(); + if($error !== null) { + $this->set_error_response( + $error['message'], + $error['type'], + true + ); + + $this->error_detail['file'] = $error['file']; + $this->error_detail['line'] = $error['line']; + $this->error_detail['trace'] = debug_backtrace(false); + try { + $database = database::get_instance(); + $this->error_detail['queries'] = $database->get_queries(); + } catch(Exception $e) {} + } + + // If the response has already been set by one of the error handlers, end + // execution here and just log & output the response. + if(isset($this->response) === true) { + + // Don't log anything for rate limit breaches. + if($this->response['data']['error_code'] !== 1005) { + $this->log(); + } + + // Override whatever headers might have already been set. + $this->set_default_headers(); + $this->output_headers(); + die(json_encode($this->response)); + } + else { + // If we got here, no errors have occurred. + + // For non-custom responses, build the response, log it, and output it. + $this->response = ['success' => true]; + + if(isset($this->request['batch']) === true) { + $this->response['data'] = []; + foreach($this->api_calls as $api_call) { + $this->response['data'][$api_call->get_alias()] = $api_call->get_response(); + } + } + else { + $this->response['data'] = $this->api_calls[0]->get_response(); + } + + // Log all of the API calls that were made. + $this->log(); + + // Output the response + $this->output_headers(); + die(json_encode($this->response)); + } + } + catch(\Exception $e) { + $this->set_error_response( + $e->getMessage(), + $e->getCode(), + (method_exists($e, 'getReportable') === true ? $e->getReportable() : true) + ); + + $this->error_detail['file'] = $e->getFile(); + $this->error_detail['line'] = $e->getLine(); + $this->error_detail['trace'] = $e->getTrace(); + $this->error_detail['extra'] = (method_exists($e, 'getExtraInfo') === true ? $e->getExtraInfo() : null); + try { + $database = database::get_instance(); + $this->error_detail['queries'] = $database->get_queries(); + } catch(Exception $e) {} + + $this->set_default_headers(); + $this->output_headers(); + die(json_encode($this->response)); + } + } + +} diff --git a/api/cora/session.php b/api/cora/session.php index 8d15c08..929b609 100644 --- a/api/cora/session.php +++ b/api/cora/session.php @@ -10,18 +10,11 @@ namespace cora; final class session { /** - * The session_key for this session. + * The current session. * - * @var string + * @var array */ - private $session_key = null; - - /** - * The user_id for this session. - * - * @var int - */ - private $user_id = null; + private $session; /** * The singleton. @@ -91,7 +84,7 @@ final class session { $database = database::get_instance(); $session_key = $this->generate_session_key(); - $database->create( + $this->session = $database->create( 'cora\session', [ 'session_key' => $session_key, @@ -126,9 +119,6 @@ final class session { } } - $this->session_key = $session_key; - $this->user_id = $user_id; - return $session_key; } @@ -182,7 +172,7 @@ final class session { return false; } - $database->update( + $this->session = $database->update( 'cora\session', [ 'session_id' => $session['session_id'], @@ -191,8 +181,7 @@ final class session { ] ); - $this->session_key = $session['session_key']; - $this->user_id = $session['user_id']; + return true; } else { $this->delete_cookie('session_key'); @@ -220,16 +209,15 @@ final class session { public function delete($session_key = null) { $database = database::get_instance(); if($session_key === null) { - $session_key = $this->session_key; + $session_key = $this->session['session_key']; } $sessions = $database->read('cora\session', ['session_key' => $session_key]); if(count($sessions) === 1) { $database->delete('cora\session', $sessions[0]['session_id']); // Remove these if the current session got logged out. - if($session_key === $this->session_key) { - $this->session_key = null; - $this->user_id = null; + if($session_key === $this->session['session_key']) { + $this->session = null; } return true; } @@ -244,11 +232,16 @@ final class session { * @return int The current user_id. */ public function get_user_id() { - return $this->user_id; + return $this->session['user_id']; } - public function delete_user_id() { - $this->user_id = null; + /** + * Get the session. + * + * @return array The session. + */ + public function get() { + return $this->session; } /** @@ -309,4 +302,12 @@ final class session { $this->set_cookie($name, '', time() - 86400); } + /** + * Check if the session is valid. + * + * @return bool Whether or not the session is valid. + */ + public function is_valid() { + return $this->session !== null; + } } diff --git a/api/cora/setting.example.php b/api/cora/setting.example.php index dcd3eb0..cd89d72 100644 --- a/api/cora/setting.example.php +++ b/api/cora/setting.example.php @@ -153,16 +153,6 @@ final class setting { 'database_password' => '', 'database_name' => '', - /** - * Influx database connection information. This is where most logging is - * sent to. - */ - 'influx_database_host' => '', - 'influx_database_port' => 8086, - 'influx_database_name' => '', - 'influx_database_username' => '', - 'influx_database_password' => '', - /** * Key and project id obtained from the Sentry DSN. See sentry.io. */ diff --git a/api/ecobee.php b/api/ecobee.php index f1fc43f..70590bd 100644 --- a/api/ecobee.php +++ b/api/ecobee.php @@ -15,10 +15,8 @@ class ecobee extends external_api { ] ]; - protected static $log_influx = true; - protected static $log_mysql = 'error'; - - protected static $influx_retention_policy = '30d'; + protected static $log_mysql = 'all'; + protected static $log_mysql_verbose = false; protected static $cache = false; protected static $cache_for = null; @@ -215,7 +213,7 @@ class ecobee extends external_api { if ($response === null) { // If this hasn't already been logged, log the error. if($this::$log_mysql !== 'all') { - $this->log_mysql($curl_response); + $this->log_mysql($curl_response, true); } throw new Exception('Invalid JSON'); } @@ -239,7 +237,7 @@ class ecobee extends external_api { else if (isset($response['status']) === true && $response['status']['code'] === 16) { // Token has been deauthorized by user. You must re-request authorization. if($this::$log_mysql !== 'all') { - $this->log_mysql($curl_response); + $this->log_mysql($curl_response, true); } $this->api('ecobee_token', 'delete', $ecobee_token['ecobee_token_id']); $this->api('user', 'log_out', ['all' => true]); @@ -248,7 +246,7 @@ class ecobee extends external_api { else if (isset($response['status']) === true && $response['status']['code'] !== 0) { // Any other error if($this::$log_mysql !== 'all') { - $this->log_mysql($curl_response); + $this->log_mysql($curl_response, true); } throw new Exception($response['status']['message']); } diff --git a/api/external_api.php b/api/external_api.php index 84c4983..c59a50e 100644 --- a/api/external_api.php +++ b/api/external_api.php @@ -9,12 +9,6 @@ */ class external_api extends cora\api { - /** - * Whether or not to log the API call to Influx. This will only log the - * event and basic timing information; no detail. - */ - protected static $log_influx = true; - /** * Whether or not to log the API call to MySQL. This will log the entire * request and response in full detail. Valid values are "error", "all", and @@ -23,10 +17,9 @@ class external_api extends cora\api { protected static $log_mysql = 'error'; /** - * Default retention policy when inserting data. Autogen is the default - * infinite one; also available is 30d. + * Whether or not to include the request and response in non-errored logs. */ - protected static $influx_retention_policy = 'autogen'; + protected static $log_mysql_verbose = true; /** * Whether or not to cache API calls. This will store a hash of the request @@ -59,7 +52,6 @@ class external_api extends cora\api { } $this->request_timestamp = time(); - $this->request_timestamp_microtime = $this->microtime(); $curl_handle = curl_init(); curl_setopt($curl_handle, CURLOPT_URL, $arguments['url']); @@ -79,7 +71,7 @@ class external_api extends cora\api { curl_setopt($curl_handle, CURLOPT_POSTFIELDS, $arguments['post_fields']); } - if($this::$log_influx !== false || $this::$log_mysql !== false) { + if($this::$log_mysql !== false) { curl_setopt($curl_handle, CURLINFO_HEADER_OUT, true); } @@ -98,34 +90,28 @@ class external_api extends cora\api { if($cache_entry === null) { $curl_response = curl_exec($curl_handle); + $this->curl_info = curl_getinfo($curl_handle); if($curl_response === false || curl_errno($curl_handle) !== 0) { - $this->cora->set_error_extra_info([ - 'curl_error' => curl_error($curl_handle) - ]); - // Error logging - if($this::$log_influx === true) { - $this->log_influx( - $this->resource . '_api_log', - true - ); - } if($this::$log_mysql === 'all' || $this::$log_mysql === 'error') { - $this->log_mysql($curl_response); + $this->log_mysql($curl_response, true); } - throw new cora\exception('Could not connect to ' . $this->resource . '.', 10600, false); + throw new cora\exception( + 'Could not connect to external API.', + 10600, + false, + [ + 'resource' => $this->resource, + 'curl_error' => curl_error($curl_handle) + ] + ); + } // General (success) logging - if($this::$log_influx === true) { - $this->log_influx( - $this->resource . '_api_log', - false - ); - } if($this::$log_mysql === 'all') { $this->log_mysql($curl_response); } @@ -218,63 +204,27 @@ class external_api extends cora\api { ); } - /** - * Log to InfluxDB/Grafana. - * - * @param array $measurement Which measurement to log as. - * @param boolean $exception Whether or not this was an exception (failure - * to connect, etc). - */ - private function log_influx($measurement, $exception) { - $this->api( - 'logger', - 'log_influx', - [ - 'measurement' => $measurement, - 'tags' => [ - 'user_id' => $this->session->get_user_id(), - 'api_user_id' => $this->cora->get_api_user()['api_user_id'], - 'exception' => $exception === true ? '1' : '0' - ], - 'fields' => [ - 'http_code' => (int) $this->curl_info['http_code'], - 'connect_time' => round($this->curl_info['connect_time'], 4) - ], - 'timestamp' => $this->request_timestamp_microtime, - 'retention_policy' => $this::$influx_retention_policy - ] - ); - } - /** * Log to MySQL with the complete details. * * @param array $curl_response The response of the cURL request. + * @param boolean $force_verbose Whether or not to force verbose logging. */ - protected function log_mysql($curl_response) { + protected function log_mysql($curl_response, $force_verbose = false) { + $attributes = [ + 'api_user_id' => $this->request->get_api_user()['api_user_id'], + 'request_timestamp' => date('Y-m-d H:i:s', $this->request_timestamp) + ]; + + if($this::$log_mysql_verbose === true || $force_verbose === true) { + $attributes['request'] = $this->curl_info; + $attributes['response'] = $curl_response; + } + $this->api( ($this->resource . '_api_log'), 'create', - [ - 'attributes' => [ - 'api_user_id' => $this->cora->get_api_user()['api_user_id'], - 'request_timestamp' => date('Y-m-d H:i:s', $this->request_timestamp), - 'request' => $this->curl_info, - 'response' => $curl_response, - ] - ] + ['attributes' => $attributes] ); } - - /** - * Get microtime for influx. - * - * @link https://github.com/influxdata/influxdb-php - * - * @return string - */ - private function microtime() { - list($usec, $sec) = explode(' ', microtime()); - return sprintf('%d%06d', $sec, $usec * 1000000); - } } diff --git a/api/index.php b/api/index.php index 4d048ae..44a5c15 100644 --- a/api/index.php +++ b/api/index.php @@ -28,13 +28,16 @@ spl_autoload_register(function($class) { include str_replace('\\', '/', $class) . '.php'; }); -// Construct cora and set up error handlers. -$cora = cora\cora::get_instance(); -set_error_handler([$cora, 'error_handler']); -set_exception_handler([$cora, 'exception_handler']); +// Construct request and set up error handlers. +$request = cora\request::get_instance(); +set_error_handler([$request, 'error_handler']); +set_exception_handler([$request, 'exception_handler']); // The shutdown handler will output the response. -register_shutdown_function([$cora, 'shutdown_handler']); +register_shutdown_function([$request, 'shutdown_handler']); + +// Go! +$request->process($_REQUEST); // Useful function function array_median($array) { @@ -57,6 +60,3 @@ function array_mean($array) { return array_sum($array) / count($array); } - -// Go! -$cora->process_request($_REQUEST); diff --git a/api/logger.php b/api/logger.php deleted file mode 100644 index 5267cb7..0000000 --- a/api/logger.php +++ /dev/null @@ -1,141 +0,0 @@ -setting->get('influx_database_host') === null || - $this->setting->get('influx_database_port') === null || - $this->setting->get('influx_database_name') === null || - $this->setting->get('influx_database_username') === null || - $this->setting->get('influx_database_password') === null - ) { - return; - } - - $tag_string = $this->get_tag_string($tags); - $field_string = $this->get_field_string($fields); - - $data_binary = - $measurement . - ($tag_string !== '' ? ',' : '') . - $tag_string . ' ' . - $field_string . ' ' . - $timestamp; - - $url = - $this->setting->get('influx_database_host') . - ':' . - $this->setting->get('influx_database_port') . - '/write' . - '?db=' . $this->setting->get('influx_database_name') . - ($retention_policy !== null ? ('&rp=' . $retention_policy) : '') . - '&precision=u'; - - exec( - 'curl ' . - '-u ' . $this->setting->get('influx_database_username') . ':' . $this->setting->get('influx_database_password') . ' ' . - '-POST "' . $url . '" ' . - '--silent ' . // silent; keeps logs out of stderr - '--show-error ' . // override silent on failure - '--max-time 10 ' . - '--connect-timeout 5 ' . - '--data-binary \'' . $data_binary . '\' > /dev/null &' - ); - } - - /** - * Convert an array into a key/value string. - * - * @param array $array The input array. Null values are removed. - * - * @return string A string like "k1=v1,k2=v2". If no non-null values are - * present this will be an empty string. - */ - private function get_field_string($fields) { - $parts = []; - - foreach($fields as $key => $value) { - if($value === null) { - continue; - } else if(is_bool($value) === true) { - $value = ($value === true) ? 'true' : 'false'; - } else if(is_int($value) === true) { - $value = $value . 'i'; - } else if(is_float($value) === true) { - $value = $value; - } else { - $value = $this->escape_field_value($value); - } - - $parts[] = $key . '=' . $value; - } - - return implode(',', $parts); - } - - /** - * Convert a tag array into a key/value string. Tags are always strings in - * Influx. - * - * @param array $array The input array. Null values are removed. - * - * @return string A string like "k1=v1,k2=v2". If no non-null values are - * present this will be an empty string. - */ - private function get_tag_string($tags) { - $parts = []; - - foreach($tags as $key => $value) { - if($value === null) { - continue; - } else { - $parts[] = $this->escape_tag_key_value($key) . '=' . $this->escape_tag_key_value($value); - } - } - - return implode(',', $parts); - } - - /** - * Add slashes where necessary to prevent injection attacks. Tag values just - * sit there unquoted (you can't quote them or the quote gets included as - * part of the value) so we have to escape other special characters in that - * context. - * - * @param string $value The value to escape. - */ - private function escape_tag_key_value($value) { - return str_replace([' ', ',', '='], ['\ ', '\,', '\='], $value); - } - - /** - * Add slashes where necessary to prevent injection attacks. Field values - * sit inside of "", so escape any " characters. At a higher level they sit - * inside of a ' from the cURL body. Escape these as well. - * - * @param string $value The value to escape. - */ - private function escape_field_value($value) { - return '"' . str_replace(['"', "'"], ['\"', "'\''"], $value) . '"'; - } -} diff --git a/api/mailchimp.php b/api/mailchimp.php index 6534cf5..dd44dfa 100644 --- a/api/mailchimp.php +++ b/api/mailchimp.php @@ -7,7 +7,6 @@ */ class mailchimp extends external_api { - protected static $log_influx = true; protected static $log_mysql = 'all'; protected static $cache = false; diff --git a/api/patreon.php b/api/patreon.php index ad77f67..ffd9654 100644 --- a/api/patreon.php +++ b/api/patreon.php @@ -15,7 +15,6 @@ class patreon extends external_api { 'public' => [] ]; - protected static $log_influx = true; protected static $log_mysql = 'all'; protected static $cache = false; diff --git a/api/smarty_streets.php b/api/smarty_streets.php index a2ec746..09115b9 100644 --- a/api/smarty_streets.php +++ b/api/smarty_streets.php @@ -15,7 +15,6 @@ */ class smarty_streets extends external_api { - protected static $log_influx = true; protected static $log_mysql = 'all'; protected static $cache = true; diff --git a/js/beestat/api.js b/js/beestat/api.js index 606c31b..1dbb432 100644 --- a/js/beestat/api.js +++ b/js/beestat/api.js @@ -168,7 +168,7 @@ beestat.api.prototype.load_ = function(response_text) { if ( response.data && ( - response.data.error_code === 1004 || // Session is expired. + response.data.error_code === 1505 || // Session is expired. response.data.error_code === 10000 || // Could not get first token. response.data.error_code === 10001 || // Could not refresh ecobee token; no token found. response.data.error_code === 10002 || // Could not refresh ecobee token; ecobee returned no token. @@ -186,7 +186,7 @@ beestat.api.prototype.load_ = function(response_text) { return; } - // Cach responses + // Cache responses var cached_until_header = this.xhr_.getResponseHeader('beestat-cached-until'); if (this.is_batch_() === true) { @@ -244,7 +244,7 @@ beestat.api.prototype.is_batch_ = function() { */ beestat.api.prototype.cache_ = function(api_call, data, until) { var server_date = moment(this.xhr_.getResponseHeader('date')); - var duration = moment.duration(moment(until).diff(server_date)); + var duration = moment.duration(moment.utc(until).diff(server_date)); beestat.api.cache[this.get_key_(api_call)] = { 'data': data,