From 03d43470fbf348d0855e47c3a2eb589b5cf85075 Mon Sep 17 00:00:00 2001 From: David Bomba Date: Tue, 1 Feb 2022 17:14:27 +1100 Subject: [PATCH] Working on csv import refactor --- app/Import/Providers/BaseImport.php | 166 +++++++++ app/Import/Providers/Csv.php | 94 +++++ app/Import/Providers/Freshbooks.php | 16 + app/Import/Providers/ImportInterface.php | 21 ++ app/Import/Providers/Invoice2Go.php | 16 + app/Import/Providers/Invoicely.php | 16 + app/Import/Providers/Wave.php | 16 + app/Import/Providers/Zoho.php | 16 + app/Import/Transformer/BaseTransformer.php | 344 ++++++++++++++++++ .../Transformer/Csv/ClientTransformer.php | 80 ++++ app/Jobs/Import/CSVIngest.php | 99 +++++ .../Stripe/Jobs/PaymentIntentWebhook.php | 3 +- .../Subscription/SubscriptionService.php | 2 +- routes/api.php | 2 +- 14 files changed, 888 insertions(+), 3 deletions(-) create mode 100644 app/Import/Providers/BaseImport.php create mode 100644 app/Import/Providers/Csv.php create mode 100644 app/Import/Providers/Freshbooks.php create mode 100644 app/Import/Providers/ImportInterface.php create mode 100644 app/Import/Providers/Invoice2Go.php create mode 100644 app/Import/Providers/Invoicely.php create mode 100644 app/Import/Providers/Wave.php create mode 100644 app/Import/Providers/Zoho.php create mode 100644 app/Import/Transformer/BaseTransformer.php create mode 100644 app/Import/Transformer/Csv/ClientTransformer.php create mode 100644 app/Jobs/Import/CSVIngest.php diff --git a/app/Import/Providers/BaseImport.php b/app/Import/Providers/BaseImport.php new file mode 100644 index 000000000000..f421bb9ea3f4 --- /dev/null +++ b/app/Import/Providers/BaseImport.php @@ -0,0 +1,166 @@ +company = $company; + $this->request = $request; + $this->hash = $request['hash']; + $this->import_type = $request['import_type']; + $this->skip_header = $request['skip_header'] ?? null; + $this->column_map = + ! empty( $request['column_map'] ) ? + array_combine( array_keys( $request['column_map'] ), array_column( $request['column_map'], 'mapping' ) ) : null; + + auth()->login( $this->company->owner(), true ); + + auth()->user()->setCompany($this->company); + } + + protected function findUser( $user_hash ) { + $user = User::where( 'account_id', $this->company->account_id ) + ->where( DB::raw( 'CONCAT_WS(" ", first_name, last_name)' ), 'like', '%' . $user_hash . '%' ) + ->first(); + + if ( $user ) { + return $user->id; + } else { + return $this->company->owner()->id; + } + } + + protected function getCsvData( $entity_type ) { + + $base64_encoded_csv = Cache::pull( $this->hash . '-' . $entity_type ); + if ( empty( $base64_encoded_csv ) ) { + return null; + } + + $csv = base64_decode( $base64_encoded_csv ); + $csv = Reader::createFromString( $csv ); + + $stmt = new Statement(); + $data = iterator_to_array( $stmt->process( $csv ) ); + + if ( count( $data ) > 0 ) { + $headers = $data[0]; + + // Remove Invoice Ninja headers + if ( count( $headers ) && count( $data ) > 4 && $this->import_type === 'csv' ) { + $first_cell = $headers[0]; + if ( strstr( $first_cell, config( 'ninja.app_name' ) ) ) { + array_shift( $data ); // Invoice Ninja... + array_shift( $data ); // + array_shift( $data ); // Enitty Type Header + } + } + } + + return $data; + } + + public function mapCSVHeaderToKeys( $csvData ) { + $keys = array_shift( $csvData ); + + return array_map( function ( $values ) use ( $keys ) { + return array_combine( $keys, $values ); + }, $csvData ); + } + + private function groupInvoices( $csvData, $key ) { + // Group by invoice. + $grouped = []; + + foreach ( $csvData as $line_item ) { + if ( empty( $line_item[ $key ] ) ) { + $this->error_array['invoice'][] = [ 'invoice' => $line_item, 'error' => 'No invoice number' ]; + } else { + $grouped[ $line_item[ $key ] ][] = $line_item; + } + } + + return $grouped; + } + + public function getErrors() + { + return $this->error_array; + } + + public function ingest($data, $entity_type) + { + foreach ( $data as $record ) { + try { + $entity = $this->transformer->transform( $record ); + + /** @var \App\Http\Requests\Request $request */ + $request = new $this->request_name(); + + // Pass entity data to request so it can be validated + $request->query = $request->request = new ParameterBag( $entity ); + $validator = Validator::make( $entity, $request->rules() ); + + if ( $validator->fails() ) { + $this->error_array[ $entity_type ][] = + [ $entity_type => $record, 'error' => $validator->errors()->all() ]; + } else { + $entity = + $this->repository->save( + array_diff_key( $entity, [ 'user_id' => false ] ), + $this->factory_name::create( $this->company->id, $this->getUserIDForRecord( $entity ) ) ); + + $entity->saveQuietly(); + + } + } catch ( \Exception $ex ) { + if ( $ex instanceof ImportException ) { + $message = $ex->getMessage(); + } else { + report( $ex ); + $message = 'Unknown error'; + } + + $this->error_array[ $entity_type ][] = [ $entity_type => $record, 'error' => $message ]; + } + } + } + +} diff --git a/app/Import/Providers/Csv.php b/app/Import/Providers/Csv.php new file mode 100644 index 000000000000..6c516f5b0089 --- /dev/null +++ b/app/Import/Providers/Csv.php @@ -0,0 +1,94 @@ +{$entity}; + + } + + private function client() + { + + $entity_type = 'client'; + + $data = $this->getCsvData($entity_type); + + $data = $this->preTransform($data); + + if(empty($data)) + return; + + $this->request_name = StoreClientRequest::class; + $this->repository_name = ClientRepository::class; + $this->factory_name = ClientFactory::class; + + $this->repository = app()->make( $this->repository_name ); + $this->repository->import_mode = true; + + $this->transformer = new ClientTransformer($this->company); + + $this->ingest($data, $entity_type); + + } + + + + + + public function preTransform(array $data) + { + + + if ( empty( $this->column_map[ 'client' ] ) ) { + return false; + } + + if ( $this->skip_header ) { + array_shift( $data ); + } + + //sort the array by key + $keys = $this->column_map[ 'client' ]; + ksort( $keys ); + + $data = array_map( function ( $row ) use ( $keys ) { + return array_combine( $keys, array_intersect_key( $row, $keys ) ); + }, $data ); + + + return $data; + + + } + + public function transform(array $data) + { + + } + +} \ No newline at end of file diff --git a/app/Import/Providers/Freshbooks.php b/app/Import/Providers/Freshbooks.php new file mode 100644 index 000000000000..3f25795848f4 --- /dev/null +++ b/app/Import/Providers/Freshbooks.php @@ -0,0 +1,16 @@ +company = $company; + } + + public function getString($data, $field) + { + return (isset($data[$field]) && $data[$field]) ? $data[$field] : ''; + } + + public function getCurrencyByCode( $data, $key = 'client.currency_id' ) + { + $code = array_key_exists( $key, $data ) ? $data[ $key ] : false; + + return $this->maps['currencies'][ $code ] ?? $this->company->settings->currency_id; + } + + public function getClient($client_name, $client_email) { + + $clients = $this->company->clients(); + + $client_id_search = $clients->where( 'id_number', $client_name ); + + if ( $client_id_search->count() >= 1 ) { + return $client_id_search->first()->id; + nlog("found via id number"); + } + + $client_name_search = $clients->where( 'name', $client_name ); + + if ( $client_name_search->count() >= 1 ) { + return $client_name_search->first()->id; + nlog("found via name"); + } + + if ( ! empty( $client_email ) ) { + $contacts = ClientContact::where( 'company_id', $this->company->id ) + ->where( 'email', $client_email ); + + if ( $contacts->count() >= 1 ) { + return $contacts->first()->client_id; + nlog("found via contact"); + } + } + nlog("did not find client"); + + return null; + } + + + + /////////////////////////////////////////////////////////////////////////////////// + /** + * @param $name + * + * @return bool + */ + public function hasClient($name) + { + return $this->company->clients()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->exists(); + } + + /** + * @param $name + * + * @return bool + */ + public function hasVendor($name) + { + return $this->company->vendors()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->exists(); + } + + /** + * @param $key + * + * @return bool + */ + public function hasProduct($key) + { + return $this->company->products()->whereRaw("LOWER(REPLACE(`product_key`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $key))])->exists(); + } + + /** + * @param $data + * @param $field + * + * @return float + */ + public function getFloat($data, $field) + { + if (array_key_exists($field, $data)) { + $number = preg_replace('/[^0-9-.]+/', '', $data[$field]); + } else { + $number = 0; + } + + return Number::parseFloat($number); + } + + /** + * @param $name + * + * @return int|null + */ + public function getClientId($name) + { + $client = $this->company->clients()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first(); + + return $client ? $client->id : null; + } + + /** + * @param $name + * + * @return string + */ + public function getProduct($data, $key, $field, $default = false) + { + + $product = $this->company->products()->whereRaw("LOWER(REPLACE(`product_key`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $data->{$key}))])->first(); + + if($product) + return $product->{$field} ?: $default; + + return $default; + + } + + /** + * @param $email + * + * @return ?Contact + */ + public function getContact($email) + { + + $contact = $this->company->client_contacts()->whereRaw("LOWER(REPLACE(`email`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $email))])->first(); + + if(!$contact) + return null; + + return $contact; + + } + + /** + * @param $name + * + * @return int|null + */ + public function getCountryId($name) + { + if(strlen($name) == 2) + return $this->getCountryIdBy2($name); + + $country = Country::whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first(); + + return $country ? $country->id : null; + } + + /** + * @param $name + * + * @return int|null + */ + public function getCountryIdBy2($name) + { + return Country::where('iso_3166_2', $name)->exists() ? Country::where('iso_3166_2', $name)->first()->id : null; + } + + /** + * @param $name + * + * @return int + */ + public function getTaxRate($name) + { + $name = strtolower(trim($name)); + + $tax_rate = $this->company->tax_rates()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first(); + + return $tax_rate ? $tax_rate->rate : 0; + } + + /** + * @param $name + * + * @return string + */ + public function getTaxName($name) + { + $name = strtolower(trim($name)); + + $tax_rate = $this->company->tax_rates()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first(); + + return $tax_rate ? $tax_rate->name : ''; + + } + + /** + * @param $date + * @param string $format + * @param mixed $data + * @param mixed $field + * + * @return null + */ + public function getDate($data, $field) + { + if ($date = data_get($data, $field)) { + try { + $date = new Carbon($date); + } catch (\Exception $e) { + // if we fail to parse return blank + $date = false; + } + } + + return $date ? $date->format('Y-m-d') : null; + } + + /** + * @param $number + * + * @return ?string + */ + public function getInvoiceNumber($number) + { + return $number ? ltrim( trim( $number ), '0' ) : null; + } + + /** + * @param $invoice_number + * + * @return int|null + */ + public function getInvoiceId($invoice_number) + { + $invoice = $this->company->invoices()->whereRaw("LOWER(REPLACE(`number`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $invoice_number))])->first(); + + return $invoice ? $invoice->id : null; + } + + /** + * @param $invoice_number + * + * @return bool + */ + public function hasInvoice($invoice_number) + { + + return $this->company->invoices()->whereRaw("LOWER(REPLACE(`number`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $invoice_number))])->exists(); + + } + + /** + * @param $invoice_number + * + * @return int|null + */ + public function getInvoiceClientId($invoice_number) + { + $invoice = $this->company->invoices()->whereRaw("LOWER(REPLACE(`number`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $invoice_number))])->first(); + + return $invoice ? $invoice->client_id : null; + } + + /** + * @param $name + * + * @return int|null + */ + public function getVendorId($name) + { + $vendor = $this->company->vendors()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first(); + + return $vendor ? $vendor->id : null; + } + + /** + * @param $name + * + * @return int|null + */ + public function getExpenseCategoryId( $name ) { + + $ec = $this->company->expense_categories()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first(); + + return $ec ? $ec->id : null; + + } + + /** + * @param $name + * + * @return int|null + */ + public function getProjectId( $name ) { + + $project = $this->company->projects()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first(); + + return $project ? $project->id : null; + } + + /** + * @param $name + * + * @return int|null + */ + public function getPaymentTypeId( $name ) { + + $pt = PaymentType::whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first(); + + return $pt ? $pt->id : null; + } +} diff --git a/app/Import/Transformer/Csv/ClientTransformer.php b/app/Import/Transformer/Csv/ClientTransformer.php new file mode 100644 index 000000000000..3659a3b6a141 --- /dev/null +++ b/app/Import/Transformer/Csv/ClientTransformer.php @@ -0,0 +1,80 @@ +name) && $this->hasClient($data->name)) { + throw new ImportException('Client already exists'); + } + + $settings = new \stdClass; + $settings->currency_id = (string)$this->getCurrencyByCode($data); + + return [ + 'company_id' => $this->company->id, + 'name' => $this->getString( $data, 'client.name' ), + 'work_phone' => $this->getString( $data, 'client.phone' ), + 'address1' => $this->getString( $data, 'client.address1' ), + 'address2' => $this->getString( $data, 'client.address2' ), + 'postal_code' => $this->getString( $data, 'client.postal_code'), + 'city' => $this->getString( $data, 'client.city' ), + 'state' => $this->getString( $data, 'client.state' ), + 'shipping_address1' => $this->getString( $data, 'client.shipping_address1' ), + 'shipping_address2' => $this->getString( $data, 'client.shipping_address2' ), + 'shipping_city' => $this->getString( $data, 'client.shipping_city' ), + 'shipping_state' => $this->getString( $data, 'client.shipping_state' ), + 'shipping_postal_code' => $this->getString( $data, 'client.shipping_postal_code' ), + 'public_notes' => $this->getString( $data, 'client.public_notes' ), + 'private_notes' => $this->getString( $data, 'client.private_notes' ), + 'website' => $this->getString( $data, 'client.website' ), + 'vat_number' => $this->getString( $data, 'client.vat_number' ), + 'id_number' => $this->getString( $data, 'client.id_number' ), + 'custom_value1' => $this->getString( $data, 'client.custom_value1' ), + 'custom_value2' => $this->getString( $data, 'client.custom_value2' ), + 'custom_value3' => $this->getString( $data, 'client.custom_value3' ), + 'custom_value4' => $this->getString( $data, 'client.custom_value4' ), + 'balance' => preg_replace( '/[^0-9,.]+/', '', $this->getFloat( $data, 'client.balance' ) ), + 'paid_to_date' => preg_replace( '/[^0-9,.]+/', '', $this->getFloat( $data, 'client.paid_to_date' ) ), + 'credit_balance' => 0, + 'settings' => $settings, + 'client_hash' => Str::random( 40 ), + 'contacts' => [ + [ + 'first_name' => $this->getString( $data, 'contact.first_name' ), + 'last_name' => $this->getString( $data, 'contact.last_name' ), + 'email' => $this->getString( $data, 'contact.email' ), + 'phone' => $this->getString( $data, 'contact.phone' ), + 'custom_value1' => $this->getString( $data, 'contact.custom_value1' ), + 'custom_value2' => $this->getString( $data, 'contact.custom_value2' ), + 'custom_value3' => $this->getString( $data, 'contact.custom_value3' ), + 'custom_value4' => $this->getString( $data, 'contact.custom_value4' ), + ], + ], + 'country_id' => isset( $data['client.country'] ) ? $this->getCountryId( $data['client.country']) : null, + 'shipping_country_id' => isset($data['client.shipping_country'] ) ? $this->getCountryId( $data['client.shipping_country'] ) : null, + ]; + } +} diff --git a/app/Jobs/Import/CSVIngest.php b/app/Jobs/Import/CSVIngest.php new file mode 100644 index 000000000000..c30871e29958 --- /dev/null +++ b/app/Jobs/Import/CSVIngest.php @@ -0,0 +1,99 @@ +company = $company; + $this->request = $request; + $this->hash = $request['hash']; + $this->import_type = $request['import_type']; + $this->skip_header = $request['skip_header'] ?? null; + $this->column_map = + ! empty( $request['column_map'] ) ? + array_combine( array_keys( $request['column_map'] ), array_column( $request['column_map'], 'mapping' ) ) : null; + } + + /** + * Execute the job. + * + * + * @return void + */ + public function handle() { + + MultiDB::setDb( $this->company->db ); + + $engine = $this->bootEngine($this->import_type); + + foreach ( [ 'client', 'product', 'invoice', 'payment', 'vendor', 'expense' ] as $entity ) { + + $engine->import($entity); + + } + + } + + private function bootEngine(string $import_type) + { + switch ($import_type) { + case 'csv': + return new Csv( $this->request, $this->company); + break; + case 'waveaccounting': + return new Wave( $this->request, $this->company); + break; + case 'invoicely': + return new Invoicely( $this->request, $this->company); + break; + case 'invoice2go': + return new Invoice2Go( $this->request, $this->company); + break; + case 'zoho': + return new Zoho( $this->request, $this->company); + break; + case 'freshbooks': + return new Freshbooks( $this->request, $this->company); + break; + default: + // code... + break; + } + } +} \ No newline at end of file diff --git a/app/PaymentDrivers/Stripe/Jobs/PaymentIntentWebhook.php b/app/PaymentDrivers/Stripe/Jobs/PaymentIntentWebhook.php index f621e923d22b..8ca8ebe1989a 100644 --- a/app/PaymentDrivers/Stripe/Jobs/PaymentIntentWebhook.php +++ b/app/PaymentDrivers/Stripe/Jobs/PaymentIntentWebhook.php @@ -119,7 +119,8 @@ class PaymentIntentWebhook implements ShouldQueue $payment_hash = PaymentHash::where('hash', $hash)->first(); - nlog("no payment found"); + if(!$payment_hash) + return; if(optional($this->stripe_request['object']['charges']['data'][0]['metadata']['payment_hash']) && in_array('card', $this->stripe_request['object']['allowed_source_types'])) { diff --git a/app/Services/Subscription/SubscriptionService.php b/app/Services/Subscription/SubscriptionService.php index 5b6e010d73c1..4dcf472ba0f5 100644 --- a/app/Services/Subscription/SubscriptionService.php +++ b/app/Services/Subscription/SubscriptionService.php @@ -91,7 +91,7 @@ class SubscriptionService 'invoice' => $this->encodePrimaryKey($payment_hash->fee_invoice_id), 'client' => $recurring_invoice->client->hashed_id, 'subscription' => $this->subscription->hashed_id, - 'contact' => auth('contact')->user()->hashed_id, + 'contact' => auth('contact')->user() ? auth('contact')->user()->hashed_id : $recurring_invoice->client->contacts()->first()->hashed_id, 'account_key' => $recurring_invoice->client->custom_value2, ]; diff --git a/routes/api.php b/routes/api.php index ff853ab9f400..c78c57f3bf2f 100644 --- a/routes/api.php +++ b/routes/api.php @@ -212,7 +212,7 @@ Route::group(['middleware' => ['throttle:300,1', 'api_db', 'token_auth', 'locale Route::resource('subscriptions', 'SubscriptionController'); Route::post('subscriptions/bulk', 'SubscriptionController@bulk')->name('subscriptions.bulk'); Route::get('statics', 'StaticController'); - Route::post('apple_pay/upload_file','ApplyPayController@upload'); + // Route::post('apple_pay/upload_file','ApplyPayController@upload'); });