From 4979109d97f937eadc24f0a52fa8f67947ac16e8 Mon Sep 17 00:00:00 2001 From: David Bomba Date: Thu, 12 Jan 2023 11:08:32 +1100 Subject: [PATCH] handle missing required keys for Invoice2Go Imports --- app/Import/Providers/BaseImport.php | 23 +++- app/Import/Providers/Csv.php | 2 +- app/Import/Providers/Wave.php | 14 ++- .../Transformer/Bank/BankTransformer.php | 81 ++++++++++++++ .../Invoice2Go/InvoiceTransformer.php | 5 + .../Transformer/Wave/ClientTransformer.php | 3 +- app/Jobs/Import/CSVIngest.php | 1 - app/Mail/Import/CsvImportCompleted.php | 93 ++++++++++++++++ lang/en/texts.php | 2 +- .../email/import/csv_completed.blade.php | 102 ++++++++++++++++++ .../email/import/csv_completed_text.blade.php | 98 +++++++++++++++++ 11 files changed, 416 insertions(+), 8 deletions(-) create mode 100644 app/Import/Transformer/Bank/BankTransformer.php create mode 100644 app/Mail/Import/CsvImportCompleted.php create mode 100644 resources/views/email/import/csv_completed.blade.php create mode 100644 resources/views/email/import/csv_completed_text.blade.php diff --git a/app/Import/Providers/BaseImport.php b/app/Import/Providers/BaseImport.php index 348cb7baf835..5fed1678b2f2 100644 --- a/app/Import/Providers/BaseImport.php +++ b/app/Import/Providers/BaseImport.php @@ -20,7 +20,7 @@ use App\Http\Requests\Quote\StoreQuoteRequest; use App\Import\ImportException; use App\Jobs\Mail\NinjaMailerJob; use App\Jobs\Mail\NinjaMailerObject; -use App\Mail\Import\ImportCompleted; +use App\Mail\Import\CsvImportCompleted; use App\Models\Company; use App\Models\Invoice; use App\Models\Quote; @@ -187,6 +187,10 @@ class BaseImport try { $entity = $this->transformer->transform($record); + + if(!$entity) + continue; + $validator = $this->runValidation($entity); if ($validator->fails()) { @@ -282,6 +286,8 @@ class BaseImport public function ingestInvoices($invoices, $invoice_number_key) { + $count = 0; + $invoice_transformer = $this->transformer; /** @var PaymentRepository $payment_repository */ @@ -343,6 +349,7 @@ class BaseImport } $invoice_repository->save($invoice_data, $invoice); + $count++; // If we're doing a generic CSV import, only import payment data if we're not importing a payment CSV. // If we're doing a platform-specific import, trust the platform to only return payment info if there's not a separate payment CSV. if ( @@ -404,6 +411,9 @@ class BaseImport ]; } } + + return $count; + } private function actionInvoiceStatus( @@ -475,6 +485,8 @@ class BaseImport public function ingestQuotes($quotes, $quote_number_key) { + $count = 0; + $quote_transformer = $this->transformer; /** @var ClientRepository $client_repository */ @@ -532,6 +544,8 @@ class BaseImport $quote->status_id = $quote_data['status_id']; } $quote_repository->save($quote_data, $quote); + + $count++; $this->actionQuoteStatus( $quote, @@ -552,7 +566,11 @@ class BaseImport 'error' => $message, ]; } + } + + return $count; + } protected function getUserIDForRecord($record) @@ -586,10 +604,11 @@ class BaseImport $data = [ 'errors' => $this->error_array, 'company' => $this->company, + 'entity_count' => $this->entity_count ]; $nmo = new NinjaMailerObject; - $nmo->mailable = new ImportCompleted($this->company, $data); + $nmo->mailable = new CsvImportCompleted($this->company, $data); $nmo->company = $this->company; $nmo->settings = $this->company->settings; $nmo->to_user = $this->company->owner(); diff --git a/app/Import/Providers/Csv.php b/app/Import/Providers/Csv.php index aee88900489b..338e45e539be 100644 --- a/app/Import/Providers/Csv.php +++ b/app/Import/Providers/Csv.php @@ -37,7 +37,7 @@ use App\Import\Transformer\Csv\PaymentTransformer; use App\Import\Transformer\Csv\ProductTransformer; use App\Import\Transformer\Csv\QuoteTransformer; use App\Import\Transformer\Csv\VendorTransformer; -use App\Import\Transformers\Bank\BankTransformer; +use App\Import\Transformer\Bank\BankTransformer; use App\Repositories\BankTransactionRepository; use App\Repositories\ClientRepository; use App\Repositories\ExpenseRepository; diff --git a/app/Import/Providers/Wave.php b/app/Import/Providers/Wave.php index 7b31b52e897a..ad0eac86cbab 100644 --- a/app/Import/Providers/Wave.php +++ b/app/Import/Providers/Wave.php @@ -66,7 +66,6 @@ class Wave extends BaseImport implements ImportInterface if (empty($data)) { $this->entity_count['clients'] = 0; - return; } @@ -170,11 +169,16 @@ class Wave extends BaseImport implements ImportInterface $entity_type = 'expense'; $data = $this->getCsvData($entity_type); + + if(!$data){ + $this->entity_count['expense'] = 0; + return; + } + $data = $this->preTransform($data, $entity_type); if (empty($data)) { $this->entity_count['expense'] = 0; - return; } @@ -212,6 +216,8 @@ class Wave extends BaseImport implements ImportInterface public function ingestExpenses($data) { + $count = 0; + $key = 'Transaction ID'; $expense_transformer = $this->transformer; @@ -255,6 +261,7 @@ class Wave extends BaseImport implements ImportInterface ); $expense_repository->save($expense_data, $expense); + $count++; } } catch (\Exception $ex) { if ($ex instanceof ImportException) { @@ -270,5 +277,8 @@ class Wave extends BaseImport implements ImportInterface ]; } } + + return $count; + } } diff --git a/app/Import/Transformer/Bank/BankTransformer.php b/app/Import/Transformer/Bank/BankTransformer.php new file mode 100644 index 000000000000..648967063dd4 --- /dev/null +++ b/app/Import/Transformer/Bank/BankTransformer.php @@ -0,0 +1,81 @@ + $transaction['transaction.bank_integration_id'], + 'transaction_id' => $this->getNumber($transaction,'transaction.transaction_id'), + 'amount' => abs($this->getFloat($transaction, 'transaction.amount')), + 'currency_id' => $this->getCurrencyByCode($transaction, 'transaction.currency'), + 'account_type' => strlen($this->getString($transaction, 'transaction.account_type')) > 1 ? $this->getString($transaction, 'transaction.account_type') : 'bank', + 'category_id' => $this->getNumber($transaction, 'transaction.category_id') > 0 ? $this->getNumber($transaction, 'transaction.category_id') : null, + 'category_type' => $this->getString($transaction, 'transaction.category_type'), + 'date' => array_key_exists('transaction.date', $transaction) ? $this->parseDate($transaction['transaction.date']) + : now()->format('Y-m-d'), + 'bank_account_id' => array_key_exists('transaction.bank_account_id', $transaction) ? $transaction['transaction.bank_account_id'] : 0, + 'description' => array_key_exists('transaction.description', $transaction) ? $transaction['transaction.description'] : '', + 'base_type' => $this->calculateType($transaction), + 'created_at' => $now, + 'updated_at' => $now, + 'company_id' => $this->company->id, + 'user_id' => $this->company->owner()->id, + ]; + + return $transformed; + } + + + private function calculateType($transaction) + { + + if(array_key_exists('transaction.base_type', $transaction) && (($transaction['transaction.base_type'] == 'CREDIT') || strtolower($transaction['transaction.base_type']) == 'deposit')) + return 'CREDIT'; + + if(array_key_exists('transaction.base_type', $transaction) && (($transaction['transaction.base_type'] == 'DEBIT') || strtolower($transaction['transaction.base_type']) == 'withdrawal')) + return 'DEBIT'; + + if(array_key_exists('transaction.category_id', $transaction)) + return 'DEBIT'; + + if(array_key_exists('transaction.category_type', $transaction) && $transaction['transaction.category_type'] == 'Income') + return 'CREDIT'; + + if(array_key_exists('transaction.category_type', $transaction)) + return 'DEBIT'; + + if(array_key_exists('transaction.amount', $transaction) && is_numeric($transaction['transaction.amount']) && $transaction['transaction.amount'] > 0) + return 'CREDIT'; + + return 'DEBIT'; + + } + +} \ No newline at end of file diff --git a/app/Import/Transformer/Invoice2Go/InvoiceTransformer.php b/app/Import/Transformer/Invoice2Go/InvoiceTransformer.php index 6d52257d6a6c..f5b18674cd95 100644 --- a/app/Import/Transformer/Invoice2Go/InvoiceTransformer.php +++ b/app/Import/Transformer/Invoice2Go/InvoiceTransformer.php @@ -28,6 +28,11 @@ class InvoiceTransformer extends BaseTransformer */ public function transform($invoice_data) { + + if (!isset($invoice_data['DocumentNumber'])) { + throw new ImportException('DocumentNumber key not found in this import file.'); + } + if ($this->hasInvoice($invoice_data['DocumentNumber'])) { throw new ImportException('Invoice number already exists'); } diff --git a/app/Import/Transformer/Wave/ClientTransformer.php b/app/Import/Transformer/Wave/ClientTransformer.php index 844560f48679..28d6439d12ac 100644 --- a/app/Import/Transformer/Wave/ClientTransformer.php +++ b/app/Import/Transformer/Wave/ClientTransformer.php @@ -28,7 +28,8 @@ class ClientTransformer extends BaseTransformer public function transform($data) { if (isset($data['customer_name']) && $this->hasClient($data['customer_name'])) { - throw new ImportException('Client already exists'); + return false; + // throw new ImportException('Client already exists'); } $settings = new \stdClass; diff --git a/app/Jobs/Import/CSVIngest.php b/app/Jobs/Import/CSVIngest.php index b4a264dd323e..75c11d632bdf 100644 --- a/app/Jobs/Import/CSVIngest.php +++ b/app/Jobs/Import/CSVIngest.php @@ -23,7 +23,6 @@ use App\Libraries\MultiDB; use App\Models\Client; use App\Models\Company; use App\Models\Vendor; -use App\Utils\Ninja; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; diff --git a/app/Mail/Import/CsvImportCompleted.php b/app/Mail/Import/CsvImportCompleted.php new file mode 100644 index 000000000000..ca0911d489a0 --- /dev/null +++ b/app/Mail/Import/CsvImportCompleted.php @@ -0,0 +1,93 @@ + (array) $errors, + * 'company' => Company $company, + * 'entity_count' => (array) $entity_count + * ]; + */ + public $data; + + /** + * Create a new message instance. + * + * @return void + */ + public function __construct(Company $company, $data) + { + $this->company = $company; + + $this->data = $data; + } + + /** + * Build the message. + * + * @return $this + */ + public function build() + { + App::forgetInstance('translator'); + App::setLocale($this->company->getLocale()); + + $t = app('translator'); + $t->replace(Ninja::transformTranslations($this->company->settings)); + + $data = array_merge($this->data, [ + 'logo' => $this->company->present()->logo(), + 'settings' => $this->company->settings, + 'company' => $this->company, + 'client_count' => isset($this->data['entity_count']['clients']) ? $this->data['entity_count']['clients'] : false, + 'product_count' => isset($this->data['entity_count']['products']) ? $this->data['entity_count']['products'] : false, + 'invoice_count' => isset($this->data['entity_count']['invoices']) ? $this->data['entity_count']['invoices'] : false, + 'quote_count' => isset($this->data['entity_count']['quotes']) ? $this->data['entity_count']['quotes'] : false, + 'credit_count' => isset($this->data['entity_count']['credits']) ? $this->data['entity_count']['credits'] : false, + 'project_count' => isset($this->data['entity_count']['projects']) ? $this->data['entity_count']['projects'] : false, + 'task_count' => isset($this->data['entity_count']['tasks']) ? $this->data['entity_count']['tasks'] : false, + 'vendor_count' => isset($this->data['entity_count']['vendors']) ? $this->data['entity_count']['vendors'] : false, + 'payment_count' => isset($this->data['entity_count']['payments']) ? $this->data['entity_count']['payments'] : false, + 'recurring_invoice_count' => isset($this->data['entity_count']['recurring_invoices']) ? $this->data['entity_count']['recurring_invoices'] : false, + 'expense_count' => isset($this->data['entity_count']['expenses']) ? $this->data['entity_count']['expenses'] : false, + 'company_gateway_count' => isset($this->data['entity_count']['company_gateways']) ? $this->data['entity_count']['company_gateways'] : false, + 'client_gateway_token_count' => isset($this->data['entity_count']['client_gateway_tokens']) ? $this->data['entity_count']['client_gateway_tokens'] : false, + 'tax_rate_count' => isset($this->data['entity_count']['tax_rates']) ? $this->data['entity_count']['tax_rates'] : false, + 'document_count' => isset($this->data['entity_count']['documents']) ? $this->data['entity_count']['documents'] : false, + 'transaction_count' => isset($this->data['entity_count']['transactions']) ? $this->data['entity_count']['transactions'] : false, + ]); + + return $this + ->subject(ctrans('texts.import_completed')) + ->from(config('mail.from.address'), config('mail.from.name')) + ->text('email.import.csv_completed_text') + ->view('email.import.csv_completed', $data); + } +} diff --git a/lang/en/texts.php b/lang/en/texts.php index 3328738d4297..388d200053c7 100644 --- a/lang/en/texts.php +++ b/lang/en/texts.php @@ -4923,7 +4923,7 @@ $LANG = array( 'matomo_id' => 'Matomo Id', 'action_add_to_invoice' => 'Add To Invoice', 'danger_zone' => 'Danger Zone', - + 'import_completed' => 'Import completed', ); diff --git a/resources/views/email/import/csv_completed.blade.php b/resources/views/email/import/csv_completed.blade.php new file mode 100644 index 000000000000..44f196105842 --- /dev/null +++ b/resources/views/email/import/csv_completed.blade.php @@ -0,0 +1,102 @@ +@component('email.template.admin', ['logo' => $logo, 'settings' => $settings, 'company' => $company ?? '']) +
+

{{ ctrans('texts.import_complete') }}

+ +

+ + @if($client_count) +

{{ ctrans('texts.clients') }}: {{ $client_count }}

+ @endif + + @if($product_count) +

{{ ctrans('texts.products') }}: {{ $product_count }}

+ @endif + + @if($invoice_count) +

{{ ctrans('texts.invoices') }}: {{ $invoice_count }}

+ @endif + + @if($payment_count) +

{{ ctrans('texts.payments') }}: {{ $payment_count }}

+ @endif + + @if($recurring_invoice_count) +

{{ ctrans('texts.recurring_invoices') }}: {{ $recurring_invoice_count }}

+ @endif + + @if($quote_count) +

{{ ctrans('texts.quotes') }}: {{ $quote_count }}

+ @endif + + @if($credit_count) +

{{ ctrans('texts.credits') }}: {{ $credit_count }}

+ @endif + + @if($project_count) +

{{ ctrans('texts.projects') }}: {{ $project_count }}

+ @endif + + @if($task_count) +

{{ ctrans('texts.tasks') }}: {{ $task_count }}

+ @endif + + @if($vendor_count) +

{{ ctrans('texts.vendors') }}: {{ $vendor_count }}

+ @endif + + @if($expense_count) +

{{ ctrans('texts.expenses') }}: {{ $expense_count }}

+ @endif + + @if($company_gateway_count) +

{{ ctrans('texts.gateways') }}: {{ $company_gateway_count }}

+ @endif + + @if($client_gateway_token_count) +

{{ ctrans('texts.tokens') }}: {{ $client_gateway_token_count }}

+ @endif + + @if($tax_rate_count) +

{{ ctrans('texts.tax_rates') }}: {{ $tax_rate_count }}

+ @endif + + @if($document_count) +

{{ ctrans('texts.documents') }}: {{ $document_count }}

+ @endif + + @if($transaction_count) +

{{ ctrans('texts.documents') }}: {{ $transaction_count }}

+ @endif + + @if(!empty($errors) ) +

{{ ctrans('texts.failed_to_import') }}

+

{{ ctrans('texts.error') }}:

+ + + + + + + + + + @foreach($errors as $entityType=>$entityErrors) + @foreach($entityErrors as $error) + + + + + + @endforeach + @endforeach + +
TypeDataError
{{$entityType}}{{json_encode($error[$entityType]??null)}}{{json_encode($error['error'])}}
+ @endif + + {{ ctrans('texts.account_login')}} + +

{{ ctrans('texts.email_signature')}}

+

{{ ctrans('texts.email_from') }}

+
+@endcomponent + diff --git a/resources/views/email/import/csv_completed_text.blade.php b/resources/views/email/import/csv_completed_text.blade.php new file mode 100644 index 000000000000..e9a367e54d43 --- /dev/null +++ b/resources/views/email/import/csv_completed_text.blade.php @@ -0,0 +1,98 @@ +{{ ctrans('texts.import_complete') }} + + @if($client_count) + {{ ctrans('texts.clients') }}: {{ $client_count }} + @endif + + @if($product_count) + {{ ctrans('texts.products') }}: {{ $product_count }} + @endif + + @if($invoice_count) + {{ ctrans('texts.invoices') }}: {{ $invoice_count }} + @endif + + @if($payment_count) + {{ ctrans('texts.payments') }}: {{ $payment_count }} + @endif + + @if($recurring_invoice_count) + {{ ctrans('texts.recurring_invoices') }}: {{ $recurring_invoice_count }} + @endif + + @if($quote_count) + {{ ctrans('texts.quotes') }}: {{ $quote_count }} + @endif + + @if($credit_count) + {{ ctrans('texts.credits') }}: {{ $credit_count }} + @endif + + @if($project_count) + {{ ctrans('texts.projects') }}: {{ $project_count }} + @endif + + @if($task_count) + {{ ctrans('texts.tasks') }}: {{ $task_count }} + @endif + + @if($vendor_count) + {{ ctrans('texts.vendors') }}: {{ $vendor_count }} + @endif + + @if($expense_count) + {{ ctrans('texts.expenses') }}: {{ $expense_count }} + @endif + + @if($company_gateway_count) + {{ ctrans('texts.gateways') }}: {{ $company_gateway_count }} + @endif + + @if($client_gateway_token_count) + {{ ctrans('texts.tokens') }}: {{ $client_gateway_token_count }} + @endif + + @if($tax_rate_count) + {{ ctrans('texts.tax_rates') }}: {{ $tax_rate_count }} + @endif + + @if($document_count) + {{ ctrans('texts.documents') }}: {{ $document_count }} + @endif + + @if($transaction_count) + {{ ctrans('texts.documents') }}: {{ $transaction_count }} + @endif + + @if(!empty($errors)) +

{{ ctrans('texts.failed_to_import') }}

+

{{ ctrans('texts.error') }}:

+ + + + + + + + + + @foreach($errors as $entityType=>$entityErrors) + @foreach($entityErrors as $error) + + + + + + @endforeach + @endforeach + +
TypeDataError
{{$entityType}}{{json_encode($error[$entityType]??null)}}{{json_encode($error['error'])}}
+ @endif + +{!! url('/') !!} + +{!! ctrans('texts.email_signature') !!} + +{!! ctrans('texts.email_from') !!} + +