Working on handling check data within migrations"

app/Console/Commands/CheckData.php

@@ -73,6 +73,7 @@ class CheckData extends Command
     protected $description = 'Check/fix data';
 
     protected $log = '';
+
     protected $isValid = true;
 
     public function handle()
@@ -90,13 +91,10 @@ class CheckData extends Command
         $this->checkContacts();
         $this->checkCompanyData();
 
-        //$this->checkLogoFiles();
 
         if (! $this->option('client_id')) {
             $this->checkOAuth();
-            //$this->checkInvitations();
+            //$this->checkFailedJobs();
-
-            $this->checkFailedJobs();
         }
 
         $this->logMessage('Done: '.strtoupper($this->isValid ? Account::RESULT_SUCCESS : Account::RESULT_FAILURE));
@@ -298,8 +296,6 @@ class CheckData extends Command
             $total_invoice_payments = 0;
 
             foreach ($client->invoices->where('is_deleted', false)->where('status_id', '>', 1) as $invoice) {
-                // $total_amount = $invoice->payments->whereNull('deleted_at')->sum('pivot.amount');
-                // $total_refund = $invoice->payments->whereNull('deleted_at')->sum('pivot.refunded');
 
                 $total_amount = $invoice->payments->where('is_deleted', false)->whereIn('status_id', [Payment::STATUS_COMPLETED, Payment:: STATUS_PENDING, Payment::STATUS_PARTIALLY_REFUNDED, Payment::STATUS_REFUNDED])->sum('pivot.amount');
                 $total_refund = $invoice->payments->where('is_deleted', false)->whereIn('status_id', [Payment::STATUS_COMPLETED, Payment:: STATUS_PENDING, Payment::STATUS_PARTIALLY_REFUNDED, Payment::STATUS_REFUNDED])->sum('pivot.refunded');
@@ -307,15 +303,15 @@ class CheckData extends Command
                 $total_invoice_payments += ($total_amount - $total_refund);
             }
 
-            foreach ($client->payments as $payment) {
+            //10/02/21
-                $credit_total_applied += $payment->paymentables->where('paymentable_type', App\Models\Credit::class)->sum(DB::raw('amount'));
+            // foreach ($client->payments as $payment) {
-            }
+            //     $credit_total_applied += $payment->paymentables->where('paymentable_type', App\Models\Credit::class)->sum(DB::raw('amount'));
+            // }
 
-            if ($credit_total_applied < 0) {
+            // if ($credit_total_applied < 0) {
-                $total_invoice_payments += $credit_total_applied;
+            //     $total_invoice_payments += $credit_total_applied;
-            } //todo this is contentious
+            // } 
 
-            // nlog("total invoice payments = {$total_invoice_payments} with client paid to date of of {$client->paid_to_date}");
 
             if (round($total_invoice_payments, 2) != round($client->paid_to_date, 2)) {
                 $wrong_paid_to_dates++;

app/Jobs/Mail/BaseMailerJob.php

@@ -42,7 +42,10 @@ class BaseMailerJob implements ShouldQueue
 
     public function setMailDriver()
     {
+        /* Singletons need to be rebooted each time just in case our Locale is changing*/
         App::forgetInstance('translator');
+
+        /* Inject custom translations if any exist */
         Lang::replace(Ninja::transformTranslations($this->settings));
 
         switch ($this->settings->email_sending_method) {
@@ -96,7 +99,7 @@ class BaseMailerJob implements ShouldQueue
 
     public function failed($exception = null)
     {
-        nlog('the job failed');
+        nlog('mailer job failed');
         nlog($exception->getMessage());
         
         $job_failure = new EmailFailure();

app/Jobs/Ninja/CheckCompanyData.php

@@ -72,8 +72,16 @@ class CheckCompanyData implements ShouldQueue
 
         Cache::put($this->hash, $cache_instance, now()->addMinutes(30));
 
-
         nlog(Cache::get($this->hash));
+
+        nlog($this->company_data);
+
+        if(!$this->isValid)
+            $this->company_data['status'] = 'errors';
+        else
+            $this->company_data['status'] = 'success';
+        
+        return $this->company_data;
         
     }
 
@@ -90,17 +98,18 @@ class CheckCompanyData implements ShouldQueue
 
         foreach ($this->company->clients->where('is_deleted', 0)->cursor() as $client) {
             $invoice_balance = $client->invoices->where('is_deleted', false)->where('status_id', '>', 1)->sum('balance');
-            $credit_balance = $client->credits->where('is_deleted', false)->sum('balance');
+            //$credit_balance = $client->credits->where('is_deleted', false)->sum('balance');
 
-            if($client->balance != $invoice_balance)
+            // if($client->balance != $invoice_balance)
-                $invoice_balance -= $credit_balance;//doesn't make sense to remove the credit amount
+            //     $invoice_balance -= $credit_balance;//doesn't make sense to remove the credit amount
 
             $ledger = CompanyLedger::where('client_id', $client->id)->orderBy('id', 'DESC')->first();
 
             if ($ledger && number_format($invoice_balance, 4) != number_format($client->balance, 4)) {
+
                 $wrong_balances++;
 
-                $this->company_data[] = "# {$client->id} " . $client->present()->name.' - '.$client->number." - Balance Failure - Invoice Balances = {$invoice_balance} Client Balance = {$client->balance} Ledger Balance = {$ledger->balance}";
+                $this->company_data[] = "# {$client->id} " . $client->present()->name.' - '.$client->number." - Balance Failure - Invoice Balances = {$invoice_balance} Client Balance = {$client->balance} Ledger Balance = {$ledger->balance} \n";
 
                 $this->isValid = false;
             }
@@ -126,14 +135,14 @@ class CheckCompanyData implements ShouldQueue
                 if ((string)$total_paid != (string)($invoice->amount - $invoice->balance - $total_credit)) {
                     $wrong_balances++;
 
-                    $this->company_data[] = $client->present()->name.' - '.$client->id." - Total Amount = {$total_amount} != Calculated Total = {$calculated_paid_amount} - Total Refund = {$total_refund} Total credit = {$total_credit}";
+                    $this->company_data[] = $client->present()->name.' - '.$client->id." - Total Amount = {$total_amount} != Calculated Total = {$calculated_paid_amount} - Total Refund = {$total_refund} Total credit = {$total_credit}\n";
 
                     $this->isValid = false;
                 }
             });
         });
 
-        $this->company_data[] = "{$wrong_balances} clients with incorrect invoice balances";
+        $this->company_data[] = "{$wrong_balances} clients with incorrect invoice balances\n";
     }
 
     private function checkPaidToDates()
@@ -145,8 +154,7 @@ class CheckCompanyData implements ShouldQueue
             $total_invoice_payments = 0;
 
             foreach ($client->invoices->where('is_deleted', false)->where('status_id', '>', 1) as $invoice) {
-                // $total_amount = $invoice->payments->whereNull('deleted_at')->sum('pivot.amount');
+
-                // $total_refund = $invoice->payments->whereNull('deleted_at')->sum('pivot.refunded');
 
                 $total_amount = $invoice->payments->where('is_deleted', false)->whereIn('status_id', [Payment::STATUS_COMPLETED, Payment:: STATUS_PENDING, Payment::STATUS_PARTIALLY_REFUNDED, Payment::STATUS_REFUNDED])->sum('pivot.amount');
                 $total_refund = $invoice->payments->where('is_deleted', false)->whereIn('status_id', [Payment::STATUS_COMPLETED, Payment:: STATUS_PENDING, Payment::STATUS_PARTIALLY_REFUNDED, Payment::STATUS_REFUNDED])->sum('pivot.refunded');
@@ -154,20 +162,21 @@ class CheckCompanyData implements ShouldQueue
                 $total_invoice_payments += ($total_amount - $total_refund);
             }
 
-            foreach ($client->payments as $payment) {
+            //10/02/21
-                $credit_total_applied += $payment->paymentables->where('paymentable_type', App\Models\Credit::class)->sum(DB::raw('amount'));
+            // foreach ($client->payments as $payment) {
-            }
+            //     $credit_total_applied += $payment->paymentables->where('paymentable_type', App\Models\Credit::class)->sum(DB::raw('amount'));
+            // }
 
-            if ($credit_total_applied < 0) {
+            // if ($credit_total_applied < 0) {
-                $total_invoice_payments += $credit_total_applied;
+            //     $total_invoice_payments += $credit_total_applied;
-            } //todo this is contentious
+            // } //todo this is contentious
 
             // nlog("total invoice payments = {$total_invoice_payments} with client paid to date of of {$client->paid_to_date}");
 
             if (round($total_invoice_payments, 2) != round($client->paid_to_date, 2)) {
                 $wrong_paid_to_dates++;
 
-                $this->company_data[] = $client->present()->name.'id = # '.$client->id." - Paid to date does not match Client Paid To Date = {$client->paid_to_date} - Invoice Payments = {$total_invoice_payments}";
+                $this->company_data[] = $client->present()->name.'id = # '.$client->id." - Paid to date does not match Client Paid To Date = {$client->paid_to_date} - Invoice Payments = {$total_invoice_payments}\n";
 
                 $this->isValid = false;
             }
@@ -187,22 +196,23 @@ class CheckCompanyData implements ShouldQueue
             $invoice_balance = Invoice::where('client_id', $client->id)->where('is_deleted', false)->where('status_id', '>', 1)->withTrashed()->sum('balance');
             $credit_balance = Credit::where('client_id', $client->id)->where('is_deleted', false)->withTrashed()->sum('balance');
 
-            /*Legacy - V4 will add credits to the balance - we may need to reverse engineer this and remove the credits from the client balance otherwise we need this hack here and in the invoice balance check.*/
+            //10/02/21
-            if($client->balance != $invoice_balance)
+            // Legacy - V4 will add credits to the balance - we may need to reverse engineer this and remove the credits from the client balance otherwise we need this hack here and in the invoice balance check.
-                $invoice_balance -= $credit_balance;
+            // if($client->balance != $invoice_balance)
+            //     $invoice_balance -= $credit_balance;
 
             $ledger = CompanyLedger::where('client_id', $client->id)->orderBy('id', 'DESC')->first();
 
             if ($ledger && (string) $invoice_balance != (string) $client->balance) {
                 $wrong_paid_to_dates++;
                 
-                $this->company_data[] = $client->present()->name.' - '.$client->id." - calculated client balances do not match {$invoice_balance} - ".rtrim($client->balance, '0');
+                $this->company_data[] = $client->present()->name.' - '.$client->id." - calculated client balances do not match {$invoice_balance} - ".rtrim($client->balance, '0')."\n";
 
                 $this->isValid = false;
             }
         }
 
-        $this->company_data[] = "{$wrong_paid_to_dates} clients with incorrect client balances";
+        $this->company_data[] = "{$wrong_paid_to_dates} clients with incorrect client balances\n";
     }
 
     private function checkContacts()
@@ -214,7 +224,7 @@ class CheckCompanyData implements ShouldQueue
                         ->orderBy('id')
                         ->get(['id']);
 
-        $this->company_data[] =  $contacts->count().' contacts without a contact_key';
+        $this->company_data[] =  $contacts->count().' contacts without a contact_key\n';
 
         if ($contacts->count() > 0) {
             $this->isValid = false;
@@ -248,7 +258,7 @@ class CheckCompanyData implements ShouldQueue
 
         $clients = $clients->get(['clients.id', 'clients.user_id', 'clients.company_id']);
         
-        $this->company_data[] = $clients->count().' clients without any contacts';
+        $this->company_data[] = $clients->count().' clients without any contacts\n';
 
         if ($clients->count() > 0) {
             $this->isValid = false;
@@ -283,7 +293,7 @@ class CheckCompanyData implements ShouldQueue
         }
 
         $clients = $clients->get(['clients.id', DB::raw('count(client_contacts.id)')]);
-        $this->company_data[] = $clients->count().' clients without a single primary contact';
+        $this->company_data[] = $clients->count().' clients without a single primary contact\n';
 
         if ($clients->count() > 0) {
             $this->isValid = false;
@@ -326,7 +336,7 @@ class CheckCompanyData implements ShouldQueue
 
                 if ($records->count()) {
                     $this->isValid = false;
-                    $this->company_data[] = $records->count()." {$table} records with incorrect {$entityType} company id";
+                    $this->company_data[] = $records->count()." {$table} records with incorrect {$entityType} company id\n";
                 }
             }
         }

app/Jobs/Util/Import.php

@@ -14,6 +14,7 @@ namespace App\Jobs\Util;
 use App\DataMapper\Analytics\MigrationFailure;
 use App\DataMapper\CompanySettings;
 use App\Exceptions\MigrationValidatorFailed;
+use App\Exceptions\ProcessingMigrationArchiveFailed;
 use App\Exceptions\ResourceDependencyMissing;
 use App\Exceptions\ResourceNotAvailableForMigration;
 use App\Factory\ClientFactory;
@@ -31,6 +32,7 @@ use App\Http\Requests\Company\UpdateCompanyRequest;
 use App\Http\ValidationRules\ValidCompanyGatewayFeesAndLimitsRule;
 use App\Http\ValidationRules\ValidUserForCompany;
 use App\Jobs\Company\CreateCompanyToken;
+use App\Jobs\Ninja\CheckCompanyData;
 use App\Jobs\Ninja\CompanySizeCheck;
 use App\Libraries\MultiDB;
 use App\Mail\MigrationCompleted;
@@ -206,9 +208,14 @@ class Import implements ShouldQueue
         $this->setInitialCompanyLedgerBalances();
         
         // $this->fixClientBalances();
+        $check_data = CheckCompanyData::dispatchNow($this->company);
+
+
+        if($check_data['status'] == 'errors')
+            throw new ProcessingMigrationArchiveFailed($check_data);
 
         Mail::to($this->user)
-            ->send(new MigrationCompleted($this->company));
+            ->send(new MigrationCompleted($this->company, $check_data));
 
         /*After a migration first some basic jobs to ensure the system is up to date*/
         VersionCheck::dispatch();

app/Jobs/Util/StartMigration.php

@@ -108,15 +108,16 @@ class StartMigration implements ShouldQueue
                 throw new NonExistingMigrationFile('Migration file does not exist, or it is corrupted.');
             }
 
-            //$data = json_decode(file_get_contents($file), 1);
-            //Import::dispatchNow($data['data'], $this->company, $this->user);
             Import::dispatchNow($file, $this->company, $this->user);
+
         } catch (NonExistingMigrationFile | ProcessingMigrationArchiveFailed | ResourceNotAvailableForMigration | MigrationValidatorFailed | ResourceDependencyMissing $e) {
+
             Mail::to($this->user)->send(new MigrationFailed($e, $e->getMessage()));
 
             if (app()->environment() !== 'production') {
                 info($e->getMessage());
             }
+            
         }
 
         //always make sure we unset the migration as running

app/Mail/MigrationCompleted.php

@@ -13,14 +13,17 @@ class MigrationCompleted extends Mailable
 
     public $company;
 
+    public $check_data;
+
     /**
      * Create a new message instance.
      *
      * @return void
      */
-    public function __construct(Company $company)
+    public function __construct(Company $company, $check_data)
     {
         $this->company = $company;
+        $this->check_data = $check_data;
     }
 
     /**
@@ -33,7 +36,8 @@ class MigrationCompleted extends Mailable
         $data['settings'] = $this->company->settings;
         $data['company'] = $this->company;
         $data['whitelabel'] = $this->company->account->isPaid() ? true : false;
-        
+        $data['check_data'] = $this->check_data;
+
         $result = $this->from(config('mail.from.address'), config('mail.from.name'))
                     ->view('email.import.completed', $data);
 

app/Mail/MigrationFailed.php

@@ -22,6 +22,7 @@ class MigrationFailed extends Mailable
     public function __construct($exception, $content = null)
     {
         $this->exception = $exception;
+        $this->content = $content;
     }
 
     /**

resources/views/email/import/completed.blade.php

@@ -73,6 +73,10 @@
         <p><b>Documents Imported:</b> {{ count($company->documents) }} </p>
     @endif
 
+    <p><b>Data Quality:</b></p>
+    
+    <p> {{ $check_data }} </p>
+
     <a href="{{ url('/') }}" target="_blank" class="button">{{ ctrans('texts.account_login')}}</a>
 
     <p>{{ ctrans('texts.email_signature')}}<br/> {{ ctrans('texts.email_from') }}</p>