From aeae0a19ab8c9ce4723d37e4305a5748ad7f888b Mon Sep 17 00:00:00 2001
From: David Bomba <turbo124@gmail.com>
Date: Sun, 15 May 2022 20:14:14 +1000
Subject: [PATCH 1/2] Adjust delays for observers

---
 app/Console/Kernel.php                            | 2 +-
 app/Http/Requests/Payment/StorePaymentRequest.php | 2 +-
 app/Mail/Engine/InvoiceEmailEngine.php            | 2 ++
 app/Models/CompanyLedger.php                      | 5 +++++
 app/Observers/ClientObserver.php                  | 6 +++---
 app/Observers/ExpenseObserver.php                 | 6 +++---
 app/Observers/InvoiceObserver.php                 | 6 +++---
 app/Observers/PaymentObserver.php                 | 4 ++--
 app/Observers/ProjectObserver.php                 | 4 ++--
 app/Observers/QuoteObserver.php                   | 6 +++---
 app/Observers/TaskObserver.php                    | 6 +++---
 app/PaymentDrivers/StripePaymentDriver.php        | 2 +-
 app/Services/Ledger/LedgerService.php             | 6 +++---
 13 files changed, 32 insertions(+), 25 deletions(-)

diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index b6e59ec00378..14445950b38f 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -55,7 +55,7 @@ class Kernel extends ConsoleKernel
 
         $schedule->job(new ReminderJob)->hourly()->withoutOverlapping();
 
-        $schedule->job(new LedgerBalanceUpdate)->everyFiveMinutes()->withoutOverlapping();
+        // $schedule->job(new LedgerBalanceUpdate)->everyFiveMinutes()->withoutOverlapping();
 
         $schedule->job(new CompanySizeCheck)->daily()->withoutOverlapping();
 
diff --git a/app/Http/Requests/Payment/StorePaymentRequest.php b/app/Http/Requests/Payment/StorePaymentRequest.php
index b26a6d5908d6..e0e52c7b25b3 100644
--- a/app/Http/Requests/Payment/StorePaymentRequest.php
+++ b/app/Http/Requests/Payment/StorePaymentRequest.php
@@ -104,7 +104,7 @@ class StorePaymentRequest extends Request
             'credits.*.credit_id' => new ValidCreditsRules($this->all()),
             'credits.*.amount' => ['required', new CreditsSumRule($this->all())],
             'invoices' => new ValidPayableInvoicesRule(),
-            'number' => ['nullable', Rule::unique('payments')->where('company_id', auth()->user()->company()->id)],
+            'number' => ['nullable', 'bail', Rule::unique('payments')->where('company_id', auth()->user()->company()->id)],
 
         ];
 
diff --git a/app/Mail/Engine/InvoiceEmailEngine.php b/app/Mail/Engine/InvoiceEmailEngine.php
index 99bf3041515f..ad0c232489cf 100644
--- a/app/Mail/Engine/InvoiceEmailEngine.php
+++ b/app/Mail/Engine/InvoiceEmailEngine.php
@@ -177,6 +177,8 @@ class InvoiceEmailEngine extends BaseEmailEngine
                                        });
                 }
 
+                $task_ids = [];
+                
                 if(property_exists($item, 'task_id'))
                 {
                     $task_ids[] = $item->task_id;
diff --git a/app/Models/CompanyLedger.php b/app/Models/CompanyLedger.php
index 4ce4690dcdde..f3f497341395 100644
--- a/app/Models/CompanyLedger.php
+++ b/app/Models/CompanyLedger.php
@@ -48,4 +48,9 @@ class CompanyLedger extends Model
     {
         return $this->morphTo();
     }
+
+    public function client()
+    {
+        return $this->belongsTo(Client::class);
+    }
 }
diff --git a/app/Observers/ClientObserver.php b/app/Observers/ClientObserver.php
index 5567a9e57faf..273b2e6c002c 100644
--- a/app/Observers/ClientObserver.php
+++ b/app/Observers/ClientObserver.php
@@ -30,7 +30,7 @@ class ClientObserver
                                     ->exists();
 
         if ($subscriptions) {
-            WebhookHandler::dispatch(Webhook::EVENT_CREATE_CLIENT, $client, $client->company);
+            WebhookHandler::dispatch(Webhook::EVENT_CREATE_CLIENT, $client, $client->company)->delay(now()->addSeconds(2));
         }
     }
 
@@ -47,7 +47,7 @@ class ClientObserver
                                     ->exists();
 
         if ($subscriptions) {
-            WebhookHandler::dispatch(Webhook::EVENT_UPDATE_CLIENT, $client, $client->company);
+            WebhookHandler::dispatch(Webhook::EVENT_UPDATE_CLIENT, $client, $client->company)->delay(now()->addSeconds(2));
         }
     }
 
@@ -64,7 +64,7 @@ class ClientObserver
                                     ->exists();
 
         if ($subscriptions) {
-            WebhookHandler::dispatch(Webhook::EVENT_DELETE_CLIENT, $client, $client->company);
+            WebhookHandler::dispatch(Webhook::EVENT_DELETE_CLIENT, $client, $client->company)->delay(now()->addSeconds(2));
         }
     }
 
diff --git a/app/Observers/ExpenseObserver.php b/app/Observers/ExpenseObserver.php
index 85d0af6171a4..2c71e5fc42da 100644
--- a/app/Observers/ExpenseObserver.php
+++ b/app/Observers/ExpenseObserver.php
@@ -30,7 +30,7 @@ class ExpenseObserver
                             ->exists();
 
         if ($subscriptions) {
-            WebhookHandler::dispatch(Webhook::EVENT_CREATE_EXPENSE, $expense, $expense->company);
+            WebhookHandler::dispatch(Webhook::EVENT_CREATE_EXPENSE, $expense, $expense->company)->delay(now()->addSeconds(2));
         }
     }
 
@@ -47,7 +47,7 @@ class ExpenseObserver
                             ->exists();
 
         if ($subscriptions) {
-            WebhookHandler::dispatch(Webhook::EVENT_UPDATE_EXPENSE, $expense, $expense->company);
+            WebhookHandler::dispatch(Webhook::EVENT_UPDATE_EXPENSE, $expense, $expense->company)->delay(now()->addSeconds(2));
         }
     }
 
@@ -64,7 +64,7 @@ class ExpenseObserver
                             ->exists();
 
         if ($subscriptions) {
-            WebhookHandler::dispatch(Webhook::EVENT_DELETE_EXPENSE, $expense, $expense->company);
+            WebhookHandler::dispatch(Webhook::EVENT_DELETE_EXPENSE, $expense, $expense->company)->delay(now()->addSeconds(2));
         }
     }
 
diff --git a/app/Observers/InvoiceObserver.php b/app/Observers/InvoiceObserver.php
index 00b18dc428c2..9650579f0e66 100644
--- a/app/Observers/InvoiceObserver.php
+++ b/app/Observers/InvoiceObserver.php
@@ -36,7 +36,7 @@ class InvoiceObserver
 
         if ($subscriptions) {
     
-            WebhookHandler::dispatch(Webhook::EVENT_CREATE_INVOICE, $invoice, $invoice->company, 'client');
+            WebhookHandler::dispatch(Webhook::EVENT_CREATE_INVOICE, $invoice, $invoice->company, 'client')->delay(now()->addSeconds(2));
         }
     }
 
@@ -55,7 +55,7 @@ class InvoiceObserver
 
         if ($subscriptions) {
                 
-            WebhookHandler::dispatch(Webhook::EVENT_UPDATE_INVOICE, $invoice, $invoice->company, 'client');
+            WebhookHandler::dispatch(Webhook::EVENT_UPDATE_INVOICE, $invoice, $invoice->company, 'client')->delay(now()->addSeconds(2));
         
         }
 
@@ -75,7 +75,7 @@ class InvoiceObserver
 
         if ($subscriptions) {
         
-            WebhookHandler::dispatch(Webhook::EVENT_DELETE_INVOICE, $invoice, $invoice->company, 'client');
+            WebhookHandler::dispatch(Webhook::EVENT_DELETE_INVOICE, $invoice, $invoice->company, 'client')->delay(now()->addSeconds(2));
         }
     }
 
diff --git a/app/Observers/PaymentObserver.php b/app/Observers/PaymentObserver.php
index efd1ac80f2a2..2f5a1d740eb6 100644
--- a/app/Observers/PaymentObserver.php
+++ b/app/Observers/PaymentObserver.php
@@ -30,7 +30,7 @@ class PaymentObserver
                             ->exists();
 
         if ($subscriptions) {
-            WebhookHandler::dispatch(Webhook::EVENT_CREATE_PAYMENT, $payment, $payment->company, 'invoices,client')->delay(5);
+            WebhookHandler::dispatch(Webhook::EVENT_CREATE_PAYMENT, $payment, $payment->company, 'invoices,client')->delay(now()->addSeconds(20));
         }
     }
 
@@ -57,7 +57,7 @@ class PaymentObserver
                         ->exists();
 
         if ($subscriptions) {
-            WebhookHandler::dispatch(Webhook::EVENT_DELETE_PAYMENT, $payment, $payment->company, 'invoices,client')->delay(5);
+            WebhookHandler::dispatch(Webhook::EVENT_DELETE_PAYMENT, $payment, $payment->company, 'invoices,client')->delay(now()->addSeconds(20));
         }
     }
 
diff --git a/app/Observers/ProjectObserver.php b/app/Observers/ProjectObserver.php
index 8adf3feee3ca..8c88a6c8eb9b 100644
--- a/app/Observers/ProjectObserver.php
+++ b/app/Observers/ProjectObserver.php
@@ -32,7 +32,7 @@ class ProjectObserver
 
         if ($subscriptions) {
     
-            WebhookHandler::dispatch(Webhook::EVENT_PROJECT_CREATE, $project, $project->company, 'client');
+            WebhookHandler::dispatch(Webhook::EVENT_PROJECT_CREATE, $project, $project->company, 'client')->delay(now()->addSeconds(2));
         }
     }
 
@@ -50,7 +50,7 @@ class ProjectObserver
 
         if ($subscriptions) {
     
-            WebhookHandler::dispatch(Webhook::EVENT_PROJECT_UPDATE, $project, $project->company, 'client');
+            WebhookHandler::dispatch(Webhook::EVENT_PROJECT_UPDATE, $project, $project->company, 'client')->delay(now()->addSeconds(2));
         }
     }
 
diff --git a/app/Observers/QuoteObserver.php b/app/Observers/QuoteObserver.php
index ba2f02fb5688..55e24d8e9b5c 100644
--- a/app/Observers/QuoteObserver.php
+++ b/app/Observers/QuoteObserver.php
@@ -33,7 +33,7 @@ class QuoteObserver
 
         if ($subscriptions) {
             $quote->load('client');
-            WebhookHandler::dispatch(Webhook::EVENT_CREATE_QUOTE, $quote, $quote->company, 'client');
+            WebhookHandler::dispatch(Webhook::EVENT_CREATE_QUOTE, $quote, $quote->company, 'client')->delay(now()->addSeconds(2));
         }
     }
 
@@ -53,7 +53,7 @@ class QuoteObserver
 
         if ($subscriptions) {
             $quote->load('client');
-            WebhookHandler::dispatch(Webhook::EVENT_UPDATE_QUOTE, $quote, $quote->company, 'client');
+            WebhookHandler::dispatch(Webhook::EVENT_UPDATE_QUOTE, $quote, $quote->company, 'client')->delay(now()->addSeconds(2));
         }
 
     }
@@ -72,7 +72,7 @@ class QuoteObserver
 
         if ($subscriptions) {
             $quote->load('client');
-            WebhookHandler::dispatch(Webhook::EVENT_DELETE_QUOTE, $quote, $quote->company, 'client');
+            WebhookHandler::dispatch(Webhook::EVENT_DELETE_QUOTE, $quote, $quote->company, 'client')->delay(now()->addSeconds(2));
         }
     }
 
diff --git a/app/Observers/TaskObserver.php b/app/Observers/TaskObserver.php
index 943ab483ebfa..5bfe4b43ef69 100644
--- a/app/Observers/TaskObserver.php
+++ b/app/Observers/TaskObserver.php
@@ -30,7 +30,7 @@ class TaskObserver
                         ->exists();
 
         if ($subscriptions) {
-            WebhookHandler::dispatch(Webhook::EVENT_CREATE_TASK, $task, $task->company);
+            WebhookHandler::dispatch(Webhook::EVENT_CREATE_TASK, $task, $task->company)->delay(now()->addSeconds(2));
         }
     }
 
@@ -47,7 +47,7 @@ class TaskObserver
                         ->exists();
 
         if ($subscriptions) {
-            WebhookHandler::dispatch(Webhook::EVENT_UPDATE_TASK, $task, $task->company);
+            WebhookHandler::dispatch(Webhook::EVENT_UPDATE_TASK, $task, $task->company)->delay(now()->addSeconds(2));
         }
     }
 
@@ -64,7 +64,7 @@ class TaskObserver
                         ->exists();
 
         if ($subscriptions) {
-            WebhookHandler::dispatch(Webhook::EVENT_DELETE_TASK, $task, $task->company);
+            WebhookHandler::dispatch(Webhook::EVENT_DELETE_TASK, $task, $task->company)->delay(now()->addSeconds(2));
         }
     }
 
diff --git a/app/PaymentDrivers/StripePaymentDriver.php b/app/PaymentDrivers/StripePaymentDriver.php
index 7f0537fe3503..c71f81655bc7 100644
--- a/app/PaymentDrivers/StripePaymentDriver.php
+++ b/app/PaymentDrivers/StripePaymentDriver.php
@@ -568,7 +568,7 @@ class StripePaymentDriver extends BaseDriver
 
         //payment_intent.succeeded - this will confirm or cancel the payment
         if($request->type === 'payment_intent.succeeded'){
-            PaymentIntentWebhook::dispatch($request->data, $request->company_key, $this->company_gateway->id)->delay(10);
+            PaymentIntentWebhook::dispatch($request->data, $request->company_key, $this->company_gateway->id)->delay(now()->addSeconds(10));
             return response()->json([], 200);
         }
 
diff --git a/app/Services/Ledger/LedgerService.php b/app/Services/Ledger/LedgerService.php
index 4925bcca997c..fe6d32097bf5 100644
--- a/app/Services/Ledger/LedgerService.php
+++ b/app/Services/Ledger/LedgerService.php
@@ -37,7 +37,7 @@ class LedgerService
 
         $this->entity->company_ledger()->save($company_ledger);
         
-        ClientLedgerBalanceUpdate::dispatch($this->entity->company, $this->entity->client)->delay(300);
+        ClientLedgerBalanceUpdate::dispatch($this->entity->company, $this->entity->client)->delay(now()->addSeconds(300));
 
         return $this;
     }
@@ -54,7 +54,7 @@ class LedgerService
 
         $this->entity->company_ledger()->save($company_ledger);
 
-        ClientLedgerBalanceUpdate::dispatch($this->entity->company, $this->entity->client)->delay(300);
+        ClientLedgerBalanceUpdate::dispatch($this->entity->company, $this->entity->client)->delay(now()->addSeconds(300));
 
         return $this;
     }
@@ -71,7 +71,7 @@ class LedgerService
 
         $this->entity->company_ledger()->save($company_ledger);
 
-        ClientLedgerBalanceUpdate::dispatch($this->entity->company, $this->entity->client)->delay(300);
+        ClientLedgerBalanceUpdate::dispatch($this->entity->company, $this->entity->client)->delay(now()->addSeconds(300));
 
         return $this;
     }

From 6a99cba813329ea5f17268a7408f1803975b6609 Mon Sep 17 00:00:00 2001
From: David Bomba <turbo124@gmail.com>
Date: Sun, 15 May 2022 20:43:50 +1000
Subject: [PATCH 2/2] Analytics on queue size

---
 app/Console/Kernel.php                 |  2 +
 app/DataMapper/Analytics/QueueSize.php | 79 ++++++++++++++++++++++++++
 app/Jobs/Ninja/QueueSize.php           | 48 ++++++++++++++++
 app/Mail/Engine/InvoiceEmailEngine.php |  6 +-
 app/Services/Credit/SendEmail.php      |  4 +-
 5 files changed, 135 insertions(+), 4 deletions(-)
 create mode 100644 app/DataMapper/Analytics/QueueSize.php
 create mode 100644 app/Jobs/Ninja/QueueSize.php

diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index 14445950b38f..b472bf3a7442 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -18,6 +18,7 @@ use App\Jobs\Cron\SubscriptionCron;
 use App\Jobs\Ledger\LedgerBalanceUpdate;
 use App\Jobs\Ninja\AdjustEmailQuota;
 use App\Jobs\Ninja\CompanySizeCheck;
+use App\Jobs\Ninja\QueueSize;
 use App\Jobs\Util\DiskCleanup;
 use App\Jobs\Util\ReminderJob;
 use App\Jobs\Util\SchedulerCheck;
@@ -56,6 +57,7 @@ class Kernel extends ConsoleKernel
         $schedule->job(new ReminderJob)->hourly()->withoutOverlapping();
 
         // $schedule->job(new LedgerBalanceUpdate)->everyFiveMinutes()->withoutOverlapping();
+        $schedule->job(new QueueSize)->everyFiveMinutes()->withoutOverlapping();
 
         $schedule->job(new CompanySizeCheck)->daily()->withoutOverlapping();
 
diff --git a/app/DataMapper/Analytics/QueueSize.php b/app/DataMapper/Analytics/QueueSize.php
new file mode 100644
index 000000000000..3266f38645e2
--- /dev/null
+++ b/app/DataMapper/Analytics/QueueSize.php
@@ -0,0 +1,79 @@
+<?php
+/**
+ * Invoice Ninja (https://invoiceninja.com).
+ *
+ * @link https://github.com/invoiceninja/invoiceninja source repository
+ *
+ * @copyright Copyright (c) 2022. Invoice Ninja LLC (https://invoiceninja.com)
+ *
+ * @license https://www.elastic.co/licensing/elastic-license
+ */
+
+namespace App\DataMapper\Analytics;
+
+use Turbo124\Beacon\ExampleMetric\GenericMixedMetric;
+
+class QueueSize extends GenericMixedMetric
+{
+
+    /**
+     * The type of Sample.
+     *
+     * Monotonically incrementing counter
+     *
+     *  - counter
+     *
+     * @var string
+     */
+    public $type = 'mixed_metric';
+
+    /**
+     * The name of the counter.
+     * @var string
+     */
+    public $name = 'ninja.queue_size';
+
+    /**
+     * The datetime of the counter measurement.
+     *
+     * date("Y-m-d H:i:s")
+     *
+     * @var DateTime
+     */
+    public $datetime;
+
+    /**
+     * The Class failure name
+     * set to 0.
+     *
+     * @var string
+     */
+    public $string_metric5 = 'stub';
+
+    /**
+     * The exception string
+     * set to 0.
+     *
+     * @var string
+     */
+    public $string_metric6 = 'stub';
+
+    /**
+     * The counter
+     * set to 1.
+     *
+     * @var string
+     */
+    public $int_metric1 = 1;
+
+    /**
+     * Company Key
+     * @var string
+     */
+    public $string_metric7 = '';
+
+    public function __construct($int_metric1) {
+        $this->int_metric1 = $int_metric1;
+    }
+    
+}
diff --git a/app/Jobs/Ninja/QueueSize.php b/app/Jobs/Ninja/QueueSize.php
new file mode 100644
index 000000000000..f5a5d6951da3
--- /dev/null
+++ b/app/Jobs/Ninja/QueueSize.php
@@ -0,0 +1,48 @@
+<?php
+/**
+ * Invoice Ninja (https://invoiceninja.com).
+ *
+ * @link https://github.com/invoiceninja/invoiceninja source repository
+ *
+ * @copyright Copyright (c) 2022. Invoice Ninja LLC (https://invoiceninja.com)
+ *
+ * @license https://www.elastic.co/licensing/elastic-license
+ */
+
+namespace App\Jobs\Ninja;
+
+use App\DataMapper\Analytics\QueueSize as QueueSizeAnalytic;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Queue;
+use Turbo124\Beacon\Facades\LightLogs;
+use Turbo124\Beacon\Jobs\Database\MySQL\DbStatus;
+
+class QueueSize implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        //
+    }
+
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        LightLogs::create(new QueueSizeAnalytic(Queue::size()))
+         ->queue();        
+    }
+}
diff --git a/app/Mail/Engine/InvoiceEmailEngine.php b/app/Mail/Engine/InvoiceEmailEngine.php
index ad0c232489cf..8ac9f89472bb 100644
--- a/app/Mail/Engine/InvoiceEmailEngine.php
+++ b/app/Mail/Engine/InvoiceEmailEngine.php
@@ -153,10 +153,12 @@ class InvoiceEmailEngine extends BaseEmailEngine
 
             $line_items = $this->invoice->line_items;
             
-            $expense_ids = [];
 
             foreach($line_items as $item)
             {
+
+                $expense_ids = [];
+
                 if(property_exists($item, 'expense_id'))
                 {
                     $expense_ids[] = $item->expense_id;
@@ -178,7 +180,7 @@ class InvoiceEmailEngine extends BaseEmailEngine
                 }
 
                 $task_ids = [];
-                
+
                 if(property_exists($item, 'task_id'))
                 {
                     $task_ids[] = $item->task_id;
diff --git a/app/Services/Credit/SendEmail.php b/app/Services/Credit/SendEmail.php
index b62122177b69..6432aaed44f2 100644
--- a/app/Services/Credit/SendEmail.php
+++ b/app/Services/Credit/SendEmail.php
@@ -45,8 +45,8 @@ class SendEmail
 
         $this->credit->invitations->each(function ($invitation) {
             if (!$invitation->contact->trashed() && $invitation->contact->email) {
-                $email_builder = (new CreditEmail())->build($invitation, $this->reminder_template);
-
+                
+                // $email_builder = (new CreditEmail())->build($invitation, $this->reminder_template);
                 // EmailCredit::dispatchNow($email_builder, $invitation, $invitation->company);
                 EmailEntity::dispatchNow($invitation, $invitation->company, $this->reminder_template);
             }