From 35775706becc36f76f9b523705ba0018df4ac2d9 Mon Sep 17 00:00:00 2001
From: David Bomba <turbo124@gmail.com>
Date: Sun, 29 Jan 2023 11:51:57 +1100
Subject: [PATCH] Improve backoff using exponential decay

---
 app/Jobs/Mail/NinjaMailerJob.php   |  9 ++++++---
 app/Jobs/Util/WebhookHandler.php   | 16 ++++++++++------
 app/Models/SystemLog.php           |  2 ++
 app/Services/Email/EmailMailer.php |  9 ++++++---
 4 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/app/Jobs/Mail/NinjaMailerJob.php b/app/Jobs/Mail/NinjaMailerJob.php
index 06df58bf3292..c1517b93dd5e 100644
--- a/app/Jobs/Mail/NinjaMailerJob.php
+++ b/app/Jobs/Mail/NinjaMailerJob.php
@@ -50,9 +50,7 @@ class NinjaMailerJob implements ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, MakesHash;
 
-    public $tries = 3; //number of retries
-
-    public $backoff = 30; //seconds to wait until retry
+    public $tries = 4; //number of retries
 
     public $deleteWhenMissingModels = true;
 
@@ -80,6 +78,11 @@ class NinjaMailerJob implements ShouldQueue
 
     }
 
+    public function backoff()
+    {
+        return [30, 60, 180, 240];
+    }
+
     public function handle()
     {
 
diff --git a/app/Jobs/Util/WebhookHandler.php b/app/Jobs/Util/WebhookHandler.php
index 25b63436f044..81ecff6bc72f 100644
--- a/app/Jobs/Util/WebhookHandler.php
+++ b/app/Jobs/Util/WebhookHandler.php
@@ -38,9 +38,7 @@ class WebhookHandler implements ShouldQueue
 
     private $company;
 
-    public $tries = 3; //number of retries
-
-    public $backoff = 10; //seconds to wait until retry
+    public $tries = 4; //number of retries
 
     public $deleteWhenMissingModels = true;
 
@@ -60,6 +58,11 @@ class WebhookHandler implements ShouldQueue
         $this->includes = $includes;
     }
 
+    public function backoff()
+    {
+        return [10, 30, 60, 180];
+    }
+
     /**
      * Execute the job.
      *
@@ -79,7 +82,7 @@ class WebhookHandler implements ShouldQueue
                                     ->where('event_id', $this->event_id)
                                     ->cursor()
                                     ->each(function ($subscription) {
-            
+
             $this->process($subscription);
 
         });
@@ -148,13 +151,14 @@ class WebhookHandler implements ShouldQueue
             }
 
         } catch (\Exception $e) {
-            nlog("429 occurred in the expcetion handler");
+            
+            nlog("429 occurred in the exception handler");
             nlog($e->getMessage());
 
             SystemLogger::dispatch(
                 $e->getMessage(),
                 SystemLog::CATEGORY_WEBHOOK,
-                SystemLog::EVENT_WEBHOOK_RESPONSE,
+                SystemLog::EVENT_WEBHOOK_FAILURE,
                 SystemLog::TYPE_WEBHOOK_RESPONSE,
                 $this->resolveClient(),
                 $this->company,
diff --git a/app/Models/SystemLog.php b/app/Models/SystemLog.php
index d166daf8dcad..ca509a65f8e5 100644
--- a/app/Models/SystemLog.php
+++ b/app/Models/SystemLog.php
@@ -69,6 +69,8 @@ class SystemLog extends Model
     const EVENT_WEBHOOK_RESPONSE = 40;
 
     const EVENT_WEBHOOK_SUCCESS = 41;
+    
+    const EVENT_WEBHOOK_FAILURE = 42;
 
     const EVENT_PDF_RESPONSE = 50;
 
diff --git a/app/Services/Email/EmailMailer.php b/app/Services/Email/EmailMailer.php
index 4f0d01071260..e04669b4e795 100644
--- a/app/Services/Email/EmailMailer.php
+++ b/app/Services/Email/EmailMailer.php
@@ -41,9 +41,7 @@ class EmailMailer implements ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, MakesHash;
 
-    public $tries = 3; //number of retries
-
-    public $backoff = 30; //seconds to wait until retry
+    public $tries = 4; //number of retries
 
     public $deleteWhenMissingModels = true;
 
@@ -59,6 +57,11 @@ class EmailMailer implements ShouldQueue
 
     public function __construct(public EmailService $email_service, public Mailable $email_mailable){}
 
+    public function backoff()
+    {
+        return [30, 60, 180, 240];
+    }
+
     public function handle(): void
     {
     	MultiDB::setDb($this->email_service->company->db);