From c80e3bf9215d5c7ff1506b484adb31f181ade1ab Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 18 Mar 2024 08:04:54 +0100 Subject: [PATCH] working mailgun inbound webhook --- .../MailgunInboundWebhookTransformer.php | 21 +++++-- app/Http/Controllers/MailgunController.php | 19 +++++- .../Mailgun/ProcessMailgunInboundWebhook.php | 61 ++++++------------- app/Services/IngresEmail/IngresEmail.php | 14 +---- .../IngresEmail/IngresEmailEngine.php | 40 ++++-------- app/Utils/TempFile.php | 29 +++++++++ routes/api.php | 2 +- 7 files changed, 95 insertions(+), 91 deletions(-) diff --git a/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php b/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php index 6871e3010168..cd50f38dfd98 100644 --- a/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php +++ b/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php @@ -13,6 +13,7 @@ namespace App\Helpers\IngresMail\Transformer; use App\Services\IngresEmail\IngresEmail; use App\Utils\TempFile; +use Illuminate\Support\Carbon; class MailgunInboundWebhookTransformer { @@ -20,15 +21,25 @@ class MailgunInboundWebhookTransformer { $ingresEmail = new IngresEmail(); - $ingresEmail->from = $data["sender"]; - $ingresEmail->subject = $data["subject"]; + $ingresEmail->from = $data["From"]; + $ingresEmail->to = $data["To"]; + $ingresEmail->subject = $data["Subject"]; $ingresEmail->plain_message = $data["body-plain"]; $ingresEmail->html_message = $data["body-html"]; - $ingresEmail->date = now(); // TODO + $ingresEmail->date = Carbon::createFromTimestamp((int) $data["timestamp"]); // parse documents as UploadedFile from webhook-data - foreach ($data["Attachments"] as $attachment) { - $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]); + foreach (json_decode($data["attachments"]) as $attachment) { + + // prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 + $url = $attachment->url; + $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + + // download file and save to tmp dir + $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + } return $ingresEmail; diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 1c5fd9dbcc0e..c4e2999ea227 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -14,6 +14,7 @@ namespace App\Http\Controllers; use App\Jobs\Mailgun\ProcessMailgunInboundWebhook; use App\Jobs\Mailgun\ProcessMailgunWebhook; use Illuminate\Http\Request; +use Log; /** * Class MailgunController. @@ -112,10 +113,22 @@ class MailgunController extends BaseController */ public function inboundWebhook(Request $request) { - if ($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.mailgun.token')) { - ProcessMailgunInboundWebhook::dispatch($request->all())->delay(10); + $input = $request->all(); - return response()->json(['message' => 'Success'], 200); + if (!array_key_exists('attachments', $input) || count(json_decode($input['attachments'])) == 0) { + Log::info('Message ignored because of missing attachments. Please ensure contacting this api-endpoint with a store & notify operation instead of a forward operation'); + return response()->json(['message' => 'Sucess. Soft Fail. Missing Attachments.'], 200); + } + + if (\abs(\time() - (int) $request['timestamp']) > 150) { + Log::info('Message ignored because of request body is too old.'); + return response()->json(['message' => 'Success. Soft Fail. Message too old.'], 200); + } + + if (\hash_equals(\hash_hmac('sha256', $input['timestamp'] . $input['token'], config('services.mailgun.webhook_signing_key')), $input['signature'])) { + ProcessMailgunInboundWebhook::dispatch($input)->delay(10); + + return response()->json(['message' => 'Success'], 201); } return response()->json(['message' => 'Unauthorized'], 403); diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index 603814263f38..b8a9ffc578d2 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -9,16 +9,17 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Jobs\PostMark; +namespace App\Jobs\Mailgun; -use App\Helpers\Mail\Webhook\Maigun\MailgunWebhookHandler; +use App\Helpers\IngresMail\Transformer\MailgunInboundWebhookTransformer; use App\Libraries\MultiDB; -use App\Models\SystemLog; +use App\Services\IngresEmail\IngresEmailEngine; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Log; class ProcessMailgunInboundWebhook implements ShouldQueue { @@ -26,18 +27,6 @@ class ProcessMailgunInboundWebhook implements ShouldQueue public $tries = 1; - public $invitation; - - private $entity; - - private array $default_response = [ - 'recipients' => '', - 'subject' => 'Message not found.', - 'entity' => '', - 'entity_id' => '', - 'events' => [], - ]; - /** * Create a new job instance. * @@ -46,23 +35,6 @@ class ProcessMailgunInboundWebhook implements ShouldQueue { } - private function getSystemLog(string $message_id): ?SystemLog - { - return SystemLog::query() - ->where('company_id', $this->invitation->company_id) - ->where('type_id', SystemLog::TYPE_WEBHOOK_RESPONSE) - ->whereJsonContains('log', ['MessageID' => $message_id]) - ->orderBy('id', 'desc') - ->first(); - - } - - private function updateSystemLog(SystemLog $system_log, array $data): void - { - $system_log->log = $data; - $system_log->save(); - } - /** * Execute the job. * @@ -71,20 +43,21 @@ class ProcessMailgunInboundWebhook implements ShouldQueue */ public function handle() { - // match companies - if (array_key_exists('ToFull', $this->request)) + if (!array_key_exists('To', $this->request) || !array_key_exists('attachments', $this->request) || !array_key_exists('timestamp', $this->request) || !array_key_exists('Subject', $this->request) || !(array_key_exists('body-html', $this->request) || array_key_exists('body-plain', $this->request))) throw new \Exception('invalid body'); - foreach ($this->request['ToFull'] as $toEmailEntry) { - $toEmail = $toEmailEntry['Email']; - - $company = MultiDB::findAndSetDbByExpenseMailbox($toEmail); - if (!$company) { - nlog('unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $toEmail); - continue; - } - - (new MailgunWebhookHandler())->process($this->request); + // match company + $company = MultiDB::findAndSetDbByExpenseMailbox($this->request["To"]); + if (!$company) { + Log::info('unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $this->request["To"]); + return; } + + // prepare + $ingresMail = (new MailgunInboundWebhookTransformer())->transform($this->request); + Log::info(json_encode($ingresMail)); + + // perform + (new IngresEmailEngine($ingresMail))->handle(); } } diff --git a/app/Services/IngresEmail/IngresEmail.php b/app/Services/IngresEmail/IngresEmail.php index 2ec889bcf6e2..728e25a41ab4 100644 --- a/app/Services/IngresEmail/IngresEmail.php +++ b/app/Services/IngresEmail/IngresEmail.php @@ -12,25 +12,17 @@ namespace App\Services\IngresEmail; use Illuminate\Http\UploadedFile; +use Illuminate\Support\Carbon; /** * EmailObject. */ class IngresEmail { - /** @var array[string] $args */ - public array $to = []; + public string $to; public string $from; - public array $reply_to = []; - - /** @var array[string] $args */ - public array $cc = []; - - /** @var array[string] $args */ - public array $bcc = []; - public ?string $subject = null; public ?string $body = null; @@ -41,7 +33,7 @@ class IngresEmail /** @var array[\Illuminate\Http\UploadedFile] $documents */ public array $documents = []; - public ?\DateTimeImmutable $date = null; + public ?Carbon $date = null; function __constructor() { diff --git a/app/Services/IngresEmail/IngresEmailEngine.php b/app/Services/IngresEmail/IngresEmailEngine.php index f89d0886314b..4ac1d9a9fe83 100644 --- a/app/Services/IngresEmail/IngresEmailEngine.php +++ b/app/Services/IngresEmail/IngresEmailEngine.php @@ -14,11 +14,9 @@ namespace App\Services\IngresEmail; use App\Events\Expense\ExpenseWasCreated; use App\Factory\ExpenseFactory; use App\Libraries\MultiDB; -use App\Models\Client; use App\Models\Company; use App\Models\Vendor; use App\Models\VendorContact; -use App\Services\Email\EmailObject; use App\Services\IngresEmail\IngresEmail; use App\Utils\Ninja; use App\Utils\TempFile; @@ -26,24 +24,18 @@ use App\Utils\Traits\GeneratesCounter; use App\Utils\Traits\SavesDocuments; use App\Utils\Traits\MakesHash; use Cache; -use Illuminate\Bus\Queueable; -use Illuminate\Contracts\Queue\ShouldQueue; -use Illuminate\Foundation\Bus\Dispatchable; -use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -class IngresEmailEngine implements ShouldQueue +class IngresEmailEngine { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, MakesHash; + use SerializesModels, MakesHash; use GeneratesCounter, SavesDocuments; - private IngresEmail $email; private ?Company $company; private ?bool $isUnknownRecipent = null; private array $globalBlacklist = []; - function __constructor(IngresEmail $email) + public function __construct(private IngresEmail $email) { - $this->email = $email; } /** * if there is not a company with an matching mailbox, we only do monitoring @@ -53,14 +45,12 @@ class IngresEmailEngine implements ShouldQueue { if ($this->isInvalidOrBlocked()) return; + $this->isUnknownRecipent = true; // Expense Mailbox => will create an expense - foreach ($this->email->to as $expense_mailbox) { - $this->company = MultiDB::findAndSetDbByExpenseMailbox($expense_mailbox); - if (!$this->company) - continue; - + $this->company = MultiDB::findAndSetDbByExpenseMailbox($this->email->to); + if ($this->company) { $this->isUnknownRecipent = false; $this->createExpense(); } @@ -112,13 +102,11 @@ class IngresEmailEngine implements ShouldQueue } // wrong recipent occurs in more than 100 emails in the last 12 hours, so the processing is blocked - foreach ($this->email->to as $recipent) { - $mailCountUnknownRecipent = Cache::get('ingresEmailUnknownRecipent:' . $recipent, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time - if ($mailCountUnknownRecipent >= 100) { - nlog('[IngressMailEngine] E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); - $this->blockSender(); - return true; - } + $mailCountUnknownRecipent = Cache::get('ingresEmailUnknownRecipent:' . $this->email->to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time + if ($mailCountUnknownRecipent >= 100) { + nlog('[IngressMailEngine] E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); + $this->blockSender(); + return true; } return false; @@ -141,10 +129,8 @@ class IngresEmailEngine implements ShouldQueue Cache::add('ingresEmailSenderUnknownRecipent:' . $this->email->from, 0, now()->addHours(6)); Cache::increment('ingresEmailSenderUnknownRecipent:' . $this->email->from); // we save the sender, to may block him - foreach ($this->email->to as $recipent) { - Cache::add('ingresEmailUnknownRecipent:' . $recipent, 0, now()->addHours(12)); - Cache::increment('ingresEmailUnknownRecipent:' . $recipent); // we save the sender, to may block him - } + Cache::add('ingresEmailUnknownRecipent:' . $this->email->to, 0, now()->addHours(12)); + Cache::increment('ingresEmailUnknownRecipent:' . $this->email->to); // we save the sender, to may block him } } diff --git a/app/Utils/TempFile.php b/app/Utils/TempFile.php index 0e8062b457ad..24df20da7271 100644 --- a/app/Utils/TempFile.php +++ b/app/Utils/TempFile.php @@ -99,4 +99,33 @@ class TempFile // return UploadedFile object return $file; } + + /* create a tmp file from a raw string: https://gist.github.com/waska14/8b3bcebfad1f86f7fcd3b82927576e38*/ + public static function UploadedFileFromUrl(string $url, string|null $fileName = null, string|null $mimeType = null): UploadedFile + { + // Create temp file and get its absolute path + $tempFile = tmpfile(); + $tempFilePath = stream_get_meta_data($tempFile)['uri']; + + // Save file data in file + file_put_contents($tempFilePath, file_get_contents($url)); + + $tempFileObject = new File($tempFilePath); + $file = new UploadedFile( + $tempFileObject->getPathname(), + $fileName ?: $tempFileObject->getFilename(), + $mimeType ?: $tempFileObject->getMimeType(), + 0, + true // Mark it as test, since the file isn't from real HTTP POST. + ); + + // Close this file after response is sent. + // Closing the file will cause to remove it from temp director! + app()->terminating(function () use ($tempFile) { + fclose($tempFile); + }); + + // return UploadedFile object + return $file; + } } diff --git a/routes/api.php b/routes/api.php index 9a45cd829754..480f1ce87e6c 100644 --- a/routes/api.php +++ b/routes/api.php @@ -71,7 +71,6 @@ use App\Http\Controllers\CompanyLedgerController; use App\Http\Controllers\PurchaseOrderController; use App\Http\Controllers\TaskSchedulerController; use App\Http\Controllers\CompanyGatewayController; -use App\Http\Controllers\MailgunWebhookController; use App\Http\Controllers\PaymentWebhookController; use App\Http\Controllers\RecurringQuoteController; use App\Http\Controllers\BankIntegrationController; @@ -426,6 +425,7 @@ Route::match(['get', 'post'], 'payment_notification_webhook/{company_key}/{compa Route::post('api/v1/postmark_webhook', [PostMarkController::class, 'webhook'])->middleware('throttle:1000,1'); Route::post('api/v1/mailgun_webhook', [MailgunController::class, 'webhook'])->middleware('throttle:1000,1'); +Route::post('api/v1/mailgun_inbound_webhook', [MailgunController::class, 'inboundWebhook'])->middleware('throttle:1000,1'); Route::get('token_hash_router', [OneTimeTokenController::class, 'router'])->middleware('throttle:500,1'); Route::get('webcron', [WebCronController::class, 'index'])->middleware('throttle:100,1'); Route::post('api/v1/get_migration_account', [HostedMigrationController::class, 'getAccount'])->middleware('guest')->middleware('throttle:100,1');