diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 8554f96b6feb..d097f61ed6ac 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -115,7 +115,7 @@ class MailgunController extends BaseController { $input = $request->all(); - if (!array_key_exists('recipient', $input) || !array_key_exists('message-url', $input)) { + if (!array_key_exists('sender', $input) || !array_key_exists('recipient', $input) || !array_key_exists('message-url', $input)) { Log::info('Failed: Message could not be parsed, because required parameters are missing. Please ensure contacting this api-endpoint with a store & notify operation instead of a forward operation!'); return response()->json(['message' => 'Failed. Missing Parameters. Use store and notify!'], 400); } @@ -123,7 +123,7 @@ class MailgunController extends BaseController // @turbo124 TODO: how to check for services.mailgun.webhook_signing_key on company level, when custom credentials are defined // TODO: validation for client mail credentials by recipient if (\hash_equals(\hash_hmac('sha256', $input['timestamp'] . $input['token'], config('services.mailgun.webhook_signing_key')), $input['signature'])) { - ProcessMailgunInboundWebhook::dispatch($input["recipient"] . "|" . $input["message-url"])->delay(10); + ProcessMailgunInboundWebhook::dispatch($input["sender"] . "|" . $input["recipient"] . "|" . $input["message-url"])->delay(10); return response()->json(['message' => 'Success'], 201); } diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index e0df719557c5..5fa1049ca607 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -13,6 +13,10 @@ namespace App\Http\Controllers; use App\Jobs\PostMark\ProcessPostmarkInboundWebhook; use App\Jobs\PostMark\ProcessPostmarkWebhook; +use App\Services\InboundMail\InboundMail; +use App\Services\InboundMail\InboundMailEngine; +use App\Utils\TempFile; +use Carbon\Carbon; use Illuminate\Http\Request; use Log; @@ -254,6 +258,13 @@ class PostMarkController extends BaseController * ), * 'Attachments' => * array ( + * array ( + * 'Content' => "base64-String", + * 'ContentLength' => 60164, + * 'Name' => 'Unbenannt.png', + * 'ContentType' => 'image/png', + * 'ContentID' => 'ii_luh2h8lg0', + * ) * ), * ) */ @@ -261,20 +272,47 @@ class PostMarkController extends BaseController { Log::info($request->all()); + Log::info($request->headers); $input = $request->all(); - if (!(array_key_exists("MessageStream", $input) && $input["MessageStream"] != "inbound") || !array_key_exists("To", $input) || !array_key_exists("MessageID", $input)) { - Log::info('Failed: Message could not be parsed, because required parameters are missing. Please ensure contacting this api-endpoint with a store & notify operation instead of a forward operation!'); + if (!(array_key_exists("MessageStream", $input) && $input["MessageStream"] == "inbound") || !array_key_exists("To", $input) || !array_key_exists("From", $input) || !array_key_exists("MessageID", $input)) { + Log::info('Failed: Message could not be parsed, because required parameters are missing.'); return response()->json(['message' => 'Failed. Missing/Invalid Parameters.'], 400); } - if ($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.postmark.token')) { - ProcessPostmarkInboundWebhook::dispatch($input["To"] . "|" . $input["MessageID"])->delay(10); + // // TODO: security + // if (!($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.postmark.token'))) + // return response()->json(['message' => 'Unauthorized'], 403); - return response()->json(['message' => 'Success'], 200); + + try { // important to save meta if something fails here to prevent spam + + // prepare data for ingresEngine + $inboundMail = new InboundMail(); + + $inboundMail->from = $input["From"]; + $inboundMail->to = $input["To"]; // usage of data-input, because we need a single email here + $inboundMail->subject = $input["Subject"]; + $inboundMail->body = $input["HtmlBody"]; + $inboundMail->text_body = $input["TextBody"]; + $inboundMail->date = Carbon::createFromTimeString($input["Date"]); + + // parse documents as UploadedFile from webhook-data + foreach ($input["Attachments"] as $attachment) { + + $inboundMail->documents[] = TempFile::UploadedFileFromBase64($attachment["Content"], $attachment["Name"], $attachment["ContentType"]); + + } + + } catch (\Exception $e) { + (new InboundMailEngine())->saveMeta($input["From"], $input["To"]); // important to save this, to protect from spam + throw $e; } - return response()->json(['message' => 'Unauthorized'], 403); + // perform + (new InboundMailEngine())->handle($inboundMail); + + return response()->json(['message' => 'Success'], 200); } } diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index 774018070fa8..706b76e8fdaa 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -125,57 +125,65 @@ class ProcessBrevoInboundWebhook implements ShouldQueue $company = MultiDB::findAndSetDbByInboundMailbox($recipient); if (!$company) { Log::info('[ProcessBrevoInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from brevo: ' . $recipient); + (new InboundMailEngine())->saveMeta($this->input["From"]["Address"], $recipient); // important to save this, to protect from spam continue; } - $company_brevo_secret = $company->settings?->email_sending_method === 'client_brevo' && $company->settings?->brevo_secret ? $company->settings?->brevo_secret : null; - if (empty ($company_brevo_secret) && empty (config('services.brevo.secret'))) - throw new \Error("[ProcessBrevoInboundWebhook] no brevo credenitals found, we cannot get the attachement"); + try { // important to save meta if something fails here to prevent spam - // prepare data for ingresEngine - $inboundMail = new InboundMail(); + $company_brevo_secret = $company->settings?->email_sending_method === 'client_brevo' && $company->settings?->brevo_secret ? $company->settings?->brevo_secret : null; + if (empty($company_brevo_secret) && empty(config('services.brevo.secret'))) + throw new \Error("[ProcessBrevoInboundWebhook] no brevo credenitals found, we cannot get the attachement"); - $inboundMail->from = $this->input["From"]["Address"]; - $inboundMail->to = $recipient; - $inboundMail->subject = $this->input["Subject"]; - $inboundMail->body = $this->input["RawHtmlBody"]; - $inboundMail->text_body = $this->input["RawTextBody"]; - $inboundMail->date = Carbon::createFromTimeString($this->input["SentAtDate"]); + // prepare data for ingresEngine + $inboundMail = new InboundMail(); - // parse documents as UploadedFile from webhook-data - foreach ($this->input["Attachments"] as $attachment) { + $inboundMail->from = $this->input["From"]["Address"]; + $inboundMail->to = $recipient; + $inboundMail->subject = $this->input["Subject"]; + $inboundMail->body = $this->input["RawHtmlBody"]; + $inboundMail->text_body = $this->input["RawTextBody"]; + $inboundMail->date = Carbon::createFromTimeString($this->input["SentAtDate"]); - // download file and save to tmp dir - if (!empty ($company_brevo_secret)) { + // parse documents as UploadedFile from webhook-data + foreach ($this->input["Attachments"] as $attachment) { - $attachment = null; - try { + // download file and save to tmp dir + if (!empty($company_brevo_secret)) { - $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", $company_brevo_secret)); - $attachment = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); + $attachment = null; + try { - } catch (\Error $e) { - if (config('services.brevo.secret')) { - Log::info("[ProcessBrevoInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); - - $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); + $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", $company_brevo_secret)); $attachment = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); - } else - throw $e; + } catch (\Error $e) { + if (config('services.brevo.secret')) { + Log::info("[ProcessBrevoInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); + + $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); + $attachment = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); + + } else + throw $e; + } + $inboundMail->documents[] = TempFile::UploadedFileFromRaw($attachment, $attachment["Name"], $attachment["ContentType"]); + + } else { + + $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); + $inboundMail->documents[] = TempFile::UploadedFileFromRaw($brevo->getInboundEmailAttachment($attachment["DownloadToken"]), $attachment["Name"], $attachment["ContentType"]); + } - $inboundMail->documents[] = TempFile::UploadedFileFromRaw($attachment, $attachment["Name"], $attachment["ContentType"]); - - } else { - - $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); - $inboundMail->documents[] = TempFile::UploadedFileFromRaw($brevo->getInboundEmailAttachment($attachment["DownloadToken"]), $attachment["Name"], $attachment["ContentType"]); } + } catch (\Exception $e) { + (new InboundMailEngine())->saveMeta($this->input["From"]["Address"], $recipient); // important to save this, to protect from spam + throw $e; } - (new InboundMailEngine($inboundMail))->handle(); + (new InboundMailEngine())->handle($inboundMail); } } diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index 4f11b6ba0d2b..a108b5f29c40 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -31,7 +31,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue /** * Create a new job instance. - * $input consists of 2 informations: recipient|messageUrl + * $input consists of 3 informations: sender/from|recipient/to|messageUrl */ public function __construct(private string $input) { @@ -163,88 +163,45 @@ class ProcessMailgunInboundWebhook implements ShouldQueue */ public function handle() { - $recipient = explode("|", $this->input)[0]; + $from = explode("|", $this->input)[0]; + $to = explode("|", $this->input)[1]; + // $messageId = explode("|", $this->input)[2]; // used as base in download function // match company - $company = MultiDB::findAndSetDbByInboundMailbox($recipient); + $company = MultiDB::findAndSetDbByInboundMailbox($to); if (!$company) { - Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $recipient); + Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $to); + (new InboundMailEngine())->saveMeta($from, $to); // important to save this, to protect from spam return; } - // fetch message from mailgun-api - $company_mailgun_domain = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_domain ? $company->settings?->mailgun_domain : null; - $company_mailgun_secret = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_secret ? $company->settings?->mailgun_secret : null; - if (!($company_mailgun_domain && $company_mailgun_secret) && !(config('services.mailgun.domain') && config('services.mailgun.secret'))) - throw new \Error("[ProcessMailgunInboundWebhook] no mailgun credenitals found, we cannot get the attachements and files"); + try { // important to save meta if something fails here to prevent spam - $mail = null; - if ($company_mailgun_domain && $company_mailgun_secret) { + // fetch message from mailgun-api + $company_mailgun_domain = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_domain ? $company->settings?->mailgun_domain : null; + $company_mailgun_secret = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_secret ? $company->settings?->mailgun_secret : null; + if (!($company_mailgun_domain && $company_mailgun_secret) && !(config('services.mailgun.domain') && config('services.mailgun.secret'))) + throw new \Error("[ProcessMailgunInboundWebhook] no mailgun credenitals found, we cannot get the attachements and files"); - $credentials = $company_mailgun_domain . ":" . $company_mailgun_secret . "@"; - $messageUrl = explode("|", $this->input)[1]; - $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); - $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); - - try { - $mail = json_decode(file_get_contents($messageUrl)); - } catch (\Error $e) { - if (config('services.mailgun.secret')) { - Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); - - $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; - $messageUrl = explode("|", $this->input)[1]; - $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); - $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); - $mail = json_decode(file_get_contents($messageUrl)); - - } else - throw $e; - } - - } else { - - $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; - $messageUrl = explode("|", $this->input)[1]; - $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); - $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); - $mail = json_decode(file_get_contents($messageUrl)); - - } - - // prepare data for ingresEngine - $inboundMail = new InboundMail(); - - $inboundMail->from = $mail->sender; - $inboundMail->to = $recipient; // usage of data-input, because we need a single email here - $inboundMail->subject = $mail->Subject; - $inboundMail->body = $mail->{"body-html"}; - $inboundMail->text_body = $mail->{"body-plain"}; - $inboundMail->date = Carbon::createFromTimeString($mail->Date); - - // parse documents as UploadedFile from webhook-data - foreach ($mail->attachments as $attachment) { // prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 - - // download file and save to tmp dir + $mail = null; if ($company_mailgun_domain && $company_mailgun_secret) { + $credentials = $company_mailgun_domain . ":" . $company_mailgun_secret . "@"; + $messageUrl = explode("|", $this->input)[2]; + $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); + $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); + try { - - $credentials = $company_mailgun_domain . ":" . $company_mailgun_secret . "@"; - $url = $attachment->url; - $url = str_replace("http://", "http://" . $credentials, $url); - $url = str_replace("https://", "https://" . $credentials, $url); - $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); - + $mail = json_decode(file_get_contents($messageUrl)); } catch (\Error $e) { if (config('services.mailgun.secret')) { Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; - $url = $attachment->url; - $url = str_replace("http://", "http://" . $credentials, $url); - $url = str_replace("https://", "https://" . $credentials, $url); - $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $messageUrl = explode("|", $this->input)[2]; + $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); + $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); + $mail = json_decode(file_get_contents($messageUrl)); } else throw $e; @@ -253,16 +210,69 @@ class ProcessMailgunInboundWebhook implements ShouldQueue } else { $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; - $url = $attachment->url; - $url = str_replace("http://", "http://" . $credentials, $url); - $url = str_replace("https://", "https://" . $credentials, $url); - $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $messageUrl = explode("|", $this->input)[2]; + $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); + $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); + $mail = json_decode(file_get_contents($messageUrl)); } + // prepare data for ingresEngine + $inboundMail = new InboundMail(); + + $inboundMail->from = $from; + $inboundMail->to = $to; // usage of data-input, because we need a single email here + $inboundMail->subject = $mail->Subject; + $inboundMail->body = $mail->{"body-html"}; + $inboundMail->text_body = $mail->{"body-plain"}; + $inboundMail->date = Carbon::createFromTimeString($mail->Date); + + // parse documents as UploadedFile from webhook-data + foreach ($mail->attachments as $attachment) { // prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 + + // download file and save to tmp dir + if ($company_mailgun_domain && $company_mailgun_secret) { + + try { + + $credentials = $company_mailgun_domain . ":" . $company_mailgun_secret . "@"; + $url = $attachment->url; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + + } catch (\Error $e) { + if (config('services.mailgun.secret')) { + Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); + + $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; + $url = $attachment->url; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + + } else + throw $e; + } + + } else { + + $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; + $url = $attachment->url; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + + } + + } + + } catch (\Exception $e) { + (new InboundMailEngine())->saveMeta($from, $to); // important to save this, to protect from spam + throw $e; } // perform - (new InboundMailEngine($inboundMail))->handle(); + (new InboundMailEngine())->handle($inboundMail); } } diff --git a/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php b/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php deleted file mode 100644 index 5904b7c5b664..000000000000 --- a/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php +++ /dev/null @@ -1,299 +0,0 @@ - 'Max Mustermann', - * 'MessageStream' => 'inbound', - * 'From' => 'max@mustermann.de', - * 'FromFull' => - * array ( - * 'Email' => 'max@mustermann.de', - * 'Name' => 'Max Mustermann', - * 'MailboxHash' => NULL, - * ), - * 'To' => '370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com', - * 'ToFull' => - * array ( - * 0 => - * array ( - * 'Email' => '370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com', - * 'Name' => NULL, - * 'MailboxHash' => NULL, - * ), - * ), - * 'Cc' => NULL, - * 'CcFull' => - * array ( - * ), - * 'Bcc' => NULL, - * 'BccFull' => - * array ( - * ), - * 'OriginalRecipient' => '370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com', - * 'Subject' => 'Re: adaw', - * 'MessageID' => 'd37fde00-b4cf-4b64-ac64-e9f6da523c25', - * 'ReplyTo' => NULL, - * 'MailboxHash' => NULL, - * 'Date' => 'Sun, 24 Mar 2024 13:17:52 +0100', - * 'TextBody' => 'wadwad - * - * Am So., 24. März 2024 um 13:17 Uhr schrieb Max Mustermann : - * - * > test - * > - * - * -- - * test.de - Max Mustermann kontakt@test.de - * ', - * 'HtmlBody' => '
wadwad

Am So., 24. März 2024 um 13:17 Uhr schrieb Max Mustermann <max@mustermann.de>:
test
- *
- * - *
- * test.de - Max Mustermann
kontakt@test.de
', - * 'StrippedTextReply' => 'wadwad - * - * Am So., 24. März 2024 um 13:17 Uhr schrieb Max Mustermann :', - * 'Tag' => NULL, - * 'Headers' => - * array ( - * 0 => - * array ( - * 'Name' => 'Return-Path', - * 'Value' => '', - * ), - * 1 => - * array ( - * 'Name' => 'Received', - * 'Value' => 'by p-pm-inboundg02a-aws-euwest1a.inbound.postmarkapp.com (Postfix, from userid 996) id 8ED1A453CA4; Sun, 24 Mar 2024 12:18:10 +0000 (UTC)', - * ), - * 2 => - * array ( - * 'Name' => 'X-Spam-Checker-Version', - * 'Value' => 'SpamAssassin 3.4.0 (2014-02-07) on p-pm-inboundg02a-aws-euwest1a', - * ), - * 3 => - * array ( - * 'Name' => 'X-Spam-Status', - * 'Value' => 'No', - * ), - * 4 => - * array ( - * 'Name' => 'X-Spam-Score', - * 'Value' => '-0.1', - * ), - * 5 => - * array ( - * 'Name' => 'X-Spam-Tests', - * 'Value' => 'DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,HTML_MESSAGE, RCVD_IN_DNSWL_NONE,RCVD_IN_MSPIKE_H2,RCVD_IN_ZEN_BLOCKED_OPENDNS, SPF_HELO_NONE,SPF_PASS,URIBL_DBL_BLOCKED_OPENDNS,URIBL_ZEN_BLOCKED_OPENDNS', - * ), - * 6 => - * array ( - * 'Name' => 'Received-SPF', - * 'Value' => 'pass (test.de: Sender is authorized to use \'max@mustermann.de\' in \'mfrom\' identity (mechanism \'include:_spf.google.com\' matched)) receiver=p-pm-inboundg02a-aws-euwest1a; identity=mailfrom; envelope-from="max@mustermann.de"; helo=mail-lf1-f51.google.com; client-ip=209.85.167.51', - * ), - * 7 => - * array ( - * 'Name' => 'Received', - * 'Value' => 'from mail-lf1-f51.google.com (mail-lf1-f51.google.com [209.85.167.51]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by p-pm-inboundg02a-aws-euwest1a.inbound.postmarkapp.com (Postfix) with ESMTPS id 437BD453CA2 for <370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com>; Sun, 24 Mar 2024 12:18:10 +0000 (UTC)', - * ), - * 8 => - * array ( - * 'Name' => 'Received', - * 'Value' => 'by mail-lf1-f51.google.com with SMTP id 2adb3069b0e04-513cf9bacf1so4773866e87.0 for <370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com>; Sun, 24 Mar 2024 05:18:10 -0700 (PDT)', - * ), - * 9 => - * array ( - * 'Name' => 'DKIM-Signature', - * 'Value' => 'v=1; a=rsa-sha256; c=relaxed/relaxed; d=test.de; s=google; t=1711282689; x=1711887489; darn=inbound.postmarkapp.com; h=to:subject:message-id:date:from:in-reply-to:references:mime-version :from:to:cc:subject:date:message-id:reply-to; bh=NvjmqLXF/5L5ZrpToR/6FgVOhTOGC9j0/B2Na5Ke6J8=; b=AMXIEoh6yGrOT6X3eBBClQ3NXFNuEoqxeM6aPONsqbpShAcT24iAJmqXylaLHv3fyX Hm6mwp3a029NnrLP/VRyKZbzIMBN2iycidtrEMXF/Eg2e42Q/08/2dZ7nxH6NqE/jz01 3M7qvwHvuoZ2Knhj7rnZc6I5m/nFxBsZc++Aj0Vv9sFoWZZooqAeTXbux1I5NyE17MrL D6byca43iINARZN7XOkoChRRZoZbOqZEtc2Va5yw7v+aYguLB4HHrIFC7G+L8hAJ0IAo 3R3DFeBw58M1xtxXCREI8Y6qMQTw60XyFw0gVmZzqR4hZiTerBSJJsZLZOBgmXxq3WLS +xVQ==', - * ), - * 10 => - * array ( - * 'Name' => 'X-Google-DKIM-Signature', - * 'Value' => 'v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1711282689; x=1711887489; h=to:subject:message-id:date:from:in-reply-to:references:mime-version :x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=NvjmqLXF/5L5ZrpToR/6FgVOhTOGC9j0/B2Na5Ke6J8=; b=uKoMhir+MX/wycNEr29Sffj45ooKksCJ1OfSRkIIGHk0rnHn8Vh+c7beYipwRPW4F2 h46K64vtIX00guYMdL2Qo2eY96+wALTqHCy67PGhvotVTROz21yxjx62pCDPGs5tefOu IkyxoybpIK8zAfLoDTd9p2GIrr5brKJyB2w1NQc1htxTQ5D4RgBxUAOKv4uVEr8r47iA MIo5d8/AifA+vCOAh7iJ7EmvDQ1R+guhQyH9m1Jo8PLapiYuHXggpBJvooyGuflKqbnt gJ/dscEr4d5aWJbw/x1dmIJ5gyJPGdBWq8NRqV/qbkXQW3H/gylifDUPXbki+EQBD5Yu EuLQ==', - * ), - * 11 => - * array ( - * 'Name' => 'X-Gm-Message-State', - * 'Value' => 'AOJu0Yxpbp1sRh17lNzg+pLnIx1jCn8ZFJQMgFuHK+6Z8RqFS5KKKTxR 8onpEbxWYYVUbrJFExNBHPD/3jdxqifCVVNaDmbpwHgmW5lHLJmA5vYRq5NFZ9OA6zKx/N6Gipr iXE4fXmSqghFNTzy9V/RT08Zp+F5RiFh/Ta6ltQl8XfCPFfSawLz6cagUgt8bBuF4RqdrYmWwzj ty86V5Br1htRNEFYivoXnNmaRcsD0tca1D23ny62O6RwWugrj1IpAYhViNyTZAWu+loKgfjJJoI MsyiSU=', - * ), - * 12 => - * array ( - * 'Name' => 'X-Google-Smtp-Source', - * 'Value' => 'AGHT+IEdtZqbVI6j7WLeaSL3dABGSnWIXaSjbYqXvFvE2H+f2zsn0gknQ4OdTJecQRCabpypVF2ue91Jb7aKl6RiyEQ=', - * ), - * 13 => - * array ( - * 'Name' => 'X-Received', - * 'Value' => 'by 2002:a19:385a:0:b0:513:c876:c80a with SMTP id d26-20020a19385a000000b00513c876c80amr2586776lfj.34.1711282689140; Sun, 24 Mar 2024 05:18:09 -0700 (PDT)', - * ), - * 14 => - * array ( - * 'Name' => 'MIME-Version', - * 'Value' => '1.0', - * ), - * 15 => - * array ( - * 'Name' => 'References', - * 'Value' => '', - * ), - * 16 => - * array ( - * 'Name' => 'In-Reply-To', - * 'Value' => '', - * ), - * 17 => - * array ( - * 'Name' => 'Message-ID', - * 'Value' => '', - * ), - * ), - * 'Attachments' => - * array ( - * ), - * ) - * @return void - */ - public function handle() - { - $recipient = explode("|", $this->input)[0]; - - // match company - $company = MultiDB::findAndSetDbByInboundMailbox($recipient); - if (!$company) { - Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $recipient); - return; - } - - // fetch message from postmark-api - $company_postmark_secret = $company->settings?->email_sending_method === 'client_postmark' && $company->settings?->postmark_secret ? $company->settings?->postmark_secret : null; - if (!($company_postmark_secret) && !(config('services.postmark.domain') && config('services.postmark.secret'))) - throw new \Error("[ProcessMailgunInboundWebhook] no postmark credenitals found, we cannot get the attachements and files"); - - $mail = null; - if ($company_postmark_secret) { - - $credentials = $company_postmark_domain . ":" . $company_postmark_secret . "@"; - $messageUrl = explode("|", $this->input)[1]; - $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); - $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); - - try { - $mail = json_decode(file_get_contents($messageUrl)); - } catch (\Error $e) { - if (config('services.postmark.secret')) { - Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); - - $credentials = config('services.postmark.domain') . ":" . config('services.postmark.secret') . "@"; - $messageUrl = explode("|", $this->input)[1]; - $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); - $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); - $mail = json_decode(file_get_contents($messageUrl)); - - } else - throw $e; - } - - } else { - - $credentials = config('services.postmark.domain') . ":" . config('services.postmark.secret') . "@"; - $messageUrl = explode("|", $this->input)[1]; - $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); - $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); - $mail = json_decode(file_get_contents($messageUrl)); - - } - - // prepare data for ingresEngine - $inboundMail = new InboundMail(); - - $inboundMail->from = $mail->sender; - $inboundMail->to = $recipient; // usage of data-input, because we need a single email here - $inboundMail->subject = $mail->Subject; - $inboundMail->body = $mail->{"body-html"}; - $inboundMail->text_body = $mail->{"body-plain"}; - $inboundMail->date = Carbon::createFromTimeString($mail->Date); - - // parse documents as UploadedFile from webhook-data - foreach ($mail->attachments as $attachment) { // prepare url with credentials before downloading :: https://github.com/postmark/postmark.js/issues/24 - - // download file and save to tmp dir - if ($company_postmark_domain && $company_postmark_secret) { - - try { - - $credentials = $company_postmark_domain . ":" . $company_postmark_secret . "@"; - $url = $attachment->url; - $url = str_replace("http://", "http://" . $credentials, $url); - $url = str_replace("https://", "https://" . $credentials, $url); - $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); - - } catch (\Error $e) { - if (config('services.postmark.secret')) { - Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); - - $credentials = config('services.postmark.domain') . ":" . config('services.postmark.secret') . "@"; - $url = $attachment->url; - $url = str_replace("http://", "http://" . $credentials, $url); - $url = str_replace("https://", "https://" . $credentials, $url); - $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); - - } else - throw $e; - } - - } else { - - $credentials = config('services.postmark.domain') . ":" . config('services.postmark.secret') . "@"; - $url = $attachment->url; - $url = str_replace("http://", "http://" . $credentials, $url); - $url = str_replace("https://", "https://" . $credentials, $url); - $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); - - } - - } - - // perform - (new InboundMailEngine($inboundMail))->handle(); - } -} diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index 1c6b2f6f0fcd..0383fbf80d3b 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -15,7 +15,6 @@ use App\Events\Expense\ExpenseWasCreated; use App\Factory\ExpenseFactory; use App\Jobs\Util\SystemLogger; use App\Libraries\MultiDB; -use App\Models\Client; use App\Models\ClientContact; use App\Models\Company; use App\Models\SystemLog; @@ -40,154 +39,146 @@ class InboundMailEngine private ?bool $isUnknownRecipent = null; private array $globalBlacklistDomains = []; private array $globalBlacklistSenders = []; - public function __construct(private InboundMail $email) + public function __construct() { } /** * if there is not a company with an matching mailbox, we only do monitoring * reuse this method to add more mail-parsing behaviors */ - public function handle() + public function handle(InboundMail $email) { - if ($this->isInvalidOrBlocked()) + if ($this->isInvalidOrBlocked($email)) return; - $this->isUnknownRecipent = true; + $isUnknownRecipent = true; // Expense Mailbox => will create an expense - $this->company = MultiDB::findAndSetDbByInboundMailbox($this->email->to); - if ($this->company) { - $this->isUnknownRecipent = false; - $this->createExpense(); + $company = MultiDB::findAndSetDbByInboundMailbox($email->to); + if ($company) { + $isUnknownRecipent = false; + $this->createExpense($company, $email); } - $this->saveMeta(); + $this->saveMeta($email->from, $email->to, $isUnknownRecipent); } // SPAM Protection - private function isInvalidOrBlocked() + private function isInvalidOrBlocked(InboundMail $email) { // invalid email - if (!filter_var($this->email->from, FILTER_VALIDATE_EMAIL)) { - $this->logBlocked('E-Mail blocked, because from e-mail has the wrong format: ' . $this->email->from); + if (!filter_var($email->from, FILTER_VALIDATE_EMAIL)) { + Log::info('E-Mail blocked, because from e-mail has the wrong format: ' . $email->from); return true; } - $parts = explode('@', $this->email->from); + $parts = explode('@', $email->from); $domain = array_pop($parts); // global blacklist if (in_array($domain, $this->globalBlacklistDomains)) { - $this->logBlocked('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $this->email->from); + Log::info('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $email->from); return true; } - if (in_array($this->email->from, $this->globalBlacklistSenders)) { - $this->logBlocked('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $this->email->from); + if (in_array($email->from, $this->globalBlacklistSenders)) { + Log::info('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $email->from); return true; } - if (Cache::has('inboundMailBlockedSender:' . $this->email->from)) { // was marked as blocked before, so we block without any console output + if (Cache::has('inboundMailBlockedSender:' . $email->from)) { // was marked as blocked before, so we block without any console output return true; } // sender occured in more than 500 emails in the last 12 hours - $senderMailCountTotal = Cache::get('inboundMailSender:' . $this->email->from, 0); + $senderMailCountTotal = Cache::get('inboundMailSender:' . $email->from, 0); if ($senderMailCountTotal >= 5000) { - $this->logBlocked('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); - $this->blockSender(); + Log::info('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $email->from); + $this->blockSender($email->from); + $this->saveMeta($email->from, $email->to); return true; } if ($senderMailCountTotal >= 1000) { - $this->logBlocked('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); - $this->saveMeta(); + Log::info('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $email->from); + $this->saveMeta($email->from, $email->to); return true; } // sender sended more than 50 emails to the wrong mailbox in the last 6 hours - $senderMailCountUnknownRecipent = Cache::get('inboundMailSenderUnknownRecipent:' . $this->email->from, 0); + $senderMailCountUnknownRecipent = Cache::get('inboundMailSenderUnknownRecipent:' . $email->from, 0); if ($senderMailCountUnknownRecipent >= 50) { - $this->logBlocked('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $this->email->from); - $this->saveMeta(); + Log::info('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $email->from); + $this->saveMeta($email->from, $email->to); return true; } // wrong recipent occurs in more than 100 emails in the last 12 hours, so the processing is blocked - $mailCountUnknownRecipent = Cache::get('inboundMailUnknownRecipent:' . $this->email->to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time + $mailCountUnknownRecipent = Cache::get('inboundMailUnknownRecipent:' . $email->to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time if ($mailCountUnknownRecipent >= 100) { - $this->logBlocked('E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); - $this->blockSender(); + Log::info('E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $email->from); + $this->blockSender($email->from); + $this->saveMeta($email->from, $email->to); return true; } return false; } - private function blockSender() + public function blockSender(string $from) { - Cache::add('inboundMailBlockedSender:' . $this->email->from, true, now()->addHours(12)); - $this->saveMeta(); + Cache::add('inboundMailBlockedSender:' . $from, true, now()->addHours(12)); // TODO: ignore, when known sender (for heavy email-usage mostly on isHosted()) // TODO: handle external blocking } - private function saveMeta() + public function saveMeta(string $from, string $to, bool $isUnknownRecipent = false) { // save cache - Cache::add('inboundMailSender:' . $this->email->from, 0, now()->addHours(12)); - Cache::increment('inboundMailSender:' . $this->email->from); + Cache::add('inboundMailSender:' . $from, 0, now()->addHours(12)); + Cache::increment('inboundMailSender:' . $from); - if ($this->isUnknownRecipent) { - Cache::add('inboundMailSenderUnknownRecipent:' . $this->email->from, 0, now()->addHours(6)); - Cache::increment('inboundMailSenderUnknownRecipent:' . $this->email->from); // we save the sender, to may block him + if ($isUnknownRecipent) { + Cache::add('inboundMailSenderUnknownRecipent:' . $from, 0, now()->addHours(6)); + Cache::increment('inboundMailSenderUnknownRecipent:' . $from); // we save the sender, to may block him - Cache::add('inboundMailUnknownRecipent:' . $this->email->to, 0, now()->addHours(12)); - Cache::increment('inboundMailUnknownRecipent:' . $this->email->to); // we save the sender, to may block him + Cache::add('inboundMailUnknownRecipent:' . $to, 0, now()->addHours(12)); + Cache::increment('inboundMailUnknownRecipent:' . $to); // we save the sender, to may block him } } - // MAIL-PARSING - private function processHtmlBodyToDocument() - { - - if ($this->email->body !== null) - $this->email->body_document = TempFile::UploadedFileFromRaw($this->email->body, "E-Mail.html", "text/html"); - - } - // MAIN-PROCESSORS - protected function createExpense() + protected function createExpense(Company $company, InboundMail $email) { // Skipping executions: will not result in not saving Metadata to prevent usage of these conditions, to spam - if (!$this->validateExpenseShouldProcess()) { - $this->logBlocked('mailbox not active for this company. from: ' . $this->email->from); + if (!($company?->inbound_mailbox_active ?: false)) { + $this->logBlocked($company, 'mailbox not active for this company. from: ' . $email->from); return; } - if (!$this->validateExpenseSender()) { - $this->logBlocked('invalid sender of an ingest email for this company. from: ' . $this->email->from); + if (!$this->validateExpenseSender($company, $email)) { + $this->logBlocked($company, 'invalid sender of an ingest email for this company. from: ' . $email->from); return; } - if (sizeOf($this->email->documents) == 0) { - $this->logBlocked('email does not contain any attachments and is likly not an expense. from: ' . $this->email->from); + if (sizeOf($email->documents) == 0) { + $this->logBlocked($company, 'email does not contain any attachments and is likly not an expense. from: ' . $email->from); return; } // create expense - $expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id); + $expense = ExpenseFactory::create($company->id, $company->owner()->id); - $expense->public_notes = $this->email->subject; - $expense->private_notes = $this->email->text_body; - $expense->date = $this->email->date; + $expense->public_notes = $email->subject; + $expense->private_notes = $email->text_body; + $expense->date = $email->date; // handle vendor assignment - $expense_vendor = $this->getVendor(); + $expense_vendor = $this->getVendor($company, $email); if ($expense_vendor) $expense->vendor_id = $expense_vendor->id; // handle documents - $this->processHtmlBodyToDocument(); + $this->processHtmlBodyToDocument($email); $documents = []; - array_push($documents, ...$this->email->documents); - if ($this->email->body_document !== null) - array_push($documents, $this->email->body_document); + array_push($documents, ...$email->documents); + if ($email->body_document !== null) + array_push($documents, $email->body_document); $expense->saveQuietly(); @@ -198,78 +189,81 @@ class InboundMailEngine } // HELPERS - private function validateExpenseShouldProcess() + private function processHtmlBodyToDocument(InboundMail $email) { - return $this->company?->inbound_mailbox_active ?: false; + + if ($email->body !== null) + $email->body_document = TempFile::UploadedFileFromRaw($email->body, "E-Mail.html", "text/html"); + } - private function validateExpenseSender() + private function validateExpenseSender(Company $company, InboundMail $email) { - $parts = explode('@', $this->email->from); + $parts = explode('@', $email->from); $domain = array_pop($parts); // whitelists - $email_whitelist = explode(",", $this->company->inbound_mailbox_whitelist_senders); - if (in_array($this->email->from, $email_whitelist)) + $email_whitelist = explode(",", $company->inbound_mailbox_whitelist_senders); + if (in_array($email->from, $email_whitelist)) return true; - $domain_whitelist = explode(",", $this->company->inbound_mailbox_whitelist_domains); + $domain_whitelist = explode(",", $company->inbound_mailbox_whitelist_domains); if (in_array($domain, $domain_whitelist)) return true; - $email_blacklist = explode(",", $this->company->inbound_mailbox_blacklist_senders); - if (in_array($this->email->from, $email_blacklist)) + $email_blacklist = explode(",", $company->inbound_mailbox_blacklist_senders); + if (in_array($email->from, $email_blacklist)) return false; - $domain_blacklist = explode(",", $this->company->inbound_mailbox_blacklist_domains); + $domain_blacklist = explode(",", $company->inbound_mailbox_blacklist_domains); if (in_array($domain, $domain_blacklist)) return false; // allow unknown - if ($this->company->inbound_mailbox_allow_unknown) + if ($company->inbound_mailbox_allow_unknown) return true; // own users - if ($this->company->inbound_mailbox_allow_company_users && $this->company->users()->where("email", $this->email->from)->exists()) + if ($company->inbound_mailbox_allow_company_users && $company->users()->where("email", $email->from)->exists()) return true; // from vendors - if ($this->company->inbound_mailbox_allow_vendors && $this->company->vendors()->where("invoicing_email", $this->email->from)->orWhere("invoicing_domain", $domain)->exists()) + if ($company->inbound_mailbox_allow_vendors && $company->vendors()->where("invoicing_email", $email->from)->orWhere("invoicing_domain", $domain)->exists()) return true; - if ($this->company->inbound_mailbox_allow_vendors && $this->company->vendors()->contacts()->where("email", $this->email->from)->exists()) + if ($company->inbound_mailbox_allow_vendors && $company->vendors()->contacts()->where("email", $email->from)->exists()) return true; // from clients - if ($this->company->inbound_mailbox_allow_clients && $this->company->clients()->contacts()->where("email", $this->email->from)->exists()) + if ($company->inbound_mailbox_allow_clients && $company->clients()->contacts()->where("email", $email->from)->exists()) return true; // denie return false; } - private function getClient() + private function getClient(Company $company, InboundMail $email) { - // $parts = explode('@', $this->email->from); + // $parts = explode('@', $email->from); // $domain = array_pop($parts); - $clientContact = ClientContact::where("company_id", $this->company->id)->where("email", $this->email->from)->first(); + $clientContact = ClientContact::where("company_id", $company->id)->where("email", $email->from)->first(); $client = $clientContact->client(); return $client; } - private function getVendor() + private function getVendor(Company $company, InboundMail $email) { - $parts = explode('@', $this->email->from); + $parts = explode('@', $email->from); $domain = array_pop($parts); - $vendor = Vendor::where("company_id", $this->company->id)->where('invoicing_email', $this->email->from)->first(); + $vendor = Vendor::where("company_id", $company->id)->where('invoicing_email', $email->from)->first(); if ($vendor == null) - $vendor = Vendor::where("company_id", $this->company->id)->where("invoicing_domain", $domain)->first(); + $vendor = Vendor::where("company_id", $company->id)->where("invoicing_domain", $domain)->first(); if ($vendor == null) { - $vendorContact = VendorContact::where("company_id", $this->company->id)->where("email", $this->email->from)->first(); + $vendorContact = VendorContact::where("company_id", $company->id)->where("email", $email->from)->first(); $vendor = $vendorContact->vendor(); } return $vendor; } - private function logBlocked(string $data) + private function logBlocked(Company $company, string $data) { - Log::info("[InboundMailEngine][company:" . $this->company->id . "] " . $data); + Log::info("[InboundMailEngine][company:" . $company->id . "] " . $data); ( new SystemLogger( @@ -278,7 +272,7 @@ class InboundMailEngine SystemLog::EVENT_INBOUND_MAIL_BLOCKED, SystemLog::TYPE_CUSTOM, null, - $this->company + $company ) )->handle(); } diff --git a/app/Utils/TempFile.php b/app/Utils/TempFile.php index 24df20da7271..5d88b911f175 100644 --- a/app/Utils/TempFile.php +++ b/app/Utils/TempFile.php @@ -40,7 +40,7 @@ class TempFile } /* create a tmp file from a base64 string: https://gist.github.com/waska14/8b3bcebfad1f86f7fcd3b82927576e38*/ - public static function UploadedFileFromBase64(string $base64File): UploadedFile + public static function UploadedFileFromBase64(string $base64File, string|null $fileName = null, string|null $mimeType = null): UploadedFile { // Get file data base64 string $fileData = base64_decode(Arr::last(explode(',', $base64File))); @@ -55,8 +55,8 @@ class TempFile $tempFileObject = new File($tempFilePath); $file = new UploadedFile( $tempFileObject->getPathname(), - $tempFileObject->getFilename(), - $tempFileObject->getMimeType(), + $fileName ?: $tempFileObject->getFilename(), + $mimeType ?: $tempFileObject->getMimeType(), 0, true // Mark it as test, since the file isn't from real HTTP POST. );