From 157037f56fc83926545517ba0dec7ff25c507517 Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 25 Mar 2024 06:41:22 +0100 Subject: [PATCH] renamings + move imap job to self-hosted only + allow by clients --- app/Console/Kernel.php | 8 +- .../Transformer/ImapMailTransformer.php | 20 +-- .../PostmarkInboundWebhookTransformer.php | 118 ------------------ .../Requests/Company/StoreCompanyRequest.php | 24 ++-- .../Requests/Company/UpdateCompanyRequest.php | 26 ++-- .../Company/ValidExpenseMailbox.php | 14 +-- app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 28 ++--- .../ProcessImapMailboxJob.php} | 22 ++-- .../Mailgun/ProcessMailgunInboundWebhook.php | 28 ++--- .../ProcessPostmarkInboundWebhook.php | 28 ++--- app/Libraries/MultiDB.php | 40 ++++-- app/Models/Company.php | 38 +++--- app/Models/SystemLog.php | 2 +- .../InboundMail.php} | 4 +- .../InboundMailEngine.php} | 107 +++++++++------- app/Transformers/CompanyTransformer.php | 11 +- config/ninja.php | 28 ++--- ...10951_create_imap_configuration_fields.php | 19 +-- lang/en/texts.php | 4 +- 19 files changed, 253 insertions(+), 316 deletions(-) rename app/Helpers/{IngresMail => InboundMail}/Transformer/ImapMailTransformer.php (56%) delete mode 100644 app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php rename app/Jobs/{Mail/ExpenseMailboxJob.php => Imap/ProcessImapMailboxJob.php} (89%) rename app/Services/{IngresEmail/IngresEmail.php => InboundMail/InboundMail.php} (93%) rename app/Services/{IngresEmail/IngresEmailEngine.php => InboundMail/InboundMailEngine.php} (62%) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index ebd7cfc4c215..9a37eaf755f4 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -17,7 +17,7 @@ use App\Jobs\Cron\RecurringInvoicesCron; use App\Jobs\Cron\SubscriptionCron; use App\Jobs\Cron\UpdateCalculatedFields; use App\Jobs\Invoice\InvoiceCheckLateWebhook; -use App\Jobs\Mail\ExpenseMailboxJob; +use App\Jobs\Imap\ProcessImapMailboxJob; use App\Jobs\Ninja\AdjustEmailQuota; use App\Jobs\Ninja\BankTransactionSync; use App\Jobs\Ninja\CheckACHStatus; @@ -98,9 +98,6 @@ class Kernel extends ConsoleKernel /* Fires webhooks for overdue Invoice */ $schedule->job(new InvoiceCheckLateWebhook())->dailyAt('07:00')->withoutOverlapping()->name('invoice-overdue-job')->onOneServer(); - /* Check ExpenseMainboxes */ - $schedule->job(new ExpenseMailboxJob)->everyThirtyMinutes()->withoutOverlapping()->name('expense-mailboxes-job')->onOneServer(); - /* Pulls in bank transactions from third party services */ $schedule->job(new BankTransactionSync())->everyFourHours()->withoutOverlapping()->name('bank-trans-sync-job')->onOneServer(); @@ -108,6 +105,9 @@ class Kernel extends ConsoleKernel $schedule->call(function () { Account::query()->whereNotNull('id')->update(['is_scheduler_running' => true]); })->everyFiveMinutes(); + + /* Check ImapMailboxes */ + $schedule->job(new ProcessImapMailboxJob)->everyFiveMinutes()->withoutOverlapping()->name('imap-mailbox-job')->onOneServer(); } /* Run hosted specific jobs */ diff --git a/app/Helpers/IngresMail/Transformer/ImapMailTransformer.php b/app/Helpers/InboundMail/Transformer/ImapMailTransformer.php similarity index 56% rename from app/Helpers/IngresMail/Transformer/ImapMailTransformer.php rename to app/Helpers/InboundMail/Transformer/ImapMailTransformer.php index 80889b3afc9c..4229d7fb5e79 100644 --- a/app/Helpers/IngresMail/Transformer/ImapMailTransformer.php +++ b/app/Helpers/InboundMail/Transformer/ImapMailTransformer.php @@ -9,9 +9,9 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Helpers\IngresMail\Transformer; +namespace App\Helpers\InboundMail\Transformer; -use App\Services\IngresEmail\IngresEmail; +use App\Services\InboundMail\InboundMail; use App\Utils\TempFile; use Ddeboer\Imap\MessageInterface; @@ -20,19 +20,19 @@ class ImapMailTransformer public function transform(MessageInterface $mail) { - $ingresEmail = new IngresEmail(); + $inboundMail = new InboundMail(); - $ingresEmail->from = $mail->getSender(); - $ingresEmail->subject = $mail->getSubject(); - $ingresEmail->plain_message = $mail->getBodyText(); - $ingresEmail->html_message = $mail->getBodyHtml(); - $ingresEmail->date = $mail->getDate(); + $inboundMail->from = $mail->getSender(); + $inboundMail->subject = $mail->getSubject(); + $inboundMail->plain_message = $mail->getBodyText(); + $inboundMail->html_message = $mail->getBodyHtml(); + $inboundMail->date = $mail->getDate(); // parse documents as UploadedFile foreach ($mail->getAttachments() as $attachment) { - $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment->getContent(), $attachment->getFilename(), $attachment->getEncoding()); + $inboundMail->documents[] = TempFile::UploadedFileFromRaw($attachment->getContent(), $attachment->getFilename(), $attachment->getEncoding()); } - return $ingresEmail; + return $inboundMail; } } diff --git a/app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php b/app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php deleted file mode 100644 index 0ad0faf71963..000000000000 --- a/app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php +++ /dev/null @@ -1,118 +0,0 @@ -from = $data["From"]; - $ingresEmail->subject = $data["Subject"]; - $ingresEmail->plain_message = $data["TextBody"]; - $ingresEmail->html_message = $data["HtmlBody"]; - $ingresEmail->date = $data["Date"]; // TODO: parsing - - // parse documents as UploadedFile from webhook-data - foreach ($data["Attachments"] as $attachment) { - $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]); - } - - return $ingresEmail; - - } - // { - // "FromName": "Postmarkapp Support", - // "MessageStream": "inbound", - // "From": "support@postmarkapp.com", - // "FromFull": { - // "Email": "support@postmarkapp.com", - // "Name": "Postmarkapp Support", - // "MailboxHash": "" - // }, - // "To": "\"Firstname Lastname\" ", - // "ToFull": [ - // { - // "Email": "yourhash+SampleHash@inbound.postmarkapp.com", - // "Name": "Firstname Lastname", - // "MailboxHash": "SampleHash" - // } - // ], - // "Cc": "\"First Cc\" , secondCc@postmarkapp.com>", - // "CcFull": [ - // { - // "Email": "firstcc@postmarkapp.com", - // "Name": "First Cc", - // "MailboxHash": "" - // }, - // { - // "Email": "secondCc@postmarkapp.com", - // "Name": "", - // "MailboxHash": "" - // } - // ], - // "Bcc": "\"First Bcc\" , secondbcc@postmarkapp.com>", - // "BccFull": [ - // { - // "Email": "firstbcc@postmarkapp.com", - // "Name": "First Bcc", - // "MailboxHash": "" - // }, - // { - // "Email": "secondbcc@postmarkapp.com", - // "Name": "", - // "MailboxHash": "" - // } - // ], - // "OriginalRecipient": "yourhash+SampleHash@inbound.postmarkapp.com", - // "Subject": "Test subject", - // "MessageID": "73e6d360-66eb-11e1-8e72-a8904824019b", - // "ReplyTo": "replyto@postmarkapp.com", - // "MailboxHash": "SampleHash", - // "Date": "Fri, 1 Aug 2014 16:45:32 -04:00", - // "TextBody": "This is a test text body.", - // "HtmlBody": "

This is a test html body.<\/p><\/body><\/html>", - // "StrippedTextReply": "This is the reply text", - // "Tag": "TestTag", - // "Headers": [ - // { - // "Name": "X-Header-Test", - // "Value": "" - // }, - // { - // "Name": "X-Spam-Status", - // "Value": "No" - // }, - // { - // "Name": "X-Spam-Score", - // "Value": "-0.1" - // }, - // { - // "Name": "X-Spam-Tests", - // "Value": "DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,SPF_PASS" - // } - // ], - // "Attachments": [ - // { - // "Name": "test.txt", - // "Content": "VGhpcyBpcyBhdHRhY2htZW50IGNvbnRlbnRzLCBiYXNlLTY0IGVuY29kZWQu", - // "ContentType": "text/plain", - // "ContentLength": 45 - // } - // ] - // } -} diff --git a/app/Http/Requests/Company/StoreCompanyRequest.php b/app/Http/Requests/Company/StoreCompanyRequest.php index fa3563db5d71..3b4725118a4d 100644 --- a/app/Http/Requests/Company/StoreCompanyRequest.php +++ b/app/Http/Requests/Company/StoreCompanyRequest.php @@ -47,7 +47,7 @@ class StoreCompanyRequest extends Request $rules['company_logo'] = 'mimes:jpeg,jpg,png,gif|max:10000'; // max 10000kb $rules['settings'] = new ValidSettingsRule(); - if (isset($input['portal_mode']) && ($input['portal_mode'] == 'domain' || $input['portal_mode'] == 'iframe')) { + if (isset ($input['portal_mode']) && ($input['portal_mode'] == 'domain' || $input['portal_mode'] == 'iframe')) { $rules['portal_domain'] = 'sometimes|url'; } else { if (Ninja::isHosted()) { @@ -57,7 +57,7 @@ class StoreCompanyRequest extends Request } } - $rules['expense_mailbox'] = new ValidExpenseMailbox($this->company->key); + $rules['inbound_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); $rules['smtp_host'] = 'sometimes|string|nullable'; $rules['smtp_port'] = 'sometimes|integer|nullable'; @@ -75,39 +75,39 @@ class StoreCompanyRequest extends Request { $input = $this->all(); - if (!isset($input['name'])) { + if (!isset ($input['name'])) { $input['name'] = 'Untitled Company'; } - if (isset($input['google_analytics_url'])) { + if (isset ($input['google_analytics_url'])) { $input['google_analytics_key'] = $input['google_analytics_url']; } - if (isset($input['portal_domain'])) { + if (isset ($input['portal_domain'])) { $input['portal_domain'] = rtrim(strtolower($input['portal_domain']), "/"); } - if (isset($input['expense_mailbox']) && Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise')) { - unset($input['expense_mailbox']); + if (isset ($input['inbound_mailbox']) && Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise')) { + unset($input['inbound_mailbox']); } - if (Ninja::isHosted() && !isset($input['subdomain'])) { + if (Ninja::isHosted() && !isset ($input['subdomain'])) { $input['subdomain'] = MultiDB::randomSubdomainGenerator(); } - if (isset($input['smtp_username']) && strlen(str_replace("*", "", $input['smtp_username'])) < 2) { + if (isset ($input['smtp_username']) && strlen(str_replace("*", "", $input['smtp_username'])) < 2) { unset($input['smtp_username']); } - if (isset($input['smtp_password']) && strlen(str_replace("*", "", $input['smtp_password'])) < 2) { + if (isset ($input['smtp_password']) && strlen(str_replace("*", "", $input['smtp_password'])) < 2) { unset($input['smtp_password']); } - if (isset($input['smtp_port'])) { + if (isset ($input['smtp_port'])) { $input['smtp_port'] = (int) $input['smtp_port']; } - if (isset($input['smtp_verify_peer']) && is_string($input['smtp_verify_peer'])) + if (isset ($input['smtp_verify_peer']) && is_string($input['smtp_verify_peer'])) $input['smtp_verify_peer'] == 'true' ? true : false; $this->replace($input); diff --git a/app/Http/Requests/Company/UpdateCompanyRequest.php b/app/Http/Requests/Company/UpdateCompanyRequest.php index 58c530c6a762..c8578d262ae1 100644 --- a/app/Http/Requests/Company/UpdateCompanyRequest.php +++ b/app/Http/Requests/Company/UpdateCompanyRequest.php @@ -66,7 +66,7 @@ class UpdateCompanyRequest extends Request // $rules['smtp_verify_peer'] = 'sometimes|string'; - if (isset($input['portal_mode']) && ($input['portal_mode'] == 'domain' || $input['portal_mode'] == 'iframe')) { + if (isset ($input['portal_mode']) && ($input['portal_mode'] == 'domain' || $input['portal_mode'] == 'iframe')) { $rules['portal_domain'] = 'bail|nullable|sometimes|url'; } @@ -74,7 +74,7 @@ class UpdateCompanyRequest extends Request $rules['subdomain'] = ['nullable', 'regex:/^[a-zA-Z0-9.-]+[a-zA-Z0-9]$/', new ValidSubdomain()]; } - $rules['expense_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); // @turbo124 check if this is right + $rules['inbound_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); // @turbo124 check if this is right return $rules; } @@ -83,36 +83,40 @@ class UpdateCompanyRequest extends Request { $input = $this->all(); - if (isset($input['portal_domain']) && strlen($input['portal_domain']) > 1) { + if (isset ($input['portal_domain']) && strlen($input['portal_domain']) > 1) { $input['portal_domain'] = $this->addScheme($input['portal_domain']); $input['portal_domain'] = rtrim(strtolower($input['portal_domain']), "/"); } - if (isset($input['settings'])) { + if (isset ($input['inbound_mailbox']) && Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise')) { + unset($input['inbound_mailbox']); + } + + if (isset ($input['settings'])) { $input['settings'] = (array) $this->filterSaveableSettings($input['settings']); } - if (isset($input['subdomain']) && $this->company->subdomain == $input['subdomain']) { + if (isset ($input['subdomain']) && $this->company->subdomain == $input['subdomain']) { unset($input['subdomain']); } - if (isset($input['e_invoice_certificate_passphrase']) && empty($input['e_invoice_certificate_passphrase'])) { + if (isset ($input['e_invoice_certificate_passphrase']) && empty ($input['e_invoice_certificate_passphrase'])) { unset($input['e_invoice_certificate_passphrase']); } - if (isset($input['smtp_username']) && strlen(str_replace("*", "", $input['smtp_username'])) < 2) { + if (isset ($input['smtp_username']) && strlen(str_replace("*", "", $input['smtp_username'])) < 2) { unset($input['smtp_username']); } - if (isset($input['smtp_password']) && strlen(str_replace("*", "", $input['smtp_password'])) < 2) { + if (isset ($input['smtp_password']) && strlen(str_replace("*", "", $input['smtp_password'])) < 2) { unset($input['smtp_password']); } - if (isset($input['smtp_port'])) { + if (isset ($input['smtp_port'])) { $input['smtp_port'] = (int) $input['smtp_port']; } - if (isset($input['smtp_verify_peer']) && is_string($input['smtp_verify_peer'])) { + if (isset ($input['smtp_verify_peer']) && is_string($input['smtp_verify_peer'])) { $input['smtp_verify_peer'] == 'true' ? true : false; } @@ -139,7 +143,7 @@ class UpdateCompanyRequest extends Request } } - if (isset($settings['email_style_custom'])) { + if (isset ($settings['email_style_custom'])) { $settings['email_style_custom'] = str_replace(['{!!', '!!}', '{{', '}}', '@if(', '@endif', '@isset', '@unless', '@auth', '@empty', '@guest', '@env', '@section', '@switch', '@foreach', '@while', '@include', '@each', '@once', '@push', '@use', '@forelse', '@verbatim', 'company_key = $company_key; - $this->endings = explode(",", config('ninja.ingest_mail.expense_mailbox_endings')); + $this->endings = explode(",", config('ninja.inbound_mailbox.inbound_mailbox_endings')); } public function passes($attribute, $value) { - if (empty($value)) { + if (empty ($value)) { return true; } // early return, if we dont have any additional validation - if (!config('ninja.ingest_mail.expense_mailbox_endings')) { + if (!config('ninja.inbound_mailbox.inbound_mailbox_endings')) { $this->validated_schema = true; - return MultiDB::checkExpenseMailboxAvailable($value); + return MultiDB::checkInboundMailboxAvailable($value); } // Validate Schema @@ -59,7 +59,7 @@ class ValidExpenseMailbox implements Rule return false; $this->validated_schema = true; - return MultiDB::checkExpenseMailboxAvailable($value); + return MultiDB::checkInboundMailboxAvailable($value); } /** @@ -68,8 +68,8 @@ class ValidExpenseMailbox implements Rule public function message() { if (!$this->validated_schema) - return ctrans('texts.expense_mailbox_invalid'); + return ctrans('texts.inbound_mailbox_invalid'); - return ctrans('texts.expense_mailbox_taken'); + return ctrans('texts.inbound_mailbox_taken'); } } diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index d97fb7245496..d8257351ca41 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -12,8 +12,8 @@ namespace App\Jobs\Brevo; use App\Libraries\MultiDB; -use App\Services\IngresEmail\IngresEmail; -use App\Services\IngresEmail\IngresEmailEngine; +use App\Services\InboundMail\InboundMail; +use App\Services\InboundMail\InboundMailEngine; use App\Utils\TempFile; use Brevo\Client\Api\InboundParsingApi; use Brevo\Client\Configuration; @@ -118,11 +118,11 @@ class ProcessBrevoInboundWebhook implements ShouldQueue public function handle() { - // brevo defines recipients as array, and we should check all of them, to be sure + // brevo defines recipients as array to enable webhook processing as batches, we check all of them foreach ($this->input["Recipients"] as $recipient) { // match company - $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); + $company = MultiDB::findAndSetDbByInboundMailbox($recipient); if (!$company) { Log::info('[ProcessBrevoInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from brevo: ' . $recipient); continue; @@ -133,14 +133,14 @@ class ProcessBrevoInboundWebhook implements ShouldQueue throw new \Error("[ProcessBrevoInboundWebhook] no brevo credenitals found, we cannot get the attachement"); // prepare data for ingresEngine - $ingresEmail = new IngresEmail(); + $inboundMail = new InboundMail(); - $ingresEmail->from = $this->input["From"]["Address"]; - $ingresEmail->to = $recipient; - $ingresEmail->subject = $this->input["Subject"]; - $ingresEmail->body = $this->input["RawHtmlBody"]; - $ingresEmail->text_body = $this->input["RawTextBody"]; - $ingresEmail->date = Carbon::createFromTimeString($this->input["SentAtDate"]); + $inboundMail->from = $this->input["From"]["Address"]; + $inboundMail->to = $recipient; + $inboundMail->subject = $this->input["Subject"]; + $inboundMail->body = $this->input["RawHtmlBody"]; + $inboundMail->text_body = $this->input["RawTextBody"]; + $inboundMail->date = Carbon::createFromTimeString($this->input["SentAtDate"]); // parse documents as UploadedFile from webhook-data foreach ($this->input["Attachments"] as $attachment) { @@ -164,18 +164,18 @@ class ProcessBrevoInboundWebhook implements ShouldQueue } else throw $e; } - $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment, $attachment["Name"], $attachment["ContentType"]); + $inboundMail->documents[] = TempFile::UploadedFileFromRaw($attachment, $attachment["Name"], $attachment["ContentType"]); } else { $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); - $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($brevo->getInboundEmailAttachment($attachment["DownloadToken"]), $attachment["Name"], $attachment["ContentType"]); + $inboundMail->documents[] = TempFile::UploadedFileFromRaw($brevo->getInboundEmailAttachment($attachment["DownloadToken"]), $attachment["Name"], $attachment["ContentType"]); } } - (new IngresEmailEngine($ingresEmail))->handle(); + (new InboundMailEngine($inboundMail))->handle(); } } diff --git a/app/Jobs/Mail/ExpenseMailboxJob.php b/app/Jobs/Imap/ProcessImapMailboxJob.php similarity index 89% rename from app/Jobs/Mail/ExpenseMailboxJob.php rename to app/Jobs/Imap/ProcessImapMailboxJob.php index 9f47582d26ee..e6aef5fd3d5e 100644 --- a/app/Jobs/Mail/ExpenseMailboxJob.php +++ b/app/Jobs/Imap/ProcessImapMailboxJob.php @@ -9,14 +9,14 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Jobs\Mail; +namespace App\Jobs\Imap; -use App\Helpers\IngresMail\Transformer\ImapMailTransformer; +use App\Helpers\InboundMail\Transformer\ImapMailTransformer; use App\Helpers\Mail\Mailbox\Imap\ImapMailbox; use App\Libraries\MultiDB; use App\Models\Company; use App\Repositories\ExpenseRepository; -use App\Services\IngresEmail\IngresEmailEngine; +use App\Services\InboundMail\InboundMailEngine; use App\Utils\Traits\MakesHash; use App\Utils\Traits\SavesDocuments; use Illuminate\Bus\Queueable; @@ -27,7 +27,7 @@ use Illuminate\Queue\SerializesModels; /*Multi Mailer implemented*/ -class ExpenseMailboxJob implements ShouldQueue +class ProcessImapMailboxJob implements ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, MakesHash, SavesDocuments; @@ -74,14 +74,14 @@ class ExpenseMailboxJob implements ShouldQueue private function getImapCredentials() { - $servers = array_map('trim', explode(",", config('ninja.ingest_mail.imap.servers'))); - $ports = array_map('trim', explode(",", config('ninja.ingest_mail.imap.ports'))); - $users = array_map('trim', explode(",", config('ninja.ingest_mail.imap.users'))); - $passwords = array_map('trim', explode(",", config('ninja.ingest_mail.imap.passwords'))); - $companies = array_map('trim', explode(",", config('ninja.ingest_mail.imap.companies'))); + $servers = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.servers'))); + $ports = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.ports'))); + $users = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.users'))); + $passwords = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.passwords'))); + $companies = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.companies'))); if (sizeOf($servers) != sizeOf($ports) || sizeOf($servers) != sizeOf($users) || sizeOf($servers) != sizeOf($passwords) || sizeOf($servers) != sizeOf($companies)) - throw new \Exception('invalid configuration ingest_mail.imap (wrong element-count)'); + throw new \Exception('invalid configuration inbound_mailbox.imap (wrong element-count)'); foreach ($companies as $index => $companyId) { @@ -116,7 +116,7 @@ class ExpenseMailboxJob implements ShouldQueue $email->markAsSeen(); - IngresEmailEngine::dispatch($transformer->transform($email)); + InboundMailEngine::dispatch($transformer->transform($email)); $imapMailbox->moveProcessed($email); diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index eac0e1bd04d2..4f11b6ba0d2b 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -12,8 +12,8 @@ namespace App\Jobs\Mailgun; use App\Libraries\MultiDB; -use App\Services\IngresEmail\IngresEmail; -use App\Services\IngresEmail\IngresEmailEngine; +use App\Services\InboundMail\InboundMail; +use App\Services\InboundMail\InboundMailEngine; use App\Utils\TempFile; use Illuminate\Support\Carbon; use Illuminate\Bus\Queueable; @@ -166,7 +166,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $recipient = explode("|", $this->input)[0]; // match company - $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); + $company = MultiDB::findAndSetDbByInboundMailbox($recipient); if (!$company) { Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $recipient); return; @@ -213,14 +213,14 @@ class ProcessMailgunInboundWebhook implements ShouldQueue } // prepare data for ingresEngine - $ingresEmail = new IngresEmail(); + $inboundMail = new InboundMail(); - $ingresEmail->from = $mail->sender; - $ingresEmail->to = $recipient; // usage of data-input, because we need a single email here - $ingresEmail->subject = $mail->Subject; - $ingresEmail->body = $mail->{"body-html"}; - $ingresEmail->text_body = $mail->{"body-plain"}; - $ingresEmail->date = Carbon::createFromTimeString($mail->Date); + $inboundMail->from = $mail->sender; + $inboundMail->to = $recipient; // usage of data-input, because we need a single email here + $inboundMail->subject = $mail->Subject; + $inboundMail->body = $mail->{"body-html"}; + $inboundMail->text_body = $mail->{"body-plain"}; + $inboundMail->date = Carbon::createFromTimeString($mail->Date); // parse documents as UploadedFile from webhook-data foreach ($mail->attachments as $attachment) { // prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 @@ -234,7 +234,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $url = $attachment->url; $url = str_replace("http://", "http://" . $credentials, $url); $url = str_replace("https://", "https://" . $credentials, $url); - $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); } catch (\Error $e) { if (config('services.mailgun.secret')) { @@ -244,7 +244,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $url = $attachment->url; $url = str_replace("http://", "http://" . $credentials, $url); $url = str_replace("https://", "https://" . $credentials, $url); - $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); } else throw $e; @@ -256,13 +256,13 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $url = $attachment->url; $url = str_replace("http://", "http://" . $credentials, $url); $url = str_replace("https://", "https://" . $credentials, $url); - $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); } } // perform - (new IngresEmailEngine($ingresEmail))->handle(); + (new InboundMailEngine($inboundMail))->handle(); } } diff --git a/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php b/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php index d616294a666c..5904b7c5b664 100644 --- a/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php +++ b/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php @@ -12,8 +12,8 @@ namespace App\Jobs\Postmark; use App\Libraries\MultiDB; -use App\Services\IngresEmail\IngresEmail; -use App\Services\IngresEmail\IngresEmailEngine; +use App\Services\InboundMail\InboundMail; +use App\Services\InboundMail\InboundMailEngine; use App\Utils\TempFile; use Illuminate\Support\Carbon; use Illuminate\Bus\Queueable; @@ -198,7 +198,7 @@ class ProcessPostmarkInboundWebhook implements ShouldQueue $recipient = explode("|", $this->input)[0]; // match company - $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); + $company = MultiDB::findAndSetDbByInboundMailbox($recipient); if (!$company) { Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $recipient); return; @@ -244,14 +244,14 @@ class ProcessPostmarkInboundWebhook implements ShouldQueue } // prepare data for ingresEngine - $ingresEmail = new IngresEmail(); + $inboundMail = new InboundMail(); - $ingresEmail->from = $mail->sender; - $ingresEmail->to = $recipient; // usage of data-input, because we need a single email here - $ingresEmail->subject = $mail->Subject; - $ingresEmail->body = $mail->{"body-html"}; - $ingresEmail->text_body = $mail->{"body-plain"}; - $ingresEmail->date = Carbon::createFromTimeString($mail->Date); + $inboundMail->from = $mail->sender; + $inboundMail->to = $recipient; // usage of data-input, because we need a single email here + $inboundMail->subject = $mail->Subject; + $inboundMail->body = $mail->{"body-html"}; + $inboundMail->text_body = $mail->{"body-plain"}; + $inboundMail->date = Carbon::createFromTimeString($mail->Date); // parse documents as UploadedFile from webhook-data foreach ($mail->attachments as $attachment) { // prepare url with credentials before downloading :: https://github.com/postmark/postmark.js/issues/24 @@ -265,7 +265,7 @@ class ProcessPostmarkInboundWebhook implements ShouldQueue $url = $attachment->url; $url = str_replace("http://", "http://" . $credentials, $url); $url = str_replace("https://", "https://" . $credentials, $url); - $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); } catch (\Error $e) { if (config('services.postmark.secret')) { @@ -275,7 +275,7 @@ class ProcessPostmarkInboundWebhook implements ShouldQueue $url = $attachment->url; $url = str_replace("http://", "http://" . $credentials, $url); $url = str_replace("https://", "https://" . $credentials, $url); - $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); } else throw $e; @@ -287,13 +287,13 @@ class ProcessPostmarkInboundWebhook implements ShouldQueue $url = $attachment->url; $url = str_replace("http://", "http://" . $credentials, $url); $url = str_replace("https://", "https://" . $credentials, $url); - $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); } } // perform - (new IngresEmailEngine($ingresEmail))->handle(); + (new InboundMailEngine($inboundMail))->handle(); } } diff --git a/app/Libraries/MultiDB.php b/app/Libraries/MultiDB.php index 3b7e5140da36..58362ab77dcd 100644 --- a/app/Libraries/MultiDB.php +++ b/app/Libraries/MultiDB.php @@ -73,7 +73,7 @@ class MultiDB 'socket', ]; - private static $protected_expense_mailboxes = []; + private static $protected_inbound_mailboxes = []; /** * @return array @@ -109,21 +109,21 @@ class MultiDB return true; } - public static function checkExpenseMailboxAvailable($expense_mailbox): bool + public static function checkInboundMailboxAvailable($inbound_mailbox): bool { if (!config('ninja.db.multi_db_enabled')) { - return Company::where("expense_mailbox", $expense_mailbox)->withTrashed()->exists(); + return Company::where("inbound_mailbox", $inbound_mailbox)->withTrashed()->exists(); } - if (in_array($expense_mailbox, self::$protected_expense_mailboxes)) { + if (in_array($inbound_mailbox, self::$protected_inbound_mailboxes)) { return false; } $current_db = config('database.default'); foreach (self::$dbs as $db) { - if (Company::on($db)->where("expense_mailbox", $expense_mailbox)->withTrashed()->exists()) { + if (Company::on($db)->where("inbound_mailbox", $inbound_mailbox)->withTrashed()->exists()) { self::setDb($current_db); return false; @@ -515,16 +515,16 @@ class MultiDB return false; } - public static function findAndSetDbByExpenseMailbox($expense_mailbox) + public static function findAndSetDbByInboundMailbox($inbound_mailbox) { if (!config('ninja.db.multi_db_enabled')) { - return Company::where("expense_mailbox", $expense_mailbox)->first(); + return Company::where("inbound_mailbox", $inbound_mailbox)->first(); } $current_db = config('database.default'); foreach (self::$dbs as $db) { - if ($company = Company::on($db)->where("expense_mailbox", $expense_mailbox)->first()) { + if ($company = Company::on($db)->where("inbound_mailbox", $inbound_mailbox)->first()) { self::setDb($db); return $company; @@ -587,7 +587,7 @@ class MultiDB $current_db = config('database.default'); - if(SMSNumbers::hasNumber($phone)){ + if (SMSNumbers::hasNumber($phone)) { return true; } @@ -615,8 +615,26 @@ class MultiDB $string = ''; $vowels = ['a', 'e', 'i', 'o', 'u', 'y']; $consonants = [ - 'b', 'c', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'm', - 'n', 'p', 'r', 's', 't', 'v', 'w', 'x', 'y', 'z', + 'b', + 'c', + 'd', + 'f', + 'g', + 'h', + 'j', + 'k', + 'l', + 'm', + 'n', + 'p', + 'r', + 's', + 't', + 'v', + 'w', + 'x', + 'y', + 'z', ]; $max = $length / 2; diff --git a/app/Models/Company.php b/app/Models/Company.php index a868093c6238..ab40b29960d0 100644 --- a/app/Models/Company.php +++ b/app/Models/Company.php @@ -111,15 +111,16 @@ use Laracasts\Presenter\PresentableTrait; * @property int $convert_expense_currency * @property int $notify_vendor_when_paid * @property int $invoice_task_hours - * @property string|null $expense_mailbox - * @property boolean $expense_mailbox_active - * @property bool $expense_mailbox_allow_company_users - * @property bool $expense_mailbox_allow_vendors - * @property bool $expense_mailbox_allow_unknown - * @property string|null $expense_mailbox_whitelist_domains - * @property string|null $expense_mailbox_whitelist_emails - * @property string|null $expense_mailbox_blacklist_domains - * @property string|null $expense_mailbox_blacklist_emails + * @property string|null $inbound_mailbox + * @property boolean $inbound_mailbox_active + * @property bool $inbound_mailbox_allow_company_users + * @property bool $inbound_mailbox_allow_vendors + * @property bool $inbound_mailbox_allow_clients + * @property bool $inbound_mailbox_allow_unknown + * @property string|null $inbound_mailbox_whitelist_domains + * @property string|null $inbound_mailbox_whitelist_senders + * @property string|null $inbound_mailbox_blacklist_domains + * @property string|null $inbound_mailbox_blacklist_senders * @property int $deleted_at * @property string $smtp_username * @property string $smtp_password @@ -368,15 +369,16 @@ class Company extends BaseModel 'calculate_taxes', 'tax_data', 'e_invoice_certificate_passphrase', - 'expense_mailbox_active', - 'expense_mailbox', // TODO: @turbo124 custom validation: self-hosted => free change, hosted => not changeable, only changeable with env-mask - 'expense_mailbox_allow_company_users', - 'expense_mailbox_allow_vendors', - 'expense_mailbox_allow_unknown', - 'expense_mailbox_whitelist_domains', - 'expense_mailbox_whitelist_emails', - 'expense_mailbox_blacklist_domains', - 'expense_mailbox_blacklist_emails', + 'inbound_mailbox_active', + 'inbound_mailbox', // TODO: @turbo124 custom validation: self-hosted => free change, hosted => not changeable, only changeable with env-mask + 'inbound_mailbox_allow_company_users', + 'inbound_mailbox_allow_vendors', + 'inbound_mailbox_allow_clients', + 'inbound_mailbox_allow_unknown', + 'inbound_mailbox_whitelist_domains', + 'inbound_mailbox_whitelist_senders', + 'inbound_mailbox_blacklist_domains', + 'inbound_mailbox_blacklist_senders', 'smtp_host', 'smtp_port', 'smtp_encryption', diff --git a/app/Models/SystemLog.php b/app/Models/SystemLog.php index 1ab3cafbae65..e71d402683df 100644 --- a/app/Models/SystemLog.php +++ b/app/Models/SystemLog.php @@ -113,7 +113,7 @@ class SystemLog extends Model public const EVENT_USER = 61; - public const EVENT_INGEST_EMAIL_FAILURE = 62; + public const EVENT_INBOUND_MAIL_BLOCKED = 62; /*Type IDs*/ public const TYPE_PAYPAL = 300; diff --git a/app/Services/IngresEmail/IngresEmail.php b/app/Services/InboundMail/InboundMail.php similarity index 93% rename from app/Services/IngresEmail/IngresEmail.php rename to app/Services/InboundMail/InboundMail.php index 72d30e84e110..2a897d2598b0 100644 --- a/app/Services/IngresEmail/IngresEmail.php +++ b/app/Services/InboundMail/InboundMail.php @@ -9,7 +9,7 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Services\IngresEmail; +namespace App\Services\InboundMail; use Illuminate\Http\UploadedFile; use Illuminate\Support\Carbon; @@ -17,7 +17,7 @@ use Illuminate\Support\Carbon; /** * EmailObject. */ -class IngresEmail +class InboundMail { public string $to; diff --git a/app/Services/IngresEmail/IngresEmailEngine.php b/app/Services/InboundMail/InboundMailEngine.php similarity index 62% rename from app/Services/IngresEmail/IngresEmailEngine.php rename to app/Services/InboundMail/InboundMailEngine.php index b8a5013a5421..c745fa2ac243 100644 --- a/app/Services/IngresEmail/IngresEmailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -9,17 +9,19 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Services\IngresEmail; +namespace App\Services\InboundMail; use App\Events\Expense\ExpenseWasCreated; use App\Factory\ExpenseFactory; use App\Jobs\Util\SystemLogger; use App\Libraries\MultiDB; +use App\Models\Client; +use App\Models\ClientContact; use App\Models\Company; use App\Models\SystemLog; use App\Models\Vendor; use App\Models\VendorContact; -use App\Services\IngresEmail\IngresEmail; +use App\Services\InboundMail\InboundMail; use App\Utils\Ninja; use App\Utils\TempFile; use App\Utils\Traits\GeneratesCounter; @@ -29,7 +31,7 @@ use Cache; use Illuminate\Queue\SerializesModels; use Log; -class IngresEmailEngine +class InboundMailEngine { use SerializesModels, MakesHash; use GeneratesCounter, SavesDocuments; @@ -37,8 +39,8 @@ class IngresEmailEngine private ?Company $company; private ?bool $isUnknownRecipent = null; private array $globalBlacklistDomains = []; - private array $globalBlacklistEmails = []; - public function __construct(private IngresEmail $email) + private array $globalBlacklistSenders = []; + public function __construct(private InboundMail $email) { } /** @@ -53,7 +55,7 @@ class IngresEmailEngine $this->isUnknownRecipent = true; // Expense Mailbox => will create an expense - $this->company = MultiDB::findAndSetDbByExpenseMailbox($this->email->to); + $this->company = MultiDB::findAndSetDbByInboundMailbox($this->email->to); if ($this->company) { $this->isUnknownRecipent = false; $this->createExpense(); @@ -67,7 +69,7 @@ class IngresEmailEngine { // invalid email if (!filter_var($this->email->from, FILTER_VALIDATE_EMAIL)) { - $this->log('E-Mail blocked, because from e-mail has the wrong format: ' . $this->email->from); + $this->logBlocked('E-Mail blocked, because from e-mail has the wrong format: ' . $this->email->from); return true; } @@ -76,43 +78,43 @@ class IngresEmailEngine // global blacklist if (in_array($domain, $this->globalBlacklistDomains)) { - $this->log('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $this->email->from); + $this->logBlocked('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $this->email->from); return true; } - if (in_array($this->email->from, $this->globalBlacklistEmails)) { - $this->log('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $this->email->from); + if (in_array($this->email->from, $this->globalBlacklistSenders)) { + $this->logBlocked('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $this->email->from); return true; } - if (Cache::has('ingresEmailBlockedSender:' . $this->email->from)) { // was marked as blocked before, so we block without any console output + if (Cache::has('inboundMailBlockedSender:' . $this->email->from)) { // was marked as blocked before, so we block without any console output return true; } // sender occured in more than 500 emails in the last 12 hours - $senderMailCountTotal = Cache::get('ingresEmailSender:' . $this->email->from, 0); + $senderMailCountTotal = Cache::get('inboundMailSender:' . $this->email->from, 0); if ($senderMailCountTotal >= 5000) { - $this->log('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); + $this->logBlocked('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); $this->blockSender(); return true; } if ($senderMailCountTotal >= 1000) { - $this->log('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); + $this->logBlocked('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); $this->saveMeta(); return true; } // sender sended more than 50 emails to the wrong mailbox in the last 6 hours - $senderMailCountUnknownRecipent = Cache::get('ingresEmailSenderUnknownRecipent:' . $this->email->from, 0); + $senderMailCountUnknownRecipent = Cache::get('inboundMailSenderUnknownRecipent:' . $this->email->from, 0); if ($senderMailCountUnknownRecipent >= 50) { - $this->log('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $this->email->from); + $this->logBlocked('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $this->email->from); $this->saveMeta(); return true; } // wrong recipent occurs in more than 100 emails in the last 12 hours, so the processing is blocked - $mailCountUnknownRecipent = Cache::get('ingresEmailUnknownRecipent:' . $this->email->to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time + $mailCountUnknownRecipent = Cache::get('inboundMailUnknownRecipent:' . $this->email->to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time if ($mailCountUnknownRecipent >= 100) { - $this->log('E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); + $this->logBlocked('E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); $this->blockSender(); return true; } @@ -121,7 +123,7 @@ class IngresEmailEngine } private function blockSender() { - Cache::add('ingresEmailBlockedSender:' . $this->email->from, true, now()->addHours(12)); + Cache::add('inboundMailBlockedSender:' . $this->email->from, true, now()->addHours(12)); $this->saveMeta(); // TODO: ignore, when known sender (for heavy email-usage mostly on isHosted()) @@ -130,15 +132,15 @@ class IngresEmailEngine private function saveMeta() { // save cache - Cache::add('ingresEmailSender:' . $this->email->from, 0, now()->addHours(12)); - Cache::increment('ingresEmailSender:' . $this->email->from); + Cache::add('inboundMailSender:' . $this->email->from, 0, now()->addHours(12)); + Cache::increment('inboundMailSender:' . $this->email->from); if ($this->isUnknownRecipent) { - Cache::add('ingresEmailSenderUnknownRecipent:' . $this->email->from, 0, now()->addHours(6)); - Cache::increment('ingresEmailSenderUnknownRecipent:' . $this->email->from); // we save the sender, to may block him + Cache::add('inboundMailSenderUnknownRecipent:' . $this->email->from, 0, now()->addHours(6)); + Cache::increment('inboundMailSenderUnknownRecipent:' . $this->email->from); // we save the sender, to may block him - Cache::add('ingresEmailUnknownRecipent:' . $this->email->to, 0, now()->addHours(12)); - Cache::increment('ingresEmailUnknownRecipent:' . $this->email->to); // we save the sender, to may block him + Cache::add('inboundMailUnknownRecipent:' . $this->email->to, 0, now()->addHours(12)); + Cache::increment('inboundMailUnknownRecipent:' . $this->email->to); // we save the sender, to may block him } } @@ -156,15 +158,15 @@ class IngresEmailEngine { // Skipping executions: will not result in not saving Metadata to prevent usage of these conditions, to spam if (!$this->validateExpenseShouldProcess()) { - $this->log('mailbox not active for this company. from: ' . $this->email->from); + $this->logBlocked('mailbox not active for this company. from: ' . $this->email->from); return; } if (!$this->validateExpenseSender()) { - $this->log('invalid sender of an ingest email for this company. from: ' . $this->email->from); + $this->logBlocked('invalid sender of an ingest email for this company. from: ' . $this->email->from); return; } if (sizeOf($this->email->documents) == 0) { - $this->log('email does not contain any attachments and is likly not an expense. from: ' . $this->email->from); + $this->logBlocked('email does not contain any attachments and is likly not an expense. from: ' . $this->email->from); return; } @@ -176,7 +178,7 @@ class IngresEmailEngine $expense->date = $this->email->date; // handle vendor assignment - $expense_vendor = $this->getExpenseVendor(); + $expense_vendor = $this->getVendor(); if ($expense_vendor) $expense->vendor_id = $expense_vendor->id; @@ -198,7 +200,7 @@ class IngresEmailEngine // HELPERS private function validateExpenseShouldProcess() { - return $this->company?->expense_mailbox_active ?: false; + return $this->company?->inbound_mailbox_active ?: false; } private function validateExpenseSender() { @@ -206,37 +208,56 @@ class IngresEmailEngine $domain = array_pop($parts); // whitelists - $email_whitelist = explode(",", $this->company->expense_mailbox_whitelist_emails); + $email_whitelist = explode(",", $this->company->inbound_mailbox_whitelist_senders); if (in_array($this->email->from, $email_whitelist)) return true; - $domain_whitelist = explode(",", $this->company->expense_mailbox_whitelist_domains); + $domain_whitelist = explode(",", $this->company->inbound_mailbox_whitelist_domains); if (in_array($domain, $domain_whitelist)) return true; - $email_blacklist = explode(",", $this->company->expense_mailbox_blacklist_emails); + $email_blacklist = explode(",", $this->company->inbound_mailbox_blacklist_senders); if (in_array($this->email->from, $email_blacklist)) return false; - $domain_blacklist = explode(",", $this->company->expense_mailbox_blacklist_domains); + $domain_blacklist = explode(",", $this->company->inbound_mailbox_blacklist_domains); if (in_array($domain, $domain_blacklist)) return false; // allow unknown - if ($this->company->expense_mailbox_allow_unknown) + if ($this->company->inbound_mailbox_allow_unknown) return true; // own users - if ($this->company->expense_mailbox_allow_company_users && $this->company->users()->where("email", $this->email->from)->exists()) + if ($this->company->inbound_mailbox_allow_company_users && $this->company->users()->where("email", $this->email->from)->exists()) return true; - // from clients/vendors (if active) - if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->where("invoicing_email", $this->email->from)->orWhere("invoicing_domain", $domain)->exists()) + // from vendors (if active) + if ($this->company->inbound_mailbox_allow_vendors && $this->company->vendors()->where("invoicing_email", $this->email->from)->orWhere("invoicing_domain", $domain)->exists()) return true; - if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->contacts()->where("email", $this->email->from)->exists()) + if ($this->company->inbound_mailbox_allow_vendors && $this->company->vendors()->contacts()->where("email", $this->email->from)->exists()) + return true; + + // from clients (if active) + if ($this->company->inbound_mailbox_allow_clients && $this->company->clients()->where("invoicing_email", $this->email->from)->orWhere("invoicing_domain", $domain)->exists()) + return true; + if ($this->company->inbound_mailbox_allow_clients && $this->company->clients()->contacts()->where("email", $this->email->from)->exists()) return true; // denie return false; } - private function getExpenseVendor() + private function getClient() + { + $parts = explode('@', $this->email->from); + $domain = array_pop($parts); + + $client = Client::where("company_id", $this->company->id)->where("email", $domain)->first(); + if ($client == null) { + $clientContact = ClientContact::where("company_id", $this->company->id)->where("email", $this->email->from)->first(); + $client = $clientContact->client(); + } + + return $client; + } + private function getVendor() { $parts = explode('@', $this->email->from); $domain = array_pop($parts); @@ -251,15 +272,15 @@ class IngresEmailEngine return $vendor; } - private function log(string $data) + private function logBlocked(string $data) { - Log::info("[IngresEmailEngine][company:" . $this->company->id . "] " . $data); + Log::info("[InboundMailEngine][company:" . $this->company->id . "] " . $data); ( new SystemLogger( $data, SystemLog::CATEGORY_MAIL, - SystemLog::EVENT_INGEST_EMAIL_FAILURE, + SystemLog::EVENT_INBOUND_MAIL_BLOCKED, SystemLog::TYPE_CUSTOM, null, $this->company diff --git a/app/Transformers/CompanyTransformer.php b/app/Transformers/CompanyTransformer.php index b56db1ae66b1..f3ebd108fc6e 100644 --- a/app/Transformers/CompanyTransformer.php +++ b/app/Transformers/CompanyTransformer.php @@ -204,7 +204,16 @@ class CompanyTransformer extends EntityTransformer 'invoice_task_project_header' => (bool) $company->invoice_task_project_header, 'invoice_task_item_description' => (bool) $company->invoice_task_item_description, 'origin_tax_data' => $company->origin_tax_data ?: new \stdClass, - 'expense_mailbox' => $company->expense_mailbox, + 'inbound_mailbox' => (bool) $company->inbound_mailbox, + 'inbound_mailbox_active' => (bool) $company->inbound_mailbox_active, + 'inbound_mailbox_allow_company_users' => (bool) $company->inbound_mailbox_allow_company_users, + 'inbound_mailbox_allow_vendors' => (bool) $company->inbound_mailbox_allow_vendors, + 'inbound_mailbox_allow_clients' => (bool) $company->inbound_mailbox_allow_clients, + 'inbound_mailbox_allow_unknown' => (bool) $company->inbound_mailbox_allow_unknown, + 'inbound_mailbox_blacklist_domains' => $company->inbound_mailbox_blacklist_domains, + 'inbound_mailbox_blacklist_senders' => $company->inbound_mailbox_blacklist_senders, + 'inbound_mailbox_whitelist_domains' => $company->inbound_mailbox_whitelist_domains, + 'inbound_mailbox_whitelist_senders' => $company->inbound_mailbox_whitelist_senders, 'smtp_host' => (string) $company->smtp_host ?? '', 'smtp_port' => (int) $company->smtp_port ?? 25, 'smtp_encryption' => (string) $company->smtp_encryption ?? 'tls', diff --git a/config/ninja.php b/config/ninja.php index a7ac185eb8d5..7274d816ff8c 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -84,11 +84,12 @@ return [ 'username' => 'user@example.com', 'clientname' => 'client@example.com', 'password' => 'password', - 'gocardless' => env('GOCARDLESS_KEYS',''), - 'square' => env('SQUARE_KEYS',''), - 'eway' => env('EWAY_KEYS',''), - 'mollie', env('MOLLIE_KEYS',''), - 'paytrace' => env('PAYTRACE_KEYS',''), + 'gocardless' => env('GOCARDLESS_KEYS', ''), + 'square' => env('SQUARE_KEYS', ''), + 'eway' => env('EWAY_KEYS', ''), + 'mollie', + env('MOLLIE_KEYS', ''), + 'paytrace' => env('PAYTRACE_KEYS', ''), 'stripe' => env('STRIPE_KEYS', ''), 'paypal' => env('PAYPAL_KEYS', ''), 'ppcp' => env('PPCP_KEYS', ''), @@ -235,16 +236,16 @@ return [ 'client_id' => env('PAYPAL_CLIENT_ID', null), 'webhook_id' => env('PAYPAL_WEBHOOK_ID', null), ], - 'ingest_mail' => [ + 'inbound_mailbox' => [ 'imap' => [ - 'servers' => env('ingest_mail_IMAP_SERVERS', ''), - 'ports' => env('ingest_mail_IMAP_PORTS', ''), - 'users' => env('ingest_mail_IMAP_USERS', ''), - 'passwords' => env('ingest_mail_IMAP_PASSWORDS', ''), - 'companies' => env('ingest_mail_IMAP_COMPANIES', '1'), + 'servers' => env('INBOUND_MAILBOX_IMAP_SERVERS', ''), + 'ports' => env('INBOUND_MAILBOX_IMAP_PORTS', ''), + 'users' => env('INBOUND_MAILBOX_IMAP_USERS', ''), + 'passwords' => env('INBOUND_MAILBOX_IMAP_PASSWORDS', ''), + 'companies' => env('INBOUND_MAILBOX_IMAP_COMPANIES', '1'), ], - 'expense_mailbox_template' => env('ingest_mail_EXPENSE_MAILBOX_TEMPLATE', null), - 'expense_mailbox_endings' => env('ingest_mail_EXPENSE_MAILBOX_ENDINGS', '@expense.invoicing.co'), + 'inbound_mailbox_template' => env('INBOUND_MAILBOX_TEMPLATE', null), + 'inbound_mailbox_endings' => env('INBOUND_MAILBOX_ENDINGS', '@expense.invoicing.co'), ], 'cloudflare' => [ 'turnstile' => [ @@ -256,5 +257,4 @@ return [ 'private_key' => env('NINJA_PRIVATE_KEY', false), ], 'upload_extensions' => env('ADDITIONAL_UPLOAD_EXTENSIONS', false), - ]; diff --git a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php index 8a8d62e22918..f1b1c21603d7 100644 --- a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php +++ b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php @@ -12,15 +12,16 @@ return new class extends Migration { public function up(): void { Schema::table('companies', function (Blueprint $table) { - $table->boolean("expense_mailbox_active")->default(true); - $table->string("expense_mailbox")->nullable(); - $table->boolean("expense_mailbox_allow_company_users")->default(false); - $table->boolean("expense_mailbox_allow_vendors")->default(false); - $table->boolean("expense_mailbox_allow_unknown")->default(false); - $table->text("expense_mailbox_whitelist_domains")->nullable(); - $table->text("expense_mailbox_whitelist_emails")->nullable(); - $table->text("expense_mailbox_blacklist_domains")->nullable(); - $table->text("expense_mailbox_blacklist_emails")->nullable(); + $table->boolean("inbound_mailbox_active")->default(true); + $table->string("inbound_mailbox")->nullable(); + $table->boolean("inbound_mailbox_allow_company_users")->default(false); + $table->boolean("inbound_mailbox_allow_vendors")->default(false); + $table->boolean("inbound_mailbox_allow_clients")->default(false); + $table->boolean("inbound_mailbox_allow_unknown")->default(false); + $table->text("inbound_mailbox_whitelist_domains")->nullable(); + $table->text("inbound_mailbox_whitelist_senders")->nullable(); + $table->text("inbound_mailbox_blacklist_domains")->nullable(); + $table->text("inbound_mailbox_blacklist_senders")->nullable(); }); Schema::table('vendors', function (Blueprint $table) { $table->string("invoicing_email")->nullable(); diff --git a/lang/en/texts.php b/lang/en/texts.php index 7ec111242ccb..140099bf538d 100644 --- a/lang/en/texts.php +++ b/lang/en/texts.php @@ -2494,8 +2494,8 @@ $lang = array( 'local_storage_required' => 'Error: local storage is not available.', 'your_password_reset_link' => 'Your Password Reset Link', 'subdomain_taken' => 'The subdomain is already in use', - 'expense_mailbox_taken' => 'The mailbox is already in use', - 'expense_mailbox_invalid' => 'The mailbox does not match the required schema', + 'inbound_mailbox_taken' => 'The inbound mailbox is already in use', + 'inbound_mailbox_invalid' => 'The inbound mailbox does not match the required schema', 'client_login' => 'Client Login', 'converted_amount' => 'Converted Amount', 'default' => 'Default',