diff --git a/app/Helpers/IngresMail/Transformer/ImapMailTransformer.php b/app/Helpers/IngresMail/Transformer/ImapMailTransformer.php new file mode 100644 index 000000000000..80889b3afc9c --- /dev/null +++ b/app/Helpers/IngresMail/Transformer/ImapMailTransformer.php @@ -0,0 +1,38 @@ +from = $mail->getSender(); + $ingresEmail->subject = $mail->getSubject(); + $ingresEmail->plain_message = $mail->getBodyText(); + $ingresEmail->html_message = $mail->getBodyHtml(); + $ingresEmail->date = $mail->getDate(); + + // parse documents as UploadedFile + foreach ($mail->getAttachments() as $attachment) { + $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment->getContent(), $attachment->getFilename(), $attachment->getEncoding()); + } + + return $ingresEmail; + } +} diff --git a/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php b/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php new file mode 100644 index 000000000000..6871e3010168 --- /dev/null +++ b/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php @@ -0,0 +1,36 @@ +from = $data["sender"]; + $ingresEmail->subject = $data["subject"]; + $ingresEmail->plain_message = $data["body-plain"]; + $ingresEmail->html_message = $data["body-html"]; + $ingresEmail->date = now(); // TODO + + // parse documents as UploadedFile from webhook-data + foreach ($data["Attachments"] as $attachment) { + $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]); + } + + return $ingresEmail; + } +} diff --git a/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php b/app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php similarity index 81% rename from app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php rename to app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php index 1cceb5a69753..0ad0faf71963 100644 --- a/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php +++ b/app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php @@ -9,13 +9,32 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Helpers\Mail\Webhook\Postmark; +namespace App\Helpers\IngresMail\Transformer; -use App\Helpers\Mail\Webhook\BaseWebhookHandler; +use App\Services\IngresEmail\IngresEmail; use App\Utils\TempFile; -class PostmarkWebhookHandler extends BaseWebhookHandler +class PostmarkInboundWebhookTransformer { + public function process($data) + { + + $ingresEmail = new IngresEmail(); + + $ingresEmail->from = $data["From"]; + $ingresEmail->subject = $data["Subject"]; + $ingresEmail->plain_message = $data["TextBody"]; + $ingresEmail->html_message = $data["HtmlBody"]; + $ingresEmail->date = $data["Date"]; // TODO: parsing + + // parse documents as UploadedFile from webhook-data + foreach ($data["Attachments"] as $attachment) { + $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]); + } + + return $ingresEmail; + + } // { // "FromName": "Postmarkapp Support", // "MessageStream": "inbound", @@ -96,29 +115,4 @@ class PostmarkWebhookHandler extends BaseWebhookHandler // } // ] // } - public function process($data) - { - - $from = $data["From"]; - $subject = $data["Subject"]; - $plain_message = $data["TextBody"]; - $html_message = $data["HtmlBody"]; - $date = $data["Date"]; // TODO - - // parse documents as UploadedFile from webhook-data - $documents = []; - foreach ($data["Attachments"] as $attachment) { - $documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]); - } - - return $this->createExpense( - $from, - $subject, - $plain_message, - $html_message, - $date, - $documents, - ); - - } } diff --git a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php b/app/Helpers/Mail/Webhook/BaseWebhookHandler.php deleted file mode 100644 index 0967e81388aa..000000000000 --- a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php +++ /dev/null @@ -1,58 +0,0 @@ -matchCompany($email); - if (!$company) - return false; - - $expense = ExpenseFactory::create($company->id, $company->owner()->id); - - $expense->public_notes = $subject; - $expense->private_notes = $plain_message; - $expense->date = $date; - - // add html_message as document to the expense - $documents[] = TempFile::UploadedFileFromRaw($html_message, "E-Mail.html", "text/html"); - - $this->saveDocuments($documents, $expense); - - $expense->saveQuietly(); - - event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); - event('eloquent.created: App\Models\Expense', $expense); - - return $expense; - } - - private function matchCompany(string $email) - { - return Company::where("expense_mailbox", $email)->first(); - } -} diff --git a/app/Helpers/Mail/Webhook/Mailgun/MailgunWebhookHandler.php b/app/Helpers/Mail/Webhook/Mailgun/MailgunWebhookHandler.php deleted file mode 100644 index e3efdc9986a0..000000000000 --- a/app/Helpers/Mail/Webhook/Mailgun/MailgunWebhookHandler.php +++ /dev/null @@ -1,44 +0,0 @@ -createExpense( - $from, - $subject, - $plain_message, - $html_message, - $date, - $documents, - ); - - } -} diff --git a/app/Jobs/Mail/ExpenseMailboxJob.php b/app/Jobs/Mail/ExpenseMailboxJob.php index 997d2ccca2aa..9319c1b08391 100644 --- a/app/Jobs/Mail/ExpenseMailboxJob.php +++ b/app/Jobs/Mail/ExpenseMailboxJob.php @@ -11,15 +11,12 @@ namespace App\Jobs\Mail; -use App\Events\Expense\ExpenseWasCreated; -use App\Factory\ExpenseFactory; +use App\Helpers\IngresMail\Transformer\ImapMailTransformer; use App\Helpers\Mail\Mailbox\Imap\ImapMailbox; use App\Libraries\MultiDB; use App\Models\Company; -use App\Models\Vendor; use App\Repositories\ExpenseRepository; -use App\Utils\Ninja; -use App\Utils\TempFile; +use App\Services\IngresEmail\IngresEmailEngine; use App\Utils\Traits\MakesHash; use App\Utils\Traits\SavesDocuments; use Illuminate\Bus\Queueable; @@ -87,6 +84,7 @@ class ExpenseMailboxJob implements ShouldQueue throw new \Exception('invalid configuration inbound_expense.imap (wrong element-count)'); foreach ($companies as $index => $companyId) { + if ($servers[$index] == '') // if property is empty, ignore => this happens exspecialy when no config is provided and it enabled us to set a single default company for env (usefull on self-hosted) continue; @@ -97,6 +95,7 @@ class ExpenseMailboxJob implements ShouldQueue "password" => $passwords[$index], ]; $this->imap_companies[] = $companyId; + } } @@ -106,55 +105,25 @@ class ExpenseMailboxJob implements ShouldQueue $credentials = $this->imap_credentials[$company->id]; $imapMailbox = new ImapMailbox($credentials->server, $credentials->port, $credentials->user, $credentials->password); + $transformer = new ImapMailTransformer(); $emails = $imapMailbox->getUnprocessedEmails(); - foreach ($emails as $mail) { + + foreach ($emails as $email) { try { - $sender = $mail->getSender(); + $email->markAsSeen(); - $vendor = Vendor::where('expense_sender_email', $sender)->first(); - if ($vendor == null) - $vendor = Vendor::where($sender, 'LIKE', "CONCAT('%',expense_sender_domain)")->first(); - if ($vendor == null) - $vendor = Vendor::where("email", $sender)->first(); + IngresEmailEngine::dispatch($transformer->transform($email)); - $documents = []; // TODO: $mail->getAttachments() + save email as document (.html) - - $data = [ - "vendor_id" => $vendor !== null ? $vendor->id : null, - "date" => $mail->getDate(), - "public_notes" => $mail->getSubject(), - "private_notes" => $mail->getCompleteBodyText(), - "documents" => $documents, // FIXME: https://github.com/ddeboer/imap?tab=readme-ov-file#message-attachments - ]; - - $expense = ExpenseFactory::create($company->company->id, $company->company->owner()->id); - - $expense->vendor_id = $vendor !== null ? $vendor->id : null; - $expense->public_notes = $mail->getSubject(); - $expense->private_notes = $mail->getBodyText(); - $expense->date = $mail->getDate(); - - // add html_message as document to the expense - $documents[] = TempFile::UploadedFileFromRaw($mail->getBodyHtml(), "E-Mail.html", "text/html"); - - $this->saveDocuments($documents, $expense); - - $expense->saveQuietly(); - - event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); - event('eloquent.created: App\Models\Expense', $expense); - - $mail->markAsSeen(); - $imapMailbox->moveProcessed($mail); + $imapMailbox->moveProcessed($email); } catch (\Exception $e) { - $imapMailbox->moveFailed($mail); + $imapMailbox->moveFailed($email); - nlog("processing of an email failed upnormally: " . $company->id . " message: " . $e->getMessage()); // @turbo124 @todo should this be handled in an other way? + nlog("processing of an email failed upnormally: " . $company->id . " message: " . $e->getMessage()); } } diff --git a/app/Models/Company.php b/app/Models/Company.php index f8b331ba9e4e..3c24d5b31fd4 100644 --- a/app/Models/Company.php +++ b/app/Models/Company.php @@ -111,8 +111,13 @@ use Laracasts\Presenter\PresentableTrait; * @property int $convert_expense_currency * @property int $notify_vendor_when_paid * @property int $invoice_task_hours - * @property boolean $expense_import * @property string|null $expense_mailbox + * @property boolean $expense_mailbox_active + * @property bool $expense_mailbox_allow_company_users + * @property bool $expense_mailbox_allow_vendors + * @property bool $expense_mailbox_allow_unknown + * @property string|null $expense_mailbox_whitelist_domains + * @property string|null $expense_mailbox_whitelist_emails * @property int $deleted_at * @property-read \App\Models\Account $account * @property-read \Illuminate\Database\Eloquent\Collection $activities @@ -354,8 +359,14 @@ class Company extends BaseModel 'calculate_taxes', 'tax_data', 'e_invoice_certificate_passphrase', - 'expense_import', + 'expense_mailbox_active', 'expense_mailbox', // TODO: @turbo124 custom validation: self-hosted => free change, hosted => not changeable, only changeable with env-mask + 'expense_mailbox_allow_company_users', + 'expense_mailbox_allow_vendors', + 'expense_mailbox_allow_unknown', + 'expense_mailbox_whitelist_domains', + 'expense_mailbox_whitelist_emails', + 'expense_mailbox_whitelist' ]; protected $hidden = [ diff --git a/app/Models/Vendor.php b/app/Models/Vendor.php index 0e7aec98deac..51223b449a84 100644 --- a/app/Models/Vendor.php +++ b/app/Models/Vendor.php @@ -54,8 +54,8 @@ use Laracasts\Presenter\PresentableTrait; * @property string|null $id_number * @property string|null $language_id * @property int|null $last_login - * @property string|null $expense_sender_email - * @property string|null $expense_sender_domain + * @property string|null $invoicing_email + * @property string|null $invoicing_domain * @property-read \Illuminate\Database\Eloquent\Collection $activities * @property-read int|null $activities_count * @property-read \App\Models\User|null $assigned_user @@ -117,8 +117,8 @@ class Vendor extends BaseModel 'number', 'language_id', 'classification', - 'expense_sender_email', - 'expense_sender_domain', + 'invoicing_email', + 'invoicing_domain', ]; protected $casts = [ diff --git a/app/Services/IngresEmail/IngresEmail.php b/app/Services/IngresEmail/IngresEmail.php new file mode 100644 index 000000000000..2ec889bcf6e2 --- /dev/null +++ b/app/Services/IngresEmail/IngresEmail.php @@ -0,0 +1,50 @@ +email = $email; + } + /** + * if there is not a company with an matching mailbox, we do nothing + */ + public function handle() + { + // Expense Mailbox => will create an expense + foreach ($this->email->to as $expense_mailbox) { + $this->company = MultiDB::findAndSetDbByExpenseMailbox($expense_mailbox); + if (!$this->company || !$this->validateExpenseActive()) + continue; + + $this->createExpense(); + } + + // TODO reuse this method to add more mail-parsing behaviors + } + + // MAIN-PROCESSORS + protected function createExpense() + { + if (!$this->validateExpenseSender()) { + nlog('invalid sender of an ingest email to company: ' . $this->company->id . ' from: ' . $this->email->from); + return; + } + + $expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id); + + $expense->public_notes = $this->email->subject; + $expense->private_notes = $this->email->text_body; + $expense->date = $this->email->date; + + // handle vendor assignment + $expense_vendor = $this->getExpenseVendor(); + if ($expense_vendor) + $expense->vendor_id = $expense_vendor->id; + + // handle documents + $this->processHtmlBodyToDocument(); + $documents = []; + array_push($documents, ...$this->email->documents); + if ($this->email->body_document) + $documents[] = $this->email->body_document; + $this->saveDocuments($documents, $expense); + + $expense->saveQuietly(); + + event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); // @turbo124 please check, I copied from API + event('eloquent.created: App\Models\Expense', $expense); // @turbo124 please check, I copied from API + + return $expense; + } + + // HELPERS + private function processHtmlBodyToDocument() + { + if (!$this->email->body_document && property_exists($this->email, "body")) { + $this->email->body_document = TempFile::UploadedFileFromRaw($this->email->body, "E-Mail.html", "text/html"); + } + } + private function validateExpenseActive() + { + return $this->company?->expense_mailbox_active ?: false; + } + private function validateExpenseSender() + { + // invalid email + if (!filter_var($this->email->from, FILTER_VALIDATE_EMAIL)) + return false; + + $parts = explode('@', $this->email->from); + $domain = array_pop($parts); + + // global blacklist + if (in_array($domain, $this->globalBlacklist)) + return false; + + // whitelists + $email_whitelist = explode(",", $this->company->expense_mailbox_whitelist_emails); + if (in_array($this->email->from, $email_whitelist)) + return true; + $domain_whitelist = explode(",", $this->company->expense_mailbox_whitelist_domains); + if (in_array($domain, $domain_whitelist)) + return true; + if ($this->company->expense_mailbox_allow_unknown && sizeOf($email_whitelist) == 0 && sizeOf($domain_whitelist) == 0) // from unknown only, when no whitelists are defined + return true; + + // own users + if ($this->company->expense_mailbox_allow_company_users && $this->company->users()->where("email", $this->email->from)->exists()) + return true; + + // from clients/vendors (if active) + if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->where("invoicing_email", $this->email->from)->orWhere($this->email->from, 'LIKE', "CONCAT('%',invoicing_domain)")->exists()) + return true; + if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->contacts()->where("email", $this->email->from)->exists()) // TODO + return true; + + // denie + return false; + } + private function getExpenseVendor() + { + $vendor = Vendor::where("company_id", $this->company->id)->where('invoicing_email', $this->email->from)->first(); + if ($vendor == null) + $vendor = Vendor::where("company_id", $this->company->id)->where($this->email->from, 'LIKE', "CONCAT('%',invoicing_domain)")->first(); + if ($vendor == null) { + $vendorContact = VendorContact::where("company_id", $this->company->id)->where("email", $this->email->from)->first(); + $vendor = $vendorContact->vendor(); + } + // TODO: from contacts + + return $vendor; + } +} diff --git a/app/Transformers/VendorTransformer.php b/app/Transformers/VendorTransformer.php index 5ec3c9a19175..de73267182fe 100644 --- a/app/Transformers/VendorTransformer.php +++ b/app/Transformers/VendorTransformer.php @@ -105,8 +105,8 @@ class VendorTransformer extends EntityTransformer 'language_id' => (string) $vendor->language_id ?: '', 'classification' => (string) $vendor->classification ?: '', 'display_name' => (string) $vendor->present()->name(), - 'expense_sender_email' => (string) $vendor->expense_sender_email ?: '', - 'expense_sender_domain' => (string) $vendor->expense_sender_domain ?: '', + 'invoicing_email' => (string) $vendor->invoicing_email ?: '', + 'invoicing_domain' => (string) $vendor->invoicing_domain ?: '', ]; } } diff --git a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php index f750b9b07435..91a698511324 100644 --- a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php +++ b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php @@ -12,8 +12,13 @@ return new class extends Migration { public function up(): void { Schema::table('company', function (Blueprint $table) { - $table->boolean("expense_import")->default(true); + $table->boolean("expense_mailbox_active")->default(true); $table->string("expense_mailbox")->nullable(); + $table->boolean("expense_mailbox_allow_company_users")->default(false); + $table->boolean("expense_mailbox_allow_vendors")->default(false); + $table->boolean("expense_mailbox_allow_unknown")->default(false); + $table->string("expense_mailbox_whitelist_domains")->nullable(); + $table->string("expense_mailbox_whitelist_emails")->nullable(); }); Company::query()->cursor()->each(function ($company) { // TODO: @turbo124 check migration on staging environment with real data to ensure, this works as exspected $company->expense_mailbox = config('ninja.inbound_expense.webhook.mailbox_template') != '' ? @@ -22,8 +27,8 @@ return new class extends Migration { $company->save(); }); Schema::table('vendor', function (Blueprint $table) { - $table->string("expense_sender_email")->nullable(); - $table->string("expense_sender_domain")->nullable(); + $table->string("invoicing_email")->nullable(); + $table->string("invoicing_domain")->nullable(); }); }