Cutlery/invoiceninja
1
0
Fork 0
mirror of https://github.com/invoiceninja/invoiceninja.git synced 2025-07-09 03:14:30 -04:00
Code Issues Packages Projects Releases Wiki Activity

wip: restruct and init IngresEmailEngine

Browse Source
This commit is contained in:
paulwer 2023-12-18 15:05:15 +01:00
parent f0415b6b20
commit 5d70daaaaa
12 changed files with 343 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
app/Helpers/IngresMail/Transformer/ImapMailTransformer.php Normal file
View File

@ -0,0 +1,38 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Helpers\IngresMail\Transformer;
use App\Services\IngresEmail\IngresEmail;
use App\Utils\TempFile;
use Ddeboer\Imap\MessageInterface;
class ImapMailTransformer
{
public function transform(MessageInterface $mail)
{
$ingresEmail = new IngresEmail();
$ingresEmail->from = $mail->getSender();
$ingresEmail->subject = $mail->getSubject();
$ingresEmail->plain_message = $mail->getBodyText();
$ingresEmail->html_message = $mail->getBodyHtml();
$ingresEmail->date = $mail->getDate();
// parse documents as UploadedFile
foreach ($mail->getAttachments() as $attachment) {
$ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment->getContent(), $attachment->getFilename(), $attachment->getEncoding());
}
return $ingresEmail;
}
}

36
app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php Normal file
View File

@ -0,0 +1,36 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Helpers\IngresMail\Transformer;
use App\Services\IngresEmail\IngresEmail;
use App\Utils\TempFile;
class MailgunInboundWebhookTransformer
{
public function transform($data)
{
$ingresEmail = new IngresEmail();
$ingresEmail->from = $data["sender"];
$ingresEmail->subject = $data["subject"];
$ingresEmail->plain_message = $data["body-plain"];
$ingresEmail->html_message = $data["body-html"];
$ingresEmail->date = now(); // TODO
// parse documents as UploadedFile from webhook-data
foreach ($data["Attachments"] as $attachment) {
$ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]);
}
return $ingresEmail;
}
}

50
app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php → app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php
View File

@ -9,13 +9,32 @@
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Helpers\Mail\Webhook\Postmark;
namespace App\Helpers\IngresMail\Transformer;
use App\Helpers\Mail\Webhook\BaseWebhookHandler;
use App\Services\IngresEmail\IngresEmail;
use App\Utils\TempFile;
class PostmarkWebhookHandler extends BaseWebhookHandler
class PostmarkInboundWebhookTransformer
{
public function process($data)
{
$ingresEmail = new IngresEmail();
$ingresEmail->from = $data["From"];
$ingresEmail->subject = $data["Subject"];
$ingresEmail->plain_message = $data["TextBody"];
$ingresEmail->html_message = $data["HtmlBody"];
$ingresEmail->date = $data["Date"]; // TODO: parsing
// parse documents as UploadedFile from webhook-data
foreach ($data["Attachments"] as $attachment) {
$ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]);
}
return $ingresEmail;
}
// {
// "FromName": "Postmarkapp Support",
// "MessageStream": "inbound",
@ -96,29 +115,4 @@ class PostmarkWebhookHandler extends BaseWebhookHandler
// }
// ]
// }
public function process($data)
{
$from = $data["From"];
$subject = $data["Subject"];
$plain_message = $data["TextBody"];
$html_message = $data["HtmlBody"];
$date = $data["Date"]; // TODO
// parse documents as UploadedFile from webhook-data
$documents = [];
foreach ($data["Attachments"] as $attachment) {
$documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]);
}
return $this->createExpense(
$from,
$subject,
$plain_message,
$html_message,
$date,
$documents,
);
}
}

58
app/Helpers/Mail/Webhook/BaseWebhookHandler.php
View File

@ -1,58 +0,0 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Helpers\Mail\Webhook;
use App\Events\Expense\ExpenseWasCreated;
use App\Factory\ExpenseFactory;
use App\Models\Company;
use App\Utils\Ninja;
use App\Utils\TempFile;
use App\Utils\Traits\GeneratesCounter;
use App\Utils\Traits\SavesDocuments;
abstract class BaseWebhookHandler
{
use GeneratesCounter, SavesDocuments;
public function process()
{
}
protected function createExpense(string $email, string $subject, string $plain_message, string $html_message, string $date, array $documents)
{
$company = $this->matchCompany($email);
if (!$company)
return false;
$expense = ExpenseFactory::create($company->id, $company->owner()->id);
$expense->public_notes = $subject;
$expense->private_notes = $plain_message;
$expense->date = $date;
// add html_message as document to the expense
$documents[] = TempFile::UploadedFileFromRaw($html_message, "E-Mail.html", "text/html");
$this->saveDocuments($documents, $expense);
$expense->saveQuietly();
event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null)));
event('eloquent.created: App\Models\Expense', $expense);
return $expense;
}
private function matchCompany(string $email)
{
return Company::where("expense_mailbox", $email)->first();
}
}

44
app/Helpers/Mail/Webhook/Mailgun/MailgunWebhookHandler.php
View File

@ -1,44 +0,0 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Helpers\Mail\Webhook\Maigun;
use App\Helpers\Mail\Webhook\BaseWebhookHandler;
use App\Utils\TempFile;
class MailgunWebhookHandler extends BaseWebhookHandler
{
public function process($data)
{
$from = $data["sender"];
$subject = $data["subject"];
$plain_message = $data["body-plain"];
$html_message = $data["body-html"];
$date = now(); // TODO
// parse documents as UploadedFile from webhook-data
$documents = [];
foreach ($data["Attachments"] as $attachment) {
$documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]);
}
return $this->createExpense(
$from,
$subject,
$plain_message,
$html_message,
$date,
$documents,
);
}
}

55
app/Jobs/Mail/ExpenseMailboxJob.php
View File

@ -11,15 +11,12 @@
namespace App\Jobs\Mail;
use App\Events\Expense\ExpenseWasCreated;
use App\Factory\ExpenseFactory;
use App\Helpers\IngresMail\Transformer\ImapMailTransformer;
use App\Helpers\Mail\Mailbox\Imap\ImapMailbox;
use App\Libraries\MultiDB;
use App\Models\Company;
use App\Models\Vendor;
use App\Repositories\ExpenseRepository;
use App\Utils\Ninja;
use App\Utils\TempFile;
use App\Services\IngresEmail\IngresEmailEngine;
use App\Utils\Traits\MakesHash;
use App\Utils\Traits\SavesDocuments;
use Illuminate\Bus\Queueable;
@ -87,6 +84,7 @@ class ExpenseMailboxJob implements ShouldQueue
throw new \Exception('invalid configuration inbound_expense.imap (wrong element-count)');
foreach ($companies as $index => $companyId) {
if ($servers[$index] == '') // if property is empty, ignore => this happens exspecialy when no config is provided and it enabled us to set a single default company for env (usefull on self-hosted)
continue;
@ -97,6 +95,7 @@ class ExpenseMailboxJob implements ShouldQueue
"password" => $passwords[$index],
];
$this->imap_companies[] = $companyId;
}
}
@ -106,55 +105,25 @@ class ExpenseMailboxJob implements ShouldQueue
$credentials = $this->imap_credentials[$company->id];
$imapMailbox = new ImapMailbox($credentials->server, $credentials->port, $credentials->user, $credentials->password);
$transformer = new ImapMailTransformer();
$emails = $imapMailbox->getUnprocessedEmails();
foreach ($emails as $mail) {
foreach ($emails as $email) {
try {
$sender = $mail->getSender();
$email->markAsSeen();
$vendor = Vendor::where('expense_sender_email', $sender)->first();
if ($vendor == null)
$vendor = Vendor::where($sender, 'LIKE', "CONCAT('%',expense_sender_domain)")->first();
if ($vendor == null)
$vendor = Vendor::where("email", $sender)->first();
IngresEmailEngine::dispatch($transformer->transform($email));
$documents = []; // TODO: $mail->getAttachments() + save email as document (.html)
$data = [
"vendor_id" => $vendor !== null ? $vendor->id : null,
"date" => $mail->getDate(),
"public_notes" => $mail->getSubject(),
"private_notes" => $mail->getCompleteBodyText(),
"documents" => $documents, // FIXME: https://github.com/ddeboer/imap?tab=readme-ov-file#message-attachments
];
$expense = ExpenseFactory::create($company->company->id, $company->company->owner()->id);
$expense->vendor_id = $vendor !== null ? $vendor->id : null;
$expense->public_notes = $mail->getSubject();
$expense->private_notes = $mail->getBodyText();
$expense->date = $mail->getDate();
// add html_message as document to the expense
$documents[] = TempFile::UploadedFileFromRaw($mail->getBodyHtml(), "E-Mail.html", "text/html");
$this->saveDocuments($documents, $expense);
$expense->saveQuietly();
event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null)));
event('eloquent.created: App\Models\Expense', $expense);
$mail->markAsSeen();
$imapMailbox->moveProcessed($mail);
$imapMailbox->moveProcessed($email);
} catch (\Exception $e) {
$imapMailbox->moveFailed($mail);
$imapMailbox->moveFailed($email);
nlog("processing of an email failed upnormally: " . $company->id . " message: " . $e->getMessage()); // @turbo124 @todo should this be handled in an other way?
nlog("processing of an email failed upnormally: " . $company->id . " message: " . $e->getMessage());
}
}

15
app/Models/Company.php
View File

@ -111,8 +111,13 @@ use Laracasts\Presenter\PresentableTrait;
* @property int $convert_expense_currency
* @property int $notify_vendor_when_paid
* @property int $invoice_task_hours
* @property boolean $expense_import
* @property string|null $expense_mailbox
* @property boolean $expense_mailbox_active
* @property bool $expense_mailbox_allow_company_users
* @property bool $expense_mailbox_allow_vendors
* @property bool $expense_mailbox_allow_unknown
* @property string|null $expense_mailbox_whitelist_domains
* @property string|null $expense_mailbox_whitelist_emails
* @property int $deleted_at
* @property-read \App\Models\Account $account
* @property-read \Illuminate\Database\Eloquent\Collection<int, \App\Models\Activity> $activities
@ -354,8 +359,14 @@ class Company extends BaseModel
'calculate_taxes',
'tax_data',
'e_invoice_certificate_passphrase',
'expense_import',
'expense_mailbox_active',
'expense_mailbox', // TODO: @turbo124 custom validation: self-hosted => free change, hosted => not changeable, only changeable with env-mask
'expense_mailbox_allow_company_users',
'expense_mailbox_allow_vendors',
'expense_mailbox_allow_unknown',
'expense_mailbox_whitelist_domains',
'expense_mailbox_whitelist_emails',
'expense_mailbox_whitelist'
];
protected $hidden = [

8
app/Models/Vendor.php
View File

@ -54,8 +54,8 @@ use Laracasts\Presenter\PresentableTrait;
* @property string|null $id_number
* @property string|null $language_id
* @property int|null $last_login
* @property string|null $expense_sender_email
* @property string|null $expense_sender_domain
* @property string|null $invoicing_email
* @property string|null $invoicing_domain
* @property-read \Illuminate\Database\Eloquent\Collection<int, \App\Models\Activity> $activities
* @property-read int|null $activities_count
* @property-read \App\Models\User|null $assigned_user
@ -117,8 +117,8 @@ class Vendor extends BaseModel
'number',
'language_id',
'classification',
'expense_sender_email',
'expense_sender_domain',
'invoicing_email',
'invoicing_domain',
];
protected $casts = [

50
app/Services/IngresEmail/IngresEmail.php Normal file
View File

@ -0,0 +1,50 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Services\IngresEmail;
use Illuminate\Http\UploadedFile;
/**
* EmailObject.
*/
class IngresEmail
{
/** @var array[string] $args */
public array $to = [];
public string $from;
public array $reply_to = [];
/** @var array[string] $args */
public array $cc = [];
/** @var array[string] $args */
public array $bcc = [];
public ?string $subject = null;
public ?string $body = null;
public ?UploadedFile $body_document;
public string $text_body;
/** @var array[\Illuminate\Http\UploadedFile] $documents */
public array $documents = [];
public ?\DateTimeImmutable $date = null;
function __constructor()
{
}
}

158
app/Services/IngresEmail/IngresEmailEngine.php Normal file
View File

@ -0,0 +1,158 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Services\IngresEmail;
use App\Events\Expense\ExpenseWasCreated;
use App\Factory\ExpenseFactory;
use App\Libraries\MultiDB;
use App\Models\Client;
use App\Models\Company;
use App\Models\Vendor;
use App\Models\VendorContact;
use App\Services\Email\EmailObject;
use App\Services\IngresEmail\IngresEmail;
use App\Utils\Ninja;
use App\Utils\TempFile;
use App\Utils\Traits\GeneratesCounter;
use App\Utils\Traits\SavesDocuments;
use App\Utils\Traits\MakesHash;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class IngresEmailEngine implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, MakesHash;
use GeneratesCounter, SavesDocuments;
private IngresEmail $email;
private ?Company $company;
private array $globalBlacklist = [];
function __constructor(IngresEmail $email)
{
$this->email = $email;
}
/**
* if there is not a company with an matching mailbox, we do nothing
*/
public function handle()
{
// Expense Mailbox => will create an expense
foreach ($this->email->to as $expense_mailbox) {
$this->company = MultiDB::findAndSetDbByExpenseMailbox($expense_mailbox);
if (!$this->company || !$this->validateExpenseActive())
continue;
$this->createExpense();
}
// TODO reuse this method to add more mail-parsing behaviors
}
// MAIN-PROCESSORS
protected function createExpense()
{
if (!$this->validateExpenseSender()) {
nlog('invalid sender of an ingest email to company: ' . $this->company->id . ' from: ' . $this->email->from);
return;
}
$expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id);
$expense->public_notes = $this->email->subject;
$expense->private_notes = $this->email->text_body;
$expense->date = $this->email->date;
// handle vendor assignment
$expense_vendor = $this->getExpenseVendor();
if ($expense_vendor)
$expense->vendor_id = $expense_vendor->id;
// handle documents
$this->processHtmlBodyToDocument();
$documents = [];
array_push($documents, ...$this->email->documents);
if ($this->email->body_document)
$documents[] = $this->email->body_document;
$this->saveDocuments($documents, $expense);
$expense->saveQuietly();
event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); // @turbo124 please check, I copied from API
event('eloquent.created: App\Models\Expense', $expense); // @turbo124 please check, I copied from API
return $expense;
}
// HELPERS
private function processHtmlBodyToDocument()
{
if (!$this->email->body_document && property_exists($this->email, "body")) {
$this->email->body_document = TempFile::UploadedFileFromRaw($this->email->body, "E-Mail.html", "text/html");
}
}
private function validateExpenseActive()
{
return $this->company?->expense_mailbox_active ?: false;
}
private function validateExpenseSender()
{
// invalid email
if (!filter_var($this->email->from, FILTER_VALIDATE_EMAIL))
return false;
$parts = explode('@', $this->email->from);
$domain = array_pop($parts);
// global blacklist
if (in_array($domain, $this->globalBlacklist))
return false;
// whitelists
$email_whitelist = explode(",", $this->company->expense_mailbox_whitelist_emails);
if (in_array($this->email->from, $email_whitelist))
return true;
$domain_whitelist = explode(",", $this->company->expense_mailbox_whitelist_domains);
if (in_array($domain, $domain_whitelist))
return true;
if ($this->company->expense_mailbox_allow_unknown && sizeOf($email_whitelist) == 0 && sizeOf($domain_whitelist) == 0) // from unknown only, when no whitelists are defined
return true;
// own users
if ($this->company->expense_mailbox_allow_company_users && $this->company->users()->where("email", $this->email->from)->exists())
return true;
// from clients/vendors (if active)
if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->where("invoicing_email", $this->email->from)->orWhere($this->email->from, 'LIKE', "CONCAT('%',invoicing_domain)")->exists())
return true;
if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->contacts()->where("email", $this->email->from)->exists()) // TODO
return true;
// denie
return false;
}
private function getExpenseVendor()
{
$vendor = Vendor::where("company_id", $this->company->id)->where('invoicing_email', $this->email->from)->first();
if ($vendor == null)
$vendor = Vendor::where("company_id", $this->company->id)->where($this->email->from, 'LIKE', "CONCAT('%',invoicing_domain)")->first();
if ($vendor == null) {
$vendorContact = VendorContact::where("company_id", $this->company->id)->where("email", $this->email->from)->first();
$vendor = $vendorContact->vendor();
}
// TODO: from contacts
return $vendor;
}
}

4
app/Transformers/VendorTransformer.php
View File

@ -105,8 +105,8 @@ class VendorTransformer extends EntityTransformer
'language_id' => (string) $vendor->language_id ?: '',
'classification' => (string) $vendor->classification ?: '',
'display_name' => (string) $vendor->present()->name(),
'expense_sender_email' => (string) $vendor->expense_sender_email ?: '',
'expense_sender_domain' => (string) $vendor->expense_sender_domain ?: '',
'invoicing_email' => (string) $vendor->invoicing_email ?: '',
'invoicing_domain' => (string) $vendor->invoicing_domain ?: '',
];
}
}

11
database/migrations/2023_12_10_110951_create_imap_configuration_fields.php
View File

@ -12,8 +12,13 @@ return new class extends Migration {
public function up(): void
{
Schema::table('company', function (Blueprint $table) {
$table->boolean("expense_import")->default(true);
$table->boolean("expense_mailbox_active")->default(true);
$table->string("expense_mailbox")->nullable();
$table->boolean("expense_mailbox_allow_company_users")->default(false);
$table->boolean("expense_mailbox_allow_vendors")->default(false);
$table->boolean("expense_mailbox_allow_unknown")->default(false);
$table->string("expense_mailbox_whitelist_domains")->nullable();
$table->string("expense_mailbox_whitelist_emails")->nullable();
});
Company::query()->cursor()->each(function ($company) { // TODO: @turbo124 check migration on staging environment with real data to ensure, this works as exspected
$company->expense_mailbox = config('ninja.inbound_expense.webhook.mailbox_template') != '' ?
@ -22,8 +27,8 @@ return new class extends Migration {
$company->save();
});
Schema::table('vendor', function (Blueprint $table) {
$table->string("expense_sender_email")->nullable();
$table->string("expense_sender_domain")->nullable();
$table->string("invoicing_email")->nullable();
$table->string("invoicing_domain")->nullable();
});
}