wip: restruct and init IngresEmailEngine

app/Helpers/IngresMail/Transformer/ImapMailTransformer.php

@@ -0,0 +1,38 @@
+<?php
+/**
+ * Invoice Ninja (https://invoiceninja.com).
+ *
+ * @link https://github.com/invoiceninja/invoiceninja source repository
+ *
+ * @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
+ *
+ * @license https://www.elastic.co/licensing/elastic-license
+ */
+
+namespace App\Helpers\IngresMail\Transformer;
+
+use App\Services\IngresEmail\IngresEmail;
+use App\Utils\TempFile;
+use Ddeboer\Imap\MessageInterface;
+
+class ImapMailTransformer
+{
+
+    public function transform(MessageInterface $mail)
+    {
+        $ingresEmail = new IngresEmail();
+
+        $ingresEmail->from = $mail->getSender();
+        $ingresEmail->subject = $mail->getSubject();
+        $ingresEmail->plain_message = $mail->getBodyText();
+        $ingresEmail->html_message = $mail->getBodyHtml();
+        $ingresEmail->date = $mail->getDate();
+
+        // parse documents as UploadedFile
+        foreach ($mail->getAttachments() as $attachment) {
+            $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment->getContent(), $attachment->getFilename(), $attachment->getEncoding());
+        }
+
+        return $ingresEmail;
+    }
+}

app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php

@@ -0,0 +1,36 @@
+<?php
+/**
+ * Invoice Ninja (https://invoiceninja.com).
+ *
+ * @link https://github.com/invoiceninja/invoiceninja source repository
+ *
+ * @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
+ *
+ * @license https://www.elastic.co/licensing/elastic-license
+ */
+
+namespace App\Helpers\IngresMail\Transformer;
+
+use App\Services\IngresEmail\IngresEmail;
+use App\Utils\TempFile;
+
+class MailgunInboundWebhookTransformer
+{
+    public function transform($data)
+    {
+        $ingresEmail = new IngresEmail();
+
+        $ingresEmail->from = $data["sender"];
+        $ingresEmail->subject = $data["subject"];
+        $ingresEmail->plain_message = $data["body-plain"];
+        $ingresEmail->html_message = $data["body-html"];
+        $ingresEmail->date = now(); // TODO
+
+        // parse documents as UploadedFile from webhook-data
+        foreach ($data["Attachments"] as $attachment) {
+            $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]);
+        }
+
+        return $ingresEmail;
+    }
+}

app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php

@@ -9,13 +9,32 @@
  * @license https://www.elastic.co/licensing/elastic-license
  */
 
-namespace App\Helpers\Mail\Webhook\Postmark;
+namespace App\Helpers\IngresMail\Transformer;
 
-use App\Helpers\Mail\Webhook\BaseWebhookHandler;
+use App\Services\IngresEmail\IngresEmail;
 use App\Utils\TempFile;
 
-class PostmarkWebhookHandler extends BaseWebhookHandler
+class PostmarkInboundWebhookTransformer
 {
+    public function process($data)
+    {
+
+        $ingresEmail = new IngresEmail();
+
+        $ingresEmail->from = $data["From"];
+        $ingresEmail->subject = $data["Subject"];
+        $ingresEmail->plain_message = $data["TextBody"];
+        $ingresEmail->html_message = $data["HtmlBody"];
+        $ingresEmail->date = $data["Date"]; // TODO: parsing
+
+        // parse documents as UploadedFile from webhook-data
+        foreach ($data["Attachments"] as $attachment) {
+            $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]);
+        }
+
+        return $ingresEmail;
+
+    }
     // {
     //   "FromName": "Postmarkapp Support",
     //   "MessageStream": "inbound",
@@ -96,29 +115,4 @@ class PostmarkWebhookHandler extends BaseWebhookHandler
     //     }
     //   ]
     // }
-    public function process($data)
-    {
-
-        $from = $data["From"];
-        $subject = $data["Subject"];
-        $plain_message = $data["TextBody"];
-        $html_message = $data["HtmlBody"];
-        $date = $data["Date"]; // TODO
-
-        // parse documents as UploadedFile from webhook-data
-        $documents = [];
-        foreach ($data["Attachments"] as $attachment) {
-            $documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]);
-        }
-
-        return $this->createExpense(
-            $from,
-            $subject,
-            $plain_message,
-            $html_message,
-            $date,
-            $documents,
-        );
-
-    }
 }

app/Helpers/Mail/Webhook/BaseWebhookHandler.php

@@ -1,58 +0,0 @@
-<?php
-/**
- * Invoice Ninja (https://invoiceninja.com).
- *
- * @link https://github.com/invoiceninja/invoiceninja source repository
- *
- * @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
- *
- * @license https://www.elastic.co/licensing/elastic-license
- */
-
-namespace App\Helpers\Mail\Webhook;
-
-use App\Events\Expense\ExpenseWasCreated;
-use App\Factory\ExpenseFactory;
-use App\Models\Company;
-use App\Utils\Ninja;
-use App\Utils\TempFile;
-use App\Utils\Traits\GeneratesCounter;
-use App\Utils\Traits\SavesDocuments;
-
-abstract class BaseWebhookHandler
-{
-    use GeneratesCounter, SavesDocuments;
-    public function process()
-    {
-
-    }
-    protected function createExpense(string $email, string $subject, string $plain_message, string $html_message, string $date, array $documents)
-    {
-        $company = $this->matchCompany($email);
-        if (!$company)
-            return false;
-
-        $expense = ExpenseFactory::create($company->id, $company->owner()->id);
-
-        $expense->public_notes = $subject;
-        $expense->private_notes = $plain_message;
-        $expense->date = $date;
-
-        // add html_message as document to the expense
-        $documents[] = TempFile::UploadedFileFromRaw($html_message, "E-Mail.html", "text/html");
-
-        $this->saveDocuments($documents, $expense);
-
-        $expense->saveQuietly();
-
-        event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null)));
-        event('eloquent.created: App\Models\Expense', $expense);
-
-        return $expense;
-    }
-
-    private function matchCompany(string $email)
-    {
-        return Company::where("expense_mailbox", $email)->first();
-    }
-}

app/Helpers/Mail/Webhook/Mailgun/MailgunWebhookHandler.php

@@ -1,44 +0,0 @@
-<?php
-/**
- * Invoice Ninja (https://invoiceninja.com).
- *
- * @link https://github.com/invoiceninja/invoiceninja source repository
- *
- * @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
- *
- * @license https://www.elastic.co/licensing/elastic-license
- */
-
-namespace App\Helpers\Mail\Webhook\Maigun;
-
-use App\Helpers\Mail\Webhook\BaseWebhookHandler;
-use App\Utils\TempFile;
-
-class MailgunWebhookHandler extends BaseWebhookHandler
-{
-    public function process($data)
-    {
-
-        $from = $data["sender"];
-        $subject = $data["subject"];
-        $plain_message = $data["body-plain"];
-        $html_message = $data["body-html"];
-        $date = now(); // TODO
-
-        // parse documents as UploadedFile from webhook-data
-        $documents = [];
-        foreach ($data["Attachments"] as $attachment) {
-            $documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]);
-        }
-
-        return $this->createExpense(
-            $from,
-            $subject,
-            $plain_message,
-            $html_message,
-            $date,
-            $documents,
-        );
-
-    }
-}

app/Jobs/Mail/ExpenseMailboxJob.php

@@ -11,15 +11,12 @@
 
 namespace App\Jobs\Mail;
 
-use App\Events\Expense\ExpenseWasCreated;
+use App\Helpers\IngresMail\Transformer\ImapMailTransformer;
-use App\Factory\ExpenseFactory;
 use App\Helpers\Mail\Mailbox\Imap\ImapMailbox;
 use App\Libraries\MultiDB;
 use App\Models\Company;
-use App\Models\Vendor;
 use App\Repositories\ExpenseRepository;
-use App\Utils\Ninja;
+use App\Services\IngresEmail\IngresEmailEngine;
-use App\Utils\TempFile;
 use App\Utils\Traits\MakesHash;
 use App\Utils\Traits\SavesDocuments;
 use Illuminate\Bus\Queueable;
@@ -87,6 +84,7 @@ class ExpenseMailboxJob implements ShouldQueue
             throw new \Exception('invalid configuration inbound_expense.imap (wrong element-count)');
 
         foreach ($companies as $index => $companyId) {
+
             if ($servers[$index] == '') // if property is empty, ignore => this happens exspecialy when no config is provided and it enabled us to set a single default company for env (usefull on self-hosted)
                 continue;
 
@@ -97,6 +95,7 @@ class ExpenseMailboxJob implements ShouldQueue
                 "password" => $passwords[$index],
             ];
             $this->imap_companies[] = $companyId;
+
         }
     }
 
@@ -106,55 +105,25 @@ class ExpenseMailboxJob implements ShouldQueue
 
         $credentials = $this->imap_credentials[$company->id];
         $imapMailbox = new ImapMailbox($credentials->server, $credentials->port, $credentials->user, $credentials->password);
+        $transformer = new ImapMailTransformer();
 
         $emails = $imapMailbox->getUnprocessedEmails();
 
-        foreach ($emails as $mail) {
+
+        foreach ($emails as $email) {
 
             try {
 
-                $sender = $mail->getSender();
+                $email->markAsSeen();
 
-                $vendor = Vendor::where('expense_sender_email', $sender)->first();
+                IngresEmailEngine::dispatch($transformer->transform($email));
-                if ($vendor == null)
-                    $vendor = Vendor::where($sender, 'LIKE', "CONCAT('%',expense_sender_domain)")->first();
-                if ($vendor == null)
-                    $vendor = Vendor::where("email", $sender)->first();
 
-                $documents = []; // TODO: $mail->getAttachments() + save email as document (.html)
+                $imapMailbox->moveProcessed($email);
-
-                $data = [
-                    "vendor_id" => $vendor !== null ? $vendor->id : null,
-                    "date" => $mail->getDate(),
-                    "public_notes" => $mail->getSubject(),
-                    "private_notes" => $mail->getCompleteBodyText(),
-                    "documents" => $documents, // FIXME: https://github.com/ddeboer/imap?tab=readme-ov-file#message-attachments
-                ];
-
-                $expense = ExpenseFactory::create($company->company->id, $company->company->owner()->id);
-
-                $expense->vendor_id = $vendor !== null ? $vendor->id : null;
-                $expense->public_notes = $mail->getSubject();
-                $expense->private_notes = $mail->getBodyText();
-                $expense->date = $mail->getDate();
-
-                // add html_message as document to the expense
-                $documents[] = TempFile::UploadedFileFromRaw($mail->getBodyHtml(), "E-Mail.html", "text/html");
-
-                $this->saveDocuments($documents, $expense);
-
-                $expense->saveQuietly();
-
-                event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null)));
-                event('eloquent.created: App\Models\Expense', $expense);
-
-                $mail->markAsSeen();
-                $imapMailbox->moveProcessed($mail);
 
             } catch (\Exception $e) {
-                $imapMailbox->moveFailed($mail);
+                $imapMailbox->moveFailed($email);
 
-                nlog("processing of an email failed upnormally: " . $company->id . " message: " . $e->getMessage()); // @turbo124 @todo should this be handled in an other way?
+                nlog("processing of an email failed upnormally: " . $company->id . " message: " . $e->getMessage());
             }
 
         }

app/Models/Company.php

@@ -111,8 +111,13 @@ use Laracasts\Presenter\PresentableTrait;
  * @property int $convert_expense_currency
  * @property int $notify_vendor_when_paid
  * @property int $invoice_task_hours
- * @property boolean $expense_import
  * @property string|null $expense_mailbox
+ * @property boolean $expense_mailbox_active
+ * @property bool $expense_mailbox_allow_company_users
+ * @property bool $expense_mailbox_allow_vendors
+ * @property bool $expense_mailbox_allow_unknown
+ * @property string|null $expense_mailbox_whitelist_domains
+ * @property string|null $expense_mailbox_whitelist_emails
  * @property int $deleted_at
  * @property-read \App\Models\Account $account
  * @property-read \Illuminate\Database\Eloquent\Collection<int, \App\Models\Activity> $activities
@@ -354,8 +359,14 @@ class Company extends BaseModel
         'calculate_taxes',
         'tax_data',
         'e_invoice_certificate_passphrase',
-        'expense_import',
+        'expense_mailbox_active',
         'expense_mailbox', // TODO: @turbo124 custom validation: self-hosted => free change, hosted => not changeable, only changeable with env-mask
+        'expense_mailbox_allow_company_users',
+        'expense_mailbox_allow_vendors',
+        'expense_mailbox_allow_unknown',
+        'expense_mailbox_whitelist_domains',
+        'expense_mailbox_whitelist_emails',
+        'expense_mailbox_whitelist'
     ];
 
     protected $hidden = [

app/Models/Vendor.php

@@ -54,8 +54,8 @@ use Laracasts\Presenter\PresentableTrait;
  * @property string|null $id_number
  * @property string|null $language_id
  * @property int|null $last_login
- * @property string|null $expense_sender_email
+ * @property string|null $invoicing_email
- * @property string|null $expense_sender_domain
+ * @property string|null $invoicing_domain
  * @property-read \Illuminate\Database\Eloquent\Collection<int, \App\Models\Activity> $activities
  * @property-read int|null $activities_count
  * @property-read \App\Models\User|null $assigned_user
@@ -117,8 +117,8 @@ class Vendor extends BaseModel
         'number',
         'language_id',
         'classification',
-        'expense_sender_email',
+        'invoicing_email',
-        'expense_sender_domain',
+        'invoicing_domain',
     ];
 
     protected $casts = [

app/Services/IngresEmail/IngresEmail.php

@@ -0,0 +1,50 @@
+<?php
+/**
+ * Invoice Ninja (https://invoiceninja.com).
+ *
+ * @link https://github.com/invoiceninja/invoiceninja source repository
+ *
+ * @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
+ *
+ * @license https://www.elastic.co/licensing/elastic-license
+ */
+
+namespace App\Services\IngresEmail;
+
+use Illuminate\Http\UploadedFile;
+
+/**
+ * EmailObject.
+ */
+class IngresEmail
+{
+    /** @var array[string] $args */
+    public array $to = [];
+
+    public string $from;
+
+    public array $reply_to = [];
+
+    /** @var array[string] $args */
+    public array $cc = [];
+
+    /** @var array[string] $args */
+    public array $bcc = [];
+
+    public ?string $subject = null;
+
+    public ?string $body = null;
+    public ?UploadedFile $body_document;
+
+    public string $text_body;
+
+    /** @var array[\Illuminate\Http\UploadedFile] $documents */
+    public array $documents = [];
+
+    public ?\DateTimeImmutable $date = null;
+
+    function __constructor()
+    {
+
+    }
+}

app/Services/IngresEmail/IngresEmailEngine.php

@@ -0,0 +1,158 @@
+<?php
+/**
+ * Invoice Ninja (https://invoiceninja.com).
+ *
+ * @link https://github.com/invoiceninja/invoiceninja source repository
+ *
+ * @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
+ *
+ * @license https://www.elastic.co/licensing/elastic-license
+ */
+
+namespace App\Services\IngresEmail;
+
+use App\Events\Expense\ExpenseWasCreated;
+use App\Factory\ExpenseFactory;
+use App\Libraries\MultiDB;
+use App\Models\Client;
+use App\Models\Company;
+use App\Models\Vendor;
+use App\Models\VendorContact;
+use App\Services\Email\EmailObject;
+use App\Services\IngresEmail\IngresEmail;
+use App\Utils\Ninja;
+use App\Utils\TempFile;
+use App\Utils\Traits\GeneratesCounter;
+use App\Utils\Traits\SavesDocuments;
+use App\Utils\Traits\MakesHash;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+class IngresEmailEngine implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, MakesHash;
+    use GeneratesCounter, SavesDocuments;
+
+    private IngresEmail $email;
+    private ?Company $company;
+    private array $globalBlacklist = [];
+    function __constructor(IngresEmail $email)
+    {
+        $this->email = $email;
+    }
+    /**
+     * if there is not a company with an matching mailbox, we do nothing
+     */
+    public function handle()
+    {
+        // Expense Mailbox => will create an expense
+        foreach ($this->email->to as $expense_mailbox) {
+            $this->company = MultiDB::findAndSetDbByExpenseMailbox($expense_mailbox);
+            if (!$this->company || !$this->validateExpenseActive())
+                continue;
+
+            $this->createExpense();
+        }
+
+        // TODO reuse this method to add more mail-parsing behaviors
+    }
+
+    // MAIN-PROCESSORS
+    protected function createExpense()
+    {
+        if (!$this->validateExpenseSender()) {
+            nlog('invalid sender of an ingest email to company: ' . $this->company->id . ' from: ' . $this->email->from);
+            return;
+        }
+
+        $expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id);
+
+        $expense->public_notes = $this->email->subject;
+        $expense->private_notes = $this->email->text_body;
+        $expense->date = $this->email->date;
+
+        // handle vendor assignment
+        $expense_vendor = $this->getExpenseVendor();
+        if ($expense_vendor)
+            $expense->vendor_id = $expense_vendor->id;
+
+        // handle documents
+        $this->processHtmlBodyToDocument();
+        $documents = [];
+        array_push($documents, ...$this->email->documents);
+        if ($this->email->body_document)
+            $documents[] = $this->email->body_document;
+        $this->saveDocuments($documents, $expense);
+
+        $expense->saveQuietly();
+
+        event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); // @turbo124 please check, I copied from API
+        event('eloquent.created: App\Models\Expense', $expense); // @turbo124 please check, I copied from API
+
+        return $expense;
+    }
+
+    // HELPERS
+    private function processHtmlBodyToDocument()
+    {
+        if (!$this->email->body_document && property_exists($this->email, "body")) {
+            $this->email->body_document = TempFile::UploadedFileFromRaw($this->email->body, "E-Mail.html", "text/html");
+        }
+    }
+    private function validateExpenseActive()
+    {
+        return $this->company?->expense_mailbox_active ?: false;
+    }
+    private function validateExpenseSender()
+    {
+        // invalid email
+        if (!filter_var($this->email->from, FILTER_VALIDATE_EMAIL))
+            return false;
+
+        $parts = explode('@', $this->email->from);
+        $domain = array_pop($parts);
+
+        // global blacklist
+        if (in_array($domain, $this->globalBlacklist))
+            return false;
+
+        // whitelists
+        $email_whitelist = explode(",", $this->company->expense_mailbox_whitelist_emails);
+        if (in_array($this->email->from, $email_whitelist))
+            return true;
+        $domain_whitelist = explode(",", $this->company->expense_mailbox_whitelist_domains);
+        if (in_array($domain, $domain_whitelist))
+            return true;
+        if ($this->company->expense_mailbox_allow_unknown && sizeOf($email_whitelist) == 0 && sizeOf($domain_whitelist) == 0) // from unknown only, when no whitelists are defined
+            return true;
+
+        // own users
+        if ($this->company->expense_mailbox_allow_company_users && $this->company->users()->where("email", $this->email->from)->exists())
+            return true;
+
+        // from clients/vendors (if active)
+        if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->where("invoicing_email", $this->email->from)->orWhere($this->email->from, 'LIKE', "CONCAT('%',invoicing_domain)")->exists())
+            return true;
+        if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->contacts()->where("email", $this->email->from)->exists()) // TODO
+            return true;
+
+        // denie
+        return false;
+    }
+    private function getExpenseVendor()
+    {
+        $vendor = Vendor::where("company_id", $this->company->id)->where('invoicing_email', $this->email->from)->first();
+        if ($vendor == null)
+            $vendor = Vendor::where("company_id", $this->company->id)->where($this->email->from, 'LIKE', "CONCAT('%',invoicing_domain)")->first();
+        if ($vendor == null) {
+            $vendorContact = VendorContact::where("company_id", $this->company->id)->where("email", $this->email->from)->first();
+            $vendor = $vendorContact->vendor();
+        }
+        // TODO: from contacts
+
+        return $vendor;
+    }
+}

app/Transformers/VendorTransformer.php

@@ -105,8 +105,8 @@ class VendorTransformer extends EntityTransformer
             'language_id' => (string) $vendor->language_id ?: '',
             'classification' => (string) $vendor->classification ?: '',
             'display_name' => (string) $vendor->present()->name(),
-            'expense_sender_email' => (string) $vendor->expense_sender_email ?: '',
+            'invoicing_email' => (string) $vendor->invoicing_email ?: '',
-            'expense_sender_domain' => (string) $vendor->expense_sender_domain ?: '',
+            'invoicing_domain' => (string) $vendor->invoicing_domain ?: '',
         ];
     }
 }

database/migrations/2023_12_10_110951_create_imap_configuration_fields.php

@@ -12,8 +12,13 @@ return new class extends Migration {
     public function up(): void
     {
         Schema::table('company', function (Blueprint $table) {
-            $table->boolean("expense_import")->default(true);
+            $table->boolean("expense_mailbox_active")->default(true);
             $table->string("expense_mailbox")->nullable();
+            $table->boolean("expense_mailbox_allow_company_users")->default(false);
+            $table->boolean("expense_mailbox_allow_vendors")->default(false);
+            $table->boolean("expense_mailbox_allow_unknown")->default(false);
+            $table->string("expense_mailbox_whitelist_domains")->nullable();
+            $table->string("expense_mailbox_whitelist_emails")->nullable();
         });
         Company::query()->cursor()->each(function ($company) { // TODO: @turbo124 check migration on staging environment with real data to ensure, this works as exspected
             $company->expense_mailbox = config('ninja.inbound_expense.webhook.mailbox_template') != '' ?
@@ -22,8 +27,8 @@ return new class extends Migration {
             $company->save();
         });
         Schema::table('vendor', function (Blueprint $table) {
-            $table->string("expense_sender_email")->nullable();
+            $table->string("invoicing_email")->nullable();
-            $table->string("expense_sender_domain")->nullable();
+            $table->string("invoicing_domain")->nullable();
         });
     }