initial commit

app/Console/Kernel.php

@@ -17,6 +17,7 @@ use App\Jobs\Cron\RecurringInvoicesCron;
 use App\Jobs\Cron\SubscriptionCron;
 use App\Jobs\Cron\UpdateCalculatedFields;
 use App\Jobs\Invoice\InvoiceCheckLateWebhook;
+use App\Jobs\Mail\ExpenseImportJob;
 use App\Jobs\Ninja\AdjustEmailQuota;
 use App\Jobs\Ninja\BankTransactionSync;
 use App\Jobs\Ninja\CheckACHStatus;
@@ -120,11 +121,13 @@ class Kernel extends ConsoleKernel
             $schedule->command('ninja:s3-cleanup')->dailyAt('23:15')->withoutOverlapping()->name('s3-cleanup-job')->onOneServer();
         }
 
-        if (config('queue.default') == 'database' && Ninja::isSelfHost() && config('ninja.internal_queue_enabled') && ! config('ninja.is_docker')) {
+        if (config('queue.default') == 'database' && Ninja::isSelfHost() && config('ninja.internal_queue_enabled') && !config('ninja.is_docker')) {
             $schedule->command('queue:work database --stop-when-empty --memory=256')->everyMinute()->withoutOverlapping();
 
             $schedule->command('queue:restart')->everyFiveMinutes()->withoutOverlapping();
         }
+
+        $schedule->job(new ExpenseImportJob)->everyThirtyMinutes()->withoutOverlapping()->name('expense-import-job')->onOneServer();
     }
 
     /**
@@ -134,7 +137,7 @@ class Kernel extends ConsoleKernel
      */
     protected function commands()
     {
-        $this->load(__DIR__.'/Commands');
+        $this->load(__DIR__ . '/Commands');
 
         require base_path('routes/console.php');
     }

app/Helpers/Mail/IncomingMailHandler.php

@@ -0,0 +1,57 @@
+<?php
+/**
+ * Invoice Ninja (https://invoiceninja.com).
+ *
+ * @link https://github.com/invoiceninja/invoiceninja source repository
+ *
+ * @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
+ *
+ * @license https://www.elastic.co/licensing/elastic-license
+ */
+
+namespace App\Helpers\Mail;
+
+use Ddeboer\Imap\MessageInterface;
+use Ddeboer\Imap\Server;
+use Ddeboer\Imap\SearchExpression;
+use Ddeboer\Imap\Search\Date\Since;
+use Ddeboer\Imap\Search\Flag\Unflagged;
+
+/**
+ * GmailTransport.
+ */
+class IncomingMailHandler
+{
+    private $server;
+    public $connection;
+    public function __construct(string $server, string $user, string $password)
+    {
+        $this->server = new Server($server);
+
+        $this->connection = $this->server->authenticate($user, $password);
+    }
+
+
+    public function getUnprocessedEmails()
+    {
+        $mailbox = $this->connection->getMailbox('INBOX');
+
+        $search = new SearchExpression();
+
+        // not older than 30days
+        $today = new \DateTimeImmutable();
+        $thirtyDaysAgo = $today->sub(new \DateInterval('P30D'));
+        $search->addCondition(new Since($thirtyDaysAgo));
+
+        // not flagged with IN-PARSED
+        $search->addCondition(new Unflagged());
+
+
+        return $mailbox->getMessages($search);
+    }
+
+    public function moveProcessed(MessageInterface $mail)
+    {
+        return $mail->move($this->connection->getMailbox('PROCESSED'));
+    }
+}

app/Jobs/Mail/ExpenseImportJob.php

@@ -0,0 +1,131 @@
+<?php
+/**
+ * Invoice Ninja (https://invoiceninja.com).
+ *
+ * @link https://github.com/invoiceninja/invoiceninja source repository
+ *
+ * @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
+ *
+ * @license https://www.elastic.co/licensing/elastic-license
+ */
+
+namespace App\Jobs\Mail;
+
+use App\DataMapper\Analytics\EmailFailure;
+use App\DataMapper\Analytics\EmailSuccess;
+use App\Events\Expense\ExpenseWasCreated;
+use App\Events\Invoice\InvoiceWasEmailedAndFailed;
+use App\Events\Payment\PaymentWasEmailedAndFailed;
+use App\Factory\ExpenseFactory;
+use App\Helpers\Mail\IncomingMailHandler;
+use App\Jobs\Util\SystemLogger;
+use App\Libraries\Google\Google;
+use App\Libraries\MultiDB;
+use App\Models\Account;
+use App\Models\ClientContact;
+use App\Models\Company;
+use App\Models\Expense;
+use App\Models\Invoice;
+use App\Models\Payment;
+use App\Models\SystemLog;
+use App\Models\User;
+use App\Models\Vendor;
+use App\Repositories\ExpenseRepository;
+use App\Utils\Ninja;
+use App\Utils\Traits\MakesHash;
+use GuzzleHttp\Exception\ClientException;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\App;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Mail;
+use Turbo124\Beacon\Facades\LightLogs;
+
+/*Multi Mailer implemented*/
+
+class ExpenseImportJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, MakesHash;
+
+    public $tries = 4; //number of retries
+
+    public $deleteWhenMissingModels = true;
+
+    /** @var null|\App\Models\Company $company  **/
+    public Company $company;
+
+    private $expense_repo;
+
+    public function __construct()
+    {
+
+        $this->expense_repo = new ExpenseRepository();
+    }
+
+    public function backoff()
+    {
+        // return [5, 10, 30, 240];
+        return [rand(5, 10), rand(30, 40), rand(60, 79), rand(160, 400)];
+
+    }
+
+    public function handle()
+    {
+
+        //multiDB environment, need to
+        foreach (MultiDB::$dbs as $db) {
+            MultiDB::setDB($db);
+
+            nlog("importing expenses from imap-servers");
+
+            $a = Account::with('companies')->cursor()->each(function ($account) {
+                $account->companies()->where('expense_import', true)->whereNotNull('expense_mailbox_imap_host')->whereNotNull('expense_mailbox_imap_user')->whereNotNull('expense_mailbox_imap_password')->cursor()->each(function ($company) {
+                    $this->handleCompanyImap($company);
+                });
+            });
+        }
+
+    }
+
+    private function handleCompanyImap(Company $company)
+    {
+        $incommingMails = new IncomingMailHandler($company->expense_mailbox_imap_host, $company->company->expense_mailbox_imap_user, $company->company->expense_mailbox_imap_password);
+
+        $emails = $incommingMails->getUnprocessedEmails();
+
+        foreach ($emails as $mail) {
+
+            $sender = $mail->getSender();
+
+            $vendor = Vendor::where('expense_sender_email', $sender)->orWhere($sender, 'LIKE', "CONCAT('%',expense_sender_email)")->first();
+
+            if ($vendor !== null)
+                $vendor = Vendor::where("email", $sender)->first();
+
+            // TODO: check email for existing vendor?!
+            $data = [
+                "vendor_id" => $vendor !== null ? $vendor->id : null,
+                "date" => $mail->getDate(),
+                "public_notes" => $mail->getSubject(),
+                "private_notes" => $mail->getCompleteBodyText(),
+                "documents" => $mail->getAttachments(), // FIXME: https://github.com/ddeboer/imap?tab=readme-ov-file#message-attachments
+            ];
+
+            $expense = $this->expense_repo->save($data, ExpenseFactory::create($company->company->id, $company->company->owner()->id)); // TODO: dont assign a new number at beginning
+
+            // TODO: check for recurring expense?! => maybe replace existing ?!
+
+            event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null)));
+
+            event('eloquent.created: App\Models\Expense', $expense);
+
+            $mail->markAsSeen();
+            $incommingMails->moveProcessed($mail);
+
+        }
+    }
+
+}

composer.json

@@ -44,6 +44,7 @@
         "braintree/braintree_php": "^6.0",
         "checkout/checkout-sdk-php": "^3.0",
         "cleverit/ubl_invoice": "^1.3",
+        "ddeboer/imap": "^1.19",
         "doctrine/dbal": "^3.0",
         "eway/eway-rapid-php": "^1.3",
         "fakerphp/faker": "^1.14",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "28b57fe6eac3d71c607125cda9a6a537",
+    "content-hash": "ef5c36f71295ade916c3b7084642f0b4",
     "packages": [
         {
             "name": "afosto/yaac",
@@ -1176,6 +1176,82 @@
             },
             "time": "2023-08-25T16:18:39+00:00"
         },
+        {
+            "name": "ddeboer/imap",
+            "version": "1.19.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/ddeboer/imap.git",
+                "reference": "30800b1cfeacc4add5bb418e40a8b6e95a8a04ac"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/ddeboer/imap/zipball/30800b1cfeacc4add5bb418e40a8b6e95a8a04ac",
+                "reference": "30800b1cfeacc4add5bb418e40a8b6e95a8a04ac",
+                "shasum": ""
+            },
+            "require": {
+                "ext-dom": "*",
+                "ext-iconv": "*",
+                "ext-imap": "*",
+                "ext-libxml": "*",
+                "ext-mbstring": "*",
+                "php": "~8.2.0 || ~8.3.0"
+            },
+            "require-dev": {
+                "friendsofphp/php-cs-fixer": "^3.38.2",
+                "laminas/laminas-mail": "^2.25.1",
+                "phpstan/phpstan": "^1.10.43",
+                "phpstan/phpstan-phpunit": "^1.3.15",
+                "phpstan/phpstan-strict-rules": "^1.5.2",
+                "phpunit/phpunit": "^10.4.2"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Ddeboer\\Imap\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "David de Boer",
+                    "email": "david@ddeboer.nl"
+                },
+                {
+                    "name": "Filippo Tessarotto",
+                    "email": "zoeslam@gmail.com"
+                },
+                {
+                    "name": "Community contributors",
+                    "homepage": "https://github.com/ddeboer/imap/graphs/contributors"
+                }
+            ],
+            "description": "Object-oriented IMAP for PHP",
+            "keywords": [
+                "email",
+                "imap",
+                "mail"
+            ],
+            "support": {
+                "issues": "https://github.com/ddeboer/imap/issues",
+                "source": "https://github.com/ddeboer/imap/tree/1.19.0"
+            },
+            "funding": [
+                {
+                    "url": "https://github.com/Slamdunk",
+                    "type": "github"
+                },
+                {
+                    "url": "https://github.com/ddeboer",
+                    "type": "github"
+                }
+            ],
+            "time": "2023-11-20T14:41:54+00:00"
+        },
         {
             "name": "dflydev/dot-access-data",
             "version": "v3.0.2",
@@ -18046,5 +18122,5 @@
     "platform-dev": {
         "php": "^8.1|^8.2"
     },
-    "plugin-api-version": "2.3.0"
+    "plugin-api-version": "2.6.0"
 }

database/migrations/2023_12_10_110951_create_imap_configuration_fields.php

@@ -0,0 +1,32 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration {
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('company', function (Blueprint $table) {
+            $table->string("expense_mailbox_imap_host")->nullable();
+            $table->string("expense_mailbox_imap_port")->nullable();
+            $table->string("expense_mailbox_imap_user")->nullable();
+            $table->string("expense_mailbox_imap_password")->nullable();
+        });
+        Schema::table('vendor', function (Blueprint $table) {
+            $table->string("expense_sender_email")->nullable();
+            $table->string("expense_sender_url")->nullable();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        //
+    }
+};