From b2b1cce085708093019a64e6aa7a13a6637cf686 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 10 Dec 2023 16:06:33 +0100 Subject: [PATCH] initial commit --- app/Console/Kernel.php | 7 +- app/Helpers/Mail/IncomingMailHandler.php | 57 ++++++++ app/Jobs/Mail/ExpenseImportJob.php | 131 ++++++++++++++++++ composer.json | 3 +- composer.lock | 80 ++++++++++- ...10951_create_imap_configuration_fields.php | 32 +++++ 6 files changed, 305 insertions(+), 5 deletions(-) create mode 100644 app/Helpers/Mail/IncomingMailHandler.php create mode 100644 app/Jobs/Mail/ExpenseImportJob.php create mode 100644 database/migrations/2023_12_10_110951_create_imap_configuration_fields.php diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index f67dc4af685b..7d883dcbf945 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -17,6 +17,7 @@ use App\Jobs\Cron\RecurringInvoicesCron; use App\Jobs\Cron\SubscriptionCron; use App\Jobs\Cron\UpdateCalculatedFields; use App\Jobs\Invoice\InvoiceCheckLateWebhook; +use App\Jobs\Mail\ExpenseImportJob; use App\Jobs\Ninja\AdjustEmailQuota; use App\Jobs\Ninja\BankTransactionSync; use App\Jobs\Ninja\CheckACHStatus; @@ -120,11 +121,13 @@ class Kernel extends ConsoleKernel $schedule->command('ninja:s3-cleanup')->dailyAt('23:15')->withoutOverlapping()->name('s3-cleanup-job')->onOneServer(); } - if (config('queue.default') == 'database' && Ninja::isSelfHost() && config('ninja.internal_queue_enabled') && ! config('ninja.is_docker')) { + if (config('queue.default') == 'database' && Ninja::isSelfHost() && config('ninja.internal_queue_enabled') && !config('ninja.is_docker')) { $schedule->command('queue:work database --stop-when-empty --memory=256')->everyMinute()->withoutOverlapping(); $schedule->command('queue:restart')->everyFiveMinutes()->withoutOverlapping(); } + + $schedule->job(new ExpenseImportJob)->everyThirtyMinutes()->withoutOverlapping()->name('expense-import-job')->onOneServer(); } /** @@ -134,7 +137,7 @@ class Kernel extends ConsoleKernel */ protected function commands() { - $this->load(__DIR__.'/Commands'); + $this->load(__DIR__ . '/Commands'); require base_path('routes/console.php'); } diff --git a/app/Helpers/Mail/IncomingMailHandler.php b/app/Helpers/Mail/IncomingMailHandler.php new file mode 100644 index 000000000000..3c98293d7714 --- /dev/null +++ b/app/Helpers/Mail/IncomingMailHandler.php @@ -0,0 +1,57 @@ +server = new Server($server); + + $this->connection = $this->server->authenticate($user, $password); + } + + + public function getUnprocessedEmails() + { + $mailbox = $this->connection->getMailbox('INBOX'); + + $search = new SearchExpression(); + + // not older than 30days + $today = new \DateTimeImmutable(); + $thirtyDaysAgo = $today->sub(new \DateInterval('P30D')); + $search->addCondition(new Since($thirtyDaysAgo)); + + // not flagged with IN-PARSED + $search->addCondition(new Unflagged()); + + + return $mailbox->getMessages($search); + } + + public function moveProcessed(MessageInterface $mail) + { + return $mail->move($this->connection->getMailbox('PROCESSED')); + } +} diff --git a/app/Jobs/Mail/ExpenseImportJob.php b/app/Jobs/Mail/ExpenseImportJob.php new file mode 100644 index 000000000000..a31c2bace073 --- /dev/null +++ b/app/Jobs/Mail/ExpenseImportJob.php @@ -0,0 +1,131 @@ +expense_repo = new ExpenseRepository(); + } + + public function backoff() + { + // return [5, 10, 30, 240]; + return [rand(5, 10), rand(30, 40), rand(60, 79), rand(160, 400)]; + + } + + public function handle() + { + + //multiDB environment, need to + foreach (MultiDB::$dbs as $db) { + MultiDB::setDB($db); + + nlog("importing expenses from imap-servers"); + + $a = Account::with('companies')->cursor()->each(function ($account) { + $account->companies()->where('expense_import', true)->whereNotNull('expense_mailbox_imap_host')->whereNotNull('expense_mailbox_imap_user')->whereNotNull('expense_mailbox_imap_password')->cursor()->each(function ($company) { + $this->handleCompanyImap($company); + }); + }); + } + + } + + private function handleCompanyImap(Company $company) + { + $incommingMails = new IncomingMailHandler($company->expense_mailbox_imap_host, $company->company->expense_mailbox_imap_user, $company->company->expense_mailbox_imap_password); + + $emails = $incommingMails->getUnprocessedEmails(); + + foreach ($emails as $mail) { + + $sender = $mail->getSender(); + + $vendor = Vendor::where('expense_sender_email', $sender)->orWhere($sender, 'LIKE', "CONCAT('%',expense_sender_email)")->first(); + + if ($vendor !== null) + $vendor = Vendor::where("email", $sender)->first(); + + // TODO: check email for existing vendor?! + $data = [ + "vendor_id" => $vendor !== null ? $vendor->id : null, + "date" => $mail->getDate(), + "public_notes" => $mail->getSubject(), + "private_notes" => $mail->getCompleteBodyText(), + "documents" => $mail->getAttachments(), // FIXME: https://github.com/ddeboer/imap?tab=readme-ov-file#message-attachments + ]; + + $expense = $this->expense_repo->save($data, ExpenseFactory::create($company->company->id, $company->company->owner()->id)); // TODO: dont assign a new number at beginning + + // TODO: check for recurring expense?! => maybe replace existing ?! + + event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); + + event('eloquent.created: App\Models\Expense', $expense); + + $mail->markAsSeen(); + $incommingMails->moveProcessed($mail); + + } + } + +} diff --git a/composer.json b/composer.json index 1ef66cdd1760..7a52c900c649 100644 --- a/composer.json +++ b/composer.json @@ -44,6 +44,7 @@ "braintree/braintree_php": "^6.0", "checkout/checkout-sdk-php": "^3.0", "cleverit/ubl_invoice": "^1.3", + "ddeboer/imap": "^1.19", "doctrine/dbal": "^3.0", "eway/eway-rapid-php": "^1.3", "fakerphp/faker": "^1.14", @@ -179,4 +180,4 @@ ], "minimum-stability": "dev", "prefer-stable": true -} \ No newline at end of file +} diff --git a/composer.lock b/composer.lock index 27dc537efdb9..09d27aa2b06b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "28b57fe6eac3d71c607125cda9a6a537", + "content-hash": "ef5c36f71295ade916c3b7084642f0b4", "packages": [ { "name": "afosto/yaac", @@ -1176,6 +1176,82 @@ }, "time": "2023-08-25T16:18:39+00:00" }, + { + "name": "ddeboer/imap", + "version": "1.19.0", + "source": { + "type": "git", + "url": "https://github.com/ddeboer/imap.git", + "reference": "30800b1cfeacc4add5bb418e40a8b6e95a8a04ac" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/ddeboer/imap/zipball/30800b1cfeacc4add5bb418e40a8b6e95a8a04ac", + "reference": "30800b1cfeacc4add5bb418e40a8b6e95a8a04ac", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-iconv": "*", + "ext-imap": "*", + "ext-libxml": "*", + "ext-mbstring": "*", + "php": "~8.2.0 || ~8.3.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.38.2", + "laminas/laminas-mail": "^2.25.1", + "phpstan/phpstan": "^1.10.43", + "phpstan/phpstan-phpunit": "^1.3.15", + "phpstan/phpstan-strict-rules": "^1.5.2", + "phpunit/phpunit": "^10.4.2" + }, + "type": "library", + "autoload": { + "psr-4": { + "Ddeboer\\Imap\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "David de Boer", + "email": "david@ddeboer.nl" + }, + { + "name": "Filippo Tessarotto", + "email": "zoeslam@gmail.com" + }, + { + "name": "Community contributors", + "homepage": "https://github.com/ddeboer/imap/graphs/contributors" + } + ], + "description": "Object-oriented IMAP for PHP", + "keywords": [ + "email", + "imap", + "mail" + ], + "support": { + "issues": "https://github.com/ddeboer/imap/issues", + "source": "https://github.com/ddeboer/imap/tree/1.19.0" + }, + "funding": [ + { + "url": "https://github.com/Slamdunk", + "type": "github" + }, + { + "url": "https://github.com/ddeboer", + "type": "github" + } + ], + "time": "2023-11-20T14:41:54+00:00" + }, { "name": "dflydev/dot-access-data", "version": "v3.0.2", @@ -18046,5 +18122,5 @@ "platform-dev": { "php": "^8.1|^8.2" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.6.0" } diff --git a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php new file mode 100644 index 000000000000..a4a788c38553 --- /dev/null +++ b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php @@ -0,0 +1,32 @@ +string("expense_mailbox_imap_host")->nullable(); + $table->string("expense_mailbox_imap_port")->nullable(); + $table->string("expense_mailbox_imap_user")->nullable(); + $table->string("expense_mailbox_imap_password")->nullable(); + }); + Schema::table('vendor', function (Blueprint $table) { + $table->string("expense_sender_email")->nullable(); + $table->string("expense_sender_url")->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + // + } +};