From b2b1cce085708093019a64e6aa7a13a6637cf686 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 10 Dec 2023 16:06:33 +0100 Subject: [PATCH 001/119] initial commit --- app/Console/Kernel.php | 7 +- app/Helpers/Mail/IncomingMailHandler.php | 57 ++++++++ app/Jobs/Mail/ExpenseImportJob.php | 131 ++++++++++++++++++ composer.json | 3 +- composer.lock | 80 ++++++++++- ...10951_create_imap_configuration_fields.php | 32 +++++ 6 files changed, 305 insertions(+), 5 deletions(-) create mode 100644 app/Helpers/Mail/IncomingMailHandler.php create mode 100644 app/Jobs/Mail/ExpenseImportJob.php create mode 100644 database/migrations/2023_12_10_110951_create_imap_configuration_fields.php diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index f67dc4af685b..7d883dcbf945 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -17,6 +17,7 @@ use App\Jobs\Cron\RecurringInvoicesCron; use App\Jobs\Cron\SubscriptionCron; use App\Jobs\Cron\UpdateCalculatedFields; use App\Jobs\Invoice\InvoiceCheckLateWebhook; +use App\Jobs\Mail\ExpenseImportJob; use App\Jobs\Ninja\AdjustEmailQuota; use App\Jobs\Ninja\BankTransactionSync; use App\Jobs\Ninja\CheckACHStatus; @@ -120,11 +121,13 @@ class Kernel extends ConsoleKernel $schedule->command('ninja:s3-cleanup')->dailyAt('23:15')->withoutOverlapping()->name('s3-cleanup-job')->onOneServer(); } - if (config('queue.default') == 'database' && Ninja::isSelfHost() && config('ninja.internal_queue_enabled') && ! config('ninja.is_docker')) { + if (config('queue.default') == 'database' && Ninja::isSelfHost() && config('ninja.internal_queue_enabled') && !config('ninja.is_docker')) { $schedule->command('queue:work database --stop-when-empty --memory=256')->everyMinute()->withoutOverlapping(); $schedule->command('queue:restart')->everyFiveMinutes()->withoutOverlapping(); } + + $schedule->job(new ExpenseImportJob)->everyThirtyMinutes()->withoutOverlapping()->name('expense-import-job')->onOneServer(); } /** @@ -134,7 +137,7 @@ class Kernel extends ConsoleKernel */ protected function commands() { - $this->load(__DIR__.'/Commands'); + $this->load(__DIR__ . '/Commands'); require base_path('routes/console.php'); } diff --git a/app/Helpers/Mail/IncomingMailHandler.php b/app/Helpers/Mail/IncomingMailHandler.php new file mode 100644 index 000000000000..3c98293d7714 --- /dev/null +++ b/app/Helpers/Mail/IncomingMailHandler.php @@ -0,0 +1,57 @@ +server = new Server($server); + + $this->connection = $this->server->authenticate($user, $password); + } + + + public function getUnprocessedEmails() + { + $mailbox = $this->connection->getMailbox('INBOX'); + + $search = new SearchExpression(); + + // not older than 30days + $today = new \DateTimeImmutable(); + $thirtyDaysAgo = $today->sub(new \DateInterval('P30D')); + $search->addCondition(new Since($thirtyDaysAgo)); + + // not flagged with IN-PARSED + $search->addCondition(new Unflagged()); + + + return $mailbox->getMessages($search); + } + + public function moveProcessed(MessageInterface $mail) + { + return $mail->move($this->connection->getMailbox('PROCESSED')); + } +} diff --git a/app/Jobs/Mail/ExpenseImportJob.php b/app/Jobs/Mail/ExpenseImportJob.php new file mode 100644 index 000000000000..a31c2bace073 --- /dev/null +++ b/app/Jobs/Mail/ExpenseImportJob.php @@ -0,0 +1,131 @@ +expense_repo = new ExpenseRepository(); + } + + public function backoff() + { + // return [5, 10, 30, 240]; + return [rand(5, 10), rand(30, 40), rand(60, 79), rand(160, 400)]; + + } + + public function handle() + { + + //multiDB environment, need to + foreach (MultiDB::$dbs as $db) { + MultiDB::setDB($db); + + nlog("importing expenses from imap-servers"); + + $a = Account::with('companies')->cursor()->each(function ($account) { + $account->companies()->where('expense_import', true)->whereNotNull('expense_mailbox_imap_host')->whereNotNull('expense_mailbox_imap_user')->whereNotNull('expense_mailbox_imap_password')->cursor()->each(function ($company) { + $this->handleCompanyImap($company); + }); + }); + } + + } + + private function handleCompanyImap(Company $company) + { + $incommingMails = new IncomingMailHandler($company->expense_mailbox_imap_host, $company->company->expense_mailbox_imap_user, $company->company->expense_mailbox_imap_password); + + $emails = $incommingMails->getUnprocessedEmails(); + + foreach ($emails as $mail) { + + $sender = $mail->getSender(); + + $vendor = Vendor::where('expense_sender_email', $sender)->orWhere($sender, 'LIKE', "CONCAT('%',expense_sender_email)")->first(); + + if ($vendor !== null) + $vendor = Vendor::where("email", $sender)->first(); + + // TODO: check email for existing vendor?! + $data = [ + "vendor_id" => $vendor !== null ? $vendor->id : null, + "date" => $mail->getDate(), + "public_notes" => $mail->getSubject(), + "private_notes" => $mail->getCompleteBodyText(), + "documents" => $mail->getAttachments(), // FIXME: https://github.com/ddeboer/imap?tab=readme-ov-file#message-attachments + ]; + + $expense = $this->expense_repo->save($data, ExpenseFactory::create($company->company->id, $company->company->owner()->id)); // TODO: dont assign a new number at beginning + + // TODO: check for recurring expense?! => maybe replace existing ?! + + event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); + + event('eloquent.created: App\Models\Expense', $expense); + + $mail->markAsSeen(); + $incommingMails->moveProcessed($mail); + + } + } + +} diff --git a/composer.json b/composer.json index 1ef66cdd1760..7a52c900c649 100644 --- a/composer.json +++ b/composer.json @@ -44,6 +44,7 @@ "braintree/braintree_php": "^6.0", "checkout/checkout-sdk-php": "^3.0", "cleverit/ubl_invoice": "^1.3", + "ddeboer/imap": "^1.19", "doctrine/dbal": "^3.0", "eway/eway-rapid-php": "^1.3", "fakerphp/faker": "^1.14", @@ -179,4 +180,4 @@ ], "minimum-stability": "dev", "prefer-stable": true -} \ No newline at end of file +} diff --git a/composer.lock b/composer.lock index 27dc537efdb9..09d27aa2b06b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "28b57fe6eac3d71c607125cda9a6a537", + "content-hash": "ef5c36f71295ade916c3b7084642f0b4", "packages": [ { "name": "afosto/yaac", @@ -1176,6 +1176,82 @@ }, "time": "2023-08-25T16:18:39+00:00" }, + { + "name": "ddeboer/imap", + "version": "1.19.0", + "source": { + "type": "git", + "url": "https://github.com/ddeboer/imap.git", + "reference": "30800b1cfeacc4add5bb418e40a8b6e95a8a04ac" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/ddeboer/imap/zipball/30800b1cfeacc4add5bb418e40a8b6e95a8a04ac", + "reference": "30800b1cfeacc4add5bb418e40a8b6e95a8a04ac", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-iconv": "*", + "ext-imap": "*", + "ext-libxml": "*", + "ext-mbstring": "*", + "php": "~8.2.0 || ~8.3.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.38.2", + "laminas/laminas-mail": "^2.25.1", + "phpstan/phpstan": "^1.10.43", + "phpstan/phpstan-phpunit": "^1.3.15", + "phpstan/phpstan-strict-rules": "^1.5.2", + "phpunit/phpunit": "^10.4.2" + }, + "type": "library", + "autoload": { + "psr-4": { + "Ddeboer\\Imap\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "David de Boer", + "email": "david@ddeboer.nl" + }, + { + "name": "Filippo Tessarotto", + "email": "zoeslam@gmail.com" + }, + { + "name": "Community contributors", + "homepage": "https://github.com/ddeboer/imap/graphs/contributors" + } + ], + "description": "Object-oriented IMAP for PHP", + "keywords": [ + "email", + "imap", + "mail" + ], + "support": { + "issues": "https://github.com/ddeboer/imap/issues", + "source": "https://github.com/ddeboer/imap/tree/1.19.0" + }, + "funding": [ + { + "url": "https://github.com/Slamdunk", + "type": "github" + }, + { + "url": "https://github.com/ddeboer", + "type": "github" + } + ], + "time": "2023-11-20T14:41:54+00:00" + }, { "name": "dflydev/dot-access-data", "version": "v3.0.2", @@ -18046,5 +18122,5 @@ "platform-dev": { "php": "^8.1|^8.2" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.6.0" } diff --git a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php new file mode 100644 index 000000000000..a4a788c38553 --- /dev/null +++ b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php @@ -0,0 +1,32 @@ +string("expense_mailbox_imap_host")->nullable(); + $table->string("expense_mailbox_imap_port")->nullable(); + $table->string("expense_mailbox_imap_user")->nullable(); + $table->string("expense_mailbox_imap_password")->nullable(); + }); + Schema::table('vendor', function (Blueprint $table) { + $table->string("expense_sender_email")->nullable(); + $table->string("expense_sender_url")->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + // + } +}; From 1125f57a7f42185df1d93685734a41d1fa6fe363 Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 14 Dec 2023 07:51:38 +0100 Subject: [PATCH 002/119] init inbound webhook --- app/Http/Controllers/PostMarkController.php | 46 +++++ .../ProcessPostmarkInboundWebhook.php | 180 ++++++++++++++++++ routes/api.php | 5 +- 3 files changed, 229 insertions(+), 2 deletions(-) create mode 100644 app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index b7a630cb51d1..44227ce9c8f1 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -11,6 +11,7 @@ namespace App\Http\Controllers; +use App\Jobs\PostMark\ProcessPostmarkInboundWebhook; use App\Jobs\PostMark\ProcessPostmarkWebhook; use Illuminate\Http\Request; @@ -69,4 +70,49 @@ class PostMarkController extends BaseController return response()->json(['message' => 'Unauthorized'], 403); } + + /** + * Process Postmark Webhook. + * + * + * @OA\Post( + * path="/api/v1/postmark_inbound_webhook", + * operationId="postmarkInboundWebhook", + * tags={"postmark"}, + * summary="Processing inbound webhooks from PostMark", + * description="Adds an credit to the system", + * @OA\Parameter(ref="#/components/parameters/X-API-TOKEN"), + * @OA\Parameter(ref="#/components/parameters/X-Requested-With"), + * @OA\Parameter(ref="#/components/parameters/include"), + * @OA\Response( + * response=200, + * description="Returns the saved credit object", + * @OA\Header(header="X-MINIMUM-CLIENT-VERSION", ref="#/components/headers/X-MINIMUM-CLIENT-VERSION"), + * @OA\Header(header="X-RateLimit-Remaining", ref="#/components/headers/X-RateLimit-Remaining"), + * @OA\Header(header="X-RateLimit-Limit", ref="#/components/headers/X-RateLimit-Limit"), + * @OA\JsonContent(ref="#/components/schemas/Credit"), + * ), + * @OA\Response( + * response=422, + * description="Validation error", + * @OA\JsonContent(ref="#/components/schemas/ValidationError"), + * + * ), + * @OA\Response( + * response="default", + * description="Unexpected Error", + * @OA\JsonContent(ref="#/components/schemas/Error"), + * ), + * ) + */ + public function inboundWebhook(Request $request) + { + if ($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.postmark.token')) { + ProcessPostmarkInboundWebhook::dispatch($request->all())->delay(10); + + return response()->json(['message' => 'Success'], 200); + } + + return response()->json(['message' => 'Unauthorized'], 403); + } } diff --git a/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php b/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php new file mode 100644 index 000000000000..2d2355b234bc --- /dev/null +++ b/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php @@ -0,0 +1,180 @@ + '', + 'subject' => 'Message not found.', + 'entity' => '', + 'entity_id' => '', + 'events' => [], + ]; + + /** + * Create a new job instance. + * + */ + public function __construct(private array $request) + { + } + + private function getSystemLog(string $message_id): ?SystemLog + { + return SystemLog::query() + ->where('company_id', $this->invitation->company_id) + ->where('type_id', SystemLog::TYPE_WEBHOOK_RESPONSE) + ->whereJsonContains('log', ['MessageID' => $message_id]) + ->orderBy('id', 'desc') + ->first(); + + } + + private function updateSystemLog(SystemLog $system_log, array $data): void + { + $system_log->log = $data; + $system_log->save(); + } + + /** + * Execute the job. + * + * + * @return void + */ + public function handle() + { + MultiDB::findAndSetDbByCompanyKey($this->request['Tag']); + + // match companies + if (array_key_exists('ToFull', $this->request)) + throw new \Exception('invalid body'); + + $toEmails = []; + foreach ($this->request['ToFull'] as $toEmailEntry) + $toEmails[] = $toEmailEntry['Email']; + + // create expense for each company + $expense = new Expense(); + + $expense->company_id; + } + // { + // "FromName": "Postmarkapp Support", + // "MessageStream": "inbound", + // "From": "support@postmarkapp.com", + // "FromFull": { + // "Email": "support@postmarkapp.com", + // "Name": "Postmarkapp Support", + // "MailboxHash": "" + // }, + // "To": "\"Firstname Lastname\" ", + // "ToFull": [ + // { + // "Email": "yourhash+SampleHash@inbound.postmarkapp.com", + // "Name": "Firstname Lastname", + // "MailboxHash": "SampleHash" + // } + // ], + // "Cc": "\"First Cc\" , secondCc@postmarkapp.com>", + // "CcFull": [ + // { + // "Email": "firstcc@postmarkapp.com", + // "Name": "First Cc", + // "MailboxHash": "" + // }, + // { + // "Email": "secondCc@postmarkapp.com", + // "Name": "", + // "MailboxHash": "" + // } + // ], + // "Bcc": "\"First Bcc\" , secondbcc@postmarkapp.com>", + // "BccFull": [ + // { + // "Email": "firstbcc@postmarkapp.com", + // "Name": "First Bcc", + // "MailboxHash": "" + // }, + // { + // "Email": "secondbcc@postmarkapp.com", + // "Name": "", + // "MailboxHash": "" + // } + // ], + // "OriginalRecipient": "yourhash+SampleHash@inbound.postmarkapp.com", + // "Subject": "Test subject", + // "MessageID": "73e6d360-66eb-11e1-8e72-a8904824019b", + // "ReplyTo": "replyto@postmarkapp.com", + // "MailboxHash": "SampleHash", + // "Date": "Fri, 1 Aug 2014 16:45:32 -04:00", + // "TextBody": "This is a test text body.", + // "HtmlBody": "

This is a test html body.<\/p><\/body><\/html>", + // "StrippedTextReply": "This is the reply text", + // "Tag": "TestTag", + // "Headers": [ + // { + // "Name": "X-Header-Test", + // "Value": "" + // }, + // { + // "Name": "X-Spam-Status", + // "Value": "No" + // }, + // { + // "Name": "X-Spam-Score", + // "Value": "-0.1" + // }, + // { + // "Name": "X-Spam-Tests", + // "Value": "DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,SPF_PASS" + // } + // ], + // "Attachments": [ + // { + // "Name": "test.txt", + // "Content": "VGhpcyBpcyBhdHRhY2htZW50IGNvbnRlbnRzLCBiYXNlLTY0IGVuY29kZWQu", + // "ContentType": "text/plain", + // "ContentLength": 45 + // } + // ] + // } +} diff --git a/routes/api.php b/routes/api.php index 4d648054e298..078274348f40 100644 --- a/routes/api.php +++ b/routes/api.php @@ -120,7 +120,7 @@ Route::group(['middleware' => ['throttle:api', 'api_secret_check']], function () Route::post('api/v1/oauth_login', [LoginController::class, 'oauthApiLogin']); }); -Route::group(['middleware' => ['throttle:login','api_secret_check','email_db']], function () { +Route::group(['middleware' => ['throttle:login', 'api_secret_check', 'email_db']], function () { Route::post('api/v1/login', [LoginController::class, 'apiLogin'])->name('login.submit'); Route::post('api/v1/reset_password', [ForgotPasswordController::class, 'sendResetLinkEmail']); }); @@ -324,7 +324,7 @@ Route::group(['middleware' => ['throttle:api', 'api_db', 'token_auth', 'locale'] Route::post('reports/user_sales_report', UserSalesReportController::class); Route::post('reports/preview/{hash}', ReportPreviewController::class); Route::post('exports/preview/{hash}', ReportExportController::class); - + Route::post('templates/preview/{hash}', TemplatePreviewController::class); Route::post('search', SearchController::class); @@ -414,6 +414,7 @@ Route::match(['get', 'post'], 'payment_notification_webhook/{company_key}/{compa ->name('payment_notification_webhook'); Route::post('api/v1/postmark_webhook', [PostMarkController::class, 'webhook'])->middleware('throttle:1000,1'); +Route::post('api/v1/postmark_inbound_webhook', [PostMarkController::class, 'inboundWebhook'])->middleware('throttle:1000,1'); Route::get('token_hash_router', [OneTimeTokenController::class, 'router'])->middleware('throttle:500,1'); Route::get('webcron', [WebCronController::class, 'index'])->middleware('throttle:100,1'); Route::post('api/v1/get_migration_account', [HostedMigrationController::class, 'getAccount'])->middleware('guest')->middleware('throttle:100,1'); From b065542020556f09264484642285d7a74cc27448 Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 14 Dec 2023 08:32:15 +0100 Subject: [PATCH 003/119] env-config --- app/Helpers/Mail/IncomingMailHandler.php | 4 +- ...portJob.php => ImapInboundExpensesJob.php} | 62 +++++++++++++------ config/ninja.php | 13 +++- ...10951_create_imap_configuration_fields.php | 7 +-- 4 files changed, 58 insertions(+), 28 deletions(-) rename app/Jobs/Mail/{ExpenseImportJob.php => ImapInboundExpensesJob.php} (63%) diff --git a/app/Helpers/Mail/IncomingMailHandler.php b/app/Helpers/Mail/IncomingMailHandler.php index 3c98293d7714..1da4e1437fe9 100644 --- a/app/Helpers/Mail/IncomingMailHandler.php +++ b/app/Helpers/Mail/IncomingMailHandler.php @@ -24,9 +24,9 @@ class IncomingMailHandler { private $server; public $connection; - public function __construct(string $server, string $user, string $password) + public function __construct(string $server, string $port, string $user, string $password) { - $this->server = new Server($server); + $this->server = new Server($server, $port == '' ? null : $port); $this->connection = $this->server->authenticate($user, $password); } diff --git a/app/Jobs/Mail/ExpenseImportJob.php b/app/Jobs/Mail/ImapInboundExpensesJob.php similarity index 63% rename from app/Jobs/Mail/ExpenseImportJob.php rename to app/Jobs/Mail/ImapInboundExpensesJob.php index a31c2bace073..0a04c7d53c7e 100644 --- a/app/Jobs/Mail/ExpenseImportJob.php +++ b/app/Jobs/Mail/ImapInboundExpensesJob.php @@ -46,32 +46,26 @@ use Turbo124\Beacon\Facades\LightLogs; /*Multi Mailer implemented*/ -class ExpenseImportJob implements ShouldQueue +class InboundExpensesJob implements ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, MakesHash; public $tries = 4; //number of retries public $deleteWhenMissingModels = true; - - /** @var null|\App\Models\Company $company **/ - public Company $company; - + private array $imap_companies; + private array $imap_credentials; private $expense_repo; public function __construct() { + $this->credentials = []; + + $this->getImapCredentials(); $this->expense_repo = new ExpenseRepository(); } - public function backoff() - { - // return [5, 10, 30, 240]; - return [rand(5, 10), rand(30, 40), rand(60, 79), rand(160, 400)]; - - } - public function handle() { @@ -81,8 +75,8 @@ class ExpenseImportJob implements ShouldQueue nlog("importing expenses from imap-servers"); - $a = Account::with('companies')->cursor()->each(function ($account) { - $account->companies()->where('expense_import', true)->whereNotNull('expense_mailbox_imap_host')->whereNotNull('expense_mailbox_imap_user')->whereNotNull('expense_mailbox_imap_password')->cursor()->each(function ($company) { + Account::with('companies')->cursor()->each(function ($account) { + $account->companies()->whereIn('id', $this->imap_companies)->cursor()->each(function ($company) { $this->handleCompanyImap($company); }); }); @@ -90,9 +84,33 @@ class ExpenseImportJob implements ShouldQueue } + private function getImapCredentials() + { + $servers = explode(",", config('ninja.imap_inbound_expense.servers')); + $ports = explode(",", config('ninja.imap_inbound_expense.servers')); + $users = explode(",", config('ninja.imap_inbound_expense.servers')); + $passwords = explode(",", config('ninja.imap_inbound_expense.servers')); + $companies = explode(",", config('ninja.imap_inbound_expense.servers')); + + if (sizeOf($servers) != sizeOf($ports) || sizeOf($servers) != sizeOf($users) || sizeOf($servers) != sizeOf($passwords) || sizeOf($servers) != sizeOf($companies)) + throw new \Exception('invalid configuration imap_inbound_expenses (wrong element-count)'); + + foreach ($companies as $index => $companyId) { + $this->imap_credentials[$companyId] = [ + "server" => $servers[$index], + "port" => $servers[$index], + "user" => $servers[$index], + "password" => $servers[$index], + ]; + $this->imap_companies[] = $companyId; + } + } + private function handleCompanyImap(Company $company) { - $incommingMails = new IncomingMailHandler($company->expense_mailbox_imap_host, $company->company->expense_mailbox_imap_user, $company->company->expense_mailbox_imap_password); + $credentials = $this->imap_credentials[$company->id]; + + $incommingMails = new IncomingMailHandler($credentials->server, $credentials->port, $credentials->user, $credentials->password); $emails = $incommingMails->getUnprocessedEmails(); @@ -100,18 +118,19 @@ class ExpenseImportJob implements ShouldQueue $sender = $mail->getSender(); - $vendor = Vendor::where('expense_sender_email', $sender)->orWhere($sender, 'LIKE', "CONCAT('%',expense_sender_email)")->first(); + $vendor = Vendor::where('expense_sender_email', $sender)->orWhere($sender, 'LIKE', "CONCAT('%',expense_sender_domain)")->first(); if ($vendor !== null) $vendor = Vendor::where("email", $sender)->first(); - // TODO: check email for existing vendor?! + $documents = []; // TODO: $mail->getAttachments() + save email as document (.html) + $data = [ "vendor_id" => $vendor !== null ? $vendor->id : null, "date" => $mail->getDate(), "public_notes" => $mail->getSubject(), "private_notes" => $mail->getCompleteBodyText(), - "documents" => $mail->getAttachments(), // FIXME: https://github.com/ddeboer/imap?tab=readme-ov-file#message-attachments + "documents" => $documents, // FIXME: https://github.com/ddeboer/imap?tab=readme-ov-file#message-attachments ]; $expense = $this->expense_repo->save($data, ExpenseFactory::create($company->company->id, $company->company->owner()->id)); // TODO: dont assign a new number at beginning @@ -128,4 +147,11 @@ class ExpenseImportJob implements ShouldQueue } } + public function backoff() + { + // return [5, 10, 30, 240]; + return [rand(5, 10), rand(30, 40), rand(60, 79), rand(160, 400)]; + + } + } diff --git a/config/ninja.php b/config/ninja.php index 94d0a307fc6a..d94ec91a01b1 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -7,7 +7,7 @@ return [ 'license_url' => 'https://app.invoiceninja.com', 'react_url' => env('REACT_URL', 'https://app.invoicing.co'), 'production' => env('NINJA_PROD', false), - 'license' => env('NINJA_LICENSE', ''), + 'license' => env('NINJA_LICENSE', ''), 'version_url' => 'https://pdf.invoicing.co/api/version', 'app_name' => env('APP_NAME', 'Invoice Ninja'), 'app_env' => env('APP_ENV', 'selfhosted'), @@ -86,7 +86,7 @@ return [ 'password' => 'password', 'stripe' => env('STRIPE_KEYS', ''), 'paypal' => env('PAYPAL_KEYS', ''), - 'ppcp' => env('PPCP_KEYS', ''), + 'ppcp' => env('PPCP_KEYS', ''), 'paypal_rest' => env('PAYPAL_REST_KEYS', ''), 'authorize' => env('AUTHORIZE_KEYS', ''), 'checkout' => env('CHECKOUT_KEYS', ''), @@ -196,7 +196,7 @@ return [ 'ninja_default_company_id' => env('NINJA_COMPANY_ID', null), 'ninja_default_company_gateway_id' => env('NINJA_COMPANY_GATEWAY_ID', null), 'ninja_hosted_secret' => env('NINJA_HOSTED_SECRET', ''), - 'ninja_hosted_header' =>env('NINJA_HEADER', ''), + 'ninja_hosted_header' => env('NINJA_HEADER', ''), 'ninja_connect_secret' => env('NINJA_CONNECT_SECRET', ''), 'internal_queue_enabled' => env('INTERNAL_QUEUE_ENABLED', true), 'ninja_apple_api_key' => env('APPLE_API_KEY', false), @@ -227,5 +227,12 @@ return [ 'paypal' => [ 'secret' => env('PAYPAL_SECRET', null), 'client_id' => env('PAYPAL_CLIENT_ID', null), + ], + 'imap_inbound_expense' => [ + 'servers' => env('IMAP_INBOUND_EXPENSE_SERVERS', ''), + 'ports' => env('IMAP_INBOUND_EXPENSE_PORTS', ''), + 'users' => env('IMAP_INBOUND_EXPENSE_USERS', ''), + 'passwords' => env('IMAP_INBOUND_EXPENSE_PASSWORDS', ''), + 'companies' => env('IMAP_INBOUND_EXPENSE_COMPANIES', ''), ] ]; diff --git a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php index a4a788c38553..e5b8a03b3abe 100644 --- a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php +++ b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php @@ -11,14 +11,11 @@ return new class extends Migration { public function up(): void { Schema::table('company', function (Blueprint $table) { - $table->string("expense_mailbox_imap_host")->nullable(); - $table->string("expense_mailbox_imap_port")->nullable(); - $table->string("expense_mailbox_imap_user")->nullable(); - $table->string("expense_mailbox_imap_password")->nullable(); + $table->string("expense_import")->default(true); }); Schema::table('vendor', function (Blueprint $table) { $table->string("expense_sender_email")->nullable(); - $table->string("expense_sender_url")->nullable(); + $table->string("expense_sender_domain")->nullable(); }); } From d8064a9a35387025eae614d1c1942f9a5498f029 Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 14 Dec 2023 10:33:49 +0100 Subject: [PATCH 004/119] improv --- app/Console/Kernel.php | 4 + ...ncomingMailHandler.php => ImapMailbox.php} | 9 ++- ...dExpensesJob.php => ExpenseMailboxJob.php} | 76 ++++++++++--------- 3 files changed, 53 insertions(+), 36 deletions(-) rename app/Helpers/Mail/{IncomingMailHandler.php => ImapMailbox.php} (85%) rename app/Jobs/Mail/{ImapInboundExpensesJob.php => ExpenseMailboxJob.php} (53%) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 7d883dcbf945..20612d16ef6c 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -18,6 +18,7 @@ use App\Jobs\Cron\SubscriptionCron; use App\Jobs\Cron\UpdateCalculatedFields; use App\Jobs\Invoice\InvoiceCheckLateWebhook; use App\Jobs\Mail\ExpenseImportJob; +use App\Jobs\Mail\ExpenseMailboxJob; use App\Jobs\Ninja\AdjustEmailQuota; use App\Jobs\Ninja\BankTransactionSync; use App\Jobs\Ninja\CheckACHStatus; @@ -98,6 +99,9 @@ class Kernel extends ConsoleKernel /* Fires webhooks for overdue Invoice */ $schedule->job(new InvoiceCheckLateWebhook)->dailyAt('07:00')->withoutOverlapping()->name('invoice-overdue-job')->onOneServer(); + /* Check ExpenseMainboxes */ + $schedule->job(new ExpenseMailboxJob)->everyThirtyMinutes()->withoutOverlapping()->name('expense-mailboxes-job')->onOneServer(); + if (Ninja::isSelfHost()) { $schedule->call(function () { Account::whereNotNull('id')->update(['is_scheduler_running' => true]); diff --git a/app/Helpers/Mail/IncomingMailHandler.php b/app/Helpers/Mail/ImapMailbox.php similarity index 85% rename from app/Helpers/Mail/IncomingMailHandler.php rename to app/Helpers/Mail/ImapMailbox.php index 1da4e1437fe9..2747dc4964a8 100644 --- a/app/Helpers/Mail/IncomingMailHandler.php +++ b/app/Helpers/Mail/ImapMailbox.php @@ -20,13 +20,13 @@ use Ddeboer\Imap\Search\Flag\Unflagged; /** * GmailTransport. */ -class IncomingMailHandler +class ImapMailbox { private $server; public $connection; public function __construct(string $server, string $port, string $user, string $password) { - $this->server = new Server($server, $port == '' ? null : $port); + $this->server = new Server($server, $port != '' ? $port : null); $this->connection = $this->server->authenticate($user, $password); } @@ -54,4 +54,9 @@ class IncomingMailHandler { return $mail->move($this->connection->getMailbox('PROCESSED')); } + + public function moveFailed(MessageInterface $mail) + { + return $mail->move($this->connection->getMailbox('FAILED')); + } } diff --git a/app/Jobs/Mail/ImapInboundExpensesJob.php b/app/Jobs/Mail/ExpenseMailboxJob.php similarity index 53% rename from app/Jobs/Mail/ImapInboundExpensesJob.php rename to app/Jobs/Mail/ExpenseMailboxJob.php index 0a04c7d53c7e..60dd6a5dc468 100644 --- a/app/Jobs/Mail/ImapInboundExpensesJob.php +++ b/app/Jobs/Mail/ExpenseMailboxJob.php @@ -17,7 +17,7 @@ use App\Events\Expense\ExpenseWasCreated; use App\Events\Invoice\InvoiceWasEmailedAndFailed; use App\Events\Payment\PaymentWasEmailedAndFailed; use App\Factory\ExpenseFactory; -use App\Helpers\Mail\IncomingMailHandler; +use App\Helpers\Mail\ImapMailbox; use App\Jobs\Util\SystemLogger; use App\Libraries\Google\Google; use App\Libraries\MultiDB; @@ -46,7 +46,7 @@ use Turbo124\Beacon\Facades\LightLogs; /*Multi Mailer implemented*/ -class InboundExpensesJob implements ShouldQueue +class ExpenseMailboxJob implements ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, MakesHash; @@ -63,7 +63,7 @@ class InboundExpensesJob implements ShouldQueue $this->getImapCredentials(); - $this->expense_repo = new ExpenseRepository(); + $this->expense_repo = new ExpenseRepository(); // @turbo124 @todo is this the right aproach? should it be handled just with the model? } public function handle() @@ -73,20 +73,20 @@ class InboundExpensesJob implements ShouldQueue foreach (MultiDB::$dbs as $db) { MultiDB::setDB($db); - nlog("importing expenses from imap-servers"); + if (sizeOf($this->imap_credentials) != 0) { + nlog("importing expenses from imap-servers"); - Account::with('companies')->cursor()->each(function ($account) { - $account->companies()->whereIn('id', $this->imap_companies)->cursor()->each(function ($company) { - $this->handleCompanyImap($company); + Company::whereIn('id', $this->imap_companies)->cursor()->each(function ($company) { + $this->handleImapCompany($company); }); - }); + } } } private function getImapCredentials() { - $servers = explode(",", config('ninja.imap_inbound_expense.servers')); + $servers = array_map('trim', explode(",", config('ninja.imap_inbound_expense.servers'))); $ports = explode(",", config('ninja.imap_inbound_expense.servers')); $users = explode(",", config('ninja.imap_inbound_expense.servers')); $passwords = explode(",", config('ninja.imap_inbound_expense.servers')); @@ -98,51 +98,59 @@ class InboundExpensesJob implements ShouldQueue foreach ($companies as $index => $companyId) { $this->imap_credentials[$companyId] = [ "server" => $servers[$index], - "port" => $servers[$index], - "user" => $servers[$index], - "password" => $servers[$index], + "port" => $ports[$index] != '' ? $ports[$index] : null, + "user" => $users[$index], + "password" => $passwords[$index], ]; $this->imap_companies[] = $companyId; } } - private function handleCompanyImap(Company $company) + private function handleImapCompany(Company $company) { + nlog("importing expenses for company: " . $company->id); + $credentials = $this->imap_credentials[$company->id]; + $imapMailbox = new ImapMailbox($credentials->server, $credentials->port, $credentials->user, $credentials->password); - $incommingMails = new IncomingMailHandler($credentials->server, $credentials->port, $credentials->user, $credentials->password); - - $emails = $incommingMails->getUnprocessedEmails(); + $emails = $imapMailbox->getUnprocessedEmails(); foreach ($emails as $mail) { - $sender = $mail->getSender(); + try { - $vendor = Vendor::where('expense_sender_email', $sender)->orWhere($sender, 'LIKE', "CONCAT('%',expense_sender_domain)")->first(); + $sender = $mail->getSender(); - if ($vendor !== null) - $vendor = Vendor::where("email", $sender)->first(); + $vendor = Vendor::where('expense_sender_email', $sender)->first(); + if ($vendor == null) + $vendor = Vendor::where($sender, 'LIKE', "CONCAT('%',expense_sender_domain)")->first(); + if ($vendor == null) + $vendor = Vendor::where("email", $sender)->first(); - $documents = []; // TODO: $mail->getAttachments() + save email as document (.html) + $documents = []; // TODO: $mail->getAttachments() + save email as document (.html) - $data = [ - "vendor_id" => $vendor !== null ? $vendor->id : null, - "date" => $mail->getDate(), - "public_notes" => $mail->getSubject(), - "private_notes" => $mail->getCompleteBodyText(), - "documents" => $documents, // FIXME: https://github.com/ddeboer/imap?tab=readme-ov-file#message-attachments - ]; + $data = [ + "vendor_id" => $vendor !== null ? $vendor->id : null, + "date" => $mail->getDate(), + "public_notes" => $mail->getSubject(), + "private_notes" => $mail->getCompleteBodyText(), + "documents" => $documents, // FIXME: https://github.com/ddeboer/imap?tab=readme-ov-file#message-attachments + ]; - $expense = $this->expense_repo->save($data, ExpenseFactory::create($company->company->id, $company->company->owner()->id)); // TODO: dont assign a new number at beginning + $expense = $this->expense_repo->save($data, ExpenseFactory::create($company->company->id, $company->company->owner()->id)); // TODO: dont assign a new number at beginning - // TODO: check for recurring expense?! => maybe replace existing ?! + event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); - event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); + event('eloquent.created: App\Models\Expense', $expense); - event('eloquent.created: App\Models\Expense', $expense); + $mail->markAsSeen(); + $imapMailbox->moveProcessed($mail); - $mail->markAsSeen(); - $incommingMails->moveProcessed($mail); + } catch (\Exception $e) { + $imapMailbox->moveFailed($mail); + + nlog("processing of an email failed upnormally: " . $company->id . " message: " . $e->getMessage()); // @turbo124 @todo should this be handled in an other way? + } } } From 8bd9f34d9834f94b9c31ddeba79c8848d433d359 Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 14 Dec 2023 10:39:55 +0100 Subject: [PATCH 005/119] vendor updates --- app/Models/Vendor.php | 26 +++++++++++++++----------- app/Transformers/VendorTransformer.php | 4 +++- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/app/Models/Vendor.php b/app/Models/Vendor.php index d6365f7e65b7..0e7aec98deac 100644 --- a/app/Models/Vendor.php +++ b/app/Models/Vendor.php @@ -54,6 +54,8 @@ use Laracasts\Presenter\PresentableTrait; * @property string|null $id_number * @property string|null $language_id * @property int|null $last_login + * @property string|null $expense_sender_email + * @property string|null $expense_sender_domain * @property-read \Illuminate\Database\Eloquent\Collection $activities * @property-read int|null $activities_count * @property-read \App\Models\User|null $assigned_user @@ -115,6 +117,8 @@ class Vendor extends BaseModel 'number', 'language_id', 'classification', + 'expense_sender_email', + 'expense_sender_domain', ]; protected $casts = [ @@ -169,11 +173,11 @@ class Vendor extends BaseModel { $currencies = Cache::get('currencies'); - if (! $currencies) { + if (!$currencies) { $this->buildCache(true); } - if (! $this->currency_id) { + if (!$this->currency_id) { return $this->company->currency(); } @@ -205,18 +209,18 @@ class Vendor extends BaseModel return ctrans('texts.vendor'); } - public function setCompanyDefaults($data, $entity_name) :array + public function setCompanyDefaults($data, $entity_name): array { $defaults = []; - if (! (array_key_exists('terms', $data) && strlen($data['terms']) > 1)) { - $defaults['terms'] = $this->getSetting($entity_name.'_terms'); + if (!(array_key_exists('terms', $data) && strlen($data['terms']) > 1)) { + $defaults['terms'] = $this->getSetting($entity_name . '_terms'); } elseif (array_key_exists('terms', $data)) { $defaults['terms'] = $data['terms']; } - if (! (array_key_exists('footer', $data) && strlen($data['footer']) > 1)) { - $defaults['footer'] = $this->getSetting($entity_name.'_footer'); + if (!(array_key_exists('footer', $data) && strlen($data['footer']) > 1)) { + $defaults['footer'] = $this->getSetting($entity_name . '_footer'); } elseif (array_key_exists('footer', $data)) { $defaults['footer'] = $data['footer']; } @@ -245,7 +249,7 @@ class Vendor extends BaseModel return ''; } - public function getMergedSettings() :object + public function getMergedSettings(): object { return $this->company->settings; } @@ -254,7 +258,7 @@ class Vendor extends BaseModel { $contact_key = $invitation->contact->contact_key; - return $this->company->company_key.'/'.$this->vendor_hash.'/'.$contact_key.'/purchase_orders/'; + return $this->company->company_key . '/' . $this->vendor_hash . '/' . $contact_key . '/purchase_orders/'; } public function locale(): string @@ -277,9 +281,9 @@ class Vendor extends BaseModel return $this->company->date_format(); } - public function backup_path() :string + public function backup_path(): string { - return $this->company->company_key.'/'.$this->vendor_hash.'/backups'; + return $this->company->company_key . '/' . $this->vendor_hash . '/backups'; } public function service() diff --git a/app/Transformers/VendorTransformer.php b/app/Transformers/VendorTransformer.php index aa3a28669234..5ec3c9a19175 100644 --- a/app/Transformers/VendorTransformer.php +++ b/app/Transformers/VendorTransformer.php @@ -104,7 +104,9 @@ class VendorTransformer extends EntityTransformer 'number' => (string) $vendor->number ?: '', 'language_id' => (string) $vendor->language_id ?: '', 'classification' => (string) $vendor->classification ?: '', - 'display_name' => (string) $vendor->present()->name(), + 'display_name' => (string) $vendor->present()->name(), + 'expense_sender_email' => (string) $vendor->expense_sender_email ?: '', + 'expense_sender_domain' => (string) $vendor->expense_sender_domain ?: '', ]; } } From a6f92751446b9702018ab86540645140f0706e85 Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 14 Dec 2023 13:16:14 +0100 Subject: [PATCH 006/119] save mailbox at company --- app/Factory/CompanyFactory.php | 7 +- app/Jobs/Mail/ExpenseMailboxJob.php | 12 +-- app/Models/Company.php | 80 ++++++++++--------- config/ninja.php | 20 +++-- ...10951_create_imap_configuration_fields.php | 10 ++- 5 files changed, 76 insertions(+), 53 deletions(-) diff --git a/app/Factory/CompanyFactory.php b/app/Factory/CompanyFactory.php index 0f68ec941896..00f2874ffd48 100644 --- a/app/Factory/CompanyFactory.php +++ b/app/Factory/CompanyFactory.php @@ -27,7 +27,7 @@ class CompanyFactory * @param int $account_id * @return Company */ - public function create(int $account_id) :Company + public function create(int $account_id): Company { $company = new Company; $company->account_id = $account_id; @@ -49,6 +49,11 @@ class CompanyFactory $company->markdown_enabled = false; $company->tax_data = new TaxModel(); $company->first_month_of_year = 1; + + // default mailbox + $company->expense_mailbox = config('ninja.inbound_expense.webhook.mailbox_template') != '' ? + str_replace('{{company_key}}', $company->company_key, config('ninja.inbound_expense.webhook.mailbox_template')) : null; + return $company; } } diff --git a/app/Jobs/Mail/ExpenseMailboxJob.php b/app/Jobs/Mail/ExpenseMailboxJob.php index 60dd6a5dc468..956d29c95619 100644 --- a/app/Jobs/Mail/ExpenseMailboxJob.php +++ b/app/Jobs/Mail/ExpenseMailboxJob.php @@ -86,14 +86,14 @@ class ExpenseMailboxJob implements ShouldQueue private function getImapCredentials() { - $servers = array_map('trim', explode(",", config('ninja.imap_inbound_expense.servers'))); - $ports = explode(",", config('ninja.imap_inbound_expense.servers')); - $users = explode(",", config('ninja.imap_inbound_expense.servers')); - $passwords = explode(",", config('ninja.imap_inbound_expense.servers')); - $companies = explode(",", config('ninja.imap_inbound_expense.servers')); + $servers = array_map('trim', explode(",", config('ninja.inbound_expense.imap.servers'))); + $ports = explode(",", config('ninja.inbound_expense.imap.servers')); + $users = explode(",", config('ninja.inbound_expense.imap.servers')); + $passwords = explode(",", config('ninja.inbound_expense.imap.servers')); + $companies = explode(",", config('ninja.inbound_expense.imap.servers')); if (sizeOf($servers) != sizeOf($ports) || sizeOf($servers) != sizeOf($users) || sizeOf($servers) != sizeOf($passwords) || sizeOf($servers) != sizeOf($companies)) - throw new \Exception('invalid configuration imap_inbound_expenses (wrong element-count)'); + throw new \Exception('invalid configuration inbound_expense.imap (wrong element-count)'); foreach ($companies as $index => $companyId) { $this->imap_credentials[$companyId] = [ diff --git a/app/Models/Company.php b/app/Models/Company.php index 8f37c7d063bc..f8b331ba9e4e 100644 --- a/app/Models/Company.php +++ b/app/Models/Company.php @@ -111,6 +111,8 @@ use Laracasts\Presenter\PresentableTrait; * @property int $convert_expense_currency * @property int $notify_vendor_when_paid * @property int $invoice_task_hours + * @property boolean $expense_import + * @property string|null $expense_mailbox * @property int $deleted_at * @property-read \App\Models\Account $account * @property-read \Illuminate\Database\Eloquent\Collection $activities @@ -352,6 +354,8 @@ class Company extends BaseModel 'calculate_taxes', 'tax_data', 'e_invoice_certificate_passphrase', + 'expense_import', + 'expense_mailbox', // TODO: @turbo124 custom validation: self-hosted => free change, hosted => not changeable, only changeable with env-mask ]; protected $hidden = [ @@ -404,17 +408,17 @@ class Company extends BaseModel return $this->morphMany(Document::class, 'documentable'); } - public function schedulers() :HasMany + public function schedulers(): HasMany { return $this->hasMany(Scheduler::class); } - public function task_schedulers() :HasMany + public function task_schedulers(): HasMany { return $this->hasMany(Scheduler::class); } - public function all_documents() :HasMany + public function all_documents(): HasMany { return $this->hasMany(Document::class); } @@ -424,22 +428,22 @@ class Company extends BaseModel return self::class; } - public function ledger() :HasMany + public function ledger(): HasMany { return $this->hasMany(CompanyLedger::class); } - public function bank_integrations() :HasMany + public function bank_integrations(): HasMany { return $this->hasMany(BankIntegration::class); } - public function bank_transactions() :HasMany + public function bank_transactions(): HasMany { return $this->hasMany(BankTransaction::class); } - public function bank_transaction_rules() :HasMany + public function bank_transaction_rules(): HasMany { return $this->hasMany(BankTransactionRule::class); } @@ -454,7 +458,7 @@ class Company extends BaseModel return $this->belongsTo(Account::class); } - public function client_contacts() :HasMany + public function client_contacts(): HasMany { return $this->hasMany(ClientContact::class)->withTrashed(); } @@ -467,27 +471,27 @@ class Company extends BaseModel return $this->hasManyThrough(User::class, CompanyUser::class, 'company_id', 'id', 'id', 'user_id')->withTrashed(); } - public function expense_categories() :HasMany + public function expense_categories(): HasMany { return $this->hasMany(ExpenseCategory::class)->withTrashed(); } - public function subscriptions() :HasMany + public function subscriptions(): HasMany { return $this->hasMany(Subscription::class)->withTrashed(); } - public function purchase_orders() :HasMany + public function purchase_orders(): HasMany { return $this->hasMany(PurchaseOrder::class)->withTrashed(); } - public function task_statuses() :HasMany + public function task_statuses(): HasMany { return $this->hasMany(TaskStatus::class)->withTrashed(); } - public function clients() :HasMany + public function clients(): HasMany { return $this->hasMany(Client::class)->withTrashed(); } @@ -495,12 +499,12 @@ class Company extends BaseModel /** * @return HasMany */ - public function tasks() :HasMany + public function tasks(): HasMany { return $this->hasMany(Task::class)->withTrashed(); } - public function webhooks() :HasMany + public function webhooks(): HasMany { return $this->hasMany(Webhook::class); } @@ -508,7 +512,7 @@ class Company extends BaseModel /** * @return HasMany */ - public function projects() :HasMany + public function projects(): HasMany { return $this->hasMany(Project::class)->withTrashed(); } @@ -516,7 +520,7 @@ class Company extends BaseModel /** * @return HasMany */ - public function vendor_contacts() :HasMany + public function vendor_contacts(): HasMany { return $this->hasMany(VendorContact::class)->withTrashed(); } @@ -524,17 +528,17 @@ class Company extends BaseModel /** * @return HasMany */ - public function vendors() :HasMany + public function vendors(): HasMany { return $this->hasMany(Vendor::class)->withTrashed(); } - public function all_activities() :\Illuminate\Database\Eloquent\Relations\HasMany + public function all_activities(): \Illuminate\Database\Eloquent\Relations\HasMany { return $this->hasMany(Activity::class); } - public function activities() :HasMany + public function activities(): HasMany { return $this->hasMany(Activity::class)->orderBy('id', 'DESC')->take(50); } @@ -620,7 +624,7 @@ class Company extends BaseModel { $companies = Cache::get('countries'); - if (! $companies) { + if (!$companies) { $this->buildCache(true); $companies = Cache::get('countries'); @@ -643,7 +647,7 @@ class Company extends BaseModel { $timezones = Cache::get('timezones'); - if (! $timezones) { + if (!$timezones) { $this->buildCache(true); } @@ -679,7 +683,7 @@ class Company extends BaseModel $languages = Cache::get('languages'); //build cache and reinit - if (! $languages) { + if (!$languages) { $this->buildCache(true); $languages = Cache::get('languages'); } @@ -699,7 +703,7 @@ class Company extends BaseModel return isset($this->settings->language_id) && $this->language() ? $this->language()->locale : config('ninja.i18n.locale'); } - public function getLogo() :?string + public function getLogo(): ?string { return $this->settings->company_logo ?: null; } @@ -713,7 +717,7 @@ class Company extends BaseModel { App::setLocale($this->getLocale()); } - + public function getSetting($setting) { //todo $this->setting ?? false @@ -809,7 +813,7 @@ class Company extends BaseModel { return $this->hasMany(CreditInvitation::class); } - + public function purchase_order_invitations(): HasMany { return $this->hasMany(PurchaseOrderInvitation::class); @@ -826,25 +830,25 @@ class Company extends BaseModel public function credit_rules() { return BankTransactionRule::query() - ->where('company_id', $this->id) - ->where('applies_to', 'CREDIT') - ->get(); + ->where('company_id', $this->id) + ->where('applies_to', 'CREDIT') + ->get(); } public function debit_rules() { return BankTransactionRule::query() - ->where('company_id', $this->id) - ->where('applies_to', 'DEBIT') - ->get(); + ->where('company_id', $this->id) + ->where('applies_to', 'DEBIT') + ->get(); } public function resolveRouteBinding($value, $field = null) { return $this->where('id', $this->decodePrimaryKey($value)) - ->where('account_id', auth()->user()->account_id) - ->firstOrFail(); + ->where('account_id', auth()->user()->account_id) + ->firstOrFail(); } public function domain(): string @@ -854,7 +858,7 @@ class Company extends BaseModel return $this->portal_domain; } - return "https://{$this->subdomain}.".config('ninja.app_domain'); + return "https://{$this->subdomain}." . config('ninja.app_domain'); } return config('ninja.app_url'); @@ -872,7 +876,7 @@ class Company extends BaseModel public function file_path(): string { - return $this->company_key.'/'; + return $this->company_key . '/'; } public function rBits() @@ -951,7 +955,7 @@ class Company extends BaseModel { $date_formats = Cache::get('date_formats'); - if (! $date_formats) { + if (!$date_formats) { $this->buildCache(true); } @@ -962,7 +966,7 @@ class Company extends BaseModel public function getInvoiceCert() { - if($this->e_invoice_certificate) { + if ($this->e_invoice_certificate) { return base64_decode($this->e_invoice_certificate); } diff --git a/config/ninja.php b/config/ninja.php index d94ec91a01b1..25505a49c6a2 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -228,11 +228,17 @@ return [ 'secret' => env('PAYPAL_SECRET', null), 'client_id' => env('PAYPAL_CLIENT_ID', null), ], - 'imap_inbound_expense' => [ - 'servers' => env('IMAP_INBOUND_EXPENSE_SERVERS', ''), - 'ports' => env('IMAP_INBOUND_EXPENSE_PORTS', ''), - 'users' => env('IMAP_INBOUND_EXPENSE_USERS', ''), - 'passwords' => env('IMAP_INBOUND_EXPENSE_PASSWORDS', ''), - 'companies' => env('IMAP_INBOUND_EXPENSE_COMPANIES', ''), - ] + 'inbound_expense' => [ + 'imap' => [ + 'servers' => env('INBOUND_EXPENSE_IMAP_SERVERS', ''), + 'ports' => env('INBOUND_EXPENSE_IMAP_PORTS', ''), + 'users' => env('INBOUND_EXPENSE_IMAP_USERS', ''), + 'passwords' => env('INBOUND_EXPENSE_IMAP_PASSWORDS', ''), + 'companies' => env('INBOUND_EXPENSE_IMAP_COMPANIES', ''), + ], + 'webhook' => [ + 'mailbox_template' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOXTEMPLATE', null), + 'mailbox_template_enterprise' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOXTEMPLATE_ENTERPRISE', '{{input}}@expense.invoicing.co'), + ], + ], ]; diff --git a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php index e5b8a03b3abe..f750b9b07435 100644 --- a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php +++ b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php @@ -1,5 +1,6 @@ string("expense_import")->default(true); + $table->boolean("expense_import")->default(true); + $table->string("expense_mailbox")->nullable(); + }); + Company::query()->cursor()->each(function ($company) { // TODO: @turbo124 check migration on staging environment with real data to ensure, this works as exspected + $company->expense_mailbox = config('ninja.inbound_expense.webhook.mailbox_template') != '' ? + str_replace('{{company_key}}', $company->company_key, config('ninja.inbound_expense.webhook.mailbox_template')) : null; + + $company->save(); }); Schema::table('vendor', function (Blueprint $table) { $table->string("expense_sender_email")->nullable(); From 8d6925afd38b461e20572ba0418109c52f1c90af Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 14 Dec 2023 16:40:43 +0100 Subject: [PATCH 007/119] restruct files --- app/Helpers/Mail/Mailbox/BaseMailbox.php | 19 ++ .../Mail/{ => Mailbox/Imap}/ImapMailbox.php | 8 - app/Helpers/Mail/Mailbox/Pop3/Pop3Mailbox.php | 57 ++++++ .../Mail/Webhook/BaseWebhookHandler.php | 20 ++ .../Webhook/Mailgun/MailgunWebhookHandler.php | 22 +++ .../Postmark/PostmarkWebhookHandler.php | 22 +++ app/Http/Controllers/MailgunController.php | 72 +++++++ app/Jobs/Mail/ExpenseMailboxJob.php | 2 +- .../Mailgun/ProcessMailgunInboundWebhook.php | 180 ++++++++++++++++++ routes/api.php | 2 + 10 files changed, 395 insertions(+), 9 deletions(-) create mode 100644 app/Helpers/Mail/Mailbox/BaseMailbox.php rename app/Helpers/Mail/{ => Mailbox/Imap}/ImapMailbox.php (90%) create mode 100644 app/Helpers/Mail/Mailbox/Pop3/Pop3Mailbox.php create mode 100644 app/Helpers/Mail/Webhook/BaseWebhookHandler.php create mode 100644 app/Helpers/Mail/Webhook/Mailgun/MailgunWebhookHandler.php create mode 100644 app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php create mode 100644 app/Http/Controllers/MailgunController.php create mode 100644 app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php diff --git a/app/Helpers/Mail/Mailbox/BaseMailbox.php b/app/Helpers/Mail/Mailbox/BaseMailbox.php new file mode 100644 index 000000000000..6ab63b11acac --- /dev/null +++ b/app/Helpers/Mail/Mailbox/BaseMailbox.php @@ -0,0 +1,19 @@ +sub(new \DateInterval('P30D')); $search->addCondition(new Since($thirtyDaysAgo)); - // not flagged with IN-PARSED - $search->addCondition(new Unflagged()); - - return $mailbox->getMessages($search); } diff --git a/app/Helpers/Mail/Mailbox/Pop3/Pop3Mailbox.php b/app/Helpers/Mail/Mailbox/Pop3/Pop3Mailbox.php new file mode 100644 index 000000000000..658b14f08eb4 --- /dev/null +++ b/app/Helpers/Mail/Mailbox/Pop3/Pop3Mailbox.php @@ -0,0 +1,57 @@ +server = new Server($server, $port != '' ? $port : null); + + $this->connection = $this->server->authenticate($user, $password); + } + + + public function getUnprocessedEmails() + { + $mailbox = $this->connection->getMailbox('INBOX'); + + $search = new SearchExpression(); + + // not older than 30days + $today = new \DateTimeImmutable(); + $thirtyDaysAgo = $today->sub(new \DateInterval('P30D')); + $search->addCondition(new Since($thirtyDaysAgo)); + + return $mailbox->getMessages($search); + } + + public function moveProcessed(MessageInterface $mail) + { + return $mail->move($this->connection->getMailbox('PROCESSED')); + } + + public function moveFailed(MessageInterface $mail) + { + return $mail->move($this->connection->getMailbox('FAILED')); + } +} diff --git a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php b/app/Helpers/Mail/Webhook/BaseWebhookHandler.php new file mode 100644 index 000000000000..cb839f4b5ef4 --- /dev/null +++ b/app/Helpers/Mail/Webhook/BaseWebhookHandler.php @@ -0,0 +1,20 @@ +header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.postmark.token')) { + ProcessMailgunInboundWebhook::dispatch($request->all())->delay(10); + + return response()->json(['message' => 'Success'], 200); + } + + return response()->json(['message' => 'Unauthorized'], 403); + } +} diff --git a/app/Jobs/Mail/ExpenseMailboxJob.php b/app/Jobs/Mail/ExpenseMailboxJob.php index 956d29c95619..972d835b45ef 100644 --- a/app/Jobs/Mail/ExpenseMailboxJob.php +++ b/app/Jobs/Mail/ExpenseMailboxJob.php @@ -17,7 +17,7 @@ use App\Events\Expense\ExpenseWasCreated; use App\Events\Invoice\InvoiceWasEmailedAndFailed; use App\Events\Payment\PaymentWasEmailedAndFailed; use App\Factory\ExpenseFactory; -use App\Helpers\Mail\ImapMailbox; +use App\Helpers\Mail\Mailbox\Imap\ImapMailbox; use App\Jobs\Util\SystemLogger; use App\Libraries\Google\Google; use App\Libraries\MultiDB; diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php new file mode 100644 index 000000000000..069b11003b4c --- /dev/null +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -0,0 +1,180 @@ + '', + 'subject' => 'Message not found.', + 'entity' => '', + 'entity_id' => '', + 'events' => [], + ]; + + /** + * Create a new job instance. + * + */ + public function __construct(private array $request) + { + } + + private function getSystemLog(string $message_id): ?SystemLog + { + return SystemLog::query() + ->where('company_id', $this->invitation->company_id) + ->where('type_id', SystemLog::TYPE_WEBHOOK_RESPONSE) + ->whereJsonContains('log', ['MessageID' => $message_id]) + ->orderBy('id', 'desc') + ->first(); + + } + + private function updateSystemLog(SystemLog $system_log, array $data): void + { + $system_log->log = $data; + $system_log->save(); + } + + /** + * Execute the job. + * + * + * @return void + */ + public function handle() + { + MultiDB::findAndSetDbByCompanyKey($this->request['Tag']); + + // match companies + if (array_key_exists('ToFull', $this->request)) + throw new \Exception('invalid body'); + + $toEmails = []; + foreach ($this->request['ToFull'] as $toEmailEntry) + $toEmails[] = $toEmailEntry['Email']; + + // create expense for each company + $expense = new Expense(); + + $expense->company_id; + } + // { + // "FromName": "Postmarkapp Support", + // "MessageStream": "inbound", + // "From": "support@postmarkapp.com", + // "FromFull": { + // "Email": "support@postmarkapp.com", + // "Name": "Postmarkapp Support", + // "MailboxHash": "" + // }, + // "To": "\"Firstname Lastname\" ", + // "ToFull": [ + // { + // "Email": "yourhash+SampleHash@inbound.postmarkapp.com", + // "Name": "Firstname Lastname", + // "MailboxHash": "SampleHash" + // } + // ], + // "Cc": "\"First Cc\" , secondCc@postmarkapp.com>", + // "CcFull": [ + // { + // "Email": "firstcc@postmarkapp.com", + // "Name": "First Cc", + // "MailboxHash": "" + // }, + // { + // "Email": "secondCc@postmarkapp.com", + // "Name": "", + // "MailboxHash": "" + // } + // ], + // "Bcc": "\"First Bcc\" , secondbcc@postmarkapp.com>", + // "BccFull": [ + // { + // "Email": "firstbcc@postmarkapp.com", + // "Name": "First Bcc", + // "MailboxHash": "" + // }, + // { + // "Email": "secondbcc@postmarkapp.com", + // "Name": "", + // "MailboxHash": "" + // } + // ], + // "OriginalRecipient": "yourhash+SampleHash@inbound.postmarkapp.com", + // "Subject": "Test subject", + // "MessageID": "73e6d360-66eb-11e1-8e72-a8904824019b", + // "ReplyTo": "replyto@postmarkapp.com", + // "MailboxHash": "SampleHash", + // "Date": "Fri, 1 Aug 2014 16:45:32 -04:00", + // "TextBody": "This is a test text body.", + // "HtmlBody": "

This is a test html body.<\/p><\/body><\/html>", + // "StrippedTextReply": "This is the reply text", + // "Tag": "TestTag", + // "Headers": [ + // { + // "Name": "X-Header-Test", + // "Value": "" + // }, + // { + // "Name": "X-Spam-Status", + // "Value": "No" + // }, + // { + // "Name": "X-Spam-Score", + // "Value": "-0.1" + // }, + // { + // "Name": "X-Spam-Tests", + // "Value": "DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,SPF_PASS" + // } + // ], + // "Attachments": [ + // { + // "Name": "test.txt", + // "Content": "VGhpcyBpcyBhdHRhY2htZW50IGNvbnRlbnRzLCBiYXNlLTY0IGVuY29kZWQu", + // "ContentType": "text/plain", + // "ContentLength": 45 + // } + // ] + // } +} diff --git a/routes/api.php b/routes/api.php index 078274348f40..69b8530bf312 100644 --- a/routes/api.php +++ b/routes/api.php @@ -388,6 +388,7 @@ Route::group(['middleware' => ['throttle:api', 'api_db', 'token_auth', 'locale'] // Route::post('hooks', [SubscriptionController::class, 'subscribe'])->name('hooks.subscribe'); // Route::delete('hooks/{subscription_id}', [SubscriptionController::class, 'unsubscribe'])->name('hooks.unsubscribe'); + Route::post('stripe/update_payment_methods', [StripeController::class, 'update'])->middleware('password_protected')->name('stripe.update'); Route::post('stripe/import_customers', [StripeController::class, 'import'])->middleware('password_protected')->name('stripe.import'); @@ -415,6 +416,7 @@ Route::match(['get', 'post'], 'payment_notification_webhook/{company_key}/{compa Route::post('api/v1/postmark_webhook', [PostMarkController::class, 'webhook'])->middleware('throttle:1000,1'); Route::post('api/v1/postmark_inbound_webhook', [PostMarkController::class, 'inboundWebhook'])->middleware('throttle:1000,1'); +Route::post('api/v1/mailgun_inbound_webhook', [MailgunController::class, 'inboundWebhook'])->middleware('throttle:1000,1'); Route::get('token_hash_router', [OneTimeTokenController::class, 'router'])->middleware('throttle:500,1'); Route::get('webcron', [WebCronController::class, 'index'])->middleware('throttle:100,1'); Route::post('api/v1/get_migration_account', [HostedMigrationController::class, 'getAccount'])->middleware('guest')->middleware('throttle:100,1'); From 5adb799100542e030416c11f1ee47ab2778ff659 Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 14 Dec 2023 18:25:38 +0100 Subject: [PATCH 008/119] removing pop3 integration, becasue we cannot move mails --- app/Helpers/Mail/Mailbox/Pop3/Pop3Mailbox.php | 57 ------------------- 1 file changed, 57 deletions(-) delete mode 100644 app/Helpers/Mail/Mailbox/Pop3/Pop3Mailbox.php diff --git a/app/Helpers/Mail/Mailbox/Pop3/Pop3Mailbox.php b/app/Helpers/Mail/Mailbox/Pop3/Pop3Mailbox.php deleted file mode 100644 index 658b14f08eb4..000000000000 --- a/app/Helpers/Mail/Mailbox/Pop3/Pop3Mailbox.php +++ /dev/null @@ -1,57 +0,0 @@ -server = new Server($server, $port != '' ? $port : null); - - $this->connection = $this->server->authenticate($user, $password); - } - - - public function getUnprocessedEmails() - { - $mailbox = $this->connection->getMailbox('INBOX'); - - $search = new SearchExpression(); - - // not older than 30days - $today = new \DateTimeImmutable(); - $thirtyDaysAgo = $today->sub(new \DateInterval('P30D')); - $search->addCondition(new Since($thirtyDaysAgo)); - - return $mailbox->getMessages($search); - } - - public function moveProcessed(MessageInterface $mail) - { - return $mail->move($this->connection->getMailbox('PROCESSED')); - } - - public function moveFailed(MessageInterface $mail) - { - return $mail->move($this->connection->getMailbox('FAILED')); - } -} From 6df213956f668e0b035ee0e711a6808d8bf43401 Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 14 Dec 2023 19:17:29 +0100 Subject: [PATCH 009/119] wip: updates --- app/Http/Controllers/MailgunController.php | 60 ++- .../Mailgun/ProcessMailgunInboundWebhook.php | 2 +- app/Jobs/Mailgun/ProcessMailgunWebhook.php | 413 ++++++++++++++++++ routes/api.php | 3 + 4 files changed, 470 insertions(+), 8 deletions(-) create mode 100644 app/Jobs/Mailgun/ProcessMailgunWebhook.php diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 11b7fb68cb48..1855934bae47 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -12,10 +12,11 @@ namespace App\Http\Controllers; use App\Jobs\Mailgun\ProcessMailgunInboundWebhook; +use App\Jobs\Mailgun\ProcessMailgunWebhook; use Illuminate\Http\Request; /** - * Class PostMarkController. + * Class MailgunController. */ class MailgunController extends BaseController { @@ -26,14 +27,59 @@ class MailgunController extends BaseController } /** - * Process Postmark Webhook. + * Process Mailgun Webhook. * * * @OA\Post( - * path="/api/v1/postmark_inbound_webhook", - * operationId="postmarkInboundWebhook", - * tags={"postmark"}, - * summary="Processing inbound webhooks from PostMark", + * path="/api/v1/mailgun_webhook", + * operationId="mailgunWebhook", + * tags={"mailgun"}, + * summary="Processing webhooks from Mailgun", + * description="Adds an credit to the system", + * @OA\Parameter(ref="#/components/parameters/X-API-TOKEN"), + * @OA\Parameter(ref="#/components/parameters/X-Requested-With"), + * @OA\Parameter(ref="#/components/parameters/include"), + * @OA\Response( + * response=200, + * description="Returns the saved credit object", + * @OA\Header(header="X-MINIMUM-CLIENT-VERSION", ref="#/components/headers/X-MINIMUM-CLIENT-VERSION"), + * @OA\Header(header="X-RateLimit-Remaining", ref="#/components/headers/X-RateLimit-Remaining"), + * @OA\Header(header="X-RateLimit-Limit", ref="#/components/headers/X-RateLimit-Limit"), + * @OA\JsonContent(ref="#/components/schemas/Credit"), + * ), + * @OA\Response( + * response=422, + * description="Validation error", + * @OA\JsonContent(ref="#/components/schemas/ValidationError"), + * + * ), + * @OA\Response( + * response="default", + * description="Unexpected Error", + * @OA\JsonContent(ref="#/components/schemas/Error"), + * ), + * ) + */ + public function webhook(Request $request) + { + if ($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.mailgun.token')) { + ProcessMailgunWebhook::dispatch($request->all())->delay(10); + + return response()->json(['message' => 'Success'], 200); + } + + return response()->json(['message' => 'Unauthorized'], 403); + } + + /** + * Process Mailgun Webhook. + * + * + * @OA\Post( + * path="/api/v1/mailgun_inbound_webhook", + * operationId="mailgunInboundWebhook", + * tags={"mailgun"}, + * summary="Processing inbound webhooks from Mailgun", * description="Adds an credit to the system", * @OA\Parameter(ref="#/components/parameters/X-API-TOKEN"), * @OA\Parameter(ref="#/components/parameters/X-Requested-With"), @@ -61,7 +107,7 @@ class MailgunController extends BaseController */ public function inboundWebhook(Request $request) { - if ($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.postmark.token')) { + if ($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.mailgun.token')) { ProcessMailgunInboundWebhook::dispatch($request->all())->delay(10); return response()->json(['message' => 'Success'], 200); diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index 069b11003b4c..5640d72c1392 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -9,7 +9,7 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Jobs\PostMark; +namespace App\Jobs\Mailgun; use App\DataMapper\Analytics\Mail\EmailBounce; use App\DataMapper\Analytics\Mail\EmailSpam; diff --git a/app/Jobs/Mailgun/ProcessMailgunWebhook.php b/app/Jobs/Mailgun/ProcessMailgunWebhook.php new file mode 100644 index 000000000000..f5bf05f69352 --- /dev/null +++ b/app/Jobs/Mailgun/ProcessMailgunWebhook.php @@ -0,0 +1,413 @@ + '', + 'subject' => 'Message not found.', + 'entity' => '', + 'entity_id' => '', + 'events' => [], + ]; + + /** + * Create a new job instance. + * + */ + public function __construct(private array $request) + { + } + + private function getSystemLog(string $message_id): ?SystemLog + { + return SystemLog::query() + ->where('company_id', $this->invitation->company_id) + ->where('type_id', SystemLog::TYPE_WEBHOOK_RESPONSE) + ->whereJsonContains('log', ['MessageID' => $message_id]) + ->orderBy('id', 'desc') + ->first(); + + } + + private function updateSystemLog(SystemLog $system_log, array $data): void + { + $system_log->log = $data; + $system_log->save(); + } + + /** + * Execute the job. + * + * + * @return void + */ + public function handle() + { + MultiDB::findAndSetDbByCompanyKey($this->request['Tag']); + + $this->invitation = $this->discoverInvitation($this->request['MessageID']); + + if (!$this->invitation) { + return; + } + + if (array_key_exists('Details', $this->request)) { + $this->invitation->email_error = $this->request['Details']; + } + + switch ($this->request['RecordType']) { + case 'delivered': + return $this->processDelivery(); + case 'permanent_fail': + case 'temporary_fail': + return $this->processBounce(); + case 'complained': + return $this->processSpamComplaint(); + case 'opened': + return $this->processOpen(); + default: + # code... + break; + } + } + + // { + // "Metadata": { + // "example": "value", + // "example_2": "value" + // }, + // "RecordType": "Open", + // "FirstOpen": true, + // "Client": { + // "Name": "Chrome 35.0.1916.153", + // "Company": "Google", + // "Family": "Chrome" + // }, + // "OS": { + // "Name": "OS X 10.7 Lion", + // "Company": "Apple Computer, Inc.", + // "Family": "OS X 10" + // }, + // "Platform": "WebMail", + // "UserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36", + // "ReadSeconds": 5, + // "Geo": { + // "CountryISOCode": "RS", + // "Country": "Serbia", + // "RegionISOCode": "VO", + // "Region": "Autonomna Pokrajina Vojvodina", + // "City": "Novi Sad", + // "Zip": "21000", + // "Coords": "45.2517,19.8369", + // "IP": "188.2.95.4" + // }, + // "MessageID": "00000000-0000-0000-0000-000000000000", + // "MessageStream": "outbound", + // "ReceivedAt": "2022-02-06T06:37:48Z", + // "Tag": "welcome-email", + // "Recipient": "john@example.com" + // } + + private function processOpen() + { + $this->invitation->opened_date = now(); + $this->invitation->save(); + + $data = array_merge($this->request, ['history' => $this->fetchMessage()]); + + $sl = $this->getSystemLog($this->request['MessageID']); + + if ($sl) { + $this->updateSystemLog($sl, $data); + return; + } + + (new SystemLogger( + $data, + SystemLog::CATEGORY_MAIL, + SystemLog::EVENT_MAIL_OPENED, + SystemLog::TYPE_WEBHOOK_RESPONSE, + $this->invitation->contact->client, + $this->invitation->company + ))->handle(); + } + + // { + // "RecordType": "Delivery", + // "ServerID": 23, + // "MessageStream": "outbound", + // "MessageID": "00000000-0000-0000-0000-000000000000", + // "Recipient": "john@example.com", + // "Tag": "welcome-email", + // "DeliveredAt": "2021-02-21T16:34:52Z", + // "Details": "Test delivery webhook details", + // "Metadata": { + // "example": "value", + // "example_2": "value" + // } + // } + private function processDelivery() + { + $this->invitation->email_status = 'delivered'; + $this->invitation->save(); + + $data = array_merge($this->request, ['history' => $this->fetchMessage()]); + + $sl = $this->getSystemLog($this->request['MessageID']); + + if ($sl) { + $this->updateSystemLog($sl, $data); + return; + } + + (new SystemLogger( + $data, + SystemLog::CATEGORY_MAIL, + SystemLog::EVENT_MAIL_DELIVERY, + SystemLog::TYPE_WEBHOOK_RESPONSE, + $this->invitation->contact->client, + $this->invitation->company + ))->handle(); + } + + // { + // "Metadata": { + // "example": "value", + // "example_2": "value" + // }, + // "RecordType": "Bounce", + // "ID": 42, + // "Type": "HardBounce", + // "TypeCode": 1, + // "Name": "Hard bounce", + // "Tag": "Test", + // "MessageID": "00000000-0000-0000-0000-000000000000", + // "ServerID": 1234, + // "MessageStream": "outbound", + // "Description": "The server was unable to deliver your message (ex: unknown user, mailbox not found).", + // "Details": "Test bounce details", + // "Email": "john@example.com", + // "From": "sender@example.com", + // "BouncedAt": "2021-02-21T16:34:52Z", + // "DumpAvailable": true, + // "Inactive": true, + // "CanActivate": true, + // "Subject": "Test subject", + // "Content": "Test content" + // } + + private function processBounce() + { + $this->invitation->email_status = 'bounced'; + $this->invitation->save(); + + $bounce = new EmailBounce( + $this->request['Tag'], + $this->request['From'], + $this->request['MessageID'] + ); + + LightLogs::create($bounce)->send(); + + $data = array_merge($this->request, ['history' => $this->fetchMessage()]); + + $sl = $this->getSystemLog($this->request['MessageID']); + + if ($sl) { + $this->updateSystemLog($sl, $data); + return; + } + + (new SystemLogger($data, SystemLog::CATEGORY_MAIL, SystemLog::EVENT_MAIL_BOUNCED, SystemLog::TYPE_WEBHOOK_RESPONSE, $this->invitation->contact->client, $this->invitation->company))->handle(); + + // if(config('ninja.notification.slack')) + // $this->invitation->company->notification(new EmailBounceNotification($this->invitation->company->account))->ninja(); + } + + // { + // "Metadata": { + // "example": "value", + // "example_2": "value" + // }, + // "RecordType": "SpamComplaint", + // "ID": 42, + // "Type": "SpamComplaint", + // "TypeCode": 100001, + // "Name": "Spam complaint", + // "Tag": "Test", + // "MessageID": "00000000-0000-0000-0000-000000000000", + // "ServerID": 1234, + // "MessageStream": "outbound", + // "Description": "The subscriber explicitly marked this message as spam.", + // "Details": "Test spam complaint details", + // "Email": "john@example.com", + // "From": "sender@example.com", + // "BouncedAt": "2021-02-21T16:34:52Z", + // "DumpAvailable": true, + // "Inactive": true, + // "CanActivate": false, + // "Subject": "Test subject", + // "Content": "Test content" + // } + private function processSpamComplaint() + { + $this->invitation->email_status = 'spam'; + $this->invitation->save(); + + $spam = new EmailSpam( + $this->request['Tag'], + $this->request['From'], + $this->request['MessageID'] + ); + + LightLogs::create($spam)->send(); + + $data = array_merge($this->request, ['history' => $this->fetchMessage()]); + + $sl = $this->getSystemLog($this->request['MessageID']); + + if ($sl) { + $this->updateSystemLog($sl, $data); + return; + } + + (new SystemLogger($data, SystemLog::CATEGORY_MAIL, SystemLog::EVENT_MAIL_SPAM_COMPLAINT, SystemLog::TYPE_WEBHOOK_RESPONSE, $this->invitation->contact->client, $this->invitation->company))->handle(); + + if (config('ninja.notification.slack')) { + $this->invitation->company->notification(new EmailSpamNotification($this->invitation->company->account))->ninja(); + } + } + + private function discoverInvitation($message_id) + { + $invitation = false; + + if ($invitation = InvoiceInvitation::where('message_id', $message_id)->first()) { + $this->entity = 'invoice'; + return $invitation; + } elseif ($invitation = QuoteInvitation::where('message_id', $message_id)->first()) { + $this->entity = 'quote'; + return $invitation; + } elseif ($invitation = RecurringInvoiceInvitation::where('message_id', $message_id)->first()) { + $this->entity = 'recurring_invoice'; + return $invitation; + } elseif ($invitation = CreditInvitation::where('message_id', $message_id)->first()) { + $this->entity = 'credit'; + return $invitation; + } elseif ($invitation = PurchaseOrderInvitation::where('message_id', $message_id)->first()) { + $this->entity = 'purchase_order'; + return $invitation; + } else { + return $invitation; + } + } + + public function getRawMessage(string $message_id) + { + + $postmark = new PostmarkClient(config('services.postmark.token')); + $messageDetail = $postmark->getOutboundMessageDetails($message_id); + return $messageDetail; + + } + + + public function getBounceId(string $message_id): ?int + { + + $messageDetail = $this->getRawMessage($message_id); + + + $event = collect($messageDetail->messageevents)->first(function ($event) { + + return $event?->Details?->BounceID ?? false; + + }); + + return $event?->Details?->BounceID ?? null; + + } + + private function fetchMessage(): array + { + if (strlen($this->request['MessageID']) < 1) { + return $this->default_response; + } + + try { + + $postmark = new PostmarkClient(config('services.postmark.token')); + $messageDetail = $postmark->getOutboundMessageDetails($this->request['MessageID']); + + $recipients = collect($messageDetail['recipients'])->flatten()->implode(','); + $subject = $messageDetail->subject ?? ''; + + $events = collect($messageDetail->messageevents)->map(function ($event) { + + return [ + 'bounce_id' => $event?->Details?->BounceID ?? '', + 'recipient' => $event->Recipient ?? '', + 'status' => $event->Type ?? '', + 'delivery_message' => $event->Details->DeliveryMessage ?? $event->Details->Summary ?? '', + 'server' => $event->Details->DestinationServer ?? '', + 'server_ip' => $event->Details->DestinationIP ?? '', + 'date' => \Carbon\Carbon::parse($event->ReceivedAt)->format('Y-m-d H:i:s') ?? '', + ]; + + })->toArray(); + + return [ + 'recipients' => $recipients, + 'subject' => $subject, + 'entity' => $this->entity ?? '', + 'entity_id' => $this->invitation->{$this->entity}->hashed_id ?? '', + 'events' => $events, + ]; + + } catch (\Exception $e) { + + return $this->default_response; + + } + } +} diff --git a/routes/api.php b/routes/api.php index 69b8530bf312..ae6fbdd28dc8 100644 --- a/routes/api.php +++ b/routes/api.php @@ -46,6 +46,7 @@ use App\Http\Controllers\InAppPurchase\AppleController; use App\Http\Controllers\InvoiceController; use App\Http\Controllers\LicenseController; use App\Http\Controllers\LogoutController; +use App\Http\Controllers\MailgunController; use App\Http\Controllers\MigrationController; use App\Http\Controllers\OneTimeTokenController; use App\Http\Controllers\PaymentController; @@ -416,7 +417,9 @@ Route::match(['get', 'post'], 'payment_notification_webhook/{company_key}/{compa Route::post('api/v1/postmark_webhook', [PostMarkController::class, 'webhook'])->middleware('throttle:1000,1'); Route::post('api/v1/postmark_inbound_webhook', [PostMarkController::class, 'inboundWebhook'])->middleware('throttle:1000,1'); +Route::post('api/v1/mailgun_webhook', [MailgunController::class, 'webhook'])->middleware('throttle:1000,1'); Route::post('api/v1/mailgun_inbound_webhook', [MailgunController::class, 'inboundWebhook'])->middleware('throttle:1000,1'); + Route::get('token_hash_router', [OneTimeTokenController::class, 'router'])->middleware('throttle:500,1'); Route::get('webcron', [WebCronController::class, 'index'])->middleware('throttle:100,1'); Route::post('api/v1/get_migration_account', [HostedMigrationController::class, 'getAccount'])->middleware('guest')->middleware('throttle:100,1'); From 58a7456087c273879720d51990fd421c56169346 Mon Sep 17 00:00:00 2001 From: paulwer Date: Fri, 15 Dec 2023 07:22:24 +0100 Subject: [PATCH 010/119] wip process normal mailgun webhooks --- app/Jobs/Mailgun/ProcessMailgunWebhook.php | 286 ++++++++++++--------- composer.json | 4 +- composer.lock | 146 +++++++++-- 3 files changed, 305 insertions(+), 131 deletions(-) diff --git a/app/Jobs/Mailgun/ProcessMailgunWebhook.php b/app/Jobs/Mailgun/ProcessMailgunWebhook.php index f5bf05f69352..296aef9f56a5 100644 --- a/app/Jobs/Mailgun/ProcessMailgunWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunWebhook.php @@ -27,6 +27,7 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Mailgun\Mailgun; use Postmark\PostmarkClient; use Turbo124\Beacon\Facades\LightLogs; @@ -63,7 +64,7 @@ class ProcessMailgunWebhook implements ShouldQueue return SystemLog::query() ->where('company_id', $this->invitation->company_id) ->where('type_id', SystemLog::TYPE_WEBHOOK_RESPONSE) - ->whereJsonContains('log', ['MessageID' => $message_id]) + ->whereJsonContains('log', ['id' => $message_id]) ->orderBy('id', 'desc') ->first(); @@ -85,7 +86,7 @@ class ProcessMailgunWebhook implements ShouldQueue { MultiDB::findAndSetDbByCompanyKey($this->request['Tag']); - $this->invitation = $this->discoverInvitation($this->request['MessageID']); + $this->invitation = $this->discoverInvitation($this->request['message']['headers']['message-id']); if (!$this->invitation) { return; @@ -95,11 +96,12 @@ class ProcessMailgunWebhook implements ShouldQueue $this->invitation->email_error = $this->request['Details']; } - switch ($this->request['RecordType']) { + switch ($this->request['event'] ?? $this->request['severity']) { case 'delivered': return $this->processDelivery(); - case 'permanent_fail': - case 'temporary_fail': + case 'failed': + case 'permanent': + case 'temporary': return $this->processBounce(); case 'complained': return $this->processSpamComplaint(); @@ -112,40 +114,33 @@ class ProcessMailgunWebhook implements ShouldQueue } // { - // "Metadata": { - // "example": "value", - // "example_2": "value" + // "event": "opened", + // "id": "-laxIqj9QWubsjY_3pTq_g", + // "timestamp": 1377047343.042277, + // "log-level": "info", + // "recipient": "recipient@example.com", + // "geolocation": { + // "country": "US", + // "region": "Texas", + // "city": "Austin" // }, - // "RecordType": "Open", - // "FirstOpen": true, - // "Client": { - // "Name": "Chrome 35.0.1916.153", - // "Company": "Google", - // "Family": "Chrome" + // "tags": [], + // "campaigns": [], + // "user-variables": {}, + // "ip": "111.111.111.111", + // "client-info": { + // "client-type": "mobile browser", + // "client-os": "iOS", + // "device-type": "mobile", + // "client-name": "Mobile Safari", + // "user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 6_1 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Mobile/10B143", + // "bot": "" // }, - // "OS": { - // "Name": "OS X 10.7 Lion", - // "Company": "Apple Computer, Inc.", - // "Family": "OS X 10" + // "message": { + // "headers": { + // "message-id": "20130821005614.19826.35976@samples.mailgun.org" + // } // }, - // "Platform": "WebMail", - // "UserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36", - // "ReadSeconds": 5, - // "Geo": { - // "CountryISOCode": "RS", - // "Country": "Serbia", - // "RegionISOCode": "VO", - // "Region": "Autonomna Pokrajina Vojvodina", - // "City": "Novi Sad", - // "Zip": "21000", - // "Coords": "45.2517,19.8369", - // "IP": "188.2.95.4" - // }, - // "MessageID": "00000000-0000-0000-0000-000000000000", - // "MessageStream": "outbound", - // "ReceivedAt": "2022-02-06T06:37:48Z", - // "Tag": "welcome-email", - // "Recipient": "john@example.com" // } private function processOpen() @@ -155,7 +150,7 @@ class ProcessMailgunWebhook implements ShouldQueue $data = array_merge($this->request, ['history' => $this->fetchMessage()]); - $sl = $this->getSystemLog($this->request['MessageID']); + $sl = $this->getSystemLog($this->request['message']['headers']['message-id']); if ($sl) { $this->updateSystemLog($sl, $data); @@ -173,18 +168,54 @@ class ProcessMailgunWebhook implements ShouldQueue } // { - // "RecordType": "Delivery", - // "ServerID": 23, - // "MessageStream": "outbound", - // "MessageID": "00000000-0000-0000-0000-000000000000", - // "Recipient": "john@example.com", - // "Tag": "welcome-email", - // "DeliveredAt": "2021-02-21T16:34:52Z", - // "Details": "Test delivery webhook details", - // "Metadata": { - // "example": "value", - // "example_2": "value" - // } + // "event": "delivered", + // "id": "hK7mQVt1QtqRiOfQXta4sw", + // "timestamp": 1529692199.626182, + // "log-level": "info", + // "envelope": { + // "transport": "smtp", + // "sender": "sender@example.org", + // "sending-ip": "123.123.123.123", + // "targets": "john@example.com" + // }, + // "flags": { + // "is-routed": false, + // "is-authenticated": false, + // "is-system-test": false, + // "is-test-mode": false + // }, + // "delivery-status": { + // "tls": true, + // "mx-host": "aspmx.l.example.com", + // "code": 250, + // "description": "", + // "session-seconds": 0.4367079734802246, + // "utf8": true, + // "attempt-no": 1, + // "message": "OK", + // "certificate-verified": true + // }, + // "message": { + // "headers": { + // "to": "team@example.org", + // "message-id": "20180622182958.1.48906CB188F1A454@exmple.org", + // "from": "sender@exmple.org", + // "subject": "Test Subject" + // }, + // "attachments": [], + // "size": 586 + // }, + // "storage": { + // "url": "https://storage-us-west1.api.mailgun.net/v3/domains/...", + // "region": "us-west1", + // "key": "AwABB...", + // "env": "production" + // }, + // "recipient": "john@example.com", + // "recipient-domain": "example.com", + // "campaigns": [], + // "tags": [], + // "user-variables": {} // } private function processDelivery() { @@ -193,7 +224,7 @@ class ProcessMailgunWebhook implements ShouldQueue $data = array_merge($this->request, ['history' => $this->fetchMessage()]); - $sl = $this->getSystemLog($this->request['MessageID']); + $sl = $this->getSystemLog($this->request['message']['headers']['message-id']); if ($sl) { $this->updateSystemLog($sl, $data); @@ -211,47 +242,66 @@ class ProcessMailgunWebhook implements ShouldQueue } // { - // "Metadata": { - // "example": "value", - // "example_2": "value" + // "event": "failed", || "temporary" || "permanent" + // "id": "pl271FzxTTmGRW8Uj3dUWw", + // "timestamp": 1529701969.818328, + // "log-level": "error", + // "severity": "permanent", + // "reason": "suppress-bounce", + // "envelope": { + // "sender": "john@example.org", + // "transport": "smtp", + // "targets": "joan@example.com" // }, - // "RecordType": "Bounce", - // "ID": 42, - // "Type": "HardBounce", - // "TypeCode": 1, - // "Name": "Hard bounce", - // "Tag": "Test", - // "MessageID": "00000000-0000-0000-0000-000000000000", - // "ServerID": 1234, - // "MessageStream": "outbound", - // "Description": "The server was unable to deliver your message (ex: unknown user, mailbox not found).", - // "Details": "Test bounce details", - // "Email": "john@example.com", - // "From": "sender@example.com", - // "BouncedAt": "2021-02-21T16:34:52Z", - // "DumpAvailable": true, - // "Inactive": true, - // "CanActivate": true, - // "Subject": "Test subject", - // "Content": "Test content" + // "flags": { + // "is-routed": false, + // "is-authenticated": true, + // "is-system-test": false, + // "is-test-mode": false + // }, + // "delivery-status": { + // "attempt-no": 1, + // "message": "", + // "code": 605, + // "description": "Not delivering to previously bounced address", + // "session-seconds": 0.0 + // }, + // "message": { + // "headers": { + // "to": "joan@example.com", + // "message-id": "20180622211249.1.2A6098970A380E12@example.org", + // "from": "john@example.org", + // "subject": "Test Subject" + // }, + // "attachments": [], + // "size": 867 + // }, + // "storage": { + // "url": "https://se.api.mailgun.net/v3/domains/example.org/messages/eyJwI...", + // "key": "eyJwI..." + // }, + // "recipient": "slava@mailgun.com", + // "recipient-domain": "mailgun.com", + // "campaigns": [], + // "tags": [], + // "user-variables": {} // } - private function processBounce() { $this->invitation->email_status = 'bounced'; $this->invitation->save(); $bounce = new EmailBounce( - $this->request['Tag'], - $this->request['From'], - $this->request['MessageID'] + $this->request['tags']->implode(','), + $this->request['message']['headers']['from'], + $this->request['message']['headers']['message-id'] ); LightLogs::create($bounce)->send(); $data = array_merge($this->request, ['history' => $this->fetchMessage()]); - $sl = $this->getSystemLog($this->request['MessageID']); + $sl = $this->getSystemLog($this->request['message']['headers']['message-id']); if ($sl) { $this->updateSystemLog($sl, $data); @@ -265,29 +315,33 @@ class ProcessMailgunWebhook implements ShouldQueue } // { - // "Metadata": { - // "example": "value", - // "example_2": "value" + // "event": "opened", + // "id": "-laxIqj9QWubsjY_3pTq_g", + // "timestamp": 1377047343.042277, + // "log-level": "info", + // "recipient": "recipient@example.com", + // "geolocation": { + // "country": "US", + // "region": "Texas", + // "city": "Austin" + // }, + // "tags": [], + // "campaigns": [], + // "user-variables": {}, + // "ip": "111.111.111.111", + // "client-info": { + // "client-type": "mobile browser", + // "client-os": "iOS", + // "device-type": "mobile", + // "client-name": "Mobile Safari", + // "user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 6_1 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Mobile/10B143", + // "bot": "" + // }, + // "message": { + // "headers": { + // "message-id": "20130821005614.19826.35976@samples.mailgun.org" + // } // }, - // "RecordType": "SpamComplaint", - // "ID": 42, - // "Type": "SpamComplaint", - // "TypeCode": 100001, - // "Name": "Spam complaint", - // "Tag": "Test", - // "MessageID": "00000000-0000-0000-0000-000000000000", - // "ServerID": 1234, - // "MessageStream": "outbound", - // "Description": "The subscriber explicitly marked this message as spam.", - // "Details": "Test spam complaint details", - // "Email": "john@example.com", - // "From": "sender@example.com", - // "BouncedAt": "2021-02-21T16:34:52Z", - // "DumpAvailable": true, - // "Inactive": true, - // "CanActivate": false, - // "Subject": "Test subject", - // "Content": "Test content" // } private function processSpamComplaint() { @@ -295,16 +349,16 @@ class ProcessMailgunWebhook implements ShouldQueue $this->invitation->save(); $spam = new EmailSpam( - $this->request['Tag'], + $this->request['tags'], $this->request['From'], - $this->request['MessageID'] + $this->request['message']['headers']['message-id'] ); LightLogs::create($spam)->send(); $data = array_merge($this->request, ['history' => $this->fetchMessage()]); - $sl = $this->getSystemLog($this->request['MessageID']); + $sl = $this->getSystemLog($this->request['message']['headers']['message-id']); if ($sl) { $this->updateSystemLog($sl, $data); @@ -370,28 +424,30 @@ class ProcessMailgunWebhook implements ShouldQueue private function fetchMessage(): array { - if (strlen($this->request['MessageID']) < 1) { + if (strlen($this->request['message']['headers']['message-id']) < 1) { return $this->default_response; } try { - $postmark = new PostmarkClient(config('services.postmark.token')); - $messageDetail = $postmark->getOutboundMessageDetails($this->request['MessageID']); + $mailgun = new Mailgun(config('services.mailgun.token'), config('services.mailgun.endpoint')); + $messageDetail = $mailgun->messages()->show($this->request['message']['headers']['message-id']); - $recipients = collect($messageDetail['recipients'])->flatten()->implode(','); - $subject = $messageDetail->subject ?? ''; + $recipients = collect($messageDetail->getRecipients())->flatten()->implode(','); + $subject = $messageDetail->getSubject() ?? ''; - $events = collect($messageDetail->messageevents)->map(function ($event) { + $events = collect($mailgun->events()->get(config('services.mailgun.domain'), [ + "message-id" => $this->request['message']['headers']['message-id'], + ])->getItems())->map(function ($event) { return [ - 'bounce_id' => $event?->Details?->BounceID ?? '', - 'recipient' => $event->Recipient ?? '', - 'status' => $event->Type ?? '', - 'delivery_message' => $event->Details->DeliveryMessage ?? $event->Details->Summary ?? '', - 'server' => $event->Details->DestinationServer ?? '', - 'server_ip' => $event->Details->DestinationIP ?? '', - 'date' => \Carbon\Carbon::parse($event->ReceivedAt)->format('Y-m-d H:i:s') ?? '', + 'bounce_id' => array_key_exists("id", $event) ? $event["id"] : '', + 'recipient' => array_key_exists("recipient", $event) ? $event["recipient"] : '', + 'status' => array_key_exists("delivery-status", $event) && array_key_exists("code", $event["delivery-status"]) ? $event["delivery-status"]["code"] : '', + 'delivery_message' => array_key_exists("delivery-status", $event) && array_key_exists("message", $event["delivery-status"]) ? $event["delivery-status"]["message"] : (array_key_exists("delivery-status", $event) && array_key_exists("description", $event["delivery-status"]) ? $event["delivery-status"]["description"] : ''), + 'server' => array_key_exists("delivery-status", $event) && array_key_exists("mx-host", $event["delivery-status"]) ? $event["delivery-status"]["mx-host"] : '', + 'server_ip' => array_key_exists("ip", $event) ? $event["ip"] : '', + 'date' => \Carbon\Carbon::parse($event["timestamp"])->format('Y-m-d H:i:s') ?? '', ]; })->toArray(); diff --git a/composer.json b/composer.json index 7a52c900c649..f34e970897f5 100644 --- a/composer.json +++ b/composer.json @@ -70,10 +70,12 @@ "league/fractal": "^0.20.0", "league/omnipay": "^3.1", "livewire/livewire": "^2.10", + "mailgun/mailgun-php": "^3.6", "microsoft/microsoft-graph": "^1.69", "mollie/mollie-api-php": "^2.36", "nelexa/zip": "^4.0", "nwidart/laravel-modules": "^10.0", + "nyholm/psr7": "^1.8", "omnipay/paypal": "^3.0", "payfast/payfast-php-sdk": "^1.1", "phpoffice/phpspreadsheet": "^1.29", @@ -92,7 +94,7 @@ "sprain/swiss-qr-bill": "^4.3", "square/square": "30.0.0.*", "stripe/stripe-php": "^12", - "symfony/http-client": "^6.0", + "symfony/http-client": "^7.0", "symfony/mailgun-mailer": "^6.1", "symfony/postmark-mailer": "^6.1", "turbo124/beacon": "^1.5", diff --git a/composer.lock b/composer.lock index 09d27aa2b06b..4c0450f960b0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ef5c36f71295ade916c3b7084642f0b4", + "content-hash": "a1427d5a8467c0c3dffac544c6da5f65", "packages": [ { "name": "afosto/yaac", @@ -6071,6 +6071,67 @@ ], "time": "2023-06-21T14:59:35+00:00" }, + { + "name": "mailgun/mailgun-php", + "version": "v3.6.3", + "source": { + "type": "git", + "url": "https://github.com/mailgun/mailgun-php.git", + "reference": "3dbdc2f220fa64e78e903477efa22858c72509be" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/mailgun/mailgun-php/zipball/3dbdc2f220fa64e78e903477efa22858c72509be", + "reference": "3dbdc2f220fa64e78e903477efa22858c72509be", + "shasum": "" + }, + "require": { + "php": "^7.3 || ^8.0", + "php-http/client-common": "^2.2.1", + "php-http/discovery": "^1.19", + "php-http/multipart-stream-builder": "^1.1.2", + "psr/http-client": "^1.0", + "webmozart/assert": "^1.9.1" + }, + "require-dev": { + "nyholm/nsa": "^1.2.1", + "nyholm/psr7": "^1.3.1", + "phpunit/phpunit": "^9.3", + "squizlabs/php_codesniffer": "^3.7", + "symfony/http-client": "^5.4 || ^6.3" + }, + "suggest": { + "nyholm/psr7": "PSR-7 message implementation", + "symfony/http-client": "HTTP client" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, + "autoload": { + "psr-4": { + "Mailgun\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Travis Swientek", + "email": "travis@mailgunhq.com" + } + ], + "description": "The Mailgun SDK provides methods for all API functions.", + "support": { + "issues": "https://github.com/mailgun/mailgun-php/issues", + "source": "https://github.com/mailgun/mailgun-php/tree/v3.6.3" + }, + "time": "2023-12-01T10:04:01+00:00" + }, { "name": "markbaker/complex", "version": "3.0.2", @@ -8012,6 +8073,62 @@ "abandoned": "psr/http-factory", "time": "2023-04-14T14:16:17+00:00" }, + { + "name": "php-http/multipart-stream-builder", + "version": "1.3.0", + "source": { + "type": "git", + "url": "https://github.com/php-http/multipart-stream-builder.git", + "reference": "f5938fd135d9fa442cc297dc98481805acfe2b6a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-http/multipart-stream-builder/zipball/f5938fd135d9fa442cc297dc98481805acfe2b6a", + "reference": "f5938fd135d9fa442cc297dc98481805acfe2b6a", + "shasum": "" + }, + "require": { + "php": "^7.1 || ^8.0", + "php-http/discovery": "^1.15", + "psr/http-factory-implementation": "^1.0" + }, + "require-dev": { + "nyholm/psr7": "^1.0", + "php-http/message": "^1.5", + "php-http/message-factory": "^1.0.2", + "phpunit/phpunit": "^7.5.15 || ^8.5 || ^9.3" + }, + "type": "library", + "autoload": { + "psr-4": { + "Http\\Message\\MultipartStream\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com" + } + ], + "description": "A builder class that help you create a multipart stream", + "homepage": "http://php-http.org", + "keywords": [ + "factory", + "http", + "message", + "multipart stream", + "stream" + ], + "support": { + "issues": "https://github.com/php-http/multipart-stream-builder/issues", + "source": "https://github.com/php-http/multipart-stream-builder/tree/1.3.0" + }, + "time": "2023-04-28T14:10:22+00:00" + }, { "name": "php-http/promise", "version": "1.2.1", @@ -11378,28 +11495,27 @@ }, { "name": "symfony/http-client", - "version": "v6.4.0", + "version": "v7.0.0", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "5c584530b77aa10ae216989ffc48b4bedc9c0b29" + "reference": "c3e90d09b3c45a5d47170e81a712d51c352cbc68" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/5c584530b77aa10ae216989ffc48b4bedc9c0b29", - "reference": "5c584530b77aa10ae216989ffc48b4bedc9c0b29", + "url": "https://api.github.com/repos/symfony/http-client/zipball/c3e90d09b3c45a5d47170e81a712d51c352cbc68", + "reference": "c3e90d09b3c45a5d47170e81a712d51c352cbc68", "shasum": "" }, "require": { - "php": ">=8.1", + "php": ">=8.2", "psr/log": "^1|^2|^3", - "symfony/deprecation-contracts": "^2.5|^3", "symfony/http-client-contracts": "^3", "symfony/service-contracts": "^2.5|^3" }, "conflict": { "php-http/discovery": "<1.15", - "symfony/http-foundation": "<6.3" + "symfony/http-foundation": "<6.4" }, "provide": { "php-http/async-client-implementation": "*", @@ -11416,11 +11532,11 @@ "nyholm/psr7": "^1.0", "php-http/httplug": "^1.0|^2.0", "psr/http-client": "^1.0", - "symfony/dependency-injection": "^5.4|^6.0|^7.0", - "symfony/http-kernel": "^5.4|^6.0|^7.0", - "symfony/messenger": "^5.4|^6.0|^7.0", - "symfony/process": "^5.4|^6.0|^7.0", - "symfony/stopwatch": "^5.4|^6.0|^7.0" + "symfony/dependency-injection": "^6.4|^7.0", + "symfony/http-kernel": "^6.4|^7.0", + "symfony/messenger": "^6.4|^7.0", + "symfony/process": "^6.4|^7.0", + "symfony/stopwatch": "^6.4|^7.0" }, "type": "library", "autoload": { @@ -11451,7 +11567,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v6.4.0" + "source": "https://github.com/symfony/http-client/tree/v7.0.0" }, "funding": [ { @@ -11467,7 +11583,7 @@ "type": "tidelift" } ], - "time": "2023-11-28T20:55:58+00:00" + "time": "2023-11-29T08:40:23+00:00" }, { "name": "symfony/http-client-contracts", From dd9727a7010ffd0b357b98fe71466c5f25668bd7 Mon Sep 17 00:00:00 2001 From: paulwer Date: Fri, 15 Dec 2023 07:24:31 +0100 Subject: [PATCH 011/119] minor comments --- app/Jobs/Mailgun/ProcessMailgunWebhook.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/Jobs/Mailgun/ProcessMailgunWebhook.php b/app/Jobs/Mailgun/ProcessMailgunWebhook.php index 296aef9f56a5..dc23e836d8dc 100644 --- a/app/Jobs/Mailgun/ProcessMailgunWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunWebhook.php @@ -406,6 +406,7 @@ class ProcessMailgunWebhook implements ShouldQueue } + // TODO: unknown public function getBounceId(string $message_id): ?int { @@ -441,7 +442,7 @@ class ProcessMailgunWebhook implements ShouldQueue ])->getItems())->map(function ($event) { return [ - 'bounce_id' => array_key_exists("id", $event) ? $event["id"] : '', + 'bounce_id' => array_key_exists("id", $event) ? $event["id"] : '', // TODO: unknown 'recipient' => array_key_exists("recipient", $event) ? $event["recipient"] : '', 'status' => array_key_exists("delivery-status", $event) && array_key_exists("code", $event["delivery-status"]) ? $event["delivery-status"]["code"] : '', 'delivery_message' => array_key_exists("delivery-status", $event) && array_key_exists("message", $event["delivery-status"]) ? $event["delivery-status"]["message"] : (array_key_exists("delivery-status", $event) && array_key_exists("description", $event["delivery-status"]) ? $event["delivery-status"]["description"] : ''), From 36279be694aea3f97f3ce9a7f0d4b5f8a8264b2d Mon Sep 17 00:00:00 2001 From: paulwer Date: Fri, 15 Dec 2023 18:15:55 +0100 Subject: [PATCH 012/119] validator for ExpenseMailbox property --- .../Mail/Webhook/BaseWebhookHandler.php | 7 + .../Postmark/PostmarkWebhookHandler.php | 11 +- .../Requests/Company/StoreCompanyRequest.php | 5 +- .../Requests/Company/UpdateCompanyRequest.php | 19 +- .../Company/ValidExpenseMailbox.php | 64 ++ app/Libraries/MultiDB.php | 94 ++- app/Transformers/AccountTransformer.php | 2 +- app/Transformers/CompanyTransformer.php | 11 +- config/ninja.php | 4 +- lang/en/texts.php | 606 +++++++++--------- 10 files changed, 471 insertions(+), 352 deletions(-) create mode 100644 app/Http/ValidationRules/Company/ValidExpenseMailbox.php diff --git a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php b/app/Helpers/Mail/Webhook/BaseWebhookHandler.php index cb839f4b5ef4..b579ad3bd9e6 100644 --- a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php +++ b/app/Helpers/Mail/Webhook/BaseWebhookHandler.php @@ -11,10 +11,17 @@ namespace App\Helpers\Mail\Webhook; +use App\Models\Company; + interface BaseWebhookHandler { public function process() { } + + protected function matchCompany(string $email) + { + return Company::where("expense_mailbox", $email)->first(); + } } diff --git a/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php b/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php index 64920a7ecb0d..51e458cb57d8 100644 --- a/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php +++ b/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php @@ -11,12 +11,21 @@ namespace App\Helpers\Mail\Webhook\Postmark; +use App\Factory\ExpenseFactory; use App\Helpers\Mail\Webhook\BaseWebhookHandler; interface PostmarkWebhookHandler extends BaseWebhookHandler { - public function process() + public function process($data) { + $email = ''; + + $company = $this->matchCompany($email); + if (!$company) + return false; + + $expense = ExpenseFactory::create($company->id, $company->owner()->id); + } } diff --git a/app/Http/Requests/Company/StoreCompanyRequest.php b/app/Http/Requests/Company/StoreCompanyRequest.php index 70e116d60eea..ab5e2201baaf 100644 --- a/app/Http/Requests/Company/StoreCompanyRequest.php +++ b/app/Http/Requests/Company/StoreCompanyRequest.php @@ -13,6 +13,7 @@ namespace App\Http\Requests\Company; use App\Http\Requests\Request; use App\Http\ValidationRules\Company\ValidCompanyQuantity; +use App\Http\ValidationRules\Company\ValidExpenseMailbox; use App\Http\ValidationRules\Company\ValidSubdomain; use App\Http\ValidationRules\ValidSettingsRule; use App\Models\Company; @@ -28,7 +29,7 @@ class StoreCompanyRequest extends Request * * @return bool */ - public function authorize() : bool + public function authorize(): bool { /** @var \App\Models\User auth()->user */ $user = auth()->user(); @@ -55,6 +56,8 @@ class StoreCompanyRequest extends Request } } + $rules['expense_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); // @turbo124 check if this is right + return $rules; } diff --git a/app/Http/Requests/Company/UpdateCompanyRequest.php b/app/Http/Requests/Company/UpdateCompanyRequest.php index 4ffa0e33093d..65150e376ca1 100644 --- a/app/Http/Requests/Company/UpdateCompanyRequest.php +++ b/app/Http/Requests/Company/UpdateCompanyRequest.php @@ -13,6 +13,7 @@ namespace App\Http\Requests\Company; use App\DataMapper\CompanySettings; use App\Http\Requests\Request; +use App\Http\ValidationRules\Company\ValidExpenseMailbox; use App\Http\ValidationRules\Company\ValidSubdomain; use App\Http\ValidationRules\ValidSettingsRule; use App\Utils\Ninja; @@ -35,7 +36,7 @@ class UpdateCompanyRequest extends Request * * @return bool */ - public function authorize() : bool + public function authorize(): bool { /** @var \App\Models\User $user */ $user = auth()->user(); @@ -67,6 +68,8 @@ class UpdateCompanyRequest extends Request $rules['subdomain'] = ['nullable', 'regex:/^[a-zA-Z0-9.-]+[a-zA-Z0-9]$/', new ValidSubdomain()]; } + $rules['expense_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); // @turbo124 check if this is right + return $rules; } @@ -80,14 +83,14 @@ class UpdateCompanyRequest extends Request } if (array_key_exists('settings', $input)) { - $input['settings'] = (array)$this->filterSaveableSettings($input['settings']); + $input['settings'] = (array) $this->filterSaveableSettings($input['settings']); } - if(array_key_exists('subdomain', $input) && $this->company->subdomain == $input['subdomain']) { + if (array_key_exists('subdomain', $input) && $this->company->subdomain == $input['subdomain']) { unset($input['subdomain']); } - if(array_key_exists('e_invoice_certificate_passphrase', $input) && empty($input['e_invoice_certificate_passphrase'])) { + if (array_key_exists('e_invoice_certificate_passphrase', $input) && empty($input['e_invoice_certificate_passphrase'])) { unset($input['e_invoice_certificate_passphrase']); } @@ -115,17 +118,17 @@ class UpdateCompanyRequest extends Request } if (isset($settings['email_style_custom'])) { - $settings['email_style_custom'] = str_replace(['{{','}}'], ['',''], $settings['email_style_custom']); + $settings['email_style_custom'] = str_replace(['{{', '}}'], ['', ''], $settings['email_style_custom']); } - if (! $account->isFreeHostedClient()) { + if (!$account->isFreeHostedClient()) { return $settings; } $saveable_casts = CompanySettings::$free_plan_casts; foreach ($settings as $key => $value) { - if (! array_key_exists($key, $saveable_casts)) { + if (!array_key_exists($key, $saveable_casts)) { unset($settings->{$key}); } } @@ -137,7 +140,7 @@ class UpdateCompanyRequest extends Request { if (Ninja::isHosted()) { $url = str_replace('http://', '', $url); - $url = parse_url($url, PHP_URL_SCHEME) === null ? $scheme.$url : $url; + $url = parse_url($url, PHP_URL_SCHEME) === null ? $scheme . $url : $url; } return rtrim($url, '/'); diff --git a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php new file mode 100644 index 000000000000..bb1b90212e22 --- /dev/null +++ b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php @@ -0,0 +1,64 @@ +company_key = $company_key; + $this->isEnterprise = $isEnterprise; + } + + public function passes($attribute, $value) + { + if (empty($value)) { + return true; + } + + // early return, if we dont have any additional validation + if (!config('ninja.inbound_expense.webhook.mailbox_schema') && !(Ninja::isHosted() && config('ninja.inbound_expense.webhook.mailbox_schema_enterprise'))) { + $this->validated_schema = true; + return MultiDB::checkExpenseMailboxAvailable($value); + } + + // Validate Schema + $validated = !config('ninja.inbound_expense.webhook.mailbox_schema') || (preg_match(config('ninja.inbound_expense.webhook.mailbox_schema'), $value) && (!config('ninja.inbound_expense.webhook.mailbox_schema_hascompanykey') || str_contains($value, $this->company_key))) ? true : false; + $validated_enterprise = !config('ninja.inbound_expense.webhook.mailbox_schema_enterprise') || (Ninja::isHosted() && $this->isEnterprise && preg_match(config('ninja.inbound_expense.webhook.mailbox_schema_enterprise'), $value)); + + if (!$validated && !$validated_enterprise) + return false; + + $this->validated_schema = true; + return MultiDB::checkExpenseMailboxAvailable($value); + } + + /** + * @return string + */ + public function message() + { + return $this->validated_schema ? ctrans('texts.expense_mailbox_taken') : ctrans('texts.expense_mailbox_invalid'); + } +} diff --git a/app/Libraries/MultiDB.php b/app/Libraries/MultiDB.php index 57e734253bcf..f58841800c7b 100644 --- a/app/Libraries/MultiDB.php +++ b/app/Libraries/MultiDB.php @@ -71,18 +71,20 @@ class MultiDB 'socket', ]; + private static $protected_expense_mailboxes = []; + /** * @return array */ - public static function getDbs() : array + public static function getDbs(): array { return self::$dbs; } - public static function checkDomainAvailable($subdomain) : bool + public static function checkDomainAvailable($subdomain): bool { - if (! config('ninja.db.multi_db_enabled')) { + if (!config('ninja.db.multi_db_enabled')) { return Company::whereSubdomain($subdomain)->count() == 0; } @@ -105,9 +107,35 @@ class MultiDB return true; } - public static function checkUserEmailExists($email) : bool + public static function checkExpenseMailboxAvailable($expense_mailbox): bool { - if (! config('ninja.db.multi_db_enabled')) { + + if (!config('ninja.db.multi_db_enabled')) { + return Company::where("expense_mailbox", $expense_mailbox)->withTrashed()->exists(); + } + + if (in_array($expense_mailbox, self::$protected_expense_mailboxes)) { + return false; + } + + $current_db = config('database.default'); + + foreach (self::$dbs as $db) { + if (Company::on($db)->where("expense_mailbox", $expense_mailbox)->withTrashed()->exists()) { + self::setDb($current_db); + + return false; + } + } + + self::setDb($current_db); + + return true; + } + + public static function checkUserEmailExists($email): bool + { + if (!config('ninja.db.multi_db_enabled')) { return User::where(['email' => $email])->withTrashed()->exists(); } // true >= 1 emails found / false -> == emails found @@ -139,7 +167,7 @@ class MultiDB * @param string $company_key The company key * @return bool True|False */ - public static function checkUserAndCompanyCoExist($email, $company_key) :bool + public static function checkUserAndCompanyCoExist($email, $company_key): bool { $current_db = config('database.default'); @@ -166,9 +194,9 @@ class MultiDB * @param array $data * @return User|null */ - public static function hasUser(array $data) : ?User + public static function hasUser(array $data): ?User { - if (! config('ninja.db.multi_db_enabled')) { + if (!config('ninja.db.multi_db_enabled')) { return User::where($data)->withTrashed()->first(); } @@ -190,9 +218,9 @@ class MultiDB * @param string $email * @return ClientContact|null */ - public static function hasContact(string $email) : ?ClientContact + public static function hasContact(string $email): ?ClientContact { - if (! config('ninja.db.multi_db_enabled')) { + if (!config('ninja.db.multi_db_enabled')) { return ClientContact::where('email', $email)->withTrashed()->first(); } @@ -217,9 +245,9 @@ class MultiDB * @param array $search * @return ClientContact|null */ - public static function findContact(array $search) : ?ClientContact + public static function findContact(array $search): ?ClientContact { - if (! config('ninja.db.multi_db_enabled')) { + if (!config('ninja.db.multi_db_enabled')) { return ClientContact::where($search)->first(); } @@ -240,7 +268,7 @@ class MultiDB return null; } - public static function contactFindAndSetDb($token) :bool + public static function contactFindAndSetDb($token): bool { $current_db = config('database.default'); @@ -257,7 +285,7 @@ class MultiDB return false; } - public static function userFindAndSetDb($email) : bool + public static function userFindAndSetDb($email): bool { $current_db = config('database.default'); @@ -275,7 +303,7 @@ class MultiDB return false; } - public static function documentFindAndSetDb($hash) : bool + public static function documentFindAndSetDb($hash): bool { $current_db = config('database.default'); @@ -293,7 +321,7 @@ class MultiDB return false; } - public static function findAndSetDb($token) :bool + public static function findAndSetDb($token): bool { $current_db = config('database.default'); @@ -310,7 +338,7 @@ class MultiDB return false; } - public static function findAndSetDbByCompanyKey($company_key) :bool + public static function findAndSetDbByCompanyKey($company_key): bool { $current_db = config('database.default'); @@ -327,7 +355,7 @@ class MultiDB return false; } - public static function findAndSetDbByCompanyId($company_id) :?Company + public static function findAndSetDbByCompanyId($company_id): ?Company { $current_db = config('database.default'); @@ -344,7 +372,7 @@ class MultiDB return null; } - public static function findAndSetDbByShopifyName($shopify_name) :?Company + public static function findAndSetDbByShopifyName($shopify_name): ?Company { $current_db = config('database.default'); @@ -361,7 +389,7 @@ class MultiDB return null; } - public static function findAndSetDbByAccountKey($account_key) :bool + public static function findAndSetDbByAccountKey($account_key): bool { $current_db = config('database.default'); @@ -378,7 +406,7 @@ class MultiDB return false; } - public static function findAndSetDbByInappTransactionId($transaction_id) :bool + public static function findAndSetDbByInappTransactionId($transaction_id): bool { $current_db = config('database.default'); @@ -396,7 +424,7 @@ class MultiDB } - public static function findAndSetDbByContactKey($contact_key) :bool + public static function findAndSetDbByContactKey($contact_key): bool { $current_db = config('database.default'); @@ -413,7 +441,7 @@ class MultiDB return false; } - public static function findAndSetDbByVendorContactKey($contact_key) :bool + public static function findAndSetDbByVendorContactKey($contact_key): bool { $current_db = config('database.default'); @@ -430,7 +458,7 @@ class MultiDB return false; } - public static function findAndSetDbByClientHash($client_hash) :bool + public static function findAndSetDbByClientHash($client_hash): bool { $current_db = config('database.default'); @@ -447,7 +475,7 @@ class MultiDB return false; } - public static function findAndSetDbByClientId($client_id) :?Client + public static function findAndSetDbByClientId($client_id): ?Client { $current_db = config('database.default'); @@ -466,7 +494,7 @@ class MultiDB public static function findAndSetDbByDomain($query_array) { - if (! config('ninja.db.multi_db_enabled')) { + if (!config('ninja.db.multi_db_enabled')) { return Company::where($query_array)->first(); } @@ -487,7 +515,7 @@ class MultiDB public static function findAndSetDbByInvitation($entity, $invitation_key) { - $class = 'App\Models\\'.ucfirst(Str::camel($entity)).'Invitation'; + $class = 'App\Models\\' . ucfirst(Str::camel($entity)) . 'Invitation'; $current_db = config('database.default'); foreach (self::$dbs as $db) { @@ -507,12 +535,12 @@ class MultiDB * @param string $phone * @return bool */ - public static function hasPhoneNumber(string $phone) : bool + public static function hasPhoneNumber(string $phone): bool { - if (! config('ninja.db.multi_db_enabled')) { + if (!config('ninja.db.multi_db_enabled')) { return Account::where('account_sms_verification_number', $phone)->where('account_sms_verified', true)->exists(); } - + $current_db = config('database.default'); foreach (self::$dbs as $db) { @@ -528,7 +556,7 @@ class MultiDB return false; } - + public static function randomSubdomainGenerator(): string { @@ -548,7 +576,7 @@ class MultiDB $string .= $consonants[rand(0, 19)]; $string .= $vowels[rand(0, 4)]; } - } while (! self::checkDomainAvailable($string)); + } while (!self::checkDomainAvailable($string)); self::setDb($current_db); @@ -559,7 +587,7 @@ class MultiDB * @param $database * @return void */ - public static function setDB(string $database) : void + public static function setDB(string $database): void { /* This will set the database connection for the request */ config(['database.default' => $database]); diff --git a/app/Transformers/AccountTransformer.php b/app/Transformers/AccountTransformer.php index 15797cff2f01..c467cdee0d6e 100644 --- a/app/Transformers/AccountTransformer.php +++ b/app/Transformers/AccountTransformer.php @@ -90,7 +90,7 @@ class AccountTransformer extends EntityTransformer 'set_react_as_default_ap' => (bool) $account->set_react_as_default_ap, 'trial_days_left' => Ninja::isHosted() ? (int) $account->getTrialDays() : 0, 'account_sms_verified' => (bool) $account->account_sms_verified, - 'has_iap_plan' => (bool)$account->inapp_transaction_id, + 'has_iap_plan' => (bool) $account->inapp_transaction_id, 'tax_api_enabled' => (bool) config('services.tax.zip_tax.key') ? true : false ]; diff --git a/app/Transformers/CompanyTransformer.php b/app/Transformers/CompanyTransformer.php index 0ea80a1e33ec..943414879b20 100644 --- a/app/Transformers/CompanyTransformer.php +++ b/app/Transformers/CompanyTransformer.php @@ -133,7 +133,7 @@ class CompanyTransformer extends EntityTransformer 'show_product_details' => (bool) $company->show_product_details, 'enable_product_quantity' => (bool) $company->enable_product_quantity, 'default_quantity' => (bool) $company->default_quantity, - 'custom_fields' => (object) $company->custom_fields ?? $std, + 'custom_fields' => (object) $company->custom_fields ?? $std, 'size_id' => (string) $company->size_id ?: '', 'industry_id' => (string) $company->industry_id ?: '', 'first_month_of_year' => (string) $company->first_month_of_year ?: '1', @@ -146,7 +146,7 @@ class CompanyTransformer extends EntityTransformer 'enabled_modules' => (int) $company->enabled_modules, 'updated_at' => (int) $company->updated_at, 'archived_at' => (int) $company->deleted_at, - 'created_at' =>(int) $company->created_at, + 'created_at' => (int) $company->created_at, 'slack_webhook_url' => (string) $company->slack_webhook_url, 'google_analytics_url' => (string) $company->google_analytics_key, //@deprecate 1-2-2021 'google_analytics_key' => (string) $company->google_analytics_key, @@ -158,7 +158,7 @@ class CompanyTransformer extends EntityTransformer 'is_large' => (bool) $this->isLarge($company), 'is_disabled' => (bool) $company->is_disabled, 'enable_shop_api' => (bool) $company->enable_shop_api, - 'mark_expenses_invoiceable'=> (bool) $company->mark_expenses_invoiceable, + 'mark_expenses_invoiceable' => (bool) $company->mark_expenses_invoiceable, 'mark_expenses_paid' => (bool) $company->mark_expenses_paid, 'invoice_expense_documents' => (bool) $company->invoice_expense_documents, 'invoice_task_timelog' => (bool) $company->invoice_task_timelog, @@ -168,10 +168,10 @@ class CompanyTransformer extends EntityTransformer 'use_credits_payment' => 'always', // @deprecate 1-2-2021 'default_task_is_date_based' => (bool) $company->default_task_is_date_based, 'enable_product_discount' => (bool) $company->enable_product_discount, - 'calculate_expense_tax_by_amount' =>(bool) $company->calculate_expense_tax_by_amount, + 'calculate_expense_tax_by_amount' => (bool) $company->calculate_expense_tax_by_amount, 'hide_empty_columns_on_pdf' => false, // @deprecate 1-2-2021 'expense_inclusive_taxes' => (bool) $company->expense_inclusive_taxes, - 'expense_amount_is_pretax' =>(bool) true, //@deprecate 1-2-2021 + 'expense_amount_is_pretax' => (bool) true, //@deprecate 1-2-2021 'oauth_password_required' => (bool) $company->oauth_password_required, 'session_timeout' => (int) $company->session_timeout, 'default_password_timeout' => (int) $company->default_password_timeout, @@ -204,6 +204,7 @@ class CompanyTransformer extends EntityTransformer 'invoice_task_project_header' => (bool) $company->invoice_task_project_header, 'invoice_task_item_description' => (bool) $company->invoice_task_item_description, 'origin_tax_data' => $company->origin_tax_data ?: new \stdClass, + 'expense_mailbox' => $company->expense_mailbox, ]; } diff --git a/config/ninja.php b/config/ninja.php index 25505a49c6a2..df09c38580d9 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -238,7 +238,9 @@ return [ ], 'webhook' => [ 'mailbox_template' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOXTEMPLATE', null), - 'mailbox_template_enterprise' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOXTEMPLATE_ENTERPRISE', '{{input}}@expense.invoicing.co'), + 'mailbox_schema' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_SCHEMA', null), + 'mailbox_schema_hascompanykey' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_SCHEMA_HASCOMPANYKEY', false), + 'mailbox_schema_enterprise' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_SCHEMA_ENTERPRISE', '.*@expense\.invoicing\.co$'), ], ], ]; diff --git a/lang/en/texts.php b/lang/en/texts.php index a4bc7734fc4b..f34136ad2d7f 100644 --- a/lang/en/texts.php +++ b/lang/en/texts.php @@ -2534,6 +2534,8 @@ $lang = array( 'local_storage_required' => 'Error: local storage is not available.', 'your_password_reset_link' => 'Your Password Reset Link', 'subdomain_taken' => 'The subdomain is already in use', + 'expense_mailbox_taken' => 'The mailbox is already in use', + 'expense_mailbox_invalid' => 'The mailbox does not match the required schema', 'client_login' => 'Client Login', 'converted_amount' => 'Converted Amount', 'default' => 'Default', @@ -3857,308 +3859,308 @@ $lang = array( 'registration_url' => 'Registration URL', 'show_product_cost' => 'Show Product Cost', 'complete' => 'Complete', - 'next' => 'Next', - 'next_step' => 'Next step', - 'notification_credit_sent_subject' => 'Credit :invoice was sent to :client', - 'notification_credit_viewed_subject' => 'Credit :invoice was viewed by :client', - 'notification_credit_sent' => 'The following client :client was emailed Credit :invoice for :amount.', - 'notification_credit_viewed' => 'The following client :client viewed Credit :credit for :amount.', - 'reset_password_text' => 'Enter your email to reset your password.', - 'password_reset' => 'Password reset', - 'account_login_text' => 'Welcome! Glad to see you.', - 'request_cancellation' => 'Request cancellation', - 'delete_payment_method' => 'Delete Payment Method', - 'about_to_delete_payment_method' => 'You are about to delete the payment method.', - 'action_cant_be_reversed' => 'Action can\'t be reversed', - 'profile_updated_successfully' => 'The profile has been updated successfully.', - 'currency_ethiopian_birr' => 'Ethiopian Birr', - 'client_information_text' => 'Use a permanent address where you can receive mail.', - 'status_id' => 'Invoice Status', - 'email_already_register' => 'This email is already linked to an account', - 'locations' => 'Locations', - 'freq_indefinitely' => 'Indefinitely', - 'cycles_remaining' => 'Cycles remaining', - 'i_understand_delete' => 'I understand, delete', - 'download_files' => 'Download Files', - 'download_timeframe' => 'Use this link to download your files, the link will expire in 1 hour.', - 'new_signup' => 'New Signup', - 'new_signup_text' => 'A new account has been created by :user - :email - from IP address: :ip', - 'notification_payment_paid_subject' => 'Payment was made by :client', - 'notification_partial_payment_paid_subject' => 'Partial payment was made by :client', - 'notification_payment_paid' => 'A payment of :amount was made by client :client towards :invoice', - 'notification_partial_payment_paid' => 'A partial payment of :amount was made by client :client towards :invoice', - 'notification_bot' => 'Notification Bot', - 'invoice_number_placeholder' => 'Invoice # :invoice', - 'entity_number_placeholder' => ':entity # :entity_number', - 'email_link_not_working' => 'If the button above isn\'t working for you, please click on the link', - 'display_log' => 'Display Log', - 'send_fail_logs_to_our_server' => 'Report errors in realtime', - 'setup' => 'Setup', - 'quick_overview_statistics' => 'Quick overview & statistics', - 'update_your_personal_info' => 'Update your personal information', - 'name_website_logo' => 'Name, website & logo', - 'make_sure_use_full_link' => 'Make sure you use full link to your site', - 'personal_address' => 'Personal address', - 'enter_your_personal_address' => 'Enter your personal address', - 'enter_your_shipping_address' => 'Enter your shipping address', - 'list_of_invoices' => 'List of invoices', - 'with_selected' => 'With selected', - 'invoice_still_unpaid' => 'This invoice is still not paid. Click the button to complete the payment', - 'list_of_recurring_invoices' => 'List of recurring invoices', - 'details_of_recurring_invoice' => 'Here are some details about recurring invoice', - 'cancellation' => 'Cancellation', - 'about_cancellation' => 'In case you want to stop the recurring invoice, please click to request the cancellation.', - 'cancellation_warning' => 'Warning! You are requesting a cancellation of this service. Your service may be cancelled with no further notification to you.', - 'cancellation_pending' => 'Cancellation pending, we\'ll be in touch!', - 'list_of_payments' => 'List of payments', - 'payment_details' => 'Details of the payment', - 'list_of_payment_invoices' => 'List of invoices affected by the payment', - 'list_of_payment_methods' => 'List of payment methods', - 'payment_method_details' => 'Details of payment method', - 'permanently_remove_payment_method' => 'Permanently remove this payment method.', - 'warning_action_cannot_be_reversed' => 'Warning! This action can not be reversed!', - 'confirmation' => 'Confirmation', - 'list_of_quotes' => 'Quotes', - 'waiting_for_approval' => 'Waiting for approval', - 'quote_still_not_approved' => 'This quote is still not approved', - 'list_of_credits' => 'Credits', - 'required_extensions' => 'Required extensions', - 'php_version' => 'PHP version', - 'writable_env_file' => 'Writable .env file', - 'env_not_writable' => '.env file is not writable by the current user.', - 'minumum_php_version' => 'Minimum PHP version', - 'satisfy_requirements' => 'Make sure all requirements are satisfied.', - 'oops_issues' => 'Oops, something does not look right!', - 'open_in_new_tab' => 'Open in new tab', - 'complete_your_payment' => 'Complete payment', - 'authorize_for_future_use' => 'Authorize payment method for future use', - 'page' => 'Page', - 'per_page' => 'Per page', - 'of' => 'Of', - 'view_credit' => 'View Credit', - 'to_view_entity_password' => 'To view the :entity you need to enter password.', - 'showing_x_of' => 'Showing :first to :last out of :total results', - 'no_results' => 'No results found.', - 'payment_failed_subject' => 'Payment failed for Client :client', - 'payment_failed_body' => 'A payment made by client :client failed with message :message', - 'register' => 'Register', - 'register_label' => 'Create your account in seconds', - 'password_confirmation' => 'Confirm your password', - 'verification' => 'Verification', - 'complete_your_bank_account_verification' => 'Before using a bank account it must be verified.', - 'checkout_com' => 'Checkout.com', - 'footer_label' => 'Copyright © :year :company.', - 'credit_card_invalid' => 'Provided credit card number is not valid.', - 'month_invalid' => 'Provided month is not valid.', - 'year_invalid' => 'Provided year is not valid.', - 'https_required' => 'HTTPS is required, form will fail', - 'if_you_need_help' => 'If you need help you can post to our', - 'update_password_on_confirm' => 'After updating password, your account will be confirmed.', - 'bank_account_not_linked' => 'To pay with a bank account, first you have to add it as payment method.', - 'application_settings_label' => 'Let\'s store basic information about your Invoice Ninja!', - 'recommended_in_production' => 'Highly recommended in production', - 'enable_only_for_development' => 'Enable only for development', - 'test_pdf' => 'Test PDF', - 'checkout_authorize_label' => 'Checkout.com can be can saved as payment method for future use, once you complete your first transaction. Don\'t forget to check "Store credit card details" during payment process.', - 'sofort_authorize_label' => 'Bank account (SOFORT) can be can saved as payment method for future use, once you complete your first transaction. Don\'t forget to check "Store payment details" during payment process.', - 'node_status' => 'Node status', - 'npm_status' => 'NPM status', - 'node_status_not_found' => 'I could not find Node anywhere. Is it installed?', - 'npm_status_not_found' => 'I could not find NPM anywhere. Is it installed?', - 'locked_invoice' => 'This invoice is locked and unable to be modified', - 'downloads' => 'Downloads', - 'resource' => 'Resource', - 'document_details' => 'Details about the document', - 'hash' => 'Hash', - 'resources' => 'Resources', - 'allowed_file_types' => 'Allowed file types:', - 'common_codes' => 'Common codes and their meanings', - 'payment_error_code_20087' => '20087: Bad Track Data (invalid CVV and/or expiry date)', - 'download_selected' => 'Download selected', - 'to_pay_invoices' => 'To pay invoices, you have to', - 'add_payment_method_first' => 'add payment method', - 'no_items_selected' => 'No items selected.', - 'payment_due' => 'Payment due', - 'account_balance' => 'Account Balance', - 'thanks' => 'Thanks', - 'minimum_required_payment' => 'Minimum required payment is :amount', - 'under_payments_disabled' => 'Company doesn\'t support underpayments.', - 'over_payments_disabled' => 'Company doesn\'t support overpayments.', - 'saved_at' => 'Saved at :time', - 'credit_payment' => 'Credit applied to Invoice :invoice_number', - 'credit_subject' => 'New credit :number from :account', - 'credit_message' => 'To view your credit for :amount, click the link below.', - 'payment_type_Crypto' => 'Cryptocurrency', - 'payment_type_Credit' => 'Credit', - 'store_for_future_use' => 'Store for future use', - 'pay_with_credit' => 'Pay with credit', - 'payment_method_saving_failed' => 'Payment method can\'t be saved for future use.', - 'pay_with' => 'Pay with', - 'n/a' => 'N/A', - 'by_clicking_next_you_accept_terms' => 'By clicking "Next step" you accept terms.', - 'not_specified' => 'Not specified', - 'before_proceeding_with_payment_warning' => 'Before proceeding with payment, you have to fill following fields', - 'after_completing_go_back_to_previous_page' => 'After completing, go back to previous page.', - 'pay' => 'Pay', - 'instructions' => 'Instructions', - 'notification_invoice_reminder1_sent_subject' => 'Reminder 1 for Invoice :invoice was sent to :client', - 'notification_invoice_reminder2_sent_subject' => 'Reminder 2 for Invoice :invoice was sent to :client', - 'notification_invoice_reminder3_sent_subject' => 'Reminder 3 for Invoice :invoice was sent to :client', - 'notification_invoice_custom_sent_subject' => 'Custom reminder for Invoice :invoice was sent to :client', - 'notification_invoice_reminder_endless_sent_subject' => 'Endless reminder for Invoice :invoice was sent to :client', - 'assigned_user' => 'Assigned User', - 'setup_steps_notice' => 'To proceed to next step, make sure you test each section.', - 'setup_phantomjs_note' => 'Note about Phantom JS. Read more.', - 'minimum_payment' => 'Minimum Payment', - 'no_action_provided' => 'No action provided. If you believe this is wrong, please contact the support.', - 'no_payable_invoices_selected' => 'No payable invoices selected. Make sure you are not trying to pay draft invoice or invoice with zero balance due.', - 'required_payment_information' => 'Required payment details', - 'required_payment_information_more' => 'To complete a payment we need more details about you.', - 'required_client_info_save_label' => 'We will save this, so you don\'t have to enter it next time.', - 'notification_credit_bounced' => 'We were unable to deliver Credit :invoice to :contact. \n :error', - 'notification_credit_bounced_subject' => 'Unable to deliver Credit :invoice', - 'save_payment_method_details' => 'Save payment method details', - 'new_card' => 'New card', - 'new_bank_account' => 'New bank account', - 'company_limit_reached' => 'Limit of :limit companies per account.', - 'credits_applied_validation' => 'Total credits applied cannot be MORE than total of invoices', - 'credit_number_taken' => 'Credit number already taken', - 'credit_not_found' => 'Credit not found', - 'invoices_dont_match_client' => 'Selected invoices are not from a single client', - 'duplicate_credits_submitted' => 'Duplicate credits submitted.', - 'duplicate_invoices_submitted' => 'Duplicate invoices submitted.', - 'credit_with_no_invoice' => 'You must have an invoice set when using a credit in a payment', - 'client_id_required' => 'Client id is required', - 'expense_number_taken' => 'Expense number already taken', - 'invoice_number_taken' => 'Invoice number already taken', - 'payment_id_required' => 'Payment `id` required.', - 'unable_to_retrieve_payment' => 'Unable to retrieve specified payment', - 'invoice_not_related_to_payment' => 'Invoice id :invoice is not related to this payment', - 'credit_not_related_to_payment' => 'Credit id :credit is not related to this payment', - 'max_refundable_invoice' => 'Attempting to refund more than allowed for invoice id :invoice, maximum refundable amount is :amount', - 'refund_without_invoices' => 'Attempting to refund a payment with invoices attached, please specify valid invoice/s to be refunded.', - 'refund_without_credits' => 'Attempting to refund a payment with credits attached, please specify valid credits/s to be refunded.', - 'max_refundable_credit' => 'Attempting to refund more than allowed for credit :credit, maximum refundable amount is :amount', - 'project_client_do_not_match' => 'Project client does not match entity client', - 'quote_number_taken' => 'Quote number already taken', - 'recurring_invoice_number_taken' => 'Recurring Invoice number :number already taken', - 'user_not_associated_with_account' => 'User not associated with this account', - 'amounts_do_not_balance' => 'Amounts do not balance correctly.', - 'insufficient_applied_amount_remaining' => 'Insufficient applied amount remaining to cover payment.', - 'insufficient_credit_balance' => 'Insufficient balance on credit.', - 'one_or_more_invoices_paid' => 'One or more of these invoices have been paid', - 'invoice_cannot_be_refunded' => 'Invoice id :number cannot be refunded', - 'attempted_refund_failed' => 'Attempting to refund :amount only :refundable_amount available for refund', - 'user_not_associated_with_this_account' => 'This user is unable to be attached to this company. Perhaps they have already registered a user on another account?', - 'migration_completed' => 'Migration completed', - 'migration_completed_description' => 'Your migration has completed, please review your data after logging in.', - 'api_404' => '404 | Nothing to see here!', - 'large_account_update_parameter' => 'Cannot load a large account without a updated_at parameter', - 'no_backup_exists' => 'No backup exists for this activity', - 'company_user_not_found' => 'Company User record not found', - 'no_credits_found' => 'No credits found.', - 'action_unavailable' => 'The requested action :action is not available.', - 'no_documents_found' => 'No Documents Found', - 'no_group_settings_found' => 'No group settings found', - 'access_denied' => 'Insufficient privileges to access/modify this resource', - 'invoice_cannot_be_marked_paid' => 'Invoice cannot be marked as paid', - 'invoice_license_or_environment' => 'Invalid license, or invalid environment :environment', - 'route_not_available' => 'Route not available', - 'invalid_design_object' => 'Invalid custom design object', - 'quote_not_found' => 'Quote/s not found', - 'quote_unapprovable' => 'Unable to approve this quote as it has expired.', - 'scheduler_has_run' => 'Scheduler has run', - 'scheduler_has_never_run' => 'Scheduler has never run', - 'self_update_not_available' => 'Self update not available on this system.', - 'user_detached' => 'User detached from company', - 'create_webhook_failure' => 'Failed to create Webhook', - 'payment_message_extended' => 'Thank you for your payment of :amount for :invoice', - 'online_payments_minimum_note' => 'Note: Online payments are supported only if amount is bigger than $1 or currency equivalent.', - 'payment_token_not_found' => 'Payment token not found, please try again. If an issue still persist, try with another payment method', - 'vendor_address1' => 'Vendor Street', - 'vendor_address2' => 'Vendor Apt/Suite', - 'partially_unapplied' => 'Partially Unapplied', - 'select_a_gmail_user' => 'Please select a user authenticated with Gmail', - 'list_long_press' => 'List Long Press', - 'show_actions' => 'Show Actions', - 'start_multiselect' => 'Start Multiselect', - 'email_sent_to_confirm_email' => 'An email has been sent to confirm the email address', - 'converted_paid_to_date' => 'Converted Paid to Date', - 'converted_credit_balance' => 'Converted Credit Balance', - 'converted_total' => 'Converted Total', - 'reply_to_name' => 'Reply-To Name', - 'payment_status_-2' => 'Partially Unapplied', - 'color_theme' => 'Color Theme', - 'start_migration' => 'Start Migration', - 'recurring_cancellation_request' => 'Request for recurring invoice cancellation from :contact', - 'recurring_cancellation_request_body' => ':contact from Client :client requested to cancel Recurring Invoice :invoice', - 'hello' => 'Hello', - 'group_documents' => 'Group documents', - 'quote_approval_confirmation_label' => 'Are you sure you want to approve this quote?', - 'migration_select_company_label' => 'Select companies to migrate', - 'force_migration' => 'Force migration', - 'require_password_with_social_login' => 'Require Password with Social Login', - 'stay_logged_in' => 'Stay Logged In', - 'session_about_to_expire' => 'Warning: Your session is about to expire', - 'count_hours' => ':count Hours', - 'count_day' => '1 Day', - 'count_days' => ':count Days', - 'web_session_timeout' => 'Web Session Timeout', - 'security_settings' => 'Security Settings', - 'resend_email' => 'Resend Email', - 'confirm_your_email_address' => 'Please confirm your email address', - 'freshbooks' => 'FreshBooks', - 'invoice2go' => 'Invoice2go', - 'invoicely' => 'Invoicely', - 'waveaccounting' => 'Wave Accounting', - 'zoho' => 'Zoho', - 'accounting' => 'Accounting', - 'required_files_missing' => 'Please provide all CSVs.', - 'migration_auth_label' => 'Let\'s continue by authenticating.', - 'api_secret' => 'API secret', - 'migration_api_secret_notice' => 'You can find API_SECRET in the .env file or Invoice Ninja v5. If property is missing, leave field blank.', - 'billing_coupon_notice' => 'Your discount will be applied on the checkout.', - 'use_last_email' => 'Use last email', - 'activate_company' => 'Activate Company', - 'activate_company_help' => 'Enable emails, recurring invoices and notifications', - 'an_error_occurred_try_again' => 'An error occurred, please try again', - 'please_first_set_a_password' => 'Please first set a password', - 'changing_phone_disables_two_factor' => 'Warning: Changing your phone number will disable 2FA', - 'help_translate' => 'Help Translate', - 'please_select_a_country' => 'Please select a country', - 'disabled_two_factor' => 'Successfully disabled 2FA', - 'connected_google' => 'Successfully connected account', - 'disconnected_google' => 'Successfully disconnected account', - 'delivered' => 'Delivered', - 'spam' => 'Spam', - 'view_docs' => 'View Docs', - 'enter_phone_to_enable_two_factor' => 'Please provide a mobile phone number to enable two factor authentication', - 'send_sms' => 'Send SMS', - 'sms_code' => 'SMS Code', - 'connect_google' => 'Connect Google', - 'disconnect_google' => 'Disconnect Google', - 'disable_two_factor' => 'Disable Two Factor', - 'invoice_task_datelog' => 'Invoice Task Datelog', - 'invoice_task_datelog_help' => 'Add date details to the invoice line items', - 'promo_code' => 'Promo code', - 'recurring_invoice_issued_to' => 'Recurring invoice issued to', - 'subscription' => 'Subscription', - 'new_subscription' => 'New Subscription', - 'deleted_subscription' => 'Successfully deleted subscription', - 'removed_subscription' => 'Successfully removed subscription', - 'restored_subscription' => 'Successfully restored subscription', - 'search_subscription' => 'Search 1 Subscription', - 'search_subscriptions' => 'Search :count Subscriptions', - 'subdomain_is_not_available' => 'Subdomain is not available', - 'connect_gmail' => 'Connect Gmail', - 'disconnect_gmail' => 'Disconnect Gmail', - 'connected_gmail' => 'Successfully connected Gmail', - 'disconnected_gmail' => 'Successfully disconnected Gmail', - 'update_fail_help' => 'Changes to the codebase may be blocking the update, you can run this command to discard the changes:', - 'client_id_number' => 'Client ID Number', - 'count_minutes' => ':count Minutes', - 'password_timeout' => 'Password Timeout', - 'shared_invoice_credit_counter' => 'Share Invoice/Credit Counter', + 'next' => 'Next', + 'next_step' => 'Next step', + 'notification_credit_sent_subject' => 'Credit :invoice was sent to :client', + 'notification_credit_viewed_subject' => 'Credit :invoice was viewed by :client', + 'notification_credit_sent' => 'The following client :client was emailed Credit :invoice for :amount.', + 'notification_credit_viewed' => 'The following client :client viewed Credit :credit for :amount.', + 'reset_password_text' => 'Enter your email to reset your password.', + 'password_reset' => 'Password reset', + 'account_login_text' => 'Welcome! Glad to see you.', + 'request_cancellation' => 'Request cancellation', + 'delete_payment_method' => 'Delete Payment Method', + 'about_to_delete_payment_method' => 'You are about to delete the payment method.', + 'action_cant_be_reversed' => 'Action can\'t be reversed', + 'profile_updated_successfully' => 'The profile has been updated successfully.', + 'currency_ethiopian_birr' => 'Ethiopian Birr', + 'client_information_text' => 'Use a permanent address where you can receive mail.', + 'status_id' => 'Invoice Status', + 'email_already_register' => 'This email is already linked to an account', + 'locations' => 'Locations', + 'freq_indefinitely' => 'Indefinitely', + 'cycles_remaining' => 'Cycles remaining', + 'i_understand_delete' => 'I understand, delete', + 'download_files' => 'Download Files', + 'download_timeframe' => 'Use this link to download your files, the link will expire in 1 hour.', + 'new_signup' => 'New Signup', + 'new_signup_text' => 'A new account has been created by :user - :email - from IP address: :ip', + 'notification_payment_paid_subject' => 'Payment was made by :client', + 'notification_partial_payment_paid_subject' => 'Partial payment was made by :client', + 'notification_payment_paid' => 'A payment of :amount was made by client :client towards :invoice', + 'notification_partial_payment_paid' => 'A partial payment of :amount was made by client :client towards :invoice', + 'notification_bot' => 'Notification Bot', + 'invoice_number_placeholder' => 'Invoice # :invoice', + 'entity_number_placeholder' => ':entity # :entity_number', + 'email_link_not_working' => 'If the button above isn\'t working for you, please click on the link', + 'display_log' => 'Display Log', + 'send_fail_logs_to_our_server' => 'Report errors in realtime', + 'setup' => 'Setup', + 'quick_overview_statistics' => 'Quick overview & statistics', + 'update_your_personal_info' => 'Update your personal information', + 'name_website_logo' => 'Name, website & logo', + 'make_sure_use_full_link' => 'Make sure you use full link to your site', + 'personal_address' => 'Personal address', + 'enter_your_personal_address' => 'Enter your personal address', + 'enter_your_shipping_address' => 'Enter your shipping address', + 'list_of_invoices' => 'List of invoices', + 'with_selected' => 'With selected', + 'invoice_still_unpaid' => 'This invoice is still not paid. Click the button to complete the payment', + 'list_of_recurring_invoices' => 'List of recurring invoices', + 'details_of_recurring_invoice' => 'Here are some details about recurring invoice', + 'cancellation' => 'Cancellation', + 'about_cancellation' => 'In case you want to stop the recurring invoice, please click to request the cancellation.', + 'cancellation_warning' => 'Warning! You are requesting a cancellation of this service. Your service may be cancelled with no further notification to you.', + 'cancellation_pending' => 'Cancellation pending, we\'ll be in touch!', + 'list_of_payments' => 'List of payments', + 'payment_details' => 'Details of the payment', + 'list_of_payment_invoices' => 'List of invoices affected by the payment', + 'list_of_payment_methods' => 'List of payment methods', + 'payment_method_details' => 'Details of payment method', + 'permanently_remove_payment_method' => 'Permanently remove this payment method.', + 'warning_action_cannot_be_reversed' => 'Warning! This action can not be reversed!', + 'confirmation' => 'Confirmation', + 'list_of_quotes' => 'Quotes', + 'waiting_for_approval' => 'Waiting for approval', + 'quote_still_not_approved' => 'This quote is still not approved', + 'list_of_credits' => 'Credits', + 'required_extensions' => 'Required extensions', + 'php_version' => 'PHP version', + 'writable_env_file' => 'Writable .env file', + 'env_not_writable' => '.env file is not writable by the current user.', + 'minumum_php_version' => 'Minimum PHP version', + 'satisfy_requirements' => 'Make sure all requirements are satisfied.', + 'oops_issues' => 'Oops, something does not look right!', + 'open_in_new_tab' => 'Open in new tab', + 'complete_your_payment' => 'Complete payment', + 'authorize_for_future_use' => 'Authorize payment method for future use', + 'page' => 'Page', + 'per_page' => 'Per page', + 'of' => 'Of', + 'view_credit' => 'View Credit', + 'to_view_entity_password' => 'To view the :entity you need to enter password.', + 'showing_x_of' => 'Showing :first to :last out of :total results', + 'no_results' => 'No results found.', + 'payment_failed_subject' => 'Payment failed for Client :client', + 'payment_failed_body' => 'A payment made by client :client failed with message :message', + 'register' => 'Register', + 'register_label' => 'Create your account in seconds', + 'password_confirmation' => 'Confirm your password', + 'verification' => 'Verification', + 'complete_your_bank_account_verification' => 'Before using a bank account it must be verified.', + 'checkout_com' => 'Checkout.com', + 'footer_label' => 'Copyright © :year :company.', + 'credit_card_invalid' => 'Provided credit card number is not valid.', + 'month_invalid' => 'Provided month is not valid.', + 'year_invalid' => 'Provided year is not valid.', + 'https_required' => 'HTTPS is required, form will fail', + 'if_you_need_help' => 'If you need help you can post to our', + 'update_password_on_confirm' => 'After updating password, your account will be confirmed.', + 'bank_account_not_linked' => 'To pay with a bank account, first you have to add it as payment method.', + 'application_settings_label' => 'Let\'s store basic information about your Invoice Ninja!', + 'recommended_in_production' => 'Highly recommended in production', + 'enable_only_for_development' => 'Enable only for development', + 'test_pdf' => 'Test PDF', + 'checkout_authorize_label' => 'Checkout.com can be can saved as payment method for future use, once you complete your first transaction. Don\'t forget to check "Store credit card details" during payment process.', + 'sofort_authorize_label' => 'Bank account (SOFORT) can be can saved as payment method for future use, once you complete your first transaction. Don\'t forget to check "Store payment details" during payment process.', + 'node_status' => 'Node status', + 'npm_status' => 'NPM status', + 'node_status_not_found' => 'I could not find Node anywhere. Is it installed?', + 'npm_status_not_found' => 'I could not find NPM anywhere. Is it installed?', + 'locked_invoice' => 'This invoice is locked and unable to be modified', + 'downloads' => 'Downloads', + 'resource' => 'Resource', + 'document_details' => 'Details about the document', + 'hash' => 'Hash', + 'resources' => 'Resources', + 'allowed_file_types' => 'Allowed file types:', + 'common_codes' => 'Common codes and their meanings', + 'payment_error_code_20087' => '20087: Bad Track Data (invalid CVV and/or expiry date)', + 'download_selected' => 'Download selected', + 'to_pay_invoices' => 'To pay invoices, you have to', + 'add_payment_method_first' => 'add payment method', + 'no_items_selected' => 'No items selected.', + 'payment_due' => 'Payment due', + 'account_balance' => 'Account Balance', + 'thanks' => 'Thanks', + 'minimum_required_payment' => 'Minimum required payment is :amount', + 'under_payments_disabled' => 'Company doesn\'t support underpayments.', + 'over_payments_disabled' => 'Company doesn\'t support overpayments.', + 'saved_at' => 'Saved at :time', + 'credit_payment' => 'Credit applied to Invoice :invoice_number', + 'credit_subject' => 'New credit :number from :account', + 'credit_message' => 'To view your credit for :amount, click the link below.', + 'payment_type_Crypto' => 'Cryptocurrency', + 'payment_type_Credit' => 'Credit', + 'store_for_future_use' => 'Store for future use', + 'pay_with_credit' => 'Pay with credit', + 'payment_method_saving_failed' => 'Payment method can\'t be saved for future use.', + 'pay_with' => 'Pay with', + 'n/a' => 'N/A', + 'by_clicking_next_you_accept_terms' => 'By clicking "Next step" you accept terms.', + 'not_specified' => 'Not specified', + 'before_proceeding_with_payment_warning' => 'Before proceeding with payment, you have to fill following fields', + 'after_completing_go_back_to_previous_page' => 'After completing, go back to previous page.', + 'pay' => 'Pay', + 'instructions' => 'Instructions', + 'notification_invoice_reminder1_sent_subject' => 'Reminder 1 for Invoice :invoice was sent to :client', + 'notification_invoice_reminder2_sent_subject' => 'Reminder 2 for Invoice :invoice was sent to :client', + 'notification_invoice_reminder3_sent_subject' => 'Reminder 3 for Invoice :invoice was sent to :client', + 'notification_invoice_custom_sent_subject' => 'Custom reminder for Invoice :invoice was sent to :client', + 'notification_invoice_reminder_endless_sent_subject' => 'Endless reminder for Invoice :invoice was sent to :client', + 'assigned_user' => 'Assigned User', + 'setup_steps_notice' => 'To proceed to next step, make sure you test each section.', + 'setup_phantomjs_note' => 'Note about Phantom JS. Read more.', + 'minimum_payment' => 'Minimum Payment', + 'no_action_provided' => 'No action provided. If you believe this is wrong, please contact the support.', + 'no_payable_invoices_selected' => 'No payable invoices selected. Make sure you are not trying to pay draft invoice or invoice with zero balance due.', + 'required_payment_information' => 'Required payment details', + 'required_payment_information_more' => 'To complete a payment we need more details about you.', + 'required_client_info_save_label' => 'We will save this, so you don\'t have to enter it next time.', + 'notification_credit_bounced' => 'We were unable to deliver Credit :invoice to :contact. \n :error', + 'notification_credit_bounced_subject' => 'Unable to deliver Credit :invoice', + 'save_payment_method_details' => 'Save payment method details', + 'new_card' => 'New card', + 'new_bank_account' => 'New bank account', + 'company_limit_reached' => 'Limit of :limit companies per account.', + 'credits_applied_validation' => 'Total credits applied cannot be MORE than total of invoices', + 'credit_number_taken' => 'Credit number already taken', + 'credit_not_found' => 'Credit not found', + 'invoices_dont_match_client' => 'Selected invoices are not from a single client', + 'duplicate_credits_submitted' => 'Duplicate credits submitted.', + 'duplicate_invoices_submitted' => 'Duplicate invoices submitted.', + 'credit_with_no_invoice' => 'You must have an invoice set when using a credit in a payment', + 'client_id_required' => 'Client id is required', + 'expense_number_taken' => 'Expense number already taken', + 'invoice_number_taken' => 'Invoice number already taken', + 'payment_id_required' => 'Payment `id` required.', + 'unable_to_retrieve_payment' => 'Unable to retrieve specified payment', + 'invoice_not_related_to_payment' => 'Invoice id :invoice is not related to this payment', + 'credit_not_related_to_payment' => 'Credit id :credit is not related to this payment', + 'max_refundable_invoice' => 'Attempting to refund more than allowed for invoice id :invoice, maximum refundable amount is :amount', + 'refund_without_invoices' => 'Attempting to refund a payment with invoices attached, please specify valid invoice/s to be refunded.', + 'refund_without_credits' => 'Attempting to refund a payment with credits attached, please specify valid credits/s to be refunded.', + 'max_refundable_credit' => 'Attempting to refund more than allowed for credit :credit, maximum refundable amount is :amount', + 'project_client_do_not_match' => 'Project client does not match entity client', + 'quote_number_taken' => 'Quote number already taken', + 'recurring_invoice_number_taken' => 'Recurring Invoice number :number already taken', + 'user_not_associated_with_account' => 'User not associated with this account', + 'amounts_do_not_balance' => 'Amounts do not balance correctly.', + 'insufficient_applied_amount_remaining' => 'Insufficient applied amount remaining to cover payment.', + 'insufficient_credit_balance' => 'Insufficient balance on credit.', + 'one_or_more_invoices_paid' => 'One or more of these invoices have been paid', + 'invoice_cannot_be_refunded' => 'Invoice id :number cannot be refunded', + 'attempted_refund_failed' => 'Attempting to refund :amount only :refundable_amount available for refund', + 'user_not_associated_with_this_account' => 'This user is unable to be attached to this company. Perhaps they have already registered a user on another account?', + 'migration_completed' => 'Migration completed', + 'migration_completed_description' => 'Your migration has completed, please review your data after logging in.', + 'api_404' => '404 | Nothing to see here!', + 'large_account_update_parameter' => 'Cannot load a large account without a updated_at parameter', + 'no_backup_exists' => 'No backup exists for this activity', + 'company_user_not_found' => 'Company User record not found', + 'no_credits_found' => 'No credits found.', + 'action_unavailable' => 'The requested action :action is not available.', + 'no_documents_found' => 'No Documents Found', + 'no_group_settings_found' => 'No group settings found', + 'access_denied' => 'Insufficient privileges to access/modify this resource', + 'invoice_cannot_be_marked_paid' => 'Invoice cannot be marked as paid', + 'invoice_license_or_environment' => 'Invalid license, or invalid environment :environment', + 'route_not_available' => 'Route not available', + 'invalid_design_object' => 'Invalid custom design object', + 'quote_not_found' => 'Quote/s not found', + 'quote_unapprovable' => 'Unable to approve this quote as it has expired.', + 'scheduler_has_run' => 'Scheduler has run', + 'scheduler_has_never_run' => 'Scheduler has never run', + 'self_update_not_available' => 'Self update not available on this system.', + 'user_detached' => 'User detached from company', + 'create_webhook_failure' => 'Failed to create Webhook', + 'payment_message_extended' => 'Thank you for your payment of :amount for :invoice', + 'online_payments_minimum_note' => 'Note: Online payments are supported only if amount is bigger than $1 or currency equivalent.', + 'payment_token_not_found' => 'Payment token not found, please try again. If an issue still persist, try with another payment method', + 'vendor_address1' => 'Vendor Street', + 'vendor_address2' => 'Vendor Apt/Suite', + 'partially_unapplied' => 'Partially Unapplied', + 'select_a_gmail_user' => 'Please select a user authenticated with Gmail', + 'list_long_press' => 'List Long Press', + 'show_actions' => 'Show Actions', + 'start_multiselect' => 'Start Multiselect', + 'email_sent_to_confirm_email' => 'An email has been sent to confirm the email address', + 'converted_paid_to_date' => 'Converted Paid to Date', + 'converted_credit_balance' => 'Converted Credit Balance', + 'converted_total' => 'Converted Total', + 'reply_to_name' => 'Reply-To Name', + 'payment_status_-2' => 'Partially Unapplied', + 'color_theme' => 'Color Theme', + 'start_migration' => 'Start Migration', + 'recurring_cancellation_request' => 'Request for recurring invoice cancellation from :contact', + 'recurring_cancellation_request_body' => ':contact from Client :client requested to cancel Recurring Invoice :invoice', + 'hello' => 'Hello', + 'group_documents' => 'Group documents', + 'quote_approval_confirmation_label' => 'Are you sure you want to approve this quote?', + 'migration_select_company_label' => 'Select companies to migrate', + 'force_migration' => 'Force migration', + 'require_password_with_social_login' => 'Require Password with Social Login', + 'stay_logged_in' => 'Stay Logged In', + 'session_about_to_expire' => 'Warning: Your session is about to expire', + 'count_hours' => ':count Hours', + 'count_day' => '1 Day', + 'count_days' => ':count Days', + 'web_session_timeout' => 'Web Session Timeout', + 'security_settings' => 'Security Settings', + 'resend_email' => 'Resend Email', + 'confirm_your_email_address' => 'Please confirm your email address', + 'freshbooks' => 'FreshBooks', + 'invoice2go' => 'Invoice2go', + 'invoicely' => 'Invoicely', + 'waveaccounting' => 'Wave Accounting', + 'zoho' => 'Zoho', + 'accounting' => 'Accounting', + 'required_files_missing' => 'Please provide all CSVs.', + 'migration_auth_label' => 'Let\'s continue by authenticating.', + 'api_secret' => 'API secret', + 'migration_api_secret_notice' => 'You can find API_SECRET in the .env file or Invoice Ninja v5. If property is missing, leave field blank.', + 'billing_coupon_notice' => 'Your discount will be applied on the checkout.', + 'use_last_email' => 'Use last email', + 'activate_company' => 'Activate Company', + 'activate_company_help' => 'Enable emails, recurring invoices and notifications', + 'an_error_occurred_try_again' => 'An error occurred, please try again', + 'please_first_set_a_password' => 'Please first set a password', + 'changing_phone_disables_two_factor' => 'Warning: Changing your phone number will disable 2FA', + 'help_translate' => 'Help Translate', + 'please_select_a_country' => 'Please select a country', + 'disabled_two_factor' => 'Successfully disabled 2FA', + 'connected_google' => 'Successfully connected account', + 'disconnected_google' => 'Successfully disconnected account', + 'delivered' => 'Delivered', + 'spam' => 'Spam', + 'view_docs' => 'View Docs', + 'enter_phone_to_enable_two_factor' => 'Please provide a mobile phone number to enable two factor authentication', + 'send_sms' => 'Send SMS', + 'sms_code' => 'SMS Code', + 'connect_google' => 'Connect Google', + 'disconnect_google' => 'Disconnect Google', + 'disable_two_factor' => 'Disable Two Factor', + 'invoice_task_datelog' => 'Invoice Task Datelog', + 'invoice_task_datelog_help' => 'Add date details to the invoice line items', + 'promo_code' => 'Promo code', + 'recurring_invoice_issued_to' => 'Recurring invoice issued to', + 'subscription' => 'Subscription', + 'new_subscription' => 'New Subscription', + 'deleted_subscription' => 'Successfully deleted subscription', + 'removed_subscription' => 'Successfully removed subscription', + 'restored_subscription' => 'Successfully restored subscription', + 'search_subscription' => 'Search 1 Subscription', + 'search_subscriptions' => 'Search :count Subscriptions', + 'subdomain_is_not_available' => 'Subdomain is not available', + 'connect_gmail' => 'Connect Gmail', + 'disconnect_gmail' => 'Disconnect Gmail', + 'connected_gmail' => 'Successfully connected Gmail', + 'disconnected_gmail' => 'Successfully disconnected Gmail', + 'update_fail_help' => 'Changes to the codebase may be blocking the update, you can run this command to discard the changes:', + 'client_id_number' => 'Client ID Number', + 'count_minutes' => ':count Minutes', + 'password_timeout' => 'Password Timeout', + 'shared_invoice_credit_counter' => 'Share Invoice/Credit Counter', 'activity_80' => ':user created subscription :subscription', 'activity_81' => ':user updated subscription :subscription', 'activity_82' => ':user archived subscription :subscription', From 168fef71c791426113672243cc5ab69d0db18847 Mon Sep 17 00:00:00 2001 From: paulwer Date: Fri, 15 Dec 2023 18:36:17 +0100 Subject: [PATCH 013/119] switch to endings for better oportunity to display in frontend --- .../Company/ValidExpenseMailbox.php | 26 +++++++++++++++++-- config/ninja.php | 6 ++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php index bb1b90212e22..b38c1071af11 100644 --- a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php +++ b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php @@ -24,11 +24,17 @@ class ValidExpenseMailbox implements Rule private $validated_schema = false; private $company_key = false; private $isEnterprise = false; + private array $endings; + private bool $hasCompanyKey; + private array $enterprise_endings; public function __construct(string $company_key, bool $isEnterprise = false) { $this->company_key = $company_key; $this->isEnterprise = $isEnterprise; + $this->endings = explode(",", config('ninja.inbound_expense.webhook.mailbox_endings')); + $this->hasCompanyKey = config('ninja.inbound_expense.webhook.mailbox_hascompanykey'); + $this->enterprise_endings = explode(",", config('ninja.inbound_expense.webhook.mailbox_enterprise_endings')); } public function passes($attribute, $value) @@ -44,8 +50,24 @@ class ValidExpenseMailbox implements Rule } // Validate Schema - $validated = !config('ninja.inbound_expense.webhook.mailbox_schema') || (preg_match(config('ninja.inbound_expense.webhook.mailbox_schema'), $value) && (!config('ninja.inbound_expense.webhook.mailbox_schema_hascompanykey') || str_contains($value, $this->company_key))) ? true : false; - $validated_enterprise = !config('ninja.inbound_expense.webhook.mailbox_schema_enterprise') || (Ninja::isHosted() && $this->isEnterprise && preg_match(config('ninja.inbound_expense.webhook.mailbox_schema_enterprise'), $value)); + $validated_hasCompanyKey = !$this->hasCompanyKey || str_contains($value, $this->company_key); + $validated = false; + if ($validated_hasCompanyKey) + foreach ($this->endings as $ending) { + if (str_ends_with($ending, $value)) { + $validated = true; + break; + } + } + + $validated_enterprise = false; + if (Ninja::isHosted() && $this->isEnterprise) + foreach ($this->endings as $ending) { + if (str_ends_with($ending, $value)) { + $validated_enterprise = true; + break; + } + } if (!$validated && !$validated_enterprise) return false; diff --git a/config/ninja.php b/config/ninja.php index df09c38580d9..0014a82c5d59 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -238,9 +238,9 @@ return [ ], 'webhook' => [ 'mailbox_template' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOXTEMPLATE', null), - 'mailbox_schema' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_SCHEMA', null), - 'mailbox_schema_hascompanykey' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_SCHEMA_HASCOMPANYKEY', false), - 'mailbox_schema_enterprise' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_SCHEMA_ENTERPRISE', '.*@expense\.invoicing\.co$'), + 'mailbox_endings_endings' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_ENDINGS', ''), + 'mailbox_hascompanykey' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_HASCOMPANYKEY', false), + 'mailbox_endings_enterprise' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_ENTERPRISE_ENDINGS', '@expense.invoicing.co'), ], ], ]; From 0e24ac0a142f0e98386e461729de545ce7dbab90 Mon Sep 17 00:00:00 2001 From: paulwer Date: Fri, 15 Dec 2023 18:36:49 +0100 Subject: [PATCH 014/119] fixes --- config/ninja.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/ninja.php b/config/ninja.php index 0014a82c5d59..3fe96170237f 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -237,7 +237,7 @@ return [ 'companies' => env('INBOUND_EXPENSE_IMAP_COMPANIES', ''), ], 'webhook' => [ - 'mailbox_template' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOXTEMPLATE', null), + 'mailbox_template' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_TEMPLATE', null), 'mailbox_endings_endings' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_ENDINGS', ''), 'mailbox_hascompanykey' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_HASCOMPANYKEY', false), 'mailbox_endings_enterprise' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_ENTERPRISE_ENDINGS', '@expense.invoicing.co'), From e05db368411ca5e6b723868d7e20e9ad8144fe1a Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 16 Dec 2023 09:17:25 +0100 Subject: [PATCH 015/119] wip: first initial setup for creating expenses from webhooks --- .../Mail/Webhook/BaseWebhookHandler.php | 27 ++++- .../Postmark/PostmarkWebhookHandler.php | 100 ++++++++++++++++-- 2 files changed, 120 insertions(+), 7 deletions(-) diff --git a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php b/app/Helpers/Mail/Webhook/BaseWebhookHandler.php index b579ad3bd9e6..e8ae7fe3d871 100644 --- a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php +++ b/app/Helpers/Mail/Webhook/BaseWebhookHandler.php @@ -11,16 +11,41 @@ namespace App\Helpers\Mail\Webhook; +use App\Factory\ExpenseFactory; use App\Models\Company; +use App\Utils\Traits\GeneratesCounter; +use App\Utils\Traits\SavesDocuments; interface BaseWebhookHandler { + use GeneratesCounter; + use SavesDocuments; public function process() { } + protected function createExpense(string $email, string $subject, string $plain_message, string $html_message, string $date, array $documents) + { + $company = $this->matchCompany($email); + if (!$company) + return false; - protected function matchCompany(string $email) + $expense = ExpenseFactory::create($company->id, $company->owner()->id); + + $expense->public_notes = $subject; + $expense->private_notes = $plain_message; + $expense->date = $date; + + // TODO: add html_message as document to the expense + + $this->saveDocuments($documents, $expense); + + $expense->saveQuietly(); + + return $expense; + } + + private function matchCompany(string $email) { return Company::where("expense_mailbox", $email)->first(); } diff --git a/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php b/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php index 51e458cb57d8..e64064191845 100644 --- a/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php +++ b/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php @@ -16,16 +16,104 @@ use App\Helpers\Mail\Webhook\BaseWebhookHandler; interface PostmarkWebhookHandler extends BaseWebhookHandler { + // { + // "FromName": "Postmarkapp Support", + // "MessageStream": "inbound", + // "From": "support@postmarkapp.com", + // "FromFull": { + // "Email": "support@postmarkapp.com", + // "Name": "Postmarkapp Support", + // "MailboxHash": "" + // }, + // "To": "\"Firstname Lastname\" ", + // "ToFull": [ + // { + // "Email": "yourhash+SampleHash@inbound.postmarkapp.com", + // "Name": "Firstname Lastname", + // "MailboxHash": "SampleHash" + // } + // ], + // "Cc": "\"First Cc\" , secondCc@postmarkapp.com>", + // "CcFull": [ + // { + // "Email": "firstcc@postmarkapp.com", + // "Name": "First Cc", + // "MailboxHash": "" + // }, + // { + // "Email": "secondCc@postmarkapp.com", + // "Name": "", + // "MailboxHash": "" + // } + // ], + // "Bcc": "\"First Bcc\" , secondbcc@postmarkapp.com>", + // "BccFull": [ + // { + // "Email": "firstbcc@postmarkapp.com", + // "Name": "First Bcc", + // "MailboxHash": "" + // }, + // { + // "Email": "secondbcc@postmarkapp.com", + // "Name": "", + // "MailboxHash": "" + // } + // ], + // "OriginalRecipient": "yourhash+SampleHash@inbound.postmarkapp.com", + // "Subject": "Test subject", + // "MessageID": "73e6d360-66eb-11e1-8e72-a8904824019b", + // "ReplyTo": "replyto@postmarkapp.com", + // "MailboxHash": "SampleHash", + // "Date": "Fri, 1 Aug 2014 16:45:32 -04:00", + // "TextBody": "This is a test text body.", + // "HtmlBody": "

This is a test html body.<\/p><\/body><\/html>", + // "StrippedTextReply": "This is the reply text", + // "Tag": "TestTag", + // "Headers": [ + // { + // "Name": "X-Header-Test", + // "Value": "" + // }, + // { + // "Name": "X-Spam-Status", + // "Value": "No" + // }, + // { + // "Name": "X-Spam-Score", + // "Value": "-0.1" + // }, + // { + // "Name": "X-Spam-Tests", + // "Value": "DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,SPF_PASS" + // } + // ], + // "Attachments": [ + // { + // "Name": "test.txt", + // "Content": "VGhpcyBpcyBhdHRhY2htZW50IGNvbnRlbnRzLCBiYXNlLTY0IGVuY29kZWQu", + // "ContentType": "text/plain", + // "ContentLength": 45 + // } + // ] + // } public function process($data) { - $email = ''; + $from = $data["From"]; + $subject = $data["Subject"]; + $plain_message = $data["TextBody"]; + $html_message = $data["HtmlBody"]; + $date = $data["Date"]; // TODO + $attachments = $data["Attachments"]; // TODO - $company = $this->matchCompany($email); - if (!$company) - return false; - - $expense = ExpenseFactory::create($company->id, $company->owner()->id); + return $this->createExpense( + $from, // from + $subject, // subject + $plain_message, // plain_message + $html_message, // html_message + $date, // date + $attachments, // attachments + ); } } From c393fdaa9b17c6fcc64f47a963a9da9f5ee67f36 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 16 Dec 2023 16:27:40 +0100 Subject: [PATCH 016/119] handling of temporary files + wip init of webhooks processing --- .../Mail/Webhook/BaseWebhookHandler.php | 6 +- .../Webhook/Mailgun/MailgunWebhookHandler.php | 26 +++- .../Postmark/PostmarkWebhookHandler.php | 23 ++-- .../Mailgun/ProcessMailgunInboundWebhook.php | 112 ++---------------- .../ProcessPostmarkInboundWebhook.php | 110 ++--------------- app/Libraries/MultiDB.php | 21 ++++ app/Utils/TempFile.php | 75 +++++++++++- 7 files changed, 154 insertions(+), 219 deletions(-) diff --git a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php b/app/Helpers/Mail/Webhook/BaseWebhookHandler.php index e8ae7fe3d871..8d2f2f4cc784 100644 --- a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php +++ b/app/Helpers/Mail/Webhook/BaseWebhookHandler.php @@ -13,10 +13,11 @@ namespace App\Helpers\Mail\Webhook; use App\Factory\ExpenseFactory; use App\Models\Company; +use App\Utils\TempFile; use App\Utils\Traits\GeneratesCounter; use App\Utils\Traits\SavesDocuments; -interface BaseWebhookHandler +abstract class BaseWebhookHandler { use GeneratesCounter; use SavesDocuments; @@ -36,7 +37,8 @@ interface BaseWebhookHandler $expense->private_notes = $plain_message; $expense->date = $date; - // TODO: add html_message as document to the expense + // add html_message as document to the expense + $documents[] = TempFile::UploadedFileFromRaw($html_message, "E-Mail.html", "text/html"); $this->saveDocuments($documents, $expense); diff --git a/app/Helpers/Mail/Webhook/Mailgun/MailgunWebhookHandler.php b/app/Helpers/Mail/Webhook/Mailgun/MailgunWebhookHandler.php index 6c2391ea3ab3..e3efdc9986a0 100644 --- a/app/Helpers/Mail/Webhook/Mailgun/MailgunWebhookHandler.php +++ b/app/Helpers/Mail/Webhook/Mailgun/MailgunWebhookHandler.php @@ -12,11 +12,33 @@ namespace App\Helpers\Mail\Webhook\Maigun; use App\Helpers\Mail\Webhook\BaseWebhookHandler; +use App\Utils\TempFile; -interface MailgunWebhookHandler extends BaseWebhookHandler +class MailgunWebhookHandler extends BaseWebhookHandler { - public function process() + public function process($data) { + $from = $data["sender"]; + $subject = $data["subject"]; + $plain_message = $data["body-plain"]; + $html_message = $data["body-html"]; + $date = now(); // TODO + + // parse documents as UploadedFile from webhook-data + $documents = []; + foreach ($data["Attachments"] as $attachment) { + $documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]); + } + + return $this->createExpense( + $from, + $subject, + $plain_message, + $html_message, + $date, + $documents, + ); + } } diff --git a/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php b/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php index e64064191845..1cceb5a69753 100644 --- a/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php +++ b/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php @@ -11,10 +11,10 @@ namespace App\Helpers\Mail\Webhook\Postmark; -use App\Factory\ExpenseFactory; use App\Helpers\Mail\Webhook\BaseWebhookHandler; +use App\Utils\TempFile; -interface PostmarkWebhookHandler extends BaseWebhookHandler +class PostmarkWebhookHandler extends BaseWebhookHandler { // { // "FromName": "Postmarkapp Support", @@ -104,15 +104,20 @@ interface PostmarkWebhookHandler extends BaseWebhookHandler $plain_message = $data["TextBody"]; $html_message = $data["HtmlBody"]; $date = $data["Date"]; // TODO - $attachments = $data["Attachments"]; // TODO + + // parse documents as UploadedFile from webhook-data + $documents = []; + foreach ($data["Attachments"] as $attachment) { + $documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]); + } return $this->createExpense( - $from, // from - $subject, // subject - $plain_message, // plain_message - $html_message, // html_message - $date, // date - $attachments, // attachments + $from, + $subject, + $plain_message, + $html_message, + $date, + $documents, ); } diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index 5640d72c1392..603814263f38 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -9,27 +9,16 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Jobs\Mailgun; +namespace App\Jobs\PostMark; -use App\DataMapper\Analytics\Mail\EmailBounce; -use App\DataMapper\Analytics\Mail\EmailSpam; -use App\Jobs\Util\SystemLogger; +use App\Helpers\Mail\Webhook\Maigun\MailgunWebhookHandler; use App\Libraries\MultiDB; -use App\Models\CreditInvitation; -use App\Models\Expense; -use App\Models\InvoiceInvitation; -use App\Models\PurchaseOrderInvitation; -use App\Models\QuoteInvitation; -use App\Models\RecurringInvoiceInvitation; use App\Models\SystemLog; -use App\Notifications\Ninja\EmailSpamNotification; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -use Postmark\PostmarkClient; -use Turbo124\Beacon\Facades\LightLogs; class ProcessMailgunInboundWebhook implements ShouldQueue { @@ -82,99 +71,20 @@ class ProcessMailgunInboundWebhook implements ShouldQueue */ public function handle() { - MultiDB::findAndSetDbByCompanyKey($this->request['Tag']); - // match companies if (array_key_exists('ToFull', $this->request)) throw new \Exception('invalid body'); - $toEmails = []; - foreach ($this->request['ToFull'] as $toEmailEntry) - $toEmails[] = $toEmailEntry['Email']; + foreach ($this->request['ToFull'] as $toEmailEntry) { + $toEmail = $toEmailEntry['Email']; - // create expense for each company - $expense = new Expense(); + $company = MultiDB::findAndSetDbByExpenseMailbox($toEmail); + if (!$company) { + nlog('unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $toEmail); + continue; + } - $expense->company_id; + (new MailgunWebhookHandler())->process($this->request); + } } - // { - // "FromName": "Postmarkapp Support", - // "MessageStream": "inbound", - // "From": "support@postmarkapp.com", - // "FromFull": { - // "Email": "support@postmarkapp.com", - // "Name": "Postmarkapp Support", - // "MailboxHash": "" - // }, - // "To": "\"Firstname Lastname\" ", - // "ToFull": [ - // { - // "Email": "yourhash+SampleHash@inbound.postmarkapp.com", - // "Name": "Firstname Lastname", - // "MailboxHash": "SampleHash" - // } - // ], - // "Cc": "\"First Cc\" , secondCc@postmarkapp.com>", - // "CcFull": [ - // { - // "Email": "firstcc@postmarkapp.com", - // "Name": "First Cc", - // "MailboxHash": "" - // }, - // { - // "Email": "secondCc@postmarkapp.com", - // "Name": "", - // "MailboxHash": "" - // } - // ], - // "Bcc": "\"First Bcc\" , secondbcc@postmarkapp.com>", - // "BccFull": [ - // { - // "Email": "firstbcc@postmarkapp.com", - // "Name": "First Bcc", - // "MailboxHash": "" - // }, - // { - // "Email": "secondbcc@postmarkapp.com", - // "Name": "", - // "MailboxHash": "" - // } - // ], - // "OriginalRecipient": "yourhash+SampleHash@inbound.postmarkapp.com", - // "Subject": "Test subject", - // "MessageID": "73e6d360-66eb-11e1-8e72-a8904824019b", - // "ReplyTo": "replyto@postmarkapp.com", - // "MailboxHash": "SampleHash", - // "Date": "Fri, 1 Aug 2014 16:45:32 -04:00", - // "TextBody": "This is a test text body.", - // "HtmlBody": "

This is a test html body.<\/p><\/body><\/html>", - // "StrippedTextReply": "This is the reply text", - // "Tag": "TestTag", - // "Headers": [ - // { - // "Name": "X-Header-Test", - // "Value": "" - // }, - // { - // "Name": "X-Spam-Status", - // "Value": "No" - // }, - // { - // "Name": "X-Spam-Score", - // "Value": "-0.1" - // }, - // { - // "Name": "X-Spam-Tests", - // "Value": "DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,SPF_PASS" - // } - // ], - // "Attachments": [ - // { - // "Name": "test.txt", - // "Content": "VGhpcyBpcyBhdHRhY2htZW50IGNvbnRlbnRzLCBiYXNlLTY0IGVuY29kZWQu", - // "ContentType": "text/plain", - // "ContentLength": 45 - // } - // ] - // } } diff --git a/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php b/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php index 2d2355b234bc..6e54fcdf05a3 100644 --- a/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php +++ b/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php @@ -11,25 +11,14 @@ namespace App\Jobs\PostMark; -use App\DataMapper\Analytics\Mail\EmailBounce; -use App\DataMapper\Analytics\Mail\EmailSpam; -use App\Jobs\Util\SystemLogger; +use App\Helpers\Mail\Webhook\Postmark\PostmarkWebhookHandler; use App\Libraries\MultiDB; -use App\Models\CreditInvitation; -use App\Models\Expense; -use App\Models\InvoiceInvitation; -use App\Models\PurchaseOrderInvitation; -use App\Models\QuoteInvitation; -use App\Models\RecurringInvoiceInvitation; use App\Models\SystemLog; -use App\Notifications\Ninja\EmailSpamNotification; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -use Postmark\PostmarkClient; -use Turbo124\Beacon\Facades\LightLogs; class ProcessPostmarkInboundWebhook implements ShouldQueue { @@ -82,99 +71,20 @@ class ProcessPostmarkInboundWebhook implements ShouldQueue */ public function handle() { - MultiDB::findAndSetDbByCompanyKey($this->request['Tag']); - // match companies if (array_key_exists('ToFull', $this->request)) throw new \Exception('invalid body'); - $toEmails = []; - foreach ($this->request['ToFull'] as $toEmailEntry) - $toEmails[] = $toEmailEntry['Email']; + foreach ($this->request['ToFull'] as $toEmailEntry) { + $toEmail = $toEmailEntry['Email']; - // create expense for each company - $expense = new Expense(); + $company = MultiDB::findAndSetDbByExpenseMailbox($toEmail); + if (!$company) { + nlog('unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $toEmail); + continue; + } - $expense->company_id; + (new PostmarkWebhookHandler())->process($this->request); + } } - // { - // "FromName": "Postmarkapp Support", - // "MessageStream": "inbound", - // "From": "support@postmarkapp.com", - // "FromFull": { - // "Email": "support@postmarkapp.com", - // "Name": "Postmarkapp Support", - // "MailboxHash": "" - // }, - // "To": "\"Firstname Lastname\" ", - // "ToFull": [ - // { - // "Email": "yourhash+SampleHash@inbound.postmarkapp.com", - // "Name": "Firstname Lastname", - // "MailboxHash": "SampleHash" - // } - // ], - // "Cc": "\"First Cc\" , secondCc@postmarkapp.com>", - // "CcFull": [ - // { - // "Email": "firstcc@postmarkapp.com", - // "Name": "First Cc", - // "MailboxHash": "" - // }, - // { - // "Email": "secondCc@postmarkapp.com", - // "Name": "", - // "MailboxHash": "" - // } - // ], - // "Bcc": "\"First Bcc\" , secondbcc@postmarkapp.com>", - // "BccFull": [ - // { - // "Email": "firstbcc@postmarkapp.com", - // "Name": "First Bcc", - // "MailboxHash": "" - // }, - // { - // "Email": "secondbcc@postmarkapp.com", - // "Name": "", - // "MailboxHash": "" - // } - // ], - // "OriginalRecipient": "yourhash+SampleHash@inbound.postmarkapp.com", - // "Subject": "Test subject", - // "MessageID": "73e6d360-66eb-11e1-8e72-a8904824019b", - // "ReplyTo": "replyto@postmarkapp.com", - // "MailboxHash": "SampleHash", - // "Date": "Fri, 1 Aug 2014 16:45:32 -04:00", - // "TextBody": "This is a test text body.", - // "HtmlBody": "

This is a test html body.<\/p><\/body><\/html>", - // "StrippedTextReply": "This is the reply text", - // "Tag": "TestTag", - // "Headers": [ - // { - // "Name": "X-Header-Test", - // "Value": "" - // }, - // { - // "Name": "X-Spam-Status", - // "Value": "No" - // }, - // { - // "Name": "X-Spam-Score", - // "Value": "-0.1" - // }, - // { - // "Name": "X-Spam-Tests", - // "Value": "DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,SPF_PASS" - // } - // ], - // "Attachments": [ - // { - // "Name": "test.txt", - // "Content": "VGhpcyBpcyBhdHRhY2htZW50IGNvbnRlbnRzLCBiYXNlLTY0IGVuY29kZWQu", - // "ContentType": "text/plain", - // "ContentLength": 45 - // } - // ] - // } } diff --git a/app/Libraries/MultiDB.php b/app/Libraries/MultiDB.php index f58841800c7b..cef54abeb347 100644 --- a/app/Libraries/MultiDB.php +++ b/app/Libraries/MultiDB.php @@ -513,6 +513,27 @@ class MultiDB return false; } + public static function findAndSetDbByExpenseMailbox($expense_mailbox) + { + if (!config('ninja.db.multi_db_enabled')) { + return Company::where("expense_mailbox", $expense_mailbox)->first(); + } + + $current_db = config('database.default'); + + foreach (self::$dbs as $db) { + if ($company = Company::on($db)->where("expense_mailbox", $expense_mailbox)->first()) { + self::setDb($db); + + return $company; + } + } + + self::setDB($current_db); + + return false; + } + public static function findAndSetDbByInvitation($entity, $invitation_key) { $class = 'App\Models\\' . ucfirst(Str::camel($entity)) . 'Invitation'; diff --git a/app/Utils/TempFile.php b/app/Utils/TempFile.php index 9d7fe59ff3d9..0e8062b457ad 100644 --- a/app/Utils/TempFile.php +++ b/app/Utils/TempFile.php @@ -11,27 +11,92 @@ namespace App\Utils; +use Illuminate\Http\File; +use Illuminate\Http\UploadedFile; +use Illuminate\Support\Arr; + class TempFile { - public static function path($url) :string + public static function path($url): string { - $temp_path = @tempnam(sys_get_temp_dir().'/'.sha1(time()), basename($url)); + $temp_path = @tempnam(sys_get_temp_dir() . '/' . sha1(time()), basename($url)); copy($url, $temp_path); return $temp_path; } /* Downloads a file to temp storage and returns the path - used for mailers */ - public static function filePath($data, $filename) :string + public static function filePath($data, $filename): string { - $dir_hash = sys_get_temp_dir().'/'.sha1(microtime()); + $dir_hash = sys_get_temp_dir() . '/' . sha1(microtime()); mkdir($dir_hash); - $file_path = $dir_hash.'/'.$filename; + $file_path = $dir_hash . '/' . $filename; file_put_contents($file_path, $data); return $file_path; } + + /* create a tmp file from a base64 string: https://gist.github.com/waska14/8b3bcebfad1f86f7fcd3b82927576e38*/ + public static function UploadedFileFromBase64(string $base64File): UploadedFile + { + // Get file data base64 string + $fileData = base64_decode(Arr::last(explode(',', $base64File))); + + // Create temp file and get its absolute path + $tempFile = tmpfile(); + $tempFilePath = stream_get_meta_data($tempFile)['uri']; + + // Save file data in file + file_put_contents($tempFilePath, $fileData); + + $tempFileObject = new File($tempFilePath); + $file = new UploadedFile( + $tempFileObject->getPathname(), + $tempFileObject->getFilename(), + $tempFileObject->getMimeType(), + 0, + true // Mark it as test, since the file isn't from real HTTP POST. + ); + + // Close this file after response is sent. + // Closing the file will cause to remove it from temp director! + app()->terminating(function () use ($tempFile) { + fclose($tempFile); + }); + + // return UploadedFile object + return $file; + } + + /* create a tmp file from a raw string: https://gist.github.com/waska14/8b3bcebfad1f86f7fcd3b82927576e38*/ + public static function UploadedFileFromRaw(string $fileData, string|null $fileName = null, string|null $mimeType = null): UploadedFile + { + // Create temp file and get its absolute path + $tempFile = tmpfile(); + $tempFilePath = stream_get_meta_data($tempFile)['uri']; + + // Save file data in file + file_put_contents($tempFilePath, $fileData); + + $tempFileObject = new File($tempFilePath); + $file = new UploadedFile( + $tempFileObject->getPathname(), + $fileName ?: $tempFileObject->getFilename(), + $mimeType ?: $tempFileObject->getMimeType(), + 0, + true // Mark it as test, since the file isn't from real HTTP POST. + ); + + // Close this file after response is sent. + // Closing the file will cause to remove it from temp director! + app()->terminating(function () use ($tempFile) { + fclose($tempFile); + }); + + // return UploadedFile object + return $file; + } } From ad480a5b32836245e7adbd3b43ca5a6a87c7a889 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 16 Dec 2023 16:36:17 +0100 Subject: [PATCH 017/119] fixes --- app/Console/Kernel.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 20612d16ef6c..90a9185ed9bf 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -17,7 +17,6 @@ use App\Jobs\Cron\RecurringInvoicesCron; use App\Jobs\Cron\SubscriptionCron; use App\Jobs\Cron\UpdateCalculatedFields; use App\Jobs\Invoice\InvoiceCheckLateWebhook; -use App\Jobs\Mail\ExpenseImportJob; use App\Jobs\Mail\ExpenseMailboxJob; use App\Jobs\Ninja\AdjustEmailQuota; use App\Jobs\Ninja\BankTransactionSync; @@ -131,7 +130,6 @@ class Kernel extends ConsoleKernel $schedule->command('queue:restart')->everyFiveMinutes()->withoutOverlapping(); } - $schedule->job(new ExpenseImportJob)->everyThirtyMinutes()->withoutOverlapping()->name('expense-import-job')->onOneServer(); } /** From 9ef5a2501b4f55045cbc672a68f55e030e13ca03 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 16 Dec 2023 17:21:52 +0100 Subject: [PATCH 018/119] fix composer --- composer.lock | 426 +++++++++++++++++++++++++------------------------- 1 file changed, 211 insertions(+), 215 deletions(-) diff --git a/composer.lock b/composer.lock index 4c0450f960b0..1d88803d2b4f 100644 --- a/composer.lock +++ b/composer.lock @@ -485,16 +485,16 @@ }, { "name": "aws/aws-sdk-php", - "version": "3.293.5", + "version": "3.294.1", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "f2002e52b382b45231da3f9552033f769acfebd8" + "reference": "63c720229a9c9cdedff6bac98d6e72be8cc241f1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/f2002e52b382b45231da3f9552033f769acfebd8", - "reference": "f2002e52b382b45231da3f9552033f769acfebd8", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/63c720229a9c9cdedff6bac98d6e72be8cc241f1", + "reference": "63c720229a9c9cdedff6bac98d6e72be8cc241f1", "shasum": "" }, "require": { @@ -574,9 +574,9 @@ "support": { "forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80", "issues": "https://github.com/aws/aws-sdk-php/issues", - "source": "https://github.com/aws/aws-sdk-php/tree/3.293.5" + "source": "https://github.com/aws/aws-sdk-php/tree/3.294.1" }, - "time": "2023-12-06T19:09:15+00:00" + "time": "2023-12-15T19:25:52+00:00" }, { "name": "bacon/bacon-qr-code", @@ -790,16 +790,16 @@ }, { "name": "carbonphp/carbon-doctrine-types", - "version": "2.0.0", + "version": "2.1.0", "source": { "type": "git", "url": "https://github.com/CarbonPHP/carbon-doctrine-types.git", - "reference": "67a77972b9f398ae7068dabacc39c08aeee170d5" + "reference": "99f76ffa36cce3b70a4a6abce41dba15ca2e84cb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/CarbonPHP/carbon-doctrine-types/zipball/67a77972b9f398ae7068dabacc39c08aeee170d5", - "reference": "67a77972b9f398ae7068dabacc39c08aeee170d5", + "url": "https://api.github.com/repos/CarbonPHP/carbon-doctrine-types/zipball/99f76ffa36cce3b70a4a6abce41dba15ca2e84cb", + "reference": "99f76ffa36cce3b70a4a6abce41dba15ca2e84cb", "shasum": "" }, "require": { @@ -839,7 +839,7 @@ ], "support": { "issues": "https://github.com/CarbonPHP/carbon-doctrine-types/issues", - "source": "https://github.com/CarbonPHP/carbon-doctrine-types/tree/2.0.0" + "source": "https://github.com/CarbonPHP/carbon-doctrine-types/tree/2.1.0" }, "funding": [ { @@ -855,7 +855,7 @@ "type": "tidelift" } ], - "time": "2023-10-01T14:29:01+00:00" + "time": "2023-12-11T17:09:12+00:00" }, { "name": "checkout/checkout-sdk-php", @@ -2693,16 +2693,16 @@ }, { "name": "google/apiclient-services", - "version": "v0.326.1", + "version": "v0.327.0", "source": { "type": "git", "url": "https://github.com/googleapis/google-api-php-client-services.git", - "reference": "4e89c28c499f87eb517679e13356469896a119c6" + "reference": "51a11d4ff70dd9f60334525e71bf4cf592e6d282" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/4e89c28c499f87eb517679e13356469896a119c6", - "reference": "4e89c28c499f87eb517679e13356469896a119c6", + "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/51a11d4ff70dd9f60334525e71bf4cf592e6d282", + "reference": "51a11d4ff70dd9f60334525e71bf4cf592e6d282", "shasum": "" }, "require": { @@ -2731,9 +2731,9 @@ ], "support": { "issues": "https://github.com/googleapis/google-api-php-client-services/issues", - "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.326.1" + "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.327.0" }, - "time": "2023-12-04T01:18:18+00:00" + "time": "2023-12-11T00:52:16+00:00" }, { "name": "google/auth", @@ -3793,16 +3793,16 @@ }, { "name": "imdhemy/google-play-billing", - "version": "1.5.0", + "version": "1.5.1", "source": { "type": "git", "url": "https://github.com/imdhemy/google-play-billing.git", - "reference": "a227174a71bc5d7b3e5f9aa4fcad2c4a9a11a8a4" + "reference": "bb94f3b6ddb021605815e528f31b8c930c41677c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/imdhemy/google-play-billing/zipball/a227174a71bc5d7b3e5f9aa4fcad2c4a9a11a8a4", - "reference": "a227174a71bc5d7b3e5f9aa4fcad2c4a9a11a8a4", + "url": "https://api.github.com/repos/imdhemy/google-play-billing/zipball/bb94f3b6ddb021605815e528f31b8c930c41677c", + "reference": "bb94f3b6ddb021605815e528f31b8c930c41677c", "shasum": "" }, "require": { @@ -3838,22 +3838,22 @@ "description": "Google Play Billing", "support": { "issues": "https://github.com/imdhemy/google-play-billing/issues", - "source": "https://github.com/imdhemy/google-play-billing/tree/1.5.0" + "source": "https://github.com/imdhemy/google-play-billing/tree/1.5.1" }, - "time": "2023-09-17T12:33:33+00:00" + "time": "2023-12-15T10:25:05+00:00" }, { "name": "imdhemy/laravel-purchases", - "version": "1.9.0", + "version": "1.9.1", "source": { "type": "git", "url": "https://github.com/imdhemy/laravel-in-app-purchases.git", - "reference": "4471f5dc211931b847ac0bf88f78bd4fa9e3760d" + "reference": "b74e09b78fb3e0f1b1630dbcfd23d9f6fe251b90" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/imdhemy/laravel-in-app-purchases/zipball/4471f5dc211931b847ac0bf88f78bd4fa9e3760d", - "reference": "4471f5dc211931b847ac0bf88f78bd4fa9e3760d", + "url": "https://api.github.com/repos/imdhemy/laravel-in-app-purchases/zipball/b74e09b78fb3e0f1b1630dbcfd23d9f6fe251b90", + "reference": "b74e09b78fb3e0f1b1630dbcfd23d9f6fe251b90", "shasum": "" }, "require": { @@ -3909,7 +3909,7 @@ ], "support": { "issues": "https://github.com/imdhemy/laravel-in-app-purchases/issues", - "source": "https://github.com/imdhemy/laravel-in-app-purchases/tree/1.9.0" + "source": "https://github.com/imdhemy/laravel-in-app-purchases/tree/1.9.1" }, "funding": [ { @@ -3917,7 +3917,7 @@ "type": "github" } ], - "time": "2023-09-19T06:01:35+00:00" + "time": "2023-12-15T10:35:56+00:00" }, { "name": "intervention/image", @@ -4190,47 +4190,47 @@ }, { "name": "jms/serializer", - "version": "3.28.0", + "version": "3.29.1", "source": { "type": "git", "url": "https://github.com/schmittjoh/serializer.git", - "reference": "5a5a03a71a28a480189c5a0ca95893c19f1d120c" + "reference": "111451f43abb448ce297361a8ab96a9591e848cd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/schmittjoh/serializer/zipball/5a5a03a71a28a480189c5a0ca95893c19f1d120c", - "reference": "5a5a03a71a28a480189c5a0ca95893c19f1d120c", + "url": "https://api.github.com/repos/schmittjoh/serializer/zipball/111451f43abb448ce297361a8ab96a9591e848cd", + "reference": "111451f43abb448ce297361a8ab96a9591e848cd", "shasum": "" }, "require": { - "doctrine/annotations": "^1.13 || ^2.0", - "doctrine/instantiator": "^1.0.3 || ^2.0", + "doctrine/annotations": "^1.14 || ^2.0", + "doctrine/instantiator": "^1.3.1 || ^2.0", "doctrine/lexer": "^2.0 || ^3.0", "jms/metadata": "^2.6", - "php": "^7.2||^8.0", - "phpstan/phpdoc-parser": "^0.4 || ^0.5 || ^1.0" + "php": "^7.2 || ^8.0", + "phpstan/phpdoc-parser": "^1.20" }, "require-dev": { "doctrine/coding-standard": "^12.0", - "doctrine/orm": "~2.1", - "doctrine/persistence": "^1.3.3|^2.0|^3.0", - "doctrine/phpcr-odm": "^1.3|^2.0", + "doctrine/orm": "^2.14 || ^3.0", + "doctrine/persistence": "^2.5.2 || ^3.0", + "doctrine/phpcr-odm": "^1.5.2 || ^2.0", "ext-pdo_sqlite": "*", - "jackalope/jackalope-doctrine-dbal": "^1.1.5", - "ocramius/proxy-manager": "^1.0|^2.0", + "jackalope/jackalope-doctrine-dbal": "^1.3", + "ocramius/proxy-manager": "^1.0 || ^2.0", "phpbench/phpbench": "^1.0", "phpstan/phpstan": "^1.0.2", - "phpunit/phpunit": "^8.5.21||^9.0||^10.0", - "psr/container": "^1.0|^2.0", - "symfony/dependency-injection": "^3.0|^4.0|^5.0|^6.0", - "symfony/expression-language": "^3.2|^4.0|^5.0|^6.0", - "symfony/filesystem": "^3.0|^4.0|^5.0|^6.0", - "symfony/form": "^3.0|^4.0|^5.0|^6.0", - "symfony/translation": "^3.0|^4.0|^5.0|^6.0", - "symfony/uid": "^5.1|^6.0", - "symfony/validator": "^3.1.9|^4.0|^5.0|^6.0", - "symfony/yaml": "^3.3|^4.0|^5.0|^6.0", - "twig/twig": "~1.34|~2.4|^3.0" + "phpunit/phpunit": "^8.5.21 || ^9.0 || ^10.0", + "psr/container": "^1.0 || ^2.0", + "symfony/dependency-injection": "^3.4 || ^4.0 || ^5.0 || ^6.0 || ^7.0", + "symfony/expression-language": "^3.2 || ^4.0 || ^5.0 || ^6.0 || ^7.0", + "symfony/filesystem": "^4.2 || ^5.0 || ^6.0 || ^7.0", + "symfony/form": "^3.4 || ^4.0 || ^5.0 || ^6.0 || ^7.0", + "symfony/translation": "^3.0 || ^4.0 || ^5.0 || ^6.0 || ^7.0", + "symfony/uid": "^5.1 || ^6.0 || ^7.0", + "symfony/validator": "^3.1.9 || ^4.0 || ^5.0 || ^6.0 || ^7.0", + "symfony/yaml": "^3.4 || ^4.0 || ^5.0 || ^6.0 || ^7.0", + "twig/twig": "^1.34 || ^2.4 || ^3.0" }, "suggest": { "doctrine/collections": "Required if you like to use doctrine collection types as ArrayCollection.", @@ -4274,7 +4274,7 @@ ], "support": { "issues": "https://github.com/schmittjoh/serializer/issues", - "source": "https://github.com/schmittjoh/serializer/tree/3.28.0" + "source": "https://github.com/schmittjoh/serializer/tree/3.29.1" }, "funding": [ { @@ -4282,7 +4282,7 @@ "type": "github" } ], - "time": "2023-08-03T14:43:08+00:00" + "time": "2023-12-14T15:25:09+00:00" }, { "name": "josemmo/facturae-php", @@ -4442,16 +4442,16 @@ }, { "name": "laravel/framework", - "version": "v10.35.0", + "version": "v10.37.3", "source": { "type": "git", "url": "https://github.com/laravel/framework.git", - "reference": "91ec2d92d2f6007e9084fe06438b99c91845da69" + "reference": "996375dd61f8c6e4ac262b57ed485655d71fcbdc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/framework/zipball/91ec2d92d2f6007e9084fe06438b99c91845da69", - "reference": "91ec2d92d2f6007e9084fe06438b99c91845da69", + "url": "https://api.github.com/repos/laravel/framework/zipball/996375dd61f8c6e4ac262b57ed485655d71fcbdc", + "reference": "996375dd61f8c6e4ac262b57ed485655d71fcbdc", "shasum": "" }, "require": { @@ -4640,7 +4640,7 @@ "issues": "https://github.com/laravel/framework/issues", "source": "https://github.com/laravel/framework" }, - "time": "2023-12-05T14:50:33+00:00" + "time": "2023-12-13T20:10:58+00:00" }, { "name": "laravel/prompts", @@ -5023,34 +5023,34 @@ }, { "name": "lcobucci/clock", - "version": "3.0.0", + "version": "3.2.0", "source": { "type": "git", "url": "https://github.com/lcobucci/clock.git", - "reference": "039ef98c6b57b101d10bd11d8fdfda12cbd996dc" + "reference": "6f28b826ea01306b07980cb8320ab30b966cd715" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/lcobucci/clock/zipball/039ef98c6b57b101d10bd11d8fdfda12cbd996dc", - "reference": "039ef98c6b57b101d10bd11d8fdfda12cbd996dc", + "url": "https://api.github.com/repos/lcobucci/clock/zipball/6f28b826ea01306b07980cb8320ab30b966cd715", + "reference": "6f28b826ea01306b07980cb8320ab30b966cd715", "shasum": "" }, "require": { - "php": "~8.1.0 || ~8.2.0", + "php": "~8.2.0 || ~8.3.0", "psr/clock": "^1.0" }, "provide": { "psr/clock-implementation": "1.0" }, "require-dev": { - "infection/infection": "^0.26", - "lcobucci/coding-standard": "^9.0", - "phpstan/extension-installer": "^1.2", - "phpstan/phpstan": "^1.9.4", - "phpstan/phpstan-deprecation-rules": "^1.1.1", - "phpstan/phpstan-phpunit": "^1.3.2", - "phpstan/phpstan-strict-rules": "^1.4.4", - "phpunit/phpunit": "^9.5.27" + "infection/infection": "^0.27", + "lcobucci/coding-standard": "^11.0.0", + "phpstan/extension-installer": "^1.3.1", + "phpstan/phpstan": "^1.10.25", + "phpstan/phpstan-deprecation-rules": "^1.1.3", + "phpstan/phpstan-phpunit": "^1.3.13", + "phpstan/phpstan-strict-rules": "^1.5.1", + "phpunit/phpunit": "^10.2.3" }, "type": "library", "autoload": { @@ -5071,7 +5071,7 @@ "description": "Yet another clock abstraction", "support": { "issues": "https://github.com/lcobucci/clock/issues", - "source": "https://github.com/lcobucci/clock/tree/3.0.0" + "source": "https://github.com/lcobucci/clock/tree/3.2.0" }, "funding": [ { @@ -5083,7 +5083,7 @@ "type": "patreon" } ], - "time": "2022-12-19T15:00:24+00:00" + "time": "2023-11-17T17:00:27+00:00" }, { "name": "lcobucci/jwt", @@ -5349,16 +5349,16 @@ }, { "name": "league/csv", - "version": "9.12.0", + "version": "9.13.0", "source": { "type": "git", "url": "https://github.com/thephpleague/csv.git", - "reference": "c1dc31e23eb3cd0f7b537ee1a669d5f58d5cfe21" + "reference": "3690cc71bfe8dc3b6daeef356939fac95348f0a8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/csv/zipball/c1dc31e23eb3cd0f7b537ee1a669d5f58d5cfe21", - "reference": "c1dc31e23eb3cd0f7b537ee1a669d5f58d5cfe21", + "url": "https://api.github.com/repos/thephpleague/csv/zipball/3690cc71bfe8dc3b6daeef356939fac95348f0a8", + "reference": "3690cc71bfe8dc3b6daeef356939fac95348f0a8", "shasum": "" }, "require": { @@ -5373,11 +5373,11 @@ "ext-xdebug": "*", "friendsofphp/php-cs-fixer": "^v3.22.0", "phpbench/phpbench": "^1.2.15", - "phpstan/phpstan": "^1.10.46", + "phpstan/phpstan": "^1.10.50", "phpstan/phpstan-deprecation-rules": "^1.1.4", "phpstan/phpstan-phpunit": "^1.3.15", "phpstan/phpstan-strict-rules": "^1.5.2", - "phpunit/phpunit": "^10.4.2", + "phpunit/phpunit": "^10.5.3", "symfony/var-dumper": "^6.4.0" }, "suggest": { @@ -5434,7 +5434,7 @@ "type": "github" } ], - "time": "2023-12-01T17:54:07+00:00" + "time": "2023-12-16T11:03:20+00:00" }, { "name": "league/flysystem", @@ -6713,16 +6713,16 @@ }, { "name": "nesbot/carbon", - "version": "2.72.0", + "version": "2.72.1", "source": { "type": "git", "url": "https://github.com/briannesbitt/Carbon.git", - "reference": "a6885fcbad2ec4360b0e200ee0da7d9b7c90786b" + "reference": "2b3b3db0a2d0556a177392ff1a3bf5608fa09f78" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/briannesbitt/Carbon/zipball/a6885fcbad2ec4360b0e200ee0da7d9b7c90786b", - "reference": "a6885fcbad2ec4360b0e200ee0da7d9b7c90786b", + "url": "https://api.github.com/repos/briannesbitt/Carbon/zipball/2b3b3db0a2d0556a177392ff1a3bf5608fa09f78", + "reference": "2b3b3db0a2d0556a177392ff1a3bf5608fa09f78", "shasum": "" }, "require": { @@ -6816,7 +6816,7 @@ "type": "tidelift" } ], - "time": "2023-11-28T10:13:25+00:00" + "time": "2023-12-08T23:47:49+00:00" }, { "name": "nette/schema", @@ -6968,16 +6968,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.17.1", + "version": "v4.18.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d" + "reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d", - "reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/1bcbb2179f97633e98bbbc87044ee2611c7d7999", + "reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999", "shasum": "" }, "require": { @@ -7018,9 +7018,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.17.1" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.18.0" }, - "time": "2023-08-13T19:53:39+00:00" + "time": "2023-12-10T21:03:43+00:00" }, { "name": "nunomaduro/termwind", @@ -8640,16 +8640,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.24.4", + "version": "1.24.5", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "6bd0c26f3786cd9b7c359675cb789e35a8e07496" + "reference": "fedf211ff14ec8381c9bf5714e33a7a552dd1acc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/6bd0c26f3786cd9b7c359675cb789e35a8e07496", - "reference": "6bd0c26f3786cd9b7c359675cb789e35a8e07496", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/fedf211ff14ec8381c9bf5714e33a7a552dd1acc", + "reference": "fedf211ff14ec8381c9bf5714e33a7a552dd1acc", "shasum": "" }, "require": { @@ -8681,9 +8681,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.24.4" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.24.5" }, - "time": "2023-11-26T18:29:22+00:00" + "time": "2023-12-16T09:33:33+00:00" }, { "name": "pragmarx/google2fa", @@ -9341,16 +9341,16 @@ }, { "name": "pusher/pusher-php-server", - "version": "7.2.3", + "version": "7.2.4", "source": { "type": "git", "url": "https://github.com/pusher/pusher-http-php.git", - "reference": "416e68dd5f640175ad5982131c42a7a666d1d8e9" + "reference": "de2f72296808f9cafa6a4462b15a768ff130cddb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/pusher/pusher-http-php/zipball/416e68dd5f640175ad5982131c42a7a666d1d8e9", - "reference": "416e68dd5f640175ad5982131c42a7a666d1d8e9", + "url": "https://api.github.com/repos/pusher/pusher-http-php/zipball/de2f72296808f9cafa6a4462b15a768ff130cddb", + "reference": "de2f72296808f9cafa6a4462b15a768ff130cddb", "shasum": "" }, "require": { @@ -9396,9 +9396,9 @@ ], "support": { "issues": "https://github.com/pusher/pusher-http-php/issues", - "source": "https://github.com/pusher/pusher-http-php/tree/7.2.3" + "source": "https://github.com/pusher/pusher-http-php/tree/7.2.4" }, - "time": "2023-05-17T16:00:06+00:00" + "time": "2023-12-15T10:58:53+00:00" }, { "name": "ralouphie/getallheaders", @@ -10202,16 +10202,16 @@ }, { "name": "setasign/fpdi", - "version": "v2.5.0", + "version": "v2.6.0", "source": { "type": "git", "url": "https://github.com/Setasign/FPDI.git", - "reference": "ecf0459643ec963febfb9a5d529dcd93656006a4" + "reference": "a6db878129ec6c7e141316ee71872923e7f1b7ad" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Setasign/FPDI/zipball/ecf0459643ec963febfb9a5d529dcd93656006a4", - "reference": "ecf0459643ec963febfb9a5d529dcd93656006a4", + "url": "https://api.github.com/repos/Setasign/FPDI/zipball/a6db878129ec6c7e141316ee71872923e7f1b7ad", + "reference": "a6db878129ec6c7e141316ee71872923e7f1b7ad", "shasum": "" }, "require": { @@ -10223,8 +10223,8 @@ }, "require-dev": { "phpunit/phpunit": "~5.7", - "setasign/fpdf": "~1.8", - "setasign/tfpdf": "~1.31", + "setasign/fpdf": "~1.8.6", + "setasign/tfpdf": "~1.33", "squizlabs/php_codesniffer": "^3.5", "tecnickcom/tcpdf": "~6.2" }, @@ -10262,7 +10262,7 @@ ], "support": { "issues": "https://github.com/Setasign/FPDI/issues", - "source": "https://github.com/Setasign/FPDI/tree/v2.5.0" + "source": "https://github.com/Setasign/FPDI/tree/v2.6.0" }, "funding": [ { @@ -10270,7 +10270,7 @@ "type": "tidelift" } ], - "time": "2023-09-28T10:46:27+00:00" + "time": "2023-12-11T16:03:32+00:00" }, { "name": "shopify/shopify-api", @@ -11005,20 +11005,20 @@ }, { "name": "symfony/css-selector", - "version": "v6.4.0", + "version": "v7.0.0", "source": { "type": "git", "url": "https://github.com/symfony/css-selector.git", - "reference": "d036c6c0d0b09e24a14a35f8292146a658f986e4" + "reference": "bb51d46e53ef8d50d523f0c5faedba056a27943e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/css-selector/zipball/d036c6c0d0b09e24a14a35f8292146a658f986e4", - "reference": "d036c6c0d0b09e24a14a35f8292146a658f986e4", + "url": "https://api.github.com/repos/symfony/css-selector/zipball/bb51d46e53ef8d50d523f0c5faedba056a27943e", + "reference": "bb51d46e53ef8d50d523f0c5faedba056a27943e", "shasum": "" }, "require": { - "php": ">=8.1" + "php": ">=8.2" }, "type": "library", "autoload": { @@ -11050,7 +11050,7 @@ "description": "Converts CSS selectors to XPath expressions", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/css-selector/tree/v6.4.0" + "source": "https://github.com/symfony/css-selector/tree/v7.0.0" }, "funding": [ { @@ -11066,7 +11066,7 @@ "type": "tidelift" } ], - "time": "2023-10-31T08:40:20+00:00" + "time": "2023-10-31T17:59:56+00:00" }, { "name": "symfony/deprecation-contracts", @@ -11212,24 +11212,24 @@ }, { "name": "symfony/event-dispatcher", - "version": "v6.4.0", + "version": "v7.0.0", "source": { "type": "git", "url": "https://github.com/symfony/event-dispatcher.git", - "reference": "d76d2632cfc2206eecb5ad2b26cd5934082941b6" + "reference": "c459b40ffe67c49af6fd392aac374c9edf8a027e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/d76d2632cfc2206eecb5ad2b26cd5934082941b6", - "reference": "d76d2632cfc2206eecb5ad2b26cd5934082941b6", + "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/c459b40ffe67c49af6fd392aac374c9edf8a027e", + "reference": "c459b40ffe67c49af6fd392aac374c9edf8a027e", "shasum": "" }, "require": { - "php": ">=8.1", + "php": ">=8.2", "symfony/event-dispatcher-contracts": "^2.5|^3" }, "conflict": { - "symfony/dependency-injection": "<5.4", + "symfony/dependency-injection": "<6.4", "symfony/service-contracts": "<2.5" }, "provide": { @@ -11238,13 +11238,13 @@ }, "require-dev": { "psr/log": "^1|^2|^3", - "symfony/config": "^5.4|^6.0|^7.0", - "symfony/dependency-injection": "^5.4|^6.0|^7.0", - "symfony/error-handler": "^5.4|^6.0|^7.0", - "symfony/expression-language": "^5.4|^6.0|^7.0", - "symfony/http-foundation": "^5.4|^6.0|^7.0", + "symfony/config": "^6.4|^7.0", + "symfony/dependency-injection": "^6.4|^7.0", + "symfony/error-handler": "^6.4|^7.0", + "symfony/expression-language": "^6.4|^7.0", + "symfony/http-foundation": "^6.4|^7.0", "symfony/service-contracts": "^2.5|^3", - "symfony/stopwatch": "^5.4|^6.0|^7.0" + "symfony/stopwatch": "^6.4|^7.0" }, "type": "library", "autoload": { @@ -11272,7 +11272,7 @@ "description": "Provides tools that allow your application components to communicate with each other by dispatching events and listening to them", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/event-dispatcher/tree/v6.4.0" + "source": "https://github.com/symfony/event-dispatcher/tree/v7.0.0" }, "funding": [ { @@ -11288,7 +11288,7 @@ "type": "tidelift" } ], - "time": "2023-07-27T06:52:43+00:00" + "time": "2023-07-27T16:29:09+00:00" }, { "name": "symfony/event-dispatcher-contracts", @@ -11855,25 +11855,25 @@ }, { "name": "symfony/intl", - "version": "v6.4.0", + "version": "v7.0.0", "source": { "type": "git", "url": "https://github.com/symfony/intl.git", - "reference": "41d16f0294b9ca6e5540728580c65cfa3848fbf5" + "reference": "a2bf3df1fe6ca7ed9aaf2d3f7d7a33b5529b021d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/intl/zipball/41d16f0294b9ca6e5540728580c65cfa3848fbf5", - "reference": "41d16f0294b9ca6e5540728580c65cfa3848fbf5", + "url": "https://api.github.com/repos/symfony/intl/zipball/a2bf3df1fe6ca7ed9aaf2d3f7d7a33b5529b021d", + "reference": "a2bf3df1fe6ca7ed9aaf2d3f7d7a33b5529b021d", "shasum": "" }, "require": { - "php": ">=8.1" + "php": ">=8.2" }, "require-dev": { - "symfony/filesystem": "^5.4|^6.0|^7.0", - "symfony/finder": "^5.4|^6.0|^7.0", - "symfony/var-exporter": "^5.4|^6.0|^7.0" + "symfony/filesystem": "^6.4|^7.0", + "symfony/finder": "^6.4|^7.0", + "symfony/var-exporter": "^6.4|^7.0" }, "type": "library", "autoload": { @@ -11917,7 +11917,7 @@ "localization" ], "support": { - "source": "https://github.com/symfony/intl/tree/v6.4.0" + "source": "https://github.com/symfony/intl/tree/v7.0.0" }, "funding": [ { @@ -11933,7 +11933,7 @@ "type": "tidelift" } ], - "time": "2023-10-28T23:12:08+00:00" + "time": "2023-10-28T23:12:22+00:00" }, { "name": "symfony/mailer", @@ -12170,20 +12170,20 @@ }, { "name": "symfony/options-resolver", - "version": "v6.4.0", + "version": "v7.0.0", "source": { "type": "git", "url": "https://github.com/symfony/options-resolver.git", - "reference": "22301f0e7fdeaacc14318928612dee79be99860e" + "reference": "700ff4096e346f54cb628ea650767c8130f1001f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/options-resolver/zipball/22301f0e7fdeaacc14318928612dee79be99860e", - "reference": "22301f0e7fdeaacc14318928612dee79be99860e", + "url": "https://api.github.com/repos/symfony/options-resolver/zipball/700ff4096e346f54cb628ea650767c8130f1001f", + "reference": "700ff4096e346f54cb628ea650767c8130f1001f", "shasum": "" }, "require": { - "php": ">=8.1", + "php": ">=8.2", "symfony/deprecation-contracts": "^2.5|^3" }, "type": "library", @@ -12217,7 +12217,7 @@ "options" ], "support": { - "source": "https://github.com/symfony/options-resolver/tree/v6.4.0" + "source": "https://github.com/symfony/options-resolver/tree/v7.0.0" }, "funding": [ { @@ -12233,7 +12233,7 @@ "type": "tidelift" } ], - "time": "2023-08-08T10:16:24+00:00" + "time": "2023-08-08T10:20:21+00:00" }, { "name": "symfony/polyfill-ctype", @@ -13447,20 +13447,20 @@ }, { "name": "symfony/string", - "version": "v6.4.0", + "version": "v7.0.0", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "b45fcf399ea9c3af543a92edf7172ba21174d809" + "reference": "92bd2bfbba476d4a1838e5e12168bef2fd1e6620" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/b45fcf399ea9c3af543a92edf7172ba21174d809", - "reference": "b45fcf399ea9c3af543a92edf7172ba21174d809", + "url": "https://api.github.com/repos/symfony/string/zipball/92bd2bfbba476d4a1838e5e12168bef2fd1e6620", + "reference": "92bd2bfbba476d4a1838e5e12168bef2fd1e6620", "shasum": "" }, "require": { - "php": ">=8.1", + "php": ">=8.2", "symfony/polyfill-ctype": "~1.8", "symfony/polyfill-intl-grapheme": "~1.0", "symfony/polyfill-intl-normalizer": "~1.0", @@ -13470,11 +13470,11 @@ "symfony/translation-contracts": "<2.5" }, "require-dev": { - "symfony/error-handler": "^5.4|^6.0|^7.0", - "symfony/http-client": "^5.4|^6.0|^7.0", - "symfony/intl": "^6.2|^7.0", + "symfony/error-handler": "^6.4|^7.0", + "symfony/http-client": "^6.4|^7.0", + "symfony/intl": "^6.4|^7.0", "symfony/translation-contracts": "^2.5|^3.0", - "symfony/var-exporter": "^5.4|^6.0|^7.0" + "symfony/var-exporter": "^6.4|^7.0" }, "type": "library", "autoload": { @@ -13513,7 +13513,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v6.4.0" + "source": "https://github.com/symfony/string/tree/v7.0.0" }, "funding": [ { @@ -13529,7 +13529,7 @@ "type": "tidelift" } ], - "time": "2023-11-28T20:41:49+00:00" + "time": "2023-11-29T08:40:23+00:00" }, { "name": "symfony/translation", @@ -14033,23 +14033,23 @@ }, { "name": "tijsverkoyen/css-to-inline-styles", - "version": "2.2.6", + "version": "v2.2.7", "source": { "type": "git", "url": "https://github.com/tijsverkoyen/CssToInlineStyles.git", - "reference": "c42125b83a4fa63b187fdf29f9c93cb7733da30c" + "reference": "83ee6f38df0a63106a9e4536e3060458b74ccedb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/tijsverkoyen/CssToInlineStyles/zipball/c42125b83a4fa63b187fdf29f9c93cb7733da30c", - "reference": "c42125b83a4fa63b187fdf29f9c93cb7733da30c", + "url": "https://api.github.com/repos/tijsverkoyen/CssToInlineStyles/zipball/83ee6f38df0a63106a9e4536e3060458b74ccedb", + "reference": "83ee6f38df0a63106a9e4536e3060458b74ccedb", "shasum": "" }, "require": { "ext-dom": "*", "ext-libxml": "*", "php": "^5.5 || ^7.0 || ^8.0", - "symfony/css-selector": "^2.7 || ^3.0 || ^4.0 || ^5.0 || ^6.0" + "symfony/css-selector": "^2.7 || ^3.0 || ^4.0 || ^5.0 || ^6.0 || ^7.0" }, "require-dev": { "phpunit/phpunit": "^4.8.35 || ^5.7 || ^6.0 || ^7.5 || ^8.5.21 || ^9.5.10" @@ -14080,9 +14080,9 @@ "homepage": "https://github.com/tijsverkoyen/CssToInlineStyles", "support": { "issues": "https://github.com/tijsverkoyen/CssToInlineStyles/issues", - "source": "https://github.com/tijsverkoyen/CssToInlineStyles/tree/2.2.6" + "source": "https://github.com/tijsverkoyen/CssToInlineStyles/tree/v2.2.7" }, - "time": "2023-01-03T09:29:04+00:00" + "time": "2023-12-08T13:03:43+00:00" }, { "name": "turbo124/beacon", @@ -15455,16 +15455,16 @@ }, { "name": "friendsofphp/php-cs-fixer", - "version": "v3.40.2", + "version": "v3.41.1", "source": { "type": "git", "url": "https://github.com/PHP-CS-Fixer/PHP-CS-Fixer.git", - "reference": "4344562a516b76afe8f2d64b2e52214c30d64ed8" + "reference": "8b6ae8dcbaf23f09680643ab832a4a3a260265f6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PHP-CS-Fixer/PHP-CS-Fixer/zipball/4344562a516b76afe8f2d64b2e52214c30d64ed8", - "reference": "4344562a516b76afe8f2d64b2e52214c30d64ed8", + "url": "https://api.github.com/repos/PHP-CS-Fixer/PHP-CS-Fixer/zipball/8b6ae8dcbaf23f09680643ab832a4a3a260265f6", + "reference": "8b6ae8dcbaf23f09680643ab832a4a3a260265f6", "shasum": "" }, "require": { @@ -15494,8 +15494,6 @@ "php-cs-fixer/accessible-object": "^1.1", "php-cs-fixer/phpunit-constraint-isidenticalstring": "^1.4", "php-cs-fixer/phpunit-constraint-xmlmatchesxsd": "^1.4", - "phpspec/prophecy": "^1.17", - "phpspec/prophecy-phpunit": "^2.0", "phpunit/phpunit": "^9.6", "symfony/phpunit-bridge": "^6.3.8 || ^7.0", "symfony/yaml": "^5.4 || ^6.0 || ^7.0" @@ -15536,7 +15534,7 @@ ], "support": { "issues": "https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/issues", - "source": "https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/tree/v3.40.2" + "source": "https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/tree/v3.41.1" }, "funding": [ { @@ -15544,7 +15542,7 @@ "type": "github" } ], - "time": "2023-12-03T09:21:33+00:00" + "time": "2023-12-10T19:59:27+00:00" }, { "name": "hamcrest/hamcrest-php", @@ -15825,16 +15823,16 @@ }, { "name": "mockery/mockery", - "version": "1.6.6", + "version": "1.6.7", "source": { "type": "git", "url": "https://github.com/mockery/mockery.git", - "reference": "b8e0bb7d8c604046539c1115994632c74dcb361e" + "reference": "0cc058854b3195ba21dc6b1f7b1f60f4ef3a9c06" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/mockery/mockery/zipball/b8e0bb7d8c604046539c1115994632c74dcb361e", - "reference": "b8e0bb7d8c604046539c1115994632c74dcb361e", + "url": "https://api.github.com/repos/mockery/mockery/zipball/0cc058854b3195ba21dc6b1f7b1f60f4ef3a9c06", + "reference": "0cc058854b3195ba21dc6b1f7b1f60f4ef3a9c06", "shasum": "" }, "require": { @@ -15847,9 +15845,7 @@ }, "require-dev": { "phpunit/phpunit": "^8.5 || ^9.6.10", - "psalm/plugin-phpunit": "^0.18.4", - "symplify/easy-coding-standard": "^11.5.0", - "vimeo/psalm": "^4.30" + "symplify/easy-coding-standard": "^12.0.8" }, "type": "library", "autoload": { @@ -15906,7 +15902,7 @@ "security": "https://github.com/mockery/mockery/security/advisories", "source": "https://github.com/mockery/mockery" }, - "time": "2023-08-09T00:03:52+00:00" + "time": "2023-12-10T02:24:34+00:00" }, { "name": "myclabs/deep-copy", @@ -16263,16 +16259,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.10.47", + "version": "1.10.50", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "84dbb33b520ea28b6cf5676a3941f4bae1c1ff39" + "reference": "06a98513ac72c03e8366b5a0cb00750b487032e4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/84dbb33b520ea28b6cf5676a3941f4bae1c1ff39", - "reference": "84dbb33b520ea28b6cf5676a3941f4bae1c1ff39", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/06a98513ac72c03e8366b5a0cb00750b487032e4", + "reference": "06a98513ac72c03e8366b5a0cb00750b487032e4", "shasum": "" }, "require": { @@ -16321,20 +16317,20 @@ "type": "tidelift" } ], - "time": "2023-12-01T15:19:17+00:00" + "time": "2023-12-13T10:59:42+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "10.1.9", + "version": "10.1.10", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "a56a9ab2f680246adcf3db43f38ddf1765774735" + "reference": "599109c8ca6bae97b23482d557d2874c25a65e59" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/a56a9ab2f680246adcf3db43f38ddf1765774735", - "reference": "a56a9ab2f680246adcf3db43f38ddf1765774735", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/599109c8ca6bae97b23482d557d2874c25a65e59", + "reference": "599109c8ca6bae97b23482d557d2874c25a65e59", "shasum": "" }, "require": { @@ -16391,7 +16387,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/10.1.9" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/10.1.10" }, "funding": [ { @@ -16399,7 +16395,7 @@ "type": "github" } ], - "time": "2023-11-23T12:23:20+00:00" + "time": "2023-12-11T06:28:43+00:00" }, { "name": "phpunit/php-file-iterator", @@ -16646,16 +16642,16 @@ }, { "name": "phpunit/phpunit", - "version": "10.5.2", + "version": "10.5.3", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "5aedff46afba98dddecaa12349ec044d9103d4fe" + "reference": "6fce887c71076a73f32fd3e0774a6833fc5c7f19" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/5aedff46afba98dddecaa12349ec044d9103d4fe", - "reference": "5aedff46afba98dddecaa12349ec044d9103d4fe", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/6fce887c71076a73f32fd3e0774a6833fc5c7f19", + "reference": "6fce887c71076a73f32fd3e0774a6833fc5c7f19", "shasum": "" }, "require": { @@ -16727,7 +16723,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.2" + "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.3" }, "funding": [ { @@ -16743,7 +16739,7 @@ "type": "tidelift" } ], - "time": "2023-12-05T14:54:33+00:00" + "time": "2023-12-13T07:25:23+00:00" }, { "name": "sebastian/cli-parser", @@ -17877,16 +17873,16 @@ }, { "name": "spatie/laravel-ignition", - "version": "2.3.1", + "version": "2.3.2", "source": { "type": "git", "url": "https://github.com/spatie/laravel-ignition.git", - "reference": "bf21cd15aa47fa4ec5d73bbc932005c70261efc8" + "reference": "4800661a195e15783477d99f7f8f669a49793996" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/spatie/laravel-ignition/zipball/bf21cd15aa47fa4ec5d73bbc932005c70261efc8", - "reference": "bf21cd15aa47fa4ec5d73bbc932005c70261efc8", + "url": "https://api.github.com/repos/spatie/laravel-ignition/zipball/4800661a195e15783477d99f7f8f669a49793996", + "reference": "4800661a195e15783477d99f7f8f669a49793996", "shasum": "" }, "require": { @@ -17965,7 +17961,7 @@ "type": "github" } ], - "time": "2023-10-09T12:55:26+00:00" + "time": "2023-12-15T13:44:49+00:00" }, { "name": "spaze/phpstan-stripe", @@ -18110,20 +18106,20 @@ }, { "name": "symfony/stopwatch", - "version": "v6.4.0", + "version": "v7.0.0", "source": { "type": "git", "url": "https://github.com/symfony/stopwatch.git", - "reference": "fc47f1015ec80927ff64ba9094dfe8b9d48fe9f2" + "reference": "7bbfa3dd564a0ce12eb4acaaa46823c740f9cb7a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/stopwatch/zipball/fc47f1015ec80927ff64ba9094dfe8b9d48fe9f2", - "reference": "fc47f1015ec80927ff64ba9094dfe8b9d48fe9f2", + "url": "https://api.github.com/repos/symfony/stopwatch/zipball/7bbfa3dd564a0ce12eb4acaaa46823c740f9cb7a", + "reference": "7bbfa3dd564a0ce12eb4acaaa46823c740f9cb7a", "shasum": "" }, "require": { - "php": ">=8.1", + "php": ">=8.2", "symfony/service-contracts": "^2.5|^3" }, "type": "library", @@ -18152,7 +18148,7 @@ "description": "Provides a way to profile code", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/stopwatch/tree/v6.4.0" + "source": "https://github.com/symfony/stopwatch/tree/v7.0.0" }, "funding": [ { @@ -18168,7 +18164,7 @@ "type": "tidelift" } ], - "time": "2023-02-16T10:14:28+00:00" + "time": "2023-07-05T13:06:06+00:00" }, { "name": "theseer/tokenizer", From 8d43eb6664f194869d7440f3983db31d09416d40 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 16 Dec 2023 18:13:08 +0100 Subject: [PATCH 019/119] better ExpenseMailboxJob --- app/Helpers/Mail/Mailbox/Imap/ImapMailbox.php | 2 +- .../Mail/Webhook/BaseWebhookHandler.php | 5 ++ app/Jobs/Mail/ExpenseMailboxJob.php | 69 ++++++++++--------- config/ninja.php | 2 +- 4 files changed, 44 insertions(+), 34 deletions(-) diff --git a/app/Helpers/Mail/Mailbox/Imap/ImapMailbox.php b/app/Helpers/Mail/Mailbox/Imap/ImapMailbox.php index 9353227a9531..ecdc9758934c 100644 --- a/app/Helpers/Mail/Mailbox/Imap/ImapMailbox.php +++ b/app/Helpers/Mail/Mailbox/Imap/ImapMailbox.php @@ -9,7 +9,7 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Helpers\Mail; +namespace App\Helpers\Mail\Mailbox\Imap; use Ddeboer\Imap\MessageInterface; use Ddeboer\Imap\Server; diff --git a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php b/app/Helpers/Mail/Webhook/BaseWebhookHandler.php index 8d2f2f4cc784..3e65923e5284 100644 --- a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php +++ b/app/Helpers/Mail/Webhook/BaseWebhookHandler.php @@ -11,8 +11,10 @@ namespace App\Helpers\Mail\Webhook; +use App\Events\Expense\ExpenseWasCreated; use App\Factory\ExpenseFactory; use App\Models\Company; +use App\Utils\Ninja; use App\Utils\TempFile; use App\Utils\Traits\GeneratesCounter; use App\Utils\Traits\SavesDocuments; @@ -44,6 +46,9 @@ abstract class BaseWebhookHandler $expense->saveQuietly(); + event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); + event('eloquent.created: App\Models\Expense', $expense); + return $expense; } diff --git a/app/Jobs/Mail/ExpenseMailboxJob.php b/app/Jobs/Mail/ExpenseMailboxJob.php index 972d835b45ef..997d2ccca2aa 100644 --- a/app/Jobs/Mail/ExpenseMailboxJob.php +++ b/app/Jobs/Mail/ExpenseMailboxJob.php @@ -11,44 +11,28 @@ namespace App\Jobs\Mail; -use App\DataMapper\Analytics\EmailFailure; -use App\DataMapper\Analytics\EmailSuccess; use App\Events\Expense\ExpenseWasCreated; -use App\Events\Invoice\InvoiceWasEmailedAndFailed; -use App\Events\Payment\PaymentWasEmailedAndFailed; use App\Factory\ExpenseFactory; use App\Helpers\Mail\Mailbox\Imap\ImapMailbox; -use App\Jobs\Util\SystemLogger; -use App\Libraries\Google\Google; use App\Libraries\MultiDB; -use App\Models\Account; -use App\Models\ClientContact; use App\Models\Company; -use App\Models\Expense; -use App\Models\Invoice; -use App\Models\Payment; -use App\Models\SystemLog; -use App\Models\User; use App\Models\Vendor; use App\Repositories\ExpenseRepository; use App\Utils\Ninja; +use App\Utils\TempFile; use App\Utils\Traits\MakesHash; -use GuzzleHttp\Exception\ClientException; +use App\Utils\Traits\SavesDocuments; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -use Illuminate\Support\Facades\App; -use Illuminate\Support\Facades\Cache; -use Illuminate\Support\Facades\Mail; -use Turbo124\Beacon\Facades\LightLogs; /*Multi Mailer implemented*/ class ExpenseMailboxJob implements ShouldQueue { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, MakesHash; + use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, MakesHash, SavesDocuments; public $tries = 4; //number of retries @@ -70,15 +54,22 @@ class ExpenseMailboxJob implements ShouldQueue { //multiDB environment, need to - foreach (MultiDB::$dbs as $db) { - MultiDB::setDB($db); + if (sizeOf($this->imap_credentials) == 0) + return; - if (sizeOf($this->imap_credentials) != 0) { - nlog("importing expenses from imap-servers"); + foreach ($this->imap_companies as $companyId) { + $company = MultiDB::findAndSetDbByCompanyId($companyId); + if (!$company) { + nlog("processing of an imap_mailbox skipped because of unknown companyId: " . $companyId); + return; + } - Company::whereIn('id', $this->imap_companies)->cursor()->each(function ($company) { - $this->handleImapCompany($company); - }); + try { + nlog("start importing expenses from imap-server of company: " . $companyId); + $this->handleImapCompany($company); + + } catch (\Exception $e) { + nlog("processing of an imap_mailbox failed upnormally: " . $companyId . " message: " . $e->getMessage()); // @turbo124 @todo should this be handled in an other way? } } @@ -87,15 +78,18 @@ class ExpenseMailboxJob implements ShouldQueue private function getImapCredentials() { $servers = array_map('trim', explode(",", config('ninja.inbound_expense.imap.servers'))); - $ports = explode(",", config('ninja.inbound_expense.imap.servers')); - $users = explode(",", config('ninja.inbound_expense.imap.servers')); - $passwords = explode(",", config('ninja.inbound_expense.imap.servers')); - $companies = explode(",", config('ninja.inbound_expense.imap.servers')); + $ports = array_map('trim', explode(",", config('ninja.inbound_expense.imap.ports'))); + $users = array_map('trim', explode(",", config('ninja.inbound_expense.imap.users'))); + $passwords = array_map('trim', explode(",", config('ninja.inbound_expense.imap.passwords'))); + $companies = array_map('trim', explode(",", config('ninja.inbound_expense.imap.companies'))); if (sizeOf($servers) != sizeOf($ports) || sizeOf($servers) != sizeOf($users) || sizeOf($servers) != sizeOf($passwords) || sizeOf($servers) != sizeOf($companies)) throw new \Exception('invalid configuration inbound_expense.imap (wrong element-count)'); foreach ($companies as $index => $companyId) { + if ($servers[$index] == '') // if property is empty, ignore => this happens exspecialy when no config is provided and it enabled us to set a single default company for env (usefull on self-hosted) + continue; + $this->imap_credentials[$companyId] = [ "server" => $servers[$index], "port" => $ports[$index] != '' ? $ports[$index] : null, @@ -137,10 +131,21 @@ class ExpenseMailboxJob implements ShouldQueue "documents" => $documents, // FIXME: https://github.com/ddeboer/imap?tab=readme-ov-file#message-attachments ]; - $expense = $this->expense_repo->save($data, ExpenseFactory::create($company->company->id, $company->company->owner()->id)); // TODO: dont assign a new number at beginning + $expense = ExpenseFactory::create($company->company->id, $company->company->owner()->id); + + $expense->vendor_id = $vendor !== null ? $vendor->id : null; + $expense->public_notes = $mail->getSubject(); + $expense->private_notes = $mail->getBodyText(); + $expense->date = $mail->getDate(); + + // add html_message as document to the expense + $documents[] = TempFile::UploadedFileFromRaw($mail->getBodyHtml(), "E-Mail.html", "text/html"); + + $this->saveDocuments($documents, $expense); + + $expense->saveQuietly(); event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); - event('eloquent.created: App\Models\Expense', $expense); $mail->markAsSeen(); diff --git a/config/ninja.php b/config/ninja.php index 3fe96170237f..2ac6bdd30f2d 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -234,7 +234,7 @@ return [ 'ports' => env('INBOUND_EXPENSE_IMAP_PORTS', ''), 'users' => env('INBOUND_EXPENSE_IMAP_USERS', ''), 'passwords' => env('INBOUND_EXPENSE_IMAP_PASSWORDS', ''), - 'companies' => env('INBOUND_EXPENSE_IMAP_COMPANIES', ''), + 'companies' => env('INBOUND_EXPENSE_IMAP_COMPANIES', '1'), ], 'webhook' => [ 'mailbox_template' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_TEMPLATE', null), From f0415b6b20c8904323e046aae8ae219140d707ea Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 16 Dec 2023 18:13:26 +0100 Subject: [PATCH 020/119] fixes --- app/Helpers/Mail/Webhook/BaseWebhookHandler.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php b/app/Helpers/Mail/Webhook/BaseWebhookHandler.php index 3e65923e5284..0967e81388aa 100644 --- a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php +++ b/app/Helpers/Mail/Webhook/BaseWebhookHandler.php @@ -21,8 +21,7 @@ use App\Utils\Traits\SavesDocuments; abstract class BaseWebhookHandler { - use GeneratesCounter; - use SavesDocuments; + use GeneratesCounter, SavesDocuments; public function process() { From 5d70daaaaac52cc1a615ce7164a5402366e13e3e Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 18 Dec 2023 15:05:15 +0100 Subject: [PATCH 021/119] wip: restruct and init IngresEmailEngine --- .../Transformer/ImapMailTransformer.php | 38 +++++ .../MailgunInboundWebhookTransformer.php | 36 ++++ .../PostmarkInboundWebhookTransformer.php} | 50 +++--- .../Mail/Webhook/BaseWebhookHandler.php | 58 ------- .../Webhook/Mailgun/MailgunWebhookHandler.php | 44 ----- app/Jobs/Mail/ExpenseMailboxJob.php | 55 ++---- app/Models/Company.php | 15 +- app/Models/Vendor.php | 8 +- app/Services/IngresEmail/IngresEmail.php | 50 ++++++ .../IngresEmail/IngresEmailEngine.php | 158 ++++++++++++++++++ app/Transformers/VendorTransformer.php | 4 +- ...10951_create_imap_configuration_fields.php | 11 +- 12 files changed, 343 insertions(+), 184 deletions(-) create mode 100644 app/Helpers/IngresMail/Transformer/ImapMailTransformer.php create mode 100644 app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php rename app/Helpers/{Mail/Webhook/Postmark/PostmarkWebhookHandler.php => IngresMail/Transformer/PostmarkInboundWebhookTransformer.php} (81%) delete mode 100644 app/Helpers/Mail/Webhook/BaseWebhookHandler.php delete mode 100644 app/Helpers/Mail/Webhook/Mailgun/MailgunWebhookHandler.php create mode 100644 app/Services/IngresEmail/IngresEmail.php create mode 100644 app/Services/IngresEmail/IngresEmailEngine.php diff --git a/app/Helpers/IngresMail/Transformer/ImapMailTransformer.php b/app/Helpers/IngresMail/Transformer/ImapMailTransformer.php new file mode 100644 index 000000000000..80889b3afc9c --- /dev/null +++ b/app/Helpers/IngresMail/Transformer/ImapMailTransformer.php @@ -0,0 +1,38 @@ +from = $mail->getSender(); + $ingresEmail->subject = $mail->getSubject(); + $ingresEmail->plain_message = $mail->getBodyText(); + $ingresEmail->html_message = $mail->getBodyHtml(); + $ingresEmail->date = $mail->getDate(); + + // parse documents as UploadedFile + foreach ($mail->getAttachments() as $attachment) { + $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment->getContent(), $attachment->getFilename(), $attachment->getEncoding()); + } + + return $ingresEmail; + } +} diff --git a/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php b/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php new file mode 100644 index 000000000000..6871e3010168 --- /dev/null +++ b/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php @@ -0,0 +1,36 @@ +from = $data["sender"]; + $ingresEmail->subject = $data["subject"]; + $ingresEmail->plain_message = $data["body-plain"]; + $ingresEmail->html_message = $data["body-html"]; + $ingresEmail->date = now(); // TODO + + // parse documents as UploadedFile from webhook-data + foreach ($data["Attachments"] as $attachment) { + $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]); + } + + return $ingresEmail; + } +} diff --git a/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php b/app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php similarity index 81% rename from app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php rename to app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php index 1cceb5a69753..0ad0faf71963 100644 --- a/app/Helpers/Mail/Webhook/Postmark/PostmarkWebhookHandler.php +++ b/app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php @@ -9,13 +9,32 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Helpers\Mail\Webhook\Postmark; +namespace App\Helpers\IngresMail\Transformer; -use App\Helpers\Mail\Webhook\BaseWebhookHandler; +use App\Services\IngresEmail\IngresEmail; use App\Utils\TempFile; -class PostmarkWebhookHandler extends BaseWebhookHandler +class PostmarkInboundWebhookTransformer { + public function process($data) + { + + $ingresEmail = new IngresEmail(); + + $ingresEmail->from = $data["From"]; + $ingresEmail->subject = $data["Subject"]; + $ingresEmail->plain_message = $data["TextBody"]; + $ingresEmail->html_message = $data["HtmlBody"]; + $ingresEmail->date = $data["Date"]; // TODO: parsing + + // parse documents as UploadedFile from webhook-data + foreach ($data["Attachments"] as $attachment) { + $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]); + } + + return $ingresEmail; + + } // { // "FromName": "Postmarkapp Support", // "MessageStream": "inbound", @@ -96,29 +115,4 @@ class PostmarkWebhookHandler extends BaseWebhookHandler // } // ] // } - public function process($data) - { - - $from = $data["From"]; - $subject = $data["Subject"]; - $plain_message = $data["TextBody"]; - $html_message = $data["HtmlBody"]; - $date = $data["Date"]; // TODO - - // parse documents as UploadedFile from webhook-data - $documents = []; - foreach ($data["Attachments"] as $attachment) { - $documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]); - } - - return $this->createExpense( - $from, - $subject, - $plain_message, - $html_message, - $date, - $documents, - ); - - } } diff --git a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php b/app/Helpers/Mail/Webhook/BaseWebhookHandler.php deleted file mode 100644 index 0967e81388aa..000000000000 --- a/app/Helpers/Mail/Webhook/BaseWebhookHandler.php +++ /dev/null @@ -1,58 +0,0 @@ -matchCompany($email); - if (!$company) - return false; - - $expense = ExpenseFactory::create($company->id, $company->owner()->id); - - $expense->public_notes = $subject; - $expense->private_notes = $plain_message; - $expense->date = $date; - - // add html_message as document to the expense - $documents[] = TempFile::UploadedFileFromRaw($html_message, "E-Mail.html", "text/html"); - - $this->saveDocuments($documents, $expense); - - $expense->saveQuietly(); - - event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); - event('eloquent.created: App\Models\Expense', $expense); - - return $expense; - } - - private function matchCompany(string $email) - { - return Company::where("expense_mailbox", $email)->first(); - } -} diff --git a/app/Helpers/Mail/Webhook/Mailgun/MailgunWebhookHandler.php b/app/Helpers/Mail/Webhook/Mailgun/MailgunWebhookHandler.php deleted file mode 100644 index e3efdc9986a0..000000000000 --- a/app/Helpers/Mail/Webhook/Mailgun/MailgunWebhookHandler.php +++ /dev/null @@ -1,44 +0,0 @@ -createExpense( - $from, - $subject, - $plain_message, - $html_message, - $date, - $documents, - ); - - } -} diff --git a/app/Jobs/Mail/ExpenseMailboxJob.php b/app/Jobs/Mail/ExpenseMailboxJob.php index 997d2ccca2aa..9319c1b08391 100644 --- a/app/Jobs/Mail/ExpenseMailboxJob.php +++ b/app/Jobs/Mail/ExpenseMailboxJob.php @@ -11,15 +11,12 @@ namespace App\Jobs\Mail; -use App\Events\Expense\ExpenseWasCreated; -use App\Factory\ExpenseFactory; +use App\Helpers\IngresMail\Transformer\ImapMailTransformer; use App\Helpers\Mail\Mailbox\Imap\ImapMailbox; use App\Libraries\MultiDB; use App\Models\Company; -use App\Models\Vendor; use App\Repositories\ExpenseRepository; -use App\Utils\Ninja; -use App\Utils\TempFile; +use App\Services\IngresEmail\IngresEmailEngine; use App\Utils\Traits\MakesHash; use App\Utils\Traits\SavesDocuments; use Illuminate\Bus\Queueable; @@ -87,6 +84,7 @@ class ExpenseMailboxJob implements ShouldQueue throw new \Exception('invalid configuration inbound_expense.imap (wrong element-count)'); foreach ($companies as $index => $companyId) { + if ($servers[$index] == '') // if property is empty, ignore => this happens exspecialy when no config is provided and it enabled us to set a single default company for env (usefull on self-hosted) continue; @@ -97,6 +95,7 @@ class ExpenseMailboxJob implements ShouldQueue "password" => $passwords[$index], ]; $this->imap_companies[] = $companyId; + } } @@ -106,55 +105,25 @@ class ExpenseMailboxJob implements ShouldQueue $credentials = $this->imap_credentials[$company->id]; $imapMailbox = new ImapMailbox($credentials->server, $credentials->port, $credentials->user, $credentials->password); + $transformer = new ImapMailTransformer(); $emails = $imapMailbox->getUnprocessedEmails(); - foreach ($emails as $mail) { + + foreach ($emails as $email) { try { - $sender = $mail->getSender(); + $email->markAsSeen(); - $vendor = Vendor::where('expense_sender_email', $sender)->first(); - if ($vendor == null) - $vendor = Vendor::where($sender, 'LIKE', "CONCAT('%',expense_sender_domain)")->first(); - if ($vendor == null) - $vendor = Vendor::where("email", $sender)->first(); + IngresEmailEngine::dispatch($transformer->transform($email)); - $documents = []; // TODO: $mail->getAttachments() + save email as document (.html) - - $data = [ - "vendor_id" => $vendor !== null ? $vendor->id : null, - "date" => $mail->getDate(), - "public_notes" => $mail->getSubject(), - "private_notes" => $mail->getCompleteBodyText(), - "documents" => $documents, // FIXME: https://github.com/ddeboer/imap?tab=readme-ov-file#message-attachments - ]; - - $expense = ExpenseFactory::create($company->company->id, $company->company->owner()->id); - - $expense->vendor_id = $vendor !== null ? $vendor->id : null; - $expense->public_notes = $mail->getSubject(); - $expense->private_notes = $mail->getBodyText(); - $expense->date = $mail->getDate(); - - // add html_message as document to the expense - $documents[] = TempFile::UploadedFileFromRaw($mail->getBodyHtml(), "E-Mail.html", "text/html"); - - $this->saveDocuments($documents, $expense); - - $expense->saveQuietly(); - - event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); - event('eloquent.created: App\Models\Expense', $expense); - - $mail->markAsSeen(); - $imapMailbox->moveProcessed($mail); + $imapMailbox->moveProcessed($email); } catch (\Exception $e) { - $imapMailbox->moveFailed($mail); + $imapMailbox->moveFailed($email); - nlog("processing of an email failed upnormally: " . $company->id . " message: " . $e->getMessage()); // @turbo124 @todo should this be handled in an other way? + nlog("processing of an email failed upnormally: " . $company->id . " message: " . $e->getMessage()); } } diff --git a/app/Models/Company.php b/app/Models/Company.php index f8b331ba9e4e..3c24d5b31fd4 100644 --- a/app/Models/Company.php +++ b/app/Models/Company.php @@ -111,8 +111,13 @@ use Laracasts\Presenter\PresentableTrait; * @property int $convert_expense_currency * @property int $notify_vendor_when_paid * @property int $invoice_task_hours - * @property boolean $expense_import * @property string|null $expense_mailbox + * @property boolean $expense_mailbox_active + * @property bool $expense_mailbox_allow_company_users + * @property bool $expense_mailbox_allow_vendors + * @property bool $expense_mailbox_allow_unknown + * @property string|null $expense_mailbox_whitelist_domains + * @property string|null $expense_mailbox_whitelist_emails * @property int $deleted_at * @property-read \App\Models\Account $account * @property-read \Illuminate\Database\Eloquent\Collection $activities @@ -354,8 +359,14 @@ class Company extends BaseModel 'calculate_taxes', 'tax_data', 'e_invoice_certificate_passphrase', - 'expense_import', + 'expense_mailbox_active', 'expense_mailbox', // TODO: @turbo124 custom validation: self-hosted => free change, hosted => not changeable, only changeable with env-mask + 'expense_mailbox_allow_company_users', + 'expense_mailbox_allow_vendors', + 'expense_mailbox_allow_unknown', + 'expense_mailbox_whitelist_domains', + 'expense_mailbox_whitelist_emails', + 'expense_mailbox_whitelist' ]; protected $hidden = [ diff --git a/app/Models/Vendor.php b/app/Models/Vendor.php index 0e7aec98deac..51223b449a84 100644 --- a/app/Models/Vendor.php +++ b/app/Models/Vendor.php @@ -54,8 +54,8 @@ use Laracasts\Presenter\PresentableTrait; * @property string|null $id_number * @property string|null $language_id * @property int|null $last_login - * @property string|null $expense_sender_email - * @property string|null $expense_sender_domain + * @property string|null $invoicing_email + * @property string|null $invoicing_domain * @property-read \Illuminate\Database\Eloquent\Collection $activities * @property-read int|null $activities_count * @property-read \App\Models\User|null $assigned_user @@ -117,8 +117,8 @@ class Vendor extends BaseModel 'number', 'language_id', 'classification', - 'expense_sender_email', - 'expense_sender_domain', + 'invoicing_email', + 'invoicing_domain', ]; protected $casts = [ diff --git a/app/Services/IngresEmail/IngresEmail.php b/app/Services/IngresEmail/IngresEmail.php new file mode 100644 index 000000000000..2ec889bcf6e2 --- /dev/null +++ b/app/Services/IngresEmail/IngresEmail.php @@ -0,0 +1,50 @@ +email = $email; + } + /** + * if there is not a company with an matching mailbox, we do nothing + */ + public function handle() + { + // Expense Mailbox => will create an expense + foreach ($this->email->to as $expense_mailbox) { + $this->company = MultiDB::findAndSetDbByExpenseMailbox($expense_mailbox); + if (!$this->company || !$this->validateExpenseActive()) + continue; + + $this->createExpense(); + } + + // TODO reuse this method to add more mail-parsing behaviors + } + + // MAIN-PROCESSORS + protected function createExpense() + { + if (!$this->validateExpenseSender()) { + nlog('invalid sender of an ingest email to company: ' . $this->company->id . ' from: ' . $this->email->from); + return; + } + + $expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id); + + $expense->public_notes = $this->email->subject; + $expense->private_notes = $this->email->text_body; + $expense->date = $this->email->date; + + // handle vendor assignment + $expense_vendor = $this->getExpenseVendor(); + if ($expense_vendor) + $expense->vendor_id = $expense_vendor->id; + + // handle documents + $this->processHtmlBodyToDocument(); + $documents = []; + array_push($documents, ...$this->email->documents); + if ($this->email->body_document) + $documents[] = $this->email->body_document; + $this->saveDocuments($documents, $expense); + + $expense->saveQuietly(); + + event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); // @turbo124 please check, I copied from API + event('eloquent.created: App\Models\Expense', $expense); // @turbo124 please check, I copied from API + + return $expense; + } + + // HELPERS + private function processHtmlBodyToDocument() + { + if (!$this->email->body_document && property_exists($this->email, "body")) { + $this->email->body_document = TempFile::UploadedFileFromRaw($this->email->body, "E-Mail.html", "text/html"); + } + } + private function validateExpenseActive() + { + return $this->company?->expense_mailbox_active ?: false; + } + private function validateExpenseSender() + { + // invalid email + if (!filter_var($this->email->from, FILTER_VALIDATE_EMAIL)) + return false; + + $parts = explode('@', $this->email->from); + $domain = array_pop($parts); + + // global blacklist + if (in_array($domain, $this->globalBlacklist)) + return false; + + // whitelists + $email_whitelist = explode(",", $this->company->expense_mailbox_whitelist_emails); + if (in_array($this->email->from, $email_whitelist)) + return true; + $domain_whitelist = explode(",", $this->company->expense_mailbox_whitelist_domains); + if (in_array($domain, $domain_whitelist)) + return true; + if ($this->company->expense_mailbox_allow_unknown && sizeOf($email_whitelist) == 0 && sizeOf($domain_whitelist) == 0) // from unknown only, when no whitelists are defined + return true; + + // own users + if ($this->company->expense_mailbox_allow_company_users && $this->company->users()->where("email", $this->email->from)->exists()) + return true; + + // from clients/vendors (if active) + if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->where("invoicing_email", $this->email->from)->orWhere($this->email->from, 'LIKE', "CONCAT('%',invoicing_domain)")->exists()) + return true; + if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->contacts()->where("email", $this->email->from)->exists()) // TODO + return true; + + // denie + return false; + } + private function getExpenseVendor() + { + $vendor = Vendor::where("company_id", $this->company->id)->where('invoicing_email', $this->email->from)->first(); + if ($vendor == null) + $vendor = Vendor::where("company_id", $this->company->id)->where($this->email->from, 'LIKE', "CONCAT('%',invoicing_domain)")->first(); + if ($vendor == null) { + $vendorContact = VendorContact::where("company_id", $this->company->id)->where("email", $this->email->from)->first(); + $vendor = $vendorContact->vendor(); + } + // TODO: from contacts + + return $vendor; + } +} diff --git a/app/Transformers/VendorTransformer.php b/app/Transformers/VendorTransformer.php index 5ec3c9a19175..de73267182fe 100644 --- a/app/Transformers/VendorTransformer.php +++ b/app/Transformers/VendorTransformer.php @@ -105,8 +105,8 @@ class VendorTransformer extends EntityTransformer 'language_id' => (string) $vendor->language_id ?: '', 'classification' => (string) $vendor->classification ?: '', 'display_name' => (string) $vendor->present()->name(), - 'expense_sender_email' => (string) $vendor->expense_sender_email ?: '', - 'expense_sender_domain' => (string) $vendor->expense_sender_domain ?: '', + 'invoicing_email' => (string) $vendor->invoicing_email ?: '', + 'invoicing_domain' => (string) $vendor->invoicing_domain ?: '', ]; } } diff --git a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php index f750b9b07435..91a698511324 100644 --- a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php +++ b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php @@ -12,8 +12,13 @@ return new class extends Migration { public function up(): void { Schema::table('company', function (Blueprint $table) { - $table->boolean("expense_import")->default(true); + $table->boolean("expense_mailbox_active")->default(true); $table->string("expense_mailbox")->nullable(); + $table->boolean("expense_mailbox_allow_company_users")->default(false); + $table->boolean("expense_mailbox_allow_vendors")->default(false); + $table->boolean("expense_mailbox_allow_unknown")->default(false); + $table->string("expense_mailbox_whitelist_domains")->nullable(); + $table->string("expense_mailbox_whitelist_emails")->nullable(); }); Company::query()->cursor()->each(function ($company) { // TODO: @turbo124 check migration on staging environment with real data to ensure, this works as exspected $company->expense_mailbox = config('ninja.inbound_expense.webhook.mailbox_template') != '' ? @@ -22,8 +27,8 @@ return new class extends Migration { $company->save(); }); Schema::table('vendor', function (Blueprint $table) { - $table->string("expense_sender_email")->nullable(); - $table->string("expense_sender_domain")->nullable(); + $table->string("invoicing_email")->nullable(); + $table->string("invoicing_domain")->nullable(); }); } From 7245de8c4ca01d70e44e217ea2c1f430c1fec4a2 Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 18 Dec 2023 17:21:15 +0100 Subject: [PATCH 022/119] wip: spam protection --- .../IngresEmail/IngresEmailEngine.php | 105 ++++++++++++++++-- 1 file changed, 94 insertions(+), 11 deletions(-) diff --git a/app/Services/IngresEmail/IngresEmailEngine.php b/app/Services/IngresEmail/IngresEmailEngine.php index 88135f30bc55..328a98d6c9f4 100644 --- a/app/Services/IngresEmail/IngresEmailEngine.php +++ b/app/Services/IngresEmail/IngresEmailEngine.php @@ -25,6 +25,7 @@ use App\Utils\TempFile; use App\Utils\Traits\GeneratesCounter; use App\Utils\Traits\SavesDocuments; use App\Utils\Traits\MakesHash; +use Cache; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; @@ -38,6 +39,7 @@ class IngresEmailEngine implements ShouldQueue private IngresEmail $email; private ?Company $company; + private ?bool $isUnknownRecipent = null; private array $globalBlacklist = []; function __constructor(IngresEmail $email) { @@ -45,19 +47,28 @@ class IngresEmailEngine implements ShouldQueue } /** * if there is not a company with an matching mailbox, we do nothing + * reuse this method to add more mail-parsing behaviors */ public function handle() { + if ($this->isInvalidOrBlocked()) + return; + $this->isUnknownRecipent = true; + // Expense Mailbox => will create an expense foreach ($this->email->to as $expense_mailbox) { $this->company = MultiDB::findAndSetDbByExpenseMailbox($expense_mailbox); - if (!$this->company || !$this->validateExpenseActive()) + if (!$this->company) + continue; + + $this->isUnknownRecipent = false; + if (!$this->validateExpenseActive()) continue; $this->createExpense(); } - // TODO reuse this method to add more mail-parsing behaviors + $this->saveMeta(); } // MAIN-PROCESSORS @@ -95,30 +106,102 @@ class IngresEmailEngine implements ShouldQueue return $expense; } - // HELPERS + // SPAM Protection + private function isInvalidOrBlocked() + { + // invalid email + if (!filter_var($this->email->from, FILTER_VALIDATE_EMAIL)) { + nlog('[IngressMailEngine] E-Mail blocked, because from e-mail has the wrong format: ' . $this->email->from); + return true; + } + + $parts = explode('@', $this->email->from); + $domain = array_pop($parts); + + // global blacklist + if (in_array($domain, $this->globalBlacklist)) { + nlog('[IngressMailEngine] E-Mail blocked, because the domain was found on globalBlocklist: ' . $this->email->from); + return true; + } + + if (Cache::has('ingresEmailBlockedSender:' . $this->email->from)) { // was marked as blocked before, so we block without any console output + return true; + } + + // sender occured in more than 500 emails in the last 12 hours + $senderMailCountTotal = Cache::get('ingresEmailSender:' . $this->email->from, 0); + if ($senderMailCountTotal >= 5000) { + nlog('[IngressMailEngine] E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); + $this->blockSender(); + return true; + } + if ($senderMailCountTotal >= 1000) { + nlog('[IngressMailEngine] E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); + $this->saveMeta(); + return true; + } + + // sender sended more than 50 emails to the wrong mailbox in the last 6 hours + $senderMailCountUnknownRecipent = Cache::get('ingresEmailSenderUnknownRecipent:' . $this->email->from, 0); + if ($senderMailCountUnknownRecipent >= 50) { + nlog('[IngressMailEngine] E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $this->email->from); + $this->saveMeta(); + return true; + } + + // wrong recipent occurs in more than 100 emails in the last 12 hours, so the processing is blocked + foreach ($this->email->to as $recipent) { + $mailCountUnknownRecipent = Cache::get('ingresEmailUnknownRecipent:' . $recipent, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time + if ($mailCountUnknownRecipent >= 100) { + nlog('[IngressMailEngine] E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); + $this->blockSender(); + return true; + } + } + + return false; + } + private function blockSender() + { + Cache::add('ingresEmailBlockedSender:' . $this->email->from, true, now()->addHours(12)); + $this->saveMeta(); + + // TODO: ignore, when known sender + // TODO: handle external blocking + } + private function saveMeta() + { + // save cache + Cache::add('ingresEmailSender:' . $this->email->from, 0, now()->addHours(12)); + Cache::increment('ingresEmailSender:' . $this->email->from); + + if ($this->isUnknownRecipent) { + Cache::add('ingresEmailSenderUnknownRecipent:' . $this->email->from, 0, now()->addHours(6)); + Cache::increment('ingresEmailSenderUnknownRecipent:' . $this->email->from); // we save the sender, to may block him + + foreach ($this->email->to as $recipent) { + Cache::add('ingresEmailUnknownRecipent:' . $recipent, 0, now()->addHours(12)); + Cache::increment('ingresEmailUnknownRecipent:' . $recipent); // we save the sender, to may block him + } + } + } + // PARSING private function processHtmlBodyToDocument() { if (!$this->email->body_document && property_exists($this->email, "body")) { $this->email->body_document = TempFile::UploadedFileFromRaw($this->email->body, "E-Mail.html", "text/html"); } } + // HELPERS private function validateExpenseActive() { return $this->company?->expense_mailbox_active ?: false; } private function validateExpenseSender() { - // invalid email - if (!filter_var($this->email->from, FILTER_VALIDATE_EMAIL)) - return false; - $parts = explode('@', $this->email->from); $domain = array_pop($parts); - // global blacklist - if (in_array($domain, $this->globalBlacklist)) - return false; - // whitelists $email_whitelist = explode(",", $this->company->expense_mailbox_whitelist_emails); if (in_array($this->email->from, $email_whitelist)) From 08662c1595f38b7ca8b8e3a5d64decf2036ab6a4 Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 18 Dec 2023 17:24:59 +0100 Subject: [PATCH 023/119] renaming and minor changes --- .../IngresEmail/IngresEmailEngine.php | 80 ++++++++++--------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/app/Services/IngresEmail/IngresEmailEngine.php b/app/Services/IngresEmail/IngresEmailEngine.php index 328a98d6c9f4..f0891df1faaa 100644 --- a/app/Services/IngresEmail/IngresEmailEngine.php +++ b/app/Services/IngresEmail/IngresEmailEngine.php @@ -62,7 +62,7 @@ class IngresEmailEngine implements ShouldQueue continue; $this->isUnknownRecipent = false; - if (!$this->validateExpenseActive()) + if (!$this->validateExpenseShouldProcess()) continue; $this->createExpense(); @@ -71,41 +71,6 @@ class IngresEmailEngine implements ShouldQueue $this->saveMeta(); } - // MAIN-PROCESSORS - protected function createExpense() - { - if (!$this->validateExpenseSender()) { - nlog('invalid sender of an ingest email to company: ' . $this->company->id . ' from: ' . $this->email->from); - return; - } - - $expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id); - - $expense->public_notes = $this->email->subject; - $expense->private_notes = $this->email->text_body; - $expense->date = $this->email->date; - - // handle vendor assignment - $expense_vendor = $this->getExpenseVendor(); - if ($expense_vendor) - $expense->vendor_id = $expense_vendor->id; - - // handle documents - $this->processHtmlBodyToDocument(); - $documents = []; - array_push($documents, ...$this->email->documents); - if ($this->email->body_document) - $documents[] = $this->email->body_document; - $this->saveDocuments($documents, $expense); - - $expense->saveQuietly(); - - event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); // @turbo124 please check, I copied from API - event('eloquent.created: App\Models\Expense', $expense); // @turbo124 please check, I copied from API - - return $expense; - } - // SPAM Protection private function isInvalidOrBlocked() { @@ -166,7 +131,7 @@ class IngresEmailEngine implements ShouldQueue Cache::add('ingresEmailBlockedSender:' . $this->email->from, true, now()->addHours(12)); $this->saveMeta(); - // TODO: ignore, when known sender + // TODO: ignore, when known sender (for heavy email-usage mostly on isHosted()) // TODO: handle external blocking } private function saveMeta() @@ -185,15 +150,52 @@ class IngresEmailEngine implements ShouldQueue } } } - // PARSING + + // MAIL-PARSING private function processHtmlBodyToDocument() { if (!$this->email->body_document && property_exists($this->email, "body")) { $this->email->body_document = TempFile::UploadedFileFromRaw($this->email->body, "E-Mail.html", "text/html"); } } + + // MAIN-PROCESSORS + protected function createExpense() + { + if (!$this->validateExpenseSender()) { + nlog('invalid sender of an ingest email to company: ' . $this->company->id . ' from: ' . $this->email->from); + return; + } + + $expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id); + + $expense->public_notes = $this->email->subject; + $expense->private_notes = $this->email->text_body; + $expense->date = $this->email->date; + + // handle vendor assignment + $expense_vendor = $this->getExpenseVendor(); + if ($expense_vendor) + $expense->vendor_id = $expense_vendor->id; + + // handle documents + $this->processHtmlBodyToDocument(); + $documents = []; + array_push($documents, ...$this->email->documents); + if ($this->email->body_document) + $documents[] = $this->email->body_document; + $this->saveDocuments($documents, $expense); + + $expense->saveQuietly(); + + event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); // @turbo124 please check, I copied from API + event('eloquent.created: App\Models\Expense', $expense); // @turbo124 please check, I copied from API + + return $expense; + } + // HELPERS - private function validateExpenseActive() + private function validateExpenseShouldProcess() { return $this->company?->expense_mailbox_active ?: false; } From 5efb33d1d3d5388a8757b5ca68b83c9348f865fd Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 18 Dec 2023 17:28:36 +0100 Subject: [PATCH 024/119] minor changes --- app/Services/IngresEmail/IngresEmailEngine.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/Services/IngresEmail/IngresEmailEngine.php b/app/Services/IngresEmail/IngresEmailEngine.php index f0891df1faaa..024091215d9b 100644 --- a/app/Services/IngresEmail/IngresEmailEngine.php +++ b/app/Services/IngresEmail/IngresEmailEngine.php @@ -190,8 +190,6 @@ class IngresEmailEngine implements ShouldQueue event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); // @turbo124 please check, I copied from API event('eloquent.created: App\Models\Expense', $expense); // @turbo124 please check, I copied from API - - return $expense; } // HELPERS From 450569e5c4241dd288e5cd1214fc5c56c38be4c8 Mon Sep 17 00:00:00 2001 From: paulwer Date: Tue, 19 Dec 2023 08:51:50 +0100 Subject: [PATCH 025/119] ignore emails with no documents attached --- .../IngresEmail/IngresEmailEngine.php | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/app/Services/IngresEmail/IngresEmailEngine.php b/app/Services/IngresEmail/IngresEmailEngine.php index 024091215d9b..f89d0886314b 100644 --- a/app/Services/IngresEmail/IngresEmailEngine.php +++ b/app/Services/IngresEmail/IngresEmailEngine.php @@ -46,7 +46,7 @@ class IngresEmailEngine implements ShouldQueue $this->email = $email; } /** - * if there is not a company with an matching mailbox, we do nothing + * if there is not a company with an matching mailbox, we only do monitoring * reuse this method to add more mail-parsing behaviors */ public function handle() @@ -62,9 +62,6 @@ class IngresEmailEngine implements ShouldQueue continue; $this->isUnknownRecipent = false; - if (!$this->validateExpenseShouldProcess()) - continue; - $this->createExpense(); } @@ -162,11 +159,21 @@ class IngresEmailEngine implements ShouldQueue // MAIN-PROCESSORS protected function createExpense() { + // Skipping executions: will not result in not saving Metadata to prevent usage of these conditions, to spam + if (!$this->validateExpenseShouldProcess()) { + nlog('email parsing not active for this company: ' . $this->company->id . ' from: ' . $this->email->from); + return; + } if (!$this->validateExpenseSender()) { nlog('invalid sender of an ingest email to company: ' . $this->company->id . ' from: ' . $this->email->from); return; } + if (sizeOf($this->email->documents) == 0) { + nlog('email does not contain any attachments and is likly not an expense. company: ' . $this->company->id . ' from: ' . $this->email->from); + return; + } + // create expense $expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id); $expense->public_notes = $this->email->subject; @@ -188,8 +195,8 @@ class IngresEmailEngine implements ShouldQueue $expense->saveQuietly(); - event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); // @turbo124 please check, I copied from API - event('eloquent.created: App\Models\Expense', $expense); // @turbo124 please check, I copied from API + event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); // @turbo124 please check, I copied from API-Controller + event('eloquent.created: App\Models\Expense', $expense); // @turbo124 please check, I copied from API-Controller } // HELPERS From 80a9d51ffbaef3312af356ac69df9d1cd10f0c1c Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 28 Dec 2023 09:52:27 +0100 Subject: [PATCH 026/119] revert preseed mailboxes --- .../2023_12_10_110951_create_imap_configuration_fields.php | 6 ------ 1 file changed, 6 deletions(-) diff --git a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php index 91a698511324..2813be78ba60 100644 --- a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php +++ b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php @@ -20,12 +20,6 @@ return new class extends Migration { $table->string("expense_mailbox_whitelist_domains")->nullable(); $table->string("expense_mailbox_whitelist_emails")->nullable(); }); - Company::query()->cursor()->each(function ($company) { // TODO: @turbo124 check migration on staging environment with real data to ensure, this works as exspected - $company->expense_mailbox = config('ninja.inbound_expense.webhook.mailbox_template') != '' ? - str_replace('{{company_key}}', $company->company_key, config('ninja.inbound_expense.webhook.mailbox_template')) : null; - - $company->save(); - }); Schema::table('vendor', function (Blueprint $table) { $table->string("invoicing_email")->nullable(); $table->string("invoicing_domain")->nullable(); From 21a8f4da762c52539aedf363abd5a5a6db43cb87 Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 28 Dec 2023 10:00:38 +0100 Subject: [PATCH 027/119] rework env-struct & validation of expense_mailbox --- app/Factory/CompanyFactory.php | 4 +- .../Company/ValidExpenseMailbox.php | 44 +++++++++---------- app/Jobs/Mail/ExpenseMailboxJob.php | 12 ++--- config/ninja.php | 20 ++++----- lang/en/texts.php | 1 + 5 files changed, 39 insertions(+), 42 deletions(-) diff --git a/app/Factory/CompanyFactory.php b/app/Factory/CompanyFactory.php index 00f2874ffd48..ca61d1a19c1f 100644 --- a/app/Factory/CompanyFactory.php +++ b/app/Factory/CompanyFactory.php @@ -51,8 +51,8 @@ class CompanyFactory $company->first_month_of_year = 1; // default mailbox - $company->expense_mailbox = config('ninja.inbound_expense.webhook.mailbox_template') != '' ? - str_replace('{{company_key}}', $company->company_key, config('ninja.inbound_expense.webhook.mailbox_template')) : null; + $company->expense_mailbox = config('ninja.ingest_mail.expense_mailbox_template') != '' ? + str_replace('{{company_key}}', $company->company_key, config('ninja.ingest_mail.expense_mailbox_template')) : null; return $company; } diff --git a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php index b38c1071af11..4f1215db0c60 100644 --- a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php +++ b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php @@ -21,6 +21,7 @@ use Illuminate\Contracts\Validation\Rule; class ValidExpenseMailbox implements Rule { + private $is_enterprise_error = false; private $validated_schema = false; private $company_key = false; private $isEnterprise = false; @@ -32,9 +33,7 @@ class ValidExpenseMailbox implements Rule { $this->company_key = $company_key; $this->isEnterprise = $isEnterprise; - $this->endings = explode(",", config('ninja.inbound_expense.webhook.mailbox_endings')); - $this->hasCompanyKey = config('ninja.inbound_expense.webhook.mailbox_hascompanykey'); - $this->enterprise_endings = explode(",", config('ninja.inbound_expense.webhook.mailbox_enterprise_endings')); + $this->endings = explode(",", config('ninja.ingest_mail.expense_mailbox_endings')); } public function passes($attribute, $value) @@ -43,33 +42,28 @@ class ValidExpenseMailbox implements Rule return true; } + // denie on hosted and not enterprise + if (Ninja::isHosted() && !$this->isEnterprise) { + $this->is_enterprise_error = true; + return false; + } + // early return, if we dont have any additional validation - if (!config('ninja.inbound_expense.webhook.mailbox_schema') && !(Ninja::isHosted() && config('ninja.inbound_expense.webhook.mailbox_schema_enterprise'))) { + if (!config('ninja.ingest_mail.expense_mailbox_endings')) { $this->validated_schema = true; return MultiDB::checkExpenseMailboxAvailable($value); } // Validate Schema - $validated_hasCompanyKey = !$this->hasCompanyKey || str_contains($value, $this->company_key); $validated = false; - if ($validated_hasCompanyKey) - foreach ($this->endings as $ending) { - if (str_ends_with($ending, $value)) { - $validated = true; - break; - } + foreach ($this->endings as $ending) { + if (str_ends_with($ending, $value)) { + $validated = true; + break; } + } - $validated_enterprise = false; - if (Ninja::isHosted() && $this->isEnterprise) - foreach ($this->endings as $ending) { - if (str_ends_with($ending, $value)) { - $validated_enterprise = true; - break; - } - } - - if (!$validated && !$validated_enterprise) + if (!$validated) return false; $this->validated_schema = true; @@ -81,6 +75,12 @@ class ValidExpenseMailbox implements Rule */ public function message() { - return $this->validated_schema ? ctrans('texts.expense_mailbox_taken') : ctrans('texts.expense_mailbox_invalid'); + if ($this->validated_schema) + return ctrans('texts.expense_mailbox_not_available'); + + if ($this->validated_schema) + return ctrans('texts.expense_mailbox_taken'); + + return ctrans('texts.expense_mailbox_invalid'); } } diff --git a/app/Jobs/Mail/ExpenseMailboxJob.php b/app/Jobs/Mail/ExpenseMailboxJob.php index 9319c1b08391..9f47582d26ee 100644 --- a/app/Jobs/Mail/ExpenseMailboxJob.php +++ b/app/Jobs/Mail/ExpenseMailboxJob.php @@ -74,14 +74,14 @@ class ExpenseMailboxJob implements ShouldQueue private function getImapCredentials() { - $servers = array_map('trim', explode(",", config('ninja.inbound_expense.imap.servers'))); - $ports = array_map('trim', explode(",", config('ninja.inbound_expense.imap.ports'))); - $users = array_map('trim', explode(",", config('ninja.inbound_expense.imap.users'))); - $passwords = array_map('trim', explode(",", config('ninja.inbound_expense.imap.passwords'))); - $companies = array_map('trim', explode(",", config('ninja.inbound_expense.imap.companies'))); + $servers = array_map('trim', explode(",", config('ninja.ingest_mail.imap.servers'))); + $ports = array_map('trim', explode(",", config('ninja.ingest_mail.imap.ports'))); + $users = array_map('trim', explode(",", config('ninja.ingest_mail.imap.users'))); + $passwords = array_map('trim', explode(",", config('ninja.ingest_mail.imap.passwords'))); + $companies = array_map('trim', explode(",", config('ninja.ingest_mail.imap.companies'))); if (sizeOf($servers) != sizeOf($ports) || sizeOf($servers) != sizeOf($users) || sizeOf($servers) != sizeOf($passwords) || sizeOf($servers) != sizeOf($companies)) - throw new \Exception('invalid configuration inbound_expense.imap (wrong element-count)'); + throw new \Exception('invalid configuration ingest_mail.imap (wrong element-count)'); foreach ($companies as $index => $companyId) { diff --git a/config/ninja.php b/config/ninja.php index 2ac6bdd30f2d..b874c939d82d 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -228,19 +228,15 @@ return [ 'secret' => env('PAYPAL_SECRET', null), 'client_id' => env('PAYPAL_CLIENT_ID', null), ], - 'inbound_expense' => [ + 'ingest_mail' => [ 'imap' => [ - 'servers' => env('INBOUND_EXPENSE_IMAP_SERVERS', ''), - 'ports' => env('INBOUND_EXPENSE_IMAP_PORTS', ''), - 'users' => env('INBOUND_EXPENSE_IMAP_USERS', ''), - 'passwords' => env('INBOUND_EXPENSE_IMAP_PASSWORDS', ''), - 'companies' => env('INBOUND_EXPENSE_IMAP_COMPANIES', '1'), - ], - 'webhook' => [ - 'mailbox_template' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_TEMPLATE', null), - 'mailbox_endings_endings' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_ENDINGS', ''), - 'mailbox_hascompanykey' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_HASCOMPANYKEY', false), - 'mailbox_endings_enterprise' => env('INBOUND_EXPENSE_WEBHOOK_MAILBOX_ENTERPRISE_ENDINGS', '@expense.invoicing.co'), + 'servers' => env('ingest_mail_IMAP_SERVERS', ''), + 'ports' => env('ingest_mail_IMAP_PORTS', ''), + 'users' => env('ingest_mail_IMAP_USERS', ''), + 'passwords' => env('ingest_mail_IMAP_PASSWORDS', ''), + 'companies' => env('ingest_mail_IMAP_COMPANIES', '1'), ], + 'expense_mailbox_template' => env('ingest_mail_EXPENSE_MAILBOX_TEMPLATE', null), + 'expense_mailbox_endings' => env('ingest_mail_EXPENSE_MAILBOX_ENDINGS', '@expense.invoicing.co'), ], ]; diff --git a/lang/en/texts.php b/lang/en/texts.php index f34136ad2d7f..38144f3cccf9 100644 --- a/lang/en/texts.php +++ b/lang/en/texts.php @@ -2534,6 +2534,7 @@ $lang = array( 'local_storage_required' => 'Error: local storage is not available.', 'your_password_reset_link' => 'Your Password Reset Link', 'subdomain_taken' => 'The subdomain is already in use', + 'expense_mailbox_not_available' => 'You are not allowed to set an expense_inbox. Please upgrade plan to enterpise.', 'expense_mailbox_taken' => 'The mailbox is already in use', 'expense_mailbox_invalid' => 'The mailbox does not match the required schema', 'client_login' => 'Client Login', From 77a615adb61e8978bbbad00831da08d5b2e9d6cd Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 28 Dec 2023 10:05:46 +0100 Subject: [PATCH 028/119] minor updates --- app/Http/ValidationRules/Company/ValidExpenseMailbox.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php index 4f1215db0c60..d6049763bf7b 100644 --- a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php +++ b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php @@ -75,12 +75,12 @@ class ValidExpenseMailbox implements Rule */ public function message() { - if ($this->validated_schema) + if ($this->is_enterprise_error) return ctrans('texts.expense_mailbox_not_available'); - if ($this->validated_schema) - return ctrans('texts.expense_mailbox_taken'); + if (!$this->validated_schema) + return ctrans('texts.expense_mailbox_invalid'); - return ctrans('texts.expense_mailbox_invalid'); + return ctrans('texts.expense_mailbox_taken'); } } From ad009b58375cd82b64bcba5f82998b7c5bdd1e67 Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 28 Dec 2023 10:18:24 +0100 Subject: [PATCH 029/119] changes according to: https://github.com/invoiceninja/invoiceninja/pull/9042#discussion_r1432330656 --- app/Http/Requests/Company/StoreCompanyRequest.php | 6 +++++- .../ValidationRules/Company/ValidExpenseMailbox.php | 11 ----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/app/Http/Requests/Company/StoreCompanyRequest.php b/app/Http/Requests/Company/StoreCompanyRequest.php index ab5e2201baaf..b342b6730d7c 100644 --- a/app/Http/Requests/Company/StoreCompanyRequest.php +++ b/app/Http/Requests/Company/StoreCompanyRequest.php @@ -56,7 +56,7 @@ class StoreCompanyRequest extends Request } } - $rules['expense_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); // @turbo124 check if this is right + $rules['expense_mailbox'] = new ValidExpenseMailbox($this->company->key); return $rules; } @@ -77,6 +77,10 @@ class StoreCompanyRequest extends Request $input['portal_domain'] = rtrim(strtolower($input['portal_domain']), "/"); } + if (isset($input['expense_mailbox']) && Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise')) { + unset($input['expense_mailbox']); + } + $this->replace($input); } } diff --git a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php index d6049763bf7b..f3126120e8ea 100644 --- a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php +++ b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php @@ -21,7 +21,6 @@ use Illuminate\Contracts\Validation\Rule; class ValidExpenseMailbox implements Rule { - private $is_enterprise_error = false; private $validated_schema = false; private $company_key = false; private $isEnterprise = false; @@ -32,7 +31,6 @@ class ValidExpenseMailbox implements Rule public function __construct(string $company_key, bool $isEnterprise = false) { $this->company_key = $company_key; - $this->isEnterprise = $isEnterprise; $this->endings = explode(",", config('ninja.ingest_mail.expense_mailbox_endings')); } @@ -42,12 +40,6 @@ class ValidExpenseMailbox implements Rule return true; } - // denie on hosted and not enterprise - if (Ninja::isHosted() && !$this->isEnterprise) { - $this->is_enterprise_error = true; - return false; - } - // early return, if we dont have any additional validation if (!config('ninja.ingest_mail.expense_mailbox_endings')) { $this->validated_schema = true; @@ -75,9 +67,6 @@ class ValidExpenseMailbox implements Rule */ public function message() { - if ($this->is_enterprise_error) - return ctrans('texts.expense_mailbox_not_available'); - if (!$this->validated_schema) return ctrans('texts.expense_mailbox_invalid'); From eea6c3458cdfdf429524981199064d4425652fcd Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 28 Dec 2023 10:29:35 +0100 Subject: [PATCH 030/119] remove unwanted lang --- lang/en/texts.php | 1 - 1 file changed, 1 deletion(-) diff --git a/lang/en/texts.php b/lang/en/texts.php index 38144f3cccf9..f34136ad2d7f 100644 --- a/lang/en/texts.php +++ b/lang/en/texts.php @@ -2534,7 +2534,6 @@ $lang = array( 'local_storage_required' => 'Error: local storage is not available.', 'your_password_reset_link' => 'Your Password Reset Link', 'subdomain_taken' => 'The subdomain is already in use', - 'expense_mailbox_not_available' => 'You are not allowed to set an expense_inbox. Please upgrade plan to enterpise.', 'expense_mailbox_taken' => 'The mailbox is already in use', 'expense_mailbox_invalid' => 'The mailbox does not match the required schema', 'client_login' => 'Client Login', From b3986df78880f2143327b6e4cf2d5d0a20ef0936 Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 28 Dec 2023 10:53:15 +0100 Subject: [PATCH 031/119] remove mailbox preset --- app/Factory/CompanyFactory.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/Factory/CompanyFactory.php b/app/Factory/CompanyFactory.php index ca61d1a19c1f..5248cb5b435d 100644 --- a/app/Factory/CompanyFactory.php +++ b/app/Factory/CompanyFactory.php @@ -50,10 +50,6 @@ class CompanyFactory $company->tax_data = new TaxModel(); $company->first_month_of_year = 1; - // default mailbox - $company->expense_mailbox = config('ninja.ingest_mail.expense_mailbox_template') != '' ? - str_replace('{{company_key}}', $company->company_key, config('ninja.ingest_mail.expense_mailbox_template')) : null; - return $company; } } From 1d92b91fc6bca35974eaa3ab98f53b527b420405 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 20 Jan 2024 08:53:47 +0100 Subject: [PATCH 032/119] merge cleanups --- app/Http/Controllers/MailgunController.php | 26 +++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 2998a3595337..62c9062da222 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -62,13 +62,16 @@ class MailgunController extends BaseController */ public function webhook(Request $request) { - if ($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.mailgun.token')) { + + $input = $request->all(); + + if (\abs(\time() - $request['signature']['timestamp']) > 15) + return response()->json(['message' => 'Success'], 200); + + if(\hash_equals(\hash_hmac('sha256', $input['signature']['timestamp'] . $input['signature']['token'], config('services.mailgun.webhook_signing_key')), $input['signature']['signature'])) ProcessMailgunWebhook::dispatch($request->all())->delay(10); - return response()->json(['message' => 'Success'], 200); - } - - return response()->json(['message' => 'Unauthorized'], 403); + return response()->json(['message' => 'Success.'], 200); } /** @@ -105,17 +108,14 @@ class MailgunController extends BaseController * ), * ) */ - public function webhook(Request $request) + public function inboundWebhook(Request $request) { + if ($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.mailgun.token')) { + ProcessMailgunInboundWebhook::dispatch($request->all())->delay(10); - $input = $request->all(); - - if (\abs(\time() - $request['signature']['timestamp']) > 15) return response()->json(['message' => 'Success'], 200); + } - if(\hash_equals(\hash_hmac('sha256', $input['signature']['timestamp'] . $input['signature']['token'], config('services.mailgun.webhook_signing_key')), $input['signature']['signature'])) - ProcessMailgunWebhook::dispatch($request->all())->delay(10); - - return response()->json(['message' => 'Success.'], 200); + return response()->json(['message' => 'Unauthorized'], 403); } } From 2f4f547eed905d1be5e989d950905e5f966f0c87 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 17 Mar 2024 09:14:19 +0100 Subject: [PATCH 033/119] fixes --- composer.lock | 442 +++++++++--------- ...10951_create_imap_configuration_fields.php | 4 +- routes/api.php | 3 - 3 files changed, 225 insertions(+), 224 deletions(-) diff --git a/composer.lock b/composer.lock index e98d7af09bfc..7f7b39bbd5d0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "afb693368e746cc97da95e958ab309ca", + "content-hash": "63d177f80df79731528c8669d192ded6", "packages": [ { "name": "afosto/yaac", @@ -369,16 +369,16 @@ }, { "name": "amphp/parallel", - "version": "v2.2.6", + "version": "v2.2.7", "source": { "type": "git", "url": "https://github.com/amphp/parallel.git", - "reference": "5aeaad20297507cc754859236720501b54306eba" + "reference": "ffda869c33c30627b6eb5c25f096882d885681dc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/parallel/zipball/5aeaad20297507cc754859236720501b54306eba", - "reference": "5aeaad20297507cc754859236720501b54306eba", + "url": "https://api.github.com/repos/amphp/parallel/zipball/ffda869c33c30627b6eb5c25f096882d885681dc", + "reference": "ffda869c33c30627b6eb5c25f096882d885681dc", "shasum": "" }, "require": { @@ -440,7 +440,7 @@ ], "support": { "issues": "https://github.com/amphp/parallel/issues", - "source": "https://github.com/amphp/parallel/tree/v2.2.6" + "source": "https://github.com/amphp/parallel/tree/v2.2.7" }, "funding": [ { @@ -448,7 +448,7 @@ "type": "github" } ], - "time": "2024-01-07T18:12:13+00:00" + "time": "2024-03-16T16:15:46+00:00" }, { "name": "amphp/parser", @@ -514,16 +514,16 @@ }, { "name": "amphp/pipeline", - "version": "v1.1.0", + "version": "v1.2.0", "source": { "type": "git", "url": "https://github.com/amphp/pipeline.git", - "reference": "8a0ecc281bb0932d6b4a786453aff18c55756e63" + "reference": "f1c2ce35d27ae86ead018adb803eccca7421dd9b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/pipeline/zipball/8a0ecc281bb0932d6b4a786453aff18c55756e63", - "reference": "8a0ecc281bb0932d6b4a786453aff18c55756e63", + "url": "https://api.github.com/repos/amphp/pipeline/zipball/f1c2ce35d27ae86ead018adb803eccca7421dd9b", + "reference": "f1c2ce35d27ae86ead018adb803eccca7421dd9b", "shasum": "" }, "require": { @@ -569,7 +569,7 @@ ], "support": { "issues": "https://github.com/amphp/pipeline/issues", - "source": "https://github.com/amphp/pipeline/tree/v1.1.0" + "source": "https://github.com/amphp/pipeline/tree/v1.2.0" }, "funding": [ { @@ -577,7 +577,7 @@ "type": "github" } ], - "time": "2023-12-23T04:34:28+00:00" + "time": "2024-03-10T14:48:16+00:00" }, { "name": "amphp/process", @@ -791,16 +791,16 @@ }, { "name": "amphp/sync", - "version": "v2.1.0", + "version": "v2.2.0", "source": { "type": "git", "url": "https://github.com/amphp/sync.git", - "reference": "50ddc7392cc8034b3e4798cef3cc90d3f4c0441c" + "reference": "375ef5b54a0d12c38e12728dde05a55e30f2fbec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/sync/zipball/50ddc7392cc8034b3e4798cef3cc90d3f4c0441c", - "reference": "50ddc7392cc8034b3e4798cef3cc90d3f4c0441c", + "url": "https://api.github.com/repos/amphp/sync/zipball/375ef5b54a0d12c38e12728dde05a55e30f2fbec", + "reference": "375ef5b54a0d12c38e12728dde05a55e30f2fbec", "shasum": "" }, "require": { @@ -814,7 +814,7 @@ "amphp/php-cs-fixer-config": "^2", "amphp/phpunit-util": "^3", "phpunit/phpunit": "^9", - "psalm/phar": "^5.4" + "psalm/phar": "5.23" }, "type": "library", "autoload": { @@ -854,7 +854,7 @@ ], "support": { "issues": "https://github.com/amphp/sync/issues", - "source": "https://github.com/amphp/sync/tree/v2.1.0" + "source": "https://github.com/amphp/sync/tree/v2.2.0" }, "funding": [ { @@ -862,7 +862,7 @@ "type": "github" } ], - "time": "2023-08-19T13:53:40+00:00" + "time": "2024-03-12T01:00:01+00:00" }, { "name": "amphp/windows-registry", @@ -1014,16 +1014,16 @@ }, { "name": "apimatic/jsonmapper", - "version": "3.1.2", + "version": "3.1.3", "source": { "type": "git", "url": "https://github.com/apimatic/jsonmapper.git", - "reference": "6673a946c21f2ceeec0cb60d17541c11a22bc79d" + "reference": "5fe6ee7ed1857d6fed669dde935c6c6c70b637d2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/apimatic/jsonmapper/zipball/6673a946c21f2ceeec0cb60d17541c11a22bc79d", - "reference": "6673a946c21f2ceeec0cb60d17541c11a22bc79d", + "url": "https://api.github.com/repos/apimatic/jsonmapper/zipball/5fe6ee7ed1857d6fed669dde935c6c6c70b637d2", + "reference": "5fe6ee7ed1857d6fed669dde935c6c6c70b637d2", "shasum": "" }, "require": { @@ -1062,9 +1062,9 @@ "support": { "email": "mehdi.jaffery@apimatic.io", "issues": "https://github.com/apimatic/jsonmapper/issues", - "source": "https://github.com/apimatic/jsonmapper/tree/3.1.2" + "source": "https://github.com/apimatic/jsonmapper/tree/3.1.3" }, - "time": "2023-06-08T04:27:10+00:00" + "time": "2024-03-15T06:02:44+00:00" }, { "name": "apimatic/unirest-php", @@ -1343,16 +1343,16 @@ }, { "name": "aws/aws-sdk-php", - "version": "3.300.13", + "version": "3.301.1", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "b1eb7307d30ebcfa4e156971f658c2d177434db3" + "reference": "0a910d2b35e7087337cdf3569dc9b6ce232aafba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/b1eb7307d30ebcfa4e156971f658c2d177434db3", - "reference": "b1eb7307d30ebcfa4e156971f658c2d177434db3", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/0a910d2b35e7087337cdf3569dc9b6ce232aafba", + "reference": "0a910d2b35e7087337cdf3569dc9b6ce232aafba", "shasum": "" }, "require": { @@ -1432,9 +1432,9 @@ "support": { "forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80", "issues": "https://github.com/aws/aws-sdk-php/issues", - "source": "https://github.com/aws/aws-sdk-php/tree/3.300.13" + "source": "https://github.com/aws/aws-sdk-php/tree/3.301.1" }, - "time": "2024-03-07T19:14:04+00:00" + "time": "2024-03-15T18:14:42+00:00" }, { "name": "bacon/bacon-qr-code", @@ -1851,28 +1851,28 @@ }, { "name": "composer/ca-bundle", - "version": "1.4.1", + "version": "1.5.0", "source": { "type": "git", "url": "https://github.com/composer/ca-bundle.git", - "reference": "3ce240142f6d59b808dd65c1f52f7a1c252e6cfd" + "reference": "0c5ccfcfea312b5c5a190a21ac5cef93f74baf99" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/ca-bundle/zipball/3ce240142f6d59b808dd65c1f52f7a1c252e6cfd", - "reference": "3ce240142f6d59b808dd65c1f52f7a1c252e6cfd", + "url": "https://api.github.com/repos/composer/ca-bundle/zipball/0c5ccfcfea312b5c5a190a21ac5cef93f74baf99", + "reference": "0c5ccfcfea312b5c5a190a21ac5cef93f74baf99", "shasum": "" }, "require": { "ext-openssl": "*", "ext-pcre": "*", - "php": "^5.3.2 || ^7.0 || ^8.0" + "php": "^7.2 || ^8.0" }, "require-dev": { - "phpstan/phpstan": "^0.12.55", + "phpstan/phpstan": "^1.10", "psr/log": "^1.0", "symfony/phpunit-bridge": "^4.2 || ^5", - "symfony/process": "^2.5 || ^3.0 || ^4.0 || ^5.0 || ^6.0 || ^7.0" + "symfony/process": "^4.0 || ^5.0 || ^6.0 || ^7.0" }, "type": "library", "extra": { @@ -1907,7 +1907,7 @@ "support": { "irc": "irc://irc.freenode.org/composer", "issues": "https://github.com/composer/ca-bundle/issues", - "source": "https://github.com/composer/ca-bundle/tree/1.4.1" + "source": "https://github.com/composer/ca-bundle/tree/1.5.0" }, "funding": [ { @@ -1923,7 +1923,7 @@ "type": "tidelift" } ], - "time": "2024-02-23T10:16:52+00:00" + "time": "2024-03-15T14:00:32+00:00" }, { "name": "dasprid/enum", @@ -2882,16 +2882,16 @@ }, { "name": "endroid/qr-code", - "version": "5.0.6", + "version": "5.0.7", "source": { "type": "git", "url": "https://github.com/endroid/qr-code.git", - "reference": "3a9cc61d2d34df93f6edc2140e7880966ee7860f" + "reference": "0cc00f0626b73bc71a1ea17af01387d0ac75e046" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/endroid/qr-code/zipball/3a9cc61d2d34df93f6edc2140e7880966ee7860f", - "reference": "3a9cc61d2d34df93f6edc2140e7880966ee7860f", + "url": "https://api.github.com/repos/endroid/qr-code/zipball/0cc00f0626b73bc71a1ea17af01387d0ac75e046", + "reference": "0cc00f0626b73bc71a1ea17af01387d0ac75e046", "shasum": "" }, "require": { @@ -2945,7 +2945,7 @@ ], "support": { "issues": "https://github.com/endroid/qr-code/issues", - "source": "https://github.com/endroid/qr-code/tree/5.0.6" + "source": "https://github.com/endroid/qr-code/tree/5.0.7" }, "funding": [ { @@ -2953,7 +2953,7 @@ "type": "github" } ], - "time": "2024-03-06T22:34:02+00:00" + "time": "2024-03-08T11:24:40+00:00" }, { "name": "eway/eway-rapid-php", @@ -3455,16 +3455,16 @@ }, { "name": "google/apiclient-services", - "version": "v0.338.0", + "version": "v0.339.0", "source": { "type": "git", "url": "https://github.com/googleapis/google-api-php-client-services.git", - "reference": "52aeb042c8d30ac0f98f4051dd4bc523708b1306" + "reference": "5662d2ab3da41ac0e0e99db221a8c22c511c8f9c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/52aeb042c8d30ac0f98f4051dd4bc523708b1306", - "reference": "52aeb042c8d30ac0f98f4051dd4bc523708b1306", + "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/5662d2ab3da41ac0e0e99db221a8c22c511c8f9c", + "reference": "5662d2ab3da41ac0e0e99db221a8c22c511c8f9c", "shasum": "" }, "require": { @@ -3493,9 +3493,9 @@ ], "support": { "issues": "https://github.com/googleapis/google-api-php-client-services/issues", - "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.338.0" + "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.339.0" }, - "time": "2024-03-03T00:56:15+00:00" + "time": "2024-03-10T01:06:17+00:00" }, { "name": "google/auth", @@ -4371,16 +4371,16 @@ }, { "name": "horstoeko/zugferd", - "version": "v1.0.34", + "version": "v1.0.36", "source": { "type": "git", "url": "https://github.com/horstoeko/zugferd.git", - "reference": "963b8ab88374e36c056f8ebceb834075be75f0a2" + "reference": "0d15c305328c137365648fe1c34a584d877127fa" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/horstoeko/zugferd/zipball/963b8ab88374e36c056f8ebceb834075be75f0a2", - "reference": "963b8ab88374e36c056f8ebceb834075be75f0a2", + "url": "https://api.github.com/repos/horstoeko/zugferd/zipball/0d15c305328c137365648fe1c34a584d877127fa", + "reference": "0d15c305328c137365648fe1c34a584d877127fa", "shasum": "" }, "require": { @@ -4438,9 +4438,9 @@ ], "support": { "issues": "https://github.com/horstoeko/zugferd/issues", - "source": "https://github.com/horstoeko/zugferd/tree/v1.0.34" + "source": "https://github.com/horstoeko/zugferd/tree/v1.0.36" }, - "time": "2024-01-27T09:14:13+00:00" + "time": "2024-03-11T04:34:59+00:00" }, { "name": "http-interop/http-factory-guzzle", @@ -4937,16 +4937,16 @@ }, { "name": "jean85/pretty-package-versions", - "version": "2.0.5", + "version": "2.0.6", "source": { "type": "git", "url": "https://github.com/Jean85/pretty-package-versions.git", - "reference": "ae547e455a3d8babd07b96966b17d7fd21d9c6af" + "reference": "f9fdd29ad8e6d024f52678b570e5593759b550b4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Jean85/pretty-package-versions/zipball/ae547e455a3d8babd07b96966b17d7fd21d9c6af", - "reference": "ae547e455a3d8babd07b96966b17d7fd21d9c6af", + "url": "https://api.github.com/repos/Jean85/pretty-package-versions/zipball/f9fdd29ad8e6d024f52678b570e5593759b550b4", + "reference": "f9fdd29ad8e6d024f52678b570e5593759b550b4", "shasum": "" }, "require": { @@ -4954,9 +4954,9 @@ "php": "^7.1|^8.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^2.17", + "friendsofphp/php-cs-fixer": "^3.2", "jean85/composer-provided-replaced-stub-package": "^1.0", - "phpstan/phpstan": "^0.12.66", + "phpstan/phpstan": "^1.4", "phpunit/phpunit": "^7.5|^8.5|^9.4", "vimeo/psalm": "^4.3" }, @@ -4990,9 +4990,9 @@ ], "support": { "issues": "https://github.com/Jean85/pretty-package-versions/issues", - "source": "https://github.com/Jean85/pretty-package-versions/tree/2.0.5" + "source": "https://github.com/Jean85/pretty-package-versions/tree/2.0.6" }, - "time": "2021-10-08T21:21:46+00:00" + "time": "2024-03-08T09:58:59+00:00" }, { "name": "jms/metadata", @@ -5371,16 +5371,16 @@ }, { "name": "laravel/framework", - "version": "v10.47.0", + "version": "v10.48.3", "source": { "type": "git", "url": "https://github.com/laravel/framework.git", - "reference": "fce29b8de62733cdecbe12e3bae801f83fff2ea4" + "reference": "5791c052b41c6b593556adc687076bfbdd13c501" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/framework/zipball/fce29b8de62733cdecbe12e3bae801f83fff2ea4", - "reference": "fce29b8de62733cdecbe12e3bae801f83fff2ea4", + "url": "https://api.github.com/repos/laravel/framework/zipball/5791c052b41c6b593556adc687076bfbdd13c501", + "reference": "5791c052b41c6b593556adc687076bfbdd13c501", "shasum": "" }, "require": { @@ -5428,6 +5428,7 @@ "conflict": { "carbonphp/carbon-doctrine-types": ">=3.0", "doctrine/dbal": ">=4.0", + "mockery/mockery": "1.6.8", "phpunit/phpunit": ">=11.0.0", "tightenco/collect": "<5.5.33" }, @@ -5573,7 +5574,7 @@ "issues": "https://github.com/laravel/framework/issues", "source": "https://github.com/laravel/framework" }, - "time": "2024-03-05T15:18:36+00:00" + "time": "2024-03-15T10:17:07+00:00" }, { "name": "laravel/prompts", @@ -6368,16 +6369,16 @@ }, { "name": "league/flysystem", - "version": "3.24.0", + "version": "3.25.1", "source": { "type": "git", "url": "https://github.com/thephpleague/flysystem.git", - "reference": "b25a361508c407563b34fac6f64a8a17a8819675" + "reference": "abbd664eb4381102c559d358420989f835208f18" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/b25a361508c407563b34fac6f64a8a17a8819675", - "reference": "b25a361508c407563b34fac6f64a8a17a8819675", + "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/abbd664eb4381102c559d358420989f835208f18", + "reference": "abbd664eb4381102c559d358420989f835208f18", "shasum": "" }, "require": { @@ -6405,7 +6406,7 @@ "friendsofphp/php-cs-fixer": "^3.5", "google/cloud-storage": "^1.23", "microsoft/azure-storage-blob": "^1.1", - "phpseclib/phpseclib": "^3.0.34", + "phpseclib/phpseclib": "^3.0.36", "phpstan/phpstan": "^1.10", "phpunit/phpunit": "^9.5.11|^10.0", "sabre/dav": "^4.6.0" @@ -6442,7 +6443,7 @@ ], "support": { "issues": "https://github.com/thephpleague/flysystem/issues", - "source": "https://github.com/thephpleague/flysystem/tree/3.24.0" + "source": "https://github.com/thephpleague/flysystem/tree/3.25.1" }, "funding": [ { @@ -6454,20 +6455,20 @@ "type": "github" } ], - "time": "2024-02-04T12:10:17+00:00" + "time": "2024-03-16T12:53:19+00:00" }, { "name": "league/flysystem-aws-s3-v3", - "version": "3.24.0", + "version": "3.25.1", "source": { "type": "git", "url": "https://github.com/thephpleague/flysystem-aws-s3-v3.git", - "reference": "809474e37b7fb1d1f8bcc0f8a98bc1cae99aa513" + "reference": "6a5be0e6d6a93574e80805c9cc108a4b63c824d8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem-aws-s3-v3/zipball/809474e37b7fb1d1f8bcc0f8a98bc1cae99aa513", - "reference": "809474e37b7fb1d1f8bcc0f8a98bc1cae99aa513", + "url": "https://api.github.com/repos/thephpleague/flysystem-aws-s3-v3/zipball/6a5be0e6d6a93574e80805c9cc108a4b63c824d8", + "reference": "6a5be0e6d6a93574e80805c9cc108a4b63c824d8", "shasum": "" }, "require": { @@ -6507,7 +6508,7 @@ "storage" ], "support": { - "source": "https://github.com/thephpleague/flysystem-aws-s3-v3/tree/3.24.0" + "source": "https://github.com/thephpleague/flysystem-aws-s3-v3/tree/3.25.1" }, "funding": [ { @@ -6519,20 +6520,20 @@ "type": "github" } ], - "time": "2024-01-26T18:43:21+00:00" + "time": "2024-03-15T19:58:44+00:00" }, { "name": "league/flysystem-local", - "version": "3.23.1", + "version": "3.25.1", "source": { "type": "git", "url": "https://github.com/thephpleague/flysystem-local.git", - "reference": "b884d2bf9b53bb4804a56d2df4902bb51e253f00" + "reference": "61a6a90d6e999e4ddd9ce5adb356de0939060b92" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem-local/zipball/b884d2bf9b53bb4804a56d2df4902bb51e253f00", - "reference": "b884d2bf9b53bb4804a56d2df4902bb51e253f00", + "url": "https://api.github.com/repos/thephpleague/flysystem-local/zipball/61a6a90d6e999e4ddd9ce5adb356de0939060b92", + "reference": "61a6a90d6e999e4ddd9ce5adb356de0939060b92", "shasum": "" }, "require": { @@ -6566,8 +6567,7 @@ "local" ], "support": { - "issues": "https://github.com/thephpleague/flysystem-local/issues", - "source": "https://github.com/thephpleague/flysystem-local/tree/3.23.1" + "source": "https://github.com/thephpleague/flysystem-local/tree/3.25.1" }, "funding": [ { @@ -6579,7 +6579,7 @@ "type": "github" } ], - "time": "2024-01-26T18:25:23+00:00" + "time": "2024-03-15T19:58:44+00:00" }, { "name": "league/fractal", @@ -7022,16 +7022,16 @@ }, { "name": "livewire/livewire", - "version": "v3.4.7", + "version": "v3.4.9", "source": { "type": "git", "url": "https://github.com/livewire/livewire.git", - "reference": "05f25dab062cd6a1ec24d8df9e889f890c832cb0" + "reference": "c65b3f0798ab2c9338213ede3588c3cdf4e6fcc0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/livewire/livewire/zipball/05f25dab062cd6a1ec24d8df9e889f890c832cb0", - "reference": "05f25dab062cd6a1ec24d8df9e889f890c832cb0", + "url": "https://api.github.com/repos/livewire/livewire/zipball/c65b3f0798ab2c9338213ede3588c3cdf4e6fcc0", + "reference": "c65b3f0798ab2c9338213ede3588c3cdf4e6fcc0", "shasum": "" }, "require": { @@ -7085,7 +7085,7 @@ "description": "A front-end framework for Laravel.", "support": { "issues": "https://github.com/livewire/livewire/issues", - "source": "https://github.com/livewire/livewire/tree/v3.4.7" + "source": "https://github.com/livewire/livewire/tree/v3.4.9" }, "funding": [ { @@ -7093,7 +7093,7 @@ "type": "github" } ], - "time": "2024-03-05T15:54:03+00:00" + "time": "2024-03-14T14:03:32+00:00" }, { "name": "maennchen/zipstream-php", @@ -8435,16 +8435,16 @@ }, { "name": "omnipay/common", - "version": "v3.2.1", + "version": "v3.3.0", "source": { "type": "git", "url": "https://github.com/thephpleague/omnipay-common.git", - "reference": "80545e9f4faab0efad36cc5f1e11a184dda22baf" + "reference": "2eca3823e9069e2c36b6007a090577d5584f9518" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/omnipay-common/zipball/80545e9f4faab0efad36cc5f1e11a184dda22baf", - "reference": "80545e9f4faab0efad36cc5f1e11a184dda22baf", + "url": "https://api.github.com/repos/thephpleague/omnipay-common/zipball/2eca3823e9069e2c36b6007a090577d5584f9518", + "reference": "2eca3823e9069e2c36b6007a090577d5584f9518", "shasum": "" }, "require": { @@ -8454,13 +8454,14 @@ "php-http/discovery": "^1.14", "php-http/message": "^1.5", "php-http/message-factory": "^1.1", - "symfony/http-foundation": "^2.1|^3|^4|^5|^6" + "symfony/http-foundation": "^2.1|^3|^4|^5|^6|^7" }, "require-dev": { + "http-interop/http-factory-guzzle": "^1.1", "omnipay/tests": "^4.1", "php-http/guzzle7-adapter": "^1", - "php-http/mock-client": "^1", - "squizlabs/php_codesniffer": "^3.5" + "php-http/mock-client": "^1.6", + "squizlabs/php_codesniffer": "^3.8.1" }, "suggest": { "league/omnipay": "The default Omnipay package provides a default HTTP Adapter." @@ -8516,7 +8517,7 @@ ], "support": { "issues": "https://github.com/thephpleague/omnipay-common/issues", - "source": "https://github.com/thephpleague/omnipay-common/tree/v3.2.1" + "source": "https://github.com/thephpleague/omnipay-common/tree/v3.3.0" }, "funding": [ { @@ -8524,7 +8525,7 @@ "type": "github" } ], - "time": "2023-05-30T12:44:03+00:00" + "time": "2024-03-08T11:56:40+00:00" }, { "name": "omnipay/paypal", @@ -9295,16 +9296,16 @@ }, { "name": "php-http/promise", - "version": "1.3.0", + "version": "1.3.1", "source": { "type": "git", "url": "https://github.com/php-http/promise.git", - "reference": "2916a606d3b390f4e9e8e2b8dd68581508be0f07" + "reference": "fc85b1fba37c169a69a07ef0d5a8075770cc1f83" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-http/promise/zipball/2916a606d3b390f4e9e8e2b8dd68581508be0f07", - "reference": "2916a606d3b390f4e9e8e2b8dd68581508be0f07", + "url": "https://api.github.com/repos/php-http/promise/zipball/fc85b1fba37c169a69a07ef0d5a8075770cc1f83", + "reference": "fc85b1fba37c169a69a07ef0d5a8075770cc1f83", "shasum": "" }, "require": { @@ -9341,9 +9342,9 @@ ], "support": { "issues": "https://github.com/php-http/promise/issues", - "source": "https://github.com/php-http/promise/tree/1.3.0" + "source": "https://github.com/php-http/promise/tree/1.3.1" }, - "time": "2024-01-04T18:49:48+00:00" + "time": "2024-03-15T13:55:21+00:00" }, { "name": "php-jsonpointer/php-jsonpointer", @@ -10425,16 +10426,16 @@ }, { "name": "psy/psysh", - "version": "v0.12.0", + "version": "v0.12.2", "source": { "type": "git", "url": "https://github.com/bobthecow/psysh.git", - "reference": "750bf031a48fd07c673dbe3f11f72362ea306d0d" + "reference": "9185c66c2165bbf4d71de78a69dccf4974f9538d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/bobthecow/psysh/zipball/750bf031a48fd07c673dbe3f11f72362ea306d0d", - "reference": "750bf031a48fd07c673dbe3f11f72362ea306d0d", + "url": "https://api.github.com/repos/bobthecow/psysh/zipball/9185c66c2165bbf4d71de78a69dccf4974f9538d", + "reference": "9185c66c2165bbf4d71de78a69dccf4974f9538d", "shasum": "" }, "require": { @@ -10498,9 +10499,9 @@ ], "support": { "issues": "https://github.com/bobthecow/psysh/issues", - "source": "https://github.com/bobthecow/psysh/tree/v0.12.0" + "source": "https://github.com/bobthecow/psysh/tree/v0.12.2" }, - "time": "2023-12-20T15:28:09+00:00" + "time": "2024-03-17T01:53:00+00:00" }, { "name": "pusher/pusher-php-server", @@ -11965,16 +11966,16 @@ }, { "name": "spatie/php-structure-discoverer", - "version": "2.1.0", + "version": "2.1.1", "source": { "type": "git", "url": "https://github.com/spatie/php-structure-discoverer.git", - "reference": "f5b3c935dda89d6c382b27e3caf348fa80bcfa88" + "reference": "24f5221641560ec0f7dce23dd814e7d555b0098b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/spatie/php-structure-discoverer/zipball/f5b3c935dda89d6c382b27e3caf348fa80bcfa88", - "reference": "f5b3c935dda89d6c382b27e3caf348fa80bcfa88", + "url": "https://api.github.com/repos/spatie/php-structure-discoverer/zipball/24f5221641560ec0f7dce23dd814e7d555b0098b", + "reference": "24f5221641560ec0f7dce23dd814e7d555b0098b", "shasum": "" }, "require": { @@ -12033,7 +12034,7 @@ ], "support": { "issues": "https://github.com/spatie/php-structure-discoverer/issues", - "source": "https://github.com/spatie/php-structure-discoverer/tree/2.1.0" + "source": "https://github.com/spatie/php-structure-discoverer/tree/2.1.1" }, "funding": [ { @@ -12041,7 +12042,7 @@ "type": "github" } ], - "time": "2024-02-16T12:42:24+00:00" + "time": "2024-03-13T16:08:30+00:00" }, { "name": "sprain/swiss-qr-bill", @@ -12323,16 +12324,16 @@ }, { "name": "symfony/css-selector", - "version": "v7.0.0", + "version": "v7.0.3", "source": { "type": "git", "url": "https://github.com/symfony/css-selector.git", - "reference": "bb51d46e53ef8d50d523f0c5faedba056a27943e" + "reference": "ec60a4edf94e63b0556b6a0888548bb400a3a3be" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/css-selector/zipball/bb51d46e53ef8d50d523f0c5faedba056a27943e", - "reference": "bb51d46e53ef8d50d523f0c5faedba056a27943e", + "url": "https://api.github.com/repos/symfony/css-selector/zipball/ec60a4edf94e63b0556b6a0888548bb400a3a3be", + "reference": "ec60a4edf94e63b0556b6a0888548bb400a3a3be", "shasum": "" }, "require": { @@ -12368,7 +12369,7 @@ "description": "Converts CSS selectors to XPath expressions", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/css-selector/tree/v7.0.0" + "source": "https://github.com/symfony/css-selector/tree/v7.0.3" }, "funding": [ { @@ -12384,7 +12385,7 @@ "type": "tidelift" } ], - "time": "2023-10-31T17:59:56+00:00" + "time": "2024-01-23T15:02:46+00:00" }, { "name": "symfony/deprecation-contracts", @@ -12530,16 +12531,16 @@ }, { "name": "symfony/event-dispatcher", - "version": "v7.0.2", + "version": "v7.0.3", "source": { "type": "git", "url": "https://github.com/symfony/event-dispatcher.git", - "reference": "098b62ae81fdd6cbf941f355059f617db28f4f9a" + "reference": "834c28d533dd0636f910909d01b9ff45cc094b5e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/098b62ae81fdd6cbf941f355059f617db28f4f9a", - "reference": "098b62ae81fdd6cbf941f355059f617db28f4f9a", + "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/834c28d533dd0636f910909d01b9ff45cc094b5e", + "reference": "834c28d533dd0636f910909d01b9ff45cc094b5e", "shasum": "" }, "require": { @@ -12590,7 +12591,7 @@ "description": "Provides tools that allow your application components to communicate with each other by dispatching events and listening to them", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/event-dispatcher/tree/v7.0.2" + "source": "https://github.com/symfony/event-dispatcher/tree/v7.0.3" }, "funding": [ { @@ -12606,7 +12607,7 @@ "type": "tidelift" } ], - "time": "2023-12-27T22:24:19+00:00" + "time": "2024-01-23T15:02:46+00:00" }, { "name": "symfony/event-dispatcher-contracts", @@ -12813,16 +12814,16 @@ }, { "name": "symfony/http-client", - "version": "v6.4.5", + "version": "v7.0.5", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "f3c86a60a3615f466333a11fd42010d4382a82c7" + "reference": "425f462a59d8030703ee04a9e1c666575ed5db3b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/f3c86a60a3615f466333a11fd42010d4382a82c7", - "reference": "f3c86a60a3615f466333a11fd42010d4382a82c7", + "url": "https://api.github.com/repos/symfony/http-client/zipball/425f462a59d8030703ee04a9e1c666575ed5db3b", + "reference": "425f462a59d8030703ee04a9e1c666575ed5db3b", "shasum": "" }, "require": { @@ -12885,7 +12886,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v6.4.5" + "source": "https://github.com/symfony/http-client/tree/v7.0.5" }, "funding": [ { @@ -12901,7 +12902,7 @@ "type": "tidelift" } ], - "time": "2024-03-02T12:45:30+00:00" + "time": "2024-03-02T12:46:12+00:00" }, { "name": "symfony/http-client-contracts", @@ -13173,16 +13174,16 @@ }, { "name": "symfony/intl", - "version": "v7.0.2", + "version": "v7.0.3", "source": { "type": "git", "url": "https://github.com/symfony/intl.git", - "reference": "5fbee19d24354bbd77b300971eb38469ddbfd7fc" + "reference": "295995df4acf6790a35b9ce6ec32b313efb11ff8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/intl/zipball/5fbee19d24354bbd77b300971eb38469ddbfd7fc", - "reference": "5fbee19d24354bbd77b300971eb38469ddbfd7fc", + "url": "https://api.github.com/repos/symfony/intl/zipball/295995df4acf6790a35b9ce6ec32b313efb11ff8", + "reference": "295995df4acf6790a35b9ce6ec32b313efb11ff8", "shasum": "" }, "require": { @@ -13235,7 +13236,7 @@ "localization" ], "support": { - "source": "https://github.com/symfony/intl/tree/v7.0.2" + "source": "https://github.com/symfony/intl/tree/v7.0.3" }, "funding": [ { @@ -13251,7 +13252,7 @@ "type": "tidelift" } ], - "time": "2023-12-27T08:42:13+00:00" + "time": "2024-01-23T15:02:46+00:00" }, { "name": "symfony/mailer", @@ -14735,16 +14736,16 @@ }, { "name": "symfony/string", - "version": "v6.4.4", + "version": "v7.0.4", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "4e465a95bdc32f49cf4c7f07f751b843bbd6dcd9" + "reference": "f5832521b998b0bec40bee688ad5de98d4cf111b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/4e465a95bdc32f49cf4c7f07f751b843bbd6dcd9", - "reference": "4e465a95bdc32f49cf4c7f07f751b843bbd6dcd9", + "url": "https://api.github.com/repos/symfony/string/zipball/f5832521b998b0bec40bee688ad5de98d4cf111b", + "reference": "f5832521b998b0bec40bee688ad5de98d4cf111b", "shasum": "" }, "require": { @@ -14801,7 +14802,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v6.4.4" + "source": "https://github.com/symfony/string/tree/v7.0.4" }, "funding": [ { @@ -14817,7 +14818,7 @@ "type": "tidelift" } ], - "time": "2024-02-01T13:16:41+00:00" + "time": "2024-02-01T13:17:36+00:00" }, { "name": "symfony/translation", @@ -15575,7 +15576,7 @@ "version": "6.44.4", "source": { "type": "git", - "url": "git@github.com:twilio/twilio-php.git", + "url": "https://github.com/twilio/twilio-php.git", "reference": "08aad5f377e2245b9cd7508e7762d95e7392fa4d" }, "dist": { @@ -15617,6 +15618,10 @@ "sms", "twilio" ], + "support": { + "issues": "https://github.com/twilio/twilio-php/issues", + "source": "https://github.com/twilio/twilio-php/tree/6.44.4" + }, "time": "2023-02-22T19:59:53+00:00" }, { @@ -15995,23 +16000,23 @@ "packages-dev": [ { "name": "barryvdh/laravel-debugbar", - "version": "v3.10.6", + "version": "v3.12.2", "source": { "type": "git", "url": "https://github.com/barryvdh/laravel-debugbar.git", - "reference": "1fcb37307ebb32207dce16fa160a92b14d8b671f" + "reference": "43555503052443964ce2c1c1f3b0378e58219eb8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/barryvdh/laravel-debugbar/zipball/1fcb37307ebb32207dce16fa160a92b14d8b671f", - "reference": "1fcb37307ebb32207dce16fa160a92b14d8b671f", + "url": "https://api.github.com/repos/barryvdh/laravel-debugbar/zipball/43555503052443964ce2c1c1f3b0378e58219eb8", + "reference": "43555503052443964ce2c1c1f3b0378e58219eb8", "shasum": "" }, "require": { "illuminate/routing": "^9|^10|^11", "illuminate/session": "^9|^10|^11", "illuminate/support": "^9|^10|^11", - "maximebf/debugbar": "~1.20.1", + "maximebf/debugbar": "~1.21.0", "php": "^8.0", "symfony/finder": "^6|^7" }, @@ -16063,7 +16068,7 @@ ], "support": { "issues": "https://github.com/barryvdh/laravel-debugbar/issues", - "source": "https://github.com/barryvdh/laravel-debugbar/tree/v3.10.6" + "source": "https://github.com/barryvdh/laravel-debugbar/tree/v3.12.2" }, "funding": [ { @@ -16075,7 +16080,7 @@ "type": "github" } ], - "time": "2024-03-01T14:41:13+00:00" + "time": "2024-03-13T09:50:34+00:00" }, { "name": "barryvdh/laravel-ide-helper", @@ -16285,16 +16290,16 @@ }, { "name": "brianium/paratest", - "version": "v7.3.1", + "version": "v7.4.3", "source": { "type": "git", "url": "https://github.com/paratestphp/paratest.git", - "reference": "551f46f52a93177d873f3be08a1649ae886b4a30" + "reference": "64fcfd0e28a6b8078a19dbf9127be2ee645b92ec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/paratestphp/paratest/zipball/551f46f52a93177d873f3be08a1649ae886b4a30", - "reference": "551f46f52a93177d873f3be08a1649ae886b4a30", + "url": "https://api.github.com/repos/paratestphp/paratest/zipball/64fcfd0e28a6b8078a19dbf9127be2ee645b92ec", + "reference": "64fcfd0e28a6b8078a19dbf9127be2ee645b92ec", "shasum": "" }, "require": { @@ -16302,28 +16307,27 @@ "ext-pcre": "*", "ext-reflection": "*", "ext-simplexml": "*", - "fidry/cpu-core-counter": "^0.5.1 || ^1.0.0", + "fidry/cpu-core-counter": "^1.1.0", "jean85/pretty-package-versions": "^2.0.5", - "php": "~8.1.0 || ~8.2.0 || ~8.3.0", - "phpunit/php-code-coverage": "^10.1.7", - "phpunit/php-file-iterator": "^4.1.0", - "phpunit/php-timer": "^6.0", - "phpunit/phpunit": "^10.4.2", - "sebastian/environment": "^6.0.1", - "symfony/console": "^6.3.4 || ^7.0.0", - "symfony/process": "^6.3.4 || ^7.0.0" + "php": "~8.2.0 || ~8.3.0", + "phpunit/php-code-coverage": "^10.1.11 || ^11.0.0", + "phpunit/php-file-iterator": "^4.1.0 || ^5.0.0", + "phpunit/php-timer": "^6.0.0 || ^7.0.0", + "phpunit/phpunit": "^10.5.9 || ^11.0.3", + "sebastian/environment": "^6.0.1 || ^7.0.0", + "symfony/console": "^6.4.3 || ^7.0.3", + "symfony/process": "^6.4.3 || ^7.0.3" }, "require-dev": { "doctrine/coding-standard": "^12.0.0", "ext-pcov": "*", "ext-posix": "*", - "infection/infection": "^0.27.6", - "phpstan/phpstan": "^1.10.40", + "phpstan/phpstan": "^1.10.58", "phpstan/phpstan-deprecation-rules": "^1.1.4", "phpstan/phpstan-phpunit": "^1.3.15", "phpstan/phpstan-strict-rules": "^1.5.2", - "squizlabs/php_codesniffer": "^3.7.2", - "symfony/filesystem": "^6.3.1 || ^7.0.0" + "squizlabs/php_codesniffer": "^3.9.0", + "symfony/filesystem": "^6.4.3 || ^7.0.3" }, "bin": [ "bin/paratest", @@ -16364,7 +16368,7 @@ ], "support": { "issues": "https://github.com/paratestphp/paratest/issues", - "source": "https://github.com/paratestphp/paratest/tree/v7.3.1" + "source": "https://github.com/paratestphp/paratest/tree/v7.4.3" }, "funding": [ { @@ -16376,20 +16380,20 @@ "type": "paypal" } ], - "time": "2023-10-31T09:24:17+00:00" + "time": "2024-02-20T07:24:02+00:00" }, { "name": "composer/class-map-generator", - "version": "1.1.0", + "version": "1.1.1", "source": { "type": "git", "url": "https://github.com/composer/class-map-generator.git", - "reference": "953cc4ea32e0c31f2185549c7d216d7921f03da9" + "reference": "8286a62d243312ed99b3eee20d5005c961adb311" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/class-map-generator/zipball/953cc4ea32e0c31f2185549c7d216d7921f03da9", - "reference": "953cc4ea32e0c31f2185549c7d216d7921f03da9", + "url": "https://api.github.com/repos/composer/class-map-generator/zipball/8286a62d243312ed99b3eee20d5005c961adb311", + "reference": "8286a62d243312ed99b3eee20d5005c961adb311", "shasum": "" }, "require": { @@ -16433,7 +16437,7 @@ ], "support": { "issues": "https://github.com/composer/class-map-generator/issues", - "source": "https://github.com/composer/class-map-generator/tree/1.1.0" + "source": "https://github.com/composer/class-map-generator/tree/1.1.1" }, "funding": [ { @@ -16449,7 +16453,7 @@ "type": "tidelift" } ], - "time": "2023-06-30T13:58:57+00:00" + "time": "2024-03-15T12:53:41+00:00" }, { "name": "composer/pcre", @@ -17107,16 +17111,16 @@ }, { "name": "maximebf/debugbar", - "version": "v1.20.2", + "version": "v1.21.3", "source": { "type": "git", "url": "https://github.com/maximebf/php-debugbar.git", - "reference": "484625c23a4fa4f303617f29fcacd42951c9c01d" + "reference": "0b407703b08ea0cf6ebc61e267cc96ff7000911b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/maximebf/php-debugbar/zipball/484625c23a4fa4f303617f29fcacd42951c9c01d", - "reference": "484625c23a4fa4f303617f29fcacd42951c9c01d", + "url": "https://api.github.com/repos/maximebf/php-debugbar/zipball/0b407703b08ea0cf6ebc61e267cc96ff7000911b", + "reference": "0b407703b08ea0cf6ebc61e267cc96ff7000911b", "shasum": "" }, "require": { @@ -17136,7 +17140,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "1.20-dev" + "dev-master": "1.21-dev" } }, "autoload": { @@ -17167,13 +17171,13 @@ ], "support": { "issues": "https://github.com/maximebf/php-debugbar/issues", - "source": "https://github.com/maximebf/php-debugbar/tree/v1.20.2" + "source": "https://github.com/maximebf/php-debugbar/tree/v1.21.3" }, - "time": "2024-02-15T10:49:09+00:00" + "time": "2024-03-12T14:23:07+00:00" }, { "name": "mockery/mockery", - "version": "1.6.7", + "version": "1.6.9", "source": { "type": "git", "url": "https://github.com/mockery/mockery.git", @@ -17617,16 +17621,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.10.60", + "version": "1.10.62", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "95dcea7d6c628a3f2f56d091d8a0219485a86bbe" + "reference": "cd5c8a1660ed3540b211407c77abf4af193a6af9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/95dcea7d6c628a3f2f56d091d8a0219485a86bbe", - "reference": "95dcea7d6c628a3f2f56d091d8a0219485a86bbe", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/cd5c8a1660ed3540b211407c77abf4af193a6af9", + "reference": "cd5c8a1660ed3540b211407c77abf4af193a6af9", "shasum": "" }, "require": { @@ -17675,20 +17679,20 @@ "type": "tidelift" } ], - "time": "2024-03-07T13:30:19+00:00" + "time": "2024-03-13T12:27:20+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "10.1.12", + "version": "10.1.14", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "842f72662d6b9edda84c4b6f13885fd9cd53dc63" + "reference": "e3f51450ebffe8e0efdf7346ae966a656f7d5e5b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/842f72662d6b9edda84c4b6f13885fd9cd53dc63", - "reference": "842f72662d6b9edda84c4b6f13885fd9cd53dc63", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/e3f51450ebffe8e0efdf7346ae966a656f7d5e5b", + "reference": "e3f51450ebffe8e0efdf7346ae966a656f7d5e5b", "shasum": "" }, "require": { @@ -17745,7 +17749,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/10.1.12" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/10.1.14" }, "funding": [ { @@ -17753,7 +17757,7 @@ "type": "github" } ], - "time": "2024-03-02T07:22:05+00:00" + "time": "2024-03-12T15:33:41+00:00" }, { "name": "phpunit/php-file-iterator", @@ -18000,16 +18004,16 @@ }, { "name": "phpunit/phpunit", - "version": "10.5.11", + "version": "10.5.13", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "0d968f6323deb3dbfeba5bfd4929b9415eb7a9a4" + "reference": "20a63fc1c6db29b15da3bd02d4b6cf59900088a7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/0d968f6323deb3dbfeba5bfd4929b9415eb7a9a4", - "reference": "0d968f6323deb3dbfeba5bfd4929b9415eb7a9a4", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/20a63fc1c6db29b15da3bd02d4b6cf59900088a7", + "reference": "20a63fc1c6db29b15da3bd02d4b6cf59900088a7", "shasum": "" }, "require": { @@ -18081,7 +18085,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.11" + "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.13" }, "funding": [ { @@ -18097,7 +18101,7 @@ "type": "tidelift" } ], - "time": "2024-02-25T14:05:00+00:00" + "time": "2024-03-12T15:37:41+00:00" }, { "name": "sebastian/cli-parser", @@ -19461,16 +19465,16 @@ }, { "name": "symfony/stopwatch", - "version": "v7.0.0", + "version": "v7.0.3", "source": { "type": "git", "url": "https://github.com/symfony/stopwatch.git", - "reference": "7bbfa3dd564a0ce12eb4acaaa46823c740f9cb7a" + "reference": "983900d6fddf2b0cbaacacbbad07610854bd8112" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/stopwatch/zipball/7bbfa3dd564a0ce12eb4acaaa46823c740f9cb7a", - "reference": "7bbfa3dd564a0ce12eb4acaaa46823c740f9cb7a", + "url": "https://api.github.com/repos/symfony/stopwatch/zipball/983900d6fddf2b0cbaacacbbad07610854bd8112", + "reference": "983900d6fddf2b0cbaacacbbad07610854bd8112", "shasum": "" }, "require": { @@ -19503,7 +19507,7 @@ "description": "Provides a way to profile code", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/stopwatch/tree/v7.0.0" + "source": "https://github.com/symfony/stopwatch/tree/v7.0.3" }, "funding": [ { @@ -19519,7 +19523,7 @@ "type": "tidelift" } ], - "time": "2023-07-05T13:06:06+00:00" + "time": "2024-01-23T15:02:46+00:00" }, { "name": "theseer/tokenizer", diff --git a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php index 2813be78ba60..5c4937ff1bb2 100644 --- a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php +++ b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php @@ -11,7 +11,7 @@ return new class extends Migration { */ public function up(): void { - Schema::table('company', function (Blueprint $table) { + Schema::table('companies', function (Blueprint $table) { $table->boolean("expense_mailbox_active")->default(true); $table->string("expense_mailbox")->nullable(); $table->boolean("expense_mailbox_allow_company_users")->default(false); @@ -20,7 +20,7 @@ return new class extends Migration { $table->string("expense_mailbox_whitelist_domains")->nullable(); $table->string("expense_mailbox_whitelist_emails")->nullable(); }); - Schema::table('vendor', function (Blueprint $table) { + Schema::table('vendors', function (Blueprint $table) { $table->string("invoicing_email")->nullable(); $table->string("invoicing_domain")->nullable(); }); diff --git a/routes/api.php b/routes/api.php index e98f352b29de..9a45cd829754 100644 --- a/routes/api.php +++ b/routes/api.php @@ -37,7 +37,6 @@ use App\Http\Controllers\CompanyController; use App\Http\Controllers\ExpenseController; use App\Http\Controllers\InvoiceController; use App\Http\Controllers\LicenseController; -use App\Http\Controllers\LogoutController; use App\Http\Controllers\MailgunController; use App\Http\Controllers\MigrationController; use App\Http\Controllers\OneTimeTokenController; @@ -52,7 +51,6 @@ use App\Http\Controllers\ActivityController; use App\Http\Controllers\DocumentController; use App\Http\Controllers\PostMarkController; use App\Http\Controllers\TemplateController; -use App\Http\Controllers\MigrationController; use App\Http\Controllers\SchedulerController; use App\Http\Controllers\SubdomainController; use App\Http\Controllers\SystemLogController; @@ -67,7 +65,6 @@ use App\Http\Controllers\PaymentTermController; use App\PaymentDrivers\PayPalPPCPPaymentDriver; use App\Http\Controllers\EmailHistoryController; use App\Http\Controllers\GroupSettingController; -use App\Http\Controllers\OneTimeTokenController; use App\Http\Controllers\SubscriptionController; use App\Http\Controllers\Bank\NordigenController; use App\Http\Controllers\CompanyLedgerController; From c80e3bf9215d5c7ff1506b484adb31f181ade1ab Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 18 Mar 2024 08:04:54 +0100 Subject: [PATCH 034/119] working mailgun inbound webhook --- .../MailgunInboundWebhookTransformer.php | 21 +++++-- app/Http/Controllers/MailgunController.php | 19 +++++- .../Mailgun/ProcessMailgunInboundWebhook.php | 61 ++++++------------- app/Services/IngresEmail/IngresEmail.php | 14 +---- .../IngresEmail/IngresEmailEngine.php | 40 ++++-------- app/Utils/TempFile.php | 29 +++++++++ routes/api.php | 2 +- 7 files changed, 95 insertions(+), 91 deletions(-) diff --git a/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php b/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php index 6871e3010168..cd50f38dfd98 100644 --- a/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php +++ b/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php @@ -13,6 +13,7 @@ namespace App\Helpers\IngresMail\Transformer; use App\Services\IngresEmail\IngresEmail; use App\Utils\TempFile; +use Illuminate\Support\Carbon; class MailgunInboundWebhookTransformer { @@ -20,15 +21,25 @@ class MailgunInboundWebhookTransformer { $ingresEmail = new IngresEmail(); - $ingresEmail->from = $data["sender"]; - $ingresEmail->subject = $data["subject"]; + $ingresEmail->from = $data["From"]; + $ingresEmail->to = $data["To"]; + $ingresEmail->subject = $data["Subject"]; $ingresEmail->plain_message = $data["body-plain"]; $ingresEmail->html_message = $data["body-html"]; - $ingresEmail->date = now(); // TODO + $ingresEmail->date = Carbon::createFromTimestamp((int) $data["timestamp"]); // parse documents as UploadedFile from webhook-data - foreach ($data["Attachments"] as $attachment) { - $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]); + foreach (json_decode($data["attachments"]) as $attachment) { + + // prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 + $url = $attachment->url; + $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + + // download file and save to tmp dir + $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + } return $ingresEmail; diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 1c5fd9dbcc0e..c4e2999ea227 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -14,6 +14,7 @@ namespace App\Http\Controllers; use App\Jobs\Mailgun\ProcessMailgunInboundWebhook; use App\Jobs\Mailgun\ProcessMailgunWebhook; use Illuminate\Http\Request; +use Log; /** * Class MailgunController. @@ -112,10 +113,22 @@ class MailgunController extends BaseController */ public function inboundWebhook(Request $request) { - if ($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.mailgun.token')) { - ProcessMailgunInboundWebhook::dispatch($request->all())->delay(10); + $input = $request->all(); - return response()->json(['message' => 'Success'], 200); + if (!array_key_exists('attachments', $input) || count(json_decode($input['attachments'])) == 0) { + Log::info('Message ignored because of missing attachments. Please ensure contacting this api-endpoint with a store & notify operation instead of a forward operation'); + return response()->json(['message' => 'Sucess. Soft Fail. Missing Attachments.'], 200); + } + + if (\abs(\time() - (int) $request['timestamp']) > 150) { + Log::info('Message ignored because of request body is too old.'); + return response()->json(['message' => 'Success. Soft Fail. Message too old.'], 200); + } + + if (\hash_equals(\hash_hmac('sha256', $input['timestamp'] . $input['token'], config('services.mailgun.webhook_signing_key')), $input['signature'])) { + ProcessMailgunInboundWebhook::dispatch($input)->delay(10); + + return response()->json(['message' => 'Success'], 201); } return response()->json(['message' => 'Unauthorized'], 403); diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index 603814263f38..b8a9ffc578d2 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -9,16 +9,17 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Jobs\PostMark; +namespace App\Jobs\Mailgun; -use App\Helpers\Mail\Webhook\Maigun\MailgunWebhookHandler; +use App\Helpers\IngresMail\Transformer\MailgunInboundWebhookTransformer; use App\Libraries\MultiDB; -use App\Models\SystemLog; +use App\Services\IngresEmail\IngresEmailEngine; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Log; class ProcessMailgunInboundWebhook implements ShouldQueue { @@ -26,18 +27,6 @@ class ProcessMailgunInboundWebhook implements ShouldQueue public $tries = 1; - public $invitation; - - private $entity; - - private array $default_response = [ - 'recipients' => '', - 'subject' => 'Message not found.', - 'entity' => '', - 'entity_id' => '', - 'events' => [], - ]; - /** * Create a new job instance. * @@ -46,23 +35,6 @@ class ProcessMailgunInboundWebhook implements ShouldQueue { } - private function getSystemLog(string $message_id): ?SystemLog - { - return SystemLog::query() - ->where('company_id', $this->invitation->company_id) - ->where('type_id', SystemLog::TYPE_WEBHOOK_RESPONSE) - ->whereJsonContains('log', ['MessageID' => $message_id]) - ->orderBy('id', 'desc') - ->first(); - - } - - private function updateSystemLog(SystemLog $system_log, array $data): void - { - $system_log->log = $data; - $system_log->save(); - } - /** * Execute the job. * @@ -71,20 +43,21 @@ class ProcessMailgunInboundWebhook implements ShouldQueue */ public function handle() { - // match companies - if (array_key_exists('ToFull', $this->request)) + if (!array_key_exists('To', $this->request) || !array_key_exists('attachments', $this->request) || !array_key_exists('timestamp', $this->request) || !array_key_exists('Subject', $this->request) || !(array_key_exists('body-html', $this->request) || array_key_exists('body-plain', $this->request))) throw new \Exception('invalid body'); - foreach ($this->request['ToFull'] as $toEmailEntry) { - $toEmail = $toEmailEntry['Email']; - - $company = MultiDB::findAndSetDbByExpenseMailbox($toEmail); - if (!$company) { - nlog('unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $toEmail); - continue; - } - - (new MailgunWebhookHandler())->process($this->request); + // match company + $company = MultiDB::findAndSetDbByExpenseMailbox($this->request["To"]); + if (!$company) { + Log::info('unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $this->request["To"]); + return; } + + // prepare + $ingresMail = (new MailgunInboundWebhookTransformer())->transform($this->request); + Log::info(json_encode($ingresMail)); + + // perform + (new IngresEmailEngine($ingresMail))->handle(); } } diff --git a/app/Services/IngresEmail/IngresEmail.php b/app/Services/IngresEmail/IngresEmail.php index 2ec889bcf6e2..728e25a41ab4 100644 --- a/app/Services/IngresEmail/IngresEmail.php +++ b/app/Services/IngresEmail/IngresEmail.php @@ -12,25 +12,17 @@ namespace App\Services\IngresEmail; use Illuminate\Http\UploadedFile; +use Illuminate\Support\Carbon; /** * EmailObject. */ class IngresEmail { - /** @var array[string] $args */ - public array $to = []; + public string $to; public string $from; - public array $reply_to = []; - - /** @var array[string] $args */ - public array $cc = []; - - /** @var array[string] $args */ - public array $bcc = []; - public ?string $subject = null; public ?string $body = null; @@ -41,7 +33,7 @@ class IngresEmail /** @var array[\Illuminate\Http\UploadedFile] $documents */ public array $documents = []; - public ?\DateTimeImmutable $date = null; + public ?Carbon $date = null; function __constructor() { diff --git a/app/Services/IngresEmail/IngresEmailEngine.php b/app/Services/IngresEmail/IngresEmailEngine.php index f89d0886314b..4ac1d9a9fe83 100644 --- a/app/Services/IngresEmail/IngresEmailEngine.php +++ b/app/Services/IngresEmail/IngresEmailEngine.php @@ -14,11 +14,9 @@ namespace App\Services\IngresEmail; use App\Events\Expense\ExpenseWasCreated; use App\Factory\ExpenseFactory; use App\Libraries\MultiDB; -use App\Models\Client; use App\Models\Company; use App\Models\Vendor; use App\Models\VendorContact; -use App\Services\Email\EmailObject; use App\Services\IngresEmail\IngresEmail; use App\Utils\Ninja; use App\Utils\TempFile; @@ -26,24 +24,18 @@ use App\Utils\Traits\GeneratesCounter; use App\Utils\Traits\SavesDocuments; use App\Utils\Traits\MakesHash; use Cache; -use Illuminate\Bus\Queueable; -use Illuminate\Contracts\Queue\ShouldQueue; -use Illuminate\Foundation\Bus\Dispatchable; -use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -class IngresEmailEngine implements ShouldQueue +class IngresEmailEngine { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, MakesHash; + use SerializesModels, MakesHash; use GeneratesCounter, SavesDocuments; - private IngresEmail $email; private ?Company $company; private ?bool $isUnknownRecipent = null; private array $globalBlacklist = []; - function __constructor(IngresEmail $email) + public function __construct(private IngresEmail $email) { - $this->email = $email; } /** * if there is not a company with an matching mailbox, we only do monitoring @@ -53,14 +45,12 @@ class IngresEmailEngine implements ShouldQueue { if ($this->isInvalidOrBlocked()) return; + $this->isUnknownRecipent = true; // Expense Mailbox => will create an expense - foreach ($this->email->to as $expense_mailbox) { - $this->company = MultiDB::findAndSetDbByExpenseMailbox($expense_mailbox); - if (!$this->company) - continue; - + $this->company = MultiDB::findAndSetDbByExpenseMailbox($this->email->to); + if ($this->company) { $this->isUnknownRecipent = false; $this->createExpense(); } @@ -112,13 +102,11 @@ class IngresEmailEngine implements ShouldQueue } // wrong recipent occurs in more than 100 emails in the last 12 hours, so the processing is blocked - foreach ($this->email->to as $recipent) { - $mailCountUnknownRecipent = Cache::get('ingresEmailUnknownRecipent:' . $recipent, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time - if ($mailCountUnknownRecipent >= 100) { - nlog('[IngressMailEngine] E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); - $this->blockSender(); - return true; - } + $mailCountUnknownRecipent = Cache::get('ingresEmailUnknownRecipent:' . $this->email->to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time + if ($mailCountUnknownRecipent >= 100) { + nlog('[IngressMailEngine] E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); + $this->blockSender(); + return true; } return false; @@ -141,10 +129,8 @@ class IngresEmailEngine implements ShouldQueue Cache::add('ingresEmailSenderUnknownRecipent:' . $this->email->from, 0, now()->addHours(6)); Cache::increment('ingresEmailSenderUnknownRecipent:' . $this->email->from); // we save the sender, to may block him - foreach ($this->email->to as $recipent) { - Cache::add('ingresEmailUnknownRecipent:' . $recipent, 0, now()->addHours(12)); - Cache::increment('ingresEmailUnknownRecipent:' . $recipent); // we save the sender, to may block him - } + Cache::add('ingresEmailUnknownRecipent:' . $this->email->to, 0, now()->addHours(12)); + Cache::increment('ingresEmailUnknownRecipent:' . $this->email->to); // we save the sender, to may block him } } diff --git a/app/Utils/TempFile.php b/app/Utils/TempFile.php index 0e8062b457ad..24df20da7271 100644 --- a/app/Utils/TempFile.php +++ b/app/Utils/TempFile.php @@ -99,4 +99,33 @@ class TempFile // return UploadedFile object return $file; } + + /* create a tmp file from a raw string: https://gist.github.com/waska14/8b3bcebfad1f86f7fcd3b82927576e38*/ + public static function UploadedFileFromUrl(string $url, string|null $fileName = null, string|null $mimeType = null): UploadedFile + { + // Create temp file and get its absolute path + $tempFile = tmpfile(); + $tempFilePath = stream_get_meta_data($tempFile)['uri']; + + // Save file data in file + file_put_contents($tempFilePath, file_get_contents($url)); + + $tempFileObject = new File($tempFilePath); + $file = new UploadedFile( + $tempFileObject->getPathname(), + $fileName ?: $tempFileObject->getFilename(), + $mimeType ?: $tempFileObject->getMimeType(), + 0, + true // Mark it as test, since the file isn't from real HTTP POST. + ); + + // Close this file after response is sent. + // Closing the file will cause to remove it from temp director! + app()->terminating(function () use ($tempFile) { + fclose($tempFile); + }); + + // return UploadedFile object + return $file; + } } diff --git a/routes/api.php b/routes/api.php index 9a45cd829754..480f1ce87e6c 100644 --- a/routes/api.php +++ b/routes/api.php @@ -71,7 +71,6 @@ use App\Http\Controllers\CompanyLedgerController; use App\Http\Controllers\PurchaseOrderController; use App\Http\Controllers\TaskSchedulerController; use App\Http\Controllers\CompanyGatewayController; -use App\Http\Controllers\MailgunWebhookController; use App\Http\Controllers\PaymentWebhookController; use App\Http\Controllers\RecurringQuoteController; use App\Http\Controllers\BankIntegrationController; @@ -426,6 +425,7 @@ Route::match(['get', 'post'], 'payment_notification_webhook/{company_key}/{compa Route::post('api/v1/postmark_webhook', [PostMarkController::class, 'webhook'])->middleware('throttle:1000,1'); Route::post('api/v1/mailgun_webhook', [MailgunController::class, 'webhook'])->middleware('throttle:1000,1'); +Route::post('api/v1/mailgun_inbound_webhook', [MailgunController::class, 'inboundWebhook'])->middleware('throttle:1000,1'); Route::get('token_hash_router', [OneTimeTokenController::class, 'router'])->middleware('throttle:500,1'); Route::get('webcron', [WebCronController::class, 'index'])->middleware('throttle:100,1'); Route::post('api/v1/get_migration_account', [HostedMigrationController::class, 'getAccount'])->middleware('guest')->middleware('throttle:100,1'); From 7cc5ff11d4df31445bebdd08e5c97214e77d3267 Mon Sep 17 00:00:00 2001 From: paulwer Date: Tue, 19 Mar 2024 07:39:35 +0100 Subject: [PATCH 035/119] working ingres engine --- .../MailgunInboundWebhookTransformer.php | 8 ++-- app/Services/IngresEmail/IngresEmail.php | 2 +- .../IngresEmail/IngresEmailEngine.php | 43 +++++++++++-------- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php b/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php index cd50f38dfd98..45a261f18d3d 100644 --- a/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php +++ b/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php @@ -21,11 +21,11 @@ class MailgunInboundWebhookTransformer { $ingresEmail = new IngresEmail(); - $ingresEmail->from = $data["From"]; - $ingresEmail->to = $data["To"]; + $ingresEmail->from = $data["sender"]; // TODO: maybe a fallback have to be used to extract email from $data["From"] + $ingresEmail->to = $data["recipient"]; // TODO: maybe a fallback have to be used to extract email from $data["To"] $ingresEmail->subject = $data["Subject"]; - $ingresEmail->plain_message = $data["body-plain"]; - $ingresEmail->html_message = $data["body-html"]; + $ingresEmail->body = $data["body-html"]; + $ingresEmail->text_body = $data["body-plain"]; $ingresEmail->date = Carbon::createFromTimestamp((int) $data["timestamp"]); // parse documents as UploadedFile from webhook-data diff --git a/app/Services/IngresEmail/IngresEmail.php b/app/Services/IngresEmail/IngresEmail.php index 728e25a41ab4..72d30e84e110 100644 --- a/app/Services/IngresEmail/IngresEmail.php +++ b/app/Services/IngresEmail/IngresEmail.php @@ -26,7 +26,7 @@ class IngresEmail public ?string $subject = null; public ?string $body = null; - public ?UploadedFile $body_document; + public ?UploadedFile $body_document = null; public string $text_body; diff --git a/app/Services/IngresEmail/IngresEmailEngine.php b/app/Services/IngresEmail/IngresEmailEngine.php index 4ac1d9a9fe83..f98b213cad94 100644 --- a/app/Services/IngresEmail/IngresEmailEngine.php +++ b/app/Services/IngresEmail/IngresEmailEngine.php @@ -25,6 +25,7 @@ use App\Utils\Traits\SavesDocuments; use App\Utils\Traits\MakesHash; use Cache; use Illuminate\Queue\SerializesModels; +use Log; class IngresEmailEngine { @@ -63,7 +64,7 @@ class IngresEmailEngine { // invalid email if (!filter_var($this->email->from, FILTER_VALIDATE_EMAIL)) { - nlog('[IngressMailEngine] E-Mail blocked, because from e-mail has the wrong format: ' . $this->email->from); + Log::info('[IngressMailEngine] E-Mail blocked, because from e-mail has the wrong format: ' . $this->email->from); return true; } @@ -72,7 +73,7 @@ class IngresEmailEngine // global blacklist if (in_array($domain, $this->globalBlacklist)) { - nlog('[IngressMailEngine] E-Mail blocked, because the domain was found on globalBlocklist: ' . $this->email->from); + Log::info('[IngressMailEngine] E-Mail blocked, because the domain was found on globalBlocklist: ' . $this->email->from); return true; } @@ -83,12 +84,12 @@ class IngresEmailEngine // sender occured in more than 500 emails in the last 12 hours $senderMailCountTotal = Cache::get('ingresEmailSender:' . $this->email->from, 0); if ($senderMailCountTotal >= 5000) { - nlog('[IngressMailEngine] E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); + Log::info('[IngressMailEngine] E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); $this->blockSender(); return true; } if ($senderMailCountTotal >= 1000) { - nlog('[IngressMailEngine] E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); + Log::info('[IngressMailEngine] E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); $this->saveMeta(); return true; } @@ -96,7 +97,7 @@ class IngresEmailEngine // sender sended more than 50 emails to the wrong mailbox in the last 6 hours $senderMailCountUnknownRecipent = Cache::get('ingresEmailSenderUnknownRecipent:' . $this->email->from, 0); if ($senderMailCountUnknownRecipent >= 50) { - nlog('[IngressMailEngine] E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $this->email->from); + Log::info('[IngressMailEngine] E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $this->email->from); $this->saveMeta(); return true; } @@ -104,7 +105,7 @@ class IngresEmailEngine // wrong recipent occurs in more than 100 emails in the last 12 hours, so the processing is blocked $mailCountUnknownRecipent = Cache::get('ingresEmailUnknownRecipent:' . $this->email->to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time if ($mailCountUnknownRecipent >= 100) { - nlog('[IngressMailEngine] E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); + Log::info('[IngressMailEngine] E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); $this->blockSender(); return true; } @@ -137,9 +138,10 @@ class IngresEmailEngine // MAIL-PARSING private function processHtmlBodyToDocument() { - if (!$this->email->body_document && property_exists($this->email, "body")) { + + if ($this->email->body !== null) $this->email->body_document = TempFile::UploadedFileFromRaw($this->email->body, "E-Mail.html", "text/html"); - } + } // MAIN-PROCESSORS @@ -147,15 +149,15 @@ class IngresEmailEngine { // Skipping executions: will not result in not saving Metadata to prevent usage of these conditions, to spam if (!$this->validateExpenseShouldProcess()) { - nlog('email parsing not active for this company: ' . $this->company->id . ' from: ' . $this->email->from); + Log::info('email parsing not active for this company: ' . $this->company->id . ' from: ' . $this->email->from); return; } if (!$this->validateExpenseSender()) { - nlog('invalid sender of an ingest email to company: ' . $this->company->id . ' from: ' . $this->email->from); + Log::info('invalid sender of an ingest email to company: ' . $this->company->id . ' from: ' . $this->email->from); return; } if (sizeOf($this->email->documents) == 0) { - nlog('email does not contain any attachments and is likly not an expense. company: ' . $this->company->id . ' from: ' . $this->email->from); + Log::info('email does not contain any attachments and is likly not an expense. company: ' . $this->company->id . ' from: ' . $this->email->from); return; } @@ -175,12 +177,15 @@ class IngresEmailEngine $this->processHtmlBodyToDocument(); $documents = []; array_push($documents, ...$this->email->documents); - if ($this->email->body_document) - $documents[] = $this->email->body_document; - $this->saveDocuments($documents, $expense); + if ($this->email->body_document !== null) + array_push($documents, $this->email->body_document); $expense->saveQuietly(); + Log::info(json_encode($documents)); + + $this->saveDocuments($documents, $expense); + event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); // @turbo124 please check, I copied from API-Controller event('eloquent.created: App\Models\Expense', $expense); // @turbo124 please check, I copied from API-Controller } @@ -210,9 +215,9 @@ class IngresEmailEngine return true; // from clients/vendors (if active) - if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->where("invoicing_email", $this->email->from)->orWhere($this->email->from, 'LIKE', "CONCAT('%',invoicing_domain)")->exists()) + if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->where("invoicing_email", $this->email->from)->orWhere("invoicing_domain", $domain)->exists()) return true; - if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->contacts()->where("email", $this->email->from)->exists()) // TODO + if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->contacts()->where("email", $this->email->from)->exists()) return true; // denie @@ -220,14 +225,16 @@ class IngresEmailEngine } private function getExpenseVendor() { + $parts = explode('@', $this->email->from); + $domain = array_pop($parts); + $vendor = Vendor::where("company_id", $this->company->id)->where('invoicing_email', $this->email->from)->first(); if ($vendor == null) - $vendor = Vendor::where("company_id", $this->company->id)->where($this->email->from, 'LIKE', "CONCAT('%',invoicing_domain)")->first(); + $vendor = Vendor::where("company_id", $this->company->id)->where("invoicing_domain", $domain)->first(); if ($vendor == null) { $vendorContact = VendorContact::where("company_id", $this->company->id)->where("email", $this->email->from)->first(); $vendor = $vendorContact->vendor(); } - // TODO: from contacts return $vendor; } From a5b3215ff73e9b989b570d201883bda93fff443d Mon Sep 17 00:00:00 2001 From: paulwer Date: Tue, 19 Mar 2024 07:55:55 +0100 Subject: [PATCH 036/119] feat: company blacklist --- app/Models/Company.php | 7 +++++-- app/Services/IngresEmail/IngresEmailEngine.php | 8 ++++++++ ...2023_12_10_110951_create_imap_configuration_fields.php | 2 ++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/app/Models/Company.php b/app/Models/Company.php index 28b61a9d7355..a868093c6238 100644 --- a/app/Models/Company.php +++ b/app/Models/Company.php @@ -118,6 +118,8 @@ use Laracasts\Presenter\PresentableTrait; * @property bool $expense_mailbox_allow_unknown * @property string|null $expense_mailbox_whitelist_domains * @property string|null $expense_mailbox_whitelist_emails + * @property string|null $expense_mailbox_blacklist_domains + * @property string|null $expense_mailbox_blacklist_emails * @property int $deleted_at * @property string $smtp_username * @property string $smtp_password @@ -373,7 +375,8 @@ class Company extends BaseModel 'expense_mailbox_allow_unknown', 'expense_mailbox_whitelist_domains', 'expense_mailbox_whitelist_emails', - 'expense_mailbox_whitelist', + 'expense_mailbox_blacklist_domains', + 'expense_mailbox_blacklist_emails', 'smtp_host', 'smtp_port', 'smtp_encryption', @@ -727,7 +730,7 @@ class Company extends BaseModel public function getLocale() { - return isset($this->settings->language_id) && $this->language() ? $this->language()->locale : config('ninja.i18n.locale'); + return isset ($this->settings->language_id) && $this->language() ? $this->language()->locale : config('ninja.i18n.locale'); } public function getLogo(): ?string diff --git a/app/Services/IngresEmail/IngresEmailEngine.php b/app/Services/IngresEmail/IngresEmailEngine.php index f98b213cad94..42fd8a304719 100644 --- a/app/Services/IngresEmail/IngresEmailEngine.php +++ b/app/Services/IngresEmail/IngresEmailEngine.php @@ -207,6 +207,14 @@ class IngresEmailEngine $domain_whitelist = explode(",", $this->company->expense_mailbox_whitelist_domains); if (in_array($domain, $domain_whitelist)) return true; + $email_blacklist = explode(",", $this->company->expense_mailbox_blacklist_emails); + if (in_array($this->email->from, $email_blacklist)) + return false; + $domain_blacklist = explode(",", $this->company->expense_mailbox_blacklist_domains); + if (in_array($domain, $domain_blacklist)) + return false; + + // allow unknown if ($this->company->expense_mailbox_allow_unknown && sizeOf($email_whitelist) == 0 && sizeOf($domain_whitelist) == 0) // from unknown only, when no whitelists are defined return true; diff --git a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php index 5c4937ff1bb2..5a34ab745f7b 100644 --- a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php +++ b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php @@ -19,6 +19,8 @@ return new class extends Migration { $table->boolean("expense_mailbox_allow_unknown")->default(false); $table->string("expense_mailbox_whitelist_domains")->nullable(); $table->string("expense_mailbox_whitelist_emails")->nullable(); + $table->string("expense_mailbox_blacklist_domains")->nullable(); + $table->string("expense_mailbox_blacklist_emails")->nullable(); }); Schema::table('vendors', function (Blueprint $table) { $table->string("invoicing_email")->nullable(); From 74fca7de2ef474f332b66e0a899cbd46c8ea6f6f Mon Sep 17 00:00:00 2001 From: paulwer Date: Tue, 19 Mar 2024 08:10:18 +0100 Subject: [PATCH 037/119] comment --- app/Http/Controllers/MailgunController.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index c4e2999ea227..0d2e1c91e937 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -125,6 +125,7 @@ class MailgunController extends BaseController return response()->json(['message' => 'Success. Soft Fail. Message too old.'], 200); } + // @turbo124 TODO: how to check for services.mailgun.webhook_signing_key on company level, when custom credentials are defined if (\hash_equals(\hash_hmac('sha256', $input['timestamp'] . $input['token'], config('services.mailgun.webhook_signing_key')), $input['signature'])) { ProcessMailgunInboundWebhook::dispatch($input)->delay(10); From 73bcf928e499f1ae8a902fe1b35c85046ffa49aa Mon Sep 17 00:00:00 2001 From: paulwer Date: Tue, 19 Mar 2024 12:56:39 +0100 Subject: [PATCH 038/119] updates mailgun webhook loading message data on runtime --- .../MailgunInboundWebhookTransformer.php | 47 ----------------- app/Http/Controllers/MailgunController.php | 11 ++-- .../Mailgun/ProcessMailgunInboundWebhook.php | 51 +++++++++++++++---- .../IngresEmail/IngresEmailEngine.php | 2 - 4 files changed, 48 insertions(+), 63 deletions(-) delete mode 100644 app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php diff --git a/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php b/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php deleted file mode 100644 index 45a261f18d3d..000000000000 --- a/app/Helpers/IngresMail/Transformer/MailgunInboundWebhookTransformer.php +++ /dev/null @@ -1,47 +0,0 @@ -from = $data["sender"]; // TODO: maybe a fallback have to be used to extract email from $data["From"] - $ingresEmail->to = $data["recipient"]; // TODO: maybe a fallback have to be used to extract email from $data["To"] - $ingresEmail->subject = $data["Subject"]; - $ingresEmail->body = $data["body-html"]; - $ingresEmail->text_body = $data["body-plain"]; - $ingresEmail->date = Carbon::createFromTimestamp((int) $data["timestamp"]); - - // parse documents as UploadedFile from webhook-data - foreach (json_decode($data["attachments"]) as $attachment) { - - // prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 - $url = $attachment->url; - $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; - $url = str_replace("http://", "http://" . $credentials, $url); - $url = str_replace("https://", "https://" . $credentials, $url); - - // download file and save to tmp dir - $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); - - } - - return $ingresEmail; - } -} diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 0d2e1c91e937..28a257d2e4e1 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -115,19 +115,24 @@ class MailgunController extends BaseController { $input = $request->all(); + if (!array_key_exists('recipient', $input) || !array_key_exists('message-url', $input)) { + Log::info('Failed: Message could not be parsed, because required parameters are missing. Please ensure contacting this api-endpoint with a store & notify operation instead of a forward operation!'); + return response()->json(['message' => 'Failed. Missing Parameters'], 400); + } + if (!array_key_exists('attachments', $input) || count(json_decode($input['attachments'])) == 0) { - Log::info('Message ignored because of missing attachments. Please ensure contacting this api-endpoint with a store & notify operation instead of a forward operation'); + Log::info('Message ignored because of missing attachments. No Actions would have been taken...'); return response()->json(['message' => 'Sucess. Soft Fail. Missing Attachments.'], 200); } - if (\abs(\time() - (int) $request['timestamp']) > 150) { + if (\abs(\time() - (int) $input['timestamp']) > 150) { Log::info('Message ignored because of request body is too old.'); return response()->json(['message' => 'Success. Soft Fail. Message too old.'], 200); } // @turbo124 TODO: how to check for services.mailgun.webhook_signing_key on company level, when custom credentials are defined if (\hash_equals(\hash_hmac('sha256', $input['timestamp'] . $input['token'], config('services.mailgun.webhook_signing_key')), $input['signature'])) { - ProcessMailgunInboundWebhook::dispatch($input)->delay(10); + ProcessMailgunInboundWebhook::dispatch($input["recipient"] . "|" . $input["message-url"])->delay(10); return response()->json(['message' => 'Success'], 201); } diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index b8a9ffc578d2..d616598167e3 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -11,9 +11,11 @@ namespace App\Jobs\Mailgun; -use App\Helpers\IngresMail\Transformer\MailgunInboundWebhookTransformer; use App\Libraries\MultiDB; +use App\Services\IngresEmail\IngresEmail; use App\Services\IngresEmail\IngresEmailEngine; +use App\Utils\TempFile; +use Illuminate\Support\Carbon; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; @@ -29,9 +31,9 @@ class ProcessMailgunInboundWebhook implements ShouldQueue /** * Create a new job instance. - * + * $input consists of 2 informations: recipient|messageUrl */ - public function __construct(private array $request) + public function __construct(private string $input) { } @@ -43,21 +45,48 @@ class ProcessMailgunInboundWebhook implements ShouldQueue */ public function handle() { - if (!array_key_exists('To', $this->request) || !array_key_exists('attachments', $this->request) || !array_key_exists('timestamp', $this->request) || !array_key_exists('Subject', $this->request) || !(array_key_exists('body-html', $this->request) || array_key_exists('body-plain', $this->request))) - throw new \Exception('invalid body'); + $recipient = explode("|", $this->input)[0]; // match company - $company = MultiDB::findAndSetDbByExpenseMailbox($this->request["To"]); + $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); if (!$company) { - Log::info('unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $this->request["To"]); + Log::info('unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $recipient); return; } - // prepare - $ingresMail = (new MailgunInboundWebhookTransformer())->transform($this->request); - Log::info(json_encode($ingresMail)); + // fetch message from mailgun-api + $mailgun_domain = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_domain ? $company->settings?->mailgun_domain : config('services.mailgun.domain'); + $mailgun_secret = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_secret ? $company->settings?->mailgun_secret : config('services.mailgun.secret'); + $credentials = $mailgun_domain . ":" . $mailgun_secret . "@"; + $messageUrl = explode("|", $this->input)[1]; + $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); + $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); + $mail = json_decode(file_get_contents($messageUrl)); + + // prepare data for ingresEngine + $ingresEmail = new IngresEmail(); + + $ingresEmail->from = $mail->sender; + $ingresEmail->to = $recipient; // usage of data-input, because we need a single email here + $ingresEmail->subject = $mail->Subject; + $ingresEmail->body = $mail->{"body-html"}; + $ingresEmail->text_body = $mail->{"body-plain"}; + $ingresEmail->date = Carbon::createFromTimeString($mail->Date); + + // parse documents as UploadedFile from webhook-data + foreach ($mail->attachments as $attachment) { + + // prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 + $url = $attachment->url; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + + // download file and save to tmp dir + $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + + } // perform - (new IngresEmailEngine($ingresMail))->handle(); + (new IngresEmailEngine($ingresEmail))->handle(); } } diff --git a/app/Services/IngresEmail/IngresEmailEngine.php b/app/Services/IngresEmail/IngresEmailEngine.php index 42fd8a304719..bb03f9d5cce8 100644 --- a/app/Services/IngresEmail/IngresEmailEngine.php +++ b/app/Services/IngresEmail/IngresEmailEngine.php @@ -182,8 +182,6 @@ class IngresEmailEngine $expense->saveQuietly(); - Log::info(json_encode($documents)); - $this->saveDocuments($documents, $expense); event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); // @turbo124 please check, I copied from API-Controller From 16b46bc74d633f245b3429e8aa37de59d1922911 Mon Sep 17 00:00:00 2001 From: paulwer Date: Tue, 19 Mar 2024 13:02:51 +0100 Subject: [PATCH 039/119] add mail from storage as example in programm-code --- .../Mailgun/ProcessMailgunInboundWebhook.php | 136 ++++++++++++++++-- 1 file changed, 127 insertions(+), 9 deletions(-) diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index d616598167e3..1a0dcc81480d 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -40,21 +40,139 @@ class ProcessMailgunInboundWebhook implements ShouldQueue /** * Execute the job. * - * + * Mail from Storage + * { + * "Content-Type": "multipart/related; boundary=\"00000000000022bfbe0613e8b7f5\"", + * "Date": "Mon, 18 Mar 2024 06:34:09 +0100", + * "Dkim-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=wer-ner.de; s=google; t=1710740086; x=1711344886; darn=domain.example; h=to:subject:message-id:date:from:in-reply-to:references:mime-version :from:to:cc:subject:date:message-id:reply-to; bh=tkxC+ZzDSJJXLVgjDyvQZyDt6wkWKFHS50z4ZWiWT9U=; b=P1Sz54Djj1LHtPF7+cAKGRaN4IRjUT3bOyYAD/kbC0Tx2yNejPrjCPy3+a6R6MShgJ odYhoLRqylPPs1DQolNO6xgamsoEiR8jnII4QjJUBut4VirMlSO+RLxzpO7pt/Hr6j93 z0G1Yffpbz44l5GhndgXsa4Hf30Q8yy0p7fqMNABB/smscj7DJDu1os2cB1JazKYsmAE X4HtU5IgCOS++xbQPqZSNwjrFWlbgal2t2yAeTKAMdGX/nNKtfgZ5imqNwJWerpAYwgk 3qvUcgTw2MpeghcPpTiflPGp4fT/f1kUjes0dcqrvkE+6oTPvo0pi76QNoVs7peWKr/c JvaA==", + * "From": "Paul Werner ", + * "In-Reply-To": "", + * "Message-Id": "", + * "Mime-Version": "1.0", + * "Received": "by mail-lj1-f175.google.com with SMTP id 38308e7fff4ca-2d4a901e284so12524521fa.1 for ; Sun, 17 Mar 2024 22:34:47 -0700 (PDT)", + * "References": " ", + * "Subject": "Fwd: TEST", + * "To": "test@domain.example", + * "X-Envelope-From": "test@sender.example", + * "X-Gm-Message-State": "AOJu0Yy6rgBIPLGjnD293mVB5vBWQIraVAOnfa/GtyM6S/JIqe4rHbrx OqRe7oFFyCDyCjL/+2AFFkB9ljxgt7MWvpdec69dEn3BNQMlxuyGkpyxZUY8PDm4XRCyIy4vGxK 6Oddl7nWV5DM4zN4eLvZH+DPteyUq9A9ET9bowZnCrP8ZcQOP5js=", + * "X-Google-Dkim-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1710740086; x=1711344886; h=to:subject:message-id:date:from:in-reply-to:references:mime-version :x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=tkxC+ZzDSJJXLVgjDyvQZyDt6wkWKFHS50z4ZWiWT9U=; b=cyDJAeNEaU2CvWAX/d9E3LMDrceXyLe01lbsYvwY6ZNTchr/0vzrxQFTVxos2DQR7u jSKpaNqI958H1oZJY36XZV0+8MY2w6DjB1F3FUHbD1q5gxJUitXNuOvvpna/q0ZaqlQf 5n3kIkakV19uxu4pcrcLxO67744pBzEmVk+IJtI9FEoZy9253v09CfkzNZo68u2VxJVD TDFVVkZuIO5xi3flUVoD3CP0Bw/0BqpDuxVvOFy+qOaItTZ5Na+OPfUJcFG2j6T0rXFQ 1vXPxodqjllLwc/V+O1TmS46H/RhsHGAae5tWk+51KX8T2ZgTkfwKPV1YeSRl0QtDhYS gU0Q==", + * "X-Google-Smtp-Source": "AGHT+IFspt+3tKf94kXs48nOb58GzuV+pJ8oE3ZNwEcx6PG53wJeW858lyh2PiYIzSEPQTY2ykatvu2fqs8Bj+9d5rw=", + * "X-Mailgun-Incoming": "Yes", + * "X-Received": "by 2002:a2e:9847:0:b0:2d4:7455:89f6 with SMTP id e7-20020a2e9847000000b002d4745589f6mr4283454ljj.40.1710740086045; Sun, 17 Mar 2024 22:34:46 -0700 (PDT)", + * "sender": "test@sender.example", + * "recipients": "test@domain.example", + * "from": "Paul Werner ", + * "subject": "Fwd: TEST", + * "body-html": "

TESTAGAIN\"Unbenannt.png\"

---------- Forwarded message ---------
Von: Paul Werner <test@sender.example>
Date: Mo., 18. März 2024 um 06:30 Uhr
Subject: Fwd: TEST
To: <test@domain.example>


Hallöööö

---------- Forwarded message ---------
Von: Paul Werner <test@sender.example>
Date: Mo., 18. März 2024 um 06:23 Uhr
Subject: Fwd: TEST
To: <test@domain.example>


asjkdahwdaiohdawdawdawwwww!!!

---------- Forwarded message ---------
Von: Paul Werner <test@sender.example>
Date: Mo., 18. März 2024 um 06:22 Uhr
Subject: TEST
To: <test@domain.example>


TEST
\r\n
\r\n
\r\n
\r\n", + * "body-plain": "TESTAGAIN[image: Unbenannt.png]\r\n\r\n---------- Forwarded message ---------\r\nVon: Paul Werner \r\nDate: Mo., 18. März 2024 um 06:30 Uhr\r\nSubject: Fwd: TEST\r\nTo: \r\n\r\n\r\nHallöööö\r\n\r\n---------- Forwarded message ---------\r\nVon: Paul Werner \r\nDate: Mo., 18. März 2024 um 06:23 Uhr\r\nSubject: Fwd: TEST\r\nTo: \r\n\r\n\r\nasjkdahwdaiohdawdawdawwwww!!!\r\n\r\n---------- Forwarded message ---------\r\nVon: Paul Werner \r\nDate: Mo., 18. März 2024 um 06:22 Uhr\r\nSubject: TEST\r\nTo: \r\n\r\n\r\nTEST\r\n", + * "attachments": [ + * { + * "name": "Unbenannt.png", + * "content-type": "image/png", + * "size": 197753, + * "url": "https://storage-europe-west1.api.mailgun.net/v3/domains/domain.example/messages/BAAFAgVMamdcBboOIOtFyJ5B5NGEkkffYQ/attachments/0" + * } + * ], + * "content-id-map": { + * "": { + * "name": "Unbenannt.png", + * "content-type": "image/png", + * "size": 197753, + * "url": "https://storage-europe-west1.api.mailgun.net/v3/domains/domain.example/messages/BAAFAgVMamdcBboOIOtFyJ5B5NGEkkffYQ/attachments/0" + * } + * }, + * "message-headers": [ + * [ + * "Received", + * "from mail-lj1-f175.google.com (mail-lj1-f175.google.com [209.85.208.175]) by 634f26f73cf3 with SMTP id (version=TLS1.3, cipher=TLS_AES_128_GCM_SHA256); Mon, 18 Mar 2024 05:34:47 GMT" + * ], + * [ + * "Received", + * "by mail-lj1-f175.google.com with SMTP id 38308e7fff4ca-2d4a901e284so12524521fa.1 for ; Sun, 17 Mar 2024 22:34:47 -0700 (PDT)" + * ], + * [ + * "X-Envelope-From", + * "test@sender.example" + * ], + * [ + * "X-Mailgun-Incoming", + * "Yes" + * ], + * [ + * "Dkim-Signature", + * "v=1; a=rsa-sha256; c=relaxed/relaxed; d=wer-ner.de; s=google; t=1710740086; x=1711344886; darn=domain.example; h=to:subject:message-id:date:from:in-reply-to:references:mime-version :from:to:cc:subject:date:message-id:reply-to; bh=tkxC+ZzDSJJXLVgjDyvQZyDt6wkWKFHS50z4ZWiWT9U=; b=P1Sz54Djj1LHtPF7+cAKGRaN4IRjUT3bOyYAD/kbC0Tx2yNejPrjCPy3+a6R6MShgJ odYhoLRqylPPs1DQolNO6xgamsoEiR8jnII4QjJUBut4VirMlSO+RLxzpO7pt/Hr6j93 z0G1Yffpbz44l5GhndgXsa4Hf30Q8yy0p7fqMNABB/smscj7DJDu1os2cB1JazKYsmAE X4HtU5IgCOS++xbQPqZSNwjrFWlbgal2t2yAeTKAMdGX/nNKtfgZ5imqNwJWerpAYwgk 3qvUcgTw2MpeghcPpTiflPGp4fT/f1kUjes0dcqrvkE+6oTPvo0pi76QNoVs7peWKr/c JvaA==" + * ], + * [ + * "X-Google-Dkim-Signature", + * "v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1710740086; x=1711344886; h=to:subject:message-id:date:from:in-reply-to:references:mime-version :x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=tkxC+ZzDSJJXLVgjDyvQZyDt6wkWKFHS50z4ZWiWT9U=; b=cyDJAeNEaU2CvWAX/d9E3LMDrceXyLe01lbsYvwY6ZNTchr/0vzrxQFTVxos2DQR7u jSKpaNqI958H1oZJY36XZV0+8MY2w6DjB1F3FUHbD1q5gxJUitXNuOvvpna/q0ZaqlQf 5n3kIkakV19uxu4pcrcLxO67744pBzEmVk+IJtI9FEoZy9253v09CfkzNZo68u2VxJVD TDFVVkZuIO5xi3flUVoD3CP0Bw/0BqpDuxVvOFy+qOaItTZ5Na+OPfUJcFG2j6T0rXFQ 1vXPxodqjllLwc/V+O1TmS46H/RhsHGAae5tWk+51KX8T2ZgTkfwKPV1YeSRl0QtDhYS gU0Q==" + * ], + * [ + * "X-Gm-Message-State", + * "AOJu0Yy6rgBIPLGjnD293mVB5vBWQIraVAOnfa/GtyM6S/JIqe4rHbrx OqRe7oFFyCDyCjL/+2AFFkB9ljxgt7MWvpdec69dEn3BNQMlxuyGkpyxZUY8PDm4XRCyIy4vGxK 6Oddl7nWV5DM4zN4eLvZH+DPteyUq9A9ET9bowZnCrP8ZcQOP5js=" + * ], + * [ + * "X-Google-Smtp-Source", + * "AGHT+IFspt+3tKf94kXs48nOb58GzuV+pJ8oE3ZNwEcx6PG53wJeW858lyh2PiYIzSEPQTY2ykatvu2fqs8Bj+9d5rw=" + * ], + * [ + * "X-Received", + * "by 2002:a2e:9847:0:b0:2d4:7455:89f6 with SMTP id e7-20020a2e9847000000b002d4745589f6mr4283454ljj.40.1710740086045; Sun, 17 Mar 2024 22:34:46 -0700 (PDT)" + * ], + * [ + * "Mime-Version", + * "1.0" + * ], + * [ + * "References", + * " " + * ], + * [ + * "In-Reply-To", + * "" + * ], + * [ + * "From", + * "Paul Werner " + * ], + * [ + * "Date", + * "Mon, 18 Mar 2024 06:34:09 +0100" + * ], + * [ + * "Message-Id", + * "" + * ], + * [ + * "Subject", + * "Fwd: TEST" + * ], + * [ + * "To", + * "test@domain.example" + * ], + * [ + * "Content-Type", + * "multipart/related; boundary=\"00000000000022bfbe0613e8b7f5\"" + * ] + * ], + * "stripped-html": "
TESTAGAIN\"Unbenannt.png\"

\n", + * "stripped-text": "TESTAGAIN[image: Unbenannt.png]\r\n\r\n---------- Forwarded message ---------\r\nVon: Paul Werner \r\nDate: Mo., 18. März 2024 um 06:30 Uhr\r\nSubject: Fwd: TEST\r\nTo: \r\n\r\n\r\nHallöööö\r\n\r\n---------- Forwarded message ---------\r\nVon: Paul Werner \r\nDate: Mo., 18. März 2024 um 06:23 Uhr\r\nSubject: Fwd: TEST\r\nTo: \r\n\r\n\r\nasjkdahwdaiohdawdawdawwwww!!!\r\n\r\n---------- Forwarded message ---------\r\nVon: Paul Werner \r\nDate: Mo., 18. März 2024 um 06:22 Uhr\r\nSubject: TEST\r\nTo: \r\n\r\n\r\nTEST", + * "stripped-signature": "" + * } * @return void */ public function handle() { $recipient = explode("|", $this->input)[0]; - // match company + *match company $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); if (!$company) { Log::info('unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $recipient); return; } - // fetch message from mailgun-api + *fetch message from mailgun-api $mailgun_domain = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_domain ? $company->settings?->mailgun_domain : config('services.mailgun.domain'); $mailgun_secret = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_secret ? $company->settings?->mailgun_secret : config('services.mailgun.secret'); $credentials = $mailgun_domain . ":" . $mailgun_secret . "@"; @@ -63,30 +181,30 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); $mail = json_decode(file_get_contents($messageUrl)); - // prepare data for ingresEngine + *prepare data for ingresEngine $ingresEmail = new IngresEmail(); $ingresEmail->from = $mail->sender; - $ingresEmail->to = $recipient; // usage of data-input, because we need a single email here + $ingresEmail->to = $recipient; *usage of data-input, because we need a single email here $ingresEmail->subject = $mail->Subject; $ingresEmail->body = $mail->{"body-html"}; $ingresEmail->text_body = $mail->{"body-plain"}; $ingresEmail->date = Carbon::createFromTimeString($mail->Date); - // parse documents as UploadedFile from webhook-data + *parse documents as UploadedFile from webhook-data foreach ($mail->attachments as $attachment) { - // prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 + *prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 $url = $attachment->url; $url = str_replace("http://", "http://" . $credentials, $url); $url = str_replace("https://", "https://" . $credentials, $url); - // download file and save to tmp dir + *download file and save to tmp dir $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); } - // perform + *perform (new IngresEmailEngine($ingresEmail))->handle(); } } From f5dfedf8fab0456132c1f114bed55ffdd15da383 Mon Sep 17 00:00:00 2001 From: paulwer Date: Tue, 19 Mar 2024 13:05:12 +0100 Subject: [PATCH 040/119] fixes --- .../Mailgun/ProcessMailgunInboundWebhook.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index 1a0dcc81480d..80be1b354422 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -165,14 +165,14 @@ class ProcessMailgunInboundWebhook implements ShouldQueue { $recipient = explode("|", $this->input)[0]; - *match company + // match company $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); if (!$company) { Log::info('unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $recipient); return; } - *fetch message from mailgun-api + // fetch message from mailgun-api $mailgun_domain = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_domain ? $company->settings?->mailgun_domain : config('services.mailgun.domain'); $mailgun_secret = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_secret ? $company->settings?->mailgun_secret : config('services.mailgun.secret'); $credentials = $mailgun_domain . ":" . $mailgun_secret . "@"; @@ -181,30 +181,30 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); $mail = json_decode(file_get_contents($messageUrl)); - *prepare data for ingresEngine + // prepare data for ingresEngine $ingresEmail = new IngresEmail(); $ingresEmail->from = $mail->sender; - $ingresEmail->to = $recipient; *usage of data-input, because we need a single email here + $ingresEmail->to = $recipient; // usage of data-input, because we need a single email here $ingresEmail->subject = $mail->Subject; $ingresEmail->body = $mail->{"body-html"}; $ingresEmail->text_body = $mail->{"body-plain"}; $ingresEmail->date = Carbon::createFromTimeString($mail->Date); - *parse documents as UploadedFile from webhook-data + // parse documents as UploadedFile from webhook-data foreach ($mail->attachments as $attachment) { - *prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 + // prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 $url = $attachment->url; $url = str_replace("http://", "http://" . $credentials, $url); $url = str_replace("https://", "https://" . $credentials, $url); - *download file and save to tmp dir + // download file and save to tmp dir $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); } - *perform + // perform (new IngresEmailEngine($ingresEmail))->handle(); } } From 9792ab71dc1149ad5f323cddd61a5c5ef4f21e25 Mon Sep 17 00:00:00 2001 From: paulwer Date: Tue, 19 Mar 2024 13:09:03 +0100 Subject: [PATCH 041/119] minor changes --- app/Http/Controllers/MailgunController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 28a257d2e4e1..05f3d25aba40 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -117,7 +117,7 @@ class MailgunController extends BaseController if (!array_key_exists('recipient', $input) || !array_key_exists('message-url', $input)) { Log::info('Failed: Message could not be parsed, because required parameters are missing. Please ensure contacting this api-endpoint with a store & notify operation instead of a forward operation!'); - return response()->json(['message' => 'Failed. Missing Parameters'], 400); + return response()->json(['message' => 'Failed. Missing Parameters. Use store and notify!'], 400); } if (!array_key_exists('attachments', $input) || count(json_decode($input['attachments'])) == 0) { From f43909141fb71a27f9f9e2eef0ad733bab642347 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 24 Mar 2024 10:53:20 +0100 Subject: [PATCH 042/119] brevo webhook + better attachement fetching cycle + fixes for brevo key integration namings --- app/Http/Controllers/BrevoController.php | 153 +++++++++++- app/Http/Controllers/MailgunController.php | 2 +- app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 182 ++++++++++++++ app/Jobs/Brevo/ProcessBrevoWebhook.php | 2 +- .../Mailgun/ProcessMailgunInboundWebhook.php | 86 +++++-- app/Providers/AppServiceProvider.php | 6 +- composer.lock | 227 +++++++++--------- config/services.php | 8 +- routes/api.php | 1 + 9 files changed, 527 insertions(+), 140 deletions(-) create mode 100644 app/Jobs/Brevo/ProcessBrevoInboundWebhook.php diff --git a/app/Http/Controllers/BrevoController.php b/app/Http/Controllers/BrevoController.php index e1dce175f485..c87b4ff0f285 100644 --- a/app/Http/Controllers/BrevoController.php +++ b/app/Http/Controllers/BrevoController.php @@ -11,8 +11,10 @@ namespace App\Http\Controllers; +use App\Jobs\Brevo\ProcessBrevoInboundWebhook; use App\Jobs\Brevo\ProcessBrevoWebhook; use Illuminate\Http\Request; +use Log; /** * Class PostMarkController. @@ -26,13 +28,13 @@ class BrevoController extends BaseController } /** - * Process Postmark Webhook. + * Process Brevo Webhook. * * * @OA\Post( - * path="/api/v1/postmark_webhook", - * operationId="postmarkWebhook", - * tags={"postmark"}, + * path="/api/v1/brevo_webhook", + * operationId="brevoWebhook", + * tags={"brevo"}, * summary="Processing webhooks from PostMark", * description="Adds an credit to the system", * @OA\Parameter(ref="#/components/parameters/X-API-TOKEN"), @@ -61,7 +63,7 @@ class BrevoController extends BaseController */ public function webhook(Request $request) { - if ($request->has('token') && $request->get('token') == config('services.brevo.key')) { + if ($request->has('token') && $request->get('token') == config('services.brevo.secret')) { ProcessBrevoWebhook::dispatch($request->all())->delay(10); return response()->json(['message' => 'Success'], 200); @@ -69,4 +71,145 @@ class BrevoController extends BaseController return response()->json(['message' => 'Unauthorized'], 403); } + + + /** + * Process Brevo Inbound Webhook. + * + * + * @OA\Post( + * path="/api/v1/brevo_inbound_webhook", + * operationId="brevoInboundWebhook", + * tags={"brevo"}, + * summary="Processing inbound webhooks from Brevo", + * description="Adds an credit to the system", + * @OA\Parameter(ref="#/components/parameters/X-API-TOKEN"), + * @OA\Parameter(ref="#/components/parameters/X-Requested-With"), + * @OA\Parameter(ref="#/components/parameters/include"), + * @OA\Response( + * response=200, + * description="Returns the saved credit object", + * @OA\Header(header="X-MINIMUM-CLIENT-VERSION", ref="#/components/headers/X-MINIMUM-CLIENT-VERSION"), + * @OA\Header(header="X-RateLimit-Remaining", ref="#/components/headers/X-RateLimit-Remaining"), + * @OA\Header(header="X-RateLimit-Limit", ref="#/components/headers/X-RateLimit-Limit"), + * @OA\JsonContent(ref="#/components/schemas/Credit"), + * ), + * @OA\Response( + * response=422, + * description="Validation error", + * @OA\JsonContent(ref="#/components/schemas/ValidationError"), + * + * ), + * @OA\Response( + * response="default", + * description="Unexpected Error", + * @OA\JsonContent(ref="#/components/schemas/Error"), + * ), + * ) + * + * array ( + * 'items' => + * array ( + * 0 => + * array ( + * 'Uuid' => + * array ( + * 0 => 'd9f48d52-a344-42a4-9056-9733488d9fa3', + * ), + * 'Recipients' => + * array ( + * 0 => 'test@test.de', + * ), + * 'MessageId' => '', + * 'InReplyTo' => NULL, + * 'From' => + * array ( + * 'Name' => 'Max Mustermann', + * 'Address' => 'max@mustermann.de', + * ), + * 'To' => + * array ( + * 0 => + * array ( + * 'Name' => NULL, + * 'Address' => 'test@test.de', + * ), + * ), + * 'Cc' => + * array ( + * ), + * 'Bcc' => + * array ( + * ), + * 'ReplyTo' => NULL, + * 'SentAtDate' => 'Sat, 23 Mar 2024 18:18:20 +0100', + * 'Subject' => 'TEST', + * 'Attachments' => + * array ( + * 0 => + * array ( + * 'Name' => 'flag--sv-1x1.svg', + * 'ContentType' => 'image/svg+xml', + * 'ContentLength' => 79957, + * 'ContentID' => 'f_lu4ct6s20', + * 'DownloadToken' => 'eyJmb2xkZXIiOiIyMDI0MDMyMzE3MTgzNi45OS43OTgwMDM4MDQiLCJmaWxlbmFtZSI6ImZsYWctLXN2LTF4MS5zdmcifQ', + * ), + * ), + * 'Headers' => + * array ( + * 'Received' => 'by mail-ed1-f51.google.com with SMTP id 4fb4d7f45d1cf-56b0af675deso3877288a12.1 for ; Sat, 23 Mar 2024 10:18:36 -0700 (PDT)', + * 'DKIM-Signature' => 'v=1; a=rsa-sha256; c=relaxed/relaxed; d=mustermann.de; s=google; t=1711214316; x=1711819116; darn=test.de; h=to:subject:message-id:date:from:mime-version:from:to:cc:subject :date:message-id:reply-to; bh=eBSl5M0zvmTd+dFXGXMMSWrQ4nCvUdyVx+1Xpl+YuX8=; b=ackw3d+qTvZk4JKxomvH626MvfwmH23mikOUc2hWwYiO6unmQgPs2w5spnkmD9aCZ9 G+3nPSYKntugOmqWstZH3z4B063U4Y6j5hTc19WtCyyb9UR+XD+C6L10yc6ez8QUhlZT uAGqDoJ+E8+dBxiMul2pow19lC88t3QxRXU+i8zScniV7SFkwzziCEODaB61yI0DXsZB bUkx5Gx6cztKaNVF2QgguF2nQnJFUnD2nabVFsihyJ5r6y61rkSM/YTfMJuES772lnhv IeF+vwiFNEPKafrchce6YJcvo5Vd5lYFK4LtHyCy3mwJpX2QY+WnWAfferZ2YfgEL0Sf K3Pw==', + * 'X-Google-DKIM-Signature' => 'v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1711214316; x=1711819116; h=to:subject:message-id:date:from:mime-version:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=eBSl5M0zvmTd+dFXGXMMSWrQ4nCvUdyVx+1Xpl+YuX8=; b=fg4tXZnstRBexYlC6MD7C7is0kQj+xY66cSJ78tSa7PtSFQzY0zajDMsepMCGiiWmN /Pc/tRtk53pru/OtfzRT9pbM6mhM1arIt+QaQBQGU5xZVV5JXfPmdnPzXqAbQztyeHrk UcEkz+qDN3JNoidw2dJhhdt5MxdKssR572NwtBrn/rN7f1o/ThWzEz+P0o06GVBpxVYP wM0EkvcJj2SUOcn36kmp1ccbMUwYCU2h1JmniEFY8RTqu2il13iXoBvG4YPxe0c0hJ6z zw1N5rONeQM113N1rpbQzS1QLSngczuOhN24M3TOwrHJIec/BxrOW6KWl/uPUqiZAf65 f0tg==', + * 'X-Gm-Message-State' => 'AOJu0YzKhR1HY1oUXoq++LLpl6UOz1S60NfPxuPXBLcP+6aACYle8rqQ fYHe2rQYTpg4KWiOswu858STOW8qmiewXD6gH/LbmEFs7sknRyDPNr/+L0cv828A3o+SOvXu3uP SY6H1aNSwIpqTRhJ+nNjTuSUpuSoABd9fYXFwPuivV0DtBhoVmpE=', + * 'X-Google-Smtp-Source' => 'AGHT+IHdA9ZhW0dQxgOYx2OXBGmu4pzSR/zwJ0vcPNXFSqttKCPS2oTw1a9b2mMdhyUeoRAwP5TmhHlAtqUUrOPwkgg=', + * 'X-Received' => 'by 2002:a50:d74c:0:b0:567:3c07:8bbc with SMTP id i12-20020a50d74c000000b005673c078bbcmr2126401edj.21.1711214316135; Sat, 23 Mar 2024 10:18:36 -0700 (PDT)', + * 'MIME-Version' => '1.0', + * 'From' => 'Max Mustermann ', + * 'Date' => 'Sat, 23 Mar 2024 18:18:20 +0100', + * 'Message-ID' => '', + * 'Subject' => 'TEST', + * 'To' => 'test@test.de', + * 'Content-Type' => 'multipart/mixed', + * ), + * 'SpamScore' => 2.8, + * 'ExtractedMarkdownMessage' => 'TEST', + * 'ExtractedMarkdownSignature' => NULL, + * 'RawHtmlBody' => '
TEST
', + * 'RawTextBody' => 'TEST', + * 'EMLDownloadToken' => 'eyJmb2xkZXIiOiIyMDI0MDMyMzE3MTgzNi45OS43OTgwMDM4MDQiLCJmaWxlbmFtZSI6InNtdHAuZW1sIn0', + * ), + * ), + * ) + */ + public function inboundWebhook(Request $request) + { + $input = $request->all(); + + // TODO: validation for client credentials by recipient + // if (!($request->has('token') && $request->get('token') == config('services.brevo.secret'))) + // return response()->json(['message' => 'Unauthorized'], 403); + + if (!array_key_exists('items', $input)) { + Log::info('Failed: Message could not be parsed, because required parameters are missing.'); + return response()->json(['message' => 'Failed. Invalid Parameters.'], 400); + } + + foreach ($input["items"] as $item) { + + if (!array_key_exists('Recipients', $item) || !array_key_exists('MessageId', $item)) { + Log::info('Failed: Message could not be parsed, because required parameters are missing. At least one item was invalid.'); + return response()->json(['message' => 'Failed. Invalid Parameters. At least one item was invalid.'], 400); + } + + if (!array_key_exists('Attachments', $item) || count($item['Attachments']) == 0) { + Log::info('Brevo: InboundParsing: SoftFail: Message ignored because of missing attachments. No Actions would have been taken...'); + continue; + } + + ProcessBrevoInboundWebhook::dispatch($item)->delay(10); + + } + + return response()->json(['message' => 'Success'], 201); + } } diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 05f3d25aba40..b26952598005 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -78,7 +78,7 @@ class MailgunController extends BaseController } /** - * Process Mailgun Webhook. + * Process Mailgun Inbound Webhook. * * * @OA\Post( diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php new file mode 100644 index 000000000000..aab2adad3a29 --- /dev/null +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -0,0 +1,182 @@ + + * array ( + * 0 => 'd9f48d52-a344-42a4-9056-9733488d9fa3', + * ), + * 'Recipients' => + * array ( + * 0 => 'test@test.de', + * ), + * 'MessageId' => '', + * 'InReplyTo' => NULL, + * 'From' => + * array ( + * 'Name' => 'Max Mustermann', + * 'Address' => 'max@mustermann.de', + * ), + * 'To' => + * array ( + * 0 => + * array ( + * 'Name' => NULL, + * 'Address' => 'test@test.de', + * ), + * ), + * 'Cc' => + * array ( + * ), + * 'Bcc' => + * array ( + * ), + * 'ReplyTo' => NULL, + * 'SentAtDate' => 'Sat, 23 Mar 2024 18:18:20 +0100', + * 'Subject' => 'TEST', + * 'Attachments' => + * array ( + * 0 => + * array ( + * 'Name' => 'flag--sv-1x1.svg', + * 'ContentType' => 'image/svg+xml', + * 'ContentLength' => 79957, + * 'ContentID' => 'f_lu4ct6s20', + * 'DownloadToken' => 'eyJmb2xkZXIiOiIyMDI0MDMyMzE3MTgzNi45OS43OTgwMDM4MDQiLCJmaWxlbmFtZSI6ImZsYWctLXN2LTF4MS5zdmcifQ', + * ), + * ), + * 'Headers' => + * array ( + * 'Received' => 'by mail-ed1-f51.google.com with SMTP id 4fb4d7f45d1cf-56b0af675deso3877288a12.1 for ; Sat, 23 Mar 2024 10:18:36 -0700 (PDT)', + * 'DKIM-Signature' => 'v=1; a=rsa-sha256; c=relaxed/relaxed; d=mustermann.de; s=google; t=1711214316; x=1711819116; darn=test.de; h=to:subject:message-id:date:from:mime-version:from:to:cc:subject :date:message-id:reply-to; bh=eBSl5M0zvmTd+dFXGXMMSWrQ4nCvUdyVx+1Xpl+YuX8=; b=ackw3d+qTvZk4JKxomvH626MvfwmH23mikOUc2hWwYiO6unmQgPs2w5spnkmD9aCZ9 G+3nPSYKntugOmqWstZH3z4B063U4Y6j5hTc19WtCyyb9UR+XD+C6L10yc6ez8QUhlZT uAGqDoJ+E8+dBxiMul2pow19lC88t3QxRXU+i8zScniV7SFkwzziCEODaB61yI0DXsZB bUkx5Gx6cztKaNVF2QgguF2nQnJFUnD2nabVFsihyJ5r6y61rkSM/YTfMJuES772lnhv IeF+vwiFNEPKafrchce6YJcvo5Vd5lYFK4LtHyCy3mwJpX2QY+WnWAfferZ2YfgEL0Sf K3Pw==', + * 'X-Google-DKIM-Signature' => 'v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1711214316; x=1711819116; h=to:subject:message-id:date:from:mime-version:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=eBSl5M0zvmTd+dFXGXMMSWrQ4nCvUdyVx+1Xpl+YuX8=; b=fg4tXZnstRBexYlC6MD7C7is0kQj+xY66cSJ78tSa7PtSFQzY0zajDMsepMCGiiWmN /Pc/tRtk53pru/OtfzRT9pbM6mhM1arIt+QaQBQGU5xZVV5JXfPmdnPzXqAbQztyeHrk UcEkz+qDN3JNoidw2dJhhdt5MxdKssR572NwtBrn/rN7f1o/ThWzEz+P0o06GVBpxVYP wM0EkvcJj2SUOcn36kmp1ccbMUwYCU2h1JmniEFY8RTqu2il13iXoBvG4YPxe0c0hJ6z zw1N5rONeQM113N1rpbQzS1QLSngczuOhN24M3TOwrHJIec/BxrOW6KWl/uPUqiZAf65 f0tg==', + * 'X-Gm-Message-State' => 'AOJu0YzKhR1HY1oUXoq++LLpl6UOz1S60NfPxuPXBLcP+6aACYle8rqQ fYHe2rQYTpg4KWiOswu858STOW8qmiewXD6gH/LbmEFs7sknRyDPNr/+L0cv828A3o+SOvXu3uP SY6H1aNSwIpqTRhJ+nNjTuSUpuSoABd9fYXFwPuivV0DtBhoVmpE=', + * 'X-Google-Smtp-Source' => 'AGHT+IHdA9ZhW0dQxgOYx2OXBGmu4pzSR/zwJ0vcPNXFSqttKCPS2oTw1a9b2mMdhyUeoRAwP5TmhHlAtqUUrOPwkgg=', + * 'X-Received' => 'by 2002:a50:d74c:0:b0:567:3c07:8bbc with SMTP id i12-20020a50d74c000000b005673c078bbcmr2126401edj.21.1711214316135; Sat, 23 Mar 2024 10:18:36 -0700 (PDT)', + * 'MIME-Version' => '1.0', + * 'From' => 'Max Mustermann ', + * 'Date' => 'Sat, 23 Mar 2024 18:18:20 +0100', + * 'Message-ID' => '', + * 'Subject' => 'TEST', + * 'To' => 'test@test.de', + * 'Content-Type' => 'multipart/mixed', + * ), + * 'SpamScore' => 2.8, + * 'ExtractedMarkdownMessage' => 'TEST', + * 'ExtractedMarkdownSignature' => NULL, + * 'RawHtmlBody' => '
TEST
', + * 'RawTextBody' => 'TEST', + * 'EMLDownloadToken' => 'eyJmb2xkZXIiOiIyMDI0MDMyMzE3MTgzNi45OS43OTgwMDM4MDQiLCJmaWxlbmFtZSI6InNtdHAuZW1sIn0', + * ), + * ) + */ + public function __construct(private array $input) + { + } + + /** + * Execute the job. + * + * TODO: insert Mail from Storage + * + * @return void + */ + public function handle() + { + + // brevo defines recipients as array, and we should check all of them, to be sure + foreach ($this->input["Recipients"] as $recipient) { + + // match company + $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); + if (!$company) { + Log::info('unknown Expense Mailbox occured while handling an inbound email from brevo: ' . $recipient); + continue; + } + + $company_brevo_secret = $company->settings?->email_sending_method === 'client_brevo' && $company->settings?->brevo_secret ? $company->settings?->brevo_secret : null; + if (empty ($company_brevo_secret) && empty (config('services.brevo.secret'))) + throw new \Error("no brevo credenitals found, we cannot get the attachement"); + + // prepare data for ingresEngine + $ingresEmail = new IngresEmail(); + + $ingresEmail->from = $this->input["From"]["Address"]; + $ingresEmail->to = $recipient; + $ingresEmail->subject = $this->input["Subject"]; + $ingresEmail->body = $this->input["RawHtmlBody"]; + $ingresEmail->text_body = $this->input["RawTextBody"]; + $ingresEmail->date = Carbon::createFromTimeString($this->input["SentAtDate"]); + + // parse documents as UploadedFile from webhook-data + foreach ($this->input["Attachments"] as $attachment) { + + // download file and save to tmp dir + if (!empty ($company_brevo_secret)) { + + $attachment = null; + try { + + $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", $company_brevo_secret)); + $attachment = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); + + } catch (\Error $e) { + if (config('services.brevo.secret')) { + Log::info("Error while downloading with company credentials, we try to use defaul credentials now..."); + + $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); + $attachment = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); + + } else + throw $e; + } + $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment, $attachment["Name"], $attachment["ContentType"]); + + } else { + + $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); + $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($brevo->getInboundEmailAttachment($attachment["DownloadToken"]), $attachment["Name"], $attachment["ContentType"]); + + } + + } + + (new IngresEmailEngine($ingresEmail))->handle(); + + } + } +} diff --git a/app/Jobs/Brevo/ProcessBrevoWebhook.php b/app/Jobs/Brevo/ProcessBrevoWebhook.php index 92959f7d2e8b..48ddb9a87942 100644 --- a/app/Jobs/Brevo/ProcessBrevoWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoWebhook.php @@ -421,7 +421,7 @@ class ProcessBrevoWebhook implements ShouldQueue public function getRawMessage(string $message_id) { - $brevo_secret = !empty($this->company->settings->brevo_secret) ? $this->company->settings->brevo_secret : config('services.brevo.key'); + $brevo_secret = !empty ($this->company->settings->brevo_secret) ? $this->company->settings->brevo_secret : config('services.brevo.secret'); $brevo = new TransactionalEmailsApi(null, Configuration::getDefaultConfiguration()->setApiKey('api-key', $brevo_secret)); $messageDetail = $brevo->getTransacEmailContent($message_id); diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index 80be1b354422..11e81f1b0633 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -173,13 +173,44 @@ class ProcessMailgunInboundWebhook implements ShouldQueue } // fetch message from mailgun-api - $mailgun_domain = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_domain ? $company->settings?->mailgun_domain : config('services.mailgun.domain'); - $mailgun_secret = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_secret ? $company->settings?->mailgun_secret : config('services.mailgun.secret'); - $credentials = $mailgun_domain . ":" . $mailgun_secret . "@"; - $messageUrl = explode("|", $this->input)[1]; - $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); - $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); - $mail = json_decode(file_get_contents($messageUrl)); + $company_mailgun_domain = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_domain ? $company->settings?->mailgun_domain : null; + $company_mailgun_secret = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_secret ? $company->settings?->mailgun_secret : null; + if (!($company_mailgun_domain && $company_mailgun_secret) && !(config('services.mailgun.domain') && config('services.mailgun.secret'))) + throw new \Error("no mailgun credenitals found, we cannot get the attachements and files"); + + $mail = null; + if ($company_mailgun_domain && $company_mailgun_secret) { + + $credentials = $company_mailgun_domain . ":" . $company_mailgun_secret . "@"; + $messageUrl = explode("|", $this->input)[1]; + $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); + $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); + + try { + $mail = json_decode(file_get_contents($messageUrl)); + } catch (\Error $e) { + if (config('services.mailgun.secret')) { + Log::info("Error while downloading with company credentials, we try to use defaul credentials now..."); + + $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; + $messageUrl = explode("|", $this->input)[1]; + $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); + $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); + $mail = json_decode(file_get_contents($messageUrl)); + + } else + throw $e; + } + + } else { + + $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; + $messageUrl = explode("|", $this->input)[1]; + $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); + $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); + $mail = json_decode(file_get_contents($messageUrl)); + + } // prepare data for ingresEngine $ingresEmail = new IngresEmail(); @@ -192,15 +223,42 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $ingresEmail->date = Carbon::createFromTimeString($mail->Date); // parse documents as UploadedFile from webhook-data - foreach ($mail->attachments as $attachment) { - - // prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 - $url = $attachment->url; - $url = str_replace("http://", "http://" . $credentials, $url); - $url = str_replace("https://", "https://" . $credentials, $url); + foreach ($mail->attachments as $attachment) { // prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 // download file and save to tmp dir - $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + if ($company_mailgun_domain && $company_mailgun_secret) { + + try { + + $credentials = $company_mailgun_domain . ":" . $company_mailgun_secret . "@"; + $url = $attachment->url; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + + } catch (\Error $e) { + if (config('services.mailgun.secret')) { + Log::info("Error while downloading with company credentials, we try to use defaul credentials now..."); + + $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; + $url = $attachment->url; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + + } else + throw $e; + } + + } else { + + $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; + $url = $attachment->url; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + + } } diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 8ab1e78c91f6..84083f546fee 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -126,18 +126,18 @@ class AppServiceProvider extends ServiceProvider new Dsn( 'brevo+api', 'default', - config('services.brevo.key') + config('services.brevo.secret') ) ); }); - Mailer::macro('brevo_config', function (string $brevo_key) { + Mailer::macro('brevo_config', function (string $brevo_secret) { // @phpstan-ignore /** @phpstan-ignore-next-line **/ Mailer::setSymfonyTransport( (new BrevoTransportFactory)->create( new Dsn( 'brevo+api', 'default', - $brevo_key + $brevo_secret ) ) ); diff --git a/composer.lock b/composer.lock index 70f2ca5f20c4..fed2fd9e57f5 100644 --- a/composer.lock +++ b/composer.lock @@ -369,21 +369,22 @@ }, { "name": "amphp/parallel", - "version": "v2.2.7", + "version": "v2.2.8", "source": { "type": "git", "url": "https://github.com/amphp/parallel.git", - "reference": "ffda869c33c30627b6eb5c25f096882d885681dc" + "reference": "efd71b342b64c2e46d904e4eb057ed5ab20f8e2d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/parallel/zipball/ffda869c33c30627b6eb5c25f096882d885681dc", - "reference": "ffda869c33c30627b6eb5c25f096882d885681dc", + "url": "https://api.github.com/repos/amphp/parallel/zipball/efd71b342b64c2e46d904e4eb057ed5ab20f8e2d", + "reference": "efd71b342b64c2e46d904e4eb057ed5ab20f8e2d", "shasum": "" }, "require": { "amphp/amp": "^3", "amphp/byte-stream": "^2", + "amphp/cache": "^2", "amphp/parser": "^1", "amphp/pipeline": "^1", "amphp/process": "^2", @@ -440,7 +441,7 @@ ], "support": { "issues": "https://github.com/amphp/parallel/issues", - "source": "https://github.com/amphp/parallel/tree/v2.2.7" + "source": "https://github.com/amphp/parallel/tree/v2.2.8" }, "funding": [ { @@ -448,20 +449,20 @@ "type": "github" } ], - "time": "2024-03-16T16:15:46+00:00" + "time": "2024-03-19T16:09:34+00:00" }, { "name": "amphp/parser", - "version": "v1.1.0", + "version": "v1.1.1", "source": { "type": "git", "url": "https://github.com/amphp/parser.git", - "reference": "ff1de4144726c5dad5fab97f66692ebe8de3e151" + "reference": "3cf1f8b32a0171d4b1bed93d25617637a77cded7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/parser/zipball/ff1de4144726c5dad5fab97f66692ebe8de3e151", - "reference": "ff1de4144726c5dad5fab97f66692ebe8de3e151", + "url": "https://api.github.com/repos/amphp/parser/zipball/3cf1f8b32a0171d4b1bed93d25617637a77cded7", + "reference": "3cf1f8b32a0171d4b1bed93d25617637a77cded7", "shasum": "" }, "require": { @@ -502,7 +503,7 @@ ], "support": { "issues": "https://github.com/amphp/parser/issues", - "source": "https://github.com/amphp/parser/tree/v1.1.0" + "source": "https://github.com/amphp/parser/tree/v1.1.1" }, "funding": [ { @@ -510,7 +511,7 @@ "type": "github" } ], - "time": "2022-12-30T18:08:47+00:00" + "time": "2024-03-21T19:16:53+00:00" }, { "name": "amphp/pipeline", @@ -707,16 +708,16 @@ }, { "name": "amphp/socket", - "version": "v2.2.4", + "version": "v2.3.0", "source": { "type": "git", "url": "https://github.com/amphp/socket.git", - "reference": "4223324c627cc26d44800630411e64856d3344bc" + "reference": "acc0a2f65ab498025ba5641f7cce499c4b1ed4b5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/socket/zipball/4223324c627cc26d44800630411e64856d3344bc", - "reference": "4223324c627cc26d44800630411e64856d3344bc", + "url": "https://api.github.com/repos/amphp/socket/zipball/acc0a2f65ab498025ba5641f7cce499c4b1ed4b5", + "reference": "acc0a2f65ab498025ba5641f7cce499c4b1ed4b5", "shasum": "" }, "require": { @@ -779,7 +780,7 @@ ], "support": { "issues": "https://github.com/amphp/socket/issues", - "source": "https://github.com/amphp/socket/tree/v2.2.4" + "source": "https://github.com/amphp/socket/tree/v2.3.0" }, "funding": [ { @@ -787,7 +788,7 @@ "type": "github" } ], - "time": "2024-02-28T15:56:06+00:00" + "time": "2024-03-19T20:01:53+00:00" }, { "name": "amphp/sync", @@ -1343,16 +1344,16 @@ }, { "name": "aws/aws-sdk-php", - "version": "3.301.2", + "version": "3.301.6", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "7f8180275e624cb566d8af77d2f1c958bf5be35b" + "reference": "18c0ebd71d3071304f1ea02aa9af75f95863177a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/7f8180275e624cb566d8af77d2f1c958bf5be35b", - "reference": "7f8180275e624cb566d8af77d2f1c958bf5be35b", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/18c0ebd71d3071304f1ea02aa9af75f95863177a", + "reference": "18c0ebd71d3071304f1ea02aa9af75f95863177a", "shasum": "" }, "require": { @@ -1432,9 +1433,9 @@ "support": { "forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80", "issues": "https://github.com/aws/aws-sdk-php/issues", - "source": "https://github.com/aws/aws-sdk-php/tree/3.301.2" + "source": "https://github.com/aws/aws-sdk-php/tree/3.301.6" }, - "time": "2024-03-18T18:06:18+00:00" + "time": "2024-03-22T18:05:21+00:00" }, { "name": "bacon/bacon-qr-code", @@ -4662,23 +4663,23 @@ }, { "name": "imdhemy/google-play-billing", - "version": "1.5.1", + "version": "1.5.2", "source": { "type": "git", "url": "https://github.com/imdhemy/google-play-billing.git", - "reference": "bb94f3b6ddb021605815e528f31b8c930c41677c" + "reference": "7f2b032354568fa50858e0f6dd25592d975b3979" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/imdhemy/google-play-billing/zipball/bb94f3b6ddb021605815e528f31b8c930c41677c", - "reference": "bb94f3b6ddb021605815e528f31b8c930c41677c", + "url": "https://api.github.com/repos/imdhemy/google-play-billing/zipball/7f2b032354568fa50858e0f6dd25592d975b3979", + "reference": "7f2b032354568fa50858e0f6dd25592d975b3979", "shasum": "" }, "require": { "ext-json": "*", "google/auth": "^1.26", "guzzlehttp/guzzle": "^7.5.1", - "nesbot/carbon": "^2.66", + "nesbot/carbon": "^2.66|^3.0", "php": ">=8.0" }, "require-dev": { @@ -4707,9 +4708,9 @@ "description": "Google Play Billing", "support": { "issues": "https://github.com/imdhemy/google-play-billing/issues", - "source": "https://github.com/imdhemy/google-play-billing/tree/1.5.1" + "source": "https://github.com/imdhemy/google-play-billing/tree/1.5.2" }, - "time": "2023-12-15T10:25:05+00:00" + "time": "2024-03-19T17:56:34+00:00" }, { "name": "imdhemy/laravel-purchases", @@ -5434,16 +5435,16 @@ }, { "name": "laravel/framework", - "version": "v10.48.3", + "version": "v10.48.4", "source": { "type": "git", "url": "https://github.com/laravel/framework.git", - "reference": "5791c052b41c6b593556adc687076bfbdd13c501" + "reference": "7e0701bf59cb76a51f7c1f7bea51c0c0c29c0b72" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/framework/zipball/5791c052b41c6b593556adc687076bfbdd13c501", - "reference": "5791c052b41c6b593556adc687076bfbdd13c501", + "url": "https://api.github.com/repos/laravel/framework/zipball/7e0701bf59cb76a51f7c1f7bea51c0c0c29c0b72", + "reference": "7e0701bf59cb76a51f7c1f7bea51c0c0c29c0b72", "shasum": "" }, "require": { @@ -5548,7 +5549,7 @@ "league/flysystem-sftp-v3": "^3.0", "mockery/mockery": "^1.5.1", "nyholm/psr7": "^1.2", - "orchestra/testbench-core": "^8.18", + "orchestra/testbench-core": "^8.23.4", "pda/pheanstalk": "^4.0", "phpstan/phpstan": "^1.4.7", "phpunit/phpunit": "^10.0.7", @@ -5637,7 +5638,7 @@ "issues": "https://github.com/laravel/framework/issues", "source": "https://github.com/laravel/framework" }, - "time": "2024-03-15T10:17:07+00:00" + "time": "2024-03-21T13:36:36+00:00" }, { "name": "laravel/prompts", @@ -6911,16 +6912,16 @@ }, { "name": "league/uri", - "version": "7.4.0", + "version": "7.4.1", "source": { "type": "git", "url": "https://github.com/thephpleague/uri.git", - "reference": "bf414ba956d902f5d98bf9385fcf63954f09dce5" + "reference": "bedb6e55eff0c933668addaa7efa1e1f2c417cc4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/uri/zipball/bf414ba956d902f5d98bf9385fcf63954f09dce5", - "reference": "bf414ba956d902f5d98bf9385fcf63954f09dce5", + "url": "https://api.github.com/repos/thephpleague/uri/zipball/bedb6e55eff0c933668addaa7efa1e1f2c417cc4", + "reference": "bedb6e55eff0c933668addaa7efa1e1f2c417cc4", "shasum": "" }, "require": { @@ -6989,7 +6990,7 @@ "docs": "https://uri.thephpleague.com", "forum": "https://thephpleague.slack.com", "issues": "https://github.com/thephpleague/uri-src/issues", - "source": "https://github.com/thephpleague/uri/tree/7.4.0" + "source": "https://github.com/thephpleague/uri/tree/7.4.1" }, "funding": [ { @@ -6997,20 +6998,20 @@ "type": "github" } ], - "time": "2023-12-01T06:24:25+00:00" + "time": "2024-03-23T07:42:40+00:00" }, { "name": "league/uri-interfaces", - "version": "7.4.0", + "version": "7.4.1", "source": { "type": "git", "url": "https://github.com/thephpleague/uri-interfaces.git", - "reference": "bd8c487ec236930f7bbc42b8d374fa882fbba0f3" + "reference": "8d43ef5c841032c87e2de015972c06f3865ef718" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/uri-interfaces/zipball/bd8c487ec236930f7bbc42b8d374fa882fbba0f3", - "reference": "bd8c487ec236930f7bbc42b8d374fa882fbba0f3", + "url": "https://api.github.com/repos/thephpleague/uri-interfaces/zipball/8d43ef5c841032c87e2de015972c06f3865ef718", + "reference": "8d43ef5c841032c87e2de015972c06f3865ef718", "shasum": "" }, "require": { @@ -7073,7 +7074,7 @@ "docs": "https://uri.thephpleague.com", "forum": "https://thephpleague.slack.com", "issues": "https://github.com/thephpleague/uri-src/issues", - "source": "https://github.com/thephpleague/uri-interfaces/tree/7.4.0" + "source": "https://github.com/thephpleague/uri-interfaces/tree/7.4.1" }, "funding": [ { @@ -7081,7 +7082,7 @@ "type": "github" } ], - "time": "2023-11-24T15:40:42+00:00" + "time": "2024-03-23T07:42:40+00:00" }, { "name": "livewire/livewire", @@ -7461,16 +7462,16 @@ }, { "name": "mollie/mollie-api-php", - "version": "v2.65.0", + "version": "v2.66.0", "source": { "type": "git", "url": "https://github.com/mollie/mollie-api-php.git", - "reference": "3920816c311ec785f47f160204296d1b7f918da5" + "reference": "d7d09ac62a565e818bf49d04acb2f0432da758a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/mollie/mollie-api-php/zipball/3920816c311ec785f47f160204296d1b7f918da5", - "reference": "3920816c311ec785f47f160204296d1b7f918da5", + "url": "https://api.github.com/repos/mollie/mollie-api-php/zipball/d7d09ac62a565e818bf49d04acb2f0432da758a9", + "reference": "d7d09ac62a565e818bf49d04acb2f0432da758a9", "shasum": "" }, "require": { @@ -7547,9 +7548,9 @@ ], "support": { "issues": "https://github.com/mollie/mollie-api-php/issues", - "source": "https://github.com/mollie/mollie-api-php/tree/v2.65.0" + "source": "https://github.com/mollie/mollie-api-php/tree/v2.66.0" }, - "time": "2024-01-23T12:39:48+00:00" + "time": "2024-03-19T13:33:42+00:00" }, { "name": "moneyphp/money", @@ -9868,16 +9869,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.26.0", + "version": "1.27.0", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "231e3186624c03d7e7c890ec662b81e6b0405227" + "reference": "86e4d5a4b036f8f0be1464522f4c6b584c452757" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/231e3186624c03d7e7c890ec662b81e6b0405227", - "reference": "231e3186624c03d7e7c890ec662b81e6b0405227", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/86e4d5a4b036f8f0be1464522f4c6b584c452757", + "reference": "86e4d5a4b036f8f0be1464522f4c6b584c452757", "shasum": "" }, "require": { @@ -9909,9 +9910,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.26.0" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.27.0" }, - "time": "2024-02-23T16:05:55+00:00" + "time": "2024-03-21T13:14:53+00:00" }, { "name": "pragmarx/google2fa", @@ -11969,16 +11970,16 @@ }, { "name": "spatie/laravel-package-tools", - "version": "1.16.3", + "version": "1.16.4", "source": { "type": "git", "url": "https://github.com/spatie/laravel-package-tools.git", - "reference": "59db18c2e20d49a0b6d447bb1c654f6c123beb9e" + "reference": "ddf678e78d7f8b17e5cdd99c0c3413a4a6592e53" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/spatie/laravel-package-tools/zipball/59db18c2e20d49a0b6d447bb1c654f6c123beb9e", - "reference": "59db18c2e20d49a0b6d447bb1c654f6c123beb9e", + "url": "https://api.github.com/repos/spatie/laravel-package-tools/zipball/ddf678e78d7f8b17e5cdd99c0c3413a4a6592e53", + "reference": "ddf678e78d7f8b17e5cdd99c0c3413a4a6592e53", "shasum": "" }, "require": { @@ -12017,7 +12018,7 @@ ], "support": { "issues": "https://github.com/spatie/laravel-package-tools/issues", - "source": "https://github.com/spatie/laravel-package-tools/tree/1.16.3" + "source": "https://github.com/spatie/laravel-package-tools/tree/1.16.4" }, "funding": [ { @@ -12025,7 +12026,7 @@ "type": "github" } ], - "time": "2024-03-07T07:35:57+00:00" + "time": "2024-03-20T07:29:11+00:00" }, { "name": "spatie/php-structure-discoverer", @@ -16590,16 +16591,16 @@ }, { "name": "composer/pcre", - "version": "3.1.2", + "version": "3.1.3", "source": { "type": "git", "url": "https://github.com/composer/pcre.git", - "reference": "4775f35b2d70865807c89d32c8e7385b86eb0ace" + "reference": "5b16e25a5355f1f3afdfc2f954a0a80aec4826a8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/pcre/zipball/4775f35b2d70865807c89d32c8e7385b86eb0ace", - "reference": "4775f35b2d70865807c89d32c8e7385b86eb0ace", + "url": "https://api.github.com/repos/composer/pcre/zipball/5b16e25a5355f1f3afdfc2f954a0a80aec4826a8", + "reference": "5b16e25a5355f1f3afdfc2f954a0a80aec4826a8", "shasum": "" }, "require": { @@ -16641,7 +16642,7 @@ ], "support": { "issues": "https://github.com/composer/pcre/issues", - "source": "https://github.com/composer/pcre/tree/3.1.2" + "source": "https://github.com/composer/pcre/tree/3.1.3" }, "funding": [ { @@ -16657,7 +16658,7 @@ "type": "tidelift" } ], - "time": "2024-03-07T15:38:35+00:00" + "time": "2024-03-19T10:26:25+00:00" }, { "name": "composer/semver", @@ -16940,16 +16941,16 @@ }, { "name": "friendsofphp/php-cs-fixer", - "version": "v3.52.0", + "version": "v3.52.1", "source": { "type": "git", "url": "https://github.com/PHP-CS-Fixer/PHP-CS-Fixer.git", - "reference": "a3564bd66f4bce9bc871ef18b690e2dc67a7f969" + "reference": "6e77207f0d851862ceeb6da63e6e22c01b1587bc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PHP-CS-Fixer/PHP-CS-Fixer/zipball/a3564bd66f4bce9bc871ef18b690e2dc67a7f969", - "reference": "a3564bd66f4bce9bc871ef18b690e2dc67a7f969", + "url": "https://api.github.com/repos/PHP-CS-Fixer/PHP-CS-Fixer/zipball/6e77207f0d851862ceeb6da63e6e22c01b1587bc", + "reference": "6e77207f0d851862ceeb6da63e6e22c01b1587bc", "shasum": "" }, "require": { @@ -17020,7 +17021,7 @@ ], "support": { "issues": "https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/issues", - "source": "https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/tree/v3.52.0" + "source": "https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/tree/v3.52.1" }, "funding": [ { @@ -17028,7 +17029,7 @@ "type": "github" } ], - "time": "2024-03-18T18:40:11+00:00" + "time": "2024-03-19T21:02:43+00:00" }, { "name": "hamcrest/hamcrest-php", @@ -17083,25 +17084,25 @@ }, { "name": "laracasts/cypress", - "version": "3.0.1", + "version": "3.0.2", "source": { "type": "git", "url": "https://github.com/laracasts/cypress.git", - "reference": "dd4e61188d4edaf65ffa18851a5df38d0fa0619a" + "reference": "449f9d69da75091c77327093e5727a5c739a4cf8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laracasts/cypress/zipball/dd4e61188d4edaf65ffa18851a5df38d0fa0619a", - "reference": "dd4e61188d4edaf65ffa18851a5df38d0fa0619a", + "url": "https://api.github.com/repos/laracasts/cypress/zipball/449f9d69da75091c77327093e5727a5c739a4cf8", + "reference": "449f9d69da75091c77327093e5727a5c739a4cf8", "shasum": "" }, "require": { - "illuminate/support": "^6.0|^7.0|^8.0|^9.0|^10.0", + "illuminate/support": "^6.0|^7.0|^8.0|^9.0|^10.0|^11.0", "php": "^8.0" }, "require-dev": { - "orchestra/testbench": "^6.0|^7.0|^8.0", - "phpunit/phpunit": "^8.0|^9.5.10", + "orchestra/testbench": "^6.0|^7.0|^8.0|^9.0", + "phpunit/phpunit": "^8.0|^9.5.10|^10.5", "spatie/laravel-ray": "^1.29" }, "type": "library", @@ -17136,9 +17137,9 @@ ], "support": { "issues": "https://github.com/laracasts/cypress/issues", - "source": "https://github.com/laracasts/cypress/tree/3.0.1" + "source": "https://github.com/laracasts/cypress/tree/3.0.2" }, - "time": "2023-02-16T20:00:16+00:00" + "time": "2024-03-19T14:07:37+00:00" }, { "name": "larastan/larastan", @@ -17310,16 +17311,16 @@ }, { "name": "mockery/mockery", - "version": "1.6.9", + "version": "1.6.11", "source": { "type": "git", "url": "https://github.com/mockery/mockery.git", - "reference": "0cc058854b3195ba21dc6b1f7b1f60f4ef3a9c06" + "reference": "81a161d0b135df89951abd52296adf97deb0723d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/mockery/mockery/zipball/0cc058854b3195ba21dc6b1f7b1f60f4ef3a9c06", - "reference": "0cc058854b3195ba21dc6b1f7b1f60f4ef3a9c06", + "url": "https://api.github.com/repos/mockery/mockery/zipball/81a161d0b135df89951abd52296adf97deb0723d", + "reference": "81a161d0b135df89951abd52296adf97deb0723d", "shasum": "" }, "require": { @@ -17331,8 +17332,8 @@ "phpunit/phpunit": "<8.0" }, "require-dev": { - "phpunit/phpunit": "^8.5 || ^9.6.10", - "symplify/easy-coding-standard": "^12.0.8" + "phpunit/phpunit": "^8.5 || ^9.6.17", + "symplify/easy-coding-standard": "^12.1.14" }, "type": "library", "autoload": { @@ -17389,7 +17390,7 @@ "security": "https://github.com/mockery/mockery/security/advisories", "source": "https://github.com/mockery/mockery" }, - "time": "2023-12-10T02:24:34+00:00" + "time": "2024-03-21T18:34:15+00:00" }, { "name": "myclabs/deep-copy", @@ -17754,16 +17755,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.10.63", + "version": "1.10.65", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "ad12836d9ca227301f5fb9960979574ed8628339" + "reference": "3c657d057a0b7ecae19cb12db446bbc99d8839c6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/ad12836d9ca227301f5fb9960979574ed8628339", - "reference": "ad12836d9ca227301f5fb9960979574ed8628339", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/3c657d057a0b7ecae19cb12db446bbc99d8839c6", + "reference": "3c657d057a0b7ecae19cb12db446bbc99d8839c6", "shasum": "" }, "require": { @@ -17812,7 +17813,7 @@ "type": "tidelift" } ], - "time": "2024-03-18T16:53:53+00:00" + "time": "2024-03-23T10:30:26+00:00" }, { "name": "phpunit/php-code-coverage", @@ -18137,16 +18138,16 @@ }, { "name": "phpunit/phpunit", - "version": "10.5.13", + "version": "10.5.15", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "20a63fc1c6db29b15da3bd02d4b6cf59900088a7" + "reference": "86376e05e8745ed81d88232ff92fee868247b07b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/20a63fc1c6db29b15da3bd02d4b6cf59900088a7", - "reference": "20a63fc1c6db29b15da3bd02d4b6cf59900088a7", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/86376e05e8745ed81d88232ff92fee868247b07b", + "reference": "86376e05e8745ed81d88232ff92fee868247b07b", "shasum": "" }, "require": { @@ -18218,7 +18219,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.13" + "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.15" }, "funding": [ { @@ -18234,7 +18235,7 @@ "type": "tidelift" } ], - "time": "2024-03-12T15:37:41+00:00" + "time": "2024-03-22T04:17:47+00:00" }, { "name": "sebastian/cli-parser", @@ -18608,16 +18609,16 @@ }, { "name": "sebastian/environment", - "version": "6.0.1", + "version": "6.1.0", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/environment.git", - "reference": "43c751b41d74f96cbbd4e07b7aec9675651e2951" + "reference": "8074dbcd93529b357029f5cc5058fd3e43666984" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/43c751b41d74f96cbbd4e07b7aec9675651e2951", - "reference": "43c751b41d74f96cbbd4e07b7aec9675651e2951", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/8074dbcd93529b357029f5cc5058fd3e43666984", + "reference": "8074dbcd93529b357029f5cc5058fd3e43666984", "shasum": "" }, "require": { @@ -18632,7 +18633,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "6.0-dev" + "dev-main": "6.1-dev" } }, "autoload": { @@ -18660,7 +18661,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/environment/issues", "security": "https://github.com/sebastianbergmann/environment/security/policy", - "source": "https://github.com/sebastianbergmann/environment/tree/6.0.1" + "source": "https://github.com/sebastianbergmann/environment/tree/6.1.0" }, "funding": [ { @@ -18668,7 +18669,7 @@ "type": "github" } ], - "time": "2023-04-11T05:39:26+00:00" + "time": "2024-03-23T08:47:14+00:00" }, { "name": "sebastian/exporter", diff --git a/config/services.php b/config/services.php index ae434aba6575..b0d9851df34e 100644 --- a/config/services.php +++ b/config/services.php @@ -31,7 +31,7 @@ return [ ], 'brevo' => [ - 'key' => env('BREVO_SECRET', ''), + 'secret' => env('BREVO_SECRET', ''), ], 'postmark' => [ @@ -39,12 +39,12 @@ return [ ], 'postmark-outlook' => [ - 'token' => env('POSTMARK_OUTLOOK_SECRET',''), + 'token' => env('POSTMARK_OUTLOOK_SECRET', ''), 'from' => [ 'address' => env('POSTMARK_OUTLOOK_FROM_ADDRESS', '') ], ], - + 'microsoft' => [ 'client_id' => env('MICROSOFT_CLIENT_ID'), 'client_secret' => env('MICROSOFT_CLIENT_SECRET'), @@ -62,6 +62,7 @@ return [ 'secret' => env('AWS_SECRET_ACCESS_KEY'), 'region' => env('SES_REGION', 'us-east-1'), ], + 'sparkpost' => [ 'secret' => env('SPARKPOST_SECRET'), ], @@ -117,6 +118,7 @@ return [ 'key' => env('ZIP_TAX_KEY', false), ], ], + 'chorus' => [ 'client_id' => env('CHORUS_CLIENT_ID', false), 'secret' => env('CHORUS_SECRET', false), diff --git a/routes/api.php b/routes/api.php index b404ef53b465..4560a549174a 100644 --- a/routes/api.php +++ b/routes/api.php @@ -431,6 +431,7 @@ Route::post('api/v1/postmark_webhook', [PostMarkController::class, 'webhook'])-> Route::post('api/v1/mailgun_webhook', [MailgunController::class, 'webhook'])->middleware('throttle:1000,1'); Route::post('api/v1/mailgun_inbound_webhook', [MailgunController::class, 'inboundWebhook'])->middleware('throttle:1000,1'); Route::post('api/v1/brevo_webhook', [BrevoController::class, 'webhook'])->middleware('throttle:1000,1'); +Route::post('api/v1/brevo_inbound_webhook', [BrevoController::class, 'inboundWebhook'])->middleware('throttle:1000,1'); Route::get('token_hash_router', [OneTimeTokenController::class, 'router'])->middleware('throttle:500,1'); Route::get('webcron', [WebCronController::class, 'index'])->middleware('throttle:100,1'); Route::post('api/v1/get_migration_account', [HostedMigrationController::class, 'getAccount'])->middleware('guest')->middleware('throttle:100,1'); From a6d09a2ce5277946a455a700e1fe4b2172065e09 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 24 Mar 2024 11:00:29 +0100 Subject: [PATCH 043/119] feat: global blacklist for specific emails --- app/Services/IngresEmail/IngresEmailEngine.php | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/app/Services/IngresEmail/IngresEmailEngine.php b/app/Services/IngresEmail/IngresEmailEngine.php index bb03f9d5cce8..64f3d28ee165 100644 --- a/app/Services/IngresEmail/IngresEmailEngine.php +++ b/app/Services/IngresEmail/IngresEmailEngine.php @@ -34,7 +34,8 @@ class IngresEmailEngine private ?Company $company; private ?bool $isUnknownRecipent = null; - private array $globalBlacklist = []; + private array $globalBlacklistDomains = []; + private array $globalBlacklistEmails = []; public function __construct(private IngresEmail $email) { } @@ -72,8 +73,12 @@ class IngresEmailEngine $domain = array_pop($parts); // global blacklist - if (in_array($domain, $this->globalBlacklist)) { - Log::info('[IngressMailEngine] E-Mail blocked, because the domain was found on globalBlocklist: ' . $this->email->from); + if (in_array($domain, $this->globalBlacklistDomains)) { + Log::info('[IngressMailEngine] E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $this->email->from); + return true; + } + if (in_array($this->email->from, $this->globalBlacklistEmails)) { + Log::info('[IngressMailEngine] E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $this->email->from); return true; } From 8031f1c277d9683ea384686d31097de3c8dbb121 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 24 Mar 2024 11:40:17 +0100 Subject: [PATCH 044/119] fixes for company blacklists and whitelists + system logs --- app/Models/SystemLog.php | 2 + .../IngresEmail/IngresEmailEngine.php | 39 +++++++++++++------ ...10951_create_imap_configuration_fields.php | 8 ++-- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/app/Models/SystemLog.php b/app/Models/SystemLog.php index 7da6583706a1..1ab3cafbae65 100644 --- a/app/Models/SystemLog.php +++ b/app/Models/SystemLog.php @@ -113,6 +113,8 @@ class SystemLog extends Model public const EVENT_USER = 61; + public const EVENT_INGEST_EMAIL_FAILURE = 62; + /*Type IDs*/ public const TYPE_PAYPAL = 300; diff --git a/app/Services/IngresEmail/IngresEmailEngine.php b/app/Services/IngresEmail/IngresEmailEngine.php index 64f3d28ee165..e218a8d3256c 100644 --- a/app/Services/IngresEmail/IngresEmailEngine.php +++ b/app/Services/IngresEmail/IngresEmailEngine.php @@ -13,8 +13,10 @@ namespace App\Services\IngresEmail; use App\Events\Expense\ExpenseWasCreated; use App\Factory\ExpenseFactory; +use App\Jobs\Util\SystemLogger; use App\Libraries\MultiDB; use App\Models\Company; +use App\Models\SystemLog; use App\Models\Vendor; use App\Models\VendorContact; use App\Services\IngresEmail\IngresEmail; @@ -65,7 +67,7 @@ class IngresEmailEngine { // invalid email if (!filter_var($this->email->from, FILTER_VALIDATE_EMAIL)) { - Log::info('[IngressMailEngine] E-Mail blocked, because from e-mail has the wrong format: ' . $this->email->from); + $this->log('E-Mail blocked, because from e-mail has the wrong format: ' . $this->email->from); return true; } @@ -74,11 +76,11 @@ class IngresEmailEngine // global blacklist if (in_array($domain, $this->globalBlacklistDomains)) { - Log::info('[IngressMailEngine] E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $this->email->from); + $this->log('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $this->email->from); return true; } if (in_array($this->email->from, $this->globalBlacklistEmails)) { - Log::info('[IngressMailEngine] E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $this->email->from); + $this->log('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $this->email->from); return true; } @@ -89,12 +91,12 @@ class IngresEmailEngine // sender occured in more than 500 emails in the last 12 hours $senderMailCountTotal = Cache::get('ingresEmailSender:' . $this->email->from, 0); if ($senderMailCountTotal >= 5000) { - Log::info('[IngressMailEngine] E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); + $this->log('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); $this->blockSender(); return true; } if ($senderMailCountTotal >= 1000) { - Log::info('[IngressMailEngine] E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); + $this->log('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); $this->saveMeta(); return true; } @@ -102,7 +104,7 @@ class IngresEmailEngine // sender sended more than 50 emails to the wrong mailbox in the last 6 hours $senderMailCountUnknownRecipent = Cache::get('ingresEmailSenderUnknownRecipent:' . $this->email->from, 0); if ($senderMailCountUnknownRecipent >= 50) { - Log::info('[IngressMailEngine] E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $this->email->from); + $this->log('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $this->email->from); $this->saveMeta(); return true; } @@ -110,7 +112,7 @@ class IngresEmailEngine // wrong recipent occurs in more than 100 emails in the last 12 hours, so the processing is blocked $mailCountUnknownRecipent = Cache::get('ingresEmailUnknownRecipent:' . $this->email->to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time if ($mailCountUnknownRecipent >= 100) { - Log::info('[IngressMailEngine] E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); + $this->log('E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); $this->blockSender(); return true; } @@ -154,15 +156,15 @@ class IngresEmailEngine { // Skipping executions: will not result in not saving Metadata to prevent usage of these conditions, to spam if (!$this->validateExpenseShouldProcess()) { - Log::info('email parsing not active for this company: ' . $this->company->id . ' from: ' . $this->email->from); + $this->log('email parsing not active for this company. from: ' . $this->email->from); return; } if (!$this->validateExpenseSender()) { - Log::info('invalid sender of an ingest email to company: ' . $this->company->id . ' from: ' . $this->email->from); + $this->log('invalid sender of an ingest email for this company. from: ' . $this->email->from); return; } if (sizeOf($this->email->documents) == 0) { - Log::info('email does not contain any attachments and is likly not an expense. company: ' . $this->company->id . ' from: ' . $this->email->from); + $this->log('email does not contain any attachments and is likly not an expense. from: ' . $this->email->from); return; } @@ -218,7 +220,7 @@ class IngresEmailEngine return false; // allow unknown - if ($this->company->expense_mailbox_allow_unknown && sizeOf($email_whitelist) == 0 && sizeOf($domain_whitelist) == 0) // from unknown only, when no whitelists are defined + if ($this->company->expense_mailbox_allow_unknown) return true; // own users @@ -249,4 +251,19 @@ class IngresEmailEngine return $vendor; } + private function log(string $data) + { + Log::info("[IngresEmailEngine][company:" . $this->company->id . "] " . $data); + + ( + new SystemLogger( + $data, + SystemLog::CATEGORY_MAIL, + SystemLog::EVENT_INGEST_EMAIL_FAILURE, + SystemLog::TYPE_CUSTOM, + null, + $this->company + ) + )->handle(); + } } diff --git a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php index 5a34ab745f7b..8a8d62e22918 100644 --- a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php +++ b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php @@ -17,10 +17,10 @@ return new class extends Migration { $table->boolean("expense_mailbox_allow_company_users")->default(false); $table->boolean("expense_mailbox_allow_vendors")->default(false); $table->boolean("expense_mailbox_allow_unknown")->default(false); - $table->string("expense_mailbox_whitelist_domains")->nullable(); - $table->string("expense_mailbox_whitelist_emails")->nullable(); - $table->string("expense_mailbox_blacklist_domains")->nullable(); - $table->string("expense_mailbox_blacklist_emails")->nullable(); + $table->text("expense_mailbox_whitelist_domains")->nullable(); + $table->text("expense_mailbox_whitelist_emails")->nullable(); + $table->text("expense_mailbox_blacklist_domains")->nullable(); + $table->text("expense_mailbox_blacklist_emails")->nullable(); }); Schema::table('vendors', function (Blueprint $table) { $table->string("invoicing_email")->nullable(); From 2cbeb23d8fe06ec83344c06cb8a2014615a02d23 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 24 Mar 2024 12:51:24 +0100 Subject: [PATCH 045/119] minor logging improvements --- app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 6 +++--- app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index aab2adad3a29..d97fb7245496 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -124,13 +124,13 @@ class ProcessBrevoInboundWebhook implements ShouldQueue // match company $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); if (!$company) { - Log::info('unknown Expense Mailbox occured while handling an inbound email from brevo: ' . $recipient); + Log::info('[ProcessBrevoInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from brevo: ' . $recipient); continue; } $company_brevo_secret = $company->settings?->email_sending_method === 'client_brevo' && $company->settings?->brevo_secret ? $company->settings?->brevo_secret : null; if (empty ($company_brevo_secret) && empty (config('services.brevo.secret'))) - throw new \Error("no brevo credenitals found, we cannot get the attachement"); + throw new \Error("[ProcessBrevoInboundWebhook] no brevo credenitals found, we cannot get the attachement"); // prepare data for ingresEngine $ingresEmail = new IngresEmail(); @@ -156,7 +156,7 @@ class ProcessBrevoInboundWebhook implements ShouldQueue } catch (\Error $e) { if (config('services.brevo.secret')) { - Log::info("Error while downloading with company credentials, we try to use defaul credentials now..."); + Log::info("[ProcessBrevoInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); $attachment = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index 11e81f1b0633..eac0e1bd04d2 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -168,7 +168,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue // match company $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); if (!$company) { - Log::info('unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $recipient); + Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $recipient); return; } @@ -176,7 +176,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $company_mailgun_domain = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_domain ? $company->settings?->mailgun_domain : null; $company_mailgun_secret = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_secret ? $company->settings?->mailgun_secret : null; if (!($company_mailgun_domain && $company_mailgun_secret) && !(config('services.mailgun.domain') && config('services.mailgun.secret'))) - throw new \Error("no mailgun credenitals found, we cannot get the attachements and files"); + throw new \Error("[ProcessMailgunInboundWebhook] no mailgun credenitals found, we cannot get the attachements and files"); $mail = null; if ($company_mailgun_domain && $company_mailgun_secret) { @@ -190,7 +190,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $mail = json_decode(file_get_contents($messageUrl)); } catch (\Error $e) { if (config('services.mailgun.secret')) { - Log::info("Error while downloading with company credentials, we try to use defaul credentials now..."); + Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; $messageUrl = explode("|", $this->input)[1]; @@ -238,7 +238,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue } catch (\Error $e) { if (config('services.mailgun.secret')) { - Log::info("Error while downloading with company credentials, we try to use defaul credentials now..."); + Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; $url = $attachment->url; From d7121910124f9cfec8d8e679b7dcb4ed42b545ab Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 24 Mar 2024 13:10:45 +0100 Subject: [PATCH 046/119] minor changes --- app/Http/Controllers/BrevoController.php | 6 +++--- app/Http/Controllers/MailgunController.php | 1 + app/Services/IngresEmail/IngresEmailEngine.php | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/app/Http/Controllers/BrevoController.php b/app/Http/Controllers/BrevoController.php index c87b4ff0f285..a81d1320e5c5 100644 --- a/app/Http/Controllers/BrevoController.php +++ b/app/Http/Controllers/BrevoController.php @@ -185,9 +185,9 @@ class BrevoController extends BaseController { $input = $request->all(); - // TODO: validation for client credentials by recipient - // if (!($request->has('token') && $request->get('token') == config('services.brevo.secret'))) - // return response()->json(['message' => 'Unauthorized'], 403); + // TODO: validation for client mail credentials by recipient + if (!($request->has('token') && $request->get('token') == config('services.brevo.secret'))) + return response()->json(['message' => 'Unauthorized'], 403); if (!array_key_exists('items', $input)) { Log::info('Failed: Message could not be parsed, because required parameters are missing.'); diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index b26952598005..36875eb8d624 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -131,6 +131,7 @@ class MailgunController extends BaseController } // @turbo124 TODO: how to check for services.mailgun.webhook_signing_key on company level, when custom credentials are defined + // TODO: validation for client mail credentials by recipient if (\hash_equals(\hash_hmac('sha256', $input['timestamp'] . $input['token'], config('services.mailgun.webhook_signing_key')), $input['signature'])) { ProcessMailgunInboundWebhook::dispatch($input["recipient"] . "|" . $input["message-url"])->delay(10); diff --git a/app/Services/IngresEmail/IngresEmailEngine.php b/app/Services/IngresEmail/IngresEmailEngine.php index e218a8d3256c..b8a5013a5421 100644 --- a/app/Services/IngresEmail/IngresEmailEngine.php +++ b/app/Services/IngresEmail/IngresEmailEngine.php @@ -156,7 +156,7 @@ class IngresEmailEngine { // Skipping executions: will not result in not saving Metadata to prevent usage of these conditions, to spam if (!$this->validateExpenseShouldProcess()) { - $this->log('email parsing not active for this company. from: ' . $this->email->from); + $this->log('mailbox not active for this company. from: ' . $this->email->from); return; } if (!$this->validateExpenseSender()) { From e888cde7dd1face0b40ca633c3635cb5c80d4f53 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 24 Mar 2024 13:32:52 +0100 Subject: [PATCH 047/119] removing webhook soft-failes in favor for system logging --- app/Http/Controllers/BrevoController.php | 5 ----- app/Http/Controllers/MailgunController.php | 10 ---------- 2 files changed, 15 deletions(-) diff --git a/app/Http/Controllers/BrevoController.php b/app/Http/Controllers/BrevoController.php index a81d1320e5c5..bd4f1ad59480 100644 --- a/app/Http/Controllers/BrevoController.php +++ b/app/Http/Controllers/BrevoController.php @@ -201,11 +201,6 @@ class BrevoController extends BaseController return response()->json(['message' => 'Failed. Invalid Parameters. At least one item was invalid.'], 400); } - if (!array_key_exists('Attachments', $item) || count($item['Attachments']) == 0) { - Log::info('Brevo: InboundParsing: SoftFail: Message ignored because of missing attachments. No Actions would have been taken...'); - continue; - } - ProcessBrevoInboundWebhook::dispatch($item)->delay(10); } diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 36875eb8d624..8554f96b6feb 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -120,16 +120,6 @@ class MailgunController extends BaseController return response()->json(['message' => 'Failed. Missing Parameters. Use store and notify!'], 400); } - if (!array_key_exists('attachments', $input) || count(json_decode($input['attachments'])) == 0) { - Log::info('Message ignored because of missing attachments. No Actions would have been taken...'); - return response()->json(['message' => 'Sucess. Soft Fail. Missing Attachments.'], 200); - } - - if (\abs(\time() - (int) $input['timestamp']) > 150) { - Log::info('Message ignored because of request body is too old.'); - return response()->json(['message' => 'Success. Soft Fail. Message too old.'], 200); - } - // @turbo124 TODO: how to check for services.mailgun.webhook_signing_key on company level, when custom credentials are defined // TODO: validation for client mail credentials by recipient if (\hash_equals(\hash_hmac('sha256', $input['timestamp'] . $input['token'], config('services.mailgun.webhook_signing_key')), $input['signature'])) { From 91a048009a23e4251f137807f68a6b0b6e7cac5e Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 24 Mar 2024 21:53:17 +0100 Subject: [PATCH 048/119] init postmark webhook --- app/Http/Controllers/PostMarkController.php | 164 +++++++++- .../ProcessPostmarkInboundWebhook.php | 297 +++++++++++++++--- routes/api.php | 1 + 3 files changed, 417 insertions(+), 45 deletions(-) diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index 44227ce9c8f1..e0df719557c5 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -14,6 +14,7 @@ namespace App\Http\Controllers; use App\Jobs\PostMark\ProcessPostmarkInboundWebhook; use App\Jobs\PostMark\ProcessPostmarkWebhook; use Illuminate\Http\Request; +use Log; /** * Class PostMarkController. @@ -104,11 +105,172 @@ class PostMarkController extends BaseController * @OA\JsonContent(ref="#/components/schemas/Error"), * ), * ) + * + * array ( + * 'FromName' => 'Max Mustermann', + * 'MessageStream' => 'inbound', + * 'From' => 'max@mustermann.de', + * 'FromFull' => + * array ( + * 'Email' => 'max@mustermann.de', + * 'Name' => 'Max Mustermann', + * 'MailboxHash' => NULL, + * ), + * 'To' => '370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com', + * 'ToFull' => + * array ( + * 0 => + * array ( + * 'Email' => '370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com', + * 'Name' => NULL, + * 'MailboxHash' => NULL, + * ), + * ), + * 'Cc' => NULL, + * 'CcFull' => + * array ( + * ), + * 'Bcc' => NULL, + * 'BccFull' => + * array ( + * ), + * 'OriginalRecipient' => '370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com', + * 'Subject' => 'Re: adaw', + * 'MessageID' => 'd37fde00-b4cf-4b64-ac64-e9f6da523c25', + * 'ReplyTo' => NULL, + * 'MailboxHash' => NULL, + * 'Date' => 'Sun, 24 Mar 2024 13:17:52 +0100', + * 'TextBody' => 'wadwad + * + * Am So., 24. März 2024 um 13:17 Uhr schrieb Max Mustermann : + * + * > test + * > + * + * -- + * test.de - Max Mustermann kontakt@test.de + * ', + * 'HtmlBody' => '
wadwad

Am So., 24. März 2024 um 13:17 Uhr schrieb Max Mustermann <max@mustermann.de>:
test
+ *
+ * + *
+ * test.de - Max Mustermann', + * 'StrippedTextReply' => 'wadwad + * + * Am So., 24. März 2024 um 13:17 Uhr schrieb Max Mustermann :', + * 'Tag' => NULL, + * 'Headers' => + * array ( + * 0 => + * array ( + * 'Name' => 'Return-Path', + * 'Value' => '', + * ), + * 1 => + * array ( + * 'Name' => 'Received', + * 'Value' => 'by p-pm-inboundg02a-aws-euwest1a.inbound.postmarkapp.com (Postfix, from userid 996) id 8ED1A453CA4; Sun, 24 Mar 2024 12:18:10 +0000 (UTC)', + * ), + * 2 => + * array ( + * 'Name' => 'X-Spam-Checker-Version', + * 'Value' => 'SpamAssassin 3.4.0 (2014-02-07) on p-pm-inboundg02a-aws-euwest1a', + * ), + * 3 => + * array ( + * 'Name' => 'X-Spam-Status', + * 'Value' => 'No', + * ), + * 4 => + * array ( + * 'Name' => 'X-Spam-Score', + * 'Value' => '-0.1', + * ), + * 5 => + * array ( + * 'Name' => 'X-Spam-Tests', + * 'Value' => 'DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,HTML_MESSAGE, RCVD_IN_DNSWL_NONE,RCVD_IN_MSPIKE_H2,RCVD_IN_ZEN_BLOCKED_OPENDNS, SPF_HELO_NONE,SPF_PASS,URIBL_DBL_BLOCKED_OPENDNS,URIBL_ZEN_BLOCKED_OPENDNS', + * ), + * 6 => + * array ( + * 'Name' => 'Received-SPF', + * 'Value' => 'pass (test.de: Sender is authorized to use \'max@mustermann.de\' in \'mfrom\' identity (mechanism \'include:_spf.google.com\' matched)) receiver=p-pm-inboundg02a-aws-euwest1a; identity=mailfrom; envelope-from="max@mustermann.de"; helo=mail-lf1-f51.google.com; client-ip=209.85.167.51', + * ), + * 7 => + * array ( + * 'Name' => 'Received', + * 'Value' => 'from mail-lf1-f51.google.com (mail-lf1-f51.google.com [209.85.167.51]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by p-pm-inboundg02a-aws-euwest1a.inbound.postmarkapp.com (Postfix) with ESMTPS id 437BD453CA2 for <370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com>; Sun, 24 Mar 2024 12:18:10 +0000 (UTC)', + * ), + * 8 => + * array ( + * 'Name' => 'Received', + * 'Value' => 'by mail-lf1-f51.google.com with SMTP id 2adb3069b0e04-513cf9bacf1so4773866e87.0 for <370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com>; Sun, 24 Mar 2024 05:18:10 -0700 (PDT)', + * ), + * 9 => + * array ( + * 'Name' => 'DKIM-Signature', + * 'Value' => 'v=1; a=rsa-sha256; c=relaxed/relaxed; d=test.de; s=google; t=1711282689; x=1711887489; darn=inbound.postmarkapp.com; h=to:subject:message-id:date:from:in-reply-to:references:mime-version :from:to:cc:subject:date:message-id:reply-to; bh=NvjmqLXF/5L5ZrpToR/6FgVOhTOGC9j0/B2Na5Ke6J8=; b=AMXIEoh6yGrOT6X3eBBClQ3NXFNuEoqxeM6aPONsqbpShAcT24iAJmqXylaLHv3fyX Hm6mwp3a029NnrLP/VRyKZbzIMBN2iycidtrEMXF/Eg2e42Q/08/2dZ7nxH6NqE/jz01 3M7qvwHvuoZ2Knhj7rnZc6I5m/nFxBsZc++Aj0Vv9sFoWZZooqAeTXbux1I5NyE17MrL D6byca43iINARZN7XOkoChRRZoZbOqZEtc2Va5yw7v+aYguLB4HHrIFC7G+L8hAJ0IAo 3R3DFeBw58M1xtxXCREI8Y6qMQTw60XyFw0gVmZzqR4hZiTerBSJJsZLZOBgmXxq3WLS +xVQ==', + * ), + * 10 => + * array ( + * 'Name' => 'X-Google-DKIM-Signature', + * 'Value' => 'v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1711282689; x=1711887489; h=to:subject:message-id:date:from:in-reply-to:references:mime-version :x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=NvjmqLXF/5L5ZrpToR/6FgVOhTOGC9j0/B2Na5Ke6J8=; b=uKoMhir+MX/wycNEr29Sffj45ooKksCJ1OfSRkIIGHk0rnHn8Vh+c7beYipwRPW4F2 h46K64vtIX00guYMdL2Qo2eY96+wALTqHCy67PGhvotVTROz21yxjx62pCDPGs5tefOu IkyxoybpIK8zAfLoDTd9p2GIrr5brKJyB2w1NQc1htxTQ5D4RgBxUAOKv4uVEr8r47iA MIo5d8/AifA+vCOAh7iJ7EmvDQ1R+guhQyH9m1Jo8PLapiYuHXggpBJvooyGuflKqbnt gJ/dscEr4d5aWJbw/x1dmIJ5gyJPGdBWq8NRqV/qbkXQW3H/gylifDUPXbki+EQBD5Yu EuLQ==', + * ), + * 11 => + * array ( + * 'Name' => 'X-Gm-Message-State', + * 'Value' => 'AOJu0Yxpbp1sRh17lNzg+pLnIx1jCn8ZFJQMgFuHK+6Z8RqFS5KKKTxR 8onpEbxWYYVUbrJFExNBHPD/3jdxqifCVVNaDmbpwHgmW5lHLJmA5vYRq5NFZ9OA6zKx/N6Gipr iXE4fXmSqghFNTzy9V/RT08Zp+F5RiFh/Ta6ltQl8XfCPFfSawLz6cagUgt8bBuF4RqdrYmWwzj ty86V5Br1htRNEFYivoXnNmaRcsD0tca1D23ny62O6RwWugrj1IpAYhViNyTZAWu+loKgfjJJoI MsyiSU=', + * ), + * 12 => + * array ( + * 'Name' => 'X-Google-Smtp-Source', + * 'Value' => 'AGHT+IEdtZqbVI6j7WLeaSL3dABGSnWIXaSjbYqXvFvE2H+f2zsn0gknQ4OdTJecQRCabpypVF2ue91Jb7aKl6RiyEQ=', + * ), + * 13 => + * array ( + * 'Name' => 'X-Received', + * 'Value' => 'by 2002:a19:385a:0:b0:513:c876:c80a with SMTP id d26-20020a19385a000000b00513c876c80amr2586776lfj.34.1711282689140; Sun, 24 Mar 2024 05:18:09 -0700 (PDT)', + * ), + * 14 => + * array ( + * 'Name' => 'MIME-Version', + * 'Value' => '1.0', + * ), + * 15 => + * array ( + * 'Name' => 'References', + * 'Value' => '', + * ), + * 16 => + * array ( + * 'Name' => 'In-Reply-To', + * 'Value' => '', + * ), + * 17 => + * array ( + * 'Name' => 'Message-ID', + * 'Value' => '', + * ), + * ), + * 'Attachments' => + * array ( + * ), + * ) */ public function inboundWebhook(Request $request) { + + Log::info($request->all()); + + $input = $request->all(); + + if (!(array_key_exists("MessageStream", $input) && $input["MessageStream"] != "inbound") || !array_key_exists("To", $input) || !array_key_exists("MessageID", $input)) { + Log::info('Failed: Message could not be parsed, because required parameters are missing. Please ensure contacting this api-endpoint with a store & notify operation instead of a forward operation!'); + return response()->json(['message' => 'Failed. Missing/Invalid Parameters.'], 400); + } + if ($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.postmark.token')) { - ProcessPostmarkInboundWebhook::dispatch($request->all())->delay(10); + ProcessPostmarkInboundWebhook::dispatch($input["To"] . "|" . $input["MessageID"])->delay(10); return response()->json(['message' => 'Success'], 200); } diff --git a/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php b/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php index 6e54fcdf05a3..d616294a666c 100644 --- a/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php +++ b/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php @@ -9,16 +9,19 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Jobs\PostMark; +namespace App\Jobs\Postmark; -use App\Helpers\Mail\Webhook\Postmark\PostmarkWebhookHandler; use App\Libraries\MultiDB; -use App\Models\SystemLog; +use App\Services\IngresEmail\IngresEmail; +use App\Services\IngresEmail\IngresEmailEngine; +use App\Utils\TempFile; +use Illuminate\Support\Carbon; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Log; class ProcessPostmarkInboundWebhook implements ShouldQueue { @@ -26,65 +29,271 @@ class ProcessPostmarkInboundWebhook implements ShouldQueue public $tries = 1; - public $invitation; - - private $entity; - - private array $default_response = [ - 'recipients' => '', - 'subject' => 'Message not found.', - 'entity' => '', - 'entity_id' => '', - 'events' => [], - ]; - /** * Create a new job instance. - * + * $input consists of 2 informations: recipient|messageId */ - public function __construct(private array $request) + public function __construct(private string $input) { } - private function getSystemLog(string $message_id): ?SystemLog - { - return SystemLog::query() - ->where('company_id', $this->invitation->company_id) - ->where('type_id', SystemLog::TYPE_WEBHOOK_RESPONSE) - ->whereJsonContains('log', ['MessageID' => $message_id]) - ->orderBy('id', 'desc') - ->first(); - - } - - private function updateSystemLog(SystemLog $system_log, array $data): void - { - $system_log->log = $data; - $system_log->save(); - } - /** * Execute the job. * + * Mail from Storage + * array ( + * 'FromName' => 'Max Mustermann', + * 'MessageStream' => 'inbound', + * 'From' => 'max@mustermann.de', + * 'FromFull' => + * array ( + * 'Email' => 'max@mustermann.de', + * 'Name' => 'Max Mustermann', + * 'MailboxHash' => NULL, + * ), + * 'To' => '370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com', + * 'ToFull' => + * array ( + * 0 => + * array ( + * 'Email' => '370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com', + * 'Name' => NULL, + * 'MailboxHash' => NULL, + * ), + * ), + * 'Cc' => NULL, + * 'CcFull' => + * array ( + * ), + * 'Bcc' => NULL, + * 'BccFull' => + * array ( + * ), + * 'OriginalRecipient' => '370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com', + * 'Subject' => 'Re: adaw', + * 'MessageID' => 'd37fde00-b4cf-4b64-ac64-e9f6da523c25', + * 'ReplyTo' => NULL, + * 'MailboxHash' => NULL, + * 'Date' => 'Sun, 24 Mar 2024 13:17:52 +0100', + * 'TextBody' => 'wadwad * + * Am So., 24. März 2024 um 13:17 Uhr schrieb Max Mustermann : + * + * > test + * > + * + * -- + * test.de - Max Mustermann kontakt@test.de + * ', + * 'HtmlBody' => '
wadwad

Am So., 24. März 2024 um 13:17 Uhr schrieb Max Mustermann <max@mustermann.de>:
test
+ *
+ * + *
+ * test.de - Max Mustermann', + * 'StrippedTextReply' => 'wadwad + * + * Am So., 24. März 2024 um 13:17 Uhr schrieb Max Mustermann :', + * 'Tag' => NULL, + * 'Headers' => + * array ( + * 0 => + * array ( + * 'Name' => 'Return-Path', + * 'Value' => '', + * ), + * 1 => + * array ( + * 'Name' => 'Received', + * 'Value' => 'by p-pm-inboundg02a-aws-euwest1a.inbound.postmarkapp.com (Postfix, from userid 996) id 8ED1A453CA4; Sun, 24 Mar 2024 12:18:10 +0000 (UTC)', + * ), + * 2 => + * array ( + * 'Name' => 'X-Spam-Checker-Version', + * 'Value' => 'SpamAssassin 3.4.0 (2014-02-07) on p-pm-inboundg02a-aws-euwest1a', + * ), + * 3 => + * array ( + * 'Name' => 'X-Spam-Status', + * 'Value' => 'No', + * ), + * 4 => + * array ( + * 'Name' => 'X-Spam-Score', + * 'Value' => '-0.1', + * ), + * 5 => + * array ( + * 'Name' => 'X-Spam-Tests', + * 'Value' => 'DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,HTML_MESSAGE, RCVD_IN_DNSWL_NONE,RCVD_IN_MSPIKE_H2,RCVD_IN_ZEN_BLOCKED_OPENDNS, SPF_HELO_NONE,SPF_PASS,URIBL_DBL_BLOCKED_OPENDNS,URIBL_ZEN_BLOCKED_OPENDNS', + * ), + * 6 => + * array ( + * 'Name' => 'Received-SPF', + * 'Value' => 'pass (test.de: Sender is authorized to use \'max@mustermann.de\' in \'mfrom\' identity (mechanism \'include:_spf.google.com\' matched)) receiver=p-pm-inboundg02a-aws-euwest1a; identity=mailfrom; envelope-from="max@mustermann.de"; helo=mail-lf1-f51.google.com; client-ip=209.85.167.51', + * ), + * 7 => + * array ( + * 'Name' => 'Received', + * 'Value' => 'from mail-lf1-f51.google.com (mail-lf1-f51.google.com [209.85.167.51]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by p-pm-inboundg02a-aws-euwest1a.inbound.postmarkapp.com (Postfix) with ESMTPS id 437BD453CA2 for <370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com>; Sun, 24 Mar 2024 12:18:10 +0000 (UTC)', + * ), + * 8 => + * array ( + * 'Name' => 'Received', + * 'Value' => 'by mail-lf1-f51.google.com with SMTP id 2adb3069b0e04-513cf9bacf1so4773866e87.0 for <370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com>; Sun, 24 Mar 2024 05:18:10 -0700 (PDT)', + * ), + * 9 => + * array ( + * 'Name' => 'DKIM-Signature', + * 'Value' => 'v=1; a=rsa-sha256; c=relaxed/relaxed; d=test.de; s=google; t=1711282689; x=1711887489; darn=inbound.postmarkapp.com; h=to:subject:message-id:date:from:in-reply-to:references:mime-version :from:to:cc:subject:date:message-id:reply-to; bh=NvjmqLXF/5L5ZrpToR/6FgVOhTOGC9j0/B2Na5Ke6J8=; b=AMXIEoh6yGrOT6X3eBBClQ3NXFNuEoqxeM6aPONsqbpShAcT24iAJmqXylaLHv3fyX Hm6mwp3a029NnrLP/VRyKZbzIMBN2iycidtrEMXF/Eg2e42Q/08/2dZ7nxH6NqE/jz01 3M7qvwHvuoZ2Knhj7rnZc6I5m/nFxBsZc++Aj0Vv9sFoWZZooqAeTXbux1I5NyE17MrL D6byca43iINARZN7XOkoChRRZoZbOqZEtc2Va5yw7v+aYguLB4HHrIFC7G+L8hAJ0IAo 3R3DFeBw58M1xtxXCREI8Y6qMQTw60XyFw0gVmZzqR4hZiTerBSJJsZLZOBgmXxq3WLS +xVQ==', + * ), + * 10 => + * array ( + * 'Name' => 'X-Google-DKIM-Signature', + * 'Value' => 'v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1711282689; x=1711887489; h=to:subject:message-id:date:from:in-reply-to:references:mime-version :x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=NvjmqLXF/5L5ZrpToR/6FgVOhTOGC9j0/B2Na5Ke6J8=; b=uKoMhir+MX/wycNEr29Sffj45ooKksCJ1OfSRkIIGHk0rnHn8Vh+c7beYipwRPW4F2 h46K64vtIX00guYMdL2Qo2eY96+wALTqHCy67PGhvotVTROz21yxjx62pCDPGs5tefOu IkyxoybpIK8zAfLoDTd9p2GIrr5brKJyB2w1NQc1htxTQ5D4RgBxUAOKv4uVEr8r47iA MIo5d8/AifA+vCOAh7iJ7EmvDQ1R+guhQyH9m1Jo8PLapiYuHXggpBJvooyGuflKqbnt gJ/dscEr4d5aWJbw/x1dmIJ5gyJPGdBWq8NRqV/qbkXQW3H/gylifDUPXbki+EQBD5Yu EuLQ==', + * ), + * 11 => + * array ( + * 'Name' => 'X-Gm-Message-State', + * 'Value' => 'AOJu0Yxpbp1sRh17lNzg+pLnIx1jCn8ZFJQMgFuHK+6Z8RqFS5KKKTxR 8onpEbxWYYVUbrJFExNBHPD/3jdxqifCVVNaDmbpwHgmW5lHLJmA5vYRq5NFZ9OA6zKx/N6Gipr iXE4fXmSqghFNTzy9V/RT08Zp+F5RiFh/Ta6ltQl8XfCPFfSawLz6cagUgt8bBuF4RqdrYmWwzj ty86V5Br1htRNEFYivoXnNmaRcsD0tca1D23ny62O6RwWugrj1IpAYhViNyTZAWu+loKgfjJJoI MsyiSU=', + * ), + * 12 => + * array ( + * 'Name' => 'X-Google-Smtp-Source', + * 'Value' => 'AGHT+IEdtZqbVI6j7WLeaSL3dABGSnWIXaSjbYqXvFvE2H+f2zsn0gknQ4OdTJecQRCabpypVF2ue91Jb7aKl6RiyEQ=', + * ), + * 13 => + * array ( + * 'Name' => 'X-Received', + * 'Value' => 'by 2002:a19:385a:0:b0:513:c876:c80a with SMTP id d26-20020a19385a000000b00513c876c80amr2586776lfj.34.1711282689140; Sun, 24 Mar 2024 05:18:09 -0700 (PDT)', + * ), + * 14 => + * array ( + * 'Name' => 'MIME-Version', + * 'Value' => '1.0', + * ), + * 15 => + * array ( + * 'Name' => 'References', + * 'Value' => '', + * ), + * 16 => + * array ( + * 'Name' => 'In-Reply-To', + * 'Value' => '', + * ), + * 17 => + * array ( + * 'Name' => 'Message-ID', + * 'Value' => '', + * ), + * ), + * 'Attachments' => + * array ( + * ), + * ) * @return void */ public function handle() { - // match companies - if (array_key_exists('ToFull', $this->request)) - throw new \Exception('invalid body'); + $recipient = explode("|", $this->input)[0]; - foreach ($this->request['ToFull'] as $toEmailEntry) { - $toEmail = $toEmailEntry['Email']; + // match company + $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); + if (!$company) { + Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $recipient); + return; + } - $company = MultiDB::findAndSetDbByExpenseMailbox($toEmail); - if (!$company) { - nlog('unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $toEmail); - continue; + // fetch message from postmark-api + $company_postmark_secret = $company->settings?->email_sending_method === 'client_postmark' && $company->settings?->postmark_secret ? $company->settings?->postmark_secret : null; + if (!($company_postmark_secret) && !(config('services.postmark.domain') && config('services.postmark.secret'))) + throw new \Error("[ProcessMailgunInboundWebhook] no postmark credenitals found, we cannot get the attachements and files"); + + $mail = null; + if ($company_postmark_secret) { + + $credentials = $company_postmark_domain . ":" . $company_postmark_secret . "@"; + $messageUrl = explode("|", $this->input)[1]; + $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); + $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); + + try { + $mail = json_decode(file_get_contents($messageUrl)); + } catch (\Error $e) { + if (config('services.postmark.secret')) { + Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); + + $credentials = config('services.postmark.domain') . ":" . config('services.postmark.secret') . "@"; + $messageUrl = explode("|", $this->input)[1]; + $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); + $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); + $mail = json_decode(file_get_contents($messageUrl)); + + } else + throw $e; } - (new PostmarkWebhookHandler())->process($this->request); + } else { + + $credentials = config('services.postmark.domain') . ":" . config('services.postmark.secret') . "@"; + $messageUrl = explode("|", $this->input)[1]; + $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); + $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); + $mail = json_decode(file_get_contents($messageUrl)); + } + + // prepare data for ingresEngine + $ingresEmail = new IngresEmail(); + + $ingresEmail->from = $mail->sender; + $ingresEmail->to = $recipient; // usage of data-input, because we need a single email here + $ingresEmail->subject = $mail->Subject; + $ingresEmail->body = $mail->{"body-html"}; + $ingresEmail->text_body = $mail->{"body-plain"}; + $ingresEmail->date = Carbon::createFromTimeString($mail->Date); + + // parse documents as UploadedFile from webhook-data + foreach ($mail->attachments as $attachment) { // prepare url with credentials before downloading :: https://github.com/postmark/postmark.js/issues/24 + + // download file and save to tmp dir + if ($company_postmark_domain && $company_postmark_secret) { + + try { + + $credentials = $company_postmark_domain . ":" . $company_postmark_secret . "@"; + $url = $attachment->url; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + + } catch (\Error $e) { + if (config('services.postmark.secret')) { + Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); + + $credentials = config('services.postmark.domain') . ":" . config('services.postmark.secret') . "@"; + $url = $attachment->url; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + + } else + throw $e; + } + + } else { + + $credentials = config('services.postmark.domain') . ":" . config('services.postmark.secret') . "@"; + $url = $attachment->url; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + + } + + } + + // perform + (new IngresEmailEngine($ingresEmail))->handle(); } } diff --git a/routes/api.php b/routes/api.php index 4b0cc86d95a8..008fd764546c 100644 --- a/routes/api.php +++ b/routes/api.php @@ -432,6 +432,7 @@ Route::match(['get', 'post'], 'payment_notification_webhook/{company_key}/{compa Route::post('api/v1/postmark_webhook', [PostMarkController::class, 'webhook'])->middleware('throttle:1000,1'); +Route::post('api/v1/postmark_inbound_webhook', [PostMarkController::class, 'inboundWebhook'])->middleware('throttle:1000,1'); Route::post('api/v1/mailgun_webhook', [MailgunController::class, 'webhook'])->middleware('throttle:1000,1'); Route::post('api/v1/mailgun_inbound_webhook', [MailgunController::class, 'inboundWebhook'])->middleware('throttle:1000,1'); Route::post('api/v1/brevo_webhook', [BrevoController::class, 'webhook'])->middleware('throttle:1000,1'); From 157037f56fc83926545517ba0dec7ff25c507517 Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 25 Mar 2024 06:41:22 +0100 Subject: [PATCH 049/119] renamings + move imap job to self-hosted only + allow by clients --- app/Console/Kernel.php | 8 +- .../Transformer/ImapMailTransformer.php | 20 +-- .../PostmarkInboundWebhookTransformer.php | 118 ------------------ .../Requests/Company/StoreCompanyRequest.php | 24 ++-- .../Requests/Company/UpdateCompanyRequest.php | 26 ++-- .../Company/ValidExpenseMailbox.php | 14 +-- app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 28 ++--- .../ProcessImapMailboxJob.php} | 22 ++-- .../Mailgun/ProcessMailgunInboundWebhook.php | 28 ++--- .../ProcessPostmarkInboundWebhook.php | 28 ++--- app/Libraries/MultiDB.php | 40 ++++-- app/Models/Company.php | 38 +++--- app/Models/SystemLog.php | 2 +- .../InboundMail.php} | 4 +- .../InboundMailEngine.php} | 107 +++++++++------- app/Transformers/CompanyTransformer.php | 11 +- config/ninja.php | 28 ++--- ...10951_create_imap_configuration_fields.php | 19 +-- lang/en/texts.php | 4 +- 19 files changed, 253 insertions(+), 316 deletions(-) rename app/Helpers/{IngresMail => InboundMail}/Transformer/ImapMailTransformer.php (56%) delete mode 100644 app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php rename app/Jobs/{Mail/ExpenseMailboxJob.php => Imap/ProcessImapMailboxJob.php} (89%) rename app/Services/{IngresEmail/IngresEmail.php => InboundMail/InboundMail.php} (93%) rename app/Services/{IngresEmail/IngresEmailEngine.php => InboundMail/InboundMailEngine.php} (62%) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index ebd7cfc4c215..9a37eaf755f4 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -17,7 +17,7 @@ use App\Jobs\Cron\RecurringInvoicesCron; use App\Jobs\Cron\SubscriptionCron; use App\Jobs\Cron\UpdateCalculatedFields; use App\Jobs\Invoice\InvoiceCheckLateWebhook; -use App\Jobs\Mail\ExpenseMailboxJob; +use App\Jobs\Imap\ProcessImapMailboxJob; use App\Jobs\Ninja\AdjustEmailQuota; use App\Jobs\Ninja\BankTransactionSync; use App\Jobs\Ninja\CheckACHStatus; @@ -98,9 +98,6 @@ class Kernel extends ConsoleKernel /* Fires webhooks for overdue Invoice */ $schedule->job(new InvoiceCheckLateWebhook())->dailyAt('07:00')->withoutOverlapping()->name('invoice-overdue-job')->onOneServer(); - /* Check ExpenseMainboxes */ - $schedule->job(new ExpenseMailboxJob)->everyThirtyMinutes()->withoutOverlapping()->name('expense-mailboxes-job')->onOneServer(); - /* Pulls in bank transactions from third party services */ $schedule->job(new BankTransactionSync())->everyFourHours()->withoutOverlapping()->name('bank-trans-sync-job')->onOneServer(); @@ -108,6 +105,9 @@ class Kernel extends ConsoleKernel $schedule->call(function () { Account::query()->whereNotNull('id')->update(['is_scheduler_running' => true]); })->everyFiveMinutes(); + + /* Check ImapMailboxes */ + $schedule->job(new ProcessImapMailboxJob)->everyFiveMinutes()->withoutOverlapping()->name('imap-mailbox-job')->onOneServer(); } /* Run hosted specific jobs */ diff --git a/app/Helpers/IngresMail/Transformer/ImapMailTransformer.php b/app/Helpers/InboundMail/Transformer/ImapMailTransformer.php similarity index 56% rename from app/Helpers/IngresMail/Transformer/ImapMailTransformer.php rename to app/Helpers/InboundMail/Transformer/ImapMailTransformer.php index 80889b3afc9c..4229d7fb5e79 100644 --- a/app/Helpers/IngresMail/Transformer/ImapMailTransformer.php +++ b/app/Helpers/InboundMail/Transformer/ImapMailTransformer.php @@ -9,9 +9,9 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Helpers\IngresMail\Transformer; +namespace App\Helpers\InboundMail\Transformer; -use App\Services\IngresEmail\IngresEmail; +use App\Services\InboundMail\InboundMail; use App\Utils\TempFile; use Ddeboer\Imap\MessageInterface; @@ -20,19 +20,19 @@ class ImapMailTransformer public function transform(MessageInterface $mail) { - $ingresEmail = new IngresEmail(); + $inboundMail = new InboundMail(); - $ingresEmail->from = $mail->getSender(); - $ingresEmail->subject = $mail->getSubject(); - $ingresEmail->plain_message = $mail->getBodyText(); - $ingresEmail->html_message = $mail->getBodyHtml(); - $ingresEmail->date = $mail->getDate(); + $inboundMail->from = $mail->getSender(); + $inboundMail->subject = $mail->getSubject(); + $inboundMail->plain_message = $mail->getBodyText(); + $inboundMail->html_message = $mail->getBodyHtml(); + $inboundMail->date = $mail->getDate(); // parse documents as UploadedFile foreach ($mail->getAttachments() as $attachment) { - $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment->getContent(), $attachment->getFilename(), $attachment->getEncoding()); + $inboundMail->documents[] = TempFile::UploadedFileFromRaw($attachment->getContent(), $attachment->getFilename(), $attachment->getEncoding()); } - return $ingresEmail; + return $inboundMail; } } diff --git a/app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php b/app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php deleted file mode 100644 index 0ad0faf71963..000000000000 --- a/app/Helpers/IngresMail/Transformer/PostmarkInboundWebhookTransformer.php +++ /dev/null @@ -1,118 +0,0 @@ -from = $data["From"]; - $ingresEmail->subject = $data["Subject"]; - $ingresEmail->plain_message = $data["TextBody"]; - $ingresEmail->html_message = $data["HtmlBody"]; - $ingresEmail->date = $data["Date"]; // TODO: parsing - - // parse documents as UploadedFile from webhook-data - foreach ($data["Attachments"] as $attachment) { - $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment["Content"], $attachment["Name"], $attachment["ContentType"]); - } - - return $ingresEmail; - - } - // { - // "FromName": "Postmarkapp Support", - // "MessageStream": "inbound", - // "From": "support@postmarkapp.com", - // "FromFull": { - // "Email": "support@postmarkapp.com", - // "Name": "Postmarkapp Support", - // "MailboxHash": "" - // }, - // "To": "\"Firstname Lastname\" ", - // "ToFull": [ - // { - // "Email": "yourhash+SampleHash@inbound.postmarkapp.com", - // "Name": "Firstname Lastname", - // "MailboxHash": "SampleHash" - // } - // ], - // "Cc": "\"First Cc\" , secondCc@postmarkapp.com>", - // "CcFull": [ - // { - // "Email": "firstcc@postmarkapp.com", - // "Name": "First Cc", - // "MailboxHash": "" - // }, - // { - // "Email": "secondCc@postmarkapp.com", - // "Name": "", - // "MailboxHash": "" - // } - // ], - // "Bcc": "\"First Bcc\" , secondbcc@postmarkapp.com>", - // "BccFull": [ - // { - // "Email": "firstbcc@postmarkapp.com", - // "Name": "First Bcc", - // "MailboxHash": "" - // }, - // { - // "Email": "secondbcc@postmarkapp.com", - // "Name": "", - // "MailboxHash": "" - // } - // ], - // "OriginalRecipient": "yourhash+SampleHash@inbound.postmarkapp.com", - // "Subject": "Test subject", - // "MessageID": "73e6d360-66eb-11e1-8e72-a8904824019b", - // "ReplyTo": "replyto@postmarkapp.com", - // "MailboxHash": "SampleHash", - // "Date": "Fri, 1 Aug 2014 16:45:32 -04:00", - // "TextBody": "This is a test text body.", - // "HtmlBody": "

This is a test html body.<\/p><\/body><\/html>", - // "StrippedTextReply": "This is the reply text", - // "Tag": "TestTag", - // "Headers": [ - // { - // "Name": "X-Header-Test", - // "Value": "" - // }, - // { - // "Name": "X-Spam-Status", - // "Value": "No" - // }, - // { - // "Name": "X-Spam-Score", - // "Value": "-0.1" - // }, - // { - // "Name": "X-Spam-Tests", - // "Value": "DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,SPF_PASS" - // } - // ], - // "Attachments": [ - // { - // "Name": "test.txt", - // "Content": "VGhpcyBpcyBhdHRhY2htZW50IGNvbnRlbnRzLCBiYXNlLTY0IGVuY29kZWQu", - // "ContentType": "text/plain", - // "ContentLength": 45 - // } - // ] - // } -} diff --git a/app/Http/Requests/Company/StoreCompanyRequest.php b/app/Http/Requests/Company/StoreCompanyRequest.php index fa3563db5d71..3b4725118a4d 100644 --- a/app/Http/Requests/Company/StoreCompanyRequest.php +++ b/app/Http/Requests/Company/StoreCompanyRequest.php @@ -47,7 +47,7 @@ class StoreCompanyRequest extends Request $rules['company_logo'] = 'mimes:jpeg,jpg,png,gif|max:10000'; // max 10000kb $rules['settings'] = new ValidSettingsRule(); - if (isset($input['portal_mode']) && ($input['portal_mode'] == 'domain' || $input['portal_mode'] == 'iframe')) { + if (isset ($input['portal_mode']) && ($input['portal_mode'] == 'domain' || $input['portal_mode'] == 'iframe')) { $rules['portal_domain'] = 'sometimes|url'; } else { if (Ninja::isHosted()) { @@ -57,7 +57,7 @@ class StoreCompanyRequest extends Request } } - $rules['expense_mailbox'] = new ValidExpenseMailbox($this->company->key); + $rules['inbound_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); $rules['smtp_host'] = 'sometimes|string|nullable'; $rules['smtp_port'] = 'sometimes|integer|nullable'; @@ -75,39 +75,39 @@ class StoreCompanyRequest extends Request { $input = $this->all(); - if (!isset($input['name'])) { + if (!isset ($input['name'])) { $input['name'] = 'Untitled Company'; } - if (isset($input['google_analytics_url'])) { + if (isset ($input['google_analytics_url'])) { $input['google_analytics_key'] = $input['google_analytics_url']; } - if (isset($input['portal_domain'])) { + if (isset ($input['portal_domain'])) { $input['portal_domain'] = rtrim(strtolower($input['portal_domain']), "/"); } - if (isset($input['expense_mailbox']) && Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise')) { - unset($input['expense_mailbox']); + if (isset ($input['inbound_mailbox']) && Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise')) { + unset($input['inbound_mailbox']); } - if (Ninja::isHosted() && !isset($input['subdomain'])) { + if (Ninja::isHosted() && !isset ($input['subdomain'])) { $input['subdomain'] = MultiDB::randomSubdomainGenerator(); } - if (isset($input['smtp_username']) && strlen(str_replace("*", "", $input['smtp_username'])) < 2) { + if (isset ($input['smtp_username']) && strlen(str_replace("*", "", $input['smtp_username'])) < 2) { unset($input['smtp_username']); } - if (isset($input['smtp_password']) && strlen(str_replace("*", "", $input['smtp_password'])) < 2) { + if (isset ($input['smtp_password']) && strlen(str_replace("*", "", $input['smtp_password'])) < 2) { unset($input['smtp_password']); } - if (isset($input['smtp_port'])) { + if (isset ($input['smtp_port'])) { $input['smtp_port'] = (int) $input['smtp_port']; } - if (isset($input['smtp_verify_peer']) && is_string($input['smtp_verify_peer'])) + if (isset ($input['smtp_verify_peer']) && is_string($input['smtp_verify_peer'])) $input['smtp_verify_peer'] == 'true' ? true : false; $this->replace($input); diff --git a/app/Http/Requests/Company/UpdateCompanyRequest.php b/app/Http/Requests/Company/UpdateCompanyRequest.php index 58c530c6a762..c8578d262ae1 100644 --- a/app/Http/Requests/Company/UpdateCompanyRequest.php +++ b/app/Http/Requests/Company/UpdateCompanyRequest.php @@ -66,7 +66,7 @@ class UpdateCompanyRequest extends Request // $rules['smtp_verify_peer'] = 'sometimes|string'; - if (isset($input['portal_mode']) && ($input['portal_mode'] == 'domain' || $input['portal_mode'] == 'iframe')) { + if (isset ($input['portal_mode']) && ($input['portal_mode'] == 'domain' || $input['portal_mode'] == 'iframe')) { $rules['portal_domain'] = 'bail|nullable|sometimes|url'; } @@ -74,7 +74,7 @@ class UpdateCompanyRequest extends Request $rules['subdomain'] = ['nullable', 'regex:/^[a-zA-Z0-9.-]+[a-zA-Z0-9]$/', new ValidSubdomain()]; } - $rules['expense_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); // @turbo124 check if this is right + $rules['inbound_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); // @turbo124 check if this is right return $rules; } @@ -83,36 +83,40 @@ class UpdateCompanyRequest extends Request { $input = $this->all(); - if (isset($input['portal_domain']) && strlen($input['portal_domain']) > 1) { + if (isset ($input['portal_domain']) && strlen($input['portal_domain']) > 1) { $input['portal_domain'] = $this->addScheme($input['portal_domain']); $input['portal_domain'] = rtrim(strtolower($input['portal_domain']), "/"); } - if (isset($input['settings'])) { + if (isset ($input['inbound_mailbox']) && Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise')) { + unset($input['inbound_mailbox']); + } + + if (isset ($input['settings'])) { $input['settings'] = (array) $this->filterSaveableSettings($input['settings']); } - if (isset($input['subdomain']) && $this->company->subdomain == $input['subdomain']) { + if (isset ($input['subdomain']) && $this->company->subdomain == $input['subdomain']) { unset($input['subdomain']); } - if (isset($input['e_invoice_certificate_passphrase']) && empty($input['e_invoice_certificate_passphrase'])) { + if (isset ($input['e_invoice_certificate_passphrase']) && empty ($input['e_invoice_certificate_passphrase'])) { unset($input['e_invoice_certificate_passphrase']); } - if (isset($input['smtp_username']) && strlen(str_replace("*", "", $input['smtp_username'])) < 2) { + if (isset ($input['smtp_username']) && strlen(str_replace("*", "", $input['smtp_username'])) < 2) { unset($input['smtp_username']); } - if (isset($input['smtp_password']) && strlen(str_replace("*", "", $input['smtp_password'])) < 2) { + if (isset ($input['smtp_password']) && strlen(str_replace("*", "", $input['smtp_password'])) < 2) { unset($input['smtp_password']); } - if (isset($input['smtp_port'])) { + if (isset ($input['smtp_port'])) { $input['smtp_port'] = (int) $input['smtp_port']; } - if (isset($input['smtp_verify_peer']) && is_string($input['smtp_verify_peer'])) { + if (isset ($input['smtp_verify_peer']) && is_string($input['smtp_verify_peer'])) { $input['smtp_verify_peer'] == 'true' ? true : false; } @@ -139,7 +143,7 @@ class UpdateCompanyRequest extends Request } } - if (isset($settings['email_style_custom'])) { + if (isset ($settings['email_style_custom'])) { $settings['email_style_custom'] = str_replace(['{!!', '!!}', '{{', '}}', '@if(', '@endif', '@isset', '@unless', '@auth', '@empty', '@guest', '@env', '@section', '@switch', '@foreach', '@while', '@include', '@each', '@once', '@push', '@use', '@forelse', '@verbatim', 'company_key = $company_key; - $this->endings = explode(",", config('ninja.ingest_mail.expense_mailbox_endings')); + $this->endings = explode(",", config('ninja.inbound_mailbox.inbound_mailbox_endings')); } public function passes($attribute, $value) { - if (empty($value)) { + if (empty ($value)) { return true; } // early return, if we dont have any additional validation - if (!config('ninja.ingest_mail.expense_mailbox_endings')) { + if (!config('ninja.inbound_mailbox.inbound_mailbox_endings')) { $this->validated_schema = true; - return MultiDB::checkExpenseMailboxAvailable($value); + return MultiDB::checkInboundMailboxAvailable($value); } // Validate Schema @@ -59,7 +59,7 @@ class ValidExpenseMailbox implements Rule return false; $this->validated_schema = true; - return MultiDB::checkExpenseMailboxAvailable($value); + return MultiDB::checkInboundMailboxAvailable($value); } /** @@ -68,8 +68,8 @@ class ValidExpenseMailbox implements Rule public function message() { if (!$this->validated_schema) - return ctrans('texts.expense_mailbox_invalid'); + return ctrans('texts.inbound_mailbox_invalid'); - return ctrans('texts.expense_mailbox_taken'); + return ctrans('texts.inbound_mailbox_taken'); } } diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index d97fb7245496..d8257351ca41 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -12,8 +12,8 @@ namespace App\Jobs\Brevo; use App\Libraries\MultiDB; -use App\Services\IngresEmail\IngresEmail; -use App\Services\IngresEmail\IngresEmailEngine; +use App\Services\InboundMail\InboundMail; +use App\Services\InboundMail\InboundMailEngine; use App\Utils\TempFile; use Brevo\Client\Api\InboundParsingApi; use Brevo\Client\Configuration; @@ -118,11 +118,11 @@ class ProcessBrevoInboundWebhook implements ShouldQueue public function handle() { - // brevo defines recipients as array, and we should check all of them, to be sure + // brevo defines recipients as array to enable webhook processing as batches, we check all of them foreach ($this->input["Recipients"] as $recipient) { // match company - $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); + $company = MultiDB::findAndSetDbByInboundMailbox($recipient); if (!$company) { Log::info('[ProcessBrevoInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from brevo: ' . $recipient); continue; @@ -133,14 +133,14 @@ class ProcessBrevoInboundWebhook implements ShouldQueue throw new \Error("[ProcessBrevoInboundWebhook] no brevo credenitals found, we cannot get the attachement"); // prepare data for ingresEngine - $ingresEmail = new IngresEmail(); + $inboundMail = new InboundMail(); - $ingresEmail->from = $this->input["From"]["Address"]; - $ingresEmail->to = $recipient; - $ingresEmail->subject = $this->input["Subject"]; - $ingresEmail->body = $this->input["RawHtmlBody"]; - $ingresEmail->text_body = $this->input["RawTextBody"]; - $ingresEmail->date = Carbon::createFromTimeString($this->input["SentAtDate"]); + $inboundMail->from = $this->input["From"]["Address"]; + $inboundMail->to = $recipient; + $inboundMail->subject = $this->input["Subject"]; + $inboundMail->body = $this->input["RawHtmlBody"]; + $inboundMail->text_body = $this->input["RawTextBody"]; + $inboundMail->date = Carbon::createFromTimeString($this->input["SentAtDate"]); // parse documents as UploadedFile from webhook-data foreach ($this->input["Attachments"] as $attachment) { @@ -164,18 +164,18 @@ class ProcessBrevoInboundWebhook implements ShouldQueue } else throw $e; } - $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($attachment, $attachment["Name"], $attachment["ContentType"]); + $inboundMail->documents[] = TempFile::UploadedFileFromRaw($attachment, $attachment["Name"], $attachment["ContentType"]); } else { $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); - $ingresEmail->documents[] = TempFile::UploadedFileFromRaw($brevo->getInboundEmailAttachment($attachment["DownloadToken"]), $attachment["Name"], $attachment["ContentType"]); + $inboundMail->documents[] = TempFile::UploadedFileFromRaw($brevo->getInboundEmailAttachment($attachment["DownloadToken"]), $attachment["Name"], $attachment["ContentType"]); } } - (new IngresEmailEngine($ingresEmail))->handle(); + (new InboundMailEngine($inboundMail))->handle(); } } diff --git a/app/Jobs/Mail/ExpenseMailboxJob.php b/app/Jobs/Imap/ProcessImapMailboxJob.php similarity index 89% rename from app/Jobs/Mail/ExpenseMailboxJob.php rename to app/Jobs/Imap/ProcessImapMailboxJob.php index 9f47582d26ee..e6aef5fd3d5e 100644 --- a/app/Jobs/Mail/ExpenseMailboxJob.php +++ b/app/Jobs/Imap/ProcessImapMailboxJob.php @@ -9,14 +9,14 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Jobs\Mail; +namespace App\Jobs\Imap; -use App\Helpers\IngresMail\Transformer\ImapMailTransformer; +use App\Helpers\InboundMail\Transformer\ImapMailTransformer; use App\Helpers\Mail\Mailbox\Imap\ImapMailbox; use App\Libraries\MultiDB; use App\Models\Company; use App\Repositories\ExpenseRepository; -use App\Services\IngresEmail\IngresEmailEngine; +use App\Services\InboundMail\InboundMailEngine; use App\Utils\Traits\MakesHash; use App\Utils\Traits\SavesDocuments; use Illuminate\Bus\Queueable; @@ -27,7 +27,7 @@ use Illuminate\Queue\SerializesModels; /*Multi Mailer implemented*/ -class ExpenseMailboxJob implements ShouldQueue +class ProcessImapMailboxJob implements ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, MakesHash, SavesDocuments; @@ -74,14 +74,14 @@ class ExpenseMailboxJob implements ShouldQueue private function getImapCredentials() { - $servers = array_map('trim', explode(",", config('ninja.ingest_mail.imap.servers'))); - $ports = array_map('trim', explode(",", config('ninja.ingest_mail.imap.ports'))); - $users = array_map('trim', explode(",", config('ninja.ingest_mail.imap.users'))); - $passwords = array_map('trim', explode(",", config('ninja.ingest_mail.imap.passwords'))); - $companies = array_map('trim', explode(",", config('ninja.ingest_mail.imap.companies'))); + $servers = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.servers'))); + $ports = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.ports'))); + $users = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.users'))); + $passwords = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.passwords'))); + $companies = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.companies'))); if (sizeOf($servers) != sizeOf($ports) || sizeOf($servers) != sizeOf($users) || sizeOf($servers) != sizeOf($passwords) || sizeOf($servers) != sizeOf($companies)) - throw new \Exception('invalid configuration ingest_mail.imap (wrong element-count)'); + throw new \Exception('invalid configuration inbound_mailbox.imap (wrong element-count)'); foreach ($companies as $index => $companyId) { @@ -116,7 +116,7 @@ class ExpenseMailboxJob implements ShouldQueue $email->markAsSeen(); - IngresEmailEngine::dispatch($transformer->transform($email)); + InboundMailEngine::dispatch($transformer->transform($email)); $imapMailbox->moveProcessed($email); diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index eac0e1bd04d2..4f11b6ba0d2b 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -12,8 +12,8 @@ namespace App\Jobs\Mailgun; use App\Libraries\MultiDB; -use App\Services\IngresEmail\IngresEmail; -use App\Services\IngresEmail\IngresEmailEngine; +use App\Services\InboundMail\InboundMail; +use App\Services\InboundMail\InboundMailEngine; use App\Utils\TempFile; use Illuminate\Support\Carbon; use Illuminate\Bus\Queueable; @@ -166,7 +166,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $recipient = explode("|", $this->input)[0]; // match company - $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); + $company = MultiDB::findAndSetDbByInboundMailbox($recipient); if (!$company) { Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $recipient); return; @@ -213,14 +213,14 @@ class ProcessMailgunInboundWebhook implements ShouldQueue } // prepare data for ingresEngine - $ingresEmail = new IngresEmail(); + $inboundMail = new InboundMail(); - $ingresEmail->from = $mail->sender; - $ingresEmail->to = $recipient; // usage of data-input, because we need a single email here - $ingresEmail->subject = $mail->Subject; - $ingresEmail->body = $mail->{"body-html"}; - $ingresEmail->text_body = $mail->{"body-plain"}; - $ingresEmail->date = Carbon::createFromTimeString($mail->Date); + $inboundMail->from = $mail->sender; + $inboundMail->to = $recipient; // usage of data-input, because we need a single email here + $inboundMail->subject = $mail->Subject; + $inboundMail->body = $mail->{"body-html"}; + $inboundMail->text_body = $mail->{"body-plain"}; + $inboundMail->date = Carbon::createFromTimeString($mail->Date); // parse documents as UploadedFile from webhook-data foreach ($mail->attachments as $attachment) { // prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 @@ -234,7 +234,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $url = $attachment->url; $url = str_replace("http://", "http://" . $credentials, $url); $url = str_replace("https://", "https://" . $credentials, $url); - $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); } catch (\Error $e) { if (config('services.mailgun.secret')) { @@ -244,7 +244,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $url = $attachment->url; $url = str_replace("http://", "http://" . $credentials, $url); $url = str_replace("https://", "https://" . $credentials, $url); - $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); } else throw $e; @@ -256,13 +256,13 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $url = $attachment->url; $url = str_replace("http://", "http://" . $credentials, $url); $url = str_replace("https://", "https://" . $credentials, $url); - $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); } } // perform - (new IngresEmailEngine($ingresEmail))->handle(); + (new InboundMailEngine($inboundMail))->handle(); } } diff --git a/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php b/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php index d616294a666c..5904b7c5b664 100644 --- a/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php +++ b/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php @@ -12,8 +12,8 @@ namespace App\Jobs\Postmark; use App\Libraries\MultiDB; -use App\Services\IngresEmail\IngresEmail; -use App\Services\IngresEmail\IngresEmailEngine; +use App\Services\InboundMail\InboundMail; +use App\Services\InboundMail\InboundMailEngine; use App\Utils\TempFile; use Illuminate\Support\Carbon; use Illuminate\Bus\Queueable; @@ -198,7 +198,7 @@ class ProcessPostmarkInboundWebhook implements ShouldQueue $recipient = explode("|", $this->input)[0]; // match company - $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); + $company = MultiDB::findAndSetDbByInboundMailbox($recipient); if (!$company) { Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $recipient); return; @@ -244,14 +244,14 @@ class ProcessPostmarkInboundWebhook implements ShouldQueue } // prepare data for ingresEngine - $ingresEmail = new IngresEmail(); + $inboundMail = new InboundMail(); - $ingresEmail->from = $mail->sender; - $ingresEmail->to = $recipient; // usage of data-input, because we need a single email here - $ingresEmail->subject = $mail->Subject; - $ingresEmail->body = $mail->{"body-html"}; - $ingresEmail->text_body = $mail->{"body-plain"}; - $ingresEmail->date = Carbon::createFromTimeString($mail->Date); + $inboundMail->from = $mail->sender; + $inboundMail->to = $recipient; // usage of data-input, because we need a single email here + $inboundMail->subject = $mail->Subject; + $inboundMail->body = $mail->{"body-html"}; + $inboundMail->text_body = $mail->{"body-plain"}; + $inboundMail->date = Carbon::createFromTimeString($mail->Date); // parse documents as UploadedFile from webhook-data foreach ($mail->attachments as $attachment) { // prepare url with credentials before downloading :: https://github.com/postmark/postmark.js/issues/24 @@ -265,7 +265,7 @@ class ProcessPostmarkInboundWebhook implements ShouldQueue $url = $attachment->url; $url = str_replace("http://", "http://" . $credentials, $url); $url = str_replace("https://", "https://" . $credentials, $url); - $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); } catch (\Error $e) { if (config('services.postmark.secret')) { @@ -275,7 +275,7 @@ class ProcessPostmarkInboundWebhook implements ShouldQueue $url = $attachment->url; $url = str_replace("http://", "http://" . $credentials, $url); $url = str_replace("https://", "https://" . $credentials, $url); - $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); } else throw $e; @@ -287,13 +287,13 @@ class ProcessPostmarkInboundWebhook implements ShouldQueue $url = $attachment->url; $url = str_replace("http://", "http://" . $credentials, $url); $url = str_replace("https://", "https://" . $credentials, $url); - $ingresEmail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); } } // perform - (new IngresEmailEngine($ingresEmail))->handle(); + (new InboundMailEngine($inboundMail))->handle(); } } diff --git a/app/Libraries/MultiDB.php b/app/Libraries/MultiDB.php index 3b7e5140da36..58362ab77dcd 100644 --- a/app/Libraries/MultiDB.php +++ b/app/Libraries/MultiDB.php @@ -73,7 +73,7 @@ class MultiDB 'socket', ]; - private static $protected_expense_mailboxes = []; + private static $protected_inbound_mailboxes = []; /** * @return array @@ -109,21 +109,21 @@ class MultiDB return true; } - public static function checkExpenseMailboxAvailable($expense_mailbox): bool + public static function checkInboundMailboxAvailable($inbound_mailbox): bool { if (!config('ninja.db.multi_db_enabled')) { - return Company::where("expense_mailbox", $expense_mailbox)->withTrashed()->exists(); + return Company::where("inbound_mailbox", $inbound_mailbox)->withTrashed()->exists(); } - if (in_array($expense_mailbox, self::$protected_expense_mailboxes)) { + if (in_array($inbound_mailbox, self::$protected_inbound_mailboxes)) { return false; } $current_db = config('database.default'); foreach (self::$dbs as $db) { - if (Company::on($db)->where("expense_mailbox", $expense_mailbox)->withTrashed()->exists()) { + if (Company::on($db)->where("inbound_mailbox", $inbound_mailbox)->withTrashed()->exists()) { self::setDb($current_db); return false; @@ -515,16 +515,16 @@ class MultiDB return false; } - public static function findAndSetDbByExpenseMailbox($expense_mailbox) + public static function findAndSetDbByInboundMailbox($inbound_mailbox) { if (!config('ninja.db.multi_db_enabled')) { - return Company::where("expense_mailbox", $expense_mailbox)->first(); + return Company::where("inbound_mailbox", $inbound_mailbox)->first(); } $current_db = config('database.default'); foreach (self::$dbs as $db) { - if ($company = Company::on($db)->where("expense_mailbox", $expense_mailbox)->first()) { + if ($company = Company::on($db)->where("inbound_mailbox", $inbound_mailbox)->first()) { self::setDb($db); return $company; @@ -587,7 +587,7 @@ class MultiDB $current_db = config('database.default'); - if(SMSNumbers::hasNumber($phone)){ + if (SMSNumbers::hasNumber($phone)) { return true; } @@ -615,8 +615,26 @@ class MultiDB $string = ''; $vowels = ['a', 'e', 'i', 'o', 'u', 'y']; $consonants = [ - 'b', 'c', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'm', - 'n', 'p', 'r', 's', 't', 'v', 'w', 'x', 'y', 'z', + 'b', + 'c', + 'd', + 'f', + 'g', + 'h', + 'j', + 'k', + 'l', + 'm', + 'n', + 'p', + 'r', + 's', + 't', + 'v', + 'w', + 'x', + 'y', + 'z', ]; $max = $length / 2; diff --git a/app/Models/Company.php b/app/Models/Company.php index a868093c6238..ab40b29960d0 100644 --- a/app/Models/Company.php +++ b/app/Models/Company.php @@ -111,15 +111,16 @@ use Laracasts\Presenter\PresentableTrait; * @property int $convert_expense_currency * @property int $notify_vendor_when_paid * @property int $invoice_task_hours - * @property string|null $expense_mailbox - * @property boolean $expense_mailbox_active - * @property bool $expense_mailbox_allow_company_users - * @property bool $expense_mailbox_allow_vendors - * @property bool $expense_mailbox_allow_unknown - * @property string|null $expense_mailbox_whitelist_domains - * @property string|null $expense_mailbox_whitelist_emails - * @property string|null $expense_mailbox_blacklist_domains - * @property string|null $expense_mailbox_blacklist_emails + * @property string|null $inbound_mailbox + * @property boolean $inbound_mailbox_active + * @property bool $inbound_mailbox_allow_company_users + * @property bool $inbound_mailbox_allow_vendors + * @property bool $inbound_mailbox_allow_clients + * @property bool $inbound_mailbox_allow_unknown + * @property string|null $inbound_mailbox_whitelist_domains + * @property string|null $inbound_mailbox_whitelist_senders + * @property string|null $inbound_mailbox_blacklist_domains + * @property string|null $inbound_mailbox_blacklist_senders * @property int $deleted_at * @property string $smtp_username * @property string $smtp_password @@ -368,15 +369,16 @@ class Company extends BaseModel 'calculate_taxes', 'tax_data', 'e_invoice_certificate_passphrase', - 'expense_mailbox_active', - 'expense_mailbox', // TODO: @turbo124 custom validation: self-hosted => free change, hosted => not changeable, only changeable with env-mask - 'expense_mailbox_allow_company_users', - 'expense_mailbox_allow_vendors', - 'expense_mailbox_allow_unknown', - 'expense_mailbox_whitelist_domains', - 'expense_mailbox_whitelist_emails', - 'expense_mailbox_blacklist_domains', - 'expense_mailbox_blacklist_emails', + 'inbound_mailbox_active', + 'inbound_mailbox', // TODO: @turbo124 custom validation: self-hosted => free change, hosted => not changeable, only changeable with env-mask + 'inbound_mailbox_allow_company_users', + 'inbound_mailbox_allow_vendors', + 'inbound_mailbox_allow_clients', + 'inbound_mailbox_allow_unknown', + 'inbound_mailbox_whitelist_domains', + 'inbound_mailbox_whitelist_senders', + 'inbound_mailbox_blacklist_domains', + 'inbound_mailbox_blacklist_senders', 'smtp_host', 'smtp_port', 'smtp_encryption', diff --git a/app/Models/SystemLog.php b/app/Models/SystemLog.php index 1ab3cafbae65..e71d402683df 100644 --- a/app/Models/SystemLog.php +++ b/app/Models/SystemLog.php @@ -113,7 +113,7 @@ class SystemLog extends Model public const EVENT_USER = 61; - public const EVENT_INGEST_EMAIL_FAILURE = 62; + public const EVENT_INBOUND_MAIL_BLOCKED = 62; /*Type IDs*/ public const TYPE_PAYPAL = 300; diff --git a/app/Services/IngresEmail/IngresEmail.php b/app/Services/InboundMail/InboundMail.php similarity index 93% rename from app/Services/IngresEmail/IngresEmail.php rename to app/Services/InboundMail/InboundMail.php index 72d30e84e110..2a897d2598b0 100644 --- a/app/Services/IngresEmail/IngresEmail.php +++ b/app/Services/InboundMail/InboundMail.php @@ -9,7 +9,7 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Services\IngresEmail; +namespace App\Services\InboundMail; use Illuminate\Http\UploadedFile; use Illuminate\Support\Carbon; @@ -17,7 +17,7 @@ use Illuminate\Support\Carbon; /** * EmailObject. */ -class IngresEmail +class InboundMail { public string $to; diff --git a/app/Services/IngresEmail/IngresEmailEngine.php b/app/Services/InboundMail/InboundMailEngine.php similarity index 62% rename from app/Services/IngresEmail/IngresEmailEngine.php rename to app/Services/InboundMail/InboundMailEngine.php index b8a5013a5421..c745fa2ac243 100644 --- a/app/Services/IngresEmail/IngresEmailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -9,17 +9,19 @@ * @license https://www.elastic.co/licensing/elastic-license */ -namespace App\Services\IngresEmail; +namespace App\Services\InboundMail; use App\Events\Expense\ExpenseWasCreated; use App\Factory\ExpenseFactory; use App\Jobs\Util\SystemLogger; use App\Libraries\MultiDB; +use App\Models\Client; +use App\Models\ClientContact; use App\Models\Company; use App\Models\SystemLog; use App\Models\Vendor; use App\Models\VendorContact; -use App\Services\IngresEmail\IngresEmail; +use App\Services\InboundMail\InboundMail; use App\Utils\Ninja; use App\Utils\TempFile; use App\Utils\Traits\GeneratesCounter; @@ -29,7 +31,7 @@ use Cache; use Illuminate\Queue\SerializesModels; use Log; -class IngresEmailEngine +class InboundMailEngine { use SerializesModels, MakesHash; use GeneratesCounter, SavesDocuments; @@ -37,8 +39,8 @@ class IngresEmailEngine private ?Company $company; private ?bool $isUnknownRecipent = null; private array $globalBlacklistDomains = []; - private array $globalBlacklistEmails = []; - public function __construct(private IngresEmail $email) + private array $globalBlacklistSenders = []; + public function __construct(private InboundMail $email) { } /** @@ -53,7 +55,7 @@ class IngresEmailEngine $this->isUnknownRecipent = true; // Expense Mailbox => will create an expense - $this->company = MultiDB::findAndSetDbByExpenseMailbox($this->email->to); + $this->company = MultiDB::findAndSetDbByInboundMailbox($this->email->to); if ($this->company) { $this->isUnknownRecipent = false; $this->createExpense(); @@ -67,7 +69,7 @@ class IngresEmailEngine { // invalid email if (!filter_var($this->email->from, FILTER_VALIDATE_EMAIL)) { - $this->log('E-Mail blocked, because from e-mail has the wrong format: ' . $this->email->from); + $this->logBlocked('E-Mail blocked, because from e-mail has the wrong format: ' . $this->email->from); return true; } @@ -76,43 +78,43 @@ class IngresEmailEngine // global blacklist if (in_array($domain, $this->globalBlacklistDomains)) { - $this->log('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $this->email->from); + $this->logBlocked('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $this->email->from); return true; } - if (in_array($this->email->from, $this->globalBlacklistEmails)) { - $this->log('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $this->email->from); + if (in_array($this->email->from, $this->globalBlacklistSenders)) { + $this->logBlocked('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $this->email->from); return true; } - if (Cache::has('ingresEmailBlockedSender:' . $this->email->from)) { // was marked as blocked before, so we block without any console output + if (Cache::has('inboundMailBlockedSender:' . $this->email->from)) { // was marked as blocked before, so we block without any console output return true; } // sender occured in more than 500 emails in the last 12 hours - $senderMailCountTotal = Cache::get('ingresEmailSender:' . $this->email->from, 0); + $senderMailCountTotal = Cache::get('inboundMailSender:' . $this->email->from, 0); if ($senderMailCountTotal >= 5000) { - $this->log('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); + $this->logBlocked('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); $this->blockSender(); return true; } if ($senderMailCountTotal >= 1000) { - $this->log('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); + $this->logBlocked('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); $this->saveMeta(); return true; } // sender sended more than 50 emails to the wrong mailbox in the last 6 hours - $senderMailCountUnknownRecipent = Cache::get('ingresEmailSenderUnknownRecipent:' . $this->email->from, 0); + $senderMailCountUnknownRecipent = Cache::get('inboundMailSenderUnknownRecipent:' . $this->email->from, 0); if ($senderMailCountUnknownRecipent >= 50) { - $this->log('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $this->email->from); + $this->logBlocked('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $this->email->from); $this->saveMeta(); return true; } // wrong recipent occurs in more than 100 emails in the last 12 hours, so the processing is blocked - $mailCountUnknownRecipent = Cache::get('ingresEmailUnknownRecipent:' . $this->email->to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time + $mailCountUnknownRecipent = Cache::get('inboundMailUnknownRecipent:' . $this->email->to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time if ($mailCountUnknownRecipent >= 100) { - $this->log('E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); + $this->logBlocked('E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); $this->blockSender(); return true; } @@ -121,7 +123,7 @@ class IngresEmailEngine } private function blockSender() { - Cache::add('ingresEmailBlockedSender:' . $this->email->from, true, now()->addHours(12)); + Cache::add('inboundMailBlockedSender:' . $this->email->from, true, now()->addHours(12)); $this->saveMeta(); // TODO: ignore, when known sender (for heavy email-usage mostly on isHosted()) @@ -130,15 +132,15 @@ class IngresEmailEngine private function saveMeta() { // save cache - Cache::add('ingresEmailSender:' . $this->email->from, 0, now()->addHours(12)); - Cache::increment('ingresEmailSender:' . $this->email->from); + Cache::add('inboundMailSender:' . $this->email->from, 0, now()->addHours(12)); + Cache::increment('inboundMailSender:' . $this->email->from); if ($this->isUnknownRecipent) { - Cache::add('ingresEmailSenderUnknownRecipent:' . $this->email->from, 0, now()->addHours(6)); - Cache::increment('ingresEmailSenderUnknownRecipent:' . $this->email->from); // we save the sender, to may block him + Cache::add('inboundMailSenderUnknownRecipent:' . $this->email->from, 0, now()->addHours(6)); + Cache::increment('inboundMailSenderUnknownRecipent:' . $this->email->from); // we save the sender, to may block him - Cache::add('ingresEmailUnknownRecipent:' . $this->email->to, 0, now()->addHours(12)); - Cache::increment('ingresEmailUnknownRecipent:' . $this->email->to); // we save the sender, to may block him + Cache::add('inboundMailUnknownRecipent:' . $this->email->to, 0, now()->addHours(12)); + Cache::increment('inboundMailUnknownRecipent:' . $this->email->to); // we save the sender, to may block him } } @@ -156,15 +158,15 @@ class IngresEmailEngine { // Skipping executions: will not result in not saving Metadata to prevent usage of these conditions, to spam if (!$this->validateExpenseShouldProcess()) { - $this->log('mailbox not active for this company. from: ' . $this->email->from); + $this->logBlocked('mailbox not active for this company. from: ' . $this->email->from); return; } if (!$this->validateExpenseSender()) { - $this->log('invalid sender of an ingest email for this company. from: ' . $this->email->from); + $this->logBlocked('invalid sender of an ingest email for this company. from: ' . $this->email->from); return; } if (sizeOf($this->email->documents) == 0) { - $this->log('email does not contain any attachments and is likly not an expense. from: ' . $this->email->from); + $this->logBlocked('email does not contain any attachments and is likly not an expense. from: ' . $this->email->from); return; } @@ -176,7 +178,7 @@ class IngresEmailEngine $expense->date = $this->email->date; // handle vendor assignment - $expense_vendor = $this->getExpenseVendor(); + $expense_vendor = $this->getVendor(); if ($expense_vendor) $expense->vendor_id = $expense_vendor->id; @@ -198,7 +200,7 @@ class IngresEmailEngine // HELPERS private function validateExpenseShouldProcess() { - return $this->company?->expense_mailbox_active ?: false; + return $this->company?->inbound_mailbox_active ?: false; } private function validateExpenseSender() { @@ -206,37 +208,56 @@ class IngresEmailEngine $domain = array_pop($parts); // whitelists - $email_whitelist = explode(",", $this->company->expense_mailbox_whitelist_emails); + $email_whitelist = explode(",", $this->company->inbound_mailbox_whitelist_senders); if (in_array($this->email->from, $email_whitelist)) return true; - $domain_whitelist = explode(",", $this->company->expense_mailbox_whitelist_domains); + $domain_whitelist = explode(",", $this->company->inbound_mailbox_whitelist_domains); if (in_array($domain, $domain_whitelist)) return true; - $email_blacklist = explode(",", $this->company->expense_mailbox_blacklist_emails); + $email_blacklist = explode(",", $this->company->inbound_mailbox_blacklist_senders); if (in_array($this->email->from, $email_blacklist)) return false; - $domain_blacklist = explode(",", $this->company->expense_mailbox_blacklist_domains); + $domain_blacklist = explode(",", $this->company->inbound_mailbox_blacklist_domains); if (in_array($domain, $domain_blacklist)) return false; // allow unknown - if ($this->company->expense_mailbox_allow_unknown) + if ($this->company->inbound_mailbox_allow_unknown) return true; // own users - if ($this->company->expense_mailbox_allow_company_users && $this->company->users()->where("email", $this->email->from)->exists()) + if ($this->company->inbound_mailbox_allow_company_users && $this->company->users()->where("email", $this->email->from)->exists()) return true; - // from clients/vendors (if active) - if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->where("invoicing_email", $this->email->from)->orWhere("invoicing_domain", $domain)->exists()) + // from vendors (if active) + if ($this->company->inbound_mailbox_allow_vendors && $this->company->vendors()->where("invoicing_email", $this->email->from)->orWhere("invoicing_domain", $domain)->exists()) return true; - if ($this->company->expense_mailbox_allow_vendors && $this->company->vendors()->contacts()->where("email", $this->email->from)->exists()) + if ($this->company->inbound_mailbox_allow_vendors && $this->company->vendors()->contacts()->where("email", $this->email->from)->exists()) + return true; + + // from clients (if active) + if ($this->company->inbound_mailbox_allow_clients && $this->company->clients()->where("invoicing_email", $this->email->from)->orWhere("invoicing_domain", $domain)->exists()) + return true; + if ($this->company->inbound_mailbox_allow_clients && $this->company->clients()->contacts()->where("email", $this->email->from)->exists()) return true; // denie return false; } - private function getExpenseVendor() + private function getClient() + { + $parts = explode('@', $this->email->from); + $domain = array_pop($parts); + + $client = Client::where("company_id", $this->company->id)->where("email", $domain)->first(); + if ($client == null) { + $clientContact = ClientContact::where("company_id", $this->company->id)->where("email", $this->email->from)->first(); + $client = $clientContact->client(); + } + + return $client; + } + private function getVendor() { $parts = explode('@', $this->email->from); $domain = array_pop($parts); @@ -251,15 +272,15 @@ class IngresEmailEngine return $vendor; } - private function log(string $data) + private function logBlocked(string $data) { - Log::info("[IngresEmailEngine][company:" . $this->company->id . "] " . $data); + Log::info("[InboundMailEngine][company:" . $this->company->id . "] " . $data); ( new SystemLogger( $data, SystemLog::CATEGORY_MAIL, - SystemLog::EVENT_INGEST_EMAIL_FAILURE, + SystemLog::EVENT_INBOUND_MAIL_BLOCKED, SystemLog::TYPE_CUSTOM, null, $this->company diff --git a/app/Transformers/CompanyTransformer.php b/app/Transformers/CompanyTransformer.php index b56db1ae66b1..f3ebd108fc6e 100644 --- a/app/Transformers/CompanyTransformer.php +++ b/app/Transformers/CompanyTransformer.php @@ -204,7 +204,16 @@ class CompanyTransformer extends EntityTransformer 'invoice_task_project_header' => (bool) $company->invoice_task_project_header, 'invoice_task_item_description' => (bool) $company->invoice_task_item_description, 'origin_tax_data' => $company->origin_tax_data ?: new \stdClass, - 'expense_mailbox' => $company->expense_mailbox, + 'inbound_mailbox' => (bool) $company->inbound_mailbox, + 'inbound_mailbox_active' => (bool) $company->inbound_mailbox_active, + 'inbound_mailbox_allow_company_users' => (bool) $company->inbound_mailbox_allow_company_users, + 'inbound_mailbox_allow_vendors' => (bool) $company->inbound_mailbox_allow_vendors, + 'inbound_mailbox_allow_clients' => (bool) $company->inbound_mailbox_allow_clients, + 'inbound_mailbox_allow_unknown' => (bool) $company->inbound_mailbox_allow_unknown, + 'inbound_mailbox_blacklist_domains' => $company->inbound_mailbox_blacklist_domains, + 'inbound_mailbox_blacklist_senders' => $company->inbound_mailbox_blacklist_senders, + 'inbound_mailbox_whitelist_domains' => $company->inbound_mailbox_whitelist_domains, + 'inbound_mailbox_whitelist_senders' => $company->inbound_mailbox_whitelist_senders, 'smtp_host' => (string) $company->smtp_host ?? '', 'smtp_port' => (int) $company->smtp_port ?? 25, 'smtp_encryption' => (string) $company->smtp_encryption ?? 'tls', diff --git a/config/ninja.php b/config/ninja.php index a7ac185eb8d5..7274d816ff8c 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -84,11 +84,12 @@ return [ 'username' => 'user@example.com', 'clientname' => 'client@example.com', 'password' => 'password', - 'gocardless' => env('GOCARDLESS_KEYS',''), - 'square' => env('SQUARE_KEYS',''), - 'eway' => env('EWAY_KEYS',''), - 'mollie', env('MOLLIE_KEYS',''), - 'paytrace' => env('PAYTRACE_KEYS',''), + 'gocardless' => env('GOCARDLESS_KEYS', ''), + 'square' => env('SQUARE_KEYS', ''), + 'eway' => env('EWAY_KEYS', ''), + 'mollie', + env('MOLLIE_KEYS', ''), + 'paytrace' => env('PAYTRACE_KEYS', ''), 'stripe' => env('STRIPE_KEYS', ''), 'paypal' => env('PAYPAL_KEYS', ''), 'ppcp' => env('PPCP_KEYS', ''), @@ -235,16 +236,16 @@ return [ 'client_id' => env('PAYPAL_CLIENT_ID', null), 'webhook_id' => env('PAYPAL_WEBHOOK_ID', null), ], - 'ingest_mail' => [ + 'inbound_mailbox' => [ 'imap' => [ - 'servers' => env('ingest_mail_IMAP_SERVERS', ''), - 'ports' => env('ingest_mail_IMAP_PORTS', ''), - 'users' => env('ingest_mail_IMAP_USERS', ''), - 'passwords' => env('ingest_mail_IMAP_PASSWORDS', ''), - 'companies' => env('ingest_mail_IMAP_COMPANIES', '1'), + 'servers' => env('INBOUND_MAILBOX_IMAP_SERVERS', ''), + 'ports' => env('INBOUND_MAILBOX_IMAP_PORTS', ''), + 'users' => env('INBOUND_MAILBOX_IMAP_USERS', ''), + 'passwords' => env('INBOUND_MAILBOX_IMAP_PASSWORDS', ''), + 'companies' => env('INBOUND_MAILBOX_IMAP_COMPANIES', '1'), ], - 'expense_mailbox_template' => env('ingest_mail_EXPENSE_MAILBOX_TEMPLATE', null), - 'expense_mailbox_endings' => env('ingest_mail_EXPENSE_MAILBOX_ENDINGS', '@expense.invoicing.co'), + 'inbound_mailbox_template' => env('INBOUND_MAILBOX_TEMPLATE', null), + 'inbound_mailbox_endings' => env('INBOUND_MAILBOX_ENDINGS', '@expense.invoicing.co'), ], 'cloudflare' => [ 'turnstile' => [ @@ -256,5 +257,4 @@ return [ 'private_key' => env('NINJA_PRIVATE_KEY', false), ], 'upload_extensions' => env('ADDITIONAL_UPLOAD_EXTENSIONS', false), - ]; diff --git a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php index 8a8d62e22918..f1b1c21603d7 100644 --- a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php +++ b/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php @@ -12,15 +12,16 @@ return new class extends Migration { public function up(): void { Schema::table('companies', function (Blueprint $table) { - $table->boolean("expense_mailbox_active")->default(true); - $table->string("expense_mailbox")->nullable(); - $table->boolean("expense_mailbox_allow_company_users")->default(false); - $table->boolean("expense_mailbox_allow_vendors")->default(false); - $table->boolean("expense_mailbox_allow_unknown")->default(false); - $table->text("expense_mailbox_whitelist_domains")->nullable(); - $table->text("expense_mailbox_whitelist_emails")->nullable(); - $table->text("expense_mailbox_blacklist_domains")->nullable(); - $table->text("expense_mailbox_blacklist_emails")->nullable(); + $table->boolean("inbound_mailbox_active")->default(true); + $table->string("inbound_mailbox")->nullable(); + $table->boolean("inbound_mailbox_allow_company_users")->default(false); + $table->boolean("inbound_mailbox_allow_vendors")->default(false); + $table->boolean("inbound_mailbox_allow_clients")->default(false); + $table->boolean("inbound_mailbox_allow_unknown")->default(false); + $table->text("inbound_mailbox_whitelist_domains")->nullable(); + $table->text("inbound_mailbox_whitelist_senders")->nullable(); + $table->text("inbound_mailbox_blacklist_domains")->nullable(); + $table->text("inbound_mailbox_blacklist_senders")->nullable(); }); Schema::table('vendors', function (Blueprint $table) { $table->string("invoicing_email")->nullable(); diff --git a/lang/en/texts.php b/lang/en/texts.php index 7ec111242ccb..140099bf538d 100644 --- a/lang/en/texts.php +++ b/lang/en/texts.php @@ -2494,8 +2494,8 @@ $lang = array( 'local_storage_required' => 'Error: local storage is not available.', 'your_password_reset_link' => 'Your Password Reset Link', 'subdomain_taken' => 'The subdomain is already in use', - 'expense_mailbox_taken' => 'The mailbox is already in use', - 'expense_mailbox_invalid' => 'The mailbox does not match the required schema', + 'inbound_mailbox_taken' => 'The inbound mailbox is already in use', + 'inbound_mailbox_invalid' => 'The inbound mailbox does not match the required schema', 'client_login' => 'Client Login', 'converted_amount' => 'Converted Amount', 'default' => 'Default', From 1db0350273c871ca5e1c7de4f5815723b0845fba Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 25 Mar 2024 07:08:41 +0100 Subject: [PATCH 050/119] changes related to allow_clients --- app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 2 +- app/Services/InboundMail/InboundMailEngine.php | 17 ++++++----------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index d8257351ca41..774018070fa8 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -118,7 +118,7 @@ class ProcessBrevoInboundWebhook implements ShouldQueue public function handle() { - // brevo defines recipients as array to enable webhook processing as batches, we check all of them + // brevo defines recipients as array, we check all of them, to be sure foreach ($this->input["Recipients"] as $recipient) { // match company diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index c745fa2ac243..1c6b2f6f0fcd 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -229,15 +229,13 @@ class InboundMailEngine if ($this->company->inbound_mailbox_allow_company_users && $this->company->users()->where("email", $this->email->from)->exists()) return true; - // from vendors (if active) + // from vendors if ($this->company->inbound_mailbox_allow_vendors && $this->company->vendors()->where("invoicing_email", $this->email->from)->orWhere("invoicing_domain", $domain)->exists()) return true; if ($this->company->inbound_mailbox_allow_vendors && $this->company->vendors()->contacts()->where("email", $this->email->from)->exists()) return true; - // from clients (if active) - if ($this->company->inbound_mailbox_allow_clients && $this->company->clients()->where("invoicing_email", $this->email->from)->orWhere("invoicing_domain", $domain)->exists()) - return true; + // from clients if ($this->company->inbound_mailbox_allow_clients && $this->company->clients()->contacts()->where("email", $this->email->from)->exists()) return true; @@ -246,14 +244,11 @@ class InboundMailEngine } private function getClient() { - $parts = explode('@', $this->email->from); - $domain = array_pop($parts); + // $parts = explode('@', $this->email->from); + // $domain = array_pop($parts); - $client = Client::where("company_id", $this->company->id)->where("email", $domain)->first(); - if ($client == null) { - $clientContact = ClientContact::where("company_id", $this->company->id)->where("email", $this->email->from)->first(); - $client = $clientContact->client(); - } + $clientContact = ClientContact::where("company_id", $this->company->id)->where("email", $this->email->from)->first(); + $client = $clientContact->client(); return $client; } From a1b35909b062fbb5a70a4284bc2e831402b6c6b6 Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 25 Mar 2024 07:16:26 +0100 Subject: [PATCH 051/119] brevo webhook validation --- app/Http/Controllers/BrevoController.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/Http/Controllers/BrevoController.php b/app/Http/Controllers/BrevoController.php index bd4f1ad59480..6a1e8ddfc0b4 100644 --- a/app/Http/Controllers/BrevoController.php +++ b/app/Http/Controllers/BrevoController.php @@ -13,6 +13,7 @@ namespace App\Http\Controllers; use App\Jobs\Brevo\ProcessBrevoInboundWebhook; use App\Jobs\Brevo\ProcessBrevoWebhook; +use App\Libraries\MultiDB; use Illuminate\Http\Request; use Log; @@ -185,8 +186,17 @@ class BrevoController extends BaseController { $input = $request->all(); - // TODO: validation for client mail credentials by recipient - if (!($request->has('token') && $request->get('token') == config('services.brevo.secret'))) + // validation for client mail credentials by recipient + if ($request->has('company')) { + if (!($request->has('token'))) + return response()->json(['message' => 'Unauthorized'], 403); + + $company = MultiDB::findAndSetDbByCompanyId($request->has('company')); + $company_brevo_secret = $company->settings?->email_sending_method === 'client_brevo' && $company->settings?->brevo_secret ? $company->settings?->brevo_secret : null; + if (!$company || $request->get('token') !== $company_brevo_secret) + return response()->json(['message' => 'Unauthorized'], 403); + + } else if (!($request->has('token') && $request->get('token') == config('services.brevo.secret'))) return response()->json(['message' => 'Unauthorized'], 403); if (!array_key_exists('items', $input)) { From 098333018788ffb17e239607fbcef434ce9fcdea Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 25 Mar 2024 07:18:32 +0100 Subject: [PATCH 052/119] minor fixes --- app/Http/Controllers/BrevoController.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Http/Controllers/BrevoController.php b/app/Http/Controllers/BrevoController.php index 6a1e8ddfc0b4..c8db34674645 100644 --- a/app/Http/Controllers/BrevoController.php +++ b/app/Http/Controllers/BrevoController.php @@ -192,8 +192,8 @@ class BrevoController extends BaseController return response()->json(['message' => 'Unauthorized'], 403); $company = MultiDB::findAndSetDbByCompanyId($request->has('company')); - $company_brevo_secret = $company->settings?->email_sending_method === 'client_brevo' && $company->settings?->brevo_secret ? $company->settings?->brevo_secret : null; - if (!$company || $request->get('token') !== $company_brevo_secret) + $company_brevo_secret = $company?->settings?->email_sending_method === 'client_brevo' && $company?->settings?->brevo_secret ? $company->settings->brevo_secret : null; + if (!$company || !$company_brevo_secret || $request->get('token') !== $company_brevo_secret) return response()->json(['message' => 'Unauthorized'], 403); } else if (!($request->has('token') && $request->get('token') == config('services.brevo.secret'))) From ab8d960384038d6b2d9dfe4aac332abce66c7536 Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 1 Apr 2024 16:04:49 +0200 Subject: [PATCH 053/119] drop imap support classes --- app/Console/Kernel.php | 4 - .../Transformer/ImapMailTransformer.php | 38 --- app/Helpers/Mail/Mailbox/BaseMailbox.php | 19 -- app/Helpers/Mail/Mailbox/Imap/ImapMailbox.php | 54 --- app/Jobs/Imap/ProcessImapMailboxJob.php | 139 -------- composer.json | 1 - composer.lock | 318 +++++++----------- config/ninja.php | 7 - 8 files changed, 122 insertions(+), 458 deletions(-) delete mode 100644 app/Helpers/InboundMail/Transformer/ImapMailTransformer.php delete mode 100644 app/Helpers/Mail/Mailbox/BaseMailbox.php delete mode 100644 app/Helpers/Mail/Mailbox/Imap/ImapMailbox.php delete mode 100644 app/Jobs/Imap/ProcessImapMailboxJob.php diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 9a37eaf755f4..ed0d7e512b59 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -17,7 +17,6 @@ use App\Jobs\Cron\RecurringInvoicesCron; use App\Jobs\Cron\SubscriptionCron; use App\Jobs\Cron\UpdateCalculatedFields; use App\Jobs\Invoice\InvoiceCheckLateWebhook; -use App\Jobs\Imap\ProcessImapMailboxJob; use App\Jobs\Ninja\AdjustEmailQuota; use App\Jobs\Ninja\BankTransactionSync; use App\Jobs\Ninja\CheckACHStatus; @@ -105,9 +104,6 @@ class Kernel extends ConsoleKernel $schedule->call(function () { Account::query()->whereNotNull('id')->update(['is_scheduler_running' => true]); })->everyFiveMinutes(); - - /* Check ImapMailboxes */ - $schedule->job(new ProcessImapMailboxJob)->everyFiveMinutes()->withoutOverlapping()->name('imap-mailbox-job')->onOneServer(); } /* Run hosted specific jobs */ diff --git a/app/Helpers/InboundMail/Transformer/ImapMailTransformer.php b/app/Helpers/InboundMail/Transformer/ImapMailTransformer.php deleted file mode 100644 index 4229d7fb5e79..000000000000 --- a/app/Helpers/InboundMail/Transformer/ImapMailTransformer.php +++ /dev/null @@ -1,38 +0,0 @@ -from = $mail->getSender(); - $inboundMail->subject = $mail->getSubject(); - $inboundMail->plain_message = $mail->getBodyText(); - $inboundMail->html_message = $mail->getBodyHtml(); - $inboundMail->date = $mail->getDate(); - - // parse documents as UploadedFile - foreach ($mail->getAttachments() as $attachment) { - $inboundMail->documents[] = TempFile::UploadedFileFromRaw($attachment->getContent(), $attachment->getFilename(), $attachment->getEncoding()); - } - - return $inboundMail; - } -} diff --git a/app/Helpers/Mail/Mailbox/BaseMailbox.php b/app/Helpers/Mail/Mailbox/BaseMailbox.php deleted file mode 100644 index 6ab63b11acac..000000000000 --- a/app/Helpers/Mail/Mailbox/BaseMailbox.php +++ /dev/null @@ -1,19 +0,0 @@ -server = new Server($server, $port != '' ? $port : null); - - $this->connection = $this->server->authenticate($user, $password); - } - - - public function getUnprocessedEmails() - { - $mailbox = $this->connection->getMailbox('INBOX'); - - $search = new SearchExpression(); - - // not older than 30days - $today = new \DateTimeImmutable(); - $thirtyDaysAgo = $today->sub(new \DateInterval('P30D')); - $search->addCondition(new Since($thirtyDaysAgo)); - - return $mailbox->getMessages($search); - } - - public function moveProcessed(MessageInterface $mail) - { - return $mail->move($this->connection->getMailbox('PROCESSED')); - } - - public function moveFailed(MessageInterface $mail) - { - return $mail->move($this->connection->getMailbox('FAILED')); - } -} diff --git a/app/Jobs/Imap/ProcessImapMailboxJob.php b/app/Jobs/Imap/ProcessImapMailboxJob.php deleted file mode 100644 index e6aef5fd3d5e..000000000000 --- a/app/Jobs/Imap/ProcessImapMailboxJob.php +++ /dev/null @@ -1,139 +0,0 @@ -credentials = []; - - $this->getImapCredentials(); - - $this->expense_repo = new ExpenseRepository(); // @turbo124 @todo is this the right aproach? should it be handled just with the model? - } - - public function handle() - { - - //multiDB environment, need to - if (sizeOf($this->imap_credentials) == 0) - return; - - foreach ($this->imap_companies as $companyId) { - $company = MultiDB::findAndSetDbByCompanyId($companyId); - if (!$company) { - nlog("processing of an imap_mailbox skipped because of unknown companyId: " . $companyId); - return; - } - - try { - nlog("start importing expenses from imap-server of company: " . $companyId); - $this->handleImapCompany($company); - - } catch (\Exception $e) { - nlog("processing of an imap_mailbox failed upnormally: " . $companyId . " message: " . $e->getMessage()); // @turbo124 @todo should this be handled in an other way? - } - } - - } - - private function getImapCredentials() - { - $servers = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.servers'))); - $ports = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.ports'))); - $users = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.users'))); - $passwords = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.passwords'))); - $companies = array_map('trim', explode(",", config('ninja.inbound_mailbox.imap.companies'))); - - if (sizeOf($servers) != sizeOf($ports) || sizeOf($servers) != sizeOf($users) || sizeOf($servers) != sizeOf($passwords) || sizeOf($servers) != sizeOf($companies)) - throw new \Exception('invalid configuration inbound_mailbox.imap (wrong element-count)'); - - foreach ($companies as $index => $companyId) { - - if ($servers[$index] == '') // if property is empty, ignore => this happens exspecialy when no config is provided and it enabled us to set a single default company for env (usefull on self-hosted) - continue; - - $this->imap_credentials[$companyId] = [ - "server" => $servers[$index], - "port" => $ports[$index] != '' ? $ports[$index] : null, - "user" => $users[$index], - "password" => $passwords[$index], - ]; - $this->imap_companies[] = $companyId; - - } - } - - private function handleImapCompany(Company $company) - { - nlog("importing expenses for company: " . $company->id); - - $credentials = $this->imap_credentials[$company->id]; - $imapMailbox = new ImapMailbox($credentials->server, $credentials->port, $credentials->user, $credentials->password); - $transformer = new ImapMailTransformer(); - - $emails = $imapMailbox->getUnprocessedEmails(); - - - foreach ($emails as $email) { - - try { - - $email->markAsSeen(); - - InboundMailEngine::dispatch($transformer->transform($email)); - - $imapMailbox->moveProcessed($email); - - } catch (\Exception $e) { - $imapMailbox->moveFailed($email); - - nlog("processing of an email failed upnormally: " . $company->id . " message: " . $e->getMessage()); - } - - } - } - - public function backoff() - { - // return [5, 10, 30, 240]; - return [rand(5, 10), rand(30, 40), rand(60, 79), rand(160, 400)]; - - } - -} diff --git a/composer.json b/composer.json index bca5882fba7f..b22ac63d55f6 100644 --- a/composer.json +++ b/composer.json @@ -44,7 +44,6 @@ "braintree/braintree_php": "^6.0", "checkout/checkout-sdk-php": "^3.0", "invoiceninja/ubl_invoice": "^2", - "ddeboer/imap": "^1.19", "doctrine/dbal": "^3.0", "eway/eway-rapid-php": "^1.3", "fakerphp/faker": "^1.14", diff --git a/composer.lock b/composer.lock index df236124fc88..0dc8a7205e8e 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "a7fb762c099a95d6168ef390844bb87b", + "content-hash": "64e61e6a7e50b5bb69dd08c1f8dbc38c", "packages": [ { "name": "adrienrn/php-mimetyper", @@ -409,16 +409,16 @@ }, { "name": "amphp/parallel", - "version": "v2.2.8", + "version": "v2.2.9", "source": { "type": "git", "url": "https://github.com/amphp/parallel.git", - "reference": "efd71b342b64c2e46d904e4eb057ed5ab20f8e2d" + "reference": "73d293f1fc4df1bebc3c4fce1432e82dd7032238" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/parallel/zipball/efd71b342b64c2e46d904e4eb057ed5ab20f8e2d", - "reference": "efd71b342b64c2e46d904e4eb057ed5ab20f8e2d", + "url": "https://api.github.com/repos/amphp/parallel/zipball/73d293f1fc4df1bebc3c4fce1432e82dd7032238", + "reference": "73d293f1fc4df1bebc3c4fce1432e82dd7032238", "shasum": "" }, "require": { @@ -481,7 +481,7 @@ ], "support": { "issues": "https://github.com/amphp/parallel/issues", - "source": "https://github.com/amphp/parallel/tree/v2.2.8" + "source": "https://github.com/amphp/parallel/tree/v2.2.9" }, "funding": [ { @@ -489,7 +489,7 @@ "type": "github" } ], - "time": "2024-03-19T16:09:34+00:00" + "time": "2024-03-24T18:27:44+00:00" }, { "name": "amphp/parser", @@ -959,16 +959,16 @@ }, { "name": "apimatic/core", - "version": "0.3.6", + "version": "0.3.7", "source": { "type": "git", "url": "https://github.com/apimatic/core-lib-php.git", - "reference": "2236fa751f265397d97ab2cb8a5c72641cf9480f" + "reference": "50fafa9c463ef8440f4b8841a3ab9913be4641ee" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/apimatic/core-lib-php/zipball/2236fa751f265397d97ab2cb8a5c72641cf9480f", - "reference": "2236fa751f265397d97ab2cb8a5c72641cf9480f", + "url": "https://api.github.com/repos/apimatic/core-lib-php/zipball/50fafa9c463ef8440f4b8841a3ab9913be4641ee", + "reference": "50fafa9c463ef8440f4b8841a3ab9913be4641ee", "shasum": "" }, "require": { @@ -1006,9 +1006,9 @@ ], "support": { "issues": "https://github.com/apimatic/core-lib-php/issues", - "source": "https://github.com/apimatic/core-lib-php/tree/0.3.6" + "source": "https://github.com/apimatic/core-lib-php/tree/0.3.7" }, - "time": "2024-02-26T04:16:28+00:00" + "time": "2024-04-01T10:19:31+00:00" }, { "name": "apimatic/core-interfaces", @@ -1384,16 +1384,16 @@ }, { "name": "aws/aws-sdk-php", - "version": "3.301.6", + "version": "3.302.0", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "18c0ebd71d3071304f1ea02aa9af75f95863177a" + "reference": "cb343ed4fc5d86c0ddf8e948f0271052f183f937" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/18c0ebd71d3071304f1ea02aa9af75f95863177a", - "reference": "18c0ebd71d3071304f1ea02aa9af75f95863177a", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/cb343ed4fc5d86c0ddf8e948f0271052f183f937", + "reference": "cb343ed4fc5d86c0ddf8e948f0271052f183f937", "shasum": "" }, "require": { @@ -1473,9 +1473,9 @@ "support": { "forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80", "issues": "https://github.com/aws/aws-sdk-php/issues", - "source": "https://github.com/aws/aws-sdk-php/tree/3.301.6" + "source": "https://github.com/aws/aws-sdk-php/tree/3.302.0" }, - "time": "2024-03-22T18:05:21+00:00" + "time": "2024-03-29T18:07:04+00:00" }, { "name": "bacon/bacon-qr-code", @@ -1585,16 +1585,16 @@ }, { "name": "braintree/braintree_php", - "version": "6.17.0", + "version": "6.18.0", "source": { "type": "git", "url": "https://github.com/braintree/braintree_php.git", - "reference": "37c187c91416003708632a58c230d03dbe88fb67" + "reference": "8ca67004fe2405ef0b6b33a5897594fdcf417e0e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/braintree/braintree_php/zipball/37c187c91416003708632a58c230d03dbe88fb67", - "reference": "37c187c91416003708632a58c230d03dbe88fb67", + "url": "https://api.github.com/repos/braintree/braintree_php/zipball/8ca67004fe2405ef0b6b33a5897594fdcf417e0e", + "reference": "8ca67004fe2405ef0b6b33a5897594fdcf417e0e", "shasum": "" }, "require": { @@ -1628,9 +1628,9 @@ "description": "Braintree PHP Client Library", "support": { "issues": "https://github.com/braintree/braintree_php/issues", - "source": "https://github.com/braintree/braintree_php/tree/6.17.0" + "source": "https://github.com/braintree/braintree_php/tree/6.18.0" }, - "time": "2024-03-06T20:01:30+00:00" + "time": "2024-03-26T21:08:13+00:00" }, { "name": "brick/math", @@ -1758,16 +1758,16 @@ }, { "name": "checkout/checkout-sdk-php", - "version": "3.0.21", + "version": "3.1.0", "source": { "type": "git", "url": "https://github.com/checkout/checkout-sdk-php.git", - "reference": "0195aa0153b79b3f8350509e54a5654e57f62bd3" + "reference": "dc1b71009f2456cabde720ee38d225c7e177adfb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/checkout/checkout-sdk-php/zipball/0195aa0153b79b3f8350509e54a5654e57f62bd3", - "reference": "0195aa0153b79b3f8350509e54a5654e57f62bd3", + "url": "https://api.github.com/repos/checkout/checkout-sdk-php/zipball/dc1b71009f2456cabde720ee38d225c7e177adfb", + "reference": "dc1b71009f2456cabde720ee38d225c7e177adfb", "shasum": "" }, "require": { @@ -1820,9 +1820,9 @@ ], "support": { "issues": "https://github.com/checkout/checkout-sdk-php/issues", - "source": "https://github.com/checkout/checkout-sdk-php/tree/3.0.21" + "source": "https://github.com/checkout/checkout-sdk-php/tree/3.1.0" }, - "time": "2024-02-08T17:30:23+00:00" + "time": "2024-03-26T12:27:04+00:00" }, { "name": "clue/stream-filter", @@ -2060,82 +2060,6 @@ }, "time": "2022-09-20T18:15:38+00:00" }, - { - "name": "ddeboer/imap", - "version": "1.19.0", - "source": { - "type": "git", - "url": "https://github.com/ddeboer/imap.git", - "reference": "30800b1cfeacc4add5bb418e40a8b6e95a8a04ac" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/ddeboer/imap/zipball/30800b1cfeacc4add5bb418e40a8b6e95a8a04ac", - "reference": "30800b1cfeacc4add5bb418e40a8b6e95a8a04ac", - "shasum": "" - }, - "require": { - "ext-dom": "*", - "ext-iconv": "*", - "ext-imap": "*", - "ext-libxml": "*", - "ext-mbstring": "*", - "php": "~8.2.0 || ~8.3.0" - }, - "require-dev": { - "friendsofphp/php-cs-fixer": "^3.38.2", - "laminas/laminas-mail": "^2.25.1", - "phpstan/phpstan": "^1.10.43", - "phpstan/phpstan-phpunit": "^1.3.15", - "phpstan/phpstan-strict-rules": "^1.5.2", - "phpunit/phpunit": "^10.4.2" - }, - "type": "library", - "autoload": { - "psr-4": { - "Ddeboer\\Imap\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "David de Boer", - "email": "david@ddeboer.nl" - }, - { - "name": "Filippo Tessarotto", - "email": "zoeslam@gmail.com" - }, - { - "name": "Community contributors", - "homepage": "https://github.com/ddeboer/imap/graphs/contributors" - } - ], - "description": "Object-oriented IMAP for PHP", - "keywords": [ - "email", - "imap", - "mail" - ], - "support": { - "issues": "https://github.com/ddeboer/imap/issues", - "source": "https://github.com/ddeboer/imap/tree/1.19.0" - }, - "funding": [ - { - "url": "https://github.com/Slamdunk", - "type": "github" - }, - { - "url": "https://github.com/ddeboer", - "type": "github" - } - ], - "time": "2023-11-20T14:41:54+00:00" - }, { "name": "dflydev/apache-mime-types", "version": "v1.0.1", @@ -3618,16 +3542,16 @@ }, { "name": "google/apiclient-services", - "version": "v0.340.0", + "version": "v0.342.0", "source": { "type": "git", "url": "https://github.com/googleapis/google-api-php-client-services.git", - "reference": "c89999ea477da2b0803b2b4f14c9e7fc23b6344a" + "reference": "b75a249bd9abf0241822fa6d9d9b396583f5003f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/c89999ea477da2b0803b2b4f14c9e7fc23b6344a", - "reference": "c89999ea477da2b0803b2b4f14c9e7fc23b6344a", + "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/b75a249bd9abf0241822fa6d9d9b396583f5003f", + "reference": "b75a249bd9abf0241822fa6d9d9b396583f5003f", "shasum": "" }, "require": { @@ -3656,9 +3580,9 @@ ], "support": { "issues": "https://github.com/googleapis/google-api-php-client-services/issues", - "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.340.0" + "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.342.0" }, - "time": "2024-03-17T00:56:17+00:00" + "time": "2024-04-01T01:08:41+00:00" }, { "name": "google/auth", @@ -4778,33 +4702,33 @@ }, { "name": "imdhemy/appstore-iap", - "version": "1.6.0", + "version": "1.6.1", "source": { "type": "git", "url": "https://github.com/imdhemy/appstore-iap.git", - "reference": "ccf75f2d9fd628ebabfc7e9ee9a6754259769c1b" + "reference": "025d176a097b864f306dad7dc3506598aa6e5990" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/imdhemy/appstore-iap/zipball/ccf75f2d9fd628ebabfc7e9ee9a6754259769c1b", - "reference": "ccf75f2d9fd628ebabfc7e9ee9a6754259769c1b", + "url": "https://api.github.com/repos/imdhemy/appstore-iap/zipball/025d176a097b864f306dad7dc3506598aa6e5990", + "reference": "025d176a097b864f306dad7dc3506598aa6e5990", "shasum": "" }, "require": { "ext-json": "*", "ext-openssl": "*", "ext-sodium": "*", - "guzzlehttp/guzzle": "^6.5|^7.5", + "guzzlehttp/guzzle": "^7.6.0", "lcobucci/jwt": "^4.3", - "nesbot/carbon": "^2.66", + "nesbot/carbon": "^2.66|^3.1", "php": ">=8.0" }, "require-dev": { - "fakerphp/faker": "^1.21", + "fakerphp/faker": "^1.22", "friendsofphp/php-cs-fixer": "^3.16", "phpunit/phpunit": "^9.6", "roave/security-advisories": "dev-latest", - "vimeo/psalm": "^5.9" + "vimeo/psalm": "^5.11" }, "type": "library", "autoload": { @@ -4825,9 +4749,9 @@ "description": "PHP Appstore In-App Purchase implementation", "support": { "issues": "https://github.com/imdhemy/appstore-iap/issues", - "source": "https://github.com/imdhemy/appstore-iap/tree/1.6.0" + "source": "https://github.com/imdhemy/appstore-iap/tree/1.6.1" }, - "time": "2023-04-08T10:10:00+00:00" + "time": "2024-03-27T09:17:17+00:00" }, { "name": "imdhemy/google-play-billing", @@ -5810,16 +5734,16 @@ }, { "name": "laravel/prompts", - "version": "v0.1.16", + "version": "v0.1.17", "source": { "type": "git", "url": "https://github.com/laravel/prompts.git", - "reference": "ca6872ab6aec3ab61db3a61f83a6caf764ec7781" + "reference": "8ee9f87f7f9eadcbe21e9e72cd4176b2f06cd5b5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/prompts/zipball/ca6872ab6aec3ab61db3a61f83a6caf764ec7781", - "reference": "ca6872ab6aec3ab61db3a61f83a6caf764ec7781", + "url": "https://api.github.com/repos/laravel/prompts/zipball/8ee9f87f7f9eadcbe21e9e72cd4176b2f06cd5b5", + "reference": "8ee9f87f7f9eadcbe21e9e72cd4176b2f06cd5b5", "shasum": "" }, "require": { @@ -5861,9 +5785,9 @@ ], "support": { "issues": "https://github.com/laravel/prompts/issues", - "source": "https://github.com/laravel/prompts/tree/v0.1.16" + "source": "https://github.com/laravel/prompts/tree/v0.1.17" }, - "time": "2024-02-21T19:25:27+00:00" + "time": "2024-03-13T16:05:43+00:00" }, { "name": "laravel/serializable-closure", @@ -6124,16 +6048,16 @@ }, { "name": "laravel/ui", - "version": "v4.5.0", + "version": "v4.5.1", "source": { "type": "git", "url": "https://github.com/laravel/ui.git", - "reference": "da3811f409297d13feccd5858ce748e7474b3d11" + "reference": "a3562953123946996a503159199d6742d5534e61" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/ui/zipball/da3811f409297d13feccd5858ce748e7474b3d11", - "reference": "da3811f409297d13feccd5858ce748e7474b3d11", + "url": "https://api.github.com/repos/laravel/ui/zipball/a3562953123946996a503159199d6742d5534e61", + "reference": "a3562953123946996a503159199d6742d5534e61", "shasum": "" }, "require": { @@ -6141,7 +6065,8 @@ "illuminate/filesystem": "^9.21|^10.0|^11.0", "illuminate/support": "^9.21|^10.0|^11.0", "illuminate/validation": "^9.21|^10.0|^11.0", - "php": "^8.0" + "php": "^8.0", + "symfony/console": "^6.0|^7.0" }, "require-dev": { "orchestra/testbench": "^7.35|^8.15|^9.0", @@ -6180,9 +6105,9 @@ "ui" ], "support": { - "source": "https://github.com/laravel/ui/tree/v4.5.0" + "source": "https://github.com/laravel/ui/tree/v4.5.1" }, - "time": "2024-03-04T13:58:27+00:00" + "time": "2024-03-21T18:12:29+00:00" }, { "name": "lcobucci/clock", @@ -6601,16 +6526,16 @@ }, { "name": "league/flysystem", - "version": "3.25.1", + "version": "3.26.0", "source": { "type": "git", "url": "https://github.com/thephpleague/flysystem.git", - "reference": "abbd664eb4381102c559d358420989f835208f18" + "reference": "072735c56cc0da00e10716dd90d5a7f7b40b36be" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/abbd664eb4381102c559d358420989f835208f18", - "reference": "abbd664eb4381102c559d358420989f835208f18", + "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/072735c56cc0da00e10716dd90d5a7f7b40b36be", + "reference": "072735c56cc0da00e10716dd90d5a7f7b40b36be", "shasum": "" }, "require": { @@ -6675,7 +6600,7 @@ ], "support": { "issues": "https://github.com/thephpleague/flysystem/issues", - "source": "https://github.com/thephpleague/flysystem/tree/3.25.1" + "source": "https://github.com/thephpleague/flysystem/tree/3.26.0" }, "funding": [ { @@ -6687,20 +6612,20 @@ "type": "github" } ], - "time": "2024-03-16T12:53:19+00:00" + "time": "2024-03-25T11:49:53+00:00" }, { "name": "league/flysystem-aws-s3-v3", - "version": "3.25.1", + "version": "3.26.0", "source": { "type": "git", "url": "https://github.com/thephpleague/flysystem-aws-s3-v3.git", - "reference": "6a5be0e6d6a93574e80805c9cc108a4b63c824d8" + "reference": "885d0a758c71ae3cd6c503544573a1fdb8dc754f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem-aws-s3-v3/zipball/6a5be0e6d6a93574e80805c9cc108a4b63c824d8", - "reference": "6a5be0e6d6a93574e80805c9cc108a4b63c824d8", + "url": "https://api.github.com/repos/thephpleague/flysystem-aws-s3-v3/zipball/885d0a758c71ae3cd6c503544573a1fdb8dc754f", + "reference": "885d0a758c71ae3cd6c503544573a1fdb8dc754f", "shasum": "" }, "require": { @@ -6740,7 +6665,7 @@ "storage" ], "support": { - "source": "https://github.com/thephpleague/flysystem-aws-s3-v3/tree/3.25.1" + "source": "https://github.com/thephpleague/flysystem-aws-s3-v3/tree/3.26.0" }, "funding": [ { @@ -6752,7 +6677,7 @@ "type": "github" } ], - "time": "2024-03-15T19:58:44+00:00" + "time": "2024-03-24T21:11:18+00:00" }, { "name": "league/flysystem-local", @@ -9151,16 +9076,16 @@ }, { "name": "php-http/discovery", - "version": "1.19.2", + "version": "1.19.4", "source": { "type": "git", "url": "https://github.com/php-http/discovery.git", - "reference": "61e1a1eb69c92741f5896d9e05fb8e9d7e8bb0cb" + "reference": "0700efda8d7526335132360167315fdab3aeb599" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-http/discovery/zipball/61e1a1eb69c92741f5896d9e05fb8e9d7e8bb0cb", - "reference": "61e1a1eb69c92741f5896d9e05fb8e9d7e8bb0cb", + "url": "https://api.github.com/repos/php-http/discovery/zipball/0700efda8d7526335132360167315fdab3aeb599", + "reference": "0700efda8d7526335132360167315fdab3aeb599", "shasum": "" }, "require": { @@ -9184,7 +9109,8 @@ "php-http/httplug": "^1.0 || ^2.0", "php-http/message-factory": "^1.0", "phpspec/phpspec": "^5.1 || ^6.1 || ^7.3", - "symfony/phpunit-bridge": "^6.2" + "sebastian/comparator": "^3.0.5 || ^4.0.8", + "symfony/phpunit-bridge": "^6.4.4 || ^7.0.1" }, "type": "composer-plugin", "extra": { @@ -9223,9 +9149,9 @@ ], "support": { "issues": "https://github.com/php-http/discovery/issues", - "source": "https://github.com/php-http/discovery/tree/1.19.2" + "source": "https://github.com/php-http/discovery/tree/1.19.4" }, - "time": "2023-11-30T16:49:05+00:00" + "time": "2024-03-29T13:00:05+00:00" }, { "name": "php-http/guzzle7-adapter", @@ -11160,16 +11086,16 @@ }, { "name": "rmccue/requests", - "version": "v2.0.10", + "version": "v2.0.11", "source": { "type": "git", "url": "https://github.com/WordPress/Requests.git", - "reference": "bcf1ac7fe8c0b2b18c1df6d24694cfc96b44b391" + "reference": "31435a468e2357e68df743f2527bda32556a0818" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/WordPress/Requests/zipball/bcf1ac7fe8c0b2b18c1df6d24694cfc96b44b391", - "reference": "bcf1ac7fe8c0b2b18c1df6d24694cfc96b44b391", + "url": "https://api.github.com/repos/WordPress/Requests/zipball/31435a468e2357e68df743f2527bda32556a0818", + "reference": "31435a468e2357e68df743f2527bda32556a0818", "shasum": "" }, "require": { @@ -11243,7 +11169,7 @@ "issues": "https://github.com/WordPress/Requests/issues", "source": "https://github.com/WordPress/Requests" }, - "time": "2024-01-08T11:14:32+00:00" + "time": "2024-03-25T10:48:46+00:00" }, { "name": "sabre/uri", @@ -16305,16 +16231,16 @@ "packages-dev": [ { "name": "barryvdh/laravel-debugbar", - "version": "v3.12.2", + "version": "v3.12.4", "source": { "type": "git", "url": "https://github.com/barryvdh/laravel-debugbar.git", - "reference": "43555503052443964ce2c1c1f3b0378e58219eb8" + "reference": "e7a9a217512d8f1d07448fd241ce2ac0922ddc2c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/barryvdh/laravel-debugbar/zipball/43555503052443964ce2c1c1f3b0378e58219eb8", - "reference": "43555503052443964ce2c1c1f3b0378e58219eb8", + "url": "https://api.github.com/repos/barryvdh/laravel-debugbar/zipball/e7a9a217512d8f1d07448fd241ce2ac0922ddc2c", + "reference": "e7a9a217512d8f1d07448fd241ce2ac0922ddc2c", "shasum": "" }, "require": { @@ -16373,7 +16299,7 @@ ], "support": { "issues": "https://github.com/barryvdh/laravel-debugbar/issues", - "source": "https://github.com/barryvdh/laravel-debugbar/tree/v3.12.2" + "source": "https://github.com/barryvdh/laravel-debugbar/tree/v3.12.4" }, "funding": [ { @@ -16385,7 +16311,7 @@ "type": "github" } ], - "time": "2024-03-13T09:50:34+00:00" + "time": "2024-04-01T09:14:15+00:00" }, { "name": "barryvdh/laravel-ide-helper", @@ -16914,16 +16840,16 @@ }, { "name": "composer/xdebug-handler", - "version": "3.0.3", + "version": "3.0.4", "source": { "type": "git", "url": "https://github.com/composer/xdebug-handler.git", - "reference": "ced299686f41dce890debac69273b47ffe98a40c" + "reference": "4f988f8fdf580d53bdb2d1278fe93d1ed5462255" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/xdebug-handler/zipball/ced299686f41dce890debac69273b47ffe98a40c", - "reference": "ced299686f41dce890debac69273b47ffe98a40c", + "url": "https://api.github.com/repos/composer/xdebug-handler/zipball/4f988f8fdf580d53bdb2d1278fe93d1ed5462255", + "reference": "4f988f8fdf580d53bdb2d1278fe93d1ed5462255", "shasum": "" }, "require": { @@ -16934,7 +16860,7 @@ "require-dev": { "phpstan/phpstan": "^1.0", "phpstan/phpstan-strict-rules": "^1.1", - "symfony/phpunit-bridge": "^6.0" + "phpunit/phpunit": "^8.5 || ^9.6 || ^10.5" }, "type": "library", "autoload": { @@ -16958,9 +16884,9 @@ "performance" ], "support": { - "irc": "irc://irc.freenode.org/composer", + "irc": "ircs://irc.libera.chat:6697/composer", "issues": "https://github.com/composer/xdebug-handler/issues", - "source": "https://github.com/composer/xdebug-handler/tree/3.0.3" + "source": "https://github.com/composer/xdebug-handler/tree/3.0.4" }, "funding": [ { @@ -16976,7 +16902,7 @@ "type": "tidelift" } ], - "time": "2022-02-25T21:32:43+00:00" + "time": "2024-03-26T18:29:49+00:00" }, { "name": "fidry/cpu-core-counter", @@ -17926,16 +17852,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.10.65", + "version": "1.10.66", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "3c657d057a0b7ecae19cb12db446bbc99d8839c6" + "reference": "94779c987e4ebd620025d9e5fdd23323903950bd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/3c657d057a0b7ecae19cb12db446bbc99d8839c6", - "reference": "3c657d057a0b7ecae19cb12db446bbc99d8839c6", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/94779c987e4ebd620025d9e5fdd23323903950bd", + "reference": "94779c987e4ebd620025d9e5fdd23323903950bd", "shasum": "" }, "require": { @@ -17984,7 +17910,7 @@ "type": "tidelift" } ], - "time": "2024-03-23T10:30:26+00:00" + "time": "2024-03-28T16:17:31+00:00" }, { "name": "phpunit/php-code-coverage", @@ -18309,16 +18235,16 @@ }, { "name": "phpunit/phpunit", - "version": "10.5.15", + "version": "10.5.16", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "86376e05e8745ed81d88232ff92fee868247b07b" + "reference": "18f8d4a5f52b61fdd9370aaae3167daa0eeb69cd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/86376e05e8745ed81d88232ff92fee868247b07b", - "reference": "86376e05e8745ed81d88232ff92fee868247b07b", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/18f8d4a5f52b61fdd9370aaae3167daa0eeb69cd", + "reference": "18f8d4a5f52b61fdd9370aaae3167daa0eeb69cd", "shasum": "" }, "require": { @@ -18390,7 +18316,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.15" + "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.16" }, "funding": [ { @@ -18406,7 +18332,7 @@ "type": "tidelift" } ], - "time": "2024-03-22T04:17:47+00:00" + "time": "2024-03-28T10:08:10+00:00" }, { "name": "sebastian/cli-parser", @@ -19457,16 +19383,16 @@ }, { "name": "spatie/ignition", - "version": "1.12.0", + "version": "1.13.1", "source": { "type": "git", "url": "https://github.com/spatie/ignition.git", - "reference": "5b6f801c605a593106b623e45ca41496a6e7d56d" + "reference": "889bf1dfa59e161590f677728b47bf4a6893983b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/spatie/ignition/zipball/5b6f801c605a593106b623e45ca41496a6e7d56d", - "reference": "5b6f801c605a593106b623e45ca41496a6e7d56d", + "url": "https://api.github.com/repos/spatie/ignition/zipball/889bf1dfa59e161590f677728b47bf4a6893983b", + "reference": "889bf1dfa59e161590f677728b47bf4a6893983b", "shasum": "" }, "require": { @@ -19536,20 +19462,20 @@ "type": "github" } ], - "time": "2024-01-03T15:49:39+00:00" + "time": "2024-03-29T14:03:47+00:00" }, { "name": "spatie/laravel-ignition", - "version": "2.4.2", + "version": "2.5.0", "source": { "type": "git", "url": "https://github.com/spatie/laravel-ignition.git", - "reference": "351504f4570e32908839fc5a2dc53bf77d02f85e" + "reference": "e23f4e8ce6644dc3d68b9d8a0aed3beaca0d6ada" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/spatie/laravel-ignition/zipball/351504f4570e32908839fc5a2dc53bf77d02f85e", - "reference": "351504f4570e32908839fc5a2dc53bf77d02f85e", + "url": "https://api.github.com/repos/spatie/laravel-ignition/zipball/e23f4e8ce6644dc3d68b9d8a0aed3beaca0d6ada", + "reference": "e23f4e8ce6644dc3d68b9d8a0aed3beaca0d6ada", "shasum": "" }, "require": { @@ -19559,7 +19485,7 @@ "illuminate/support": "^10.0|^11.0", "php": "^8.1", "spatie/flare-client-php": "^1.3.5", - "spatie/ignition": "^1.9", + "spatie/ignition": "^1.13", "symfony/console": "^6.2.3|^7.0", "symfony/var-dumper": "^6.2.3|^7.0" }, @@ -19628,7 +19554,7 @@ "type": "github" } ], - "time": "2024-02-09T16:08:40+00:00" + "time": "2024-03-29T14:14:55+00:00" }, { "name": "spaze/phpstan-stripe", diff --git a/config/ninja.php b/config/ninja.php index 7274d816ff8c..7eca111ec035 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -237,13 +237,6 @@ return [ 'webhook_id' => env('PAYPAL_WEBHOOK_ID', null), ], 'inbound_mailbox' => [ - 'imap' => [ - 'servers' => env('INBOUND_MAILBOX_IMAP_SERVERS', ''), - 'ports' => env('INBOUND_MAILBOX_IMAP_PORTS', ''), - 'users' => env('INBOUND_MAILBOX_IMAP_USERS', ''), - 'passwords' => env('INBOUND_MAILBOX_IMAP_PASSWORDS', ''), - 'companies' => env('INBOUND_MAILBOX_IMAP_COMPANIES', '1'), - ], 'inbound_mailbox_template' => env('INBOUND_MAILBOX_TEMPLATE', null), 'inbound_mailbox_endings' => env('INBOUND_MAILBOX_ENDINGS', '@expense.invoicing.co'), ], From a1338cbbab72a6cef52eac39cc755fb1341aae22 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 3 Apr 2024 07:57:23 +0200 Subject: [PATCH 054/119] fixes for postmark + inbound engine rework --- app/Http/Controllers/MailgunController.php | 4 +- app/Http/Controllers/PostMarkController.php | 50 ++- app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 74 +++-- .../Mailgun/ProcessMailgunInboundWebhook.php | 154 ++++----- .../ProcessPostmarkInboundWebhook.php | 299 ------------------ .../InboundMail/InboundMailEngine.php | 176 +++++------ app/Utils/TempFile.php | 6 +- 7 files changed, 257 insertions(+), 506 deletions(-) delete mode 100644 app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 8554f96b6feb..d097f61ed6ac 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -115,7 +115,7 @@ class MailgunController extends BaseController { $input = $request->all(); - if (!array_key_exists('recipient', $input) || !array_key_exists('message-url', $input)) { + if (!array_key_exists('sender', $input) || !array_key_exists('recipient', $input) || !array_key_exists('message-url', $input)) { Log::info('Failed: Message could not be parsed, because required parameters are missing. Please ensure contacting this api-endpoint with a store & notify operation instead of a forward operation!'); return response()->json(['message' => 'Failed. Missing Parameters. Use store and notify!'], 400); } @@ -123,7 +123,7 @@ class MailgunController extends BaseController // @turbo124 TODO: how to check for services.mailgun.webhook_signing_key on company level, when custom credentials are defined // TODO: validation for client mail credentials by recipient if (\hash_equals(\hash_hmac('sha256', $input['timestamp'] . $input['token'], config('services.mailgun.webhook_signing_key')), $input['signature'])) { - ProcessMailgunInboundWebhook::dispatch($input["recipient"] . "|" . $input["message-url"])->delay(10); + ProcessMailgunInboundWebhook::dispatch($input["sender"] . "|" . $input["recipient"] . "|" . $input["message-url"])->delay(10); return response()->json(['message' => 'Success'], 201); } diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index e0df719557c5..5fa1049ca607 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -13,6 +13,10 @@ namespace App\Http\Controllers; use App\Jobs\PostMark\ProcessPostmarkInboundWebhook; use App\Jobs\PostMark\ProcessPostmarkWebhook; +use App\Services\InboundMail\InboundMail; +use App\Services\InboundMail\InboundMailEngine; +use App\Utils\TempFile; +use Carbon\Carbon; use Illuminate\Http\Request; use Log; @@ -254,6 +258,13 @@ class PostMarkController extends BaseController * ), * 'Attachments' => * array ( + * array ( + * 'Content' => "base64-String", + * 'ContentLength' => 60164, + * 'Name' => 'Unbenannt.png', + * 'ContentType' => 'image/png', + * 'ContentID' => 'ii_luh2h8lg0', + * ) * ), * ) */ @@ -261,20 +272,47 @@ class PostMarkController extends BaseController { Log::info($request->all()); + Log::info($request->headers); $input = $request->all(); - if (!(array_key_exists("MessageStream", $input) && $input["MessageStream"] != "inbound") || !array_key_exists("To", $input) || !array_key_exists("MessageID", $input)) { - Log::info('Failed: Message could not be parsed, because required parameters are missing. Please ensure contacting this api-endpoint with a store & notify operation instead of a forward operation!'); + if (!(array_key_exists("MessageStream", $input) && $input["MessageStream"] == "inbound") || !array_key_exists("To", $input) || !array_key_exists("From", $input) || !array_key_exists("MessageID", $input)) { + Log::info('Failed: Message could not be parsed, because required parameters are missing.'); return response()->json(['message' => 'Failed. Missing/Invalid Parameters.'], 400); } - if ($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.postmark.token')) { - ProcessPostmarkInboundWebhook::dispatch($input["To"] . "|" . $input["MessageID"])->delay(10); + // // TODO: security + // if (!($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.postmark.token'))) + // return response()->json(['message' => 'Unauthorized'], 403); - return response()->json(['message' => 'Success'], 200); + + try { // important to save meta if something fails here to prevent spam + + // prepare data for ingresEngine + $inboundMail = new InboundMail(); + + $inboundMail->from = $input["From"]; + $inboundMail->to = $input["To"]; // usage of data-input, because we need a single email here + $inboundMail->subject = $input["Subject"]; + $inboundMail->body = $input["HtmlBody"]; + $inboundMail->text_body = $input["TextBody"]; + $inboundMail->date = Carbon::createFromTimeString($input["Date"]); + + // parse documents as UploadedFile from webhook-data + foreach ($input["Attachments"] as $attachment) { + + $inboundMail->documents[] = TempFile::UploadedFileFromBase64($attachment["Content"], $attachment["Name"], $attachment["ContentType"]); + + } + + } catch (\Exception $e) { + (new InboundMailEngine())->saveMeta($input["From"], $input["To"]); // important to save this, to protect from spam + throw $e; } - return response()->json(['message' => 'Unauthorized'], 403); + // perform + (new InboundMailEngine())->handle($inboundMail); + + return response()->json(['message' => 'Success'], 200); } } diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index 774018070fa8..706b76e8fdaa 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -125,57 +125,65 @@ class ProcessBrevoInboundWebhook implements ShouldQueue $company = MultiDB::findAndSetDbByInboundMailbox($recipient); if (!$company) { Log::info('[ProcessBrevoInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from brevo: ' . $recipient); + (new InboundMailEngine())->saveMeta($this->input["From"]["Address"], $recipient); // important to save this, to protect from spam continue; } - $company_brevo_secret = $company->settings?->email_sending_method === 'client_brevo' && $company->settings?->brevo_secret ? $company->settings?->brevo_secret : null; - if (empty ($company_brevo_secret) && empty (config('services.brevo.secret'))) - throw new \Error("[ProcessBrevoInboundWebhook] no brevo credenitals found, we cannot get the attachement"); + try { // important to save meta if something fails here to prevent spam - // prepare data for ingresEngine - $inboundMail = new InboundMail(); + $company_brevo_secret = $company->settings?->email_sending_method === 'client_brevo' && $company->settings?->brevo_secret ? $company->settings?->brevo_secret : null; + if (empty($company_brevo_secret) && empty(config('services.brevo.secret'))) + throw new \Error("[ProcessBrevoInboundWebhook] no brevo credenitals found, we cannot get the attachement"); - $inboundMail->from = $this->input["From"]["Address"]; - $inboundMail->to = $recipient; - $inboundMail->subject = $this->input["Subject"]; - $inboundMail->body = $this->input["RawHtmlBody"]; - $inboundMail->text_body = $this->input["RawTextBody"]; - $inboundMail->date = Carbon::createFromTimeString($this->input["SentAtDate"]); + // prepare data for ingresEngine + $inboundMail = new InboundMail(); - // parse documents as UploadedFile from webhook-data - foreach ($this->input["Attachments"] as $attachment) { + $inboundMail->from = $this->input["From"]["Address"]; + $inboundMail->to = $recipient; + $inboundMail->subject = $this->input["Subject"]; + $inboundMail->body = $this->input["RawHtmlBody"]; + $inboundMail->text_body = $this->input["RawTextBody"]; + $inboundMail->date = Carbon::createFromTimeString($this->input["SentAtDate"]); - // download file and save to tmp dir - if (!empty ($company_brevo_secret)) { + // parse documents as UploadedFile from webhook-data + foreach ($this->input["Attachments"] as $attachment) { - $attachment = null; - try { + // download file and save to tmp dir + if (!empty($company_brevo_secret)) { - $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", $company_brevo_secret)); - $attachment = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); + $attachment = null; + try { - } catch (\Error $e) { - if (config('services.brevo.secret')) { - Log::info("[ProcessBrevoInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); - - $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); + $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", $company_brevo_secret)); $attachment = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); - } else - throw $e; + } catch (\Error $e) { + if (config('services.brevo.secret')) { + Log::info("[ProcessBrevoInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); + + $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); + $attachment = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); + + } else + throw $e; + } + $inboundMail->documents[] = TempFile::UploadedFileFromRaw($attachment, $attachment["Name"], $attachment["ContentType"]); + + } else { + + $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); + $inboundMail->documents[] = TempFile::UploadedFileFromRaw($brevo->getInboundEmailAttachment($attachment["DownloadToken"]), $attachment["Name"], $attachment["ContentType"]); + } - $inboundMail->documents[] = TempFile::UploadedFileFromRaw($attachment, $attachment["Name"], $attachment["ContentType"]); - - } else { - - $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); - $inboundMail->documents[] = TempFile::UploadedFileFromRaw($brevo->getInboundEmailAttachment($attachment["DownloadToken"]), $attachment["Name"], $attachment["ContentType"]); } + } catch (\Exception $e) { + (new InboundMailEngine())->saveMeta($this->input["From"]["Address"], $recipient); // important to save this, to protect from spam + throw $e; } - (new InboundMailEngine($inboundMail))->handle(); + (new InboundMailEngine())->handle($inboundMail); } } diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index 4f11b6ba0d2b..a108b5f29c40 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -31,7 +31,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue /** * Create a new job instance. - * $input consists of 2 informations: recipient|messageUrl + * $input consists of 3 informations: sender/from|recipient/to|messageUrl */ public function __construct(private string $input) { @@ -163,88 +163,45 @@ class ProcessMailgunInboundWebhook implements ShouldQueue */ public function handle() { - $recipient = explode("|", $this->input)[0]; + $from = explode("|", $this->input)[0]; + $to = explode("|", $this->input)[1]; + // $messageId = explode("|", $this->input)[2]; // used as base in download function // match company - $company = MultiDB::findAndSetDbByInboundMailbox($recipient); + $company = MultiDB::findAndSetDbByInboundMailbox($to); if (!$company) { - Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $recipient); + Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $to); + (new InboundMailEngine())->saveMeta($from, $to); // important to save this, to protect from spam return; } - // fetch message from mailgun-api - $company_mailgun_domain = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_domain ? $company->settings?->mailgun_domain : null; - $company_mailgun_secret = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_secret ? $company->settings?->mailgun_secret : null; - if (!($company_mailgun_domain && $company_mailgun_secret) && !(config('services.mailgun.domain') && config('services.mailgun.secret'))) - throw new \Error("[ProcessMailgunInboundWebhook] no mailgun credenitals found, we cannot get the attachements and files"); + try { // important to save meta if something fails here to prevent spam - $mail = null; - if ($company_mailgun_domain && $company_mailgun_secret) { + // fetch message from mailgun-api + $company_mailgun_domain = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_domain ? $company->settings?->mailgun_domain : null; + $company_mailgun_secret = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_secret ? $company->settings?->mailgun_secret : null; + if (!($company_mailgun_domain && $company_mailgun_secret) && !(config('services.mailgun.domain') && config('services.mailgun.secret'))) + throw new \Error("[ProcessMailgunInboundWebhook] no mailgun credenitals found, we cannot get the attachements and files"); - $credentials = $company_mailgun_domain . ":" . $company_mailgun_secret . "@"; - $messageUrl = explode("|", $this->input)[1]; - $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); - $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); - - try { - $mail = json_decode(file_get_contents($messageUrl)); - } catch (\Error $e) { - if (config('services.mailgun.secret')) { - Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); - - $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; - $messageUrl = explode("|", $this->input)[1]; - $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); - $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); - $mail = json_decode(file_get_contents($messageUrl)); - - } else - throw $e; - } - - } else { - - $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; - $messageUrl = explode("|", $this->input)[1]; - $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); - $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); - $mail = json_decode(file_get_contents($messageUrl)); - - } - - // prepare data for ingresEngine - $inboundMail = new InboundMail(); - - $inboundMail->from = $mail->sender; - $inboundMail->to = $recipient; // usage of data-input, because we need a single email here - $inboundMail->subject = $mail->Subject; - $inboundMail->body = $mail->{"body-html"}; - $inboundMail->text_body = $mail->{"body-plain"}; - $inboundMail->date = Carbon::createFromTimeString($mail->Date); - - // parse documents as UploadedFile from webhook-data - foreach ($mail->attachments as $attachment) { // prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 - - // download file and save to tmp dir + $mail = null; if ($company_mailgun_domain && $company_mailgun_secret) { + $credentials = $company_mailgun_domain . ":" . $company_mailgun_secret . "@"; + $messageUrl = explode("|", $this->input)[2]; + $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); + $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); + try { - - $credentials = $company_mailgun_domain . ":" . $company_mailgun_secret . "@"; - $url = $attachment->url; - $url = str_replace("http://", "http://" . $credentials, $url); - $url = str_replace("https://", "https://" . $credentials, $url); - $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); - + $mail = json_decode(file_get_contents($messageUrl)); } catch (\Error $e) { if (config('services.mailgun.secret')) { Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; - $url = $attachment->url; - $url = str_replace("http://", "http://" . $credentials, $url); - $url = str_replace("https://", "https://" . $credentials, $url); - $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $messageUrl = explode("|", $this->input)[2]; + $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); + $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); + $mail = json_decode(file_get_contents($messageUrl)); } else throw $e; @@ -253,16 +210,69 @@ class ProcessMailgunInboundWebhook implements ShouldQueue } else { $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; - $url = $attachment->url; - $url = str_replace("http://", "http://" . $credentials, $url); - $url = str_replace("https://", "https://" . $credentials, $url); - $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + $messageUrl = explode("|", $this->input)[2]; + $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); + $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); + $mail = json_decode(file_get_contents($messageUrl)); } + // prepare data for ingresEngine + $inboundMail = new InboundMail(); + + $inboundMail->from = $from; + $inboundMail->to = $to; // usage of data-input, because we need a single email here + $inboundMail->subject = $mail->Subject; + $inboundMail->body = $mail->{"body-html"}; + $inboundMail->text_body = $mail->{"body-plain"}; + $inboundMail->date = Carbon::createFromTimeString($mail->Date); + + // parse documents as UploadedFile from webhook-data + foreach ($mail->attachments as $attachment) { // prepare url with credentials before downloading :: https://github.com/mailgun/mailgun.js/issues/24 + + // download file and save to tmp dir + if ($company_mailgun_domain && $company_mailgun_secret) { + + try { + + $credentials = $company_mailgun_domain . ":" . $company_mailgun_secret . "@"; + $url = $attachment->url; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + + } catch (\Error $e) { + if (config('services.mailgun.secret')) { + Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); + + $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; + $url = $attachment->url; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + + } else + throw $e; + } + + } else { + + $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; + $url = $attachment->url; + $url = str_replace("http://", "http://" . $credentials, $url); + $url = str_replace("https://", "https://" . $credentials, $url); + $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); + + } + + } + + } catch (\Exception $e) { + (new InboundMailEngine())->saveMeta($from, $to); // important to save this, to protect from spam + throw $e; } // perform - (new InboundMailEngine($inboundMail))->handle(); + (new InboundMailEngine())->handle($inboundMail); } } diff --git a/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php b/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php deleted file mode 100644 index 5904b7c5b664..000000000000 --- a/app/Jobs/PostMark/ProcessPostmarkInboundWebhook.php +++ /dev/null @@ -1,299 +0,0 @@ - 'Max Mustermann', - * 'MessageStream' => 'inbound', - * 'From' => 'max@mustermann.de', - * 'FromFull' => - * array ( - * 'Email' => 'max@mustermann.de', - * 'Name' => 'Max Mustermann', - * 'MailboxHash' => NULL, - * ), - * 'To' => '370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com', - * 'ToFull' => - * array ( - * 0 => - * array ( - * 'Email' => '370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com', - * 'Name' => NULL, - * 'MailboxHash' => NULL, - * ), - * ), - * 'Cc' => NULL, - * 'CcFull' => - * array ( - * ), - * 'Bcc' => NULL, - * 'BccFull' => - * array ( - * ), - * 'OriginalRecipient' => '370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com', - * 'Subject' => 'Re: adaw', - * 'MessageID' => 'd37fde00-b4cf-4b64-ac64-e9f6da523c25', - * 'ReplyTo' => NULL, - * 'MailboxHash' => NULL, - * 'Date' => 'Sun, 24 Mar 2024 13:17:52 +0100', - * 'TextBody' => 'wadwad - * - * Am So., 24. März 2024 um 13:17 Uhr schrieb Max Mustermann : - * - * > test - * > - * - * -- - * test.de - Max Mustermann kontakt@test.de - * ', - * 'HtmlBody' => '

wadwad

Am So., 24. März 2024 um 13:17 Uhr schrieb Max Mustermann <max@mustermann.de>:
test
- *
- * - *
- * test.de - Max Mustermann', - * 'StrippedTextReply' => 'wadwad - * - * Am So., 24. März 2024 um 13:17 Uhr schrieb Max Mustermann :', - * 'Tag' => NULL, - * 'Headers' => - * array ( - * 0 => - * array ( - * 'Name' => 'Return-Path', - * 'Value' => '', - * ), - * 1 => - * array ( - * 'Name' => 'Received', - * 'Value' => 'by p-pm-inboundg02a-aws-euwest1a.inbound.postmarkapp.com (Postfix, from userid 996) id 8ED1A453CA4; Sun, 24 Mar 2024 12:18:10 +0000 (UTC)', - * ), - * 2 => - * array ( - * 'Name' => 'X-Spam-Checker-Version', - * 'Value' => 'SpamAssassin 3.4.0 (2014-02-07) on p-pm-inboundg02a-aws-euwest1a', - * ), - * 3 => - * array ( - * 'Name' => 'X-Spam-Status', - * 'Value' => 'No', - * ), - * 4 => - * array ( - * 'Name' => 'X-Spam-Score', - * 'Value' => '-0.1', - * ), - * 5 => - * array ( - * 'Name' => 'X-Spam-Tests', - * 'Value' => 'DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,HTML_MESSAGE, RCVD_IN_DNSWL_NONE,RCVD_IN_MSPIKE_H2,RCVD_IN_ZEN_BLOCKED_OPENDNS, SPF_HELO_NONE,SPF_PASS,URIBL_DBL_BLOCKED_OPENDNS,URIBL_ZEN_BLOCKED_OPENDNS', - * ), - * 6 => - * array ( - * 'Name' => 'Received-SPF', - * 'Value' => 'pass (test.de: Sender is authorized to use \'max@mustermann.de\' in \'mfrom\' identity (mechanism \'include:_spf.google.com\' matched)) receiver=p-pm-inboundg02a-aws-euwest1a; identity=mailfrom; envelope-from="max@mustermann.de"; helo=mail-lf1-f51.google.com; client-ip=209.85.167.51', - * ), - * 7 => - * array ( - * 'Name' => 'Received', - * 'Value' => 'from mail-lf1-f51.google.com (mail-lf1-f51.google.com [209.85.167.51]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by p-pm-inboundg02a-aws-euwest1a.inbound.postmarkapp.com (Postfix) with ESMTPS id 437BD453CA2 for <370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com>; Sun, 24 Mar 2024 12:18:10 +0000 (UTC)', - * ), - * 8 => - * array ( - * 'Name' => 'Received', - * 'Value' => 'by mail-lf1-f51.google.com with SMTP id 2adb3069b0e04-513cf9bacf1so4773866e87.0 for <370c69ad9e41d616fc914b3c60250224@inbound.postmarkapp.com>; Sun, 24 Mar 2024 05:18:10 -0700 (PDT)', - * ), - * 9 => - * array ( - * 'Name' => 'DKIM-Signature', - * 'Value' => 'v=1; a=rsa-sha256; c=relaxed/relaxed; d=test.de; s=google; t=1711282689; x=1711887489; darn=inbound.postmarkapp.com; h=to:subject:message-id:date:from:in-reply-to:references:mime-version :from:to:cc:subject:date:message-id:reply-to; bh=NvjmqLXF/5L5ZrpToR/6FgVOhTOGC9j0/B2Na5Ke6J8=; b=AMXIEoh6yGrOT6X3eBBClQ3NXFNuEoqxeM6aPONsqbpShAcT24iAJmqXylaLHv3fyX Hm6mwp3a029NnrLP/VRyKZbzIMBN2iycidtrEMXF/Eg2e42Q/08/2dZ7nxH6NqE/jz01 3M7qvwHvuoZ2Knhj7rnZc6I5m/nFxBsZc++Aj0Vv9sFoWZZooqAeTXbux1I5NyE17MrL D6byca43iINARZN7XOkoChRRZoZbOqZEtc2Va5yw7v+aYguLB4HHrIFC7G+L8hAJ0IAo 3R3DFeBw58M1xtxXCREI8Y6qMQTw60XyFw0gVmZzqR4hZiTerBSJJsZLZOBgmXxq3WLS +xVQ==', - * ), - * 10 => - * array ( - * 'Name' => 'X-Google-DKIM-Signature', - * 'Value' => 'v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1711282689; x=1711887489; h=to:subject:message-id:date:from:in-reply-to:references:mime-version :x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=NvjmqLXF/5L5ZrpToR/6FgVOhTOGC9j0/B2Na5Ke6J8=; b=uKoMhir+MX/wycNEr29Sffj45ooKksCJ1OfSRkIIGHk0rnHn8Vh+c7beYipwRPW4F2 h46K64vtIX00guYMdL2Qo2eY96+wALTqHCy67PGhvotVTROz21yxjx62pCDPGs5tefOu IkyxoybpIK8zAfLoDTd9p2GIrr5brKJyB2w1NQc1htxTQ5D4RgBxUAOKv4uVEr8r47iA MIo5d8/AifA+vCOAh7iJ7EmvDQ1R+guhQyH9m1Jo8PLapiYuHXggpBJvooyGuflKqbnt gJ/dscEr4d5aWJbw/x1dmIJ5gyJPGdBWq8NRqV/qbkXQW3H/gylifDUPXbki+EQBD5Yu EuLQ==', - * ), - * 11 => - * array ( - * 'Name' => 'X-Gm-Message-State', - * 'Value' => 'AOJu0Yxpbp1sRh17lNzg+pLnIx1jCn8ZFJQMgFuHK+6Z8RqFS5KKKTxR 8onpEbxWYYVUbrJFExNBHPD/3jdxqifCVVNaDmbpwHgmW5lHLJmA5vYRq5NFZ9OA6zKx/N6Gipr iXE4fXmSqghFNTzy9V/RT08Zp+F5RiFh/Ta6ltQl8XfCPFfSawLz6cagUgt8bBuF4RqdrYmWwzj ty86V5Br1htRNEFYivoXnNmaRcsD0tca1D23ny62O6RwWugrj1IpAYhViNyTZAWu+loKgfjJJoI MsyiSU=', - * ), - * 12 => - * array ( - * 'Name' => 'X-Google-Smtp-Source', - * 'Value' => 'AGHT+IEdtZqbVI6j7WLeaSL3dABGSnWIXaSjbYqXvFvE2H+f2zsn0gknQ4OdTJecQRCabpypVF2ue91Jb7aKl6RiyEQ=', - * ), - * 13 => - * array ( - * 'Name' => 'X-Received', - * 'Value' => 'by 2002:a19:385a:0:b0:513:c876:c80a with SMTP id d26-20020a19385a000000b00513c876c80amr2586776lfj.34.1711282689140; Sun, 24 Mar 2024 05:18:09 -0700 (PDT)', - * ), - * 14 => - * array ( - * 'Name' => 'MIME-Version', - * 'Value' => '1.0', - * ), - * 15 => - * array ( - * 'Name' => 'References', - * 'Value' => '', - * ), - * 16 => - * array ( - * 'Name' => 'In-Reply-To', - * 'Value' => '', - * ), - * 17 => - * array ( - * 'Name' => 'Message-ID', - * 'Value' => '', - * ), - * ), - * 'Attachments' => - * array ( - * ), - * ) - * @return void - */ - public function handle() - { - $recipient = explode("|", $this->input)[0]; - - // match company - $company = MultiDB::findAndSetDbByInboundMailbox($recipient); - if (!$company) { - Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $recipient); - return; - } - - // fetch message from postmark-api - $company_postmark_secret = $company->settings?->email_sending_method === 'client_postmark' && $company->settings?->postmark_secret ? $company->settings?->postmark_secret : null; - if (!($company_postmark_secret) && !(config('services.postmark.domain') && config('services.postmark.secret'))) - throw new \Error("[ProcessMailgunInboundWebhook] no postmark credenitals found, we cannot get the attachements and files"); - - $mail = null; - if ($company_postmark_secret) { - - $credentials = $company_postmark_domain . ":" . $company_postmark_secret . "@"; - $messageUrl = explode("|", $this->input)[1]; - $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); - $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); - - try { - $mail = json_decode(file_get_contents($messageUrl)); - } catch (\Error $e) { - if (config('services.postmark.secret')) { - Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); - - $credentials = config('services.postmark.domain') . ":" . config('services.postmark.secret') . "@"; - $messageUrl = explode("|", $this->input)[1]; - $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); - $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); - $mail = json_decode(file_get_contents($messageUrl)); - - } else - throw $e; - } - - } else { - - $credentials = config('services.postmark.domain') . ":" . config('services.postmark.secret') . "@"; - $messageUrl = explode("|", $this->input)[1]; - $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); - $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); - $mail = json_decode(file_get_contents($messageUrl)); - - } - - // prepare data for ingresEngine - $inboundMail = new InboundMail(); - - $inboundMail->from = $mail->sender; - $inboundMail->to = $recipient; // usage of data-input, because we need a single email here - $inboundMail->subject = $mail->Subject; - $inboundMail->body = $mail->{"body-html"}; - $inboundMail->text_body = $mail->{"body-plain"}; - $inboundMail->date = Carbon::createFromTimeString($mail->Date); - - // parse documents as UploadedFile from webhook-data - foreach ($mail->attachments as $attachment) { // prepare url with credentials before downloading :: https://github.com/postmark/postmark.js/issues/24 - - // download file and save to tmp dir - if ($company_postmark_domain && $company_postmark_secret) { - - try { - - $credentials = $company_postmark_domain . ":" . $company_postmark_secret . "@"; - $url = $attachment->url; - $url = str_replace("http://", "http://" . $credentials, $url); - $url = str_replace("https://", "https://" . $credentials, $url); - $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); - - } catch (\Error $e) { - if (config('services.postmark.secret')) { - Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); - - $credentials = config('services.postmark.domain') . ":" . config('services.postmark.secret') . "@"; - $url = $attachment->url; - $url = str_replace("http://", "http://" . $credentials, $url); - $url = str_replace("https://", "https://" . $credentials, $url); - $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); - - } else - throw $e; - } - - } else { - - $credentials = config('services.postmark.domain') . ":" . config('services.postmark.secret') . "@"; - $url = $attachment->url; - $url = str_replace("http://", "http://" . $credentials, $url); - $url = str_replace("https://", "https://" . $credentials, $url); - $inboundMail->documents[] = TempFile::UploadedFileFromUrl($url, $attachment->name, $attachment->{"content-type"}); - - } - - } - - // perform - (new InboundMailEngine($inboundMail))->handle(); - } -} diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index 1c6b2f6f0fcd..0383fbf80d3b 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -15,7 +15,6 @@ use App\Events\Expense\ExpenseWasCreated; use App\Factory\ExpenseFactory; use App\Jobs\Util\SystemLogger; use App\Libraries\MultiDB; -use App\Models\Client; use App\Models\ClientContact; use App\Models\Company; use App\Models\SystemLog; @@ -40,154 +39,146 @@ class InboundMailEngine private ?bool $isUnknownRecipent = null; private array $globalBlacklistDomains = []; private array $globalBlacklistSenders = []; - public function __construct(private InboundMail $email) + public function __construct() { } /** * if there is not a company with an matching mailbox, we only do monitoring * reuse this method to add more mail-parsing behaviors */ - public function handle() + public function handle(InboundMail $email) { - if ($this->isInvalidOrBlocked()) + if ($this->isInvalidOrBlocked($email)) return; - $this->isUnknownRecipent = true; + $isUnknownRecipent = true; // Expense Mailbox => will create an expense - $this->company = MultiDB::findAndSetDbByInboundMailbox($this->email->to); - if ($this->company) { - $this->isUnknownRecipent = false; - $this->createExpense(); + $company = MultiDB::findAndSetDbByInboundMailbox($email->to); + if ($company) { + $isUnknownRecipent = false; + $this->createExpense($company, $email); } - $this->saveMeta(); + $this->saveMeta($email->from, $email->to, $isUnknownRecipent); } // SPAM Protection - private function isInvalidOrBlocked() + private function isInvalidOrBlocked(InboundMail $email) { // invalid email - if (!filter_var($this->email->from, FILTER_VALIDATE_EMAIL)) { - $this->logBlocked('E-Mail blocked, because from e-mail has the wrong format: ' . $this->email->from); + if (!filter_var($email->from, FILTER_VALIDATE_EMAIL)) { + Log::info('E-Mail blocked, because from e-mail has the wrong format: ' . $email->from); return true; } - $parts = explode('@', $this->email->from); + $parts = explode('@', $email->from); $domain = array_pop($parts); // global blacklist if (in_array($domain, $this->globalBlacklistDomains)) { - $this->logBlocked('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $this->email->from); + Log::info('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $email->from); return true; } - if (in_array($this->email->from, $this->globalBlacklistSenders)) { - $this->logBlocked('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $this->email->from); + if (in_array($email->from, $this->globalBlacklistSenders)) { + Log::info('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $email->from); return true; } - if (Cache::has('inboundMailBlockedSender:' . $this->email->from)) { // was marked as blocked before, so we block without any console output + if (Cache::has('inboundMailBlockedSender:' . $email->from)) { // was marked as blocked before, so we block without any console output return true; } // sender occured in more than 500 emails in the last 12 hours - $senderMailCountTotal = Cache::get('inboundMailSender:' . $this->email->from, 0); + $senderMailCountTotal = Cache::get('inboundMailSender:' . $email->from, 0); if ($senderMailCountTotal >= 5000) { - $this->logBlocked('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); - $this->blockSender(); + Log::info('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $email->from); + $this->blockSender($email->from); + $this->saveMeta($email->from, $email->to); return true; } if ($senderMailCountTotal >= 1000) { - $this->logBlocked('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $this->email->from); - $this->saveMeta(); + Log::info('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $email->from); + $this->saveMeta($email->from, $email->to); return true; } // sender sended more than 50 emails to the wrong mailbox in the last 6 hours - $senderMailCountUnknownRecipent = Cache::get('inboundMailSenderUnknownRecipent:' . $this->email->from, 0); + $senderMailCountUnknownRecipent = Cache::get('inboundMailSenderUnknownRecipent:' . $email->from, 0); if ($senderMailCountUnknownRecipent >= 50) { - $this->logBlocked('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $this->email->from); - $this->saveMeta(); + Log::info('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $email->from); + $this->saveMeta($email->from, $email->to); return true; } // wrong recipent occurs in more than 100 emails in the last 12 hours, so the processing is blocked - $mailCountUnknownRecipent = Cache::get('inboundMailUnknownRecipent:' . $this->email->to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time + $mailCountUnknownRecipent = Cache::get('inboundMailUnknownRecipent:' . $email->to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time if ($mailCountUnknownRecipent >= 100) { - $this->logBlocked('E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $this->email->from); - $this->blockSender(); + Log::info('E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $email->from); + $this->blockSender($email->from); + $this->saveMeta($email->from, $email->to); return true; } return false; } - private function blockSender() + public function blockSender(string $from) { - Cache::add('inboundMailBlockedSender:' . $this->email->from, true, now()->addHours(12)); - $this->saveMeta(); + Cache::add('inboundMailBlockedSender:' . $from, true, now()->addHours(12)); // TODO: ignore, when known sender (for heavy email-usage mostly on isHosted()) // TODO: handle external blocking } - private function saveMeta() + public function saveMeta(string $from, string $to, bool $isUnknownRecipent = false) { // save cache - Cache::add('inboundMailSender:' . $this->email->from, 0, now()->addHours(12)); - Cache::increment('inboundMailSender:' . $this->email->from); + Cache::add('inboundMailSender:' . $from, 0, now()->addHours(12)); + Cache::increment('inboundMailSender:' . $from); - if ($this->isUnknownRecipent) { - Cache::add('inboundMailSenderUnknownRecipent:' . $this->email->from, 0, now()->addHours(6)); - Cache::increment('inboundMailSenderUnknownRecipent:' . $this->email->from); // we save the sender, to may block him + if ($isUnknownRecipent) { + Cache::add('inboundMailSenderUnknownRecipent:' . $from, 0, now()->addHours(6)); + Cache::increment('inboundMailSenderUnknownRecipent:' . $from); // we save the sender, to may block him - Cache::add('inboundMailUnknownRecipent:' . $this->email->to, 0, now()->addHours(12)); - Cache::increment('inboundMailUnknownRecipent:' . $this->email->to); // we save the sender, to may block him + Cache::add('inboundMailUnknownRecipent:' . $to, 0, now()->addHours(12)); + Cache::increment('inboundMailUnknownRecipent:' . $to); // we save the sender, to may block him } } - // MAIL-PARSING - private function processHtmlBodyToDocument() - { - - if ($this->email->body !== null) - $this->email->body_document = TempFile::UploadedFileFromRaw($this->email->body, "E-Mail.html", "text/html"); - - } - // MAIN-PROCESSORS - protected function createExpense() + protected function createExpense(Company $company, InboundMail $email) { // Skipping executions: will not result in not saving Metadata to prevent usage of these conditions, to spam - if (!$this->validateExpenseShouldProcess()) { - $this->logBlocked('mailbox not active for this company. from: ' . $this->email->from); + if (!($company?->inbound_mailbox_active ?: false)) { + $this->logBlocked($company, 'mailbox not active for this company. from: ' . $email->from); return; } - if (!$this->validateExpenseSender()) { - $this->logBlocked('invalid sender of an ingest email for this company. from: ' . $this->email->from); + if (!$this->validateExpenseSender($company, $email)) { + $this->logBlocked($company, 'invalid sender of an ingest email for this company. from: ' . $email->from); return; } - if (sizeOf($this->email->documents) == 0) { - $this->logBlocked('email does not contain any attachments and is likly not an expense. from: ' . $this->email->from); + if (sizeOf($email->documents) == 0) { + $this->logBlocked($company, 'email does not contain any attachments and is likly not an expense. from: ' . $email->from); return; } // create expense - $expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id); + $expense = ExpenseFactory::create($company->id, $company->owner()->id); - $expense->public_notes = $this->email->subject; - $expense->private_notes = $this->email->text_body; - $expense->date = $this->email->date; + $expense->public_notes = $email->subject; + $expense->private_notes = $email->text_body; + $expense->date = $email->date; // handle vendor assignment - $expense_vendor = $this->getVendor(); + $expense_vendor = $this->getVendor($company, $email); if ($expense_vendor) $expense->vendor_id = $expense_vendor->id; // handle documents - $this->processHtmlBodyToDocument(); + $this->processHtmlBodyToDocument($email); $documents = []; - array_push($documents, ...$this->email->documents); - if ($this->email->body_document !== null) - array_push($documents, $this->email->body_document); + array_push($documents, ...$email->documents); + if ($email->body_document !== null) + array_push($documents, $email->body_document); $expense->saveQuietly(); @@ -198,78 +189,81 @@ class InboundMailEngine } // HELPERS - private function validateExpenseShouldProcess() + private function processHtmlBodyToDocument(InboundMail $email) { - return $this->company?->inbound_mailbox_active ?: false; + + if ($email->body !== null) + $email->body_document = TempFile::UploadedFileFromRaw($email->body, "E-Mail.html", "text/html"); + } - private function validateExpenseSender() + private function validateExpenseSender(Company $company, InboundMail $email) { - $parts = explode('@', $this->email->from); + $parts = explode('@', $email->from); $domain = array_pop($parts); // whitelists - $email_whitelist = explode(",", $this->company->inbound_mailbox_whitelist_senders); - if (in_array($this->email->from, $email_whitelist)) + $email_whitelist = explode(",", $company->inbound_mailbox_whitelist_senders); + if (in_array($email->from, $email_whitelist)) return true; - $domain_whitelist = explode(",", $this->company->inbound_mailbox_whitelist_domains); + $domain_whitelist = explode(",", $company->inbound_mailbox_whitelist_domains); if (in_array($domain, $domain_whitelist)) return true; - $email_blacklist = explode(",", $this->company->inbound_mailbox_blacklist_senders); - if (in_array($this->email->from, $email_blacklist)) + $email_blacklist = explode(",", $company->inbound_mailbox_blacklist_senders); + if (in_array($email->from, $email_blacklist)) return false; - $domain_blacklist = explode(",", $this->company->inbound_mailbox_blacklist_domains); + $domain_blacklist = explode(",", $company->inbound_mailbox_blacklist_domains); if (in_array($domain, $domain_blacklist)) return false; // allow unknown - if ($this->company->inbound_mailbox_allow_unknown) + if ($company->inbound_mailbox_allow_unknown) return true; // own users - if ($this->company->inbound_mailbox_allow_company_users && $this->company->users()->where("email", $this->email->from)->exists()) + if ($company->inbound_mailbox_allow_company_users && $company->users()->where("email", $email->from)->exists()) return true; // from vendors - if ($this->company->inbound_mailbox_allow_vendors && $this->company->vendors()->where("invoicing_email", $this->email->from)->orWhere("invoicing_domain", $domain)->exists()) + if ($company->inbound_mailbox_allow_vendors && $company->vendors()->where("invoicing_email", $email->from)->orWhere("invoicing_domain", $domain)->exists()) return true; - if ($this->company->inbound_mailbox_allow_vendors && $this->company->vendors()->contacts()->where("email", $this->email->from)->exists()) + if ($company->inbound_mailbox_allow_vendors && $company->vendors()->contacts()->where("email", $email->from)->exists()) return true; // from clients - if ($this->company->inbound_mailbox_allow_clients && $this->company->clients()->contacts()->where("email", $this->email->from)->exists()) + if ($company->inbound_mailbox_allow_clients && $company->clients()->contacts()->where("email", $email->from)->exists()) return true; // denie return false; } - private function getClient() + private function getClient(Company $company, InboundMail $email) { - // $parts = explode('@', $this->email->from); + // $parts = explode('@', $email->from); // $domain = array_pop($parts); - $clientContact = ClientContact::where("company_id", $this->company->id)->where("email", $this->email->from)->first(); + $clientContact = ClientContact::where("company_id", $company->id)->where("email", $email->from)->first(); $client = $clientContact->client(); return $client; } - private function getVendor() + private function getVendor(Company $company, InboundMail $email) { - $parts = explode('@', $this->email->from); + $parts = explode('@', $email->from); $domain = array_pop($parts); - $vendor = Vendor::where("company_id", $this->company->id)->where('invoicing_email', $this->email->from)->first(); + $vendor = Vendor::where("company_id", $company->id)->where('invoicing_email', $email->from)->first(); if ($vendor == null) - $vendor = Vendor::where("company_id", $this->company->id)->where("invoicing_domain", $domain)->first(); + $vendor = Vendor::where("company_id", $company->id)->where("invoicing_domain", $domain)->first(); if ($vendor == null) { - $vendorContact = VendorContact::where("company_id", $this->company->id)->where("email", $this->email->from)->first(); + $vendorContact = VendorContact::where("company_id", $company->id)->where("email", $email->from)->first(); $vendor = $vendorContact->vendor(); } return $vendor; } - private function logBlocked(string $data) + private function logBlocked(Company $company, string $data) { - Log::info("[InboundMailEngine][company:" . $this->company->id . "] " . $data); + Log::info("[InboundMailEngine][company:" . $company->id . "] " . $data); ( new SystemLogger( @@ -278,7 +272,7 @@ class InboundMailEngine SystemLog::EVENT_INBOUND_MAIL_BLOCKED, SystemLog::TYPE_CUSTOM, null, - $this->company + $company ) )->handle(); } diff --git a/app/Utils/TempFile.php b/app/Utils/TempFile.php index 24df20da7271..5d88b911f175 100644 --- a/app/Utils/TempFile.php +++ b/app/Utils/TempFile.php @@ -40,7 +40,7 @@ class TempFile } /* create a tmp file from a base64 string: https://gist.github.com/waska14/8b3bcebfad1f86f7fcd3b82927576e38*/ - public static function UploadedFileFromBase64(string $base64File): UploadedFile + public static function UploadedFileFromBase64(string $base64File, string|null $fileName = null, string|null $mimeType = null): UploadedFile { // Get file data base64 string $fileData = base64_decode(Arr::last(explode(',', $base64File))); @@ -55,8 +55,8 @@ class TempFile $tempFileObject = new File($tempFilePath); $file = new UploadedFile( $tempFileObject->getPathname(), - $tempFileObject->getFilename(), - $tempFileObject->getMimeType(), + $fileName ?: $tempFileObject->getFilename(), + $mimeType ?: $tempFileObject->getMimeType(), 0, true // Mark it as test, since the file isn't from real HTTP POST. ); From 8e1dc42bfb5b07385447ff8b1cf5260a0688b82d Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 3 Apr 2024 08:06:39 +0200 Subject: [PATCH 055/119] prevent downloading actions, when sender is blocked --- app/Http/Controllers/PostMarkController.php | 7 ++- app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 6 +++ .../Mailgun/ProcessMailgunInboundWebhook.php | 6 +++ .../InboundMail/InboundMailEngine.php | 44 +++++++++---------- 4 files changed, 39 insertions(+), 24 deletions(-) diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index 5fa1049ca607..917c8ed44c84 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -11,12 +11,11 @@ namespace App\Http\Controllers; -use App\Jobs\PostMark\ProcessPostmarkInboundWebhook; use App\Jobs\PostMark\ProcessPostmarkWebhook; use App\Services\InboundMail\InboundMail; use App\Services\InboundMail\InboundMailEngine; use App\Utils\TempFile; -use Carbon\Carbon; +use Illuminate\Support\Carbon; use Illuminate\Http\Request; use Log; @@ -285,6 +284,10 @@ class PostMarkController extends BaseController // if (!($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.postmark.token'))) // return response()->json(['message' => 'Unauthorized'], 403); + if ((new InboundMailEngine())->isInvalidOrBlocked($input["From"], $input["To"])) { + Log::info('Failed: Sender is blocked: ' . $input["From"] . " Recipient: " . $input["To"]); + return response()->json(['message' => 'Blocked.'], 403); + } try { // important to save meta if something fails here to prevent spam diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index 706b76e8fdaa..957a7a9ae162 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -121,6 +121,12 @@ class ProcessBrevoInboundWebhook implements ShouldQueue // brevo defines recipients as array, we check all of them, to be sure foreach ($this->input["Recipients"] as $recipient) { + // Spam protection + if ((new InboundMailEngine())->isInvalidOrBlocked($this->input["From"]["Address"], $recipient)) { + Log::info('Failed: Sender is blocked: ' . $this->input["From"]["Address"] . " Recipient: " . $recipient); + throw new \Error('Sender is blocked'); + } + // match company $company = MultiDB::findAndSetDbByInboundMailbox($recipient); if (!$company) { diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index a108b5f29c40..ff0792c52e38 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -167,6 +167,12 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $to = explode("|", $this->input)[1]; // $messageId = explode("|", $this->input)[2]; // used as base in download function + // Spam protection + if ((new InboundMailEngine())->isInvalidOrBlocked($from, $to)) { + Log::info('Failed: Sender is blocked: ' . $from . " Recipient: " . $to); + throw new \Error('Sender is blocked'); + } + // match company $company = MultiDB::findAndSetDbByInboundMailbox($to); if (!$company) { diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index 0383fbf80d3b..b9f293d4f9f2 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -48,7 +48,7 @@ class InboundMailEngine */ public function handle(InboundMail $email) { - if ($this->isInvalidOrBlocked($email)) + if ($this->isInvalidOrBlocked($email->from, $email->to)) return; $isUnknownRecipent = true; @@ -64,59 +64,59 @@ class InboundMailEngine } // SPAM Protection - private function isInvalidOrBlocked(InboundMail $email) + public function isInvalidOrBlocked(string $from, string $to) { // invalid email - if (!filter_var($email->from, FILTER_VALIDATE_EMAIL)) { - Log::info('E-Mail blocked, because from e-mail has the wrong format: ' . $email->from); + if (!filter_var($from, FILTER_VALIDATE_EMAIL)) { + Log::info('E-Mail blocked, because from e-mail has the wrong format: ' . $from); return true; } - $parts = explode('@', $email->from); + $parts = explode('@', $from); $domain = array_pop($parts); // global blacklist if (in_array($domain, $this->globalBlacklistDomains)) { - Log::info('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $email->from); + Log::info('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $from); return true; } - if (in_array($email->from, $this->globalBlacklistSenders)) { - Log::info('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $email->from); + if (in_array($from, $this->globalBlacklistSenders)) { + Log::info('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $from); return true; } - if (Cache::has('inboundMailBlockedSender:' . $email->from)) { // was marked as blocked before, so we block without any console output + if (Cache::has('inboundMailBlockedSender:' . $from)) { // was marked as blocked before, so we block without any console output return true; } // sender occured in more than 500 emails in the last 12 hours - $senderMailCountTotal = Cache::get('inboundMailSender:' . $email->from, 0); + $senderMailCountTotal = Cache::get('inboundMailSender:' . $from, 0); if ($senderMailCountTotal >= 5000) { - Log::info('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $email->from); - $this->blockSender($email->from); - $this->saveMeta($email->from, $email->to); + Log::info('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $from); + $this->blockSender($from); + $this->saveMeta($from, $to); return true; } if ($senderMailCountTotal >= 1000) { - Log::info('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $email->from); - $this->saveMeta($email->from, $email->to); + Log::info('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $from); + $this->saveMeta($from, $to); return true; } // sender sended more than 50 emails to the wrong mailbox in the last 6 hours - $senderMailCountUnknownRecipent = Cache::get('inboundMailSenderUnknownRecipent:' . $email->from, 0); + $senderMailCountUnknownRecipent = Cache::get('inboundMailSenderUnknownRecipent:' . $from, 0); if ($senderMailCountUnknownRecipent >= 50) { - Log::info('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $email->from); - $this->saveMeta($email->from, $email->to); + Log::info('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $from); + $this->saveMeta($from, $to); return true; } // wrong recipent occurs in more than 100 emails in the last 12 hours, so the processing is blocked - $mailCountUnknownRecipent = Cache::get('inboundMailUnknownRecipent:' . $email->to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time + $mailCountUnknownRecipent = Cache::get('inboundMailUnknownRecipent:' . $to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time if ($mailCountUnknownRecipent >= 100) { - Log::info('E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $email->from); - $this->blockSender($email->from); - $this->saveMeta($email->from, $email->to); + Log::info('E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $from); + $this->blockSender($from); + $this->saveMeta($from, $to); return true; } From f0ba86b699d06c6545fc822e30935d3c34dcf6d6 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 3 Apr 2024 08:20:36 +0200 Subject: [PATCH 056/119] fixes --- app/Services/InboundMail/InboundMailEngine.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index b9f293d4f9f2..c45a7e62d8d8 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -226,11 +226,11 @@ class InboundMailEngine // from vendors if ($company->inbound_mailbox_allow_vendors && $company->vendors()->where("invoicing_email", $email->from)->orWhere("invoicing_domain", $domain)->exists()) return true; - if ($company->inbound_mailbox_allow_vendors && $company->vendors()->contacts()->where("email", $email->from)->exists()) + if ($company->inbound_mailbox_allow_vendors && VendorContact::where("company_id", $company->id)->where("email", $email->from)->exists()) return true; // from clients - if ($company->inbound_mailbox_allow_clients && $company->clients()->contacts()->where("email", $email->from)->exists()) + if ($company->inbound_mailbox_allow_clients && ClientContact::where("company_id", $company->id)->where("email", $email->from)->exists()) return true; // denie From f0d61a8261ca110a373432318aba9d19bba7007c Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 3 Apr 2024 08:33:40 +0200 Subject: [PATCH 057/119] remove vendor email colums --- app/Http/Controllers/BrevoController.php | 18 +++++++++--------- app/Http/Controllers/PostMarkController.php | 3 --- app/Services/InboundMail/InboundMailEngine.php | 17 ++--------------- ...2023_12_10_110951_inbound_mail_parsing.php} | 4 ---- 4 files changed, 11 insertions(+), 31 deletions(-) rename database/migrations/{2023_12_10_110951_create_imap_configuration_fields.php => 2023_12_10_110951_inbound_mail_parsing.php} (86%) diff --git a/app/Http/Controllers/BrevoController.php b/app/Http/Controllers/BrevoController.php index c8db34674645..be4c7e9de2bd 100644 --- a/app/Http/Controllers/BrevoController.php +++ b/app/Http/Controllers/BrevoController.php @@ -187,17 +187,17 @@ class BrevoController extends BaseController $input = $request->all(); // validation for client mail credentials by recipient - if ($request->has('company')) { - if (!($request->has('token'))) - return response()->json(['message' => 'Unauthorized'], 403); + // if ($request->has('company')) { + // if (!($request->has('token'))) + // return response()->json(['message' => 'Unauthorized'], 403); - $company = MultiDB::findAndSetDbByCompanyId($request->has('company')); - $company_brevo_secret = $company?->settings?->email_sending_method === 'client_brevo' && $company?->settings?->brevo_secret ? $company->settings->brevo_secret : null; - if (!$company || !$company_brevo_secret || $request->get('token') !== $company_brevo_secret) - return response()->json(['message' => 'Unauthorized'], 403); + // $company = MultiDB::findAndSetDbByCompanyId($request->has('company')); + // $company_brevo_secret = $company?->settings?->email_sending_method === 'client_brevo' && $company?->settings?->brevo_secret ? $company->settings->brevo_secret : null; + // if (!$company || !$company_brevo_secret || $request->get('token') !== $company_brevo_secret) + // return response()->json(['message' => 'Unauthorized'], 403); - } else if (!($request->has('token') && $request->get('token') == config('services.brevo.secret'))) - return response()->json(['message' => 'Unauthorized'], 403); + // } else if (!($request->has('token') && $request->get('token') == config('services.brevo.secret'))) + // return response()->json(['message' => 'Unauthorized'], 403); if (!array_key_exists('items', $input)) { Log::info('Failed: Message could not be parsed, because required parameters are missing.'); diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index 917c8ed44c84..e76782323c33 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -270,9 +270,6 @@ class PostMarkController extends BaseController public function inboundWebhook(Request $request) { - Log::info($request->all()); - Log::info($request->headers); - $input = $request->all(); if (!(array_key_exists("MessageStream", $input) && $input["MessageStream"] == "inbound") || !array_key_exists("To", $input) || !array_key_exists("From", $input) || !array_key_exists("MessageID", $input)) { diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index c45a7e62d8d8..14690a4bd5af 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -224,8 +224,6 @@ class InboundMailEngine return true; // from vendors - if ($company->inbound_mailbox_allow_vendors && $company->vendors()->where("invoicing_email", $email->from)->orWhere("invoicing_domain", $domain)->exists()) - return true; if ($company->inbound_mailbox_allow_vendors && VendorContact::where("company_id", $company->id)->where("email", $email->from)->exists()) return true; @@ -238,9 +236,6 @@ class InboundMailEngine } private function getClient(Company $company, InboundMail $email) { - // $parts = explode('@', $email->from); - // $domain = array_pop($parts); - $clientContact = ClientContact::where("company_id", $company->id)->where("email", $email->from)->first(); $client = $clientContact->client(); @@ -248,16 +243,8 @@ class InboundMailEngine } private function getVendor(Company $company, InboundMail $email) { - $parts = explode('@', $email->from); - $domain = array_pop($parts); - - $vendor = Vendor::where("company_id", $company->id)->where('invoicing_email', $email->from)->first(); - if ($vendor == null) - $vendor = Vendor::where("company_id", $company->id)->where("invoicing_domain", $domain)->first(); - if ($vendor == null) { - $vendorContact = VendorContact::where("company_id", $company->id)->where("email", $email->from)->first(); - $vendor = $vendorContact->vendor(); - } + $vendorContact = VendorContact::where("company_id", $company->id)->where("email", $email->from)->first(); + $vendor = $vendorContact->vendor(); return $vendor; } diff --git a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php b/database/migrations/2023_12_10_110951_inbound_mail_parsing.php similarity index 86% rename from database/migrations/2023_12_10_110951_create_imap_configuration_fields.php rename to database/migrations/2023_12_10_110951_inbound_mail_parsing.php index f1b1c21603d7..f750d07065b4 100644 --- a/database/migrations/2023_12_10_110951_create_imap_configuration_fields.php +++ b/database/migrations/2023_12_10_110951_inbound_mail_parsing.php @@ -23,10 +23,6 @@ return new class extends Migration { $table->text("inbound_mailbox_blacklist_domains")->nullable(); $table->text("inbound_mailbox_blacklist_senders")->nullable(); }); - Schema::table('vendors', function (Blueprint $table) { - $table->string("invoicing_email")->nullable(); - $table->string("invoicing_domain")->nullable(); - }); } /** From b7a29bb2c7855597f82201f0f16b2045a4cfa89b Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 3 Apr 2024 15:01:57 +0200 Subject: [PATCH 058/119] fix: getVendor / getClient --- app/Http/Controllers/BrevoController.php | 23 +++++++++++-------- app/Http/Controllers/MailgunController.php | 4 ++++ app/Http/Controllers/PostMarkController.php | 3 +++ app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 5 ++-- .../Mailgun/ProcessMailgunInboundWebhook.php | 2 ++ .../InboundMail/InboundMailEngine.php | 10 ++++---- 6 files changed, 32 insertions(+), 15 deletions(-) diff --git a/app/Http/Controllers/BrevoController.php b/app/Http/Controllers/BrevoController.php index be4c7e9de2bd..8cb666d24410 100644 --- a/app/Http/Controllers/BrevoController.php +++ b/app/Http/Controllers/BrevoController.php @@ -14,6 +14,7 @@ namespace App\Http\Controllers; use App\Jobs\Brevo\ProcessBrevoInboundWebhook; use App\Jobs\Brevo\ProcessBrevoWebhook; use App\Libraries\MultiDB; +use App\Models\Company; use Illuminate\Http\Request; use Log; @@ -77,6 +78,9 @@ class BrevoController extends BaseController /** * Process Brevo Inbound Webhook. * + * IMPORTANT NOTICE: brevo strips old sended emails, therefore only current attachements are present + * + * IMPORTANT NOTICE: brevo saves the message and attachemnts for later retrieval, therefore we can process it within a async job for performance reasons * * @OA\Post( * path="/api/v1/brevo_inbound_webhook", @@ -187,17 +191,18 @@ class BrevoController extends BaseController $input = $request->all(); // validation for client mail credentials by recipient - // if ($request->has('company')) { - // if (!($request->has('token'))) - // return response()->json(['message' => 'Unauthorized'], 403); + if ($request->has('company_key')) { + if (!($request->has('token'))) + return response()->json(['message' => 'Unauthorized'], 403); - // $company = MultiDB::findAndSetDbByCompanyId($request->has('company')); - // $company_brevo_secret = $company?->settings?->email_sending_method === 'client_brevo' && $company?->settings?->brevo_secret ? $company->settings->brevo_secret : null; - // if (!$company || !$company_brevo_secret || $request->get('token') !== $company_brevo_secret) - // return response()->json(['message' => 'Unauthorized'], 403); + MultiDB::findAndSetDbByCompanyKey($request->has('company_key')); + $company = Company::where('company_key', $request->has('company_key'))->first(); + $company_brevo_secret = $company?->settings?->email_sending_method === 'client_brevo' && $company?->settings?->brevo_secret ? $company->settings->brevo_secret : null; + if (!$company || !$company_brevo_secret || $request->get('token') !== $company_brevo_secret) + return response()->json(['message' => 'Unauthorized'], 403); - // } else if (!($request->has('token') && $request->get('token') == config('services.brevo.secret'))) - // return response()->json(['message' => 'Unauthorized'], 403); + } else if (!($request->has('token') && $request->get('token') == config('services.brevo.secret'))) + return response()->json(['message' => 'Unauthorized'], 403); if (!array_key_exists('items', $input)) { Log::info('Failed: Message could not be parsed, because required parameters are missing.'); diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index d097f61ed6ac..35f1c8388e6e 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -80,6 +80,10 @@ class MailgunController extends BaseController /** * Process Mailgun Inbound Webhook. * + * IMPORTANT NOTICE: mailgun does NOT strip old sended emails, therefore all past attachements are present + * + * IMPORTANT NOTICE: mailgun saves the message and attachemnts for later retrieval, therefore we can process it within a async job for performance reasons + * * * @OA\Post( * path="/api/v1/mailgun_inbound_webhook", diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index e76782323c33..ab3b9e74a56a 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -78,6 +78,9 @@ class PostMarkController extends BaseController /** * Process Postmark Webhook. * + * IMPORTANT NOTICE: postmark does NOT strip old sended emails, therefore also all past attachements are present + * + * IMPORTANT NOTICE: postmark does not saves attachements for later retrieval, therefore we cannot process it within a async job * * @OA\Post( * path="/api/v1/postmark_inbound_webhook", diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index 957a7a9ae162..6fafc74ee47d 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -33,6 +33,9 @@ class ProcessBrevoInboundWebhook implements ShouldQueue /** * Create a new job instance. + * + * IMPORTANT NOTICE: brevo strips old sended emails, therefore only current attachements are present + * * $input consists of json/serialized-array: * * array ( @@ -111,8 +114,6 @@ class ProcessBrevoInboundWebhook implements ShouldQueue /** * Execute the job. * - * TODO: insert Mail from Storage - * * @return void */ public function handle() diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index ff0792c52e38..c1846df7fcf1 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -40,6 +40,8 @@ class ProcessMailgunInboundWebhook implements ShouldQueue /** * Execute the job. * + * IMPORTANT NOTICE: mailgun does NOT strip old sended emails, therefore all past attachements are present + * * Mail from Storage * { * "Content-Type": "multipart/related; boundary=\"00000000000022bfbe0613e8b7f5\"", diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index 14690a4bd5af..3c1db5a0f0a0 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -237,16 +237,18 @@ class InboundMailEngine private function getClient(Company $company, InboundMail $email) { $clientContact = ClientContact::where("company_id", $company->id)->where("email", $email->from)->first(); - $client = $clientContact->client(); + if (!$clientContact) + return null; - return $client; + return $clientContact->client(); } private function getVendor(Company $company, InboundMail $email) { $vendorContact = VendorContact::where("company_id", $company->id)->where("email", $email->from)->first(); - $vendor = $vendorContact->vendor(); + if (!$vendorContact) + return null; - return $vendor; + return $vendorContact->vendor(); } private function logBlocked(Company $company, string $data) { From b7378b9b100a74c480f507046c76afad49d8a340 Mon Sep 17 00:00:00 2001 From: paulwer Date: Thu, 4 Apr 2024 07:27:07 +0200 Subject: [PATCH 059/119] renaming inbound_mailbox to expense_mailbox --- .../Requests/Company/StoreCompanyRequest.php | 24 ++++++++--------- .../Requests/Company/UpdateCompanyRequest.php | 26 +++++++++---------- .../Company/ValidExpenseMailbox.php | 14 +++++----- app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 2 +- .../Mailgun/ProcessMailgunInboundWebhook.php | 2 +- app/Libraries/MultiDB.php | 16 ++++++------ app/Models/Company.php | 10 +++---- .../InboundMail/InboundMailEngine.php | 4 +-- app/Transformers/CompanyTransformer.php | 4 +-- config/ninja.php | 4 +-- ...2023_12_10_110951_inbound_mail_parsing.php | 4 +-- lang/en/texts.php | 4 +-- 12 files changed, 57 insertions(+), 57 deletions(-) diff --git a/app/Http/Requests/Company/StoreCompanyRequest.php b/app/Http/Requests/Company/StoreCompanyRequest.php index 3b4725118a4d..ff09dbfacc07 100644 --- a/app/Http/Requests/Company/StoreCompanyRequest.php +++ b/app/Http/Requests/Company/StoreCompanyRequest.php @@ -47,7 +47,7 @@ class StoreCompanyRequest extends Request $rules['company_logo'] = 'mimes:jpeg,jpg,png,gif|max:10000'; // max 10000kb $rules['settings'] = new ValidSettingsRule(); - if (isset ($input['portal_mode']) && ($input['portal_mode'] == 'domain' || $input['portal_mode'] == 'iframe')) { + if (isset($input['portal_mode']) && ($input['portal_mode'] == 'domain' || $input['portal_mode'] == 'iframe')) { $rules['portal_domain'] = 'sometimes|url'; } else { if (Ninja::isHosted()) { @@ -57,7 +57,7 @@ class StoreCompanyRequest extends Request } } - $rules['inbound_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); + $rules['expense_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); $rules['smtp_host'] = 'sometimes|string|nullable'; $rules['smtp_port'] = 'sometimes|integer|nullable'; @@ -75,39 +75,39 @@ class StoreCompanyRequest extends Request { $input = $this->all(); - if (!isset ($input['name'])) { + if (!isset($input['name'])) { $input['name'] = 'Untitled Company'; } - if (isset ($input['google_analytics_url'])) { + if (isset($input['google_analytics_url'])) { $input['google_analytics_key'] = $input['google_analytics_url']; } - if (isset ($input['portal_domain'])) { + if (isset($input['portal_domain'])) { $input['portal_domain'] = rtrim(strtolower($input['portal_domain']), "/"); } - if (isset ($input['inbound_mailbox']) && Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise')) { - unset($input['inbound_mailbox']); + if (isset($input['expense_mailbox']) && Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise')) { + unset($input['expense_mailbox']); } - if (Ninja::isHosted() && !isset ($input['subdomain'])) { + if (Ninja::isHosted() && !isset($input['subdomain'])) { $input['subdomain'] = MultiDB::randomSubdomainGenerator(); } - if (isset ($input['smtp_username']) && strlen(str_replace("*", "", $input['smtp_username'])) < 2) { + if (isset($input['smtp_username']) && strlen(str_replace("*", "", $input['smtp_username'])) < 2) { unset($input['smtp_username']); } - if (isset ($input['smtp_password']) && strlen(str_replace("*", "", $input['smtp_password'])) < 2) { + if (isset($input['smtp_password']) && strlen(str_replace("*", "", $input['smtp_password'])) < 2) { unset($input['smtp_password']); } - if (isset ($input['smtp_port'])) { + if (isset($input['smtp_port'])) { $input['smtp_port'] = (int) $input['smtp_port']; } - if (isset ($input['smtp_verify_peer']) && is_string($input['smtp_verify_peer'])) + if (isset($input['smtp_verify_peer']) && is_string($input['smtp_verify_peer'])) $input['smtp_verify_peer'] == 'true' ? true : false; $this->replace($input); diff --git a/app/Http/Requests/Company/UpdateCompanyRequest.php b/app/Http/Requests/Company/UpdateCompanyRequest.php index c8578d262ae1..e4b24961e85b 100644 --- a/app/Http/Requests/Company/UpdateCompanyRequest.php +++ b/app/Http/Requests/Company/UpdateCompanyRequest.php @@ -66,7 +66,7 @@ class UpdateCompanyRequest extends Request // $rules['smtp_verify_peer'] = 'sometimes|string'; - if (isset ($input['portal_mode']) && ($input['portal_mode'] == 'domain' || $input['portal_mode'] == 'iframe')) { + if (isset($input['portal_mode']) && ($input['portal_mode'] == 'domain' || $input['portal_mode'] == 'iframe')) { $rules['portal_domain'] = 'bail|nullable|sometimes|url'; } @@ -74,7 +74,7 @@ class UpdateCompanyRequest extends Request $rules['subdomain'] = ['nullable', 'regex:/^[a-zA-Z0-9.-]+[a-zA-Z0-9]$/', new ValidSubdomain()]; } - $rules['inbound_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); // @turbo124 check if this is right + $rules['expense_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); // @turbo124 check if this is right return $rules; } @@ -83,40 +83,40 @@ class UpdateCompanyRequest extends Request { $input = $this->all(); - if (isset ($input['portal_domain']) && strlen($input['portal_domain']) > 1) { + if (isset($input['portal_domain']) && strlen($input['portal_domain']) > 1) { $input['portal_domain'] = $this->addScheme($input['portal_domain']); $input['portal_domain'] = rtrim(strtolower($input['portal_domain']), "/"); } - if (isset ($input['inbound_mailbox']) && Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise')) { - unset($input['inbound_mailbox']); + if (isset($input['expense_mailbox']) && Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise')) { + unset($input['expense_mailbox']); } - if (isset ($input['settings'])) { + if (isset($input['settings'])) { $input['settings'] = (array) $this->filterSaveableSettings($input['settings']); } - if (isset ($input['subdomain']) && $this->company->subdomain == $input['subdomain']) { + if (isset($input['subdomain']) && $this->company->subdomain == $input['subdomain']) { unset($input['subdomain']); } - if (isset ($input['e_invoice_certificate_passphrase']) && empty ($input['e_invoice_certificate_passphrase'])) { + if (isset($input['e_invoice_certificate_passphrase']) && empty($input['e_invoice_certificate_passphrase'])) { unset($input['e_invoice_certificate_passphrase']); } - if (isset ($input['smtp_username']) && strlen(str_replace("*", "", $input['smtp_username'])) < 2) { + if (isset($input['smtp_username']) && strlen(str_replace("*", "", $input['smtp_username'])) < 2) { unset($input['smtp_username']); } - if (isset ($input['smtp_password']) && strlen(str_replace("*", "", $input['smtp_password'])) < 2) { + if (isset($input['smtp_password']) && strlen(str_replace("*", "", $input['smtp_password'])) < 2) { unset($input['smtp_password']); } - if (isset ($input['smtp_port'])) { + if (isset($input['smtp_port'])) { $input['smtp_port'] = (int) $input['smtp_port']; } - if (isset ($input['smtp_verify_peer']) && is_string($input['smtp_verify_peer'])) { + if (isset($input['smtp_verify_peer']) && is_string($input['smtp_verify_peer'])) { $input['smtp_verify_peer'] == 'true' ? true : false; } @@ -143,7 +143,7 @@ class UpdateCompanyRequest extends Request } } - if (isset ($settings['email_style_custom'])) { + if (isset($settings['email_style_custom'])) { $settings['email_style_custom'] = str_replace(['{!!', '!!}', '{{', '}}', '@if(', '@endif', '@isset', '@unless', '@auth', '@empty', '@guest', '@env', '@section', '@switch', '@foreach', '@while', '@include', '@each', '@once', '@push', '@use', '@forelse', '@verbatim', 'company_key = $company_key; - $this->endings = explode(",", config('ninja.inbound_mailbox.inbound_mailbox_endings')); + $this->endings = explode(",", config('ninja.inbound_mailbox.expense_mailbox_endings')); } public function passes($attribute, $value) { - if (empty ($value)) { + if (empty($value)) { return true; } // early return, if we dont have any additional validation - if (!config('ninja.inbound_mailbox.inbound_mailbox_endings')) { + if (!config('ninja.inbound_mailbox.expense_mailbox_endings')) { $this->validated_schema = true; - return MultiDB::checkInboundMailboxAvailable($value); + return MultiDB::checkExpenseMailboxAvailable($value); } // Validate Schema @@ -59,7 +59,7 @@ class ValidExpenseMailbox implements Rule return false; $this->validated_schema = true; - return MultiDB::checkInboundMailboxAvailable($value); + return MultiDB::checkExpenseMailboxAvailable($value); } /** @@ -68,8 +68,8 @@ class ValidExpenseMailbox implements Rule public function message() { if (!$this->validated_schema) - return ctrans('texts.inbound_mailbox_invalid'); + return ctrans('texts.expense_mailbox_invalid'); - return ctrans('texts.inbound_mailbox_taken'); + return ctrans('texts.expense_mailbox_taken'); } } diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index 6fafc74ee47d..d6fc40e6f4a0 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -129,7 +129,7 @@ class ProcessBrevoInboundWebhook implements ShouldQueue } // match company - $company = MultiDB::findAndSetDbByInboundMailbox($recipient); + $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); if (!$company) { Log::info('[ProcessBrevoInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from brevo: ' . $recipient); (new InboundMailEngine())->saveMeta($this->input["From"]["Address"], $recipient); // important to save this, to protect from spam diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index c1846df7fcf1..1e786f80eeac 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -176,7 +176,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue } // match company - $company = MultiDB::findAndSetDbByInboundMailbox($to); + $company = MultiDB::findAndSetDbByExpenseMailbox($to); if (!$company) { Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $to); (new InboundMailEngine())->saveMeta($from, $to); // important to save this, to protect from spam diff --git a/app/Libraries/MultiDB.php b/app/Libraries/MultiDB.php index 58362ab77dcd..c0a6b213d241 100644 --- a/app/Libraries/MultiDB.php +++ b/app/Libraries/MultiDB.php @@ -73,7 +73,7 @@ class MultiDB 'socket', ]; - private static $protected_inbound_mailboxes = []; + private static $protected_expense_mailboxes = []; /** * @return array @@ -109,21 +109,21 @@ class MultiDB return true; } - public static function checkInboundMailboxAvailable($inbound_mailbox): bool + public static function checkExpenseMailboxAvailable($expense_mailbox): bool { if (!config('ninja.db.multi_db_enabled')) { - return Company::where("inbound_mailbox", $inbound_mailbox)->withTrashed()->exists(); + return Company::where("expense_mailbox", $expense_mailbox)->withTrashed()->exists(); } - if (in_array($inbound_mailbox, self::$protected_inbound_mailboxes)) { + if (in_array($expense_mailbox, self::$protected_expense_mailboxes)) { return false; } $current_db = config('database.default'); foreach (self::$dbs as $db) { - if (Company::on($db)->where("inbound_mailbox", $inbound_mailbox)->withTrashed()->exists()) { + if (Company::on($db)->where("expense_mailbox", $expense_mailbox)->withTrashed()->exists()) { self::setDb($current_db); return false; @@ -515,16 +515,16 @@ class MultiDB return false; } - public static function findAndSetDbByInboundMailbox($inbound_mailbox) + public static function findAndSetDbByExpenseMailbox($expense_mailbox) { if (!config('ninja.db.multi_db_enabled')) { - return Company::where("inbound_mailbox", $inbound_mailbox)->first(); + return Company::where("expense_mailbox", $expense_mailbox)->first(); } $current_db = config('database.default'); foreach (self::$dbs as $db) { - if ($company = Company::on($db)->where("inbound_mailbox", $inbound_mailbox)->first()) { + if ($company = Company::on($db)->where("expense_mailbox", $expense_mailbox)->first()) { self::setDb($db); return $company; diff --git a/app/Models/Company.php b/app/Models/Company.php index ab40b29960d0..5400d284b507 100644 --- a/app/Models/Company.php +++ b/app/Models/Company.php @@ -111,8 +111,8 @@ use Laracasts\Presenter\PresentableTrait; * @property int $convert_expense_currency * @property int $notify_vendor_when_paid * @property int $invoice_task_hours - * @property string|null $inbound_mailbox - * @property boolean $inbound_mailbox_active + * @property string|null $expense_mailbox + * @property boolean $expense_mailbox_active * @property bool $inbound_mailbox_allow_company_users * @property bool $inbound_mailbox_allow_vendors * @property bool $inbound_mailbox_allow_clients @@ -369,8 +369,8 @@ class Company extends BaseModel 'calculate_taxes', 'tax_data', 'e_invoice_certificate_passphrase', - 'inbound_mailbox_active', - 'inbound_mailbox', // TODO: @turbo124 custom validation: self-hosted => free change, hosted => not changeable, only changeable with env-mask + 'expense_mailbox_active', + 'expense_mailbox', // TODO: @turbo124 custom validation: self-hosted => free change, hosted => not changeable, only changeable with env-mask 'inbound_mailbox_allow_company_users', 'inbound_mailbox_allow_vendors', 'inbound_mailbox_allow_clients', @@ -732,7 +732,7 @@ class Company extends BaseModel public function getLocale() { - return isset ($this->settings->language_id) && $this->language() ? $this->language()->locale : config('ninja.i18n.locale'); + return isset($this->settings->language_id) && $this->language() ? $this->language()->locale : config('ninja.i18n.locale'); } public function getLogo(): ?string diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index 3c1db5a0f0a0..cdf1fe177fb7 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -54,7 +54,7 @@ class InboundMailEngine $isUnknownRecipent = true; // Expense Mailbox => will create an expense - $company = MultiDB::findAndSetDbByInboundMailbox($email->to); + $company = MultiDB::findAndSetDbByExpenseMailbox($email->to); if ($company) { $isUnknownRecipent = false; $this->createExpense($company, $email); @@ -148,7 +148,7 @@ class InboundMailEngine protected function createExpense(Company $company, InboundMail $email) { // Skipping executions: will not result in not saving Metadata to prevent usage of these conditions, to spam - if (!($company?->inbound_mailbox_active ?: false)) { + if (!($company?->expense_mailbox_active ?: false)) { $this->logBlocked($company, 'mailbox not active for this company. from: ' . $email->from); return; } diff --git a/app/Transformers/CompanyTransformer.php b/app/Transformers/CompanyTransformer.php index f3ebd108fc6e..f6cc4ae526a5 100644 --- a/app/Transformers/CompanyTransformer.php +++ b/app/Transformers/CompanyTransformer.php @@ -204,8 +204,8 @@ class CompanyTransformer extends EntityTransformer 'invoice_task_project_header' => (bool) $company->invoice_task_project_header, 'invoice_task_item_description' => (bool) $company->invoice_task_item_description, 'origin_tax_data' => $company->origin_tax_data ?: new \stdClass, - 'inbound_mailbox' => (bool) $company->inbound_mailbox, - 'inbound_mailbox_active' => (bool) $company->inbound_mailbox_active, + 'expense_mailbox' => (bool) $company->expense_mailbox, + 'expense_mailbox_active' => (bool) $company->expense_mailbox_active, 'inbound_mailbox_allow_company_users' => (bool) $company->inbound_mailbox_allow_company_users, 'inbound_mailbox_allow_vendors' => (bool) $company->inbound_mailbox_allow_vendors, 'inbound_mailbox_allow_clients' => (bool) $company->inbound_mailbox_allow_clients, diff --git a/config/ninja.php b/config/ninja.php index bdac51f8c0f1..3a05f424e7f3 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -237,8 +237,8 @@ return [ 'webhook_id' => env('PAYPAL_WEBHOOK_ID', null), ], 'inbound_mailbox' => [ - 'inbound_mailbox_template' => env('INBOUND_MAILBOX_TEMPLATE', null), - 'inbound_mailbox_endings' => env('INBOUND_MAILBOX_ENDINGS', '@expense.invoicing.co'), + 'expense_mailbox_template' => env('EXPENSE_MAILBOX_TEMPLATE', null), + 'expense_mailbox_endings' => env('EXPENSE_MAILBOX_ENDINGS', '@expense.invoicing.co'), ], 'cloudflare' => [ 'turnstile' => [ diff --git a/database/migrations/2023_12_10_110951_inbound_mail_parsing.php b/database/migrations/2023_12_10_110951_inbound_mail_parsing.php index f750d07065b4..94d647510f34 100644 --- a/database/migrations/2023_12_10_110951_inbound_mail_parsing.php +++ b/database/migrations/2023_12_10_110951_inbound_mail_parsing.php @@ -12,8 +12,8 @@ return new class extends Migration { public function up(): void { Schema::table('companies', function (Blueprint $table) { - $table->boolean("inbound_mailbox_active")->default(true); - $table->string("inbound_mailbox")->nullable(); + $table->boolean("expense_mailbox_active")->default(true); + $table->string("expense_mailbox")->nullable(); $table->boolean("inbound_mailbox_allow_company_users")->default(false); $table->boolean("inbound_mailbox_allow_vendors")->default(false); $table->boolean("inbound_mailbox_allow_clients")->default(false); diff --git a/lang/en/texts.php b/lang/en/texts.php index 627eb807284d..3f6fb79b43a5 100644 --- a/lang/en/texts.php +++ b/lang/en/texts.php @@ -2494,8 +2494,8 @@ $lang = array( 'local_storage_required' => 'Error: local storage is not available.', 'your_password_reset_link' => 'Your Password Reset Link', 'subdomain_taken' => 'The subdomain is already in use', - 'inbound_mailbox_taken' => 'The inbound mailbox is already in use', - 'inbound_mailbox_invalid' => 'The inbound mailbox does not match the required schema', + 'expense_mailbox_taken' => 'The inbound mailbox is already in use', + 'expense_mailbox_invalid' => 'The inbound mailbox does not match the required schema', 'client_login' => 'Client Login', 'converted_amount' => 'Converted Amount', 'default' => 'Default', From 832397a98ea212aa14175869ad5335a86262db49 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 7 Apr 2024 13:26:10 +0200 Subject: [PATCH 060/119] minor security updates --- app/Http/Controllers/BrevoController.php | 13 +------------ app/Http/Controllers/MailgunController.php | 11 ++++++----- app/Http/Controllers/PostMarkController.php | 14 ++++++++------ config/ninja.php | 1 + 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/app/Http/Controllers/BrevoController.php b/app/Http/Controllers/BrevoController.php index 8cb666d24410..ac6d9265cc9a 100644 --- a/app/Http/Controllers/BrevoController.php +++ b/app/Http/Controllers/BrevoController.php @@ -190,18 +190,7 @@ class BrevoController extends BaseController { $input = $request->all(); - // validation for client mail credentials by recipient - if ($request->has('company_key')) { - if (!($request->has('token'))) - return response()->json(['message' => 'Unauthorized'], 403); - - MultiDB::findAndSetDbByCompanyKey($request->has('company_key')); - $company = Company::where('company_key', $request->has('company_key'))->first(); - $company_brevo_secret = $company?->settings?->email_sending_method === 'client_brevo' && $company?->settings?->brevo_secret ? $company->settings->brevo_secret : null; - if (!$company || !$company_brevo_secret || $request->get('token') !== $company_brevo_secret) - return response()->json(['message' => 'Unauthorized'], 403); - - } else if (!($request->has('token') && $request->get('token') == config('services.brevo.secret'))) + if (!($request->has('token') && $request->get('token') == config('ninja.inbound_mailbox.inbound_webhook_token'))) return response()->json(['message' => 'Unauthorized'], 403); if (!array_key_exists('items', $input)) { diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 35f1c8388e6e..32bcac1799b8 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -126,12 +126,13 @@ class MailgunController extends BaseController // @turbo124 TODO: how to check for services.mailgun.webhook_signing_key on company level, when custom credentials are defined // TODO: validation for client mail credentials by recipient - if (\hash_equals(\hash_hmac('sha256', $input['timestamp'] . $input['token'], config('services.mailgun.webhook_signing_key')), $input['signature'])) { - ProcessMailgunInboundWebhook::dispatch($input["sender"] . "|" . $input["recipient"] . "|" . $input["message-url"])->delay(10); + $authorizedByHash = \hash_equals(\hash_hmac('sha256', $input['timestamp'] . $input['token'], config('services.mailgun.webhook_signing_key')), $input['signature']); + $authorizedByToken = $request->has('token') && $request->get('token') == config('ninja.inbound_mailbox.inbound_webhook_token'); + if (!$authorizedByHash && !$authorizedByToken) + return response()->json(['message' => 'Unauthorized'], 403); - return response()->json(['message' => 'Success'], 201); - } + ProcessMailgunInboundWebhook::dispatch($input["sender"] . "|" . $input["recipient"] . "|" . $input["message-url"])->delay(10); - return response()->json(['message' => 'Unauthorized'], 403); + return response()->json(['message' => 'Success.'], 200); } } diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index ab3b9e74a56a..1f59240a2d73 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -275,17 +275,19 @@ class PostMarkController extends BaseController $input = $request->all(); + if (!($request->has('token') && $request->get('token') == config('ninja.inbound_mailbox.inbound_webhook_token'))) + return response()->json(['message' => 'Unauthorized'], 403); + if (!(array_key_exists("MessageStream", $input) && $input["MessageStream"] == "inbound") || !array_key_exists("To", $input) || !array_key_exists("From", $input) || !array_key_exists("MessageID", $input)) { Log::info('Failed: Message could not be parsed, because required parameters are missing.'); return response()->json(['message' => 'Failed. Missing/Invalid Parameters.'], 400); } - // // TODO: security - // if (!($request->header('X-API-SECURITY') && $request->header('X-API-SECURITY') == config('services.postmark.token'))) - // return response()->json(['message' => 'Unauthorized'], 403); + $inboundEngine = new InboundMailEngine(); - if ((new InboundMailEngine())->isInvalidOrBlocked($input["From"], $input["To"])) { + if ($inboundEngine->isInvalidOrBlocked($input["From"], $input["To"])) { Log::info('Failed: Sender is blocked: ' . $input["From"] . " Recipient: " . $input["To"]); + $inboundEngine->saveMeta($input["From"], $input["To"]); return response()->json(['message' => 'Blocked.'], 403); } @@ -309,12 +311,12 @@ class PostMarkController extends BaseController } } catch (\Exception $e) { - (new InboundMailEngine())->saveMeta($input["From"], $input["To"]); // important to save this, to protect from spam + $inboundEngine->saveMeta($input["From"], $input["To"]); // important to save this, to protect from spam throw $e; } // perform - (new InboundMailEngine())->handle($inboundMail); + $inboundEngine->handle($inboundMail); return response()->json(['message' => 'Success'], 200); } diff --git a/config/ninja.php b/config/ninja.php index 3a05f424e7f3..94933bce3b59 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -239,6 +239,7 @@ return [ 'inbound_mailbox' => [ 'expense_mailbox_template' => env('EXPENSE_MAILBOX_TEMPLATE', null), 'expense_mailbox_endings' => env('EXPENSE_MAILBOX_ENDINGS', '@expense.invoicing.co'), + 'inbound_webhook_key' => env('INBOUND_WEBHOOK_KEY', null) ], 'cloudflare' => [ 'turnstile' => [ From c02a4fb08dc2391b3be7a6537eb65f2b69ca1ace Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 7 Apr 2024 13:58:10 +0200 Subject: [PATCH 061/119] minor adjustment according spam behavior --- app/Http/Controllers/PostMarkController.php | 11 +++++++++-- app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 10 ++++++---- app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php | 10 ++++++---- app/Services/InboundMail/InboundMailEngine.php | 13 ++++++------- 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index 1f59240a2d73..a6877b1e250c 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -12,6 +12,7 @@ namespace App\Http\Controllers; use App\Jobs\PostMark\ProcessPostmarkWebhook; +use App\Libraries\MultiDB; use App\Services\InboundMail\InboundMail; use App\Services\InboundMail\InboundMailEngine; use App\Utils\TempFile; @@ -287,10 +288,16 @@ class PostMarkController extends BaseController if ($inboundEngine->isInvalidOrBlocked($input["From"], $input["To"])) { Log::info('Failed: Sender is blocked: ' . $input["From"] . " Recipient: " . $input["To"]); - $inboundEngine->saveMeta($input["From"], $input["To"]); return response()->json(['message' => 'Blocked.'], 403); } + $company = MultiDB::findAndSetDbByExpenseMailbox($input["To"]); + if (!$company) { + Log::info('[PostmarkInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $input["To"]); + $inboundEngine->saveMeta($input["From"], $input["To"], true); // important to save this, to protect from spam + return; + } + try { // important to save meta if something fails here to prevent spam // prepare data for ingresEngine @@ -316,7 +323,7 @@ class PostMarkController extends BaseController } // perform - $inboundEngine->handle($inboundMail); + $inboundEngine->handleExpenseMailbox($inboundMail); return response()->json(['message' => 'Success'], 200); } diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index d6fc40e6f4a0..be11b4905f0f 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -31,6 +31,8 @@ class ProcessBrevoInboundWebhook implements ShouldQueue public $tries = 1; + private InboundMailEngine $engine = new InboundMailEngine(); + /** * Create a new job instance. * @@ -123,7 +125,7 @@ class ProcessBrevoInboundWebhook implements ShouldQueue foreach ($this->input["Recipients"] as $recipient) { // Spam protection - if ((new InboundMailEngine())->isInvalidOrBlocked($this->input["From"]["Address"], $recipient)) { + if ($this->engine->isInvalidOrBlocked($this->input["From"]["Address"], $recipient)) { Log::info('Failed: Sender is blocked: ' . $this->input["From"]["Address"] . " Recipient: " . $recipient); throw new \Error('Sender is blocked'); } @@ -132,7 +134,7 @@ class ProcessBrevoInboundWebhook implements ShouldQueue $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); if (!$company) { Log::info('[ProcessBrevoInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from brevo: ' . $recipient); - (new InboundMailEngine())->saveMeta($this->input["From"]["Address"], $recipient); // important to save this, to protect from spam + // $this->engine->saveMeta($this->input["From"]["Address"], $recipient, true); // @turbo124 disabled, because recipents contains all recipients, and will likly result in false bans?! => normally important to save this, to protect from spam continue; } @@ -186,11 +188,11 @@ class ProcessBrevoInboundWebhook implements ShouldQueue } } catch (\Exception $e) { - (new InboundMailEngine())->saveMeta($this->input["From"]["Address"], $recipient); // important to save this, to protect from spam + $this->engine->saveMeta($this->input["From"]["Address"], $recipient); // important to save this, to protect from spam throw $e; } - (new InboundMailEngine())->handle($inboundMail); + $this->engine->handleExpenseMailbox($inboundMail); } } diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index 1e786f80eeac..3ac892d8d081 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -29,6 +29,8 @@ class ProcessMailgunInboundWebhook implements ShouldQueue public $tries = 1; + private InboundMailEngine $engine = new InboundMailEngine(); + /** * Create a new job instance. * $input consists of 3 informations: sender/from|recipient/to|messageUrl @@ -170,7 +172,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue // $messageId = explode("|", $this->input)[2]; // used as base in download function // Spam protection - if ((new InboundMailEngine())->isInvalidOrBlocked($from, $to)) { + if ($this->engine->isInvalidOrBlocked($from, $to)) { Log::info('Failed: Sender is blocked: ' . $from . " Recipient: " . $to); throw new \Error('Sender is blocked'); } @@ -179,7 +181,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $company = MultiDB::findAndSetDbByExpenseMailbox($to); if (!$company) { Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $to); - (new InboundMailEngine())->saveMeta($from, $to); // important to save this, to protect from spam + $this->engine->saveMeta($from, $to, true); // important to save this, to protect from spam return; } @@ -276,11 +278,11 @@ class ProcessMailgunInboundWebhook implements ShouldQueue } } catch (\Exception $e) { - (new InboundMailEngine())->saveMeta($from, $to); // important to save this, to protect from spam + $this->engine->saveMeta($from, $to); // important to save this, to protect from spam throw $e; } // perform - (new InboundMailEngine())->handle($inboundMail); + $this->engine->handleExpenseMailbox($inboundMail); } } diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index cdf1fe177fb7..f5deef5a874b 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -46,21 +46,20 @@ class InboundMailEngine * if there is not a company with an matching mailbox, we only do monitoring * reuse this method to add more mail-parsing behaviors */ - public function handle(InboundMail $email) + public function handleExpenseMailbox(InboundMail $email) { if ($this->isInvalidOrBlocked($email->from, $email->to)) return; - $isUnknownRecipent = true; - // Expense Mailbox => will create an expense $company = MultiDB::findAndSetDbByExpenseMailbox($email->to); - if ($company) { - $isUnknownRecipent = false; - $this->createExpense($company, $email); + if (!$company) { + $this->saveMeta($email->from, $email->to, true); + return; } - $this->saveMeta($email->from, $email->to, $isUnknownRecipent); + $this->createExpense($company, $email); + $this->saveMeta($email->from, $email->to); } // SPAM Protection From 4ac3289819726312bc5f6148f95d673246b9e72e Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 7 Apr 2024 14:10:42 +0200 Subject: [PATCH 062/119] brevo fixes --- app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index be11b4905f0f..02901132fee1 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -121,6 +121,8 @@ class ProcessBrevoInboundWebhook implements ShouldQueue public function handle() { + $foundOneRecipient = false; // used for spam documentation below + // brevo defines recipients as array, we check all of them, to be sure foreach ($this->input["Recipients"] as $recipient) { @@ -134,10 +136,11 @@ class ProcessBrevoInboundWebhook implements ShouldQueue $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); if (!$company) { Log::info('[ProcessBrevoInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from brevo: ' . $recipient); - // $this->engine->saveMeta($this->input["From"]["Address"], $recipient, true); // @turbo124 disabled, because recipents contains all recipients, and will likly result in false bans?! => normally important to save this, to protect from spam continue; } + $foundOneRecipient = true; + try { // important to save meta if something fails here to prevent spam $company_brevo_secret = $company->settings?->email_sending_method === 'client_brevo' && $company->settings?->brevo_secret ? $company->settings?->brevo_secret : null; @@ -195,5 +198,11 @@ class ProcessBrevoInboundWebhook implements ShouldQueue $this->engine->handleExpenseMailbox($inboundMail); } + + // document for spam => mark all recipients as handled emails with unmatched mailbox => otherwise dont do any + if (!$foundOneRecipient) + foreach ($this->input["Recipients"] as $recipient) { + $this->engine->saveMeta($this->input["From"]["Address"], $recipient, true); + } } } From c1ec89b0a752521b0ba5c602ef276b62b2cba778 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 7 Apr 2024 16:08:34 +0200 Subject: [PATCH 063/119] fixes for spam-blocking --- app/Http/Controllers/PostMarkController.php | 1 - app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 6 ++-- .../Mailgun/ProcessMailgunInboundWebhook.php | 6 ++-- .../InboundMail/InboundMailEngine.php | 32 +++++++++++++------ 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index a6877b1e250c..81e618493c04 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -287,7 +287,6 @@ class PostMarkController extends BaseController $inboundEngine = new InboundMailEngine(); if ($inboundEngine->isInvalidOrBlocked($input["From"], $input["To"])) { - Log::info('Failed: Sender is blocked: ' . $input["From"] . " Recipient: " . $input["To"]); return response()->json(['message' => 'Blocked.'], 403); } diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index 02901132fee1..864b0bdb9af8 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -31,7 +31,7 @@ class ProcessBrevoInboundWebhook implements ShouldQueue public $tries = 1; - private InboundMailEngine $engine = new InboundMailEngine(); + private InboundMailEngine $engine; /** * Create a new job instance. @@ -111,6 +111,7 @@ class ProcessBrevoInboundWebhook implements ShouldQueue */ public function __construct(private array $input) { + $this->engine = new InboundMailEngine(); } /** @@ -128,8 +129,7 @@ class ProcessBrevoInboundWebhook implements ShouldQueue // Spam protection if ($this->engine->isInvalidOrBlocked($this->input["From"]["Address"], $recipient)) { - Log::info('Failed: Sender is blocked: ' . $this->input["From"]["Address"] . " Recipient: " . $recipient); - throw new \Error('Sender is blocked'); + return; } // match company diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index 3ac892d8d081..1bf6caeb44dc 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -29,7 +29,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue public $tries = 1; - private InboundMailEngine $engine = new InboundMailEngine(); + private InboundMailEngine $engine; /** * Create a new job instance. @@ -37,6 +37,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue */ public function __construct(private string $input) { + $this->engine = new InboundMailEngine(); } /** @@ -173,8 +174,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue // Spam protection if ($this->engine->isInvalidOrBlocked($from, $to)) { - Log::info('Failed: Sender is blocked: ' . $from . " Recipient: " . $to); - throw new \Error('Sender is blocked'); + return; } // match company diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index f5deef5a874b..5880c9ee5bfa 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -39,6 +39,8 @@ class InboundMailEngine private ?bool $isUnknownRecipent = null; private array $globalBlacklistDomains = []; private array $globalBlacklistSenders = []; + private array $globalWhitelistDomains = []; // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders + private array $globalWhitelistSenders = []; // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders public function __construct() { } @@ -70,15 +72,25 @@ class InboundMailEngine Log::info('E-Mail blocked, because from e-mail has the wrong format: ' . $from); return true; } + if (!filter_var($to, FILTER_VALIDATE_EMAIL)) { + Log::info('E-Mail blocked, because to e-mail has the wrong format: ' . $from); + return true; + } $parts = explode('@', $from); $domain = array_pop($parts); // global blacklist + if (in_array($from, $this->globalWhitelistDomains)) { + return false; + } if (in_array($domain, $this->globalBlacklistDomains)) { Log::info('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $from); return true; } + if (in_array($domain, $this->globalWhitelistSenders)) { + return false; + } if (in_array($from, $this->globalBlacklistSenders)) { Log::info('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $from); return true; @@ -89,7 +101,7 @@ class InboundMailEngine } // sender occured in more than 500 emails in the last 12 hours - $senderMailCountTotal = Cache::get('inboundMailSender:' . $from, 0); + $senderMailCountTotal = Cache::get('inboundMailCountSender:' . $from, 0); if ($senderMailCountTotal >= 5000) { Log::info('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $from); $this->blockSender($from); @@ -103,7 +115,7 @@ class InboundMailEngine } // sender sended more than 50 emails to the wrong mailbox in the last 6 hours - $senderMailCountUnknownRecipent = Cache::get('inboundMailSenderUnknownRecipent:' . $from, 0); + $senderMailCountUnknownRecipent = Cache::get('inboundMailCountSenderUnknownRecipent:' . $from, 0); if ($senderMailCountUnknownRecipent >= 50) { Log::info('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $from); $this->saveMeta($from, $to); @@ -111,8 +123,8 @@ class InboundMailEngine } // wrong recipent occurs in more than 100 emails in the last 12 hours, so the processing is blocked - $mailCountUnknownRecipent = Cache::get('inboundMailUnknownRecipent:' . $to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time - if ($mailCountUnknownRecipent >= 100) { + $mailCountUnknownRecipent = Cache::get('inboundMailCountUnknownRecipent:' . $to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time + if ($mailCountUnknownRecipent >= 200) { Log::info('E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $from); $this->blockSender($from); $this->saveMeta($from, $to); @@ -131,15 +143,15 @@ class InboundMailEngine public function saveMeta(string $from, string $to, bool $isUnknownRecipent = false) { // save cache - Cache::add('inboundMailSender:' . $from, 0, now()->addHours(12)); - Cache::increment('inboundMailSender:' . $from); + Cache::add('inboundMailCountSender:' . $from, 0, now()->addHours(12)); + Cache::increment('inboundMailCountSender:' . $from); if ($isUnknownRecipent) { - Cache::add('inboundMailSenderUnknownRecipent:' . $from, 0, now()->addHours(6)); - Cache::increment('inboundMailSenderUnknownRecipent:' . $from); // we save the sender, to may block him + Cache::add('inboundMailCountSenderUnknownRecipent:' . $from, 0, now()->addHours(6)); + Cache::increment('inboundMailCountSenderUnknownRecipent:' . $from); // we save the sender, to may block him - Cache::add('inboundMailUnknownRecipent:' . $to, 0, now()->addHours(12)); - Cache::increment('inboundMailUnknownRecipent:' . $to); // we save the sender, to may block him + Cache::add('inboundMailCountUnknownRecipent:' . $to, 0, now()->addHours(12)); + Cache::increment('inboundMailCountUnknownRecipent:' . $to); // we save the sender, to may block him } } From 8547af74ab87b0d2c68886147c758ca36be19613 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 7 Apr 2024 16:26:35 +0200 Subject: [PATCH 064/119] fixes --- composer.lock | 335 ++++++++++++++++++++++++----------------------- config/ninja.php | 1 - 2 files changed, 169 insertions(+), 167 deletions(-) diff --git a/composer.lock b/composer.lock index 0dc8a7205e8e..43ca85369c49 100644 --- a/composer.lock +++ b/composer.lock @@ -1384,16 +1384,16 @@ }, { "name": "aws/aws-sdk-php", - "version": "3.302.0", + "version": "3.303.4", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "cb343ed4fc5d86c0ddf8e948f0271052f183f937" + "reference": "9ae5429f7699701cb158780cd287d1549f45ad32" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/cb343ed4fc5d86c0ddf8e948f0271052f183f937", - "reference": "cb343ed4fc5d86c0ddf8e948f0271052f183f937", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/9ae5429f7699701cb158780cd287d1549f45ad32", + "reference": "9ae5429f7699701cb158780cd287d1549f45ad32", "shasum": "" }, "require": { @@ -1473,9 +1473,9 @@ "support": { "forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80", "issues": "https://github.com/aws/aws-sdk-php/issues", - "source": "https://github.com/aws/aws-sdk-php/tree/3.302.0" + "source": "https://github.com/aws/aws-sdk-php/tree/3.303.4" }, - "time": "2024-03-29T18:07:04+00:00" + "time": "2024-04-05T18:04:15+00:00" }, { "name": "bacon/bacon-qr-code", @@ -3586,16 +3586,16 @@ }, { "name": "google/auth", - "version": "v1.37.0", + "version": "v1.37.1", "source": { "type": "git", "url": "https://github.com/googleapis/google-auth-library-php.git", - "reference": "5f16f67375b6f202b857183d7ef4e076acd7d4aa" + "reference": "1a7de77b72e6ac60dccf0e6478c4c1005bb0ff46" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/googleapis/google-auth-library-php/zipball/5f16f67375b6f202b857183d7ef4e076acd7d4aa", - "reference": "5f16f67375b6f202b857183d7ef4e076acd7d4aa", + "url": "https://api.github.com/repos/googleapis/google-auth-library-php/zipball/1a7de77b72e6ac60dccf0e6478c4c1005bb0ff46", + "reference": "1a7de77b72e6ac60dccf0e6478c4c1005bb0ff46", "shasum": "" }, "require": { @@ -3638,9 +3638,9 @@ "support": { "docs": "https://googleapis.github.io/google-auth-library-php/main/", "issues": "https://github.com/googleapis/google-auth-library-php/issues", - "source": "https://github.com/googleapis/google-auth-library-php/tree/v1.37.0" + "source": "https://github.com/googleapis/google-auth-library-php/tree/v1.37.1" }, - "time": "2024-02-21T17:03:52+00:00" + "time": "2024-04-03T18:41:12+00:00" }, { "name": "graham-campbell/result-type", @@ -4526,16 +4526,16 @@ }, { "name": "horstoeko/zugferd", - "version": "v1.0.37", + "version": "v1.0.38", "source": { "type": "git", "url": "https://github.com/horstoeko/zugferd.git", - "reference": "05f58ad4dbcc23d767fceb15f46b46097ffd43f1" + "reference": "47730d3d01c1229f22d75193d68bc43ea16bcdb7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/horstoeko/zugferd/zipball/05f58ad4dbcc23d767fceb15f46b46097ffd43f1", - "reference": "05f58ad4dbcc23d767fceb15f46b46097ffd43f1", + "url": "https://api.github.com/repos/horstoeko/zugferd/zipball/47730d3d01c1229f22d75193d68bc43ea16bcdb7", + "reference": "47730d3d01c1229f22d75193d68bc43ea16bcdb7", "shasum": "" }, "require": { @@ -4594,9 +4594,9 @@ ], "support": { "issues": "https://github.com/horstoeko/zugferd/issues", - "source": "https://github.com/horstoeko/zugferd/tree/v1.0.37" + "source": "https://github.com/horstoeko/zugferd/tree/v1.0.38" }, - "time": "2024-03-24T11:31:03+00:00" + "time": "2024-04-05T03:38:38+00:00" }, { "name": "http-interop/http-factory-guzzle", @@ -5029,25 +5029,24 @@ }, { "name": "invoiceninja/ubl_invoice", - "version": "v2.0.3", + "version": "v2.1.0", "source": { "type": "git", "url": "https://github.com/invoiceninja/UBL_invoice.git", - "reference": "ed10f4f5804e6bcce15d0491b5d35c10ea7cd9f1" + "reference": "d41ed89d66e09fefc7d743f10e68701627d80a31" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/invoiceninja/UBL_invoice/zipball/ed10f4f5804e6bcce15d0491b5d35c10ea7cd9f1", - "reference": "ed10f4f5804e6bcce15d0491b5d35c10ea7cd9f1", + "url": "https://api.github.com/repos/invoiceninja/UBL_invoice/zipball/d41ed89d66e09fefc7d743f10e68701627d80a31", + "reference": "d41ed89d66e09fefc7d743f10e68701627d80a31", "shasum": "" }, "require": { - "goetas-webservices/xsd2php-runtime": "^0.2.2", "sabre/xml": "^4.0" }, "require-dev": { - "goetas-webservices/xsd2php": "^0.3", "greenter/ubl-validator": "^2", + "phpstan/phpstan": "^1.10", "phpunit/phpunit": "^10" }, "type": "library", @@ -5087,9 +5086,9 @@ "xml invoice" ], "support": { - "source": "https://github.com/invoiceninja/UBL_invoice/tree/v2.0.3" + "source": "https://github.com/invoiceninja/UBL_invoice/tree/v2.1.0" }, - "time": "2024-02-17T06:34:35+00:00" + "time": "2024-04-04T04:33:56+00:00" }, { "name": "jean85/pretty-package-versions", @@ -7179,16 +7178,16 @@ }, { "name": "livewire/livewire", - "version": "v3.4.9", + "version": "v3.4.10", "source": { "type": "git", "url": "https://github.com/livewire/livewire.git", - "reference": "c65b3f0798ab2c9338213ede3588c3cdf4e6fcc0" + "reference": "6f90e2d7f8e80a97a7406c22a0fbc61ca1256ed9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/livewire/livewire/zipball/c65b3f0798ab2c9338213ede3588c3cdf4e6fcc0", - "reference": "c65b3f0798ab2c9338213ede3588c3cdf4e6fcc0", + "url": "https://api.github.com/repos/livewire/livewire/zipball/6f90e2d7f8e80a97a7406c22a0fbc61ca1256ed9", + "reference": "6f90e2d7f8e80a97a7406c22a0fbc61ca1256ed9", "shasum": "" }, "require": { @@ -7198,6 +7197,7 @@ "illuminate/validation": "^10.0|^11.0", "league/mime-type-detection": "^1.9", "php": "^8.1", + "symfony/console": "^6.0|^7.0", "symfony/http-kernel": "^6.2|^7.0" }, "require-dev": { @@ -7205,8 +7205,8 @@ "laravel/framework": "^10.0|^11.0", "laravel/prompts": "^0.1.6", "mockery/mockery": "^1.3.1", - "orchestra/testbench": "8.20.0|^9.0", - "orchestra/testbench-dusk": "8.20.0|^9.0", + "orchestra/testbench": "^8.21.0|^9.0", + "orchestra/testbench-dusk": "^8.24|^9.1", "phpunit/phpunit": "^10.4", "psy/psysh": "^0.11.22|^0.12" }, @@ -7242,7 +7242,7 @@ "description": "A front-end framework for Laravel.", "support": { "issues": "https://github.com/livewire/livewire/issues", - "source": "https://github.com/livewire/livewire/tree/v3.4.9" + "source": "https://github.com/livewire/livewire/tree/v3.4.10" }, "funding": [ { @@ -7250,7 +7250,7 @@ "type": "github" } ], - "time": "2024-03-14T14:03:32+00:00" + "time": "2024-04-02T14:22:50+00:00" }, { "name": "maennchen/zipstream-php", @@ -8867,16 +8867,16 @@ }, { "name": "paragonie/sodium_compat", - "version": "v1.20.0", + "version": "v1.20.1", "source": { "type": "git", "url": "https://github.com/paragonie/sodium_compat.git", - "reference": "e592a3e06d1fa0d43988c7c7d9948ca836f644b6" + "reference": "1840b98d228bdad83869b191d7e51f9bb6624d8d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/paragonie/sodium_compat/zipball/e592a3e06d1fa0d43988c7c7d9948ca836f644b6", - "reference": "e592a3e06d1fa0d43988c7c7d9948ca836f644b6", + "url": "https://api.github.com/repos/paragonie/sodium_compat/zipball/1840b98d228bdad83869b191d7e51f9bb6624d8d", + "reference": "1840b98d228bdad83869b191d7e51f9bb6624d8d", "shasum": "" }, "require": { @@ -8947,9 +8947,9 @@ ], "support": { "issues": "https://github.com/paragonie/sodium_compat/issues", - "source": "https://github.com/paragonie/sodium_compat/tree/v1.20.0" + "source": "https://github.com/paragonie/sodium_compat/tree/v1.20.1" }, - "time": "2023-04-30T00:54:53+00:00" + "time": "2024-04-05T21:00:10+00:00" }, { "name": "payfast/payfast-php-sdk", @@ -9963,16 +9963,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.27.0", + "version": "1.28.0", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "86e4d5a4b036f8f0be1464522f4c6b584c452757" + "reference": "cd06d6b1a1b3c75b0b83f97577869fd85a3cd4fb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/86e4d5a4b036f8f0be1464522f4c6b584c452757", - "reference": "86e4d5a4b036f8f0be1464522f4c6b584c452757", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/cd06d6b1a1b3c75b0b83f97577869fd85a3cd4fb", + "reference": "cd06d6b1a1b3c75b0b83f97577869fd85a3cd4fb", "shasum": "" }, "require": { @@ -10004,9 +10004,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.27.0" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.28.0" }, - "time": "2024-03-21T13:14:53+00:00" + "time": "2024-04-03T18:51:33+00:00" }, { "name": "pragmarx/google2fa", @@ -10584,16 +10584,16 @@ }, { "name": "psy/psysh", - "version": "v0.12.2", + "version": "v0.12.3", "source": { "type": "git", "url": "https://github.com/bobthecow/psysh.git", - "reference": "9185c66c2165bbf4d71de78a69dccf4974f9538d" + "reference": "b6b6cce7d3ee8fbf31843edce5e8f5a72eff4a73" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/bobthecow/psysh/zipball/9185c66c2165bbf4d71de78a69dccf4974f9538d", - "reference": "9185c66c2165bbf4d71de78a69dccf4974f9538d", + "url": "https://api.github.com/repos/bobthecow/psysh/zipball/b6b6cce7d3ee8fbf31843edce5e8f5a72eff4a73", + "reference": "b6b6cce7d3ee8fbf31843edce5e8f5a72eff4a73", "shasum": "" }, "require": { @@ -10657,9 +10657,9 @@ ], "support": { "issues": "https://github.com/bobthecow/psysh/issues", - "source": "https://github.com/bobthecow/psysh/tree/v0.12.2" + "source": "https://github.com/bobthecow/psysh/tree/v0.12.3" }, - "time": "2024-03-17T01:53:00+00:00" + "time": "2024-04-02T15:57:53+00:00" }, { "name": "pusher/pusher-php-server", @@ -12460,16 +12460,16 @@ }, { "name": "symfony/console", - "version": "v6.4.4", + "version": "v6.4.6", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "0d9e4eb5ad413075624378f474c4167ea202de78" + "reference": "a2708a5da5c87d1d0d52937bdeac625df659e11f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/0d9e4eb5ad413075624378f474c4167ea202de78", - "reference": "0d9e4eb5ad413075624378f474c4167ea202de78", + "url": "https://api.github.com/repos/symfony/console/zipball/a2708a5da5c87d1d0d52937bdeac625df659e11f", + "reference": "a2708a5da5c87d1d0d52937bdeac625df659e11f", "shasum": "" }, "require": { @@ -12534,7 +12534,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v6.4.4" + "source": "https://github.com/symfony/console/tree/v6.4.6" }, "funding": [ { @@ -12550,7 +12550,7 @@ "type": "tidelift" } ], - "time": "2024-02-22T20:27:10+00:00" + "time": "2024-03-29T19:07:53+00:00" }, { "name": "symfony/css-selector", @@ -12686,16 +12686,16 @@ }, { "name": "symfony/error-handler", - "version": "v6.4.4", + "version": "v6.4.6", "source": { "type": "git", "url": "https://github.com/symfony/error-handler.git", - "reference": "c725219bdf2afc59423c32793d5019d2a904e13a" + "reference": "64db1c1802e3a4557e37ba33031ac39f452ac5d4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/error-handler/zipball/c725219bdf2afc59423c32793d5019d2a904e13a", - "reference": "c725219bdf2afc59423c32793d5019d2a904e13a", + "url": "https://api.github.com/repos/symfony/error-handler/zipball/64db1c1802e3a4557e37ba33031ac39f452ac5d4", + "reference": "64db1c1802e3a4557e37ba33031ac39f452ac5d4", "shasum": "" }, "require": { @@ -12741,7 +12741,7 @@ "description": "Provides tools to manage errors and ease debugging PHP code", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/error-handler/tree/v6.4.4" + "source": "https://github.com/symfony/error-handler/tree/v6.4.6" }, "funding": [ { @@ -12757,7 +12757,7 @@ "type": "tidelift" } ], - "time": "2024-02-22T20:27:10+00:00" + "time": "2024-03-19T11:56:30+00:00" }, { "name": "symfony/event-dispatcher", @@ -12841,16 +12841,16 @@ }, { "name": "symfony/event-dispatcher-contracts", - "version": "v3.4.0", + "version": "v3.4.2", "source": { "type": "git", "url": "https://github.com/symfony/event-dispatcher-contracts.git", - "reference": "a76aed96a42d2b521153fb382d418e30d18b59df" + "reference": "4e64b49bf370ade88e567de29465762e316e4224" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/event-dispatcher-contracts/zipball/a76aed96a42d2b521153fb382d418e30d18b59df", - "reference": "a76aed96a42d2b521153fb382d418e30d18b59df", + "url": "https://api.github.com/repos/symfony/event-dispatcher-contracts/zipball/4e64b49bf370ade88e567de29465762e316e4224", + "reference": "4e64b49bf370ade88e567de29465762e316e4224", "shasum": "" }, "require": { @@ -12897,7 +12897,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/event-dispatcher-contracts/tree/v3.4.0" + "source": "https://github.com/symfony/event-dispatcher-contracts/tree/v3.4.2" }, "funding": [ { @@ -12913,20 +12913,20 @@ "type": "tidelift" } ], - "time": "2023-05-23T14:45:45+00:00" + "time": "2024-01-23T14:51:35+00:00" }, { "name": "symfony/filesystem", - "version": "v6.4.3", + "version": "v6.4.6", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "7f3b1755eb49297a0827a7575d5d2b2fd11cc9fb" + "reference": "9919b5509ada52cc7f66f9a35c86a4a29955c9d3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/7f3b1755eb49297a0827a7575d5d2b2fd11cc9fb", - "reference": "7f3b1755eb49297a0827a7575d5d2b2fd11cc9fb", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/9919b5509ada52cc7f66f9a35c86a4a29955c9d3", + "reference": "9919b5509ada52cc7f66f9a35c86a4a29955c9d3", "shasum": "" }, "require": { @@ -12960,7 +12960,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v6.4.3" + "source": "https://github.com/symfony/filesystem/tree/v6.4.6" }, "funding": [ { @@ -12976,7 +12976,7 @@ "type": "tidelift" } ], - "time": "2024-01-23T14:51:35+00:00" + "time": "2024-03-21T19:36:20+00:00" }, { "name": "symfony/finder", @@ -13044,23 +13044,23 @@ }, { "name": "symfony/http-client", - "version": "v6.4.5", + "version": "v6.4.6", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "f3c86a60a3615f466333a11fd42010d4382a82c7" + "reference": "6a46c0ea9b099f9a5132d560a51833ffcbd5b0d9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/f3c86a60a3615f466333a11fd42010d4382a82c7", - "reference": "f3c86a60a3615f466333a11fd42010d4382a82c7", + "url": "https://api.github.com/repos/symfony/http-client/zipball/6a46c0ea9b099f9a5132d560a51833ffcbd5b0d9", + "reference": "6a46c0ea9b099f9a5132d560a51833ffcbd5b0d9", "shasum": "" }, "require": { "php": ">=8.1", "psr/log": "^1|^2|^3", "symfony/deprecation-contracts": "^2.5|^3", - "symfony/http-client-contracts": "^3", + "symfony/http-client-contracts": "^3.4.1", "symfony/service-contracts": "^2.5|^3" }, "conflict": { @@ -13078,7 +13078,7 @@ "amphp/http-client": "^4.2.1", "amphp/http-tunnel": "^1.0", "amphp/socket": "^1.1", - "guzzlehttp/promises": "^1.4", + "guzzlehttp/promises": "^1.4|^2.0", "nyholm/psr7": "^1.0", "php-http/httplug": "^1.0|^2.0", "psr/http-client": "^1.0", @@ -13117,7 +13117,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v6.4.5" + "source": "https://github.com/symfony/http-client/tree/v6.4.6" }, "funding": [ { @@ -13133,20 +13133,20 @@ "type": "tidelift" } ], - "time": "2024-03-02T12:45:30+00:00" + "time": "2024-04-01T20:35:50+00:00" }, { "name": "symfony/http-client-contracts", - "version": "v3.4.0", + "version": "v3.4.2", "source": { "type": "git", "url": "https://github.com/symfony/http-client-contracts.git", - "reference": "1ee70e699b41909c209a0c930f11034b93578654" + "reference": "b6b5c876b3a4ed74460e2c5ac53bbce2f12e2a7e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/1ee70e699b41909c209a0c930f11034b93578654", - "reference": "1ee70e699b41909c209a0c930f11034b93578654", + "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/b6b5c876b3a4ed74460e2c5ac53bbce2f12e2a7e", + "reference": "b6b5c876b3a4ed74460e2c5ac53bbce2f12e2a7e", "shasum": "" }, "require": { @@ -13195,7 +13195,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/http-client-contracts/tree/v3.4.0" + "source": "https://github.com/symfony/http-client-contracts/tree/v3.4.2" }, "funding": [ { @@ -13211,7 +13211,7 @@ "type": "tidelift" } ], - "time": "2023-07-30T20:28:31+00:00" + "time": "2024-04-01T18:51:09+00:00" }, { "name": "symfony/http-foundation", @@ -13292,16 +13292,16 @@ }, { "name": "symfony/http-kernel", - "version": "v6.4.5", + "version": "v6.4.6", "source": { "type": "git", "url": "https://github.com/symfony/http-kernel.git", - "reference": "f6947cb939d8efee137797382cb4db1af653ef75" + "reference": "060038863743fd0cd982be06acecccf246d35653" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-kernel/zipball/f6947cb939d8efee137797382cb4db1af653ef75", - "reference": "f6947cb939d8efee137797382cb4db1af653ef75", + "url": "https://api.github.com/repos/symfony/http-kernel/zipball/060038863743fd0cd982be06acecccf246d35653", + "reference": "060038863743fd0cd982be06acecccf246d35653", "shasum": "" }, "require": { @@ -13385,7 +13385,7 @@ "description": "Provides a structured process for converting a Request into a Response", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/http-kernel/tree/v6.4.5" + "source": "https://github.com/symfony/http-kernel/tree/v6.4.6" }, "funding": [ { @@ -13401,7 +13401,7 @@ "type": "tidelift" } ], - "time": "2024-03-04T21:00:47+00:00" + "time": "2024-04-03T06:09:15+00:00" }, { "name": "symfony/intl", @@ -13487,16 +13487,16 @@ }, { "name": "symfony/mailer", - "version": "v6.4.4", + "version": "v6.4.6", "source": { "type": "git", "url": "https://github.com/symfony/mailer.git", - "reference": "791c5d31a8204cf3db0c66faab70282307f4376b" + "reference": "677f34a6f4b4559e08acf73ae0aec460479e5859" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/mailer/zipball/791c5d31a8204cf3db0c66faab70282307f4376b", - "reference": "791c5d31a8204cf3db0c66faab70282307f4376b", + "url": "https://api.github.com/repos/symfony/mailer/zipball/677f34a6f4b4559e08acf73ae0aec460479e5859", + "reference": "677f34a6f4b4559e08acf73ae0aec460479e5859", "shasum": "" }, "require": { @@ -13547,7 +13547,7 @@ "description": "Helps sending emails", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/mailer/tree/v6.4.4" + "source": "https://github.com/symfony/mailer/tree/v6.4.6" }, "funding": [ { @@ -13563,7 +13563,7 @@ "type": "tidelift" } ], - "time": "2024-02-03T21:33:47+00:00" + "time": "2024-03-27T21:14:17+00:00" }, { "name": "symfony/mailgun-mailer", @@ -13636,16 +13636,16 @@ }, { "name": "symfony/mime", - "version": "v6.4.3", + "version": "v6.4.6", "source": { "type": "git", "url": "https://github.com/symfony/mime.git", - "reference": "5017e0a9398c77090b7694be46f20eb796262a34" + "reference": "14762b86918823cb42e3558cdcca62e58b5227fe" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/mime/zipball/5017e0a9398c77090b7694be46f20eb796262a34", - "reference": "5017e0a9398c77090b7694be46f20eb796262a34", + "url": "https://api.github.com/repos/symfony/mime/zipball/14762b86918823cb42e3558cdcca62e58b5227fe", + "reference": "14762b86918823cb42e3558cdcca62e58b5227fe", "shasum": "" }, "require": { @@ -13666,6 +13666,7 @@ "league/html-to-markdown": "^5.0", "phpdocumentor/reflection-docblock": "^3.0|^4.0|^5.0", "symfony/dependency-injection": "^5.4|^6.0|^7.0", + "symfony/process": "^5.4|^6.4|^7.0", "symfony/property-access": "^5.4|^6.0|^7.0", "symfony/property-info": "^5.4|^6.0|^7.0", "symfony/serializer": "^6.3.2|^7.0" @@ -13700,7 +13701,7 @@ "mime-type" ], "support": { - "source": "https://github.com/symfony/mime/tree/v6.4.3" + "source": "https://github.com/symfony/mime/tree/v6.4.6" }, "funding": [ { @@ -13716,7 +13717,7 @@ "type": "tidelift" } ], - "time": "2024-01-30T08:32:12+00:00" + "time": "2024-03-21T19:36:20+00:00" }, { "name": "symfony/options-resolver", @@ -14802,16 +14803,16 @@ }, { "name": "symfony/routing", - "version": "v6.4.5", + "version": "v6.4.6", "source": { "type": "git", "url": "https://github.com/symfony/routing.git", - "reference": "7fe30068e207d9c31c0138501ab40358eb2d49a4" + "reference": "f2591fd1f8c6e3734656b5d6b3829e8bf81f507c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/routing/zipball/7fe30068e207d9c31c0138501ab40358eb2d49a4", - "reference": "7fe30068e207d9c31c0138501ab40358eb2d49a4", + "url": "https://api.github.com/repos/symfony/routing/zipball/f2591fd1f8c6e3734656b5d6b3829e8bf81f507c", + "reference": "f2591fd1f8c6e3734656b5d6b3829e8bf81f507c", "shasum": "" }, "require": { @@ -14865,7 +14866,7 @@ "url" ], "support": { - "source": "https://github.com/symfony/routing/tree/v6.4.5" + "source": "https://github.com/symfony/routing/tree/v6.4.6" }, "funding": [ { @@ -14881,20 +14882,20 @@ "type": "tidelift" } ], - "time": "2024-02-27T12:33:30+00:00" + "time": "2024-03-28T13:28:49+00:00" }, { "name": "symfony/service-contracts", - "version": "v3.4.1", + "version": "v3.4.2", "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "fe07cbc8d837f60caf7018068e350cc5163681a0" + "reference": "11bbf19a0fb7b36345861e85c5768844c552906e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/fe07cbc8d837f60caf7018068e350cc5163681a0", - "reference": "fe07cbc8d837f60caf7018068e350cc5163681a0", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/11bbf19a0fb7b36345861e85c5768844c552906e", + "reference": "11bbf19a0fb7b36345861e85c5768844c552906e", "shasum": "" }, "require": { @@ -14947,7 +14948,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.4.1" + "source": "https://github.com/symfony/service-contracts/tree/v3.4.2" }, "funding": [ { @@ -14963,7 +14964,7 @@ "type": "tidelift" } ], - "time": "2023-12-26T14:02:43+00:00" + "time": "2023-12-19T21:51:00+00:00" }, { "name": "symfony/string", @@ -15148,16 +15149,16 @@ }, { "name": "symfony/translation-contracts", - "version": "v3.4.1", + "version": "v3.4.2", "source": { "type": "git", "url": "https://github.com/symfony/translation-contracts.git", - "reference": "06450585bf65e978026bda220cdebca3f867fde7" + "reference": "43810bdb2ddb5400e5c5e778e27b210a0ca83b6b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/translation-contracts/zipball/06450585bf65e978026bda220cdebca3f867fde7", - "reference": "06450585bf65e978026bda220cdebca3f867fde7", + "url": "https://api.github.com/repos/symfony/translation-contracts/zipball/43810bdb2ddb5400e5c5e778e27b210a0ca83b6b", + "reference": "43810bdb2ddb5400e5c5e778e27b210a0ca83b6b", "shasum": "" }, "require": { @@ -15206,7 +15207,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/translation-contracts/tree/v3.4.1" + "source": "https://github.com/symfony/translation-contracts/tree/v3.4.2" }, "funding": [ { @@ -15222,7 +15223,7 @@ "type": "tidelift" } ], - "time": "2023-12-26T14:02:43+00:00" + "time": "2024-01-23T14:51:35+00:00" }, { "name": "symfony/uid", @@ -15300,16 +15301,16 @@ }, { "name": "symfony/validator", - "version": "v6.4.4", + "version": "v6.4.6", "source": { "type": "git", "url": "https://github.com/symfony/validator.git", - "reference": "1cf92edc9a94d16275efef949fa6748d11cc8f47" + "reference": "ca1d78e8677e966e307a63799677b64b194d735d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/validator/zipball/1cf92edc9a94d16275efef949fa6748d11cc8f47", - "reference": "1cf92edc9a94d16275efef949fa6748d11cc8f47", + "url": "https://api.github.com/repos/symfony/validator/zipball/ca1d78e8677e966e307a63799677b64b194d735d", + "reference": "ca1d78e8677e966e307a63799677b64b194d735d", "shasum": "" }, "require": { @@ -15376,7 +15377,7 @@ "description": "Provides tools to validate values", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/validator/tree/v6.4.4" + "source": "https://github.com/symfony/validator/tree/v6.4.6" }, "funding": [ { @@ -15392,20 +15393,20 @@ "type": "tidelift" } ], - "time": "2024-02-22T20:27:10+00:00" + "time": "2024-03-27T22:00:14+00:00" }, { "name": "symfony/var-dumper", - "version": "v6.4.4", + "version": "v6.4.6", "source": { "type": "git", "url": "https://github.com/symfony/var-dumper.git", - "reference": "b439823f04c98b84d4366c79507e9da6230944b1" + "reference": "95bd2706a97fb875185b51ecaa6112ec184233d4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/var-dumper/zipball/b439823f04c98b84d4366c79507e9da6230944b1", - "reference": "b439823f04c98b84d4366c79507e9da6230944b1", + "url": "https://api.github.com/repos/symfony/var-dumper/zipball/95bd2706a97fb875185b51ecaa6112ec184233d4", + "reference": "95bd2706a97fb875185b51ecaa6112ec184233d4", "shasum": "" }, "require": { @@ -15461,7 +15462,7 @@ "dump" ], "support": { - "source": "https://github.com/symfony/var-dumper/tree/v6.4.4" + "source": "https://github.com/symfony/var-dumper/tree/v6.4.6" }, "funding": [ { @@ -15477,7 +15478,7 @@ "type": "tidelift" } ], - "time": "2024-02-15T11:23:52+00:00" + "time": "2024-03-19T11:56:30+00:00" }, { "name": "symfony/yaml", @@ -16231,23 +16232,23 @@ "packages-dev": [ { "name": "barryvdh/laravel-debugbar", - "version": "v3.12.4", + "version": "v3.13.3", "source": { "type": "git", "url": "https://github.com/barryvdh/laravel-debugbar.git", - "reference": "e7a9a217512d8f1d07448fd241ce2ac0922ddc2c" + "reference": "241e9bddb04ab42a04a5fe8b2b9654374c864229" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/barryvdh/laravel-debugbar/zipball/e7a9a217512d8f1d07448fd241ce2ac0922ddc2c", - "reference": "e7a9a217512d8f1d07448fd241ce2ac0922ddc2c", + "url": "https://api.github.com/repos/barryvdh/laravel-debugbar/zipball/241e9bddb04ab42a04a5fe8b2b9654374c864229", + "reference": "241e9bddb04ab42a04a5fe8b2b9654374c864229", "shasum": "" }, "require": { "illuminate/routing": "^9|^10|^11", "illuminate/session": "^9|^10|^11", "illuminate/support": "^9|^10|^11", - "maximebf/debugbar": "~1.21.0", + "maximebf/debugbar": "~1.22.0", "php": "^8.0", "symfony/finder": "^6|^7" }, @@ -16260,7 +16261,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "3.10-dev" + "dev-master": "3.13-dev" }, "laravel": { "providers": [ @@ -16299,7 +16300,7 @@ ], "support": { "issues": "https://github.com/barryvdh/laravel-debugbar/issues", - "source": "https://github.com/barryvdh/laravel-debugbar/tree/v3.12.4" + "source": "https://github.com/barryvdh/laravel-debugbar/tree/v3.13.3" }, "funding": [ { @@ -16311,7 +16312,7 @@ "type": "github" } ], - "time": "2024-04-01T09:14:15+00:00" + "time": "2024-04-04T02:42:49+00:00" }, { "name": "barryvdh/laravel-ide-helper", @@ -17342,25 +17343,27 @@ }, { "name": "maximebf/debugbar", - "version": "v1.21.3", + "version": "v1.22.3", "source": { "type": "git", "url": "https://github.com/maximebf/php-debugbar.git", - "reference": "0b407703b08ea0cf6ebc61e267cc96ff7000911b" + "reference": "7aa9a27a0b1158ed5ad4e7175e8d3aee9a818b96" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/maximebf/php-debugbar/zipball/0b407703b08ea0cf6ebc61e267cc96ff7000911b", - "reference": "0b407703b08ea0cf6ebc61e267cc96ff7000911b", + "url": "https://api.github.com/repos/maximebf/php-debugbar/zipball/7aa9a27a0b1158ed5ad4e7175e8d3aee9a818b96", + "reference": "7aa9a27a0b1158ed5ad4e7175e8d3aee9a818b96", "shasum": "" }, "require": { - "php": "^7.1|^8", + "php": "^7.2|^8", "psr/log": "^1|^2|^3", "symfony/var-dumper": "^4|^5|^6|^7" }, "require-dev": { - "phpunit/phpunit": ">=7.5.20 <10.0", + "dbrekelmans/bdi": "^1", + "phpunit/phpunit": "^8|^9", + "symfony/panther": "^1|^2.1", "twig/twig": "^1.38|^2.7|^3.0" }, "suggest": { @@ -17371,7 +17374,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "1.21-dev" + "dev-master": "1.22-dev" } }, "autoload": { @@ -17402,9 +17405,9 @@ ], "support": { "issues": "https://github.com/maximebf/php-debugbar/issues", - "source": "https://github.com/maximebf/php-debugbar/tree/v1.21.3" + "source": "https://github.com/maximebf/php-debugbar/tree/v1.22.3" }, - "time": "2024-03-12T14:23:07+00:00" + "time": "2024-04-03T19:39:26+00:00" }, { "name": "mockery/mockery", @@ -18235,16 +18238,16 @@ }, { "name": "phpunit/phpunit", - "version": "10.5.16", + "version": "10.5.17", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "18f8d4a5f52b61fdd9370aaae3167daa0eeb69cd" + "reference": "c1f736a473d21957ead7e94fcc029f571895abf5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/18f8d4a5f52b61fdd9370aaae3167daa0eeb69cd", - "reference": "18f8d4a5f52b61fdd9370aaae3167daa0eeb69cd", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c1f736a473d21957ead7e94fcc029f571895abf5", + "reference": "c1f736a473d21957ead7e94fcc029f571895abf5", "shasum": "" }, "require": { @@ -18316,7 +18319,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.16" + "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.17" }, "funding": [ { @@ -18332,7 +18335,7 @@ "type": "tidelift" } ], - "time": "2024-03-28T10:08:10+00:00" + "time": "2024-04-05T04:39:01+00:00" }, { "name": "sebastian/cli-parser", @@ -19466,16 +19469,16 @@ }, { "name": "spatie/laravel-ignition", - "version": "2.5.0", + "version": "2.5.1", "source": { "type": "git", "url": "https://github.com/spatie/laravel-ignition.git", - "reference": "e23f4e8ce6644dc3d68b9d8a0aed3beaca0d6ada" + "reference": "0c864b3cbd66ce67a2096c5f743e07ce8f1d6ab9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/spatie/laravel-ignition/zipball/e23f4e8ce6644dc3d68b9d8a0aed3beaca0d6ada", - "reference": "e23f4e8ce6644dc3d68b9d8a0aed3beaca0d6ada", + "url": "https://api.github.com/repos/spatie/laravel-ignition/zipball/0c864b3cbd66ce67a2096c5f743e07ce8f1d6ab9", + "reference": "0c864b3cbd66ce67a2096c5f743e07ce8f1d6ab9", "shasum": "" }, "require": { @@ -19554,7 +19557,7 @@ "type": "github" } ], - "time": "2024-03-29T14:14:55+00:00" + "time": "2024-04-02T06:30:22+00:00" }, { "name": "spaze/phpstan-stripe", diff --git a/config/ninja.php b/config/ninja.php index e07134b15e17..3a4ea9c4ebc8 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -236,7 +236,6 @@ return [ 'webhook_id' => env('PAYPAL_WEBHOOK_ID', null), ], 'inbound_mailbox' => [ - 'expense_mailbox_template' => env('EXPENSE_MAILBOX_TEMPLATE', null), 'expense_mailbox_endings' => env('EXPENSE_MAILBOX_ENDINGS', '@expense.invoicing.co'), 'inbound_webhook_key' => env('INBOUND_WEBHOOK_KEY', null) ], From 7b29ee8ebf449aa0ee69903086b2f95e192f057c Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 7 Apr 2024 16:30:04 +0200 Subject: [PATCH 065/119] remove unused variables from ExpenseMailbox check --- app/Http/Requests/Company/StoreCompanyRequest.php | 2 +- app/Http/Requests/Company/UpdateCompanyRequest.php | 2 +- app/Http/ValidationRules/Company/ValidExpenseMailbox.php | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/Http/Requests/Company/StoreCompanyRequest.php b/app/Http/Requests/Company/StoreCompanyRequest.php index ff09dbfacc07..e9b5c50b8a43 100644 --- a/app/Http/Requests/Company/StoreCompanyRequest.php +++ b/app/Http/Requests/Company/StoreCompanyRequest.php @@ -57,7 +57,7 @@ class StoreCompanyRequest extends Request } } - $rules['expense_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); + $rules['expense_mailbox'] = new ValidExpenseMailbox(); $rules['smtp_host'] = 'sometimes|string|nullable'; $rules['smtp_port'] = 'sometimes|integer|nullable'; diff --git a/app/Http/Requests/Company/UpdateCompanyRequest.php b/app/Http/Requests/Company/UpdateCompanyRequest.php index e4b24961e85b..06254af3c4eb 100644 --- a/app/Http/Requests/Company/UpdateCompanyRequest.php +++ b/app/Http/Requests/Company/UpdateCompanyRequest.php @@ -74,7 +74,7 @@ class UpdateCompanyRequest extends Request $rules['subdomain'] = ['nullable', 'regex:/^[a-zA-Z0-9.-]+[a-zA-Z0-9]$/', new ValidSubdomain()]; } - $rules['expense_mailbox'] = new ValidExpenseMailbox($this->company->key, $this->company->account->isPaid() && $this->company->account->plan == 'enterprise'); // @turbo124 check if this is right + $rules['expense_mailbox'] = new ValidExpenseMailbox(); return $rules; } diff --git a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php index a113c2fc0ea5..d9ab3727cb13 100644 --- a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php +++ b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php @@ -22,15 +22,13 @@ class ValidExpenseMailbox implements Rule { private $validated_schema = false; - private $company_key = false; private $isEnterprise = false; private array $endings; private bool $hasCompanyKey; private array $enterprise_endings; - public function __construct(string $company_key, bool $isEnterprise = false) + public function __construct() { - $this->company_key = $company_key; $this->endings = explode(",", config('ninja.inbound_mailbox.expense_mailbox_endings')); } From 9aa97d7ddab5b9f4f1cd85fea95b78d16a81c3d1 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 7 Apr 2024 16:35:57 +0200 Subject: [PATCH 066/119] spelling mistake --- app/Http/Controllers/BrevoController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Http/Controllers/BrevoController.php b/app/Http/Controllers/BrevoController.php index ac6d9265cc9a..64d06fb50024 100644 --- a/app/Http/Controllers/BrevoController.php +++ b/app/Http/Controllers/BrevoController.php @@ -37,7 +37,7 @@ class BrevoController extends BaseController * path="/api/v1/brevo_webhook", * operationId="brevoWebhook", * tags={"brevo"}, - * summary="Processing webhooks from PostMark", + * summary="Processing webhooks from Brevo", * description="Adds an credit to the system", * @OA\Parameter(ref="#/components/parameters/X-API-TOKEN"), * @OA\Parameter(ref="#/components/parameters/X-Requested-With"), From f530d95f59d5aee80a62cf9c577e855783c1d2d8 Mon Sep 17 00:00:00 2001 From: paulwer Date: Tue, 23 Apr 2024 22:40:43 +0200 Subject: [PATCH 067/119] update composer --- composer.lock | 438 +++++++++++++++++++++----------------------------- 1 file changed, 183 insertions(+), 255 deletions(-) diff --git a/composer.lock b/composer.lock index 48673499f039..7acbdb55dcc1 100644 --- a/composer.lock +++ b/composer.lock @@ -100,16 +100,16 @@ }, { "name": "amphp/amp", - "version": "v3.0.0", + "version": "v3.0.1", "source": { "type": "git", "url": "https://github.com/amphp/amp.git", - "reference": "aaf0ec1d5a2c20b523258995a10e80c1fb765871" + "reference": "ff63f10210adb6e83335e0e25522bac5cd7dc4e2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/amp/zipball/aaf0ec1d5a2c20b523258995a10e80c1fb765871", - "reference": "aaf0ec1d5a2c20b523258995a10e80c1fb765871", + "url": "https://api.github.com/repos/amphp/amp/zipball/ff63f10210adb6e83335e0e25522bac5cd7dc4e2", + "reference": "ff63f10210adb6e83335e0e25522bac5cd7dc4e2", "shasum": "" }, "require": { @@ -119,7 +119,7 @@ "require-dev": { "amphp/php-cs-fixer-config": "^2", "phpunit/phpunit": "^9", - "psalm/phar": "^4.13" + "psalm/phar": "5.23.1" }, "type": "library", "autoload": { @@ -169,7 +169,7 @@ ], "support": { "issues": "https://github.com/amphp/amp/issues", - "source": "https://github.com/amphp/amp/tree/v3.0.0" + "source": "https://github.com/amphp/amp/tree/v3.0.1" }, "funding": [ { @@ -177,7 +177,7 @@ "type": "github" } ], - "time": "2022-12-18T16:52:44+00:00" + "time": "2024-04-18T15:24:36+00:00" }, { "name": "amphp/byte-stream", @@ -256,16 +256,16 @@ }, { "name": "amphp/cache", - "version": "v2.0.0", + "version": "v2.0.1", "source": { "type": "git", "url": "https://github.com/amphp/cache.git", - "reference": "218bb3888d380eb9dd926cd06f803573c84391d3" + "reference": "46912e387e6aa94933b61ea1ead9cf7540b7797c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/cache/zipball/218bb3888d380eb9dd926cd06f803573c84391d3", - "reference": "218bb3888d380eb9dd926cd06f803573c84391d3", + "url": "https://api.github.com/repos/amphp/cache/zipball/46912e387e6aa94933b61ea1ead9cf7540b7797c", + "reference": "46912e387e6aa94933b61ea1ead9cf7540b7797c", "shasum": "" }, "require": { @@ -309,7 +309,7 @@ "homepage": "https://amphp.org/cache", "support": { "issues": "https://github.com/amphp/cache/issues", - "source": "https://github.com/amphp/cache/tree/v2.0.0" + "source": "https://github.com/amphp/cache/tree/v2.0.1" }, "funding": [ { @@ -317,20 +317,20 @@ "type": "github" } ], - "time": "2023-01-09T21:04:12+00:00" + "time": "2024-04-19T03:38:06+00:00" }, { "name": "amphp/dns", - "version": "v2.1.1", + "version": "v2.1.2", "source": { "type": "git", "url": "https://github.com/amphp/dns.git", - "reference": "3e3f413fbbaacd9632b1612941b363fa26a72e52" + "reference": "04c88e67bef804203df934703bd422ea72f46b0e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/dns/zipball/3e3f413fbbaacd9632b1612941b363fa26a72e52", - "reference": "3e3f413fbbaacd9632b1612941b363fa26a72e52", + "url": "https://api.github.com/repos/amphp/dns/zipball/04c88e67bef804203df934703bd422ea72f46b0e", + "reference": "04c88e67bef804203df934703bd422ea72f46b0e", "shasum": "" }, "require": { @@ -397,7 +397,7 @@ ], "support": { "issues": "https://github.com/amphp/dns/issues", - "source": "https://github.com/amphp/dns/tree/v2.1.1" + "source": "https://github.com/amphp/dns/tree/v2.1.2" }, "funding": [ { @@ -405,7 +405,7 @@ "type": "github" } ], - "time": "2024-01-30T23:25:30+00:00" + "time": "2024-04-19T03:49:29+00:00" }, { "name": "amphp/parallel", @@ -622,16 +622,16 @@ }, { "name": "amphp/process", - "version": "v2.0.2", + "version": "v2.0.3", "source": { "type": "git", "url": "https://github.com/amphp/process.git", - "reference": "a79dc87100be857db2c4bbfd5369585a6d1e658c" + "reference": "52e08c09dec7511d5fbc1fb00d3e4e79fc77d58d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/process/zipball/a79dc87100be857db2c4bbfd5369585a6d1e658c", - "reference": "a79dc87100be857db2c4bbfd5369585a6d1e658c", + "url": "https://api.github.com/repos/amphp/process/zipball/52e08c09dec7511d5fbc1fb00d3e4e79fc77d58d", + "reference": "52e08c09dec7511d5fbc1fb00d3e4e79fc77d58d", "shasum": "" }, "require": { @@ -678,7 +678,7 @@ "homepage": "https://amphp.org/process", "support": { "issues": "https://github.com/amphp/process/issues", - "source": "https://github.com/amphp/process/tree/v2.0.2" + "source": "https://github.com/amphp/process/tree/v2.0.3" }, "funding": [ { @@ -686,7 +686,7 @@ "type": "github" } ], - "time": "2024-02-13T20:38:21+00:00" + "time": "2024-04-19T03:13:44+00:00" }, { "name": "amphp/serialization", @@ -748,16 +748,16 @@ }, { "name": "amphp/socket", - "version": "v2.3.0", + "version": "v2.3.1", "source": { "type": "git", "url": "https://github.com/amphp/socket.git", - "reference": "acc0a2f65ab498025ba5641f7cce499c4b1ed4b5" + "reference": "58e0422221825b79681b72c50c47a930be7bf1e1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/socket/zipball/acc0a2f65ab498025ba5641f7cce499c4b1ed4b5", - "reference": "acc0a2f65ab498025ba5641f7cce499c4b1ed4b5", + "url": "https://api.github.com/repos/amphp/socket/zipball/58e0422221825b79681b72c50c47a930be7bf1e1", + "reference": "58e0422221825b79681b72c50c47a930be7bf1e1", "shasum": "" }, "require": { @@ -820,7 +820,7 @@ ], "support": { "issues": "https://github.com/amphp/socket/issues", - "source": "https://github.com/amphp/socket/tree/v2.3.0" + "source": "https://github.com/amphp/socket/tree/v2.3.1" }, "funding": [ { @@ -828,7 +828,7 @@ "type": "github" } ], - "time": "2024-03-19T20:01:53+00:00" + "time": "2024-04-21T14:33:03+00:00" }, { "name": "amphp/sync", @@ -1330,16 +1330,16 @@ }, { "name": "aws/aws-crt-php", - "version": "v1.2.4", + "version": "v1.2.5", "source": { "type": "git", "url": "https://github.com/awslabs/aws-crt-php.git", - "reference": "eb0c6e4e142224a10b08f49ebf87f32611d162b2" + "reference": "0ea1f04ec5aa9f049f97e012d1ed63b76834a31b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/awslabs/aws-crt-php/zipball/eb0c6e4e142224a10b08f49ebf87f32611d162b2", - "reference": "eb0c6e4e142224a10b08f49ebf87f32611d162b2", + "url": "https://api.github.com/repos/awslabs/aws-crt-php/zipball/0ea1f04ec5aa9f049f97e012d1ed63b76834a31b", + "reference": "0ea1f04ec5aa9f049f97e012d1ed63b76834a31b", "shasum": "" }, "require": { @@ -1378,35 +1378,22 @@ ], "support": { "issues": "https://github.com/awslabs/aws-crt-php/issues", - "source": "https://github.com/awslabs/aws-crt-php/tree/v1.2.4" + "source": "https://github.com/awslabs/aws-crt-php/tree/v1.2.5" }, - "time": "2023-11-08T00:42:13+00:00" + "time": "2024-04-19T21:30:56+00:00" }, { "name": "aws/aws-sdk-php", -<<<<<<< HEAD - "version": "3.303.4", + "version": "3.305.1", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "9ae5429f7699701cb158780cd287d1549f45ad32" + "reference": "3af1c6925b95a0f4303a1859dd56aa8374560c42" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/9ae5429f7699701cb158780cd287d1549f45ad32", - "reference": "9ae5429f7699701cb158780cd287d1549f45ad32", -======= - "version": "3.304.2", - "source": { - "type": "git", - "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "2435079c3e1a08148d955de15ec090018114f35a" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/2435079c3e1a08148d955de15ec090018114f35a", - "reference": "2435079c3e1a08148d955de15ec090018114f35a", ->>>>>>> 51ab86a2dde7bff11fc6bd7a2970463cf980a86d + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/3af1c6925b95a0f4303a1859dd56aa8374560c42", + "reference": "3af1c6925b95a0f4303a1859dd56aa8374560c42", "shasum": "" }, "require": { @@ -1486,15 +1473,9 @@ "support": { "forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80", "issues": "https://github.com/aws/aws-sdk-php/issues", -<<<<<<< HEAD - "source": "https://github.com/aws/aws-sdk-php/tree/3.303.4" + "source": "https://github.com/aws/aws-sdk-php/tree/3.305.1" }, - "time": "2024-04-05T18:04:15+00:00" -======= - "source": "https://github.com/aws/aws-sdk-php/tree/3.304.2" - }, - "time": "2024-04-10T18:05:32+00:00" ->>>>>>> 51ab86a2dde7bff11fc6bd7a2970463cf980a86d + "time": "2024-04-23T18:10:07+00:00" }, { "name": "bacon/bacon-qr-code", @@ -2037,21 +2018,21 @@ }, { "name": "daverandom/libdns", - "version": "v2.0.3", + "version": "v2.1.0", "source": { "type": "git", "url": "https://github.com/DaveRandom/LibDNS.git", - "reference": "42c2d700d1178c9f9e78664793463f7f1aea248c" + "reference": "b84c94e8fe6b7ee4aecfe121bfe3b6177d303c8a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/DaveRandom/LibDNS/zipball/42c2d700d1178c9f9e78664793463f7f1aea248c", - "reference": "42c2d700d1178c9f9e78664793463f7f1aea248c", + "url": "https://api.github.com/repos/DaveRandom/LibDNS/zipball/b84c94e8fe6b7ee4aecfe121bfe3b6177d303c8a", + "reference": "b84c94e8fe6b7ee4aecfe121bfe3b6177d303c8a", "shasum": "" }, "require": { "ext-ctype": "*", - "php": ">=7.0" + "php": ">=7.1" }, "suggest": { "ext-intl": "Required for IDN support" @@ -2075,9 +2056,9 @@ ], "support": { "issues": "https://github.com/DaveRandom/LibDNS/issues", - "source": "https://github.com/DaveRandom/LibDNS/tree/v2.0.3" + "source": "https://github.com/DaveRandom/LibDNS/tree/v2.1.0" }, - "time": "2022-09-20T18:15:38+00:00" + "time": "2024-04-12T12:12:48+00:00" }, { "name": "dflydev/apache-mime-types", @@ -3432,22 +3413,22 @@ }, { "name": "goetas-webservices/xsd2php-runtime", - "version": "v0.2.16", + "version": "v0.2.17", "source": { "type": "git", "url": "https://github.com/goetas-webservices/xsd2php-runtime.git", - "reference": "4a24dc8ead032dae6cf82168a46702a31f7db42f" + "reference": "be15c48cda6adfab82e180a69dfa1937e208cfe1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/goetas-webservices/xsd2php-runtime/zipball/4a24dc8ead032dae6cf82168a46702a31f7db42f", - "reference": "4a24dc8ead032dae6cf82168a46702a31f7db42f", + "url": "https://api.github.com/repos/goetas-webservices/xsd2php-runtime/zipball/be15c48cda6adfab82e180a69dfa1937e208cfe1", + "reference": "be15c48cda6adfab82e180a69dfa1937e208cfe1", "shasum": "" }, "require": { "jms/serializer": "^1.2|^2.0|^3.0", "php": ">=7.1", - "symfony/yaml": "^2.2|^3.0|^4.0|^5.0|^6.0" + "symfony/yaml": "^2.2|^3.0|^4.0|^5.0|^6.0|^7.0" }, "conflict": { "jms/serializer": "1.4.1|1.6.1|1.6.2" @@ -3486,9 +3467,9 @@ ], "support": { "issues": "https://github.com/goetas-webservices/xsd2php-runtime/issues", - "source": "https://github.com/goetas-webservices/xsd2php-runtime/tree/v0.2.16" + "source": "https://github.com/goetas-webservices/xsd2php-runtime/tree/v0.2.17" }, - "time": "2022-02-04T09:31:42+00:00" + "time": "2024-04-12T22:55:31+00:00" }, { "name": "google/apiclient", @@ -3561,29 +3542,16 @@ }, { "name": "google/apiclient-services", -<<<<<<< HEAD - "version": "v0.342.0", + "version": "v0.345.0", "source": { "type": "git", "url": "https://github.com/googleapis/google-api-php-client-services.git", - "reference": "b75a249bd9abf0241822fa6d9d9b396583f5003f" + "reference": "a975dae4c06d304b020eda3f53e7c3402b32132d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/b75a249bd9abf0241822fa6d9d9b396583f5003f", - "reference": "b75a249bd9abf0241822fa6d9d9b396583f5003f", -======= - "version": "v0.343.0", - "source": { - "type": "git", - "url": "https://github.com/googleapis/google-api-php-client-services.git", - "reference": "8cb869e5c413b0e9d4aeada3d83df5e2ce388154" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/8cb869e5c413b0e9d4aeada3d83df5e2ce388154", - "reference": "8cb869e5c413b0e9d4aeada3d83df5e2ce388154", ->>>>>>> 51ab86a2dde7bff11fc6bd7a2970463cf980a86d + "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/a975dae4c06d304b020eda3f53e7c3402b32132d", + "reference": "a975dae4c06d304b020eda3f53e7c3402b32132d", "shasum": "" }, "require": { @@ -3612,15 +3580,9 @@ ], "support": { "issues": "https://github.com/googleapis/google-api-php-client-services/issues", -<<<<<<< HEAD - "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.342.0" + "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.345.0" }, - "time": "2024-04-01T01:08:41+00:00" -======= - "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.343.0" - }, - "time": "2024-04-06T00:58:16+00:00" ->>>>>>> 51ab86a2dde7bff11fc6bd7a2970463cf980a86d + "time": "2024-04-21T01:02:15+00:00" }, { "name": "google/auth", @@ -4442,16 +4404,16 @@ }, { "name": "horstoeko/orderx", - "version": "v1.0.20", + "version": "v1.0.21", "source": { "type": "git", "url": "https://github.com/horstoeko/orderx.git", - "reference": "d8957cc0fea19b098d799a0c438a73504e7b326c" + "reference": "eaa2bd74b03c6845a38ef4611501cc4e70adbef7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/horstoeko/orderx/zipball/d8957cc0fea19b098d799a0c438a73504e7b326c", - "reference": "d8957cc0fea19b098d799a0c438a73504e7b326c", + "url": "https://api.github.com/repos/horstoeko/orderx/zipball/eaa2bd74b03c6845a38ef4611501cc4e70adbef7", + "reference": "eaa2bd74b03c6845a38ef4611501cc4e70adbef7", "shasum": "" }, "require": { @@ -4504,9 +4466,9 @@ ], "support": { "issues": "https://github.com/horstoeko/orderx/issues", - "source": "https://github.com/horstoeko/orderx/tree/v1.0.20" + "source": "https://github.com/horstoeko/orderx/tree/v1.0.21" }, - "time": "2024-03-21T04:28:54+00:00" + "time": "2024-04-18T04:14:03+00:00" }, { "name": "horstoeko/stringmanagement", @@ -4564,16 +4526,16 @@ }, { "name": "horstoeko/zugferd", - "version": "v1.0.38", + "version": "v1.0.41", "source": { "type": "git", "url": "https://github.com/horstoeko/zugferd.git", - "reference": "47730d3d01c1229f22d75193d68bc43ea16bcdb7" + "reference": "dee8f7efd017de6e637621e30808aff420641d5a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/horstoeko/zugferd/zipball/47730d3d01c1229f22d75193d68bc43ea16bcdb7", - "reference": "47730d3d01c1229f22d75193d68bc43ea16bcdb7", + "url": "https://api.github.com/repos/horstoeko/zugferd/zipball/dee8f7efd017de6e637621e30808aff420641d5a", + "reference": "dee8f7efd017de6e637621e30808aff420641d5a", "shasum": "" }, "require": { @@ -4586,6 +4548,7 @@ "setasign/fpdf": "^1", "setasign/fpdi": "^2", "smalot/pdfparser": "^0|^2", + "symfony/process": "^5|^6", "symfony/validator": "^5|^6", "symfony/yaml": "^5|^6" }, @@ -4632,9 +4595,9 @@ ], "support": { "issues": "https://github.com/horstoeko/zugferd/issues", - "source": "https://github.com/horstoeko/zugferd/tree/v1.0.38" + "source": "https://github.com/horstoeko/zugferd/tree/v1.0.41" }, - "time": "2024-04-05T03:38:38+00:00" + "time": "2024-04-18T03:57:58+00:00" }, { "name": "http-interop/http-factory-guzzle", @@ -5067,18 +5030,6 @@ }, { "name": "invoiceninja/ubl_invoice", -<<<<<<< HEAD - "version": "v2.1.0", - "source": { - "type": "git", - "url": "https://github.com/invoiceninja/UBL_invoice.git", - "reference": "d41ed89d66e09fefc7d743f10e68701627d80a31" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/invoiceninja/UBL_invoice/zipball/d41ed89d66e09fefc7d743f10e68701627d80a31", - "reference": "d41ed89d66e09fefc7d743f10e68701627d80a31", -======= "version": "v2.2.2", "source": { "type": "git", @@ -5089,7 +5040,6 @@ "type": "zip", "url": "https://api.github.com/repos/invoiceninja/UBL_invoice/zipball/7defd7e60363608df22bf3bc9caf749a29cde22c", "reference": "7defd7e60363608df22bf3bc9caf749a29cde22c", ->>>>>>> 51ab86a2dde7bff11fc6bd7a2970463cf980a86d "shasum": "" }, "require": { @@ -5137,15 +5087,9 @@ "xml invoice" ], "support": { -<<<<<<< HEAD - "source": "https://github.com/invoiceninja/UBL_invoice/tree/v2.1.0" - }, - "time": "2024-04-04T04:33:56+00:00" -======= "source": "https://github.com/invoiceninja/UBL_invoice/tree/v2.2.2" }, "time": "2024-04-10T11:53:16+00:00" ->>>>>>> 51ab86a2dde7bff11fc6bd7a2970463cf980a86d }, { "name": "jean85/pretty-package-versions", @@ -5583,16 +5527,16 @@ }, { "name": "laravel/framework", - "version": "v10.48.7", + "version": "v10.48.9", "source": { "type": "git", "url": "https://github.com/laravel/framework.git", - "reference": "118c686992f4b90d4da6deaf0901315c337bbaf9" + "reference": "ad758500b47964d022addf119600a1b1b0230733" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/framework/zipball/118c686992f4b90d4da6deaf0901315c337bbaf9", - "reference": "118c686992f4b90d4da6deaf0901315c337bbaf9", + "url": "https://api.github.com/repos/laravel/framework/zipball/ad758500b47964d022addf119600a1b1b0230733", + "reference": "ad758500b47964d022addf119600a1b1b0230733", "shasum": "" }, "require": { @@ -5786,20 +5730,20 @@ "issues": "https://github.com/laravel/framework/issues", "source": "https://github.com/laravel/framework" }, - "time": "2024-04-10T14:57:20+00:00" + "time": "2024-04-23T15:01:33+00:00" }, { "name": "laravel/prompts", - "version": "v0.1.18", + "version": "v0.1.20", "source": { "type": "git", "url": "https://github.com/laravel/prompts.git", - "reference": "3b5e6b03f1f1175574b5a32331d99c9819da9848" + "reference": "bf9a360c484976692de0f3792f30066f4f4b34a2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/prompts/zipball/3b5e6b03f1f1175574b5a32331d99c9819da9848", - "reference": "3b5e6b03f1f1175574b5a32331d99c9819da9848", + "url": "https://api.github.com/repos/laravel/prompts/zipball/bf9a360c484976692de0f3792f30066f4f4b34a2", + "reference": "bf9a360c484976692de0f3792f30066f4f4b34a2", "shasum": "" }, "require": { @@ -5841,9 +5785,9 @@ ], "support": { "issues": "https://github.com/laravel/prompts/issues", - "source": "https://github.com/laravel/prompts/tree/v0.1.18" + "source": "https://github.com/laravel/prompts/tree/v0.1.20" }, - "time": "2024-04-04T17:41:50+00:00" + "time": "2024-04-18T00:45:25+00:00" }, { "name": "laravel/serializable-closure", @@ -5968,26 +5912,28 @@ }, { "name": "laravel/socialite", - "version": "v5.12.1", + "version": "v5.13.0", "source": { "type": "git", "url": "https://github.com/laravel/socialite.git", - "reference": "7dae1b072573809f32ab6dcf4aebb57c8b3e8acf" + "reference": "a03e9b2f63d8125f61952fe4f5b75d70fd7c8286" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/socialite/zipball/7dae1b072573809f32ab6dcf4aebb57c8b3e8acf", - "reference": "7dae1b072573809f32ab6dcf4aebb57c8b3e8acf", + "url": "https://api.github.com/repos/laravel/socialite/zipball/a03e9b2f63d8125f61952fe4f5b75d70fd7c8286", + "reference": "a03e9b2f63d8125f61952fe4f5b75d70fd7c8286", "shasum": "" }, "require": { "ext-json": "*", + "firebase/php-jwt": "^6.4", "guzzlehttp/guzzle": "^6.0|^7.0", "illuminate/contracts": "^6.0|^7.0|^8.0|^9.0|^10.0|^11.0", "illuminate/http": "^6.0|^7.0|^8.0|^9.0|^10.0|^11.0", "illuminate/support": "^6.0|^7.0|^8.0|^9.0|^10.0|^11.0", "league/oauth1-client": "^1.10.1", - "php": "^7.2|^8.0" + "php": "^7.2|^8.0", + "phpseclib/phpseclib": "^3.0" }, "require-dev": { "mockery/mockery": "^1.0", @@ -6034,7 +5980,7 @@ "issues": "https://github.com/laravel/socialite/issues", "source": "https://github.com/laravel/socialite" }, - "time": "2024-02-16T08:58:20+00:00" + "time": "2024-04-15T18:09:46+00:00" }, { "name": "laravel/tinker", @@ -7612,16 +7558,16 @@ }, { "name": "mollie/mollie-api-php", - "version": "v2.66.0", + "version": "v2.67.0", "source": { "type": "git", "url": "https://github.com/mollie/mollie-api-php.git", - "reference": "d7d09ac62a565e818bf49d04acb2f0432da758a9" + "reference": "cf15c53127aaaac9b39d3c9772bf3f4c8ee16bd2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/mollie/mollie-api-php/zipball/d7d09ac62a565e818bf49d04acb2f0432da758a9", - "reference": "d7d09ac62a565e818bf49d04acb2f0432da758a9", + "url": "https://api.github.com/repos/mollie/mollie-api-php/zipball/cf15c53127aaaac9b39d3c9772bf3f4c8ee16bd2", + "reference": "cf15c53127aaaac9b39d3c9772bf3f4c8ee16bd2", "shasum": "" }, "require": { @@ -7698,9 +7644,9 @@ ], "support": { "issues": "https://github.com/mollie/mollie-api-php/issues", - "source": "https://github.com/mollie/mollie-api-php/tree/v2.66.0" + "source": "https://github.com/mollie/mollie-api-php/tree/v2.67.0" }, - "time": "2024-03-19T13:33:42+00:00" + "time": "2024-04-12T07:06:01+00:00" }, { "name": "moneyphp/money", @@ -7792,16 +7738,16 @@ }, { "name": "monolog/monolog", - "version": "3.5.0", + "version": "3.6.0", "source": { "type": "git", "url": "https://github.com/Seldaek/monolog.git", - "reference": "c915e2634718dbc8a4a15c61b0e62e7a44e14448" + "reference": "4b18b21a5527a3d5ffdac2fd35d3ab25a9597654" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/c915e2634718dbc8a4a15c61b0e62e7a44e14448", - "reference": "c915e2634718dbc8a4a15c61b0e62e7a44e14448", + "url": "https://api.github.com/repos/Seldaek/monolog/zipball/4b18b21a5527a3d5ffdac2fd35d3ab25a9597654", + "reference": "4b18b21a5527a3d5ffdac2fd35d3ab25a9597654", "shasum": "" }, "require": { @@ -7824,7 +7770,7 @@ "phpstan/phpstan": "^1.9", "phpstan/phpstan-deprecation-rules": "^1.0", "phpstan/phpstan-strict-rules": "^1.4", - "phpunit/phpunit": "^10.1", + "phpunit/phpunit": "^10.5.17", "predis/predis": "^1.1 || ^2", "ruflin/elastica": "^7", "symfony/mailer": "^5.4 || ^6", @@ -7877,7 +7823,7 @@ ], "support": { "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.5.0" + "source": "https://github.com/Seldaek/monolog/tree/3.6.0" }, "funding": [ { @@ -7889,7 +7835,7 @@ "type": "tidelift" } ], - "time": "2023-10-27T15:32:31+00:00" + "time": "2024-04-12T21:02:21+00:00" }, { "name": "mtdowling/jmespath.php", @@ -8924,16 +8870,16 @@ }, { "name": "paragonie/sodium_compat", - "version": "v1.20.1", + "version": "v1.21.1", "source": { "type": "git", "url": "https://github.com/paragonie/sodium_compat.git", - "reference": "1840b98d228bdad83869b191d7e51f9bb6624d8d" + "reference": "bb312875dcdd20680419564fe42ba1d9564b9e37" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/paragonie/sodium_compat/zipball/1840b98d228bdad83869b191d7e51f9bb6624d8d", - "reference": "1840b98d228bdad83869b191d7e51f9bb6624d8d", + "url": "https://api.github.com/repos/paragonie/sodium_compat/zipball/bb312875dcdd20680419564fe42ba1d9564b9e37", + "reference": "bb312875dcdd20680419564fe42ba1d9564b9e37", "shasum": "" }, "require": { @@ -9004,9 +8950,9 @@ ], "support": { "issues": "https://github.com/paragonie/sodium_compat/issues", - "source": "https://github.com/paragonie/sodium_compat/tree/v1.20.1" + "source": "https://github.com/paragonie/sodium_compat/tree/v1.21.1" }, - "time": "2024-04-05T21:00:10+00:00" + "time": "2024-04-22T22:05:04+00:00" }, { "name": "payfast/payfast-php-sdk", @@ -11290,16 +11236,16 @@ }, { "name": "sabre/xml", - "version": "4.0.4", + "version": "4.0.5", "source": { "type": "git", "url": "https://github.com/sabre-io/xml.git", - "reference": "99caa5dd248776ca6a1e1d2cfdef556a3fa63456" + "reference": "c29e49fcf9ca8ca058b1e350ee9abe4205c0de89" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sabre-io/xml/zipball/99caa5dd248776ca6a1e1d2cfdef556a3fa63456", - "reference": "99caa5dd248776ca6a1e1d2cfdef556a3fa63456", + "url": "https://api.github.com/repos/sabre-io/xml/zipball/c29e49fcf9ca8ca058b1e350ee9abe4205c0de89", + "reference": "c29e49fcf9ca8ca058b1e350ee9abe4205c0de89", "shasum": "" }, "require": { @@ -11311,7 +11257,7 @@ "sabre/uri": ">=2.0,<4.0.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.38", + "friendsofphp/php-cs-fixer": "^3.51", "phpstan/phpstan": "^1.10", "phpunit/phpunit": "^9.6" }, @@ -11355,7 +11301,7 @@ "issues": "https://github.com/sabre-io/xml/issues", "source": "https://github.com/fruux/sabre-xml" }, - "time": "2023-11-09T10:47:15+00:00" + "time": "2024-04-18T10:44:25+00:00" }, { "name": "sentry/sdk", @@ -15726,22 +15672,22 @@ }, { "name": "twig/intl-extra", - "version": "v3.8.0", + "version": "v3.9.2", "source": { "type": "git", "url": "https://github.com/twigphp/intl-extra.git", - "reference": "7b3db67c700735f473a265a97e1adaeba3e6ca0c" + "reference": "39865e5d13165016a8e7ab8cc648ad2f7aa4b639" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/intl-extra/zipball/7b3db67c700735f473a265a97e1adaeba3e6ca0c", - "reference": "7b3db67c700735f473a265a97e1adaeba3e6ca0c", + "url": "https://api.github.com/repos/twigphp/intl-extra/zipball/39865e5d13165016a8e7ab8cc648ad2f7aa4b639", + "reference": "39865e5d13165016a8e7ab8cc648ad2f7aa4b639", "shasum": "" }, "require": { "php": ">=7.2.5", - "symfony/intl": "^5.4|^6.0|^7.0", - "twig/twig": "^3.0" + "symfony/intl": "^5.4|^6.4|^7.0", + "twig/twig": "^3.9" }, "require-dev": { "symfony/phpunit-bridge": "^6.4|^7.0" @@ -15774,7 +15720,7 @@ "twig" ], "support": { - "source": "https://github.com/twigphp/intl-extra/tree/v3.8.0" + "source": "https://github.com/twigphp/intl-extra/tree/v3.9.2" }, "funding": [ { @@ -15786,34 +15732,41 @@ "type": "tidelift" } ], - "time": "2023-11-21T17:27:48+00:00" + "time": "2024-04-17T12:41:53+00:00" }, { "name": "twig/twig", - "version": "v3.8.0", + "version": "v3.9.3", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "9d15f0ac07f44dc4217883ec6ae02fd555c6f71d" + "reference": "a842d75fed59cdbcbd3a3ad7fb9eb768fc350d58" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/9d15f0ac07f44dc4217883ec6ae02fd555c6f71d", - "reference": "9d15f0ac07f44dc4217883ec6ae02fd555c6f71d", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/a842d75fed59cdbcbd3a3ad7fb9eb768fc350d58", + "reference": "a842d75fed59cdbcbd3a3ad7fb9eb768fc350d58", "shasum": "" }, "require": { "php": ">=7.2.5", + "symfony/deprecation-contracts": "^2.5|^3", "symfony/polyfill-ctype": "^1.8", "symfony/polyfill-mbstring": "^1.3", "symfony/polyfill-php80": "^1.22" }, "require-dev": { "psr/container": "^1.0|^2.0", - "symfony/phpunit-bridge": "^5.4.9|^6.3|^7.0" + "symfony/phpunit-bridge": "^5.4.9|^6.4|^7.0" }, "type": "library", "autoload": { + "files": [ + "src/Resources/core.php", + "src/Resources/debug.php", + "src/Resources/escaper.php", + "src/Resources/string_loader.php" + ], "psr-4": { "Twig\\": "src/" } @@ -15846,7 +15799,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v3.8.0" + "source": "https://github.com/twigphp/Twig/tree/v3.9.3" }, "funding": [ { @@ -15858,7 +15811,7 @@ "type": "tidelift" } ], - "time": "2023-11-21T18:54:41+00:00" + "time": "2024-04-18T11:59:33+00:00" }, { "name": "twilio/sdk", @@ -16289,18 +16242,6 @@ "packages-dev": [ { "name": "barryvdh/laravel-debugbar", -<<<<<<< HEAD - "version": "v3.13.3", - "source": { - "type": "git", - "url": "https://github.com/barryvdh/laravel-debugbar.git", - "reference": "241e9bddb04ab42a04a5fe8b2b9654374c864229" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/barryvdh/laravel-debugbar/zipball/241e9bddb04ab42a04a5fe8b2b9654374c864229", - "reference": "241e9bddb04ab42a04a5fe8b2b9654374c864229", -======= "version": "v3.13.4", "source": { "type": "git", @@ -16311,7 +16252,6 @@ "type": "zip", "url": "https://api.github.com/repos/barryvdh/laravel-debugbar/zipball/00201bcd1eaf9b1d3debddcdc13c219e4835fb61", "reference": "00201bcd1eaf9b1d3debddcdc13c219e4835fb61", ->>>>>>> 51ab86a2dde7bff11fc6bd7a2970463cf980a86d "shasum": "" }, "require": { @@ -16370,11 +16310,7 @@ ], "support": { "issues": "https://github.com/barryvdh/laravel-debugbar/issues", -<<<<<<< HEAD - "source": "https://github.com/barryvdh/laravel-debugbar/tree/v3.13.3" -======= "source": "https://github.com/barryvdh/laravel-debugbar/tree/v3.13.4" ->>>>>>> 51ab86a2dde7bff11fc6bd7a2970463cf980a86d }, "funding": [ { @@ -16386,11 +16322,7 @@ "type": "github" } ], -<<<<<<< HEAD - "time": "2024-04-04T02:42:49+00:00" -======= "time": "2024-04-10T09:15:45+00:00" ->>>>>>> 51ab86a2dde7bff11fc6bd7a2970463cf980a86d }, { "name": "barryvdh/laravel-ide-helper", @@ -17117,16 +17049,16 @@ }, { "name": "friendsofphp/php-cs-fixer", - "version": "v3.53.0", + "version": "v3.54.0", "source": { "type": "git", "url": "https://github.com/PHP-CS-Fixer/PHP-CS-Fixer.git", - "reference": "69a19093a9ded8d1baac62ed6c009b8bc148d008" + "reference": "2aecbc8640d7906c38777b3dcab6f4ca79004d08" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PHP-CS-Fixer/PHP-CS-Fixer/zipball/69a19093a9ded8d1baac62ed6c009b8bc148d008", - "reference": "69a19093a9ded8d1baac62ed6c009b8bc148d008", + "url": "https://api.github.com/repos/PHP-CS-Fixer/PHP-CS-Fixer/zipball/2aecbc8640d7906c38777b3dcab6f4ca79004d08", + "reference": "2aecbc8640d7906c38777b3dcab6f4ca79004d08", "shasum": "" }, "require": { @@ -17198,7 +17130,7 @@ ], "support": { "issues": "https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/issues", - "source": "https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/tree/v3.53.0" + "source": "https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/tree/v3.54.0" }, "funding": [ { @@ -17206,7 +17138,7 @@ "type": "github" } ], - "time": "2024-04-08T15:03:00+00:00" + "time": "2024-04-17T08:12:13+00:00" }, { "name": "hamcrest/hamcrest-php", @@ -17320,16 +17252,16 @@ }, { "name": "larastan/larastan", - "version": "v2.9.2", + "version": "v2.9.5", "source": { "type": "git", "url": "https://github.com/larastan/larastan.git", - "reference": "a79b46b96060504b400890674b83f66aa7f5db6d" + "reference": "101f1a4470f87326f4d3995411d28679d8800abe" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/larastan/larastan/zipball/a79b46b96060504b400890674b83f66aa7f5db6d", - "reference": "a79b46b96060504b400890674b83f66aa7f5db6d", + "url": "https://api.github.com/repos/larastan/larastan/zipball/101f1a4470f87326f4d3995411d28679d8800abe", + "reference": "101f1a4470f87326f4d3995411d28679d8800abe", "shasum": "" }, "require": { @@ -17342,15 +17274,15 @@ "illuminate/pipeline": "^9.52.16 || ^10.28.0 || ^11.0", "illuminate/support": "^9.52.16 || ^10.28.0 || ^11.0", "php": "^8.0.2", - "phpmyadmin/sql-parser": "^5.8.2", - "phpstan/phpstan": "^1.10.50" + "phpmyadmin/sql-parser": "^5.9.0", + "phpstan/phpstan": "^1.10.66" }, "require-dev": { "doctrine/coding-standard": "^12.0", - "nikic/php-parser": "^4.17.1", - "orchestra/canvas": "^7.11.1 || ^8.11.0 || ^9.0.0", - "orchestra/testbench": "^7.33.0 || ^8.13.0 || ^9.0.0", - "phpunit/phpunit": "^9.6.13 || ^10.5" + "nikic/php-parser": "^4.19.1", + "orchestra/canvas": "^7.11.1 || ^8.11.0 || ^9.0.2", + "orchestra/testbench": "^7.33.0 || ^8.13.0 || ^9.0.3", + "phpunit/phpunit": "^9.6.13 || ^10.5.16" }, "suggest": { "orchestra/testbench": "Using Larastan for analysing a package needs Testbench" @@ -17398,7 +17330,7 @@ ], "support": { "issues": "https://github.com/larastan/larastan/issues", - "source": "https://github.com/larastan/larastan/tree/v2.9.2" + "source": "https://github.com/larastan/larastan/tree/v2.9.5" }, "funding": [ { @@ -17418,7 +17350,7 @@ "type": "patreon" } ], - "time": "2024-02-27T03:16:03+00:00" + "time": "2024-04-16T19:13:34+00:00" }, { "name": "maximebf/debugbar", @@ -17934,16 +17866,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.10.66", + "version": "1.10.67", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "94779c987e4ebd620025d9e5fdd23323903950bd" + "reference": "16ddbe776f10da6a95ebd25de7c1dbed397dc493" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/94779c987e4ebd620025d9e5fdd23323903950bd", - "reference": "94779c987e4ebd620025d9e5fdd23323903950bd", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/16ddbe776f10da6a95ebd25de7c1dbed397dc493", + "reference": "16ddbe776f10da6a95ebd25de7c1dbed397dc493", "shasum": "" }, "require": { @@ -17986,13 +17918,9 @@ { "url": "https://github.com/phpstan", "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/phpstan/phpstan", - "type": "tidelift" } ], - "time": "2024-03-28T16:17:31+00:00" + "time": "2024-04-16T07:22:02+00:00" }, { "name": "phpunit/php-code-coverage", @@ -18317,16 +18245,16 @@ }, { "name": "phpunit/phpunit", - "version": "10.5.17", + "version": "10.5.19", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "c1f736a473d21957ead7e94fcc029f571895abf5" + "reference": "c726f0de022368f6ed103e452a765d3304a996a4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c1f736a473d21957ead7e94fcc029f571895abf5", - "reference": "c1f736a473d21957ead7e94fcc029f571895abf5", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c726f0de022368f6ed103e452a765d3304a996a4", + "reference": "c726f0de022368f6ed103e452a765d3304a996a4", "shasum": "" }, "require": { @@ -18398,7 +18326,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.17" + "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.19" }, "funding": [ { @@ -18414,7 +18342,7 @@ "type": "tidelift" } ], - "time": "2024-04-05T04:39:01+00:00" + "time": "2024-04-17T14:06:18+00:00" }, { "name": "sebastian/cli-parser", @@ -19465,16 +19393,16 @@ }, { "name": "spatie/ignition", - "version": "1.13.1", + "version": "1.13.2", "source": { "type": "git", "url": "https://github.com/spatie/ignition.git", - "reference": "889bf1dfa59e161590f677728b47bf4a6893983b" + "reference": "952798e239d9969e4e694b124c2cc222798dbb28" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/spatie/ignition/zipball/889bf1dfa59e161590f677728b47bf4a6893983b", - "reference": "889bf1dfa59e161590f677728b47bf4a6893983b", + "url": "https://api.github.com/repos/spatie/ignition/zipball/952798e239d9969e4e694b124c2cc222798dbb28", + "reference": "952798e239d9969e4e694b124c2cc222798dbb28", "shasum": "" }, "require": { @@ -19544,20 +19472,20 @@ "type": "github" } ], - "time": "2024-03-29T14:03:47+00:00" + "time": "2024-04-16T08:49:17+00:00" }, { "name": "spatie/laravel-ignition", - "version": "2.5.1", + "version": "2.5.2", "source": { "type": "git", "url": "https://github.com/spatie/laravel-ignition.git", - "reference": "0c864b3cbd66ce67a2096c5f743e07ce8f1d6ab9" + "reference": "c93fcadcc4629775c839ac9a90916f07a660266f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/spatie/laravel-ignition/zipball/0c864b3cbd66ce67a2096c5f743e07ce8f1d6ab9", - "reference": "0c864b3cbd66ce67a2096c5f743e07ce8f1d6ab9", + "url": "https://api.github.com/repos/spatie/laravel-ignition/zipball/c93fcadcc4629775c839ac9a90916f07a660266f", + "reference": "c93fcadcc4629775c839ac9a90916f07a660266f", "shasum": "" }, "require": { @@ -19567,7 +19495,7 @@ "illuminate/support": "^10.0|^11.0", "php": "^8.1", "spatie/flare-client-php": "^1.3.5", - "spatie/ignition": "^1.13", + "spatie/ignition": "^1.13.2", "symfony/console": "^6.2.3|^7.0", "symfony/var-dumper": "^6.2.3|^7.0" }, @@ -19636,7 +19564,7 @@ "type": "github" } ], - "time": "2024-04-02T06:30:22+00:00" + "time": "2024-04-16T08:57:16+00:00" }, { "name": "spaze/phpstan-stripe", @@ -19906,5 +19834,5 @@ "platform-dev": { "php": "^8.1|^8.2" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.6.0" } From b2110de1103c253d3d5d718e30f4b51b7d77a87a Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 24 Apr 2024 08:40:58 +0200 Subject: [PATCH 068/119] replace Log::info with nlog --- app/Http/Controllers/BrevoController.php | 4 ++-- app/Http/Controllers/MailgunController.php | 2 +- app/Http/Controllers/PostMarkController.php | 4 ++-- app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 4 ++-- .../Mailgun/ProcessMailgunInboundWebhook.php | 6 +++--- app/Services/InboundMail/InboundMailEngine.php | 18 +++++++++--------- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/app/Http/Controllers/BrevoController.php b/app/Http/Controllers/BrevoController.php index 53dbe9bceaf1..d462f06683c0 100644 --- a/app/Http/Controllers/BrevoController.php +++ b/app/Http/Controllers/BrevoController.php @@ -194,14 +194,14 @@ class BrevoController extends BaseController return response()->json(['message' => 'Unauthorized'], 403); if (!array_key_exists('items', $input)) { - Log::info('Failed: Message could not be parsed, because required parameters are missing.'); + nlog('Failed: Message could not be parsed, because required parameters are missing.'); return response()->json(['message' => 'Failed. Invalid Parameters.'], 400); } foreach ($input["items"] as $item) { if (!array_key_exists('Recipients', $item) || !array_key_exists('MessageId', $item)) { - Log::info('Failed: Message could not be parsed, because required parameters are missing. At least one item was invalid.'); + nlog('Failed: Message could not be parsed, because required parameters are missing. At least one item was invalid.'); return response()->json(['message' => 'Failed. Invalid Parameters. At least one item was invalid.'], 400); } diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 8413185db528..d8cae5c6b7e7 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -120,7 +120,7 @@ class MailgunController extends BaseController $input = $request->all(); if (!array_key_exists('sender', $input) || !array_key_exists('recipient', $input) || !array_key_exists('message-url', $input)) { - Log::info('Failed: Message could not be parsed, because required parameters are missing. Please ensure contacting this api-endpoint with a store & notify operation instead of a forward operation!'); + nlog('Failed: Message could not be parsed, because required parameters are missing. Please ensure contacting this api-endpoint with a store & notify operation instead of a forward operation!'); return response()->json(['message' => 'Failed. Missing Parameters. Use store and notify!'], 400); } diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index 88272a445e7d..7e4b967c1b7e 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -280,7 +280,7 @@ class PostMarkController extends BaseController return response()->json(['message' => 'Unauthorized'], 403); if (!(array_key_exists("MessageStream", $input) && $input["MessageStream"] == "inbound") || !array_key_exists("To", $input) || !array_key_exists("From", $input) || !array_key_exists("MessageID", $input)) { - Log::info('Failed: Message could not be parsed, because required parameters are missing.'); + nlog('Failed: Message could not be parsed, because required parameters are missing.'); return response()->json(['message' => 'Failed. Missing/Invalid Parameters.'], 400); } @@ -292,7 +292,7 @@ class PostMarkController extends BaseController $company = MultiDB::findAndSetDbByExpenseMailbox($input["To"]); if (!$company) { - Log::info('[PostmarkInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $input["To"]); + nlog('[PostmarkInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $input["To"]); $inboundEngine->saveMeta($input["From"], $input["To"], true); // important to save this, to protect from spam return; } diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index 864b0bdb9af8..4adbd567f6ed 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -135,7 +135,7 @@ class ProcessBrevoInboundWebhook implements ShouldQueue // match company $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); if (!$company) { - Log::info('[ProcessBrevoInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from brevo: ' . $recipient); + nlog('[ProcessBrevoInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from brevo: ' . $recipient); continue; } @@ -171,7 +171,7 @@ class ProcessBrevoInboundWebhook implements ShouldQueue } catch (\Error $e) { if (config('services.brevo.secret')) { - Log::info("[ProcessBrevoInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); + nlog("[ProcessBrevoInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); $attachment = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index 1bf6caeb44dc..b11078047124 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -180,7 +180,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue // match company $company = MultiDB::findAndSetDbByExpenseMailbox($to); if (!$company) { - Log::info('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $to); + nlog('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $to); $this->engine->saveMeta($from, $to, true); // important to save this, to protect from spam return; } @@ -205,7 +205,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue $mail = json_decode(file_get_contents($messageUrl)); } catch (\Error $e) { if (config('services.mailgun.secret')) { - Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); + nlog("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; $messageUrl = explode("|", $this->input)[2]; @@ -253,7 +253,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue } catch (\Error $e) { if (config('services.mailgun.secret')) { - Log::info("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); + nlog("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; $url = $attachment->url; diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index 5880c9ee5bfa..057ed43d1ac0 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -69,11 +69,11 @@ class InboundMailEngine { // invalid email if (!filter_var($from, FILTER_VALIDATE_EMAIL)) { - Log::info('E-Mail blocked, because from e-mail has the wrong format: ' . $from); + nlog('E-Mail blocked, because from e-mail has the wrong format: ' . $from); return true; } if (!filter_var($to, FILTER_VALIDATE_EMAIL)) { - Log::info('E-Mail blocked, because to e-mail has the wrong format: ' . $from); + nlog('E-Mail blocked, because to e-mail has the wrong format: ' . $from); return true; } @@ -85,14 +85,14 @@ class InboundMailEngine return false; } if (in_array($domain, $this->globalBlacklistDomains)) { - Log::info('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $from); + nlog('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $from); return true; } if (in_array($domain, $this->globalWhitelistSenders)) { return false; } if (in_array($from, $this->globalBlacklistSenders)) { - Log::info('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $from); + nlog('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $from); return true; } @@ -103,13 +103,13 @@ class InboundMailEngine // sender occured in more than 500 emails in the last 12 hours $senderMailCountTotal = Cache::get('inboundMailCountSender:' . $from, 0); if ($senderMailCountTotal >= 5000) { - Log::info('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $from); + nlog('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $from); $this->blockSender($from); $this->saveMeta($from, $to); return true; } if ($senderMailCountTotal >= 1000) { - Log::info('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $from); + nlog('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $from); $this->saveMeta($from, $to); return true; } @@ -117,7 +117,7 @@ class InboundMailEngine // sender sended more than 50 emails to the wrong mailbox in the last 6 hours $senderMailCountUnknownRecipent = Cache::get('inboundMailCountSenderUnknownRecipent:' . $from, 0); if ($senderMailCountUnknownRecipent >= 50) { - Log::info('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $from); + nlog('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $from); $this->saveMeta($from, $to); return true; } @@ -125,7 +125,7 @@ class InboundMailEngine // wrong recipent occurs in more than 100 emails in the last 12 hours, so the processing is blocked $mailCountUnknownRecipent = Cache::get('inboundMailCountUnknownRecipent:' . $to, 0); // @turbo124 maybe use many to save resources in case of spam with multiple to addresses each time if ($mailCountUnknownRecipent >= 200) { - Log::info('E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $from); + nlog('E-Mail blocked, because anyone sended more than ' . $mailCountUnknownRecipent . ' emails to the wrong mailbox in the last 12 hours. Current sender was blocked as well: ' . $from); $this->blockSender($from); $this->saveMeta($from, $to); return true; @@ -263,7 +263,7 @@ class InboundMailEngine } private function logBlocked(Company $company, string $data) { - Log::info("[InboundMailEngine][company:" . $company->id . "] " . $data); + nlog("[InboundMailEngine][company:" . $company->id . "] " . $data); ( new SystemLogger( From 5a5df06cdaf34a108168f5f6e2cc2aa5c79bc2d2 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 24 Apr 2024 08:42:38 +0200 Subject: [PATCH 069/119] fix ninja.php --- config/ninja.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/ninja.php b/config/ninja.php index 085a4f92fb12..0a5ea312d77d 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -87,8 +87,6 @@ return [ 'gocardless' => env('GOCARDLESS_KEYS', ''), 'square' => env('SQUARE_KEYS', ''), 'eway' => env('EWAY_KEYS', ''), - 'mollie', - env('MOLLIE_KEYS', ''), 'paytrace' => env('PAYTRACE_KEYS', ''), 'stripe' => env('STRIPE_KEYS', ''), 'paypal' => env('PAYPAL_KEYS', ''), From 21aa38c52ff60a98372c0af61029e8f96d5d4613 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 24 Apr 2024 08:44:28 +0200 Subject: [PATCH 070/119] fix invalid fields in VendorTransformer --- app/Transformers/VendorTransformer.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/Transformers/VendorTransformer.php b/app/Transformers/VendorTransformer.php index 3e10568b09a9..c849fd223290 100644 --- a/app/Transformers/VendorTransformer.php +++ b/app/Transformers/VendorTransformer.php @@ -105,8 +105,6 @@ class VendorTransformer extends EntityTransformer 'language_id' => (string) $vendor->language_id ?: '', 'classification' => (string) $vendor->classification ?: '', 'display_name' => (string) $vendor->present()->name(), - 'invoicing_email' => (string) $vendor->invoicing_email ?: '', - 'invoicing_domain' => (string) $vendor->invoicing_domain ?: '', 'routing_id' => (string) $vendor->routing_id ?: '', ]; } From 8378f9c1d13559abb098126bfcb3c90f16755bd0 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 24 Apr 2024 08:49:02 +0200 Subject: [PATCH 071/119] fixes --- app/Http/Controllers/BrevoController.php | 3 --- app/Http/Controllers/MailgunController.php | 1 - app/Http/Controllers/PostMarkController.php | 1 - app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 1 - app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php | 1 - app/Services/InboundMail/InboundMailEngine.php | 3 --- 6 files changed, 10 deletions(-) diff --git a/app/Http/Controllers/BrevoController.php b/app/Http/Controllers/BrevoController.php index d462f06683c0..d4363e54a6f5 100644 --- a/app/Http/Controllers/BrevoController.php +++ b/app/Http/Controllers/BrevoController.php @@ -13,10 +13,7 @@ namespace App\Http\Controllers; use App\Jobs\Brevo\ProcessBrevoInboundWebhook; use App\Jobs\Brevo\ProcessBrevoWebhook; -use App\Libraries\MultiDB; -use App\Models\Company; use Illuminate\Http\Request; -use Log; /** * Class PostMarkController. diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index d8cae5c6b7e7..44ac87420fe4 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -14,7 +14,6 @@ namespace App\Http\Controllers; use App\Jobs\Mailgun\ProcessMailgunInboundWebhook; use App\Jobs\Mailgun\ProcessMailgunWebhook; use Illuminate\Http\Request; -use Log; /** * Class MailgunController. diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index 7e4b967c1b7e..6014f059afc0 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -18,7 +18,6 @@ use App\Services\InboundMail\InboundMailEngine; use App\Utils\TempFile; use Illuminate\Support\Carbon; use Illuminate\Http\Request; -use Log; /** * Class PostMarkController. diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index 4adbd567f6ed..39617fb03282 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -23,7 +23,6 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -use Log; class ProcessBrevoInboundWebhook implements ShouldQueue { diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index b11078047124..f278cdba0153 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -21,7 +21,6 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -use Log; class ProcessMailgunInboundWebhook implements ShouldQueue { diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index 057ed43d1ac0..31f242476094 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -18,7 +18,6 @@ use App\Libraries\MultiDB; use App\Models\ClientContact; use App\Models\Company; use App\Models\SystemLog; -use App\Models\Vendor; use App\Models\VendorContact; use App\Services\InboundMail\InboundMail; use App\Utils\Ninja; @@ -28,14 +27,12 @@ use App\Utils\Traits\SavesDocuments; use App\Utils\Traits\MakesHash; use Cache; use Illuminate\Queue\SerializesModels; -use Log; class InboundMailEngine { use SerializesModels, MakesHash; use GeneratesCounter, SavesDocuments; - private ?Company $company; private ?bool $isUnknownRecipent = null; private array $globalBlacklistDomains = []; private array $globalBlacklistSenders = []; From 96d4a250cf0754de1f99acc0556b7ce92492db4d Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 24 Apr 2024 08:56:53 +0200 Subject: [PATCH 072/119] fixes on MultiDB --- app/Libraries/MultiDB.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Libraries/MultiDB.php b/app/Libraries/MultiDB.php index ea482b225edb..ca1f0bb5ace3 100644 --- a/app/Libraries/MultiDB.php +++ b/app/Libraries/MultiDB.php @@ -113,7 +113,7 @@ class MultiDB { if (!config('ninja.db.multi_db_enabled')) { - return Company::where("expense_mailbox", $expense_mailbox)->withTrashed()->exists(); + return Company::where("expense_mailbox", $expense_mailbox)->exists(); } if (in_array($expense_mailbox, self::$protected_expense_mailboxes)) { @@ -123,7 +123,7 @@ class MultiDB $current_db = config('database.default'); foreach (self::$dbs as $db) { - if (Company::on($db)->where("expense_mailbox", $expense_mailbox)->withTrashed()->exists()) { + if (Company::on($db)->where("expense_mailbox", $expense_mailbox)->exists()) { self::setDb($current_db); return false; From e08e377b3492ac6c72a31dd223b3bf47be9e777e Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 19 May 2024 06:31:26 +0200 Subject: [PATCH 073/119] fixes: env vars --- app/Services/InboundMail/InboundMailEngine.php | 8 ++++---- config/ninja.php | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index 31f242476094..f207d47f216e 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -16,7 +16,7 @@ use App\Factory\ExpenseFactory; use App\Jobs\Util\SystemLogger; use App\Libraries\MultiDB; use App\Models\ClientContact; -use App\Models\Company; +use App\Models\Company; use App\Models\SystemLog; use App\Models\VendorContact; use App\Services\InboundMail\InboundMail; @@ -99,13 +99,13 @@ class InboundMailEngine // sender occured in more than 500 emails in the last 12 hours $senderMailCountTotal = Cache::get('inboundMailCountSender:' . $from, 0); - if ($senderMailCountTotal >= 5000) { + if ($senderMailCountTotal >= config('global_inbound_sender_permablock_mailcount')) { nlog('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $from); $this->blockSender($from); $this->saveMeta($from, $to); return true; } - if ($senderMailCountTotal >= 1000) { + if ($senderMailCountTotal >= config('global_inbound_sender_block_mailcount')) { nlog('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $from); $this->saveMeta($from, $to); return true; @@ -113,7 +113,7 @@ class InboundMailEngine // sender sended more than 50 emails to the wrong mailbox in the last 6 hours $senderMailCountUnknownRecipent = Cache::get('inboundMailCountSenderUnknownRecipent:' . $from, 0); - if ($senderMailCountUnknownRecipent >= 50) { + if ($senderMailCountUnknownRecipent >= config('company_inbound_sender_block_unknown_reciepent')) { nlog('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $from); $this->saveMeta($from, $to); return true; diff --git a/config/ninja.php b/config/ninja.php index 0a5ea312d77d..f800871d3a48 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -235,7 +235,11 @@ return [ ], 'inbound_mailbox' => [ 'expense_mailbox_endings' => env('EXPENSE_MAILBOX_ENDINGS', '@expense.invoicing.co'), - 'inbound_webhook_key' => env('INBOUND_WEBHOOK_KEY', null) + 'inbound_webhook_key' => env('INBOUND_WEBHOOK_KEY', null), + 'global_inbound_sender_block_mailcount' => env('GLOBAL_INBOUND_SENDER_BLOCK_MAILCOUNT', 1000), + 'global_inbound_sender_permablock_mailcount' => env('GLOBAL_INBOUND_SENDER_PERMABLOCK_MAILCOUNT', 5000), + 'company_inbound_sender_block_unknown_reciepent' => env('COMPANY_INBOUND_SENDER_BLOCK_UNKNOWN_RECIEPENT', 50), + 'global_inbound_sender_permablock_unknown_reciepent' => env('GLOBAL_INBOUND_SENDER_PERMABLOCK_UNKNOWN_RECIEPENT', 5000), ], 'cloudflare' => [ 'turnstile' => [ From ef609928439e75e5b04f8979cde45a72fad0361b Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 19 May 2024 06:55:15 +0200 Subject: [PATCH 074/119] fix: reduce vars in favor for db --- app/Models/Company.php | 12 +++---- .../InboundMail/InboundMailEngine.php | 34 ++++++++----------- app/Transformers/CompanyTransformer.php | 6 ++-- ...2023_12_10_110951_inbound_mail_parsing.php | 7 ++-- 4 files changed, 23 insertions(+), 36 deletions(-) diff --git a/app/Models/Company.php b/app/Models/Company.php index 2ee97801137b..718dc7b47fce 100644 --- a/app/Models/Company.php +++ b/app/Models/Company.php @@ -117,10 +117,8 @@ use Laracasts\Presenter\PresentableTrait; * @property bool $inbound_mailbox_allow_vendors * @property bool $inbound_mailbox_allow_clients * @property bool $inbound_mailbox_allow_unknown - * @property string|null $inbound_mailbox_whitelist_domains - * @property string|null $inbound_mailbox_whitelist_senders - * @property string|null $inbound_mailbox_blacklist_domains - * @property string|null $inbound_mailbox_blacklist_senders + * @property string|null $inbound_mailbox_whitelist + * @property string|null $inbound_mailbox_blacklist * @property int $deleted_at * @property string $smtp_username * @property string $smtp_password @@ -375,10 +373,8 @@ class Company extends BaseModel 'inbound_mailbox_allow_vendors', 'inbound_mailbox_allow_clients', 'inbound_mailbox_allow_unknown', - 'inbound_mailbox_whitelist_domains', - 'inbound_mailbox_whitelist_senders', - 'inbound_mailbox_blacklist_domains', - 'inbound_mailbox_blacklist_senders', + 'inbound_mailbox_whitelist', + 'inbound_mailbox_blacklist', 'smtp_host', 'smtp_port', 'smtp_encryption', diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index f207d47f216e..c1d2c6820923 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -16,7 +16,7 @@ use App\Factory\ExpenseFactory; use App\Jobs\Util\SystemLogger; use App\Libraries\MultiDB; use App\Models\ClientContact; -use App\Models\Company; +use App\Models\Company; use App\Models\SystemLog; use App\Models\VendorContact; use App\Services\InboundMail\InboundMail; @@ -34,10 +34,8 @@ class InboundMailEngine use GeneratesCounter, SavesDocuments; private ?bool $isUnknownRecipent = null; - private array $globalBlacklistDomains = []; - private array $globalBlacklistSenders = []; - private array $globalWhitelistDomains = []; // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders - private array $globalWhitelistSenders = []; // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders + private array $globalBlacklist = []; + private array $globalWhitelist = []; // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders public function __construct() { } @@ -78,17 +76,17 @@ class InboundMailEngine $domain = array_pop($parts); // global blacklist - if (in_array($from, $this->globalWhitelistDomains)) { + if (in_array($from, $this->globalWhitelist)) { return false; } - if (in_array($domain, $this->globalBlacklistDomains)) { + if (in_array($domain, $this->globalWhitelist)) { + return false; + } + if (in_array($domain, $this->globalBlacklist)) { nlog('E-Mail blocked, because the domain was found on globalBlocklistDomains: ' . $from); return true; } - if (in_array($domain, $this->globalWhitelistSenders)) { - return false; - } - if (in_array($from, $this->globalBlacklistSenders)) { + if (in_array($from, $this->globalBlacklist)) { nlog('E-Mail blocked, because the email was found on globalBlocklistEmails: ' . $from); return true; } @@ -210,17 +208,15 @@ class InboundMailEngine $domain = array_pop($parts); // whitelists - $email_whitelist = explode(",", $company->inbound_mailbox_whitelist_senders); - if (in_array($email->from, $email_whitelist)) + $whitelist = explode(",", $company->inbound_mailbox_whitelist); + if (in_array($email->from, $whitelist)) return true; - $domain_whitelist = explode(",", $company->inbound_mailbox_whitelist_domains); - if (in_array($domain, $domain_whitelist)) + if (in_array($domain, $whitelist)) return true; - $email_blacklist = explode(",", $company->inbound_mailbox_blacklist_senders); - if (in_array($email->from, $email_blacklist)) + $blacklist = explode(",", $company->inbound_mailbox_blacklist); + if (in_array($email->from, $blacklist)) return false; - $domain_blacklist = explode(",", $company->inbound_mailbox_blacklist_domains); - if (in_array($domain, $domain_blacklist)) + if (in_array($domain, $blacklist)) return false; // allow unknown diff --git a/app/Transformers/CompanyTransformer.php b/app/Transformers/CompanyTransformer.php index 603e2562a27d..475e17fd8714 100644 --- a/app/Transformers/CompanyTransformer.php +++ b/app/Transformers/CompanyTransformer.php @@ -210,10 +210,8 @@ class CompanyTransformer extends EntityTransformer 'inbound_mailbox_allow_vendors' => (bool) $company->inbound_mailbox_allow_vendors, 'inbound_mailbox_allow_clients' => (bool) $company->inbound_mailbox_allow_clients, 'inbound_mailbox_allow_unknown' => (bool) $company->inbound_mailbox_allow_unknown, - 'inbound_mailbox_blacklist_domains' => $company->inbound_mailbox_blacklist_domains, - 'inbound_mailbox_blacklist_senders' => $company->inbound_mailbox_blacklist_senders, - 'inbound_mailbox_whitelist_domains' => $company->inbound_mailbox_whitelist_domains, - 'inbound_mailbox_whitelist_senders' => $company->inbound_mailbox_whitelist_senders, + 'inbound_mailbox_blacklist' => $company->inbound_mailbox_blacklist, + 'inbound_mailbox_whitelist' => $company->inbound_mailbox_whitelist, 'smtp_host' => (string) $company->smtp_host ?? '', 'smtp_port' => (int) $company->smtp_port ?? 25, 'smtp_encryption' => (string) $company->smtp_encryption ?? 'tls', diff --git a/database/migrations/2023_12_10_110951_inbound_mail_parsing.php b/database/migrations/2023_12_10_110951_inbound_mail_parsing.php index 94d647510f34..056096254f4a 100644 --- a/database/migrations/2023_12_10_110951_inbound_mail_parsing.php +++ b/database/migrations/2023_12_10_110951_inbound_mail_parsing.php @@ -1,6 +1,5 @@ boolean("inbound_mailbox_allow_vendors")->default(false); $table->boolean("inbound_mailbox_allow_clients")->default(false); $table->boolean("inbound_mailbox_allow_unknown")->default(false); - $table->text("inbound_mailbox_whitelist_domains")->nullable(); - $table->text("inbound_mailbox_whitelist_senders")->nullable(); - $table->text("inbound_mailbox_blacklist_domains")->nullable(); - $table->text("inbound_mailbox_blacklist_senders")->nullable(); + $table->text("inbound_mailbox_whitelist")->nullable(); + $table->text("inbound_mailbox_blacklist")->nullable(); }); } From 22bc9425a2b3b7322ea7fc49de3eee9735bab59b Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 19 May 2024 06:59:37 +0200 Subject: [PATCH 075/119] fix: env for black and whitelist --- app/Services/InboundMail/InboundMailEngine.php | 4 ++-- config/ninja.php | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index c1d2c6820923..f442529bac42 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -34,8 +34,8 @@ class InboundMailEngine use GeneratesCounter, SavesDocuments; private ?bool $isUnknownRecipent = null; - private array $globalBlacklist = []; - private array $globalWhitelist = []; // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders + private array $globalBlacklist = explode(",", config('global_inbound_blocklist')); + private array $globalWhitelist = explode(",", config('global_inbound_whitelist')); // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders public function __construct() { } diff --git a/config/ninja.php b/config/ninja.php index ec196a51b6ab..9628d985d34b 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -236,6 +236,8 @@ return [ 'inbound_mailbox' => [ 'expense_mailbox_endings' => env('EXPENSE_MAILBOX_ENDINGS', '@expense.invoicing.co'), 'inbound_webhook_key' => env('INBOUND_WEBHOOK_KEY', null), + 'global_inbound_blacklist' => env('GLOBAL_INBOUND_BLACKLIST', ''), + 'global_inbound_whitelist' => env('GLOBAL_INBOUND_WHITELIST', ''), 'global_inbound_sender_block_mailcount' => env('GLOBAL_INBOUND_SENDER_BLOCK_MAILCOUNT', 1000), 'global_inbound_sender_permablock_mailcount' => env('GLOBAL_INBOUND_SENDER_PERMABLOCK_MAILCOUNT', 5000), 'company_inbound_sender_block_unknown_reciepent' => env('COMPANY_INBOUND_SENDER_BLOCK_UNKNOWN_RECIEPENT', 50), From 8e850ea2a631deca858b21bc57801ac6bcbc5578 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 22 May 2024 07:07:27 +0200 Subject: [PATCH 076/119] wip: ZugferdEDocument implementation --- .../InboundMail/InboundMailEngine.php | 84 ++++++++++++++----- 1 file changed, 65 insertions(+), 19 deletions(-) diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index f442529bac42..cb3399dba2f0 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -55,7 +55,7 @@ class InboundMailEngine return; } - $this->createExpense($company, $email); + $this->createExpenses($company, $email); $this->saveMeta($email->from, $email->to); } @@ -151,7 +151,7 @@ class InboundMailEngine } // MAIN-PROCESSORS - protected function createExpense(Company $company, InboundMail $email) + protected function createExpenses(Company $company, InboundMail $email) { // Skipping executions: will not result in not saving Metadata to prevent usage of these conditions, to spam if (!($company?->expense_mailbox_active ?: false)) { @@ -167,31 +167,77 @@ class InboundMailEngine return; } - // create expense - $expense = ExpenseFactory::create($company->id, $company->owner()->id); - - $expense->public_notes = $email->subject; - $expense->private_notes = $email->text_body; - $expense->date = $email->date; - // handle vendor assignment $expense_vendor = $this->getVendor($company, $email); - if ($expense_vendor) - $expense->vendor_id = $expense_vendor->id; // handle documents $this->processHtmlBodyToDocument($email); - $documents = []; - array_push($documents, ...$email->documents); - if ($email->body_document !== null) - array_push($documents, $email->body_document); - $expense->saveQuietly(); + // handle documents seperatly + foreach ($email->documents as $document) { - $this->saveDocuments($documents, $expense); + $expense = null; - event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); // @turbo124 please check, I copied from API-Controller - event('eloquent.created: App\Models\Expense', $expense); // @turbo124 please check, I copied from API-Controller + // TODO: check if document is a ZugferdEDocument and can be handled that way + if ($document->extension() === 'pdf' || $document->extension() === 'xml') { + try { + + $expense = (new ZugferdEDocument($document->getContent()))->run(); + + } catch (\Exception $err) { + + // throw error, when its not the DocumentNotFoundError + if (!($exception instanceof \horstoeko\zugferd\exception\ZugferdFileNotFoundException)) { + nlog("An error occured while processing InboundEmail document with ZugferdEDocument: {$err}"); + throw $err; + } + + } + + // save additional context of the email to the document + if ($expense) { + + if (!$expense->vendor_id && $expense_vendor) + $expense->vendor_id = $expense_vendor->id; + + $documents = []; + if ($email->body_document !== null) + array_push($documents, $email->body_document); + + $expense->saveQuietly(); + + $this->saveDocuments($documents, $expense); + + continue; + + } + } + + // TODO: check if document can be handled by OCR solution + + // create expense just from document + $expense = ExpenseFactory::create($company->id, $company->owner()->id); + + $expense->public_notes = $email->subject; + $expense->private_notes = $email->text_body; + $expense->date = $email->date; + + if ($expense_vendor) + $expense->vendor_id = $expense_vendor->id; + + $documents = []; + array_push($documents, $document); + if ($email->body_document !== null) + array_push($documents, $email->body_document); + + $expense->saveQuietly(); + + $this->saveDocuments($documents, $expense); + + event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); // @turbo124 please check, I copied from API-Controller + event('eloquent.created: App\Models\Expense', $expense); // @turbo124 please check, I copied from API-Controller + + } } // HELPERS From e204788ef61b0998212b5ae1bfa0f53433ba29d3 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 22 May 2024 07:14:53 +0200 Subject: [PATCH 077/119] minor code cleanups --- app/Services/InboundMail/InboundMailEngine.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index cb3399dba2f0..9d94d7bff8bc 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -176,13 +176,14 @@ class InboundMailEngine // handle documents seperatly foreach ($email->documents as $document) { + /** @var \App\Models\Expense $expense */ $expense = null; // TODO: check if document is a ZugferdEDocument and can be handled that way if ($document->extension() === 'pdf' || $document->extension() === 'xml') { try { - $expense = (new ZugferdEDocument($document->getContent()))->run(); + $expense = (new ZugferdEDocument($document->get()))->run(); } catch (\Exception $err) { From e586455be3dcbda3ca18eba2fe5d8e9b1fc3ed5d Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 22 Jun 2024 15:02:53 +0200 Subject: [PATCH 078/119] rewrite EDocument service structure --- app/Console/Kernel.php | 1 - app/Jobs/EDocument/ImportEDocument.php | 25 ++------- .../EDocument/Imports/ParseEDocument.php | 52 +++++++++++++++++++ 3 files changed, 56 insertions(+), 22 deletions(-) create mode 100644 app/Services/EDocument/Imports/ParseEDocument.php diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 092404a870f9..2c1a731228ad 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -129,7 +129,6 @@ class Kernel extends ConsoleKernel $schedule->command('queue:restart')->everyFiveMinutes()->withoutOverlapping(); } - } /** diff --git a/app/Jobs/EDocument/ImportEDocument.php b/app/Jobs/EDocument/ImportEDocument.php index 13fb22de94f8..59d8e7b99596 100644 --- a/app/Jobs/EDocument/ImportEDocument.php +++ b/app/Jobs/EDocument/ImportEDocument.php @@ -12,7 +12,7 @@ namespace App\Jobs\EDocument; use App\Models\Expense; -use App\Services\EDocument\Imports\ZugferdEDocument; +use App\Services\EDocument\Imports\ParseEDocument; use Exception; use Illuminate\Bus\Queueable; use Illuminate\Queue\SerializesModels; @@ -28,13 +28,10 @@ class ImportEDocument implements ShouldQueue use SerializesModels; public $deleteWhenMissingModels = true; - private string $file_name; - private readonly string $file_content; - public function __construct(string $file_content, string $file_name) + public function __construct(private readonly string $file_content, private string $file_name) { - $this->file_content = $file_content; - $this->file_name = $file_name; + } /** @@ -45,20 +42,6 @@ class ImportEDocument implements ShouldQueue */ public function handle(): Expense { - if (str_contains($this->file_name, ".xml")){ - switch (true) { - case stristr($this->file_content, "urn:cen.eu:en16931:2017"): - case stristr($this->file_content, "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_3.0"): - case stristr($this->file_content, "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.1"): - case stristr($this->file_content, "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.0"): - return (new ZugferdEDocument($this->file_content, $this->file_name))->run(); - default: - throw new Exception("E-Invoice standard not supported"); - } - } - else { - throw new Exception("File type not supported"); - } - + return (new ParseEDocument($this->file_content, $this->file_name))->run(); } } diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php new file mode 100644 index 000000000000..89ad20986eb7 --- /dev/null +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -0,0 +1,52 @@ +file_name, ".xml")) { + switch (true) { + case stristr($this->file_content, "urn:cen.eu:en16931:2017"): + case stristr($this->file_content, "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_3.0"): + case stristr($this->file_content, "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.1"): + case stristr($this->file_content, "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.0"): + return (new ZugferdEDocument($this->file_content, $this->file_name))->run(); + default: + throw new Exception("E-Invoice standard not supported"); + } + } else { + throw new Exception("File type not supported"); + } + } +} + From 45e2b6aeb8262893feeb872c21d1382ef9a5cd0a Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 22 Jun 2024 15:33:53 +0200 Subject: [PATCH 079/119] integrate ParseEDocument to InboundMailEngine --- .../InboundMail/InboundMailEngine.php | 96 +++++++++---------- 1 file changed, 43 insertions(+), 53 deletions(-) diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index 9d94d7bff8bc..c7eb03b87117 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -11,7 +11,6 @@ namespace App\Services\InboundMail; -use App\Events\Expense\ExpenseWasCreated; use App\Factory\ExpenseFactory; use App\Jobs\Util\SystemLogger; use App\Libraries\MultiDB; @@ -19,8 +18,8 @@ use App\Models\ClientContact; use App\Models\Company; use App\Models\SystemLog; use App\Models\VendorContact; +use App\Services\EDocument\Imports\ParseEDocument; use App\Services\InboundMail\InboundMail; -use App\Utils\Ninja; use App\Utils\TempFile; use App\Utils\Traits\GeneratesCounter; use App\Utils\Traits\SavesDocuments; @@ -167,78 +166,69 @@ class InboundMailEngine return; } - // handle vendor assignment - $expense_vendor = $this->getVendor($company, $email); + /** @var \App\Models\Expense $expense */ + $expense = null; - // handle documents - $this->processHtmlBodyToDocument($email); - - // handle documents seperatly + // check documents for EDocument xml foreach ($email->documents as $document) { - /** @var \App\Models\Expense $expense */ - $expense = null; + // check if document can be parsed to an expense + try { - // TODO: check if document is a ZugferdEDocument and can be handled that way - if ($document->extension() === 'pdf' || $document->extension() === 'xml') { - try { + $expense_obj = (new ParseEDocument($document->get(), $document->getFilename()))->run(); - $expense = (new ZugferdEDocument($document->get()))->run(); + // throw error, when multiple parseable files are registered + if ($expense && $expense_obj) + throw new \Exception('Multiple parseable Invoice documents found in email. Please use only one Invoice document per email.'); - } catch (\Exception $err) { + $expense = $expense_obj; - // throw error, when its not the DocumentNotFoundError - if (!($exception instanceof \horstoeko\zugferd\exception\ZugferdFileNotFoundException)) { - nlog("An error occured while processing InboundEmail document with ZugferdEDocument: {$err}"); + } catch (\Exception $err) { + // throw error, only, when its not expected + switch (true) { + case ($err->getMessage() === 'E-Invoice standard not supported'): + case ($err->getMessage() === 'File type not supported'): + break; + default: throw $err; - } - - } - - // save additional context of the email to the document - if ($expense) { - - if (!$expense->vendor_id && $expense_vendor) - $expense->vendor_id = $expense_vendor->id; - - $documents = []; - if ($email->body_document !== null) - array_push($documents, $email->body_document); - - $expense->saveQuietly(); - - $this->saveDocuments($documents, $expense); - - continue; - } } - // TODO: check if document can be handled by OCR solution + } - // create expense just from document + // populate missing data with data from email + if (!$expense) $expense = ExpenseFactory::create($company->id, $company->owner()->id); + if (!$expense->public_notes) $expense->public_notes = $email->subject; + + if (!$expense->private_notes) $expense->private_notes = $email->text_body; + + if (!$expense->date) $expense->date = $email->date; + if (!$expense->vendor_id) { + $expense_vendor = $this->getVendor($company, $email); + if ($expense_vendor) $expense->vendor_id = $expense_vendor->id; - - $documents = []; - array_push($documents, $document); - if ($email->body_document !== null) - array_push($documents, $email->body_document); - - $expense->saveQuietly(); - - $this->saveDocuments($documents, $expense); - - event(new ExpenseWasCreated($expense, $expense->company, Ninja::eventVars(null))); // @turbo124 please check, I copied from API-Controller - event('eloquent.created: App\Models\Expense', $expense); // @turbo124 please check, I copied from API-Controller - } + + // handle documents + $documents = []; + array_push($documents, $document); + + // handle email document + $this->processHtmlBodyToDocument($email); + if ($email->body_document !== null) + array_push($documents, $email->body_document); + + $expense->saveQuietly(); + + $this->saveDocuments($documents, $expense); + } // HELPERS From d0cad92fce0b266e49572acc72900c2ef99e8ebe Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 22 Jun 2024 15:54:33 +0200 Subject: [PATCH 080/119] fix composer issue --- composer.lock | 83 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 7 deletions(-) diff --git a/composer.lock b/composer.lock index 5e620fc4ea5d..55c00a171257 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "628af85e7b23ea07b68733208d210c92", + "content-hash": "f0c032698a2cafaf6d112d0fb27cb010", "packages": [ { "name": "adrienrn/php-mimetyper", @@ -8296,6 +8296,75 @@ }, "time": "2024-04-08T12:52:34+00:00" }, + { + "name": "php-http/client-common", + "version": "2.7.1", + "source": { + "type": "git", + "url": "https://github.com/php-http/client-common.git", + "reference": "1e19c059b0e4d5f717bf5d524d616165aeab0612" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-http/client-common/zipball/1e19c059b0e4d5f717bf5d524d616165aeab0612", + "reference": "1e19c059b0e4d5f717bf5d524d616165aeab0612", + "shasum": "" + }, + "require": { + "php": "^7.1 || ^8.0", + "php-http/httplug": "^2.0", + "php-http/message": "^1.6", + "psr/http-client": "^1.0", + "psr/http-factory": "^1.0", + "psr/http-message": "^1.0 || ^2.0", + "symfony/options-resolver": "~4.0.15 || ~4.1.9 || ^4.2.1 || ^5.0 || ^6.0 || ^7.0", + "symfony/polyfill-php80": "^1.17" + }, + "require-dev": { + "doctrine/instantiator": "^1.1", + "guzzlehttp/psr7": "^1.4", + "nyholm/psr7": "^1.2", + "phpspec/phpspec": "^5.1 || ^6.3 || ^7.1", + "phpspec/prophecy": "^1.10.2", + "phpunit/phpunit": "^7.5.20 || ^8.5.33 || ^9.6.7" + }, + "suggest": { + "ext-json": "To detect JSON responses with the ContentTypePlugin", + "ext-libxml": "To detect XML responses with the ContentTypePlugin", + "php-http/cache-plugin": "PSR-6 Cache plugin", + "php-http/logger-plugin": "PSR-3 Logger plugin", + "php-http/stopwatch-plugin": "Symfony Stopwatch plugin" + }, + "type": "library", + "autoload": { + "psr-4": { + "Http\\Client\\Common\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com" + } + ], + "description": "Common HTTP Client implementations and tools for HTTPlug", + "homepage": "http://httplug.io", + "keywords": [ + "client", + "common", + "http", + "httplug" + ], + "support": { + "issues": "https://github.com/php-http/client-common/issues", + "source": "https://github.com/php-http/client-common/tree/2.7.1" + }, + "time": "2023-11-30T10:31:25+00:00" + }, { "name": "php-http/discovery", "version": "1.19.4", @@ -8620,16 +8689,16 @@ }, { "name": "php-http/multipart-stream-builder", - "version": "1.3.0", + "version": "1.3.1", "source": { "type": "git", "url": "https://github.com/php-http/multipart-stream-builder.git", - "reference": "f5938fd135d9fa442cc297dc98481805acfe2b6a" + "reference": "ed56da23b95949ae4747378bed8a5b61a2fdae24" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-http/multipart-stream-builder/zipball/f5938fd135d9fa442cc297dc98481805acfe2b6a", - "reference": "f5938fd135d9fa442cc297dc98481805acfe2b6a", + "url": "https://api.github.com/repos/php-http/multipart-stream-builder/zipball/ed56da23b95949ae4747378bed8a5b61a2fdae24", + "reference": "ed56da23b95949ae4747378bed8a5b61a2fdae24", "shasum": "" }, "require": { @@ -8670,9 +8739,9 @@ ], "support": { "issues": "https://github.com/php-http/multipart-stream-builder/issues", - "source": "https://github.com/php-http/multipart-stream-builder/tree/1.3.0" + "source": "https://github.com/php-http/multipart-stream-builder/tree/1.3.1" }, - "time": "2023-04-28T14:10:22+00:00" + "time": "2024-06-10T14:51:55+00:00" }, { "name": "php-http/promise", From 46f3fd38661dd8c5acf8e0d5110a18a15b75f229 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 22 Jun 2024 17:55:07 +0200 Subject: [PATCH 081/119] wrong implementation use each document for its own expense (or parse) --- .../InboundMail/InboundMailEngine.php | 77 ++++++++++--------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index c7eb03b87117..f2973556811d 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -166,22 +166,29 @@ class InboundMailEngine return; } - /** @var \App\Models\Expense $expense */ - $expense = null; + // prepare data + $expense_vendor = $this->getVendor($company, $email); + $this->processHtmlBodyToDocument($email); - // check documents for EDocument xml + $parsed_expense_ids = []; // used to check if an expense was already matched within this job + + // check documents => optimal when parsed from any source => else create an expense for each document foreach ($email->documents as $document) { + /** @var \App\Models\Expense $expense */ + $expense = null; + // check if document can be parsed to an expense try { - $expense_obj = (new ParseEDocument($document->get(), $document->getFilename()))->run(); + $expense = (new ParseEDocument($document->get(), $document->getFilename()))->run(); - // throw error, when multiple parseable files are registered - if ($expense && $expense_obj) - throw new \Exception('Multiple parseable Invoice documents found in email. Please use only one Invoice document per email.'); - - $expense = $expense_obj; + // check if expense was already matched within this job and skip if true + if (array_search($expense->id, $parsed_expense_ids)) { + $this->saveDocument($document, $expense); + continue; + } + array_push($parsed_expenses, $expense->id); } catch (\Exception $err) { // throw error, only, when its not expected @@ -194,41 +201,35 @@ class InboundMailEngine } } - } + // populate missing data with data from email + if (!$expense) + $expense = ExpenseFactory::create($company->id, $company->owner()->id); - // populate missing data with data from email - if (!$expense) - $expense = ExpenseFactory::create($company->id, $company->owner()->id); + if (!$expense->public_notes) + $expense->public_notes = $email->subject; - if (!$expense->public_notes) - $expense->public_notes = $email->subject; + if (!$expense->private_notes) + $expense->private_notes = $email->text_body; - if (!$expense->private_notes) - $expense->private_notes = $email->text_body; + if (!$expense->date) + $expense->date = $email->date; - if (!$expense->date) - $expense->date = $email->date; - - if (!$expense->vendor_id) { - $expense_vendor = $this->getVendor($company, $email); - - if ($expense_vendor) + if (!$expense->vendor_id && $expense_vendor) $expense->vendor_id = $expense_vendor->id; + + // handle documents + $documents = []; + array_push($documents, $document); + + // handle email document + if ($email->body_document !== null) + array_push($documents, $email->body_document); + + $expense->saveQuietly(); + + $this->saveDocuments($documents, $expense); + } - - // handle documents - $documents = []; - array_push($documents, $document); - - // handle email document - $this->processHtmlBodyToDocument($email); - if ($email->body_document !== null) - array_push($documents, $email->body_document); - - $expense->saveQuietly(); - - $this->saveDocuments($documents, $expense); - } // HELPERS From fc7d84dc244ac0a3ea03d2d69e3a19eeba071761 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 22 Jun 2024 18:04:35 +0200 Subject: [PATCH 082/119] minor fixes --- .../EDocument/Imports/ZugferdEDocument.php | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/app/Services/EDocument/Imports/ZugferdEDocument.php b/app/Services/EDocument/Imports/ZugferdEDocument.php index 49fb780ffab3..0794a2a8a0b4 100644 --- a/app/Services/EDocument/Imports/ZugferdEDocument.php +++ b/app/Services/EDocument/Imports/ZugferdEDocument.php @@ -20,12 +20,15 @@ use App\Models\Expense; use App\Models\Vendor; use App\Services\AbstractService; use App\Utils\TempFile; +use App\Utils\Traits\SavesDocuments; use Exception; use horstoeko\zugferd\ZugferdDocumentReader; use horstoeko\zugferdvisualizer\renderer\ZugferdVisualizerLaravelRenderer; use horstoeko\zugferdvisualizer\ZugferdVisualizer; -class ZugferdEDocument extends AbstractService { +class ZugferdEDocument extends AbstractService +{ + use SavesDocuments; public ZugferdDocumentReader|string $document; /** @@ -66,10 +69,10 @@ class ZugferdEDocument extends AbstractService { $expense->save(); $origin_file = TempFile::UploadedFileFromRaw($this->tempdocument, $this->documentname, "application/xml"); - (new UploadFile($origin_file, UploadFile::DOCUMENT, $user, $expense->company, $expense, null, false))->handle(); - $uploaded_file = TempFile::UploadedFileFromRaw($visualizer->renderPdf(), $documentno."_visualiser.pdf", "application/pdf"); - (new UploadFile($uploaded_file, UploadFile::DOCUMENT, $user, $expense->company, $expense, null, false))->handle(); + $uploaded_file = TempFile::UploadedFileFromRaw($visualizer->renderPdf(), $documentno . "_visualiser.pdf", "application/pdf"); + $this->saveDocuments([$origin_file, $uploaded_file], $expense); $expense->save(); + if ($taxCurrency && $taxCurrency != $invoiceCurrency) { $expense->private_notes = ctrans("texts.tax_currency_mismatch"); } @@ -115,8 +118,7 @@ class ZugferdEDocument extends AbstractService { $expense->vendor_id = $vendor->id; } $expense->transaction_reference = $documentno; - } - else { + } else { // The document exists as an expense // Handle accordingly nlog("Document already exists"); From 2a7eb83965b81b06b3702da53415e3c4b883dc91 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 22 Jun 2024 18:52:25 +0200 Subject: [PATCH 083/119] several reworks + wip mindee --- app/Http/Controllers/ExpenseController.php | 9 +- app/Jobs/EDocument/ImportEDocument.php | 9 +- .../EDocument/Imports/MindeeEDocument.php | 148 ++++++++++++++++++ .../EDocument/Imports/ParseEDocument.php | 45 ++++-- .../EDocument/Imports/ZugferdEDocument.php | 15 +- .../InboundMail/InboundMailEngine.php | 23 +-- config/services.php | 7 + 7 files changed, 221 insertions(+), 35 deletions(-) create mode 100644 app/Services/EDocument/Imports/MindeeEDocument.php diff --git a/app/Http/Controllers/ExpenseController.php b/app/Http/Controllers/ExpenseController.php index 0553b5dd3aec..44444990de0e 100644 --- a/app/Http/Controllers/ExpenseController.php +++ b/app/Http/Controllers/ExpenseController.php @@ -497,7 +497,7 @@ class ExpenseController extends BaseController $expenses = Expense::withTrashed()->find($request->ids); - if($request->action == 'bulk_categorize' && $user->can('edit', $expenses->first())) { + if ($request->action == 'bulk_categorize' && $user->can('edit', $expenses->first())) { $this->expense_repo->categorize($expenses, $request->category_id); $expenses = collect([]); } @@ -573,7 +573,7 @@ class ExpenseController extends BaseController */ public function upload(UploadExpenseRequest $request, Expense $expense) { - if (! $this->checkFeature(Account::FEATURE_DOCUMENTS)) { + if (!$this->checkFeature(Account::FEATURE_DOCUMENTS)) { return $this->featureFailure(); } @@ -587,9 +587,8 @@ class ExpenseController extends BaseController public function edocument(EDocumentRequest $request): string { if ($request->hasFile("documents")) { - return (new ImportEDocument($request->file("documents")[0]->get(), $request->file("documents")[0]->getClientOriginalName()))->handle(); - } - else { + return (new ImportEDocument($request->file("documents")[0]->get(), $request->file("documents")[0]->getClientOriginalName(), $request->file("documents")[0]->getMimeType()))->handle(); + } else { return "No file found"; } diff --git a/app/Jobs/EDocument/ImportEDocument.php b/app/Jobs/EDocument/ImportEDocument.php index 59d8e7b99596..1d5e379db683 100644 --- a/app/Jobs/EDocument/ImportEDocument.php +++ b/app/Jobs/EDocument/ImportEDocument.php @@ -13,6 +13,7 @@ namespace App\Jobs\EDocument; use App\Models\Expense; use App\Services\EDocument\Imports\ParseEDocument; +use App\Utils\TempFile; use Exception; use Illuminate\Bus\Queueable; use Illuminate\Queue\SerializesModels; @@ -29,7 +30,7 @@ class ImportEDocument implements ShouldQueue public $deleteWhenMissingModels = true; - public function __construct(private readonly string $file_content, private string $file_name) + public function __construct(private readonly string $file_content, private string $file_name, private string $file_mime_type) { } @@ -42,6 +43,10 @@ class ImportEDocument implements ShouldQueue */ public function handle(): Expense { - return (new ParseEDocument($this->file_content, $this->file_name))->run(); + + $file = TempFile::UploadedFileFromRaw($this->file_content, $this->file_name, $this->file_mime_type); + + return (new ParseEDocument($file))->run(); + } } diff --git a/app/Services/EDocument/Imports/MindeeEDocument.php b/app/Services/EDocument/Imports/MindeeEDocument.php new file mode 100644 index 000000000000..456e393cf499 --- /dev/null +++ b/app/Services/EDocument/Imports/MindeeEDocument.php @@ -0,0 +1,148 @@ +user(); + + $api_key = config('services.mindee.api_key'); + + if (!$api_key) + throw new Exception('Mindee API key not configured'); + + // check global contingent + // TODO: add contingent for each company + + + $mindeeClient = new Client($api_key); + + + // Load a file from disk + $inputSource = $mindeeClient->sourceFromFile($this->file); + + // Parse the file + $apiResponse = $mindeeClient->parse(InvoiceV4::class, $inputSource); + + $expense = Expense::where('amount', $grandTotalAmount)->where("transaction_reference", $documentno)->whereDate("date", $documentdate)->first(); + if (empty($expense)) { + // The document does not exist as an expense + // Handle accordingly + $visualizer = new ZugferdVisualizer($this->document); + $visualizer->setDefaultTemplate(); + $visualizer->setRenderer(app(ZugferdVisualizerLaravelRenderer::class)); + $visualizer->setPdfFontDefault("arial"); + $visualizer->setPdfPaperSize('A4-P'); + $visualizer->setTemplate('edocument.xinvoice'); + + $expense = ExpenseFactory::create($user->company()->id, $user->id); + $expense->date = $documentdate; + $expense->user_id = $user->id; + $expense->company_id = $user->company->id; + $expense->public_notes = $documentno; + $expense->currency_id = Currency::whereCode($invoiceCurrency)->first()->id; + $expense->save(); + + $documents = []; + array_push($documents, $this->file); + if ($this->file->getExtension() == "xml") + array_push($documents, TempFile::UploadedFileFromRaw($visualizer->renderPdf(), $documentno . "_visualiser.pdf", "application/pdf")); + $this->saveDocuments($documents, $expense); + $expense->saveQuietly(); + + if ($taxCurrency && $taxCurrency != $invoiceCurrency) { + $expense->private_notes = ctrans("texts.tax_currency_mismatch"); + } + $expense->uses_inclusive_taxes = True; + $expense->amount = $grandTotalAmount; + $counter = 1; + if ($this->document->firstDocumentTax()) { + do { + $this->document->getDocumentTax($categoryCode, $typeCode, $basisAmount, $calculatedAmount, $rateApplicablePercent, $exemptionReason, $exemptionReasonCode, $lineTotalBasisAmount, $allowanceChargeBasisAmount, $taxPointDate, $dueDateTypeCode); + $expense->{"tax_amount$counter"} = $calculatedAmount; + $expense->{"tax_rate$counter"} = $rateApplicablePercent; + $counter++; + } while ($this->document->nextDocumentTax()); + } + $this->document->getDocumentSeller($name, $buyer_id, $buyer_description); + $this->document->getDocumentSellerContact($person_name, $person_department, $contact_phone, $contact_fax, $contact_email); + $this->document->getDocumentSellerAddress($address_1, $address_2, $address_3, $postcode, $city, $country, $subdivision); + $this->document->getDocumentSellerTaxRegistration($taxtype); + $taxid = null; + if (array_key_exists("VA", $taxtype)) { + $taxid = $taxtype["VA"]; + } + $vendor = Vendor::where('vat_number', $taxid)->first(); + + if (!empty($vendor)) { + // Vendor found + $expense->vendor_id = $vendor->id; + } else { + $vendor = VendorFactory::create($user->company()->id, $user->id); + $vendor->name = $name; + if ($taxid != null) { + $vendor->vat_number = $taxid; + } + $vendor->currency_id = Currency::whereCode($invoiceCurrency)->first()->id; + $vendor->phone = $contact_phone; + $vendor->address1 = $address_1; + $vendor->address2 = $address_2; + $vendor->city = $city; + $vendor->postal_code = $postcode; + $vendor->country_id = Country::where('iso_3166_2', $country)->first()->id; + + $vendor->save(); + $expense->vendor_id = $vendor->id; + } + $expense->transaction_reference = $documentno; + } else { + // The document exists as an expense + // Handle accordingly + nlog("Document already exists"); + $expense->private_notes = $expense->private_notes . ctrans("texts.edocument_import_already_exists", ["date" => time()]); + } + $expense->save(); + return $expense; + } +} + diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index 89ad20986eb7..bcda0ece3c6f 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -14,6 +14,7 @@ namespace App\Services\EDocument\Imports; use App\Models\Expense; use App\Services\AbstractService; use Exception; +use Illuminate\Http\UploadedFile; class ParseEDocument extends AbstractService { @@ -21,32 +22,50 @@ class ParseEDocument extends AbstractService /** * @throws Exception */ - public function __construct(private string $file_content, private string $file_name) + public function __construct(private UploadedFile $file) { } /** * Execute the service. + * the service will parse the file with all available libraries of the system and will return an expense, when possible * * @return Expense * @throws \Exception */ public function run(): Expense { - if (str_contains($this->file_name, ".xml")) { - switch (true) { - case stristr($this->file_content, "urn:cen.eu:en16931:2017"): - case stristr($this->file_content, "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_3.0"): - case stristr($this->file_content, "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.1"): - case stristr($this->file_content, "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.0"): - return (new ZugferdEDocument($this->file_content, $this->file_name))->run(); - default: - throw new Exception("E-Invoice standard not supported"); - } - } else { - throw new Exception("File type not supported"); + + $expense = null; + + // try to parse via Zugferd lib + $zugferd_exception = null; + try { + $expense = (new ZugferdEDocument($this->file))->run(); + } catch (Exception $e) { + $zugferd_exception = $e; } + + // try to parse via mindee lib + $mindee_exception = null; + try { + $expense = (new MindeeEDocument($this->file))->run(); + } catch (Exception $e) { + // ignore not available exceptions + $mindee_exception = $e; + } + + // return expense, when available and supress any errors occured before + if ($expense) + return $expense; + + // log exceptions and throw error + if ($zugferd_exception) + nlog("Zugferd Exception: " . $zugferd_exception->getMessage()); + if ($mindee_exception) + nlog("Mindee Exception: " . $zugferd_exception->getMessage()); + throw new Exception("File type not supported or issue while parsing"); } } diff --git a/app/Services/EDocument/Imports/ZugferdEDocument.php b/app/Services/EDocument/Imports/ZugferdEDocument.php index 0794a2a8a0b4..21a6388ff3ce 100644 --- a/app/Services/EDocument/Imports/ZugferdEDocument.php +++ b/app/Services/EDocument/Imports/ZugferdEDocument.php @@ -25,6 +25,7 @@ use Exception; use horstoeko\zugferd\ZugferdDocumentReader; use horstoeko\zugferdvisualizer\renderer\ZugferdVisualizerLaravelRenderer; use horstoeko\zugferdvisualizer\ZugferdVisualizer; +use Illuminate\Http\UploadedFile; class ZugferdEDocument extends AbstractService { @@ -34,7 +35,7 @@ class ZugferdEDocument extends AbstractService /** * @throws Exception */ - public function __construct(public string $tempdocument, public string $documentname) + public function __construct(public UploadedFile $file) { # curl -X POST http://localhost:8000/api/v1/edocument/upload -H "Content-Type: multipart/form-data" -H "X-API-TOKEN: 7tdDdkz987H3AYIWhNGXy8jTjJIoDhkAclCDLE26cTCj1KYX7EBHC66VEitJwWhn" -H "X-Requested-With: XMLHttpRequest" -F _method=PUT -F documents[]=@einvoice.xml } @@ -45,7 +46,7 @@ class ZugferdEDocument extends AbstractService public function run(): Expense { $user = auth()->user(); - $this->document = ZugferdDocumentReader::readAndGuessFromContent($this->tempdocument); + $this->document = ZugferdDocumentReader::readAndGuessFromContent($this->file->get()); $this->document->getDocumentInformation($documentno, $documenttypecode, $documentdate, $invoiceCurrency, $taxCurrency, $documentname, $documentlanguage, $effectiveSpecifiedPeriod); $this->document->getDocumentSummation($grandTotalAmount, $duePayableAmount, $lineTotalAmount, $chargeTotalAmount, $allowanceTotalAmount, $taxBasisTotalAmount, $taxTotalAmount, $roundingAmount, $totalPrepaidAmount); @@ -68,10 +69,12 @@ class ZugferdEDocument extends AbstractService $expense->currency_id = Currency::whereCode($invoiceCurrency)->first()->id; $expense->save(); - $origin_file = TempFile::UploadedFileFromRaw($this->tempdocument, $this->documentname, "application/xml"); - $uploaded_file = TempFile::UploadedFileFromRaw($visualizer->renderPdf(), $documentno . "_visualiser.pdf", "application/pdf"); - $this->saveDocuments([$origin_file, $uploaded_file], $expense); - $expense->save(); + $documents = []; + array_push($documents, $this->file); + if ($this->file->getExtension() == "xml") + array_push($documents, TempFile::UploadedFileFromRaw($visualizer->renderPdf(), $documentno . "_visualiser.pdf", "application/pdf")); + $this->saveDocuments($documents, $expense); + $expense->saveQuietly(); if ($taxCurrency && $taxCurrency != $invoiceCurrency) { $expense->private_notes = ctrans("texts.tax_currency_mismatch"); diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index f2973556811d..8c5acf764443 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -181,13 +181,12 @@ class InboundMailEngine // check if document can be parsed to an expense try { - $expense = (new ParseEDocument($document->get(), $document->getFilename()))->run(); + $expense = (new ParseEDocument($document))->run(); // check if expense was already matched within this job and skip if true - if (array_search($expense->id, $parsed_expense_ids)) { - $this->saveDocument($document, $expense); + if (array_search($expense->id, $parsed_expense_ids)) continue; - } + array_push($parsed_expenses, $expense->id); } catch (\Exception $err) { @@ -201,6 +200,8 @@ class InboundMailEngine } } + $is_imported_by_parser = array_search($expense->id, $parsed_expense_ids); + // populate missing data with data from email if (!$expense) $expense = ExpenseFactory::create($company->id, $company->owner()->id); @@ -217,18 +218,22 @@ class InboundMailEngine if (!$expense->vendor_id && $expense_vendor) $expense->vendor_id = $expense_vendor->id; - // handle documents + // save document only, when not imported by parser $documents = []; - array_push($documents, $document); + if ($is_imported_by_parser) + array_push($documents, $document); - // handle email document + // email document if ($email->body_document !== null) array_push($documents, $email->body_document); - $expense->saveQuietly(); - $this->saveDocuments($documents, $expense); + if ($is_imported_by_parser) + $expense->saveQuietly(); + else + $expense->save(); + } } diff --git a/config/services.php b/config/services.php index b0d9851df34e..48e8b75f4769 100644 --- a/config/services.php +++ b/config/services.php @@ -51,6 +51,13 @@ return [ 'redirect' => env('MICROSOFT_REDIRECT_URI'), ], + 'mindee' => [ + 'api_key' => env('MINDEE_API_KEY'), + 'global_contingent_month' => env('MINDEE_GLOBAL_CONTINGENT_MONTH', 1000), + 'company_contingent_month' => env('MINDEE_COMPANY_CONTINGENT_MONTH', 500), + 'company_contingent_month_enterprise' => env('MINDEE_COMPANY_CONTINGENT_MONTH', 500), + ], + 'apple' => [ 'client_id' => env('APPLE_CLIENT_ID'), 'client_secret' => env('APPLE_CLIENT_SECRET'), From f480108c25fb7e59085afdb4cb8853cf1488330d Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 22 Jun 2024 19:58:55 +0200 Subject: [PATCH 084/119] wip: adding mindee --- .../EDocument/Imports/MindeeEDocument.php | 36 +++++-------- .../InboundMail/InboundMailEngine.php | 8 +-- composer.json | 7 +-- composer.lock | 52 +++++++++++++++++-- 4 files changed, 71 insertions(+), 32 deletions(-) diff --git a/app/Services/EDocument/Imports/MindeeEDocument.php b/app/Services/EDocument/Imports/MindeeEDocument.php index 456e393cf499..a1d8488cc0b4 100644 --- a/app/Services/EDocument/Imports/MindeeEDocument.php +++ b/app/Services/EDocument/Imports/MindeeEDocument.php @@ -13,7 +13,6 @@ namespace App\Services\EDocument\Imports; use App\Factory\ExpenseFactory; use App\Factory\VendorFactory; -use App\Jobs\Util\UploadFile; use App\Models\Country; use App\Models\Currency; use App\Models\Expense; @@ -29,7 +28,6 @@ use Illuminate\Http\UploadedFile; class MindeeEDocument extends AbstractService { use SavesDocuments; - public ZugferdDocumentReader|string $document; /** * @throws Exception @@ -62,19 +60,21 @@ class MindeeEDocument extends AbstractService $inputSource = $mindeeClient->sourceFromFile($this->file); // Parse the file - $apiResponse = $mindeeClient->parse(InvoiceV4::class, $inputSource); + $result = $mindeeClient->parse(InvoiceV4::class, $inputSource); + + /** @var \Mindee\Product\Invoice\InvoiceV4Document $prediction */ + $prediction = $result->document->inference->prediction; + + $grandTotalAmount = $prediction->totalAmount->value; + $documentno = $prediction->invoiceNumber->value; + $documentdate = $prediction->date->value; + $invoiceCurrency = $prediction->locale->currency; + $country = $prediction->locale->country; $expense = Expense::where('amount', $grandTotalAmount)->where("transaction_reference", $documentno)->whereDate("date", $documentdate)->first(); if (empty($expense)) { // The document does not exist as an expense // Handle accordingly - $visualizer = new ZugferdVisualizer($this->document); - $visualizer->setDefaultTemplate(); - $visualizer->setRenderer(app(ZugferdVisualizerLaravelRenderer::class)); - $visualizer->setPdfFontDefault("arial"); - $visualizer->setPdfPaperSize('A4-P'); - $visualizer->setTemplate('edocument.xinvoice'); - $expense = ExpenseFactory::create($user->company()->id, $user->id); $expense->date = $documentdate; $expense->user_id = $user->id; @@ -83,11 +83,7 @@ class MindeeEDocument extends AbstractService $expense->currency_id = Currency::whereCode($invoiceCurrency)->first()->id; $expense->save(); - $documents = []; - array_push($documents, $this->file); - if ($this->file->getExtension() == "xml") - array_push($documents, TempFile::UploadedFileFromRaw($visualizer->renderPdf(), $documentno . "_visualiser.pdf", "application/pdf")); - $this->saveDocuments($documents, $expense); + $this->saveDocument($this->file, $expense); $expense->saveQuietly(); if ($taxCurrency && $taxCurrency != $invoiceCurrency) { @@ -104,10 +100,6 @@ class MindeeEDocument extends AbstractService $counter++; } while ($this->document->nextDocumentTax()); } - $this->document->getDocumentSeller($name, $buyer_id, $buyer_description); - $this->document->getDocumentSellerContact($person_name, $person_department, $contact_phone, $contact_fax, $contact_email); - $this->document->getDocumentSellerAddress($address_1, $address_2, $address_3, $postcode, $city, $country, $subdivision); - $this->document->getDocumentSellerTaxRegistration($taxtype); $taxid = null; if (array_key_exists("VA", $taxtype)) { $taxid = $taxtype["VA"]; @@ -124,12 +116,12 @@ class MindeeEDocument extends AbstractService $vendor->vat_number = $taxid; } $vendor->currency_id = Currency::whereCode($invoiceCurrency)->first()->id; - $vendor->phone = $contact_phone; - $vendor->address1 = $address_1; + $vendor->phone = $prediction->supplierPhoneNumber; + $vendor->address1 = $address_1; // TODO: we only have the full address string $vendor->address2 = $address_2; $vendor->city = $city; $vendor->postal_code = $postcode; - $vendor->country_id = Country::where('iso_3166_2', $country)->first()->id; + $vendor->country_id = Country::where('iso_3166_2', $country)->first()->id; // could be 2 or 3 length $vendor->save(); $expense->vendor_id = $vendor->id; diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index 8c5acf764443..d9a6deb8423c 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -206,11 +206,11 @@ class InboundMailEngine if (!$expense) $expense = ExpenseFactory::create($company->id, $company->owner()->id); - if (!$expense->public_notes) - $expense->public_notes = $email->subject; + if ($is_imported_by_parser) + $expense->public_notes = $expense->public_notes . $email->subject; - if (!$expense->private_notes) - $expense->private_notes = $email->text_body; + if ($is_imported_by_parser) + $expense->private_notes = $expense->private_notes . $email->text_body; if (!$expense->date) $expense->date = $email->date; diff --git a/composer.json b/composer.json index 0b57e8ccacb5..3c6c9d9699d0 100644 --- a/composer.json +++ b/composer.json @@ -32,10 +32,10 @@ "type": "project", "require": { "php": "^8.2", + "ext-curl": "*", "ext-dom": "*", "ext-json": "*", "ext-libxml": "*", - "ext-curl": "*", "afosto/yaac": "^1.4", "asm/php-ansible": "dev-main", "authorizenet/authorizenet": "^2.0", @@ -57,12 +57,12 @@ "hedii/laravel-gelf-logger": "^9", "horstoeko/orderx": "dev-master", "horstoeko/zugferd": "^1", - "horstoeko/zugferdvisualizer":"^1", + "horstoeko/zugferdvisualizer": "^1", "hyvor/php-json-exporter": "^0.0.3", "imdhemy/laravel-purchases": "^1.7", "intervention/image": "^2.5", - "invoiceninja/inspector": "^3.0", "invoiceninja/einvoice": "dev-main", + "invoiceninja/inspector": "^3.0", "invoiceninja/ubl_invoice": "^2", "josemmo/facturae-php": "^1.7", "laracasts/presenter": "^0.2.1", @@ -78,6 +78,7 @@ "livewire/livewire": "^3.0", "mailgun/mailgun-php": "^3.6", "microsoft/microsoft-graph": "^1.69", + "mindee/mindee": "^1.8", "mollie/mollie-api-php": "^2.36", "nelexa/zip": "^4.0", "nordigen/nordigen-php": "^1.1", diff --git a/composer.lock b/composer.lock index 55c00a171257..e0170e6def50 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f0c032698a2cafaf6d112d0fb27cb010", + "content-hash": "bea0d5e6d14872ba2000632b2b8d287d", "packages": [ { "name": "adrienrn/php-mimetyper", @@ -6572,6 +6572,52 @@ }, "time": "2024-01-15T18:49:30+00:00" }, + { + "name": "mindee/mindee", + "version": "v1.8.0", + "source": { + "type": "git", + "url": "https://github.com/mindee/mindee-api-php.git", + "reference": "cf7de6c6e23fa81372f1f1d314bbf93e1f811953" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/mindee/mindee-api-php/zipball/cf7de6c6e23fa81372f1f1d314bbf93e1f811953", + "reference": "cf7de6c6e23fa81372f1f1d314bbf93e1f811953", + "shasum": "" + }, + "require": { + "ext-curl": "*", + "ext-fileinfo": "*", + "ext-json": "*", + "php": ">=7.4", + "symfony/console": ">=5.4" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.38", + "phpunit/phpunit": "^9.6", + "squizlabs/php_codesniffer": "^3.7" + }, + "bin": [ + "bin/mindee" + ], + "type": "library", + "autoload": { + "psr-4": { + "Mindee\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "Mindee Client Library for PHP", + "support": { + "issues": "https://github.com/mindee/mindee-api-php/issues", + "source": "https://github.com/mindee/mindee-api-php/tree/v1.8.0" + }, + "time": "2024-05-31T15:24:47+00:00" + }, { "name": "mollie/mollie-api-php", "version": "v2.68.0", @@ -19770,10 +19816,10 @@ "prefer-lowest": false, "platform": { "php": "^8.2", + "ext-curl": "*", "ext-dom": "*", "ext-json": "*", - "ext-libxml": "*", - "ext-curl": "*" + "ext-libxml": "*" }, "platform-dev": [], "plugin-api-version": "2.6.0" From aa12fe3976c7be9b1303e21ae8afc10f08c7d863 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sat, 22 Jun 2024 20:57:11 +0200 Subject: [PATCH 085/119] wip mindee --- .../EDocument/Imports/MindeeEDocument.php | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/app/Services/EDocument/Imports/MindeeEDocument.php b/app/Services/EDocument/Imports/MindeeEDocument.php index a1d8488cc0b4..e352bf6df39d 100644 --- a/app/Services/EDocument/Imports/MindeeEDocument.php +++ b/app/Services/EDocument/Imports/MindeeEDocument.php @@ -65,6 +65,9 @@ class MindeeEDocument extends AbstractService /** @var \Mindee\Product\Invoice\InvoiceV4Document $prediction */ $prediction = $result->document->inference->prediction; + if ($prediction->documentType !== 'INVOICE') + throw new Exception('Unsupported document type'); + $grandTotalAmount = $prediction->totalAmount->value; $documentno = $prediction->invoiceNumber->value; $documentdate = $prediction->date->value; @@ -86,42 +89,38 @@ class MindeeEDocument extends AbstractService $this->saveDocument($this->file, $expense); $expense->saveQuietly(); - if ($taxCurrency && $taxCurrency != $invoiceCurrency) { - $expense->private_notes = ctrans("texts.tax_currency_mismatch"); - } + // if ($taxCurrency && $taxCurrency != $invoiceCurrency) { + // $expense->private_notes = ctrans("texts.tax_currency_mismatch"); + // } $expense->uses_inclusive_taxes = True; $expense->amount = $grandTotalAmount; $counter = 1; - if ($this->document->firstDocumentTax()) { - do { - $this->document->getDocumentTax($categoryCode, $typeCode, $basisAmount, $calculatedAmount, $rateApplicablePercent, $exemptionReason, $exemptionReasonCode, $lineTotalBasisAmount, $allowanceChargeBasisAmount, $taxPointDate, $dueDateTypeCode); - $expense->{"tax_amount$counter"} = $calculatedAmount; - $expense->{"tax_rate$counter"} = $rateApplicablePercent; - $counter++; - } while ($this->document->nextDocumentTax()); + foreach ($prediction->taxes as $taxesElem) { + $expense->{"tax_amount$counter"} = $taxesElem->amount; + $expense->{"tax_rate$counter"} = $taxesElem->rate; + $counter++; } $taxid = null; if (array_key_exists("VA", $taxtype)) { $taxid = $taxtype["VA"]; } - $vendor = Vendor::where('vat_number', $taxid)->first(); + $vendor = Vendor::where('email', $prediction->supplierEmail)->first(); if (!empty($vendor)) { // Vendor found $expense->vendor_id = $vendor->id; } else { $vendor = VendorFactory::create($user->company()->id, $user->id); - $vendor->name = $name; - if ($taxid != null) { - $vendor->vat_number = $taxid; - } + $vendor->name = $prediction->supplierName; + $vendor->email = $prediction->supplierEmail; + $vendor->currency_id = Currency::whereCode($invoiceCurrency)->first()->id; $vendor->phone = $prediction->supplierPhoneNumber; - $vendor->address1 = $address_1; // TODO: we only have the full address string - $vendor->address2 = $address_2; - $vendor->city = $city; - $vendor->postal_code = $postcode; - $vendor->country_id = Country::where('iso_3166_2', $country)->first()->id; // could be 2 or 3 length + // $vendor->address1 = $address_1; // TODO: we only have the full address string + // $vendor->address2 = $address_2; + // $vendor->city = $city; + // $vendor->postal_code = $postcode; + $vendor->country_id = Country::where('iso_3166_2', $country)->first()->id || Country::where('iso_3166_3', $country)->first()->id; // could be 2 or 3 length $vendor->save(); $expense->vendor_id = $vendor->id; From e3da9914ba66dba493ab282a507ffd4313853c28 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 23 Jun 2024 09:32:51 +0200 Subject: [PATCH 086/119] rework mindee limits --- .../EDocument/Imports/MindeeEDocument.php | 61 ++++++++++++------- .../EDocument/Imports/ZugferdEDocument.php | 3 +- config/services.php | 7 ++- 3 files changed, 45 insertions(+), 26 deletions(-) diff --git a/app/Services/EDocument/Imports/MindeeEDocument.php b/app/Services/EDocument/Imports/MindeeEDocument.php index e352bf6df39d..7e5245337998 100644 --- a/app/Services/EDocument/Imports/MindeeEDocument.php +++ b/app/Services/EDocument/Imports/MindeeEDocument.php @@ -20,6 +20,7 @@ use App\Models\Vendor; use App\Services\AbstractService; use App\Utils\TempFile; use App\Utils\Traits\SavesDocuments; +use Cache; use Exception; use Mindee\Client; use Mindee\Product\Invoice\InvoiceV4; @@ -45,22 +46,15 @@ class MindeeEDocument extends AbstractService $user = auth()->user(); $api_key = config('services.mindee.api_key'); - if (!$api_key) throw new Exception('Mindee API key not configured'); + $this->checkLimits(); - // check global contingent - // TODO: add contingent for each company - - + // perform parsing $mindeeClient = new Client($api_key); - - - // Load a file from disk $inputSource = $mindeeClient->sourceFromFile($this->file); - - // Parse the file $result = $mindeeClient->parse(InvoiceV4::class, $inputSource); + $this->incrementRequestCounts(); /** @var \Mindee\Product\Invoice\InvoiceV4Document $prediction */ $prediction = $result->document->inference->prediction; @@ -86,12 +80,12 @@ class MindeeEDocument extends AbstractService $expense->currency_id = Currency::whereCode($invoiceCurrency)->first()->id; $expense->save(); - $this->saveDocument($this->file, $expense); + $this->saveDocuments([ + $this->file, + TempFile::UploadedFileFromRaw(strval($result->document), $documentno . "_mindee_orc_result.txt", "text/plain") + ], $expense); $expense->saveQuietly(); - // if ($taxCurrency && $taxCurrency != $invoiceCurrency) { - // $expense->private_notes = ctrans("texts.tax_currency_mismatch"); - // } $expense->uses_inclusive_taxes = True; $expense->amount = $grandTotalAmount; $counter = 1; @@ -100,10 +94,7 @@ class MindeeEDocument extends AbstractService $expense->{"tax_rate$counter"} = $taxesElem->rate; $counter++; } - $taxid = null; - if (array_key_exists("VA", $taxtype)) { - $taxid = $taxtype["VA"]; - } + $vendor = Vendor::where('email', $prediction->supplierEmail)->first(); if (!empty($vendor)) { @@ -114,13 +105,13 @@ class MindeeEDocument extends AbstractService $vendor->name = $prediction->supplierName; $vendor->email = $prediction->supplierEmail; - $vendor->currency_id = Currency::whereCode($invoiceCurrency)->first()->id; + $vendor->currency_id = Currency::whereCode($invoiceCurrency)->first()?->id; $vendor->phone = $prediction->supplierPhoneNumber; - // $vendor->address1 = $address_1; // TODO: we only have the full address string + // $vendor->address1 = $address_1; // TODO: we only have the full address string from mindee returned // $vendor->address2 = $address_2; // $vendor->city = $city; // $vendor->postal_code = $postcode; - $vendor->country_id = Country::where('iso_3166_2', $country)->first()->id || Country::where('iso_3166_3', $country)->first()->id; // could be 2 or 3 length + $vendor->country_id = Country::where('iso_3166_2', $country)->first()?->id || Country::where('iso_3166_3', $country)->first()?->id || null; // could be 2 or 3 length $vendor->save(); $expense->vendor_id = $vendor->id; @@ -135,5 +126,33 @@ class MindeeEDocument extends AbstractService $expense->save(); return $expense; } + + private function checkLimits() + { + $user = auth()->user(); + + Cache::add('mindeeTotalDailyRequests', 0, now()->endOfDay()); + Cache::add('mindeeTotalMonthlyRequests', 0, now()->endOfMonth()); + Cache::add('mindeeAccountDailyRequests' . $user->company->account->id, 0, now()->endOfDay()); + Cache::add('mindeeAccountMonthlyRequests' . $user->company->account->id, 0, now()->endOfMonth()); + if (config('services.mindee.daily_limit') != 0 && Cache::get('mindeeTotalDailyRequests') > config('services.mindee.daily_limit')) + throw new Exception('Mindee daily limit reached'); + if (config('services.mindee.monthly_limit') != 0 && Cache::get('mindeeTotalMonthlyRequests') > config('services.mindee.monthly_limit')) + throw new Exception('Mindee monthly limit reached'); + if (config('services.mindee.account_daily_limit') != 0 && Cache::get('mindeeAccountDailyRequests' . $user->company->account->id) > config('services.mindee.account_daily_limit')) + throw new Exception('Mindee daily limit reached for account: ' . $user->company->account->id); + if (config('services.mindee.account_monthly_limit') != 0 && Cache::get('mindeeAccountMonthlyRequests' . $user->company->account->id) > config('services.mindee.account_monthly_limit')) + throw new Exception('Mindee monthly limit reached for account: ' . $user->company->account->id); + } + + private function incrementRequestCounts() + { + $user = auth()->user(); + + Cache::increment('mindeeTotalDailyRequests'); + Cache::increment('mindeeTotalMonthlyRequests'); + Cache::increment('mindeeAccountDailyRequests' . $user->company->account->id); + Cache::increment('mindeeAccountMonthlyRequests' . $user->company->account->id); + } } diff --git a/app/Services/EDocument/Imports/ZugferdEDocument.php b/app/Services/EDocument/Imports/ZugferdEDocument.php index 21a6388ff3ce..36fe1b77d124 100644 --- a/app/Services/EDocument/Imports/ZugferdEDocument.php +++ b/app/Services/EDocument/Imports/ZugferdEDocument.php @@ -69,8 +69,7 @@ class ZugferdEDocument extends AbstractService $expense->currency_id = Currency::whereCode($invoiceCurrency)->first()->id; $expense->save(); - $documents = []; - array_push($documents, $this->file); + $documents = [$this->file]; if ($this->file->getExtension() == "xml") array_push($documents, TempFile::UploadedFileFromRaw($visualizer->renderPdf(), $documentno . "_visualiser.pdf", "application/pdf")); $this->saveDocuments($documents, $expense); diff --git a/config/services.php b/config/services.php index 48e8b75f4769..f7f2d0ecea44 100644 --- a/config/services.php +++ b/config/services.php @@ -53,9 +53,10 @@ return [ 'mindee' => [ 'api_key' => env('MINDEE_API_KEY'), - 'global_contingent_month' => env('MINDEE_GLOBAL_CONTINGENT_MONTH', 1000), - 'company_contingent_month' => env('MINDEE_COMPANY_CONTINGENT_MONTH', 500), - 'company_contingent_month_enterprise' => env('MINDEE_COMPANY_CONTINGENT_MONTH', 500), + 'daily_limit' => env('MINDEE_DAILY_LIMIT', 100), + 'monthly_limit' => env('MINDEE_MONTHLY_LIMIT', 250), + 'account_daily_limit' => env('MINDEE_ACCOUNT_DAILY_LIMIT', 0), + 'account_monthly_limit' => env('MINDEE_ACCOUNT_MONTHLY_LIMIT', 0), ], 'apple' => [ From 682d4883ff80f7daefe01967f36e64d1c0fa16f7 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 23 Jun 2024 09:36:54 +0200 Subject: [PATCH 087/119] readd case --- app/Services/EDocument/Imports/ParseEDocument.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index bcda0ece3c6f..2c1e74e85487 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -42,7 +42,14 @@ class ParseEDocument extends AbstractService // try to parse via Zugferd lib $zugferd_exception = null; try { - $expense = (new ZugferdEDocument($this->file))->run(); + switch (true) { + case $this->file->getExtension() == 'pdf': + case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017"): + case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_3.0"): + case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.1"): + case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.0"): + $expense = (new ZugferdEDocument($this->file))->run(); + } } catch (Exception $e) { $zugferd_exception = $e; } From 371eb688567fa2a44b9b1f60f8de2b8480381025 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 23 Jun 2024 11:24:40 +0200 Subject: [PATCH 088/119] remove old field data --- app/Models/Vendor.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/Models/Vendor.php b/app/Models/Vendor.php index 5c5606e3589f..ba1ee4a098e9 100644 --- a/app/Models/Vendor.php +++ b/app/Models/Vendor.php @@ -55,8 +55,6 @@ use Laracasts\Presenter\PresentableTrait; * @property string|null $id_number * @property int|null $language_id * @property int|null $last_login - * @property string|null $invoicing_email - * @property string|null $invoicing_domain * @property-read \Illuminate\Database\Eloquent\Collection $activities * @property-read int|null $activities_count * @property-read \App\Models\User|null $assigned_user @@ -118,8 +116,6 @@ class Vendor extends BaseModel 'number', 'language_id', 'classification', - 'invoicing_email', - 'invoicing_domain', ]; protected $casts = [ From a31e810ec9968e6ba182ab47d29e7d86278a155e Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 24 Jun 2024 16:30:00 +0200 Subject: [PATCH 089/119] fixes from tests --- app/Http/Controllers/ExpenseController.php | 5 ++-- .../EDocument/Imports/MindeeEDocument.php | 29 +++++++++++++++---- .../EDocument/Imports/ParseEDocument.php | 2 +- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/app/Http/Controllers/ExpenseController.php b/app/Http/Controllers/ExpenseController.php index 44444990de0e..091eb519ca5d 100644 --- a/app/Http/Controllers/ExpenseController.php +++ b/app/Http/Controllers/ExpenseController.php @@ -587,10 +587,9 @@ class ExpenseController extends BaseController public function edocument(EDocumentRequest $request): string { if ($request->hasFile("documents")) { - return (new ImportEDocument($request->file("documents")[0]->get(), $request->file("documents")[0]->getClientOriginalName(), $request->file("documents")[0]->getMimeType()))->handle(); - } else { - return "No file found"; + return (new ImportEDocument($request->file("documents")->get(), $request->file("documents")->getClientOriginalName(), $request->file("documents")->getMimeType()))->handle(); } + return "No file found"; } } diff --git a/app/Services/EDocument/Imports/MindeeEDocument.php b/app/Services/EDocument/Imports/MindeeEDocument.php index 7e5245337998..d73c7546c97b 100644 --- a/app/Services/EDocument/Imports/MindeeEDocument.php +++ b/app/Services/EDocument/Imports/MindeeEDocument.php @@ -12,11 +12,13 @@ namespace App\Services\EDocument\Imports; use App\Factory\ExpenseFactory; +use App\Factory\VendorContactFactory; use App\Factory\VendorFactory; use App\Models\Country; use App\Models\Currency; use App\Models\Expense; use App\Models\Vendor; +use App\Models\VendorContact; use App\Services\AbstractService; use App\Utils\TempFile; use App\Utils\Traits\SavesDocuments; @@ -52,14 +54,14 @@ class MindeeEDocument extends AbstractService // perform parsing $mindeeClient = new Client($api_key); - $inputSource = $mindeeClient->sourceFromFile($this->file); + $inputSource = $mindeeClient->sourceFromBytes($this->file->get(), $this->file->getClientOriginalName()); $result = $mindeeClient->parse(InvoiceV4::class, $inputSource); $this->incrementRequestCounts(); /** @var \Mindee\Product\Invoice\InvoiceV4Document $prediction */ $prediction = $result->document->inference->prediction; - if ($prediction->documentType !== 'INVOICE') + if ($prediction->documentType->value !== 'INVOICE') throw new Exception('Unsupported document type'); $grandTotalAmount = $prediction->totalAmount->value; @@ -95,15 +97,20 @@ class MindeeEDocument extends AbstractService $counter++; } - $vendor = Vendor::where('email', $prediction->supplierEmail)->first(); + $vendor = null; + $vendor_contact = VendorContact::where("company_id", $user->company()->id)->where("email", $prediction->supplierEmail)->first(); + if ($vendor_contact) + return $vendor = $vendor_contact->vendor; - if (!empty($vendor)) { + if ($vendor) + $vendor = Vendor::where("company_id", $user->company()->id)->where("name", $prediction->supplierName)->first(); + + if ($vendor) { // Vendor found $expense->vendor_id = $vendor->id; } else { $vendor = VendorFactory::create($user->company()->id, $user->id); $vendor->name = $prediction->supplierName; - $vendor->email = $prediction->supplierEmail; $vendor->currency_id = Currency::whereCode($invoiceCurrency)->first()?->id; $vendor->phone = $prediction->supplierPhoneNumber; @@ -111,9 +118,19 @@ class MindeeEDocument extends AbstractService // $vendor->address2 = $address_2; // $vendor->city = $city; // $vendor->postal_code = $postcode; - $vendor->country_id = Country::where('iso_3166_2', $country)->first()?->id || Country::where('iso_3166_3', $country)->first()?->id || null; // could be 2 or 3 length + $country = Country::where('iso_3166_2', $country)->first()?->id || Country::where('iso_3166_3', $country)->first()?->id || null; + if ($country) + $vendor->country_id = Country::where('iso_3166_2', $country)->first()?->id || Country::where('iso_3166_3', $country)->first()?->id || null; // could be 2 or 3 length $vendor->save(); + + if ($prediction->supplierEmail) { + $vendor_contact = VendorContactFactory::create($user->company()->id, $user->id); + $vendor_contact->vendor_id = $vendor->id; + $vendor_contact->email = $prediction->supplierEmail; + $vendor_contact->save(); + } + $expense->vendor_id = $vendor->id; } $expense->transaction_reference = $documentno; diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index 2c1e74e85487..647c6b5b52b6 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -71,7 +71,7 @@ class ParseEDocument extends AbstractService if ($zugferd_exception) nlog("Zugferd Exception: " . $zugferd_exception->getMessage()); if ($mindee_exception) - nlog("Mindee Exception: " . $zugferd_exception->getMessage()); + nlog("Mindee Exception: " . $mindee_exception->getMessage()); throw new Exception("File type not supported or issue while parsing"); } } From a626736f2ddb12b9d207f1c5f3b3404fb73e4a60 Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 24 Jun 2024 17:40:27 +0200 Subject: [PATCH 090/119] feature flag --- app/Services/EDocument/Imports/ParseEDocument.php | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index 647c6b5b52b6..a2786409444b 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -13,6 +13,7 @@ namespace App\Services\EDocument\Imports; use App\Models\Expense; use App\Services\AbstractService; +use App\Utils\Ninja; use Exception; use Illuminate\Http\UploadedFile; @@ -56,12 +57,13 @@ class ParseEDocument extends AbstractService // try to parse via mindee lib $mindee_exception = null; - try { - $expense = (new MindeeEDocument($this->file))->run(); - } catch (Exception $e) { - // ignore not available exceptions - $mindee_exception = $e; - } + if (Ninja::isSelfHost()) + try { + $expense = (new MindeeEDocument($this->file))->run(); + } catch (Exception $e) { + // ignore not available exceptions + $mindee_exception = $e; + } // return expense, when available and supress any errors occured before if ($expense) From 74b7581354f359a5cceb20a315710ddd9e62453e Mon Sep 17 00:00:00 2001 From: paulwer Date: Tue, 25 Jun 2024 05:48:16 +0200 Subject: [PATCH 091/119] allow hosted + enterprise --- app/Services/EDocument/Imports/ParseEDocument.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index a2786409444b..89480728d3c8 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -38,6 +38,9 @@ class ParseEDocument extends AbstractService public function run(): Expense { + /** @var \App\Models\Account $account */ + $account = auth()->user()->account; + $expense = null; // try to parse via Zugferd lib @@ -57,7 +60,7 @@ class ParseEDocument extends AbstractService // try to parse via mindee lib $mindee_exception = null; - if (Ninja::isSelfHost()) + if (config('services.mindee.api_key') && (Ninja::isSelfHost() || (Ninja::isHosted() && $account->isPaid() && $account->plan == 'enterprise'))) try { $expense = (new MindeeEDocument($this->file))->run(); } catch (Exception $e) { From 7e9e33b8466b3d980136bfeab2bc51848db405dd Mon Sep 17 00:00:00 2001 From: paulwer Date: Tue, 25 Jun 2024 05:50:28 +0200 Subject: [PATCH 092/119] fixes --- app/Services/EDocument/Imports/ParseEDocument.php | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index 89480728d3c8..40afcb8f3149 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -41,8 +41,6 @@ class ParseEDocument extends AbstractService /** @var \App\Models\Account $account */ $account = auth()->user()->account; - $expense = null; - // try to parse via Zugferd lib $zugferd_exception = null; try { @@ -52,7 +50,7 @@ class ParseEDocument extends AbstractService case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_3.0"): case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.1"): case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.0"): - $expense = (new ZugferdEDocument($this->file))->run(); + return (new ZugferdEDocument($this->file))->run(); } } catch (Exception $e) { $zugferd_exception = $e; @@ -62,16 +60,12 @@ class ParseEDocument extends AbstractService $mindee_exception = null; if (config('services.mindee.api_key') && (Ninja::isSelfHost() || (Ninja::isHosted() && $account->isPaid() && $account->plan == 'enterprise'))) try { - $expense = (new MindeeEDocument($this->file))->run(); + return (new MindeeEDocument($this->file))->run(); } catch (Exception $e) { // ignore not available exceptions $mindee_exception = $e; } - // return expense, when available and supress any errors occured before - if ($expense) - return $expense; - // log exceptions and throw error if ($zugferd_exception) nlog("Zugferd Exception: " . $zugferd_exception->getMessage()); From 96b60f4ee23d4a80125c2a4837cfe7a28a857df8 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 26 Jun 2024 17:34:32 +0200 Subject: [PATCH 093/119] fix for api error and requested changes --- app/Http/Controllers/ExpenseController.php | 14 +++++++++++--- app/Services/EDocument/Imports/MindeeEDocument.php | 8 +++++--- app/Services/EDocument/Imports/ParseEDocument.php | 2 +- .../EDocument/Imports/ZugferdEDocument.php | 8 ++++++-- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/app/Http/Controllers/ExpenseController.php b/app/Http/Controllers/ExpenseController.php index 091eb519ca5d..cc127864026f 100644 --- a/app/Http/Controllers/ExpenseController.php +++ b/app/Http/Controllers/ExpenseController.php @@ -586,9 +586,17 @@ class ExpenseController extends BaseController public function edocument(EDocumentRequest $request): string { - if ($request->hasFile("documents")) { - return (new ImportEDocument($request->file("documents")->get(), $request->file("documents")->getClientOriginalName(), $request->file("documents")->getMimeType()))->handle(); - } + if ($request->hasFile("documents")) + try { + + return (new ImportEDocument($request->file("documents")->get(), $request->file("documents")->getClientOriginalName(), $request->file("documents")->getMimeType()))->handle(); + + } catch (\Exception $e) { + if ($e->getCode() == 409) + return $e->getMessage(); + + throw $e; + } return "No file found"; } diff --git a/app/Services/EDocument/Imports/MindeeEDocument.php b/app/Services/EDocument/Imports/MindeeEDocument.php index d73c7546c97b..a37e0bf9197a 100644 --- a/app/Services/EDocument/Imports/MindeeEDocument.php +++ b/app/Services/EDocument/Imports/MindeeEDocument.php @@ -79,7 +79,7 @@ class MindeeEDocument extends AbstractService $expense->user_id = $user->id; $expense->company_id = $user->company->id; $expense->public_notes = $documentno; - $expense->currency_id = Currency::whereCode($invoiceCurrency)->first()->id; + $expense->currency_id = Currency::whereCode($invoiceCurrency)->first()?->id || $user->company->settings->currency_id; $expense->save(); $this->saveDocuments([ @@ -118,9 +118,11 @@ class MindeeEDocument extends AbstractService // $vendor->address2 = $address_2; // $vendor->city = $city; // $vendor->postal_code = $postcode; - $country = Country::where('iso_3166_2', $country)->first()?->id || Country::where('iso_3166_3', $country)->first()?->id || null; + $country = app('countries')->first(function ($c) use ($country) { + return $c->iso_3166_2 == $country || $c->iso_3166_3 == $country; + }); if ($country) - $vendor->country_id = Country::where('iso_3166_2', $country)->first()?->id || Country::where('iso_3166_3', $country)->first()?->id || null; // could be 2 or 3 length + $vendor->country_id = $country->id; $vendor->save(); diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index 40afcb8f3149..aed13a9fda7f 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -71,7 +71,7 @@ class ParseEDocument extends AbstractService nlog("Zugferd Exception: " . $zugferd_exception->getMessage()); if ($mindee_exception) nlog("Mindee Exception: " . $mindee_exception->getMessage()); - throw new Exception("File type not supported or issue while parsing"); + throw new Exception("File type not supported or issue while parsing", 409); } } diff --git a/app/Services/EDocument/Imports/ZugferdEDocument.php b/app/Services/EDocument/Imports/ZugferdEDocument.php index 36fe1b77d124..e685cbe6ad8e 100644 --- a/app/Services/EDocument/Imports/ZugferdEDocument.php +++ b/app/Services/EDocument/Imports/ZugferdEDocument.php @@ -66,7 +66,7 @@ class ZugferdEDocument extends AbstractService $expense->user_id = $user->id; $expense->company_id = $user->company->id; $expense->public_notes = $documentno; - $expense->currency_id = Currency::whereCode($invoiceCurrency)->first()->id; + $expense->currency_id = Currency::whereCode($invoiceCurrency)->first()?->id || $user->company->settings->currency_id; $expense->save(); $documents = [$this->file]; @@ -114,7 +114,11 @@ class ZugferdEDocument extends AbstractService $vendor->address2 = $address_2; $vendor->city = $city; $vendor->postal_code = $postcode; - $vendor->country_id = Country::where('iso_3166_2', $country)->first()->id; + $country = app('countries')->first(function ($c) use ($country) { + return $c->iso_3166_2 == $country || $c->iso_3166_3 == $country; + }); + if ($country) + $vendor->country_id = $country->id; $vendor->save(); $expense->vendor_id = $vendor->id; From 8616adb4373cf8ec13ef554fc86fd43ee2483204 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 26 Jun 2024 19:00:09 +0200 Subject: [PATCH 094/119] fixes --- app/Services/InboundMail/InboundMailEngine.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index d9a6deb8423c..bd3b95d865d7 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -32,11 +32,14 @@ class InboundMailEngine use SerializesModels, MakesHash; use GeneratesCounter, SavesDocuments; - private ?bool $isUnknownRecipent = null; - private array $globalBlacklist = explode(",", config('global_inbound_blocklist')); - private array $globalWhitelist = explode(",", config('global_inbound_whitelist')); // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders + private array $globalBlacklist; + private array $globalWhitelist; // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders public function __construct() { + + $this->globalBlacklist = explode(",", config('global_inbound_blocklist')); + $this->globalWhitelist = explode(",", config('global_inbound_whitelist')); // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders + } /** * if there is not a company with an matching mailbox, we only do monitoring From f44559869a2779a6672801bf60760c6a917dd0d9 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 26 Jun 2024 19:05:14 +0200 Subject: [PATCH 095/119] comments --- app/Services/EDocument/Imports/ParseEDocument.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index aed13a9fda7f..73776e4aaa9e 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -32,6 +32,8 @@ class ParseEDocument extends AbstractService * Execute the service. * the service will parse the file with all available libraries of the system and will return an expense, when possible * + * @developer the function should be implemented with local first aproach to save costs of external providers (like mindee ocr) + * * @return Expense * @throws \Exception */ @@ -41,7 +43,7 @@ class ParseEDocument extends AbstractService /** @var \App\Models\Account $account */ $account = auth()->user()->account; - // try to parse via Zugferd lib + // ZUGFERD - try to parse via Zugferd lib $zugferd_exception = null; try { switch (true) { @@ -56,7 +58,7 @@ class ParseEDocument extends AbstractService $zugferd_exception = $e; } - // try to parse via mindee lib + // MINDEE OCR - try to parse via mindee external service $mindee_exception = null; if (config('services.mindee.api_key') && (Ninja::isSelfHost() || (Ninja::isHosted() && $account->isPaid() && $account->plan == 'enterprise'))) try { From ec0df164ce7732dcac7aea073710a3888b3cd6d1 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 26 Jun 2024 19:24:57 +0200 Subject: [PATCH 096/119] log error in all cases for better debugging experience + minor find vendor improvements --- .../EDocument/Imports/MindeeEDocument.php | 5 ++- .../EDocument/Imports/ParseEDocument.php | 32 ++++++++----------- .../EDocument/Imports/ZugferdEDocument.php | 12 ++++++- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/app/Services/EDocument/Imports/MindeeEDocument.php b/app/Services/EDocument/Imports/MindeeEDocument.php index a37e0bf9197a..7bc3b962354c 100644 --- a/app/Services/EDocument/Imports/MindeeEDocument.php +++ b/app/Services/EDocument/Imports/MindeeEDocument.php @@ -100,9 +100,8 @@ class MindeeEDocument extends AbstractService $vendor = null; $vendor_contact = VendorContact::where("company_id", $user->company()->id)->where("email", $prediction->supplierEmail)->first(); if ($vendor_contact) - return $vendor = $vendor_contact->vendor; - - if ($vendor) + $vendor = $vendor_contact->vendor; + if (!$vendor) $vendor = Vendor::where("company_id", $user->company()->id)->where("name", $prediction->supplierName)->first(); if ($vendor) { diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index 73776e4aaa9e..b790caaf026c 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -44,35 +44,29 @@ class ParseEDocument extends AbstractService $account = auth()->user()->account; // ZUGFERD - try to parse via Zugferd lib - $zugferd_exception = null; - try { - switch (true) { - case $this->file->getExtension() == 'pdf': - case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017"): - case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_3.0"): - case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.1"): - case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.0"): + switch (true) { + case $this->file->getExtension() == 'pdf': + case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017"): + case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_3.0"): + case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.1"): + case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.0"): + try { return (new ZugferdEDocument($this->file))->run(); - } - } catch (Exception $e) { - $zugferd_exception = $e; + } catch (Exception $e) { + nlog("Zugferd Exception: " . $e->getMessage()); + } } // MINDEE OCR - try to parse via mindee external service - $mindee_exception = null; if (config('services.mindee.api_key') && (Ninja::isSelfHost() || (Ninja::isHosted() && $account->isPaid() && $account->plan == 'enterprise'))) try { return (new MindeeEDocument($this->file))->run(); } catch (Exception $e) { - // ignore not available exceptions - $mindee_exception = $e; + if (!($e->getMessage() == 'Unsupported document type')) + nlog("Mindee Exception: " . $e->getMessage()); } - // log exceptions and throw error - if ($zugferd_exception) - nlog("Zugferd Exception: " . $zugferd_exception->getMessage()); - if ($mindee_exception) - nlog("Mindee Exception: " . $mindee_exception->getMessage()); + // NO PARSER OR ERROR throw new Exception("File type not supported or issue while parsing", 409); } } diff --git a/app/Services/EDocument/Imports/ZugferdEDocument.php b/app/Services/EDocument/Imports/ZugferdEDocument.php index e685cbe6ad8e..f654708d548a 100644 --- a/app/Services/EDocument/Imports/ZugferdEDocument.php +++ b/app/Services/EDocument/Imports/ZugferdEDocument.php @@ -18,6 +18,7 @@ use App\Models\Country; use App\Models\Currency; use App\Models\Expense; use App\Models\Vendor; +use App\Models\VendorContact; use App\Services\AbstractService; use App\Utils\TempFile; use App\Utils\Traits\SavesDocuments; @@ -46,6 +47,7 @@ class ZugferdEDocument extends AbstractService public function run(): Expense { $user = auth()->user(); + $this->document = ZugferdDocumentReader::readAndGuessFromContent($this->file->get()); $this->document->getDocumentInformation($documentno, $documenttypecode, $documentdate, $invoiceCurrency, $taxCurrency, $documentname, $documentlanguage, $effectiveSpecifiedPeriod); $this->document->getDocumentSummation($grandTotalAmount, $duePayableAmount, $lineTotalAmount, $chargeTotalAmount, $allowanceTotalAmount, $taxBasisTotalAmount, $taxTotalAmount, $roundingAmount, $totalPrepaidAmount); @@ -93,11 +95,19 @@ class ZugferdEDocument extends AbstractService $this->document->getDocumentSellerContact($person_name, $person_department, $contact_phone, $contact_fax, $contact_email); $this->document->getDocumentSellerAddress($address_1, $address_2, $address_3, $postcode, $city, $country, $subdivision); $this->document->getDocumentSellerTaxRegistration($taxtype); + $taxid = null; if (array_key_exists("VA", $taxtype)) { $taxid = $taxtype["VA"]; } - $vendor = Vendor::where('vat_number', $taxid)->first(); + $vendor = Vendor::where("company_id", $user->company()->id)->where('vat_number', $taxid)->first(); + if (!$vendor) { + $vendor_contact = VendorContact::where("company_id", $user->company()->id)->where("email", $contact_email)->first(); + if ($vendor_contact) + $vendor = $vendor_contact->vendor; + } + if (!$vendor) + $vendor = Vendor::where("company_id", $user->company()->id)->where("name", $person_name)->first(); if (!empty($vendor)) { // Vendor found From dbcf32da26dbbdc8c2f8a8a913eea2fdf0eb5484 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 30 Jun 2024 13:06:30 +0200 Subject: [PATCH 097/119] minor fixes --- app/Services/EDocument/Imports/ParseEDocument.php | 2 +- app/Services/InboundMail/InboundMailEngine.php | 7 +++++++ app/Transformers/CompanyTransformer.php | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index b790caaf026c..39d0b4cd99cd 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -58,7 +58,7 @@ class ParseEDocument extends AbstractService } // MINDEE OCR - try to parse via mindee external service - if (config('services.mindee.api_key') && (Ninja::isSelfHost() || (Ninja::isHosted() && $account->isPaid() && $account->plan == 'enterprise'))) + if (config('services.mindee.api_key') && !(Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise'))) try { return (new MindeeEDocument($this->file))->run(); } catch (Exception $e) { diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index bd3b95d865d7..979ab57a73be 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -20,6 +20,7 @@ use App\Models\SystemLog; use App\Models\VendorContact; use App\Services\EDocument\Imports\ParseEDocument; use App\Services\InboundMail\InboundMail; +use App\Utils\Ninja; use App\Utils\TempFile; use App\Utils\Traits\GeneratesCounter; use App\Utils\Traits\SavesDocuments; @@ -57,6 +58,12 @@ class InboundMailEngine return; } + // check if company plan matches requirements + if (Ninja::isHosted() && !($company->account->isPaid() && $company->account->plan == 'enterprise')) { + $this->saveMeta($email->from, $email->to); + return; + } + $this->createExpenses($company, $email); $this->saveMeta($email->from, $email->to); } diff --git a/app/Transformers/CompanyTransformer.php b/app/Transformers/CompanyTransformer.php index 183253f5d923..87d61a8d7fe7 100644 --- a/app/Transformers/CompanyTransformer.php +++ b/app/Transformers/CompanyTransformer.php @@ -204,7 +204,7 @@ class CompanyTransformer extends EntityTransformer 'invoice_task_project_header' => (bool) $company->invoice_task_project_header, 'invoice_task_item_description' => (bool) $company->invoice_task_item_description, 'origin_tax_data' => $company->origin_tax_data ?: new \stdClass, - 'expense_mailbox' => (bool) $company->expense_mailbox, + 'expense_mailbox' => (string) $company->expense_mailbox, 'expense_mailbox_active' => (bool) $company->expense_mailbox_active, 'inbound_mailbox_allow_company_users' => (bool) $company->inbound_mailbox_allow_company_users, 'inbound_mailbox_allow_vendors' => (bool) $company->inbound_mailbox_allow_vendors, From b85e846de5e5be27c003ec3a5264e651bc0da9e8 Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 30 Jun 2024 13:07:09 +0200 Subject: [PATCH 098/119] fixes --- app/Services/EDocument/Imports/ParseEDocument.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index 39d0b4cd99cd..0c4f91d36779 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -58,7 +58,7 @@ class ParseEDocument extends AbstractService } // MINDEE OCR - try to parse via mindee external service - if (config('services.mindee.api_key') && !(Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise'))) + if (config('services.mindee.api_key') && !(Ninja::isHosted() && !($account->isPaid() && $account->plan == 'enterprise'))) try { return (new MindeeEDocument($this->file))->run(); } catch (Exception $e) { From f07fde39aa7a09fd91996b13c092f8752c4fe635 Mon Sep 17 00:00:00 2001 From: paulwer Date: Tue, 2 Jul 2024 20:20:28 +0200 Subject: [PATCH 099/119] fixes --- app/Http/Controllers/MailgunController.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 44ac87420fe4..f82df4c67706 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -20,8 +20,6 @@ use Illuminate\Http\Request; */ class MailgunController extends BaseController { - private $invitation; - public function __construct() { } From 1f92ea01080ea80a89f8a6bd2d100f423c230051 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 28 Aug 2024 07:33:03 +0200 Subject: [PATCH 100/119] fixes for validating expense_mailbox in company request --- app/Http/Requests/Company/UpdateCompanyRequest.php | 2 +- app/Http/ValidationRules/Company/ValidExpenseMailbox.php | 3 ++- app/Libraries/MultiDB.php | 2 +- app/Transformers/CompanyTransformer.php | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/Http/Requests/Company/UpdateCompanyRequest.php b/app/Http/Requests/Company/UpdateCompanyRequest.php index fa8927a8c83c..52d6d80aa9d4 100644 --- a/app/Http/Requests/Company/UpdateCompanyRequest.php +++ b/app/Http/Requests/Company/UpdateCompanyRequest.php @@ -76,7 +76,7 @@ class UpdateCompanyRequest extends Request $rules['subdomain'] = ['nullable', 'regex:/^[a-zA-Z0-9.-]+[a-zA-Z0-9]$/', new ValidSubdomain()]; } - $rules['expense_mailbox'] = new ValidExpenseMailbox(); + $rules['expense_mailbox'] = ['email', 'nullable', new ValidExpenseMailbox()]; return $rules; } diff --git a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php index d9ab3727cb13..aa333fddc0e6 100644 --- a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php +++ b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php @@ -14,6 +14,7 @@ namespace App\Http\ValidationRules\Company; use App\Libraries\MultiDB; use App\Utils\Ninja; use Illuminate\Contracts\Validation\Rule; +use Symfony\Component\Validator\Constraints\EmailValidator; /** * Class ValidCompanyQuantity. @@ -47,7 +48,7 @@ class ValidExpenseMailbox implements Rule // Validate Schema $validated = false; foreach ($this->endings as $ending) { - if (str_ends_with($ending, $value)) { + if (str_ends_with($value, $ending)) { $validated = true; break; } diff --git a/app/Libraries/MultiDB.php b/app/Libraries/MultiDB.php index 3e190a3c611b..cc3545b92764 100644 --- a/app/Libraries/MultiDB.php +++ b/app/Libraries/MultiDB.php @@ -113,7 +113,7 @@ class MultiDB { if (!config('ninja.db.multi_db_enabled')) { - return Company::where("expense_mailbox", $expense_mailbox)->exists(); + return !Company::where("expense_mailbox", $expense_mailbox)->exists(); } if (in_array($expense_mailbox, self::$protected_expense_mailboxes)) { diff --git a/app/Transformers/CompanyTransformer.php b/app/Transformers/CompanyTransformer.php index 4d3dacc65aae..7c28215870b9 100644 --- a/app/Transformers/CompanyTransformer.php +++ b/app/Transformers/CompanyTransformer.php @@ -210,8 +210,8 @@ class CompanyTransformer extends EntityTransformer 'inbound_mailbox_allow_vendors' => (bool) $company->inbound_mailbox_allow_vendors, 'inbound_mailbox_allow_clients' => (bool) $company->inbound_mailbox_allow_clients, 'inbound_mailbox_allow_unknown' => (bool) $company->inbound_mailbox_allow_unknown, - 'inbound_mailbox_blacklist' => $company->inbound_mailbox_blacklist, - 'inbound_mailbox_whitelist' => $company->inbound_mailbox_whitelist, + 'inbound_mailbox_blacklist' => (string) $company->inbound_mailbox_blacklist, + 'inbound_mailbox_whitelist' => (string) $company->inbound_mailbox_whitelist, 'smtp_host' => (string) $company->smtp_host ?? '', 'smtp_port' => (int) $company->smtp_port ?? 25, 'smtp_encryption' => (string) $company->smtp_encryption ?? 'tls', From b9f56f0e3084f1dcd5224c4c3873d929802cf552 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 28 Aug 2024 08:07:37 +0200 Subject: [PATCH 101/119] fixes postmark --- app/Http/Controllers/PostMarkController.php | 9 ++++++++- app/Services/InboundMail/InboundMailEngine.php | 13 +++++++------ config/ninja.php | 2 +- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index 9daa9e28908e..7eac5dfaf3f5 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -319,7 +319,14 @@ class PostMarkController extends BaseController } // perform - $inboundEngine->handleExpenseMailbox($inboundMail); + try { + $inboundEngine->handleExpenseMailbox($inboundMail); + } catch (\Exception $e) { + if ($e->getCode() == 409) + return response()->json(['message' => $e->getMessage()], 409); + + throw $e; + } return response()->json(['message' => 'Success'], 200); } diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index 979ab57a73be..5d4df9e8c970 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -38,8 +38,8 @@ class InboundMailEngine public function __construct() { - $this->globalBlacklist = explode(",", config('global_inbound_blocklist')); - $this->globalWhitelist = explode(",", config('global_inbound_whitelist')); // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders + $this->globalBlacklist = explode(",", config('ninja.inbound_mailbox.global_inbound_blocklist')); + $this->globalWhitelist = explode(",", config('ninja.inbound_mailbox.global_inbound_whitelist')); // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders } /** @@ -101,18 +101,19 @@ class InboundMailEngine } if (Cache::has('inboundMailBlockedSender:' . $from)) { // was marked as blocked before, so we block without any console output + // nlog('E-Mail was marked as blocked before: ' . $from); return true; } // sender occured in more than 500 emails in the last 12 hours $senderMailCountTotal = Cache::get('inboundMailCountSender:' . $from, 0); - if ($senderMailCountTotal >= config('global_inbound_sender_permablock_mailcount')) { + if ($senderMailCountTotal >= config('ninja.inbound_mailbox.global_inbound_sender_permablock_mailcount')) { nlog('E-Mail blocked permanent, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $from); $this->blockSender($from); $this->saveMeta($from, $to); return true; } - if ($senderMailCountTotal >= config('global_inbound_sender_block_mailcount')) { + if ($senderMailCountTotal >= config('ninja.inbound_mailbox.global_inbound_sender_block_mailcount')) { nlog('E-Mail blocked, because the sender sended more than ' . $senderMailCountTotal . ' emails in the last 12 hours: ' . $from); $this->saveMeta($from, $to); return true; @@ -120,7 +121,7 @@ class InboundMailEngine // sender sended more than 50 emails to the wrong mailbox in the last 6 hours $senderMailCountUnknownRecipent = Cache::get('inboundMailCountSenderUnknownRecipent:' . $from, 0); - if ($senderMailCountUnknownRecipent >= config('company_inbound_sender_block_unknown_reciepent')) { + if ($senderMailCountUnknownRecipent >= config('ninja.inbound_mailbox.company_inbound_sender_block_unknown_reciepent')) { nlog('E-Mail blocked, because the sender sended more than ' . $senderMailCountUnknownRecipent . ' emails to the wrong mailbox in the last 6 hours: ' . $from); $this->saveMeta($from, $to); return true; @@ -191,7 +192,7 @@ class InboundMailEngine // check if document can be parsed to an expense try { - $expense = (new ParseEDocument($document))->run(); + $expense = (new ParseEDocument($document, $company))->run(); // check if expense was already matched within this job and skip if true if (array_search($expense->id, $parsed_expense_ids)) diff --git a/config/ninja.php b/config/ninja.php index 1c06947ad280..16b0c0f3d551 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -232,7 +232,7 @@ return [ ], 'inbound_mailbox' => [ 'expense_mailbox_endings' => env('EXPENSE_MAILBOX_ENDINGS', '@expense.invoicing.co'), - 'inbound_webhook_key' => env('INBOUND_WEBHOOK_KEY', null), + 'inbound_webhook_token' => env('INBOUND_WEBHOOK_TOKEN', null), 'global_inbound_blacklist' => env('GLOBAL_INBOUND_BLACKLIST', ''), 'global_inbound_whitelist' => env('GLOBAL_INBOUND_WHITELIST', ''), 'global_inbound_sender_block_mailcount' => env('GLOBAL_INBOUND_SENDER_BLOCK_MAILCOUNT', 1000), From 2c7bf4f44df8b3653ddae2e1397e6ef34f8439b2 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 28 Aug 2024 08:18:39 +0200 Subject: [PATCH 102/119] fixes --- app/Http/Controllers/PostMarkController.php | 2 ++ app/Services/InboundMail/InboundMailEngine.php | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index 7eac5dfaf3f5..97b487450d58 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -320,7 +320,9 @@ class PostMarkController extends BaseController // perform try { + $inboundEngine->handleExpenseMailbox($inboundMail); + } catch (\Exception $e) { if ($e->getCode() == 409) return response()->json(['message' => $e->getMessage()], 409); diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index 5d4df9e8c970..f40fbf50a0df 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -204,19 +204,19 @@ class InboundMailEngine // throw error, only, when its not expected switch (true) { case ($err->getMessage() === 'E-Invoice standard not supported'): - case ($err->getMessage() === 'File type not supported'): + case ($err->getMessage() === 'File type not supported or issue while parsing'): break; default: throw $err; } } - $is_imported_by_parser = array_search($expense->id, $parsed_expense_ids); - // populate missing data with data from email if (!$expense) $expense = ExpenseFactory::create($company->id, $company->owner()->id); + $is_imported_by_parser = array_search($expense->id, $parsed_expense_ids); + if ($is_imported_by_parser) $expense->public_notes = $expense->public_notes . $email->subject; @@ -238,13 +238,13 @@ class InboundMailEngine if ($email->body_document !== null) array_push($documents, $email->body_document); - $this->saveDocuments($documents, $expense); - if ($is_imported_by_parser) $expense->saveQuietly(); else $expense->save(); + $this->saveDocuments($documents, $expense); + } } From 144d52b444799169db4ce5386cef51989cefe7dc Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 28 Aug 2024 08:44:08 +0200 Subject: [PATCH 103/119] fixes --- app/Services/EDocument/Imports/ParseEDocument.php | 10 +++++----- app/Services/InboundMail/InboundMailEngine.php | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index 3c9f8f048483..fbd739122ee4 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -46,11 +46,11 @@ class ParseEDocument extends AbstractService // ZUGFERD - try to parse via Zugferd lib switch (true) { - case $this->file->getExtension() == 'pdf': - case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017"): - case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_3.0"): - case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.1"): - case $this->file->getExtension() == 'xml' && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.0"): + case ($this->file->getExtension() == 'pdf' || $this->file->getMimeType() == 'application/pdf'): + case ($this->file->getExtension() == 'xml' || $this->file->getMimeType() == 'application/xml') && stristr($this->file->get(), "urn:cen.eu:en16931:2017"): + case ($this->file->getExtension() == 'xml' || $this->file->getMimeType() == 'application/xml') && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_3.0"): + case ($this->file->getExtension() == 'xml' || $this->file->getMimeType() == 'application/xml') && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.1"): + case ($this->file->getExtension() == 'xml' || $this->file->getMimeType() == 'application/xml') && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.0"): try { return (new ZugferdEDocument($this->file, $this->company))->run(); } catch (Exception $e) { diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index f40fbf50a0df..d4d4a9d322e6 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -229,20 +229,20 @@ class InboundMailEngine if (!$expense->vendor_id && $expense_vendor) $expense->vendor_id = $expense_vendor->id; + if ($is_imported_by_parser) + $expense->saveQuietly(); + else + $expense->save(); + // save document only, when not imported by parser $documents = []; - if ($is_imported_by_parser) + if (!$is_imported_by_parser) array_push($documents, $document); // email document if ($email->body_document !== null) array_push($documents, $email->body_document); - if ($is_imported_by_parser) - $expense->saveQuietly(); - else - $expense->save(); - $this->saveDocuments($documents, $expense); } From 5da3ae77707b6885ebc0cb2f153179275683c1d3 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 28 Aug 2024 09:29:11 +0200 Subject: [PATCH 104/119] fixes --- app/Http/Controllers/BrevoController.php | 4 +- app/Http/Controllers/PostMarkController.php | 2 +- .../EDocument/Imports/MindeeEDocument.php | 41 ++++++++----------- .../EDocument/Imports/ParseEDocument.php | 27 +++++++----- .../EDocument/Imports/ZugferdEDocument.php | 2 +- 5 files changed, 39 insertions(+), 37 deletions(-) diff --git a/app/Http/Controllers/BrevoController.php b/app/Http/Controllers/BrevoController.php index 81c02bdcd376..f295a317c7de 100644 --- a/app/Http/Controllers/BrevoController.php +++ b/app/Http/Controllers/BrevoController.php @@ -185,8 +185,8 @@ class BrevoController extends BaseController { $input = $request->all(); - if (!($request->has('token') && $request->get('token') == config('ninja.inbound_mailbox.inbound_webhook_token'))) - return response()->json(['message' => 'Unauthorized'], 403); + // if (!($request->has('token') && $request->get('token') == config('ninja.inbound_mailbox.inbound_webhook_token'))) + // return response()->json(['message' => 'Unauthorized'], 403); if (!array_key_exists('items', $input)) { nlog('Failed: Message could not be parsed, because required parameters are missing.'); diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index 97b487450d58..4ed1434c66f8 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -289,7 +289,7 @@ class PostMarkController extends BaseController $company = MultiDB::findAndSetDbByExpenseMailbox($input["To"]); if (!$company) { - nlog('[PostmarkInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $input["To"]); + nlog('[PostmarkInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $input["To"]); $inboundEngine->saveMeta($input["From"], $input["To"], true); // important to save this, to protect from spam return; } diff --git a/app/Services/EDocument/Imports/MindeeEDocument.php b/app/Services/EDocument/Imports/MindeeEDocument.php index 7bc3b962354c..934582e76f8a 100644 --- a/app/Services/EDocument/Imports/MindeeEDocument.php +++ b/app/Services/EDocument/Imports/MindeeEDocument.php @@ -14,6 +14,7 @@ namespace App\Services\EDocument\Imports; use App\Factory\ExpenseFactory; use App\Factory\VendorContactFactory; use App\Factory\VendorFactory; +use App\Models\Company; use App\Models\Country; use App\Models\Currency; use App\Models\Expense; @@ -35,7 +36,7 @@ class MindeeEDocument extends AbstractService /** * @throws Exception */ - public function __construct(public UploadedFile $file) + public function __construct(public UploadedFile $file, public Company $company) { # curl -X POST http://localhost:8000/api/v1/edocument/upload -H "Content-Type: multipart/form-data" -H "X-API-TOKEN: 7tdDdkz987H3AYIWhNGXy8jTjJIoDhkAclCDLE26cTCj1KYX7EBHC66VEitJwWhn" -H "X-Requested-With: XMLHttpRequest" -F _method=PUT -F documents[]=@einvoice.xml } @@ -45,8 +46,6 @@ class MindeeEDocument extends AbstractService */ public function run(): Expense { - $user = auth()->user(); - $api_key = config('services.mindee.api_key'); if (!$api_key) throw new Exception('Mindee API key not configured'); @@ -74,12 +73,12 @@ class MindeeEDocument extends AbstractService if (empty($expense)) { // The document does not exist as an expense // Handle accordingly - $expense = ExpenseFactory::create($user->company()->id, $user->id); + $expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id); $expense->date = $documentdate; - $expense->user_id = $user->id; - $expense->company_id = $user->company->id; + $expense->user_id = $this->company->owner()->id; + $expense->company_id = $this->company->id; $expense->public_notes = $documentno; - $expense->currency_id = Currency::whereCode($invoiceCurrency)->first()?->id || $user->company->settings->currency_id; + $expense->currency_id = Currency::whereCode($invoiceCurrency)->first()?->id || $this->company->settings->currency_id; $expense->save(); $this->saveDocuments([ @@ -98,17 +97,17 @@ class MindeeEDocument extends AbstractService } $vendor = null; - $vendor_contact = VendorContact::where("company_id", $user->company()->id)->where("email", $prediction->supplierEmail)->first(); + $vendor_contact = VendorContact::where("company_id", $this->company->id)->where("email", $prediction->supplierEmail)->first(); if ($vendor_contact) $vendor = $vendor_contact->vendor; if (!$vendor) - $vendor = Vendor::where("company_id", $user->company()->id)->where("name", $prediction->supplierName)->first(); + $vendor = Vendor::where("company_id", $this->company->id)->where("name", $prediction->supplierName)->first(); if ($vendor) { // Vendor found $expense->vendor_id = $vendor->id; } else { - $vendor = VendorFactory::create($user->company()->id, $user->id); + $vendor = VendorFactory::create($this->company->id, $this->company->owner()->id); $vendor->name = $prediction->supplierName; $vendor->currency_id = Currency::whereCode($invoiceCurrency)->first()?->id; @@ -126,7 +125,7 @@ class MindeeEDocument extends AbstractService $vendor->save(); if ($prediction->supplierEmail) { - $vendor_contact = VendorContactFactory::create($user->company()->id, $user->id); + $vendor_contact = VendorContactFactory::create($this->company->id, $this->company->owner()->id); $vendor_contact->vendor_id = $vendor->id; $vendor_contact->email = $prediction->supplierEmail; $vendor_contact->save(); @@ -147,30 +146,26 @@ class MindeeEDocument extends AbstractService private function checkLimits() { - $user = auth()->user(); - Cache::add('mindeeTotalDailyRequests', 0, now()->endOfDay()); Cache::add('mindeeTotalMonthlyRequests', 0, now()->endOfMonth()); - Cache::add('mindeeAccountDailyRequests' . $user->company->account->id, 0, now()->endOfDay()); - Cache::add('mindeeAccountMonthlyRequests' . $user->company->account->id, 0, now()->endOfMonth()); + Cache::add('mindeeAccountDailyRequests' . $this->company->account->id, 0, now()->endOfDay()); + Cache::add('mindeeAccountMonthlyRequests' . $this->company->account->id, 0, now()->endOfMonth()); if (config('services.mindee.daily_limit') != 0 && Cache::get('mindeeTotalDailyRequests') > config('services.mindee.daily_limit')) throw new Exception('Mindee daily limit reached'); if (config('services.mindee.monthly_limit') != 0 && Cache::get('mindeeTotalMonthlyRequests') > config('services.mindee.monthly_limit')) throw new Exception('Mindee monthly limit reached'); - if (config('services.mindee.account_daily_limit') != 0 && Cache::get('mindeeAccountDailyRequests' . $user->company->account->id) > config('services.mindee.account_daily_limit')) - throw new Exception('Mindee daily limit reached for account: ' . $user->company->account->id); - if (config('services.mindee.account_monthly_limit') != 0 && Cache::get('mindeeAccountMonthlyRequests' . $user->company->account->id) > config('services.mindee.account_monthly_limit')) - throw new Exception('Mindee monthly limit reached for account: ' . $user->company->account->id); + if (config('services.mindee.account_daily_limit') != 0 && Cache::get('mindeeAccountDailyRequests' . $this->company->account->id) > config('services.mindee.account_daily_limit')) + throw new Exception('Mindee daily limit reached for account: ' . $this->company->account->id); + if (config('services.mindee.account_monthly_limit') != 0 && Cache::get('mindeeAccountMonthlyRequests' . $this->company->account->id) > config('services.mindee.account_monthly_limit')) + throw new Exception('Mindee monthly limit reached for account: ' . $this->company->account->id); } private function incrementRequestCounts() { - $user = auth()->user(); - Cache::increment('mindeeTotalDailyRequests'); Cache::increment('mindeeTotalMonthlyRequests'); - Cache::increment('mindeeAccountDailyRequests' . $user->company->account->id); - Cache::increment('mindeeAccountMonthlyRequests' . $user->company->account->id); + Cache::increment('mindeeAccountDailyRequests' . $this->company->account->id); + Cache::increment('mindeeAccountMonthlyRequests' . $this->company->account->id); } } diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index fbd739122ee4..b58d4d7664b1 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -44,13 +44,16 @@ class ParseEDocument extends AbstractService /** @var \App\Models\Account $account */ $account = $this->company->owner()->account; + $extension = $this->file->getClientOriginalExtension() ?: $this->file->getExtension(); + $mimetype = $this->file->getClientMimeType() ?: $$this->file->getMimeType(); + // ZUGFERD - try to parse via Zugferd lib switch (true) { - case ($this->file->getExtension() == 'pdf' || $this->file->getMimeType() == 'application/pdf'): - case ($this->file->getExtension() == 'xml' || $this->file->getMimeType() == 'application/xml') && stristr($this->file->get(), "urn:cen.eu:en16931:2017"): - case ($this->file->getExtension() == 'xml' || $this->file->getMimeType() == 'application/xml') && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_3.0"): - case ($this->file->getExtension() == 'xml' || $this->file->getMimeType() == 'application/xml') && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.1"): - case ($this->file->getExtension() == 'xml' || $this->file->getMimeType() == 'application/xml') && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.0"): + case ($extension == 'pdf' || $mimetype == 'application/pdf'): + case ($extension == 'xml' || $mimetype == 'application/xml') && stristr($this->file->get(), "urn:cen.eu:en16931:2017"): + case ($extension == 'xml' || $mimetype == 'application/xml') && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_3.0"): + case ($extension == 'xml' || $mimetype == 'application/xml') && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.1"): + case ($extension == 'xml' || $mimetype == 'application/xml') && stristr($this->file->get(), "urn:cen.eu:en16931:2017#compliant#urn:xeinkauf.de:kosit:xrechnung_2.0"): try { return (new ZugferdEDocument($this->file, $this->company))->run(); } catch (Exception $e) { @@ -60,11 +63,15 @@ class ParseEDocument extends AbstractService // MINDEE OCR - try to parse via mindee external service if (config('services.mindee.api_key') && !(Ninja::isHosted() && !($account->isPaid() && $account->plan == 'enterprise'))) - try { - return (new MindeeEDocument($this->file))->run(); - } catch (Exception $e) { - if (!($e->getMessage() == 'Unsupported document type')) - nlog("Mindee Exception: " . $e->getMessage()); + switch (true) { + case ($extension == 'pdf' || $mimetype == 'application/pdf'): + case ($extension == 'heic' || $extension == 'heic' || $extension == 'png' || $extension == 'jpg' || $extension == 'jpeg' || $extension == 'webp' || str_starts_with($mimetype, 'image/')): + try { + return (new MindeeEDocument($this->file, $this->company))->run(); + } catch (Exception $e) { + if (!($e->getMessage() == 'Unsupported document type')) + nlog("Mindee Exception: " . $e->getMessage()); + } } // NO PARSER OR ERROR diff --git a/app/Services/EDocument/Imports/ZugferdEDocument.php b/app/Services/EDocument/Imports/ZugferdEDocument.php index c16f9b862903..ce88ddcfd62e 100644 --- a/app/Services/EDocument/Imports/ZugferdEDocument.php +++ b/app/Services/EDocument/Imports/ZugferdEDocument.php @@ -50,7 +50,7 @@ class ZugferdEDocument extends AbstractService /** @var \App\Models\User $user */ $user = $this->company->owner(); - $this->document = ZugferdDocumentReader::readAndGuessFromContent($this->tempdocument); + $this->document = ZugferdDocumentReader::readAndGuessFromContent($this->file->get()); $this->document->getDocumentInformation($documentno, $documenttypecode, $documentdate, $invoiceCurrency, $taxCurrency, $documentname, $documentlanguage, $effectiveSpecifiedPeriod); $this->document->getDocumentSummation($grandTotalAmount, $duePayableAmount, $lineTotalAmount, $chargeTotalAmount, $allowanceTotalAmount, $taxBasisTotalAmount, $taxTotalAmount, $roundingAmount, $totalPrepaidAmount); From 2a28d8c97638165d286f26b4342af74d51521cd4 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 28 Aug 2024 09:29:50 +0200 Subject: [PATCH 105/119] fixes --- app/Http/Controllers/BrevoController.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Http/Controllers/BrevoController.php b/app/Http/Controllers/BrevoController.php index f295a317c7de..81c02bdcd376 100644 --- a/app/Http/Controllers/BrevoController.php +++ b/app/Http/Controllers/BrevoController.php @@ -185,8 +185,8 @@ class BrevoController extends BaseController { $input = $request->all(); - // if (!($request->has('token') && $request->get('token') == config('ninja.inbound_mailbox.inbound_webhook_token'))) - // return response()->json(['message' => 'Unauthorized'], 403); + if (!($request->has('token') && $request->get('token') == config('ninja.inbound_mailbox.inbound_webhook_token'))) + return response()->json(['message' => 'Unauthorized'], 403); if (!array_key_exists('items', $input)) { nlog('Failed: Message could not be parsed, because required parameters are missing.'); From 5ecb2033dbf5b642d7bdfee67f24c925c3100a59 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 28 Aug 2024 09:53:38 +0200 Subject: [PATCH 106/119] fixes --- app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 8 ++++---- app/Services/EDocument/Imports/MindeeEDocument.php | 2 +- app/Services/EDocument/Imports/ZugferdEDocument.php | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index 39617fb03282..30f741c69ed6 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -162,23 +162,23 @@ class ProcessBrevoInboundWebhook implements ShouldQueue // download file and save to tmp dir if (!empty($company_brevo_secret)) { - $attachment = null; + $data = null; try { $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", $company_brevo_secret)); - $attachment = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); + $data = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); } catch (\Error $e) { if (config('services.brevo.secret')) { nlog("[ProcessBrevoInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); - $attachment = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); + $data = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); } else throw $e; } - $inboundMail->documents[] = TempFile::UploadedFileFromRaw($attachment, $attachment["Name"], $attachment["ContentType"]); + $inboundMail->documents[] = TempFile::UploadedFileFromRaw($data, $attachment["Name"], $attachment["ContentType"]); } else { diff --git a/app/Services/EDocument/Imports/MindeeEDocument.php b/app/Services/EDocument/Imports/MindeeEDocument.php index 934582e76f8a..a6e72f33ba5e 100644 --- a/app/Services/EDocument/Imports/MindeeEDocument.php +++ b/app/Services/EDocument/Imports/MindeeEDocument.php @@ -137,7 +137,7 @@ class MindeeEDocument extends AbstractService } else { // The document exists as an expense // Handle accordingly - nlog("Document already exists"); + nlog("Mindee: Document already exists"); $expense->private_notes = $expense->private_notes . ctrans("texts.edocument_import_already_exists", ["date" => time()]); } $expense->save(); diff --git a/app/Services/EDocument/Imports/ZugferdEDocument.php b/app/Services/EDocument/Imports/ZugferdEDocument.php index ce88ddcfd62e..ed7215e54e6c 100644 --- a/app/Services/EDocument/Imports/ZugferdEDocument.php +++ b/app/Services/EDocument/Imports/ZugferdEDocument.php @@ -140,7 +140,7 @@ class ZugferdEDocument extends AbstractService } else { // The document exists as an expense // Handle accordingly - nlog("Document already exists"); + nlog("Zugferd: Document already exists"); $expense->private_notes = $expense->private_notes . ctrans("texts.edocument_import_already_exists", ["date" => time()]); } $expense->save(); From 984af8750a228b7820d2146dcd0e29e48dd9a240 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 28 Aug 2024 09:56:01 +0200 Subject: [PATCH 107/119] fixes --- app/Services/InboundMail/InboundMailEngine.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index d4d4a9d322e6..c4b80ffebee5 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -198,7 +198,7 @@ class InboundMailEngine if (array_search($expense->id, $parsed_expense_ids)) continue; - array_push($parsed_expenses, $expense->id); + array_push($parsed_expense_ids, $expense->id); } catch (\Exception $err) { // throw error, only, when its not expected From 7ff7ce6f191373eb1e65bda61f934e77ae2c9a07 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 28 Aug 2024 10:08:48 +0200 Subject: [PATCH 108/119] brevo fixes --- app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index 30f741c69ed6..b0ea10eaa47c 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -23,6 +23,7 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Http\UploadedFile; class ProcessBrevoInboundWebhook implements ShouldQueue { @@ -162,29 +163,44 @@ class ProcessBrevoInboundWebhook implements ShouldQueue // download file and save to tmp dir if (!empty($company_brevo_secret)) { - $data = null; try { $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", $company_brevo_secret)); - $data = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); + $inboundMail->documents[] = new UploadedFile( + $brevo->getInboundEmailAttachment($attachment["DownloadToken"])->getPathname(), + $attachment["Name"], + $attachment["ContentType"], + 0, + true // Mark it as test, since the file isn't from real HTTP POST. + ); } catch (\Error $e) { if (config('services.brevo.secret')) { nlog("[ProcessBrevoInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); - $data = $brevo->getInboundEmailAttachment($attachment["DownloadToken"]); + $inboundMail->documents[] = new UploadedFile( + $brevo->getInboundEmailAttachment($attachment["DownloadToken"])->getPathname(), + $attachment["Name"], + $attachment["ContentType"], + 0, + true // Mark it as test, since the file isn't from real HTTP POST. + ); } else throw $e; } - $inboundMail->documents[] = TempFile::UploadedFileFromRaw($data, $attachment["Name"], $attachment["ContentType"]); } else { $brevo = new InboundParsingApi(null, Configuration::getDefaultConfiguration()->setApiKey("api-key", config('services.brevo.secret'))); - $inboundMail->documents[] = TempFile::UploadedFileFromRaw($brevo->getInboundEmailAttachment($attachment["DownloadToken"]), $attachment["Name"], $attachment["ContentType"]); - + $inboundMail->documents[] = new UploadedFile( + $brevo->getInboundEmailAttachment($attachment["DownloadToken"])->getPathname(), + $attachment["Name"], + $attachment["ContentType"], + 0, + true // Mark it as test, since the file isn't from real HTTP POST. + ); } } From 0b4eaf9d4f572e242cbb4cd05d7e32d1e6459e67 Mon Sep 17 00:00:00 2001 From: paulwer Date: Wed, 28 Aug 2024 10:20:43 +0200 Subject: [PATCH 109/119] fixes --- config/ninja.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/ninja.php b/config/ninja.php index 16b0c0f3d551..76234c55a83f 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -84,6 +84,7 @@ return [ 'gocardless' => env('GOCARDLESS_KEYS', ''), 'square' => env('SQUARE_KEYS', ''), 'eway' => env('EWAY_KEYS', ''), + 'mollie' => env('MOLLIE_KEYS', ''), 'paytrace' => env('PAYTRACE_KEYS', ''), 'stripe' => env('STRIPE_KEYS', ''), 'paypal' => env('PAYPAL_KEYS', ''), @@ -96,7 +97,6 @@ return [ 'test_email' => env('TEST_EMAIL', 'test@example.com'), 'wepay' => env('WEPAY_KEYS', ''), 'braintree' => env('BRAINTREE_KEYS', ''), - 'mollie' => env('MOLLIE_KEYS', ''), ], 'contact' => [ 'email' => env('MAIL_FROM_ADDRESS'), From 918258948b39f2120c5d71a1d6913c944d2828cf Mon Sep 17 00:00:00 2001 From: David Bomba Date: Thu, 29 Aug 2024 16:57:16 +1000 Subject: [PATCH 110/119] Hosted changes --- app/Services/InboundMail/InboundMail.php | 2 +- app/Services/InboundMail/InboundMailEngine.php | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Services/InboundMail/InboundMail.php b/app/Services/InboundMail/InboundMail.php index 2a897d2598b0..4532a041dedb 100644 --- a/app/Services/InboundMail/InboundMail.php +++ b/app/Services/InboundMail/InboundMail.php @@ -15,7 +15,7 @@ use Illuminate\Http\UploadedFile; use Illuminate\Support\Carbon; /** - * EmailObject. + * InboundMail. */ class InboundMail { diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index c4b80ffebee5..c6934b50decc 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -38,9 +38,9 @@ class InboundMailEngine public function __construct() { - $this->globalBlacklist = explode(",", config('ninja.inbound_mailbox.global_inbound_blocklist')); - $this->globalWhitelist = explode(",", config('ninja.inbound_mailbox.global_inbound_whitelist')); // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders - + // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders + $this->globalBlacklist = Ninja::isSelfHost() ? explode(",", config('ninja.inbound_mailbox.global_inbound_blocklist')) : []; + $this->globalWhitelist = Ninja::isSelfHost() ? explode(",", config('ninja.inbound_mailbox.global_inbound_whitelist')) : []; } /** * if there is not a company with an matching mailbox, we only do monitoring From 4ea7c7ef632bb22c53a7925d7452b230e2ad57f6 Mon Sep 17 00:00:00 2001 From: David Bomba Date: Thu, 29 Aug 2024 17:12:12 +1000 Subject: [PATCH 111/119] Split parameters --- app/Http/Controllers/MailgunController.php | 2 +- app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php | 12 ++++++------ app/Services/InboundMail/InboundMailEngine.php | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index d2bc0bffefc1..7db6f3dc773b 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -128,7 +128,7 @@ class MailgunController extends BaseController if (!$authorizedByHash && !$authorizedByToken) return response()->json(['message' => 'Unauthorized'], 403); - ProcessMailgunInboundWebhook::dispatch($input["sender"] . "|" . $input["recipient"] . "|" . $input["message-url"])->delay(rand(2, 10)); + ProcessMailgunInboundWebhook::dispatch($input["sender"], $input["recipient"], $input["message-url"])->delay(rand(2, 10)); return response()->json(['message' => 'Success.'], 200); } diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index f278cdba0153..565cf2af4e34 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -34,7 +34,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue * Create a new job instance. * $input consists of 3 informations: sender/from|recipient/to|messageUrl */ - public function __construct(private string $input) + public function __construct(private string $sender, private string $recipient, private string $message_url) { $this->engine = new InboundMailEngine(); } @@ -167,8 +167,8 @@ class ProcessMailgunInboundWebhook implements ShouldQueue */ public function handle() { - $from = explode("|", $this->input)[0]; - $to = explode("|", $this->input)[1]; + $from = $this->sender;//explode("|", $this->input)[0]; + $to = $this->recipient; //explode("|", $this->input)[1]; // $messageId = explode("|", $this->input)[2]; // used as base in download function // Spam protection @@ -196,7 +196,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue if ($company_mailgun_domain && $company_mailgun_secret) { $credentials = $company_mailgun_domain . ":" . $company_mailgun_secret . "@"; - $messageUrl = explode("|", $this->input)[2]; + $messageUrl = $this->message_url;//explode("|", $this->input)[2]; $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); @@ -207,7 +207,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue nlog("[ProcessMailgunInboundWebhook] Error while downloading with company credentials, we try to use default credentials now..."); $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; - $messageUrl = explode("|", $this->input)[2]; + $messageUrl = $this->message_url;//explode("|", $this->input)[2]; $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); $mail = json_decode(file_get_contents($messageUrl)); @@ -219,7 +219,7 @@ class ProcessMailgunInboundWebhook implements ShouldQueue } else { $credentials = config('services.mailgun.domain') . ":" . config('services.mailgun.secret') . "@"; - $messageUrl = explode("|", $this->input)[2]; + $messageUrl = $this->message_url; //explode("|", $this->input)[2]; $messageUrl = str_replace("http://", "http://" . $credentials, $messageUrl); $messageUrl = str_replace("https://", "https://" . $credentials, $messageUrl); $mail = json_decode(file_get_contents($messageUrl)); diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index c6934b50decc..8173f5286861 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -310,7 +310,7 @@ class InboundMailEngine } private function logBlocked(Company $company, string $data) { - nlog("[InboundMailEngine][company:" . $company->id . "] " . $data); + nlog("[InboundMailEngine][company:" . $company->company_key . "] " . $data); ( new SystemLogger( From 8f88c408f7f204547ba15c9a60fb34d77061da88 Mon Sep 17 00:00:00 2001 From: David Bomba Date: Thu, 29 Aug 2024 17:17:52 +1000 Subject: [PATCH 112/119] minor fixeS --- app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index 565cf2af4e34..961e489a2b64 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -187,10 +187,10 @@ class ProcessMailgunInboundWebhook implements ShouldQueue try { // important to save meta if something fails here to prevent spam // fetch message from mailgun-api - $company_mailgun_domain = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_domain ? $company->settings?->mailgun_domain : null; - $company_mailgun_secret = $company->settings?->email_sending_method === 'client_mailgun' && $company->settings?->mailgun_secret ? $company->settings?->mailgun_secret : null; + $company_mailgun_domain = $company->getSetting('email_sending_method') == 'client_mailgun' && strlen($company->getSetting('mailgun_domain') ?? '') > 2 ? $company->getSetting('mailgun_domain') : null; + $company_mailgun_secret = $company->getSetting('email_sending_method') == 'client_mailgun' && strlen($company->getSetting('mailgun_secret') ?? '') > 2 ? $company->getSetting('mailgun_secret') : null; if (!($company_mailgun_domain && $company_mailgun_secret) && !(config('services.mailgun.domain') && config('services.mailgun.secret'))) - throw new \Error("[ProcessMailgunInboundWebhook] no mailgun credenitals found, we cannot get the attachements and files"); + throw new \Error("[ProcessMailgunInboundWebhook] no mailgun credentials found, we cannot get the attachements and files"); $mail = null; if ($company_mailgun_domain && $company_mailgun_secret) { From 93c382eae119101b13a965b113ae44f1e1c848fc Mon Sep 17 00:00:00 2001 From: David Bomba Date: Fri, 30 Aug 2024 10:57:43 +1000 Subject: [PATCH 113/119] Minor refactors for inbound email processing --- app/Http/Controllers/MailgunController.php | 16 +++- app/Http/Controllers/PostMarkController.php | 17 ++-- .../Company/ValidExpenseMailbox.php | 3 - app/Jobs/Brevo/ProcessBrevoInboundWebhook.php | 47 +++++++---- .../Mailgun/ProcessMailgunInboundWebhook.php | 34 ++++---- .../EDocument/Imports/MindeeEDocument.php | 53 ++++++++----- .../EDocument/Imports/ZugferdEDocument.php | 23 +++--- .../InboundMail/InboundMailEngine.php | 78 ++++++++++--------- ...2023_12_10_110951_inbound_mail_parsing.php | 2 +- 9 files changed, 162 insertions(+), 111 deletions(-) diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 7db6f3dc773b..06a37b0cff55 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -11,9 +11,11 @@ namespace App\Http\Controllers; -use App\Jobs\Mailgun\ProcessMailgunInboundWebhook; -use App\Jobs\Mailgun\ProcessMailgunWebhook; +use App\Models\Company; +use App\Libraries\MultiDB; use Illuminate\Http\Request; +use App\Jobs\Mailgun\ProcessMailgunWebhook; +use App\Jobs\Mailgun\ProcessMailgunInboundWebhook; /** * Class MailgunController. @@ -126,9 +128,15 @@ class MailgunController extends BaseController $authorizedByHash = \hash_equals(\hash_hmac('sha256', $input['timestamp'] . $input['token'], config('services.mailgun.webhook_signing_key')), $input['signature']); $authorizedByToken = $request->has('token') && $request->get('token') == config('ninja.inbound_mailbox.inbound_webhook_token'); if (!$authorizedByHash && !$authorizedByToken) - return response()->json(['message' => 'Unauthorized'], 403); + return response()->json(['message' => 'Unauthorized'], 403); - ProcessMailgunInboundWebhook::dispatch($input["sender"], $input["recipient"], $input["message-url"])->delay(rand(2, 10)); + /** @var \App\Models\Company $company */ + $company = MultiDB::findAndSetDbByExpenseMailbox($input["recipient"]); + + if(!$company) + return response()->json(['message' => 'Ok'], 200); // Fail gracefully + + ProcessMailgunInboundWebhook::dispatch($input["sender"], $input["recipient"], $input["message-url"], $company)->delay(rand(2, 10)); return response()->json(['message' => 'Success.'], 200); } diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index 4ed1434c66f8..25bf7976d43a 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -280,20 +280,21 @@ class PostMarkController extends BaseController nlog('Failed: Message could not be parsed, because required parameters are missing.'); return response()->json(['message' => 'Failed. Missing/Invalid Parameters.'], 400); } + + $company = MultiDB::findAndSetDbByExpenseMailbox($input["To"]); - $inboundEngine = new InboundMailEngine(); + if (!$company) { + nlog('[PostmarkInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $input["To"]); + // $inboundEngine->saveMeta($input["From"], $input["To"], true); // important to save this, to protect from spam + return response()->json(['message' => 'Ok'], 200); + } + + $inboundEngine = new InboundMailEngine($company); if ($inboundEngine->isInvalidOrBlocked($input["From"], $input["To"])) { return response()->json(['message' => 'Blocked.'], 403); } - $company = MultiDB::findAndSetDbByExpenseMailbox($input["To"]); - if (!$company) { - nlog('[PostmarkInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $input["To"]); - $inboundEngine->saveMeta($input["From"], $input["To"], true); // important to save this, to protect from spam - return; - } - try { // important to save meta if something fails here to prevent spam // prepare data for ingresEngine diff --git a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php index aa333fddc0e6..8ae3b97129bb 100644 --- a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php +++ b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php @@ -23,10 +23,7 @@ class ValidExpenseMailbox implements Rule { private $validated_schema = false; - private $isEnterprise = false; private array $endings; - private bool $hasCompanyKey; - private array $enterprise_endings; public function __construct() { diff --git a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php index b0ea10eaa47c..ae6660b7efe3 100644 --- a/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php +++ b/app/Jobs/Brevo/ProcessBrevoInboundWebhook.php @@ -11,19 +11,20 @@ namespace App\Jobs\Brevo; -use App\Libraries\MultiDB; -use App\Services\InboundMail\InboundMail; -use App\Services\InboundMail\InboundMailEngine; use App\Utils\TempFile; -use Brevo\Client\Api\InboundParsingApi; -use Brevo\Client\Configuration; -use Illuminate\Support\Carbon; +use App\Libraries\MultiDB; use Illuminate\Bus\Queueable; +use Illuminate\Support\Carbon; +use Brevo\Client\Configuration; +use Illuminate\Http\UploadedFile; +use Illuminate\Queue\SerializesModels; +use Brevo\Client\Api\InboundParsingApi; +use Illuminate\Queue\InteractsWithQueue; +use App\Services\InboundMail\InboundMail; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; -use Illuminate\Queue\InteractsWithQueue; -use Illuminate\Queue\SerializesModels; -use Illuminate\Http\UploadedFile; +use App\Services\InboundMail\InboundMailEngine; +use Illuminate\Queue\Middleware\WithoutOverlapping; class ProcessBrevoInboundWebhook implements ShouldQueue { @@ -111,7 +112,6 @@ class ProcessBrevoInboundWebhook implements ShouldQueue */ public function __construct(private array $input) { - $this->engine = new InboundMailEngine(); } /** @@ -134,18 +134,24 @@ class ProcessBrevoInboundWebhook implements ShouldQueue // match company $company = MultiDB::findAndSetDbByExpenseMailbox($recipient); + if (!$company) { nlog('[ProcessBrevoInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from brevo: ' . $recipient); continue; } + $this->engine = new InboundMailEngine($company); + $foundOneRecipient = true; try { // important to save meta if something fails here to prevent spam - $company_brevo_secret = $company->settings?->email_sending_method === 'client_brevo' && $company->settings?->brevo_secret ? $company->settings?->brevo_secret : null; - if (empty($company_brevo_secret) && empty(config('services.brevo.secret'))) + if(strlen($company->getSetting('brevo_secret') ?? '') < 2 && empty(config('services.brevo.secret'))){ + nlog("No Brevo Configuration available for this company"); throw new \Error("[ProcessBrevoInboundWebhook] no brevo credenitals found, we cannot get the attachement"); + } + + $company_brevo_secret = strlen($company->getSetting('brevo_secret') ?? '') < 2 ? $company->getSetting('brevo_secret') : config('services.brevo.secret'); // prepare data for ingresEngine $inboundMail = new InboundMail(); @@ -160,8 +166,10 @@ class ProcessBrevoInboundWebhook implements ShouldQueue // parse documents as UploadedFile from webhook-data foreach ($this->input["Attachments"] as $attachment) { + // @todo - i think this allows switching between client configured brevo AND system configured brevo // download file and save to tmp dir - if (!empty($company_brevo_secret)) { + if (!empty($company_brevo_secret)) + { try { @@ -220,4 +228,17 @@ class ProcessBrevoInboundWebhook implements ShouldQueue $this->engine->saveMeta($this->input["From"]["Address"], $recipient, true); } } + + public function middleware() + { + return [new WithoutOverlapping($this->input["From"]["Address"])]; + } + + public function failed($exception) + { + nlog("BREVO:: Ingest Exception:: => ".$exception->getMessage()); + config(['queue.failed.driver' => null]); + } + + } diff --git a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php index 961e489a2b64..84ecc06d5ba2 100644 --- a/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php +++ b/app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php @@ -11,16 +11,17 @@ namespace App\Jobs\Mailgun; -use App\Libraries\MultiDB; -use App\Services\InboundMail\InboundMail; -use App\Services\InboundMail\InboundMailEngine; +use App\Models\Company; use App\Utils\TempFile; -use Illuminate\Support\Carbon; +use App\Libraries\MultiDB; use Illuminate\Bus\Queueable; +use Illuminate\Support\Carbon; +use Illuminate\Queue\SerializesModels; +use Illuminate\Queue\InteractsWithQueue; +use App\Services\InboundMail\InboundMail; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; -use Illuminate\Queue\InteractsWithQueue; -use Illuminate\Queue\SerializesModels; +use App\Services\InboundMail\InboundMailEngine; class ProcessMailgunInboundWebhook implements ShouldQueue { @@ -34,9 +35,9 @@ class ProcessMailgunInboundWebhook implements ShouldQueue * Create a new job instance. * $input consists of 3 informations: sender/from|recipient/to|messageUrl */ - public function __construct(private string $sender, private string $recipient, private string $message_url) + public function __construct(private string $sender, private string $recipient, private string $message_url, private Company $company) { - $this->engine = new InboundMailEngine(); + $this->engine = new InboundMailEngine($company); } /** @@ -176,19 +177,20 @@ class ProcessMailgunInboundWebhook implements ShouldQueue return; } + // lets assess this at a higher level to ensure that only valid email inboxes are processed. // match company - $company = MultiDB::findAndSetDbByExpenseMailbox($to); - if (!$company) { - nlog('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $to); - $this->engine->saveMeta($from, $to, true); // important to save this, to protect from spam - return; - } + // $company = MultiDB::findAndSetDbByExpenseMailbox($to); + // if (!$company) { + // nlog('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $to); + // $this->engine->saveMeta($from, $to, true); // important to save this, to protect from spam + // return; + // } try { // important to save meta if something fails here to prevent spam // fetch message from mailgun-api - $company_mailgun_domain = $company->getSetting('email_sending_method') == 'client_mailgun' && strlen($company->getSetting('mailgun_domain') ?? '') > 2 ? $company->getSetting('mailgun_domain') : null; - $company_mailgun_secret = $company->getSetting('email_sending_method') == 'client_mailgun' && strlen($company->getSetting('mailgun_secret') ?? '') > 2 ? $company->getSetting('mailgun_secret') : null; + $company_mailgun_domain = $this->company->getSetting('email_sending_method') == 'client_mailgun' && strlen($this->company->getSetting('mailgun_domain') ?? '') > 2 ? $this->company->getSetting('mailgun_domain') : null; + $company_mailgun_secret = $this->company->getSetting('email_sending_method') == 'client_mailgun' && strlen($this->company->getSetting('mailgun_secret') ?? '') > 2 ? $this->company->getSetting('mailgun_secret') : null; if (!($company_mailgun_domain && $company_mailgun_secret) && !(config('services.mailgun.domain') && config('services.mailgun.secret'))) throw new \Error("[ProcessMailgunInboundWebhook] no mailgun credentials found, we cannot get the attachements and files"); diff --git a/app/Services/EDocument/Imports/MindeeEDocument.php b/app/Services/EDocument/Imports/MindeeEDocument.php index a6e72f33ba5e..1b238c28824a 100644 --- a/app/Services/EDocument/Imports/MindeeEDocument.php +++ b/app/Services/EDocument/Imports/MindeeEDocument.php @@ -47,8 +47,10 @@ class MindeeEDocument extends AbstractService public function run(): Expense { $api_key = config('services.mindee.api_key'); + if (!$api_key) throw new Exception('Mindee API key not configured'); + $this->checkLimits(); // perform parsing @@ -69,42 +71,52 @@ class MindeeEDocument extends AbstractService $invoiceCurrency = $prediction->locale->currency; $country = $prediction->locale->country; - $expense = Expense::where('amount', $grandTotalAmount)->where("transaction_reference", $documentno)->whereDate("date", $documentdate)->first(); - if (empty($expense)) { + $expense = Expense::query() + ->where('company_id', $this->company->id) + ->where('amount', $grandTotalAmount) + ->where("transaction_reference", $documentno) + ->whereDate("date", $documentdate) + ->first(); + + if (!$expense) { // The document does not exist as an expense // Handle accordingly + + /** @var \App\Models\Currency $currency */ + $currency = app('currencies')->first(function ($c) use ($invoiceCurrency){ + /** @var \App\Models\Currency $c */ + return $c->code == $invoiceCurrency; + }); + $expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id); $expense->date = $documentdate; - $expense->user_id = $this->company->owner()->id; - $expense->company_id = $this->company->id; $expense->public_notes = $documentno; - $expense->currency_id = Currency::whereCode($invoiceCurrency)->first()?->id || $this->company->settings->currency_id; + $expense->currency_id = $currency ? $currency->id : $this->company->settings->currency_id; $expense->save(); $this->saveDocuments([ $this->file, TempFile::UploadedFileFromRaw(strval($result->document), $documentno . "_mindee_orc_result.txt", "text/plain") ], $expense); - $expense->saveQuietly(); + // $expense->saveQuietly(); $expense->uses_inclusive_taxes = True; $expense->amount = $grandTotalAmount; $counter = 1; + foreach ($prediction->taxes as $taxesElem) { - $expense->{"tax_amount$counter"} = $taxesElem->amount; - $expense->{"tax_rate$counter"} = $taxesElem->rate; + $expense->{"tax_amount{$counter}"} = $taxesElem->amount; + $expense->{"tax_rate{$counter}"} = $taxesElem->rate; $counter++; } - - $vendor = null; - $vendor_contact = VendorContact::where("company_id", $this->company->id)->where("email", $prediction->supplierEmail)->first(); - if ($vendor_contact) - $vendor = $vendor_contact->vendor; - if (!$vendor) - $vendor = Vendor::where("company_id", $this->company->id)->where("name", $prediction->supplierName)->first(); + + /** @var \App\Models\VendorContact $vendor_contact */ + $vendor_contact = VendorContact::query()->where("company_id", $this->company->id)->where("email", $prediction->supplierEmail)->first(); + + /** @var \App\Models\Vendor|null $vendor */ + $vendor = $vendor_contact ? $vendor_contact->vendor : Vendor::query()->where("company_id", $this->company->id)->where("name", $prediction->supplierName)->first(); if ($vendor) { - // Vendor found $expense->vendor_id = $vendor->id; } else { $vendor = VendorFactory::create($this->company->id, $this->company->owner()->id); @@ -116,15 +128,19 @@ class MindeeEDocument extends AbstractService // $vendor->address2 = $address_2; // $vendor->city = $city; // $vendor->postal_code = $postcode; + + /** @var ?\App\Models\Country $country */ $country = app('countries')->first(function ($c) use ($country) { + /** @var \App\Models\Country $c */ return $c->iso_3166_2 == $country || $c->iso_3166_3 == $country; }); + if ($country) $vendor->country_id = $country->id; $vendor->save(); - if ($prediction->supplierEmail) { + if (strlen($prediction->supplierEmail ?? '') > 2) { $vendor_contact = VendorContactFactory::create($this->company->id, $this->company->owner()->id); $vendor_contact->vendor_id = $vendor->id; $vendor_contact->email = $prediction->supplierEmail; @@ -138,8 +154,9 @@ class MindeeEDocument extends AbstractService // The document exists as an expense // Handle accordingly nlog("Mindee: Document already exists"); - $expense->private_notes = $expense->private_notes . ctrans("texts.edocument_import_already_exists", ["date" => time()]); + $expense->private_notes = $expense->private_notes . ctrans("texts.edocument_import_already_exists", ["date" => now()->format('Y-m-d')]); } + $expense->save(); return $expense; } diff --git a/app/Services/EDocument/Imports/ZugferdEDocument.php b/app/Services/EDocument/Imports/ZugferdEDocument.php index ed7215e54e6c..f221b3547f64 100644 --- a/app/Services/EDocument/Imports/ZugferdEDocument.php +++ b/app/Services/EDocument/Imports/ZugferdEDocument.php @@ -102,17 +102,20 @@ class ZugferdEDocument extends AbstractService if (array_key_exists("VA", $taxtype)) { $taxid = $taxtype["VA"]; } - $vendor = Vendor::where("company_id", $user->company()->id)->where('vat_number', $taxid)->first(); - if (!$vendor) { - $vendor_contact = VendorContact::where("company_id", $user->company()->id)->where("email", $contact_email)->first(); - if ($vendor_contact) - $vendor = $vendor_contact->vendor; - } - if (!$vendor) - $vendor = Vendor::where("company_id", $user->company()->id)->where("name", $person_name)->first(); - if (!empty($vendor)) { - // Vendor found + $vendor = Vendor::query() + ->where("company_id", $user->company()->id) + ->where(function ($q) use($taxid, $person_name, $contact_email){ + $q->when(!is_null($taxid), function ($when_query) use($taxid){ + $when_query->orWhere('vat_number', $taxid); + }) + ->orWhere("name", $person_name) + ->orWhereHas('contacts', function ($qq) use ($contact_email){ + $qq->where("email", $contact_email); + }); + })->first(); + + if ($vendor) { $expense->vendor_id = $vendor->id; } else { $vendor = VendorFactory::create($this->company->id, $user->id); diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index 8173f5286861..3aee96e2fdc0 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -35,7 +35,7 @@ class InboundMailEngine private array $globalBlacklist; private array $globalWhitelist; // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders - public function __construct() + public function __construct(private Company $company) { // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders @@ -52,19 +52,19 @@ class InboundMailEngine return; // Expense Mailbox => will create an expense - $company = MultiDB::findAndSetDbByExpenseMailbox($email->to); - if (!$company) { - $this->saveMeta($email->from, $email->to, true); - return; - } + // $company = MultiDB::findAndSetDbByExpenseMailbox($email->to); + // if (!$company) { + // $this->saveMeta($email->from, $email->to, true); + // return; + // } // check if company plan matches requirements - if (Ninja::isHosted() && !($company->account->isPaid() && $company->account->plan == 'enterprise')) { + if (Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise')) { $this->saveMeta($email->from, $email->to); return; } - $this->createExpenses($company, $email); + $this->createExpenses($email); $this->saveMeta($email->from, $email->to); } @@ -145,6 +145,8 @@ class InboundMailEngine // TODO: ignore, when known sender (for heavy email-usage mostly on isHosted()) // TODO: handle external blocking } + + //@todo - refactor public function saveMeta(string $from, string $to, bool $isUnknownRecipent = false) { // save cache @@ -161,24 +163,24 @@ class InboundMailEngine } // MAIN-PROCESSORS - protected function createExpenses(Company $company, InboundMail $email) + protected function createExpenses(InboundMail $email) { // Skipping executions: will not result in not saving Metadata to prevent usage of these conditions, to spam - if (!($company?->expense_mailbox_active ?: false)) { - $this->logBlocked($company, 'mailbox not active for this company. from: ' . $email->from); + if (!$this->company->expense_mailbox_active) { + $this->logBlocked($this->company, 'mailbox not active for this company. from: ' . $email->from); return; } - if (!$this->validateExpenseSender($company, $email)) { - $this->logBlocked($company, 'invalid sender of an ingest email for this company. from: ' . $email->from); + if (!$this->validateExpenseSender($email)) { + $this->logBlocked($this->company, 'invalid sender of an ingest email for this company. from: ' . $email->from); return; } if (sizeOf($email->documents) == 0) { - $this->logBlocked($company, 'email does not contain any attachments and is likly not an expense. from: ' . $email->from); + $this->logBlocked($this->company, 'email does not contain any attachments and is likly not an expense. from: ' . $email->from); return; } // prepare data - $expense_vendor = $this->getVendor($company, $email); + $expense_vendor = $this->getVendor($email); $this->processHtmlBodyToDocument($email); $parsed_expense_ids = []; // used to check if an expense was already matched within this job @@ -192,7 +194,7 @@ class InboundMailEngine // check if document can be parsed to an expense try { - $expense = (new ParseEDocument($document, $company))->run(); + $expense = (new ParseEDocument($document, $this->company))->run(); // check if expense was already matched within this job and skip if true if (array_search($expense->id, $parsed_expense_ids)) @@ -213,7 +215,7 @@ class InboundMailEngine // populate missing data with data from email if (!$expense) - $expense = ExpenseFactory::create($company->id, $company->owner()->id); + $expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id); $is_imported_by_parser = array_search($expense->id, $parsed_expense_ids); @@ -256,61 +258,61 @@ class InboundMailEngine $email->body_document = TempFile::UploadedFileFromRaw($email->body, "E-Mail.html", "text/html"); } - private function validateExpenseSender(Company $company, InboundMail $email) + private function validateExpenseSender(InboundMail $email) { $parts = explode('@', $email->from); $domain = array_pop($parts); // whitelists - $whitelist = explode(",", $company->inbound_mailbox_whitelist); + $whitelist = explode(",", $this->company->inbound_mailbox_whitelist); if (in_array($email->from, $whitelist)) return true; if (in_array($domain, $whitelist)) return true; - $blacklist = explode(",", $company->inbound_mailbox_blacklist); + $blacklist = explode(",", $this->company->inbound_mailbox_blacklist); if (in_array($email->from, $blacklist)) return false; if (in_array($domain, $blacklist)) return false; // allow unknown - if ($company->inbound_mailbox_allow_unknown) + if ($this->company->inbound_mailbox_allow_unknown) return true; // own users - if ($company->inbound_mailbox_allow_company_users && $company->users()->where("email", $email->from)->exists()) + if ($this->company->inbound_mailbox_allow_company_users && $this->company->users()->where("email", $email->from)->exists()) return true; // from vendors - if ($company->inbound_mailbox_allow_vendors && VendorContact::where("company_id", $company->id)->where("email", $email->from)->exists()) + if ($this->company->inbound_mailbox_allow_vendors && VendorContact::where("company_id", $this->company->id)->where("email", $email->from)->exists()) return true; // from clients - if ($company->inbound_mailbox_allow_clients && ClientContact::where("company_id", $company->id)->where("email", $email->from)->exists()) + if ($this->company->inbound_mailbox_allow_clients && ClientContact::where("company_id", $this->company->id)->where("email", $email->from)->exists()) return true; // denie return false; } - private function getClient(Company $company, InboundMail $email) - { - $clientContact = ClientContact::where("company_id", $company->id)->where("email", $email->from)->first(); - if (!$clientContact) - return null; - return $clientContact->client(); - } - private function getVendor(Company $company, InboundMail $email) - { - $vendorContact = VendorContact::where("company_id", $company->id)->where("email", $email->from)->first(); - if (!$vendorContact) - return null; + // private function getClient(InboundMail $email) + // { + // $clientContact = ClientContact::where("company_id", $this->company->id)->where("email", $email->from)->first(); + // if (!$clientContact) + // return null; - return $vendorContact->vendor(); + // return $clientContact->client(); + // } + private function getVendor(InboundMail $email) + { + $vendorContact = VendorContact::with('vendor')->where("company_id", $this->company->id)->where("email", $email->from)->first(); + + return $vendorContact ? $vendorContact->vendor : null; } + private function logBlocked(Company $company, string $data) { - nlog("[InboundMailEngine][company:" . $company->company_key . "] " . $data); + nlog("[InboundMailEngine][company:" . $this->company->company_key . "] " . $data); ( new SystemLogger( diff --git a/database/migrations/2023_12_10_110951_inbound_mail_parsing.php b/database/migrations/2023_12_10_110951_inbound_mail_parsing.php index 056096254f4a..7bbed2aab785 100644 --- a/database/migrations/2023_12_10_110951_inbound_mail_parsing.php +++ b/database/migrations/2023_12_10_110951_inbound_mail_parsing.php @@ -11,7 +11,7 @@ return new class extends Migration { public function up(): void { Schema::table('companies', function (Blueprint $table) { - $table->boolean("expense_mailbox_active")->default(true); + $table->boolean("expense_mailbox_active")->default(false); $table->string("expense_mailbox")->nullable(); $table->boolean("inbound_mailbox_allow_company_users")->default(false); $table->boolean("inbound_mailbox_allow_vendors")->default(false); From f117e79d43663cd9c39360974a084762aa09e7b8 Mon Sep 17 00:00:00 2001 From: David Bomba Date: Mon, 2 Sep 2024 13:07:01 +1000 Subject: [PATCH 114/119] Patches for inbound emails --- app/Http/Controllers/PostMarkController.php | 10 +++++----- app/Services/EDocument/Imports/ParseEDocument.php | 1 + app/Services/EDocument/Imports/ZugferdEDocument.php | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index 25bf7976d43a..88e6812fd38e 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -300,11 +300,11 @@ class PostMarkController extends BaseController // prepare data for ingresEngine $inboundMail = new InboundMail(); - $inboundMail->from = $input["From"]; - $inboundMail->to = $input["To"]; // usage of data-input, because we need a single email here - $inboundMail->subject = $input["Subject"]; - $inboundMail->body = $input["HtmlBody"]; - $inboundMail->text_body = $input["TextBody"]; + $inboundMail->from = $input["From"] ?? ''; + $inboundMail->to = $input["To"] ; // usage of data-input, because we need a single email here + $inboundMail->subject = $input["Subject"] ?? ''; + $inboundMail->body = $input["HtmlBody"] ?? ''; + $inboundMail->text_body = $input["TextBody"] ?? ''; $inboundMail->date = Carbon::createFromTimeString($input["Date"]); // parse documents as UploadedFile from webhook-data diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index b58d4d7664b1..e49d39d68b9e 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -57,6 +57,7 @@ class ParseEDocument extends AbstractService try { return (new ZugferdEDocument($this->file, $this->company))->run(); } catch (Exception $e) { + nlog($this->file->get()); nlog("Zugferd Exception: " . $e->getMessage()); } } diff --git a/app/Services/EDocument/Imports/ZugferdEDocument.php b/app/Services/EDocument/Imports/ZugferdEDocument.php index f221b3547f64..cc70e4cd6694 100644 --- a/app/Services/EDocument/Imports/ZugferdEDocument.php +++ b/app/Services/EDocument/Imports/ZugferdEDocument.php @@ -104,7 +104,7 @@ class ZugferdEDocument extends AbstractService } $vendor = Vendor::query() - ->where("company_id", $user->company()->id) + ->where("company_id", $this->company->id) ->where(function ($q) use($taxid, $person_name, $contact_email){ $q->when(!is_null($taxid), function ($when_query) use($taxid){ $when_query->orWhere('vat_number', $taxid); From 3aa17bd6cded3553c30b0d8a4547e58a1fb7a77b Mon Sep 17 00:00:00 2001 From: paulwer Date: Sun, 15 Sep 2024 08:18:07 +0200 Subject: [PATCH 115/119] remove mimes validation within edocument endpoint, because ParseEDocument handles this + minor code cleanups --- app/Http/Controllers/ExpenseController.php | 46 +++++++++++++++++++ app/Http/Controllers/PostMarkController.php | 4 +- .../Requests/Expense/EDocumentRequest.php | 4 +- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/app/Http/Controllers/ExpenseController.php b/app/Http/Controllers/ExpenseController.php index a6b887f58f00..5a702c6842e8 100644 --- a/app/Http/Controllers/ExpenseController.php +++ b/app/Http/Controllers/ExpenseController.php @@ -584,6 +584,52 @@ class ExpenseController extends BaseController return $this->itemResponse($expense->fresh()); } + /** + * @OA\Post( + * path="/api/v1/expenses/edocument", + * operationId="edocumentExpense", + * tags={"expenses"}, + * summary="Uploads an electronic document to a expense", + * description="Handles the uploading of an electronic document to a expense", + * @OA\Parameter(ref="#/components/parameters/X-API-TOKEN"), + * @OA\Parameter(ref="#/components/parameters/X-Requested-With"), + * @OA\Parameter(ref="#/components/parameters/include"), + * @OA\RequestBody( + * description="User credentials", + * required=true, + * @OA\MediaType( + * mediaType="multipart/form-data", + * @OA\Schema( + * type="array", + * @OA\Items( + * type="string", + * format="binary", + * description="The files to be uploaded", + * ), + * ) + * ) + * ), + * @OA\Response( + * response=200, + * description="Returns a HTTP status", + * @OA\Header(header="X-MINIMUM-CLIENT-VERSION", ref="#/components/headers/X-MINIMUM-CLIENT-VERSION"), + * @OA\Header(header="X-RateLimit-Remaining", ref="#/components/headers/X-RateLimit-Remaining"), + * @OA\Header(header="X-RateLimit-Limit", ref="#/components/headers/X-RateLimit-Limit"), + * ), + * @OA\Response( + * response=422, + * description="Validation error", + * @OA\JsonContent(ref="#/components/schemas/ValidationError"), + * + * ), + * @OA\Response( + * response="default", + * response="default", + * description="Unexpected Error", + * @OA\JsonContent(ref="#/components/schemas/Error"), + * ), + * ) + */ public function edocument(EDocumentRequest $request) { $user = auth()->user(); diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index 88e6812fd38e..0358a610d397 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -280,7 +280,7 @@ class PostMarkController extends BaseController nlog('Failed: Message could not be parsed, because required parameters are missing.'); return response()->json(['message' => 'Failed. Missing/Invalid Parameters.'], 400); } - + $company = MultiDB::findAndSetDbByExpenseMailbox($input["To"]); if (!$company) { @@ -301,7 +301,7 @@ class PostMarkController extends BaseController $inboundMail = new InboundMail(); $inboundMail->from = $input["From"] ?? ''; - $inboundMail->to = $input["To"] ; // usage of data-input, because we need a single email here + $inboundMail->to = $input["To"]; // usage of data-input, because we need a single email here $inboundMail->subject = $input["Subject"] ?? ''; $inboundMail->body = $input["HtmlBody"] ?? ''; $inboundMail->text_body = $input["TextBody"] ?? ''; diff --git a/app/Http/Requests/Expense/EDocumentRequest.php b/app/Http/Requests/Expense/EDocumentRequest.php index 643c3cd6fc33..410c9a716fe8 100644 --- a/app/Http/Requests/Expense/EDocumentRequest.php +++ b/app/Http/Requests/Expense/EDocumentRequest.php @@ -25,9 +25,9 @@ class EDocumentRequest extends Request $rules = []; if ($this->file('documents') && is_array($this->file('documents'))) { - $rules['documents.*'] = 'required|file|max:1000000|mimes:xml'; + $rules['documents.*'] = 'required|file|max:1000000'; } elseif ($this->file('documents')) { - $rules['documents'] = 'required|file|max:1000000|mimes:xml'; + $rules['documents'] = 'required|file|max:1000000'; } return $rules; } From 10f69770234929d282c4c17f22c726c0ff791b4c Mon Sep 17 00:00:00 2001 From: David Bomba Date: Sun, 15 Sep 2024 17:32:43 +1000 Subject: [PATCH 116/119] Adjustments for expense imports --- app/Http/Controllers/MailgunController.php | 4 ++++ app/Http/Controllers/PostMarkController.php | 9 +++++---- .../Requests/Company/UpdateCompanyRequest.php | 15 ++++++++------- .../Company/ValidExpenseMailbox.php | 15 +++------------ app/Services/InboundMail/InboundMail.php | 2 +- app/Services/InboundMail/InboundMailEngine.php | 8 ++++---- 6 files changed, 25 insertions(+), 28 deletions(-) diff --git a/app/Http/Controllers/MailgunController.php b/app/Http/Controllers/MailgunController.php index 06a37b0cff55..d44b5e3ad2ea 100644 --- a/app/Http/Controllers/MailgunController.php +++ b/app/Http/Controllers/MailgunController.php @@ -64,6 +64,8 @@ class MailgunController extends BaseController { $input = $request->all(); + + nlog($input); if (\abs(\time() - $request['signature']['timestamp']) > 15) { return response()->json(['message' => 'Success'], 200); @@ -118,6 +120,8 @@ class MailgunController extends BaseController { $input = $request->all(); + nlog($input); + if (!array_key_exists('sender', $input) || !array_key_exists('recipient', $input) || !array_key_exists('message-url', $input)) { nlog('Failed: Message could not be parsed, because required parameters are missing. Please ensure contacting this api-endpoint with a store & notify operation instead of a forward operation!'); return response()->json(['message' => 'Failed. Missing Parameters. Use store and notify!'], 400); diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index 88e6812fd38e..82905513fcc7 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -273,7 +273,9 @@ class PostMarkController extends BaseController $input = $request->all(); - if (!($request->has('token') && $request->get('token') == config('ninja.inbound_mailbox.inbound_webhook_token'))) + nlog($input); + + if (!$request->has('token') || $request->token != config('ninja.inbound_mailbox.inbound_webhook_token')) return response()->json(['message' => 'Unauthorized'], 403); if (!(array_key_exists("MessageStream", $input) && $input["MessageStream"] == "inbound") || !array_key_exists("To", $input) || !array_key_exists("From", $input) || !array_key_exists("MessageID", $input)) { @@ -281,17 +283,16 @@ class PostMarkController extends BaseController return response()->json(['message' => 'Failed. Missing/Invalid Parameters.'], 400); } - $company = MultiDB::findAndSetDbByExpenseMailbox($input["To"]); + $company = MultiDB::findAndSetDbByExpenseMailbox($input["ToFull"][0]["Email"]); if (!$company) { nlog('[PostmarkInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $input["To"]); - // $inboundEngine->saveMeta($input["From"], $input["To"], true); // important to save this, to protect from spam return response()->json(['message' => 'Ok'], 200); } $inboundEngine = new InboundMailEngine($company); - if ($inboundEngine->isInvalidOrBlocked($input["From"], $input["To"])) { + if ($inboundEngine->isInvalidOrBlocked($input["From"], $input["ToFull"][0]["Email"])) { return response()->json(['message' => 'Blocked.'], 403); } diff --git a/app/Http/Requests/Company/UpdateCompanyRequest.php b/app/Http/Requests/Company/UpdateCompanyRequest.php index 52d6d80aa9d4..44ef906d4476 100644 --- a/app/Http/Requests/Company/UpdateCompanyRequest.php +++ b/app/Http/Requests/Company/UpdateCompanyRequest.php @@ -11,14 +11,15 @@ namespace App\Http\Requests\Company; -use App\DataMapper\CompanySettings; -use App\Http\Requests\Request; -use App\Http\ValidationRules\Company\ValidExpenseMailbox; -use App\Http\ValidationRules\ValidSettingsRule; -use App\Http\ValidationRules\EInvoice\ValidCompanyScheme; -use App\Http\ValidationRules\Company\ValidSubdomain; use App\Utils\Ninja; +use App\Http\Requests\Request; use App\Utils\Traits\MakesHash; +use Illuminate\Validation\Rule; +use App\DataMapper\CompanySettings; +use App\Http\ValidationRules\ValidSettingsRule; +use App\Http\ValidationRules\Company\ValidSubdomain; +use App\Http\ValidationRules\Company\ValidExpenseMailbox; +use App\Http\ValidationRules\EInvoice\ValidCompanyScheme; class UpdateCompanyRequest extends Request { @@ -76,7 +77,7 @@ class UpdateCompanyRequest extends Request $rules['subdomain'] = ['nullable', 'regex:/^[a-zA-Z0-9.-]+[a-zA-Z0-9]$/', new ValidSubdomain()]; } - $rules['expense_mailbox'] = ['email', 'nullable', new ValidExpenseMailbox()]; + $rules['expense_mailbox'] = ['sometimes','email', 'nullable', new ValidExpenseMailbox(), Rule::unique('companies')->ignore($this->company->id)]; return $rules; } diff --git a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php index 8ae3b97129bb..eb7be7c3e447 100644 --- a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php +++ b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php @@ -32,30 +32,21 @@ class ValidExpenseMailbox implements Rule public function passes($attribute, $value) { - if (empty($value)) { + if (empty($value) || !config('ninja.inbound_mailbox.expense_mailbox_endings')) { return true; } - // early return, if we dont have any additional validation - if (!config('ninja.inbound_mailbox.expense_mailbox_endings')) { - $this->validated_schema = true; - return MultiDB::checkExpenseMailboxAvailable($value); - } // Validate Schema $validated = false; foreach ($this->endings as $ending) { if (str_ends_with($value, $ending)) { - $validated = true; - break; + return true; } } - if (!$validated) - return false; + return false; - $this->validated_schema = true; - return MultiDB::checkExpenseMailboxAvailable($value); } /** diff --git a/app/Services/InboundMail/InboundMail.php b/app/Services/InboundMail/InboundMail.php index 4532a041dedb..f7ab8652ea95 100644 --- a/app/Services/InboundMail/InboundMail.php +++ b/app/Services/InboundMail/InboundMail.php @@ -30,7 +30,7 @@ class InboundMail public string $text_body; - /** @var array[\Illuminate\Http\UploadedFile] $documents */ + /** @var array[?\Illuminate\Http\UploadedFile] $documents */ public array $documents = []; public ?Carbon $date = null; diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index 3aee96e2fdc0..aa08eee23d0e 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -265,14 +265,14 @@ class InboundMailEngine // whitelists $whitelist = explode(",", $this->company->inbound_mailbox_whitelist); - if (in_array($email->from, $whitelist)) + if (is_array($whitelist) && in_array($email->from, $whitelist)) return true; - if (in_array($domain, $whitelist)) + if (is_array($whitelist) && in_array($domain, $whitelist)) return true; $blacklist = explode(",", $this->company->inbound_mailbox_blacklist); - if (in_array($email->from, $blacklist)) + if (is_array($blacklist) && in_array($email->from, $blacklist)) return false; - if (in_array($domain, $blacklist)) + if (is_array($blacklist) && in_array($domain, $blacklist)) return false; // allow unknown From 3f0f5663a93955709161be26e8e735da9a50571b Mon Sep 17 00:00:00 2001 From: paulwer Date: Mon, 16 Sep 2024 06:57:02 +0200 Subject: [PATCH 117/119] fix: mindee taxation transform issue --- app/Http/Controllers/PostMarkController.php | 4 +--- .../EDocument/Imports/MindeeEDocument.php | 24 +++++++++---------- .../EDocument/Imports/ParseEDocument.php | 1 - 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/app/Http/Controllers/PostMarkController.php b/app/Http/Controllers/PostMarkController.php index e76d7eb72526..5e972cdfcb7a 100644 --- a/app/Http/Controllers/PostMarkController.php +++ b/app/Http/Controllers/PostMarkController.php @@ -273,8 +273,6 @@ class PostMarkController extends BaseController $input = $request->all(); - nlog($input); - if (!$request->has('token') || $request->token != config('ninja.inbound_mailbox.inbound_webhook_token')) return response()->json(['message' => 'Unauthorized'], 403); @@ -282,7 +280,7 @@ class PostMarkController extends BaseController nlog('Failed: Message could not be parsed, because required parameters are missing.'); return response()->json(['message' => 'Failed. Missing/Invalid Parameters.'], 400); } - + $company = MultiDB::findAndSetDbByExpenseMailbox($input["ToFull"][0]["Email"]); if (!$company) { diff --git a/app/Services/EDocument/Imports/MindeeEDocument.php b/app/Services/EDocument/Imports/MindeeEDocument.php index 1b238c28824a..0a92b403e48a 100644 --- a/app/Services/EDocument/Imports/MindeeEDocument.php +++ b/app/Services/EDocument/Imports/MindeeEDocument.php @@ -47,7 +47,7 @@ class MindeeEDocument extends AbstractService public function run(): Expense { $api_key = config('services.mindee.api_key'); - + if (!$api_key) throw new Exception('Mindee API key not configured'); @@ -72,22 +72,22 @@ class MindeeEDocument extends AbstractService $country = $prediction->locale->country; $expense = Expense::query() - ->where('company_id', $this->company->id) - ->where('amount', $grandTotalAmount) - ->where("transaction_reference", $documentno) - ->whereDate("date", $documentdate) - ->first(); + ->where('company_id', $this->company->id) + ->where('amount', $grandTotalAmount) + ->where("transaction_reference", $documentno) + ->whereDate("date", $documentdate) + ->first(); if (!$expense) { // The document does not exist as an expense // Handle accordingly /** @var \App\Models\Currency $currency */ - $currency = app('currencies')->first(function ($c) use ($invoiceCurrency){ - /** @var \App\Models\Currency $c */ + $currency = app('currencies')->first(function ($c) use ($invoiceCurrency) { + /** @var \App\Models\Currency $c */ return $c->code == $invoiceCurrency; }); - + $expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id); $expense->date = $documentdate; $expense->public_notes = $documentno; @@ -105,14 +105,14 @@ class MindeeEDocument extends AbstractService $counter = 1; foreach ($prediction->taxes as $taxesElem) { - $expense->{"tax_amount{$counter}"} = $taxesElem->amount; + $expense->{"tax_amount{$counter}"} = $taxesElem->value; $expense->{"tax_rate{$counter}"} = $taxesElem->rate; $counter++; } - + /** @var \App\Models\VendorContact $vendor_contact */ $vendor_contact = VendorContact::query()->where("company_id", $this->company->id)->where("email", $prediction->supplierEmail)->first(); - + /** @var \App\Models\Vendor|null $vendor */ $vendor = $vendor_contact ? $vendor_contact->vendor : Vendor::query()->where("company_id", $this->company->id)->where("name", $prediction->supplierName)->first(); diff --git a/app/Services/EDocument/Imports/ParseEDocument.php b/app/Services/EDocument/Imports/ParseEDocument.php index e49d39d68b9e..b58d4d7664b1 100644 --- a/app/Services/EDocument/Imports/ParseEDocument.php +++ b/app/Services/EDocument/Imports/ParseEDocument.php @@ -57,7 +57,6 @@ class ParseEDocument extends AbstractService try { return (new ZugferdEDocument($this->file, $this->company))->run(); } catch (Exception $e) { - nlog($this->file->get()); nlog("Zugferd Exception: " . $e->getMessage()); } } From ff695cdad00069b378a0de1553c9259fc8957b91 Mon Sep 17 00:00:00 2001 From: David Bomba Date: Mon, 16 Sep 2024 15:59:39 +1000 Subject: [PATCH 118/119] improve validation --- .../Requests/Company/UpdateCompanyRequest.php | 7 +++ .../Company/ValidExpenseMailbox.php | 13 ++---- config/ninja.php | 3 +- tests/Feature/CompanyTest.php | 45 +++++++++++++++++++ 4 files changed, 57 insertions(+), 11 deletions(-) diff --git a/app/Http/Requests/Company/UpdateCompanyRequest.php b/app/Http/Requests/Company/UpdateCompanyRequest.php index 44ef906d4476..1d26fb044b0d 100644 --- a/app/Http/Requests/Company/UpdateCompanyRequest.php +++ b/app/Http/Requests/Company/UpdateCompanyRequest.php @@ -78,6 +78,13 @@ class UpdateCompanyRequest extends Request } $rules['expense_mailbox'] = ['sometimes','email', 'nullable', new ValidExpenseMailbox(), Rule::unique('companies')->ignore($this->company->id)]; + $rules['expense_mailbox_active'] = ['sometimes','boolean']; + $rules['inbound_mailbox_allow_company_users'] = ['sometimes','boolean']; + $rules['inbound_mailbox_allow_vendors'] = ['sometimes','boolean']; + $rules['inbound_mailbox_allow_clients'] = ['sometimes','boolean']; + $rules['inbound_mailbox_allow_unknown'] = ['sometimes','boolean']; + $rules['inbound_mailbox_whitelist'] = ['sometimes', 'string', 'nullable', 'regex:/^[\w\-\.\+]+@([\w-]+\.)+[\w-]{2,4}(,[\w\-\.\+]+@([\w-]+\.)+[\w-]{2,4})*$/']; + $rules['inbound_mailbox_blacklist'] = ['sometimes', 'string', 'nullable', 'regex:/^[\w\-\.\+]+@([\w-]+\.)+[\w-]{2,4}(,[\w\-\.\+]+@([\w-]+\.)+[\w-]{2,4})*$/']; return $rules; } diff --git a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php index eb7be7c3e447..6d3a26a70a60 100644 --- a/app/Http/ValidationRules/Company/ValidExpenseMailbox.php +++ b/app/Http/ValidationRules/Company/ValidExpenseMailbox.php @@ -22,8 +22,7 @@ use Symfony\Component\Validator\Constraints\EmailValidator; class ValidExpenseMailbox implements Rule { - private $validated_schema = false; - private array $endings; + private array $endings = []; public function __construct() { @@ -35,10 +34,7 @@ class ValidExpenseMailbox implements Rule if (empty($value) || !config('ninja.inbound_mailbox.expense_mailbox_endings')) { return true; } - - - // Validate Schema - $validated = false; + foreach ($this->endings as $ending) { if (str_ends_with($value, $ending)) { return true; @@ -54,9 +50,6 @@ class ValidExpenseMailbox implements Rule */ public function message() { - if (!$this->validated_schema) - return ctrans('texts.expense_mailbox_invalid'); - - return ctrans('texts.expense_mailbox_taken'); + return ctrans('texts.expense_mailbox_invalid'); } } diff --git a/config/ninja.php b/config/ninja.php index a0444ce2ab8e..db302f6dbf0e 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -232,7 +232,8 @@ return [ 'webhook_id' => env('PAYPAL_WEBHOOK_ID', null), ], 'inbound_mailbox' => [ - 'expense_mailbox_endings' => env('EXPENSE_MAILBOX_ENDINGS', '@expense.invoicing.co'), + 'expense_mailbox_endings' => env('EXPENSE_MAILBOX_ENDINGS', false), + // 'expense_mailbox_endings' => env('EXPENSE_MAILBOX_ENDINGS', '@expense.invoicing.co'), 'inbound_webhook_token' => env('INBOUND_WEBHOOK_TOKEN', null), 'global_inbound_blacklist' => env('GLOBAL_INBOUND_BLACKLIST', ''), 'global_inbound_whitelist' => env('GLOBAL_INBOUND_WHITELIST', ''), diff --git a/tests/Feature/CompanyTest.php b/tests/Feature/CompanyTest.php index b6ca9508ec2f..55b15b45ef67 100644 --- a/tests/Feature/CompanyTest.php +++ b/tests/Feature/CompanyTest.php @@ -50,6 +50,49 @@ class CompanyTest extends TestCase $this->makeTestData(); } + + public function testCompanyExpenseMailbox() + { + // Test valid email address + $company_update = [ + 'expense_mailbox' => 'valid@example.com', + ]; + + $response = $this->withHeaders([ + 'X-API-SECRET' => config('ninja.api_secret'), + 'X-API-TOKEN' => $this->token, + ])->putJson('/api/v1/companies/'.$this->encodePrimaryKey($this->company->id), $company_update); + + $response->assertStatus(200); + $this->assertEquals('valid@example.com', $response->json('data.expense_mailbox')); + + // Test invalid email address + $company_update = [ + 'expense_mailbox' => 'invalid-email', + ]; + + $response = $this->withHeaders([ + 'X-API-SECRET' => config('ninja.api_secret'), + 'X-API-TOKEN' => $this->token, + ])->putJson('/api/v1/companies/'.$this->encodePrimaryKey($this->company->id), $company_update); + + $response->assertStatus(422); + $response->assertJsonValidationErrors(['expense_mailbox']); + + // Test empty email address + $company_update = [ + 'expense_mailbox' => '', + ]; + + $response = $this->withHeaders([ + 'X-API-SECRET' => config('ninja.api_secret'), + 'X-API-TOKEN' => $this->token, + ])->putJson('/api/v1/companies/'.$this->encodePrimaryKey($this->company->id), $company_update); + + $response->assertStatus(200); + $this->assertEmpty($response->json('data.expense_mailbox')); + } + public function testEnsureStrReplace() { $x = '**********'; @@ -216,4 +259,6 @@ class CompanyTest extends TestCase ])->delete('/api/v1/companies/'.$this->encodePrimaryKey($company->id)) ->assertStatus(200); } + + } From db7c21e761417745be6ef35a5309d796a96f2d48 Mon Sep 17 00:00:00 2001 From: David Bomba Date: Mon, 16 Sep 2024 16:47:57 +1000 Subject: [PATCH 119/119] Fixes for expense engine --- app/Services/InboundMail/InboundMail.php | 6 +-- .../InboundMail/InboundMailEngine.php | 42 +++++++------------ 2 files changed, 19 insertions(+), 29 deletions(-) diff --git a/app/Services/InboundMail/InboundMail.php b/app/Services/InboundMail/InboundMail.php index f7ab8652ea95..2430df047ed6 100644 --- a/app/Services/InboundMail/InboundMail.php +++ b/app/Services/InboundMail/InboundMail.php @@ -26,17 +26,17 @@ class InboundMail public ?string $subject = null; public ?string $body = null; + public ?UploadedFile $body_document = null; public string $text_body; - /** @var array[?\Illuminate\Http\UploadedFile] $documents */ + /** @var array $documents */ public array $documents = []; public ?Carbon $date = null; - function __constructor() + public function __construct() { - } } diff --git a/app/Services/InboundMail/InboundMailEngine.php b/app/Services/InboundMail/InboundMailEngine.php index aa08eee23d0e..b420a851f243 100644 --- a/app/Services/InboundMail/InboundMailEngine.php +++ b/app/Services/InboundMail/InboundMailEngine.php @@ -34,14 +34,15 @@ class InboundMailEngine use GeneratesCounter, SavesDocuments; private array $globalBlacklist; - private array $globalWhitelist; // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders + + private array $globalWhitelist; + public function __construct(private Company $company) { - - // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders $this->globalBlacklist = Ninja::isSelfHost() ? explode(",", config('ninja.inbound_mailbox.global_inbound_blocklist')) : []; $this->globalWhitelist = Ninja::isSelfHost() ? explode(",", config('ninja.inbound_mailbox.global_inbound_whitelist')) : []; } + /** * if there is not a company with an matching mailbox, we only do monitoring * reuse this method to add more mail-parsing behaviors @@ -51,20 +52,14 @@ class InboundMailEngine if ($this->isInvalidOrBlocked($email->from, $email->to)) return; - // Expense Mailbox => will create an expense - // $company = MultiDB::findAndSetDbByExpenseMailbox($email->to); - // if (!$company) { - // $this->saveMeta($email->from, $email->to, true); - // return; - // } // check if company plan matches requirements if (Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise')) { - $this->saveMeta($email->from, $email->to); return; } $this->createExpenses($email); + $this->saveMeta($email->from, $email->to); } @@ -149,7 +144,9 @@ class InboundMailEngine //@todo - refactor public function saveMeta(string $from, string $to, bool $isUnknownRecipent = false) { - // save cache + if(Ninja::isHosted()) + return; + Cache::add('inboundMailCountSender:' . $from, 0, now()->addHours(12)); Cache::increment('inboundMailCountSender:' . $from); @@ -167,20 +164,21 @@ class InboundMailEngine { // Skipping executions: will not result in not saving Metadata to prevent usage of these conditions, to spam if (!$this->company->expense_mailbox_active) { - $this->logBlocked($this->company, 'mailbox not active for this company. from: ' . $email->from); + $this->logBlocked('mailbox not active for this company. from: ' . $email->from); return; } if (!$this->validateExpenseSender($email)) { - $this->logBlocked($this->company, 'invalid sender of an ingest email for this company. from: ' . $email->from); + $this->logBlocked('invalid sender of an ingest email for this company. from: ' . $email->from); return; } - if (sizeOf($email->documents) == 0) { - $this->logBlocked($this->company, 'email does not contain any attachments and is likly not an expense. from: ' . $email->from); + if (count($email->documents) == 0) { + $this->logBlocked('email does not contain any attachments and is likly not an expense. from: ' . $email->from); return; } // prepare data $expense_vendor = $this->getVendor($email); + $this->processHtmlBodyToDocument($email); $parsed_expense_ids = []; // used to check if an expense was already matched within this job @@ -254,7 +252,7 @@ class InboundMailEngine private function processHtmlBodyToDocument(InboundMail $email) { - if ($email->body !== null) + if (!is_null($email->body)) $email->body_document = TempFile::UploadedFileFromRaw($email->body, "E-Mail.html", "text/html"); } @@ -295,14 +293,6 @@ class InboundMailEngine return false; } - // private function getClient(InboundMail $email) - // { - // $clientContact = ClientContact::where("company_id", $this->company->id)->where("email", $email->from)->first(); - // if (!$clientContact) - // return null; - - // return $clientContact->client(); - // } private function getVendor(InboundMail $email) { $vendorContact = VendorContact::with('vendor')->where("company_id", $this->company->id)->where("email", $email->from)->first(); @@ -310,7 +300,7 @@ class InboundMailEngine return $vendorContact ? $vendorContact->vendor : null; } - private function logBlocked(Company $company, string $data) + private function logBlocked(string $data) { nlog("[InboundMailEngine][company:" . $this->company->company_key . "] " . $data); @@ -321,7 +311,7 @@ class InboundMailEngine SystemLog::EVENT_INBOUND_MAIL_BLOCKED, SystemLog::TYPE_CUSTOM, null, - $company + $this->company ) )->handle(); }