From 7a13d9308249269ab138762c32fed2778812d60a Mon Sep 17 00:00:00 2001 From: Hillel Coren Date: Sun, 2 Apr 2017 16:54:07 +0300 Subject: [PATCH] Added ImportData job to use queue --- app/Http/Controllers/ImportController.php | 114 ++++++++++-------- .../Controllers/OnlinePaymentController.php | 1 + app/Jobs/ImportData.php | 80 ++++++++++++ app/Ninja/Mailers/UserMailer.php | 4 +- app/Services/ImportService.php | 94 ++++++++++----- resources/lang/en/texts.php | 2 + .../views/accounts/import_export.blade.php | 2 +- resources/views/accounts/import_map.blade.php | 5 + 8 files changed, 219 insertions(+), 83 deletions(-) create mode 100644 app/Jobs/ImportData.php diff --git a/app/Http/Controllers/ImportController.php b/app/Http/Controllers/ImportController.php index 55002d9f9ce3..80577479521a 100644 --- a/app/Http/Controllers/ImportController.php +++ b/app/Http/Controllers/ImportController.php @@ -2,59 +2,88 @@ namespace App\Http\Controllers; +use Illuminate\Http\Request; use App\Services\ImportService; +use App\Jobs\ImportData; use Exception; use Input; use Redirect; use Session; use Utils; use View; +use Auth; class ImportController extends BaseController { public function __construct(ImportService $importService) { - //parent::__construct(); - $this->importService = $importService; } - public function doImport() + public function doImport(Request $request) { $source = Input::get('source'); $files = []; + $timestamp = time(); foreach (ImportService::$entityTypes as $entityType) { - if (Input::file("{$entityType}_file")) { - $files[$entityType] = Input::file("{$entityType}_file")->getRealPath(); - if ($source === IMPORT_CSV) { - Session::forget("{$entityType}-data"); + $fileName = $entityType; + if ($request->hasFile($fileName)) { + $file = $request->file($fileName); + $destinationPath = storage_path() . '/import'; + $extension = $file->getClientOriginalExtension(); + + if (! in_array($extension, ['csv', 'xls', 'json'])) { + continue; } + + $newFileName = sprintf('%s_%s_%s.%s', Auth::user()->account_id, $timestamp, $fileName, $extension); + $file->move($destinationPath, $newFileName); + $files[$entityType] = $newFileName; } } if (! count($files)) { Session::flash('error', trans('texts.select_file')); - return Redirect::to('/settings/' . ACCOUNT_IMPORT_EXPORT); } try { if ($source === IMPORT_CSV) { $data = $this->importService->mapCSV($files); - - return View::make('accounts.import_map', ['data' => $data]); + return View::make('accounts.import_map', [ + 'data' => $data, + 'timestamp' => $timestamp, + ]); } elseif ($source === IMPORT_JSON) { $includeData = filter_var(Input::get('data'), FILTER_VALIDATE_BOOLEAN); $includeSettings = filter_var(Input::get('settings'), FILTER_VALIDATE_BOOLEAN); - $results = $this->importService->importJSON($files[IMPORT_JSON], $includeData, $includeSettings); - - return $this->showResult($results, $includeSettings); + if (config('queue.default') === 'sync') { + $results = $this->importService->importJSON($files[IMPORT_JSON], $includeData, $includeSettings); + $message = $this->importService->presentResults($results, $includeSettings); + } else { + $settings = [ + 'files' => $files, + 'include_data' => $includeData, + 'include_settings' => $includeSettings, + ]; + $this->dispatch(new ImportData(Auth::user(), IMPORT_JSON, $settings)); + $message = 'started...'; + } } else { - $results = $this->importService->importFiles($source, $files); - - return $this->showResult($results); + if (config('queue.default') === 'sync') { + $results = $this->importService->importFiles($source, $files); + $message = $this->importService->presentResults($results); + } else { + $settings = [ + 'files' => $files, + 'source' => $source, + ]; + $this->dispatch(new ImportData(Auth::user(), false, $settings)); + $message = 'started...'; + } } + return redirect('/settings/' . ACCOUNT_IMPORT_EXPORT)->withWarning($message); } catch (Exception $exception) { Utils::logError($exception); Session::flash('error', $exception->getMessage()); @@ -65,13 +94,24 @@ class ImportController extends BaseController public function doImportCSV() { - $map = Input::get('map'); - $headers = Input::get('headers'); - try { - $results = $this->importService->importCSV($map, $headers); + $map = Input::get('map'); + $headers = Input::get('headers'); + $timestamp = Input::get('timestamp'); + if (config('queue.default') === 'sync') { + $results = $this->importService->importCSV($map, $headers, $timestamp); + $message = $this->importService->presentResults($results); + } else { + $settings = [ + 'timestamp' => $timestamp, + 'map' => $map, + 'headers' => $headers, + ]; + $this->dispatch(new ImportData(Auth::user(), IMPORT_CSV, $settings)); + $message = 'started...'; + } - return $this->showResult($results); + return redirect('/settings/' . ACCOUNT_IMPORT_EXPORT)->withWarning($message); } catch (Exception $exception) { Utils::logError($exception); Session::flash('error', $exception->getMessage()); @@ -79,36 +119,4 @@ class ImportController extends BaseController return Redirect::to('/settings/' . ACCOUNT_IMPORT_EXPORT); } } - - private function showResult($results, $includeSettings = false) - { - $message = ''; - $skipped = []; - - if ($includeSettings) { - $message = trans('texts.imported_settings') . '
'; - } - - foreach ($results as $entityType => $entityResults) { - if ($count = count($entityResults[RESULT_SUCCESS])) { - $message .= trans("texts.created_{$entityType}s", ['count' => $count]) . '
'; - } - if (count($entityResults[RESULT_FAILURE])) { - $skipped = array_merge($skipped, $entityResults[RESULT_FAILURE]); - } - } - - if (count($skipped)) { - $message .= '

' . trans('texts.failed_to_import') . '
'; - foreach ($skipped as $skip) { - $message .= json_encode($skip) . '
'; - } - } - - if ($message) { - Session::flash('warning', $message); - } - - return Redirect::to('/settings/' . ACCOUNT_IMPORT_EXPORT); - } } diff --git a/app/Http/Controllers/OnlinePaymentController.php b/app/Http/Controllers/OnlinePaymentController.php index d31078956c7c..399e73cdeb6a 100644 --- a/app/Http/Controllers/OnlinePaymentController.php +++ b/app/Http/Controllers/OnlinePaymentController.php @@ -302,6 +302,7 @@ class OnlinePaymentController extends BaseController } Auth::onceUsingId($account->users[0]->id); + $account->loadLocalizationSettings(); $product = Product::scope(Input::get('product_id'))->first(); if (! $product) { diff --git a/app/Jobs/ImportData.php b/app/Jobs/ImportData.php new file mode 100644 index 000000000000..9ffe334138be --- /dev/null +++ b/app/Jobs/ImportData.php @@ -0,0 +1,80 @@ +user = $user; + $this->type = $type; + $this->settings = $settings; + } + + /** + * Execute the job. + * + * @param ContactMailer $mailer + */ + public function handle(ImportService $importService, UserMailer $userMailer) + { + $includeSettings = false; + + Auth::onceUsingId($this->user->id); + $this->user->account->loadLocalizationSettings(); + + if ($this->type === IMPORT_JSON) { + $includeData = $this->settings['include_data']; + $includeSettings = $this->settings['include_settings']; + $files = $this->settings['files']; + $results = $importService->importJSON($files[IMPORT_JSON], $includeData, $includeSettings); + } elseif ($this->type === IMPORT_CSV) { + $map = $this->settings['map']; + $headers = $this->settings['headers']; + $timestamp = $this->settings['timestamp']; + $results = $importService->importCSV($map, $headers, $timestamp); + } else { + $source = $this->settings['source']; + $files = $this->settings['files']; + $results = $importService->importFiles($source, $files); + } + + $subject = trans('texts.import_complete'); + $message = $importService->presentResults($results, $includeSettings); + $userMailer->sendMessage($this->user, $subject, $message); + } +} diff --git a/app/Ninja/Mailers/UserMailer.php b/app/Ninja/Mailers/UserMailer.php index 8e44295cd726..302e7d6b8ed2 100644 --- a/app/Ninja/Mailers/UserMailer.php +++ b/app/Ninja/Mailers/UserMailer.php @@ -114,7 +114,7 @@ class UserMailer extends Mailer /** * @param Invitation $invitation */ - public function sendMessage($user, $subject, $message, $invoice) + public function sendMessage($user, $subject, $message, $invoice = false) { if (! $user->email) { return; @@ -125,7 +125,7 @@ class UserMailer extends Mailer 'userName' => $user->getDisplayName(), 'primaryMessage' => $subject, 'secondaryMessage' => $message, - 'invoiceLink' => $invoice->present()->multiAccountLink, + 'invoiceLink' => $invoice ? $invoice->present()->multiAccountLink : false, ]; $this->sendTo($user->email, CONTACT_EMAIL, CONTACT_NAME, $subject, $view, $data); diff --git a/app/Services/ImportService.php b/app/Services/ImportService.php index d1e1ee1f7977..62964a9d2c5a 100644 --- a/app/Services/ImportService.php +++ b/app/Services/ImportService.php @@ -149,7 +149,7 @@ class ImportService { $this->initMaps(); - $file = file_get_contents($file); + $file = file_get_contents(storage_path() . '/import/' . $file); $json = json_decode($file, true); $json = $this->removeIdFields($json); $transformer = new BaseTransformer($this->maps); @@ -284,6 +284,7 @@ class ImportService // Convert the data $row_list = []; + $file = storage_path() . '/import/' . $file; Excel::load($file, function ($reader) use ($source, $entityType, &$row_list, &$results) { $this->checkData($entityType, count($reader->all())); @@ -539,29 +540,13 @@ class ImportService */ public function mapFile($entityType, $filename, $columns, $map) { - require_once app_path().'/Includes/parsecsv.lib.php'; - $csv = new parseCSV(); - $csv->heading = false; - $csv->auto($filename); - + $data = $this->getCsvData($filename); $headers = false; $hasHeaders = false; $mapped = []; - if (count($csv->data) > 0) { - $headers = $csv->data[0]; - - // Remove Invoice Ninja headers - if (count($headers) && count($csv->data) > 4) { - $firstCell = $headers[0]; - if (strstr($firstCell, APP_NAME)) { - array_shift($csv->data); // Invoice Ninja... - array_shift($csv->data); // - array_shift($csv->data); // Enitty Type Header - } - $headers = $csv->data[0]; - } - + if (count($data) > 0) { + $headers = $data[0]; foreach ($headers as $title) { if (strpos(strtolower($title), 'name') > 0) { $hasHeaders = true; @@ -583,11 +568,11 @@ class ImportService } } - Session::put("{$entityType}-data", $csv->data); + //Session::put("{$entityType}-data", $csv->data); $data = [ 'entityType' => $entityType, - 'data' => $csv->data, + 'data' => $data, 'headers' => $headers, 'hasHeaders' => $hasHeaders, 'columns' => $columns, @@ -597,6 +582,31 @@ class ImportService return $data; } + private function getCsvData($filename) + { + require_once app_path().'/Includes/parsecsv.lib.php'; + $csv = new parseCSV(); + $csv->heading = false; + $csv->auto(storage_path() . '/import/' . $filename); + $data = $csv->data; + + if (count($data) > 0) { + $headers = $data[0]; + + // Remove Invoice Ninja headers + if (count($headers) && count($data) > 4) { + $firstCell = $headers[0]; + if (strstr($firstCell, APP_NAME)) { + array_shift($data); // Invoice Ninja... + array_shift($data); // + array_shift($data); // Enitty Type Header + } + } + } + + return $data; + } + /** * @param $column * @param $pattern @@ -642,12 +652,12 @@ class ImportService * * @return array */ - public function importCSV(array $maps, $headers) + public function importCSV(array $maps, $headers, $timestamp) { $results = []; foreach ($maps as $entityType => $map) { - $results[$entityType] = $this->executeCSV($entityType, $map, $headers[$entityType]); + $results[$entityType] = $this->executeCSV($entityType, $map, $headers[$entityType], $timestamp); } return $results; @@ -660,7 +670,7 @@ class ImportService * * @return array */ - private function executeCSV($entityType, $map, $hasHeaders) + private function executeCSV($entityType, $map, $hasHeaders, $timestamp) { $results = [ RESULT_SUCCESS => [], @@ -668,7 +678,9 @@ class ImportService ]; $source = IMPORT_CSV; - $data = Session::get("{$entityType}-data"); + //$data = Session::get("{$entityType}-data"); + $filename = sprintf('%s_%s_%s.csv', Auth::user()->account_id, $timestamp, $entityType); + $data = $this->getCsvData($filename); $this->checkData($entityType, count($data)); $this->initMaps(); @@ -707,7 +719,7 @@ class ImportService } } - Session::forget("{$entityType}-data"); + //Session::forget("{$entityType}-data"); return $results; } @@ -894,4 +906,32 @@ class ImportService return $isEmpty; } + + public function presentResults($results, $includeSettings = false) + { + $message = ''; + $skipped = []; + + if ($includeSettings) { + $message = trans('texts.imported_settings') . '
'; + } + + foreach ($results as $entityType => $entityResults) { + if ($count = count($entityResults[RESULT_SUCCESS])) { + $message .= trans("texts.created_{$entityType}s", ['count' => $count]) . '
'; + } + if (count($entityResults[RESULT_FAILURE])) { + $skipped = array_merge($skipped, $entityResults[RESULT_FAILURE]); + } + } + + if (count($skipped)) { + $message .= '

' . trans('texts.failed_to_import') . '
'; + foreach ($skipped as $skip) { + $message .= json_encode($skip) . '
'; + } + } + + return $message; + } } diff --git a/resources/lang/en/texts.php b/resources/lang/en/texts.php index 98c62f921332..b978b170381c 100644 --- a/resources/lang/en/texts.php +++ b/resources/lang/en/texts.php @@ -2460,6 +2460,8 @@ $LANG = array( 'reply_to_email' => 'Reply-To Email', 'reply_to_email_help' => 'Specify the reply-to address for client emails.', 'bcc_email_help' => 'Privately include this address with client emails.', + 'import_complete' => 'Your import has successfully completed.', + ); diff --git a/resources/views/accounts/import_export.blade.php b/resources/views/accounts/import_export.blade.php index 90e5698a2fe9..512b1d4156ef 100644 --- a/resources/views/accounts/import_export.blade.php +++ b/resources/views/accounts/import_export.blade.php @@ -33,7 +33,7 @@
@foreach (\App\Services\ImportService::$entityTypes as $entityType) - {!! Former::file("{$entityType}_file") + {!! Former::file($entityType) ->addGroupClass("import-file {$entityType}-file") !!} @endforeach diff --git a/resources/views/accounts/import_map.blade.php b/resources/views/accounts/import_map.blade.php index 5c3c76ac7c20..3bf6d2d65895 100644 --- a/resources/views/accounts/import_map.blade.php +++ b/resources/views/accounts/import_map.blade.php @@ -6,6 +6,11 @@ @include('accounts.nav', ['selected' => ACCOUNT_IMPORT_EXPORT]) {!! Former::open('/import_csv')->addClass('warn-on-exit') !!} + {!! Former::populateField('timestamp', $timestamp) !!} + +

+ {!! Former::text('timestamp') !!} +
@foreach (App\Services\ImportService::$entityTypes as $entityType) @if (isset($data[$entityType]))