Working on CSV import

app/Http/Controllers/ImportController.php

@@ -22,6 +22,7 @@ class ImportController extends BaseController
     {
         $source = Input::get('source');
         $files = [];
+        $skipped = [];
 
         foreach (ImportService::$entityTypes as $entityType) {
             if (Input::file("{$entityType}_file")) {
@@ -34,8 +35,16 @@ class ImportController extends BaseController
                 $data = $this->importService->mapCSV($files);
                 return View::make('accounts.import_map', ['data' => $data]);
             } else {
-                $result = $this->importService->import($source, $files);
+                $skipped = $this->importService->import($source, $files);
-                Session::flash('message', trans('texts.imported_file') . ' - ' . $result);
+                if (count($skipped)) {
+                    $message = trans('texts.failed_to_import');
+                    foreach ($skipped as $skip) {
+                        $message .= '<br/>' . json_encode($skip);
+                    }
+                    Session::flash('warning', $message);
+                } else {
+                    Session::flash('message', trans('texts.imported_file'));
+                }
             }
         } catch (Exception $exception) {
             Session::flash('error', $exception->getMessage());
@@ -48,15 +57,23 @@ class ImportController extends BaseController
     {
         $map = Input::get('map');
         $headers = Input::get('headers');
+        $skipped = [];
 
-        //try {
+        try {
-            $count = $this->importService->importCSV($map, $headers);
+            $skipped = $this->importService->importCSV($map, $headers);
-            $message = Utils::pluralize('created_client', $count);
 
-            Session::flash('message', $message);
+            if (count($skipped)) {
-        //} catch (Exception $exception) {
+                $message = trans('texts.failed_to_import');
-        //    Session::flash('error', $exception->getMessage());
+                foreach ($skipped as $skip) {
-        //}
+                    $message .= '<br/>' . json_encode($skip);
+                }
+                Session::flash('warning', $message);
+            } else {
+                Session::flash('message', trans('texts.imported_file'));
+            }
+        } catch (Exception $exception) {
+            Session::flash('error', $exception->getMessage());
+        }
 
         return Redirect::to('/settings/' . ACCOUNT_IMPORT_EXPORT);
     }

app/Models/Client.php

@@ -71,19 +71,18 @@ class Client extends EntityModel
     public static function getImportMap()
     {
         return [
-            'first' => Contact::$fieldFirstName,
+            'first' => 'first_name',
-            'last' => Contact::$fieldLastName,
+            'last' => 'last_name',
-            'email' => Contact::$fieldEmail,
+            'email' => 'email',
-            'mobile' => Contact::$fieldPhone,
+            'mobile|phone' => 'phone',
-            'phone' => Client::$fieldPhone,
+            'name|organization' => 'name',
-            'name|organization' => Client::$fieldName,
+            'street2|address2' => 'address2',
-            'street|address|address1' => Client::$fieldAddress1,
+            'street|address|address1' => 'address1',
-            'street2|address2' => Client::$fieldAddress2,
+            'city' => 'city',
-            'city' => Client::$fieldCity,
+            'state|province' => 'state',
-            'state|province' => Client::$fieldState,
+            'zip|postal|code' => 'postal_code',
-            'zip|postal|code' => Client::$fieldPostalCode,
+            'country' => 'country',
-            'country' => Client::$fieldCountry,
+            'note' => 'notes',
-            'note' => Client::$fieldNotes,
         ];
     }
 

app/Models/Invoice.php

@@ -65,11 +65,11 @@ class Invoice extends EntityModel implements BalanceAffecting
     public static function getImportMap()
     {
         return [
-            'number' => Invoice::$fieldInvoiceNumber,
+            'number^po' => 'invoice_number',
-            'amount' => Invoice::$fieldAmount,
+            'amount' => 'amount',
             'organization' => 'name',
-            'paid' => 'paid',
+            'paid^date' => 'paid',
-            'invoice_date' => Invoice::$fieldInvoiceDate,
+            'invoice_date|create_date' => 'invoice_date',
             'terms' => 'terms',
             'notes' => 'notes',
         ];

app/Services/ImportService.php

@@ -59,46 +59,24 @@ class ImportService
         }
     }
 
-    private function checkClientCount($count)
-    {
-        $totalClients = $count + Client::scope()->withTrashed()->count();
-        if ($totalClients > Auth::user()->getMaxNumClients()) {
-            throw new Exception(trans('texts.limit_clients', ['count' => Auth::user()->getMaxNumClients()]));
-        }
-    }
-
     private function execute($source, $entityType, $file)
     {
-        Excel::load($file, function ($reader) use ($source, $entityType) {
+        $skipped = [];
 
+        Excel::load($file, function ($reader) use ($source, $entityType, $skipped) {
             $this->checkData($entityType, count($reader->all()));
             $maps = $this->createMaps();
 
             $reader->each(function ($row) use ($source, $entityType, $maps) {
-                $this->saveData($source, $entityType, $row, $maps);
+                $result = $this->saveData($source, $entityType, $row, $maps);
+
+                if ( ! $result) {
+                    $skipped[] = $row;
+                }
             });
         });
-    }
 
-    private function executeCSV($entityType, $map, $hasHeaders)
+        return $skipped;
-    {
-        $source = IMPORT_CSV;
-
-        $data = Session::get("{$entityType}-data");
-        $this->checkData($entityType, count($data));
-        $maps = $this->createMaps();
-
-        foreach ($data as $row) {
-            if ($hasHeaders) {
-                $hasHeaders = false;
-                continue;
-            }
-
-            $row = $this->convertToObject($entityType, $row, $map);
-            $this->saveData($source, $entityType, $row, $maps);
-        }
-
-        Session::forget("{$entityType}-data");
     }
 
     private function saveData($source, $entityType, $row, $maps)
@@ -112,6 +90,7 @@ class ImportService
 
         $data = $this->fractal->createData($resource)->toArray();
 
+        // if the invoice number is blank we'll assign it
         if ($entityType == ENTITY_INVOICE && !$data['invoice_number']) {
             $account = Auth::user()->account;
             $invoice = Invoice::createNew();
@@ -123,7 +102,7 @@ class ImportService
         }
 
         $entity = $this->{"{$entityType}Repo"}->save($data);
-        
+
         // if the invoice is paid we'll also create a payment record
         if ($entityType === ENTITY_INVOICE && isset($row->paid) && $row->paid) {
             $this->createPayment($source, $row, $maps, $data['client_id'], $entity->public_id);
@@ -137,6 +116,14 @@ class ImportService
         }
     }
 
+    private function checkClientCount($count)
+    {
+        $totalClients = $count + Client::scope()->withTrashed()->count();
+        if ($totalClients > Auth::user()->getMaxNumClients()) {
+            throw new Exception(trans('texts.limit_clients', ['count' => Auth::user()->getMaxNumClients()]));
+        }
+    }
+
     public static function getTransformerClassName($source, $entityType)
     {
         return 'App\\Ninja\\Import\\'.$source.'\\'.ucwords($entityType).'Transformer';
@@ -162,15 +149,14 @@ class ImportService
         }
     }
 
-    // looking for a better solution...
-    // http://stackoverflow.com/questions/33781567/how-can-i-re-use-the-validation-code-in-my-laravel-formrequest-classes
     private function validate($data, $entityType)
     {
         if ($entityType === ENTITY_CLIENT) {
             $rules = [
                 'contacts' => 'valid_contacts',
             ];
-        } if ($entityType === ENTITY_INVOICE) {
+        }
+        if ($entityType === ENTITY_INVOICE) {
             $rules = [
                 'client.contacts' => 'valid_contacts',
                 'invoice_items' => 'valid_invoice_items',
@@ -279,13 +265,9 @@ class ImportService
 
                 if ($hasHeaders) {
                     foreach ($map as $search => $column) {
-                        foreach (explode("|", $search) as $string) {
+                        if ($this->checkForMatch($title, $search)) {
-                            if (strpos($title, 'sec') === 0) {
+                            $mapped[$i] = $column;
-                                continue;
+                            break;
-                            } elseif (strpos($title, $string) !== false) {
-                                $mapped[$i] = $column;
-                                break(2);
-                            }
                         }
                     }
                 }
@@ -304,11 +286,77 @@ class ImportService
         return $data;
     }
 
+    private function checkForMatch($column, $pattern)
+    {
+        if (strpos($column, 'sec') === 0) {
+            return false;
+        }
+
+        if (strpos($pattern, '^')) {
+            list($include, $exclude) = explode('^', $pattern);
+            $includes = explode('|', $include);
+            $excludes = explode('|', $exclude);
+        } else {
+            $includes = explode('|', $pattern);
+            $excludes = [];
+        }
+
+        foreach ($includes as $string) {
+            if (strpos($column, $string) !== false) {
+                $excluded = false;
+                foreach ($excludes as $exclude) {
+                    if (strpos($column, $exclude) !== false) {
+                        $excluded = true;
+                        break;
+                    }
+                }
+                if (!$excluded) {
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
+
     public function importCSV($maps, $headers)
     {
+        $skipped = [];
+
         foreach ($maps as $entityType => $map) {
-            $this->executeCSV($entityType, $map, $headers[$entityType]);
+            $result = $this->executeCSV($entityType, $map, $headers[$entityType]);
+            $skipped = array_merge($skipped, $result);
         }
+
+        return $skipped;
+    }
+
+    private function executeCSV($entityType, $map, $hasHeaders)
+    {
+        $skipped = [];
+        $source = IMPORT_CSV;
+
+        $data = Session::get("{$entityType}-data");
+        $this->checkData($entityType, count($data));
+        $maps = $this->createMaps();
+
+        foreach ($data as $row) {
+            if ($hasHeaders) {
+                $hasHeaders = false;
+                continue;
+            }
+
+            $row = $this->convertToObject($entityType, $row, $map);
+            $result = $this->saveData($source, $entityType, $row, $maps);
+
+            if ( ! $result) {
+                $skipped[] = $row;
+            }
+        }
+
+        Session::forget("{$entityType}-data");
+
+        return $skipped;
     }
 
     private function convertToObject($entityType, $data, $map)
@@ -330,6 +378,10 @@ class ImportService
                 continue;
             }
 
+            if (isset($obj->$field) && $obj->$field) {
+                continue;
+            }
+
             $obj->$field = $data[$index];
         }
 

resources/lang/en/texts.php

@@ -948,5 +948,6 @@ return array(
     'notes' => 'Notes',
     'invoice_will_create' => 'client will be created',
     'invoices_will_create' => 'invoices will be created',
+    'failed_to_import' => 'The following records failed to import',
 
 );