Cutlery/invoiceninja
1
0
Fork 0
mirror of https://github.com/invoiceninja/invoiceninja.git synced 2025-07-09 03:14:30 -04:00
Code Issues Packages Projects Releases Wiki Activity

Refactor code and do more work on import function

Browse Source
This commit is contained in:
Troels Liebe Bentsen 2014-10-06 20:42:07 +02:00
parent 56a296f103
commit 28b5b5f9c2
5 changed files with 189 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

326
app/commands/ImportTimesheetData.php
View File

@ -6,39 +6,31 @@ use Symfony\Component\Console\Input\InputArgument;
class ImportTimesheetData extends Command { class ImportTimesheetData extends Command {
protected $name = 'ninja:import-timesheet-data'; protected $name = 'ninja:import-timesheet-data';
protected $description = 'Import timesheet data'; protected $description = 'Import timesheet data';
public function fire() public function fire() {
{ $this->info(date('Y-m-d') . ' Running ImportTimesheetData...');
$this->info(date('Y-m-d') . ' Running ImportTimesheetData...');
/* try { /* try {
$dt = new DateTime("now"); $dt = new DateTime("now");
var_dump($dt); var_dump($dt);
echo "1:".$dt."\n"; echo "1:".$dt."\n";
echo $dt->getTimestamp()."\n"; echo $dt->getTimestamp()."\n";
} catch (Exception $ex) { } catch (Exception $ex) {
echo $ex->getMessage(); echo $ex->getMessage();
echo $ex->getTraceAsString(); echo $ex->getTraceAsString();
} }
exit(0); */ exit(0); */
// Create some initial sources we can test with // Create some initial sources we can test with
$user = User::first(); $user = User::first();
if (!$user) { if (!$user) {
$this->error("Error: please create user account by logging in"); $this->error("Error: please create user account by logging in");
return; return;
} }
// TODO: Populate with own test data until test data has been created // TODO: Populate with own test data until test data has been created
// Truncate the tables // Truncate the tables
$this->info("Truncate tables"); $this->info("Truncate tables");
DB::statement('SET FOREIGN_KEY_CHECKS=0;'); DB::statement('SET FOREIGN_KEY_CHECKS=0;');
@ -55,36 +47,37 @@ class ImportTimesheetData extends Command {
$project = Project::createNew($user); $project = Project::createNew($user);
$project->name = $options['description']; $project->name = $options['description'];
$project->save(); $project->save();
$code = ProjectCode::createNew($user); $code = ProjectCode::createNew($user);
$code->name = $name; $code->name = $name;
$project->codes()->save($code); $project->codes()->save($code);
} }
#Project::createNew($user); #Project::createNew($user);
} }
if (!TimesheetEventSource::find(1)) { if (!TimesheetEventSource::find(1)) {
$this->info("Import old event sources"); $this->info("Import old event sources");
$oldevent_sources = json_decode(file_get_contents("/home/tlb/git/itktime/employes.json"), true); $oldevent_sources = json_decode(file_get_contents("/home/tlb/git/itktime/employes.json"), true);
//array_shift($oldevent_sources); //array_shift($oldevent_sources);
//array_pop($oldevent_sources); //array_pop($oldevent_sources);
foreach ($oldevent_sources as $source) { foreach ($oldevent_sources as $source) {
$event_source = TimesheetEventSource::createNew($user); $event_source = TimesheetEventSource::createNew($user);
$event_source->name = $source['name']; $event_source->name = $source['name'];
$event_source->url = $source['url']; $event_source->url = $source['url'];
$event_source->owner = $source['owner']; $event_source->owner = $source['owner'];
$event_source->type = 'ical'; $event_source->type = 'ical';
//$event_source->from_date = new DateTime("2009-01-01");
$event_source->save(); $event_source->save();
} }
} }
// Add all URL's to Curl // Add all URL's to Curl
$this->info("Download ICAL feeds"); $this->info("Download ICAL feeds");
$event_sources = TimesheetEventSource::all(); // TODO: Filter based on ical feeds $event_sources = TimesheetEventSource::all(); // TODO: Filter based on ical feeds
$urls = []; $urls = [];
$event_sources->map(function($item) use(&$urls) { $event_sources->map(function($item) use(&$urls) {
$urls[] = $item->url; $urls[] = $item->url;
@ -92,20 +85,20 @@ class ImportTimesheetData extends Command {
$results = $this->curlGetUrls($urls); $results = $this->curlGetUrls($urls);
// Fetch all codes so we can do a quick lookup // Fetch all codes so we can do a quick lookup
$codes = array(); $codes = array();
ProjectCode::all()->map(function($item) use(&$codes) { ProjectCode::all()->map(function($item) use(&$codes) {
$codes[$item->name] = $item; $codes[$item->name] = $item;
}); });
//FIXME: Make sure we keep track of duplicate UID's so we don't fail when inserting them to the database //FIXME: Make sure we keep track of duplicate UID's so we don't fail when inserting them to the database
$this->info("Start parsing ICAL files"); $this->info("Start parsing ICAL files");
foreach($event_sources as $i => $event_source) { foreach ($event_sources as $i => $event_source) {
if(!is_array($results[$i])) { if (!is_array($results[$i])) {
$this->info("Find events in ".$event_source->name); $this->info("Find events in " . $event_source->name);
file_put_contents("/tmp/".$event_source->name.".ical", $results[$i]); file_put_contents("/tmp/" . $event_source->name . ".ical", $results[$i]);
if(preg_match_all('/BEGIN:VEVENT\r?\n(.+?)\r?\nEND:VEVENT/s', $results[$i], $icalmatches)) { if (preg_match_all('/BEGIN:VEVENT\r?\n(.+?)\r?\nEND:VEVENT/s', $results[$i], $icalmatches)) {
$uids = []; $uids = [];
foreach($icalmatches[1] as $eventstr) { foreach ($icalmatches[1] as $eventstr) {
//print "---\n"; //print "---\n";
//print $eventstr."\n"; //print $eventstr."\n";
//print "---\n"; //print "---\n";
@ -113,137 +106,116 @@ class ImportTimesheetData extends Command {
# Fix lines broken by 76 char limit # Fix lines broken by 76 char limit
$eventstr = preg_replace('/\r?\n\s/s', '', $eventstr); $eventstr = preg_replace('/\r?\n\s/s', '', $eventstr);
//$this->info("Parse data"); //$this->info("Parse data");
if(preg_match_all('/(?:^|\r?\n)([^;:]+)[;:]([^\r\n]+)/s', $eventstr, $eventmatches)) { $data = TimesheetUtils::parseICALEvent($eventstr);
// Build ICAL event array if ($data) {
$data = ['summary' => ''];
foreach($eventmatches[1] as $i => $key) {
# Convert escaped linebreakes to linebreak
$value = preg_replace("/\r?\n\s/", "", $eventmatches[2][$i]);
# Unescape , and ;
$value = preg_replace('/\\\\([,;])/s', '$1', $value);
$data[strtolower($key)] = $value;
}
// Extract code for summary so we only import events we use // Extract code for summary so we only import events we use
//$this->info("Match summary"); list($codename, $tags, $title) = TimesheetUtils::parseEventSummary($data['summary']);
if(preg_match('/^\s*([^\s:\/]+)(?:\/([^:]+))?\s*:\s*(.*?)\s*$/s', $data['summary'], $matches)) { if ($codename != null) {
$codename = strtoupper($matches[1]); $event = TimesheetEvent::createNew($user);
$tags = strtolower($matches[2]); $event->uid = $data['uid'];
$title = $matches[3];
# Add RECURRENCE-ID to the UID to make sure the event is unique
//$this->info("Check code"); if (isset($data['recurrence-id'])) {
if(isset($codes[$codename])) { $event->uid .= $data['recurrence-id'];
//var_dump($data); }
$code = $codes[$codename];
$event = TimesheetEvent::createNew($user); // Check for duplicate events in the feed
$event->summary = $title; if (isset($uids[$event->uid])) {
$event->description = $title; echo "Duplicate event found:";
$event->owner = $event_source->owner; echo "org:\n";
$event->timesheet_event_source_id = $event_source->id; var_dump($uids[$event->uid]);
$event->project_id = $code->project_id; echo "new:\n";
$event->project_code_id = $code->id; var_dump($data);
$event->uid = $data['uid']; continue;
}
# Add RECURRENCE-ID to the UID to make sure the event is unique $uids[$event->uid] = $data;
if(isset($data['recurrence-id'])) {
$event->uid .= $data['recurrence-id']; //TODO: Bail on RRULE as we don't support that
// Convert to DateTime objects
foreach (['dtstart', 'dtend', 'created', 'last-modified'] as $key) {
// Parse and create DataTime object from ICAL format
list($dt, $timezone) = TimesheetUtils::parseICALDate($data[$key]);
// Handle bad dates in created and last-modified
if ($dt == null) {
if ($key == 'created' || $key == 'last-modified') {
$dt = new DateTime('1970-01-01T00:00:00', new DateTimeZone("UTC")); // Default to UNIX epoc
echo "Could not parse date for $key: '" . $data[$key] . "' so default to UNIX Epoc\n"; // TODO write to error table
} else {
echo "Could not parse date for $key: '" . $data[$key] . "'\n"; // TODO write to error table
exit(255); // TODO: Bail on this event
}
} }
// Check for duplicate events in the feed // Assign DateTime object to
if (isset($uids[$event->uid])) { switch ($key) {
echo "Duplicate event found:"; case 'dtstart':
echo "org:\n"; $event->start_date = $dt;
var_dump($uids[$event->uid]); $event->org_start_date_timezone = $timezone;
echo "new:\n"; break;
var_dump($data); case 'dtend':
$event->end_date = $dt;
$event->org_end_date_timezone = $timezone;
break;
case 'created': $event->org_created_at = $dt;
break;
case 'last-modified': $event->org_updated_at = $dt;
break;
}
}
// Check that we are witin the range
if ($event_source->from_date != null) {
$from_date = new DateTime($event_source->from_date, new DateTimeZone('UTC'));
if ($from_date > $event->end_date) {
// Skip this event
echo "Skiped: $codename: $title\n";
continue; continue;
} }
$uids[$event->uid] = $data; }
//TODO: Bail on RRULE as we don't support that // Calculate number of hours
$di = $event->end_date->diff($event->start_date);
//$event->org_data = $eventstr; $event->hours = $di->h + $di->i / 60;
if(isset($data['location'])) {
$event->location = $data['location'];
}
foreach (['dtstart', 'dtend', 'created', 'last-modified'] as $key) { // Copy data to new object
// Parse and create DataTime object from ICAL format $event->org_data = $eventstr;
$dt = null; $event->summary = $title;
$timezone = null; $event->description = $title;
if (preg_match('/^TZID=(.+?):([12]\d\d\d)(\d\d)(\d\d)T(\d\d)(\d\d)(\d\d)$/', $data[$key], $m)) { $event->org_code = $code;
$timezone = $m[1]; $event->owner = $event_source->owner;
$dt = new DateTime("{$m[2]}-{$m[3]}-{$m[4]}T{$m[5]}:{$m[6]}:{$m[7]}", new DateTimeZone($m[1])); $event->timesheet_event_source_id = $event_source->id;
} else if (preg_match('/^VALUE=DATE:([12]\d\d\d)(\d\d)(\d\d)$/', $data[$key], $m)) { if (isset($codes[$codename])) {
$dt = new DateTime("{$m[1]}-{$m[2]}-{$m[3]}T00:00:00", new DateTimeZone("UTC")); $event->project_id = $codes[$codename]->project_id;
} else if (preg_match('/^([12]\d\d\d)(\d\d)(\d\d)T(\d\d)(\d\d)(\d\d)Z$/', $data[$key], $m)) { $event->project_code_id = $codes[$codename]->id;
$dt = new DateTime("{$m[1]}-{$m[2]}-{$m[3]}T{$m[4]}:{$m[5]}:{$m[6]}", new DateTimeZone("UTC")); }
} else if($key == 'created' || $key == 'last-modified') { if (isset($data['location'])) {
$dt = new DateTime('1970-01-01T00:00:00', new DateTimeZone("UTC")); // Default to UNIX epoc $event->location = $data['location'];
echo "Could not parse date for $key: '".$data[$key]."' so default to UNIX Epoc\n"; // TODO write to error table }
} else {
echo "Could not parse date for $key: '".$data[$key]."'\n"; // TODO write to error table try {
exit(255); // TODO: Bail onthis event
}
// Assign DateTime object to
switch ($key) {
case 'dtstart':
$event->start_date = $dt;
$event->org_start_date_timezone = $timezone;
break;
case 'dtend':
$event->end_date = $dt;
$event->org_end_date_timezone = $timezone;
break;
case 'created': $event->org_created_at = $dt; break;
case 'last-modified': $event->org_updated_at = $dt; break;
}
}
// Calculate number of hours
$di = $event->end_date->diff($event->start_date);
$event->hours = $di->h + $di->i / 60;
/*var_dump($event);
exit();*/
// Save event // Save event
$event->save();
//if(!preg_match("/forbered møde med Peter Pietras - nyt sjovt projekt./", $event->summary)) { } catch (Exception $ex) {
echo "'" . $event->summary . "'\n";
try { var_dump($data);
//$event->start_date = new DateTime(""); echo $ex->getMessage();
$event->save(); echo $ex->getTraceAsString();
} catch (Exception $ex) { //exit();
echo "'".$event->summary."'\n";
var_dump($data);
echo $ex->getMessage();
echo $ex->getTraceAsString();
//exit();
}
//}
} else {
//TODO: Add to error table so we can show user
echo "Code not found: $codename\n";
} }
} }
} }
} }
} else { } else {
// Parse error // Parse error
} }
} else { } else {
// Curl Error // Curl Error
} }
} }
$this->info('Done'); $this->info('Done');
} }
private function curlGetUrls($urls = [], $timeout = 30) { private function curlGetUrls($urls = [], $timeout = 30) {
// Create muxer // Create muxer
@ -256,10 +228,10 @@ class ImportTimesheetData extends Command {
// Create new handle and add to muxer // Create new handle and add to muxer
$ch = curl_init($url); $ch = curl_init($url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_ENCODING , "gzip"); curl_setopt($ch, CURLOPT_ENCODING, "gzip");
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); //timeout in seconds curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); //timeout in seconds
curl_multi_add_handle($multi, $ch); curl_multi_add_handle($multi, $ch);
$handles[(int) $ch] = $ch; $handles[(int) $ch] = $ch;
$ch2idx[(int) $ch] = $i; $ch2idx[(int) $ch] = $i;
@ -267,41 +239,37 @@ class ImportTimesheetData extends Command {
// Do initial connect // Do initial connect
$still_running = true; $still_running = true;
while($still_running) { while ($still_running) {
// Do curl stuff // Do curl stuff
while (($mrc = curl_multi_exec($multi, $still_running)) === CURLM_CALL_MULTI_PERFORM); while (($mrc = curl_multi_exec($multi, $still_running)) === CURLM_CALL_MULTI_PERFORM);
if ($mrc !== CURLM_OK) { break; } if ($mrc !== CURLM_OK) {
break;
}
// Try to read from handles that are ready // Try to read from handles that are ready
while ($info = curl_multi_info_read($multi)) { while ($info = curl_multi_info_read($multi)) {
if ($info["result"] == CURLE_OK) { if ($info["result"] == CURLE_OK) {
$results[$ch2idx[(int) $info["handle"]]] = curl_multi_getcontent($info["handle"]); $results[$ch2idx[(int) $info["handle"]]] = curl_multi_getcontent($info["handle"]);
} else { } else {
if(CURLE_UNSUPPORTED_PROTOCOL == $info["result"]) { if (CURLE_UNSUPPORTED_PROTOCOL == $info["result"]) {
$results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Unsupported protocol"]; $results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Unsupported protocol"];
} else if (CURLE_URL_MALFORMAT == $info["result"]) {
} else if(CURLE_URL_MALFORMAT == $info["result"]) {
$results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Malform url"]; $results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Malform url"];
} else if (CURLE_COULDNT_RESOLVE_HOST == $info["result"]) {
} else if(CURLE_COULDNT_RESOLVE_HOST == $info["result"]){
$results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Could not resolve host"]; $results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Could not resolve host"];
} else if (CURLE_OPERATION_TIMEDOUT == $info["result"]) {
} else if(CURLE_OPERATION_TIMEDOUT == $info["result"]){
$results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Timed out waiting for operations to finish"]; $results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Timed out waiting for operations to finish"];
} else { } else {
$results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Unknown curl error code"]; $results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Unknown curl error code"];
} }
} }
} }
// Sleep until // Sleep until
if (($rs = curl_multi_select($multi)) === -1) { if (($rs = curl_multi_select($multi)) === -1) {
usleep(20); // select failed for some reason, so we sleep for 20ms and run some more curl stuff usleep(20); // select failed for some reason, so we sleep for 20ms and run some more curl stuff
} }
} }
} finally { } finally {
foreach ($handles as $chi => $ch) { foreach ($handles as $chi => $ch) {
curl_multi_remove_handle($multi, $ch); curl_multi_remove_handle($multi, $ch);
@ -309,22 +277,18 @@ class ImportTimesheetData extends Command {
curl_multi_close($multi); curl_multi_close($multi);
} }
return $results; return $results;
} }
protected function getArguments()
{
return array(
);
}
protected function getOptions() protected function getArguments() {
{ return array(
return array( );
}
);
}
} protected function getOptions() {
return array(
);
}
}

17
app/database/migrations/2014_09_30_103529_add_timesheets.php → app/database/migrations/2014_10_06_103529_add_timesheets.php
View File

@ -79,6 +79,9 @@ class AddTimesheets extends Migration {
$t->string('url'); $t->string('url');
$t->enum('type', array('ical', 'googlejson')); $t->enum('type', array('ical', 'googlejson'));
$t->dateTime('from_date')->nullable();
$t->dateTime('to_date')->nullable();
$t->foreign('account_id')->references('id')->on('accounts'); $t->foreign('account_id')->references('id')->on('accounts');
$t->foreign('user_id')->references('id')->on('users')->onDelete('cascade'); $t->foreign('user_id')->references('id')->on('users')->onDelete('cascade');
}); });
@ -94,23 +97,31 @@ class AddTimesheets extends Migration {
$t->timestamps(); $t->timestamps();
$t->softDeletes(); $t->softDeletes();
$t->string('summary'); // Basic fields
$t->string('uid'); $t->string('uid');
$t->string('summary');
$t->text('description'); $t->text('description');
$t->string('location'); $t->string('location');
$t->string('owner'); $t->string('owner');
$t->dateTime('start_date'); $t->dateTime('start_date');
$t->dateTime('end_date'); $t->dateTime('end_date');
# Calculated values
$t->decimal('hours'); $t->decimal('hours');
$t->float('discount'); $t->float('discount');
// Original data
$t->string('org_code');
$t->timeStamp('org_created_at'); $t->timeStamp('org_created_at');
$t->timeStamp('org_updated_at'); $t->timeStamp('org_updated_at');
$t->string('org_start_date_timezone')->nullable(); $t->string('org_start_date_timezone')->nullable();
$t->string('org_end_date_timezone')->nullable(); $t->string('org_end_date_timezone')->nullable();
$t->text('org_data'); $t->text('org_data');
// Error and merge handling
$t->string('import_error')->nullable();
$t->text('updated_data')->nullable();
$t->foreign('account_id')->references('id')->on('accounts'); $t->foreign('account_id')->references('id')->on('accounts');
$t->foreign('user_id')->references('id')->on('users')->onDelete('cascade'); $t->foreign('user_id')->references('id')->on('users')->onDelete('cascade');
$t->foreign('timesheet_event_source_id')->references('id')->on('timesheet_event_sources')->onDelete('cascade'); $t->foreign('timesheet_event_source_id')->references('id')->on('timesheet_event_sources')->onDelete('cascade');

3
app/models/TimesheetEvent.php
View File

@ -35,6 +35,9 @@ class TimesheetEvent extends Eloquent
return $this->belongsTo('ProjectCode'); return $this->belongsTo('ProjectCode');
} }
/**
* @return TimesheetEvent
*/
public static function createNew($parent = false) public static function createNew($parent = false)
{ {
$className = get_called_class(); $className = get_called_class();

3
app/tests/ExampleTest.php
View File

@ -10,8 +10,7 @@ class ExampleTest extends TestCase {
public function testBasicExample() public function testBasicExample()
{ {
$crawler = $this->client->request('GET', '/'); $crawler = $this->client->request('GET', '/');
$this->assertTrue($this->client->getResponse()->isRedirect());
$this->assertTrue($this->client->getResponse()->isOk());
} }
} }

26
app/tests/TimesheetUtilsTest.php Normal file
View File

@ -0,0 +1,26 @@
<?php
class TimesheetUtilTest extends \PHPUnit_Framework_TestCase {
public function testParseEventSummary() {
list($code, $codes, $title) = TimesheetUtils::parseEventSummary('Riga :)');
$this->assertSame(null, $code);
list($code, $tags, $title) = TimesheetUtils::parseEventSummary('Test:');
$this->assertSame("TEST", $code);
list($code, $tags, $title) = TimesheetUtils::parseEventSummary('Test: ');
$this->assertSame("TEST", $code);
list($code, $tags, $title) = TimesheetUtils::parseEventSummary('Test::');
$this->assertSame("TEST", $code);
list($code, $tags, $title) = TimesheetUtils::parseEventSummary('TEST: Hello :)');
$this->assertSame("TEST", $code);
list($code, $tags, $title) = TimesheetUtils::parseEventSummary('Test/tags: ');
$this->assertSame('TEST', $code);
$this->assertSame('tags', $tags);
}
}