Fixed import command, added tests

config/parse/dispenser.go

@@ -37,7 +37,7 @@ func NewDispenserTokens(filename string, tokens []token) Dispenser {
 
 // Next loads the next token. Returns true if a token
 // was loaded; false otherwise. If false, all tokens
-// have already been consumed.
+// have been consumed.
 func (d *Dispenser) Next() bool {
 	if d.cursor < len(d.tokens)-1 {
 		d.cursor++
@@ -49,7 +49,7 @@ func (d *Dispenser) Next() bool {
 // NextArg loads the next token if it is on the same
 // line. Returns true if a token was loaded; false
 // otherwise. If false, all tokens on the line have
-// been consumed.
+// been consumed. It handles imported tokens correctly.
 func (d *Dispenser) NextArg() bool {
 	if d.cursor < 0 {
 		d.cursor++
@@ -59,7 +59,8 @@ func (d *Dispenser) NextArg() bool {
 		return false
 	}
 	if d.cursor < len(d.tokens)-1 &&
-		(d.tokens[d.cursor].line+d.numLineBreaks(d.cursor) == d.tokens[d.cursor+1].line) {
+		d.tokens[d.cursor].file == d.tokens[d.cursor+1].file &&
+		d.tokens[d.cursor].line+d.numLineBreaks(d.cursor) == d.tokens[d.cursor+1].line {
 		d.cursor++
 		return true
 	}
@@ -69,7 +70,7 @@ func (d *Dispenser) NextArg() bool {
 // NextLine loads the next token only if it is not on the same
 // line as the current token, and returns true if a token was
 // loaded; false otherwise. If false, there is not another token
-// or it is on the same line.
+// or it is on the same line. It handles imported tokens correctly.
 func (d *Dispenser) NextLine() bool {
 	if d.cursor < 0 {
 		d.cursor++
@@ -79,7 +80,8 @@ func (d *Dispenser) NextLine() bool {
 		return false
 	}
 	if d.cursor < len(d.tokens)-1 &&
-		d.tokens[d.cursor].line+d.numLineBreaks(d.cursor) < d.tokens[d.cursor+1].line {
+		(d.tokens[d.cursor].file != d.tokens[d.cursor+1].file ||
+			d.tokens[d.cursor].line+d.numLineBreaks(d.cursor) < d.tokens[d.cursor+1].line) {
 		d.cursor++
 		return true
 	}
@@ -135,6 +137,18 @@ func (d *Dispenser) Line() int {
 	return d.tokens[d.cursor].line
 }
 
+// File gets the filename of the current token. If there is no token loaded,
+// it returns the filename originally given when parsing started.
+func (d *Dispenser) File() string {
+	if d.cursor < 0 || d.cursor >= len(d.tokens) {
+		return d.filename
+	}
+	if tokenFilename := d.tokens[d.cursor].file; tokenFilename != "" {
+		return tokenFilename
+	}
+	return d.filename
+}
+
 // Args is a convenience function that loads the next arguments
 // (tokens on the same line) into an arbitrary number of strings
 // pointed to in targets. If there are fewer tokens available
@@ -185,7 +199,7 @@ func (d *Dispenser) ArgErr() error {
 // SyntaxErr creates a generic syntax error which explains what was
 // found and what was expected.
 func (d *Dispenser) SyntaxErr(expected string) error {
-	msg := fmt.Sprintf("%s:%d - Syntax error: Unexpected token '%s', expecting '%s'", d.filename, d.Line(), d.Val(), expected)
+	msg := fmt.Sprintf("%s:%d - Syntax error: Unexpected token '%s', expecting '%s'", d.File(), d.Line(), d.Val(), expected)
 	return errors.New(msg)
 }
 
@@ -197,7 +211,7 @@ func (d *Dispenser) EofErr() error {
 
 // Err generates a custom parse error with a message of msg.
 func (d *Dispenser) Err(msg string) error {
-	msg = fmt.Sprintf("%s:%d - Parse error: %s", d.filename, d.Line(), msg)
+	msg = fmt.Sprintf("%s:%d - Parse error: %s", d.File(), d.Line(), msg)
 	return errors.New(msg)
 }
 
@@ -215,3 +229,17 @@ func (d *Dispenser) numLineBreaks(tknIdx int) int {
 	}
 	return strings.Count(d.tokens[tknIdx].text, "\n")
 }
+
+// isNewLine determines whether the current token is on a different
+// line (higher line number) than the previous token. It handles imported
+// tokens correctly. If there isn't a previous token, it returns true.
+func (d *Dispenser) isNewLine() bool {
+	if d.cursor < 1 {
+		return true
+	}
+	if d.cursor > len(d.tokens)-1 {
+		return false
+	}
+	return d.tokens[d.cursor-1].file != d.tokens[d.cursor].file ||
+		d.tokens[d.cursor-1].line+d.numLineBreaks(d.cursor-1) < d.tokens[d.cursor].line
+}

config/parse/import_test1.txt

@@ -0,0 +1,2 @@
+dir2 arg1 arg2
+dir3

config/parse/import_test2.txt

@@ -0,0 +1,4 @@
+host1 {
+	dir1
+	dir2 arg1
+}

config/parse/lexer.go

@@ -19,6 +19,7 @@ type (
 
 	// token represents a single parsable unit.
 	token struct {
+		file string
 		line int
 		text string
 	}

config/parse/parsing.go

@@ -3,6 +3,7 @@ package parse
 import (
 	"net"
 	"os"
+	"path/filepath"
 	"strings"
 )
 
@@ -73,7 +74,16 @@ func (p *parser) addresses() error {
 	var expectingAnother bool
 
 	for {
-		tkn, startLine := p.Val(), p.Line()
+		tkn := p.Val()
+
+		// special case: import directive replaces tokens during parse-time
+		if tkn == "import" && p.isNewLine() {
+			err := p.doImport()
+			if err != nil {
+				return err
+			}
+			continue
+		}
 
 		// Open brace definitely indicates end of addresses
 		if tkn == "{" {
@@ -104,13 +114,13 @@ func (p *parser) addresses() error {
 		if expectingAnother && !hasNext {
 			return p.EofErr()
 		}
-		if !expectingAnother && p.Line() > startLine {
-			break
-		}
 		if !hasNext {
 			p.eof = true
 			break // EOF
 		}
+		if !expectingAnother && p.isNewLine() {
+			break
+		}
 	}
 
 	return nil
@@ -156,6 +166,7 @@ func (p *parser) directives() error {
 			if err != nil {
 				return err
 			}
+			p.cursor-- // cursor is advanced when we continue, so roll back one more
 			continue
 		}
 
@@ -188,12 +199,17 @@ func (p *parser) doImport() error {
 	defer file.Close()
 	importedTokens := allTokens(file)
 
+	// Tack the filename onto these tokens so any errors show the imported file's name
+	for i := 0; i < len(importedTokens); i++ {
+		importedTokens[i].file = filepath.Base(importFile)
+	}
+
 	// Splice out the import directive and its argument (2 tokens total)
-	// and insert the imported tokens.
+	// and insert the imported tokens in their place.
 	tokensBefore := p.tokens[:p.cursor-1]
 	tokensAfter := p.tokens[p.cursor+1:]
 	p.tokens = append(tokensBefore, append(importedTokens, tokensAfter...)...)
-	p.cursor -= 2
+	p.cursor-- // cursor was advanced one position to read the filename; rewind it
 
 	return nil
 }
@@ -206,7 +222,6 @@ func (p *parser) doImport() error {
 // by directive setup functions.
 func (p *parser) directive() error {
 	dir := p.Val()
-	line := p.Line()
 	nesting := 0
 
 	if _, ok := ValidDirectives[dir]; !ok {
@@ -219,7 +234,7 @@ func (p *parser) directive() error {
 	for p.Next() {
 		if p.Val() == "{" {
 			nesting++
-		} else if p.Line()+p.numLineBreaks(p.cursor) > line && nesting == 0 {
+		} else if p.isNewLine() && nesting == 0 {
 			p.cursor-- // read too far
 			break
 		} else if p.Val() == "}" && nesting > 0 {
@@ -239,7 +254,7 @@ func (p *parser) directive() error {
 // openCurlyBrace expects the current token to be an
 // opening curly brace. This acts like an assertion
 // because it returns an error if the token is not
-// a opening curly brace. It does not advance the token.
+// a opening curly brace. It does NOT advance the token.
 func (p *parser) openCurlyBrace() error {
 	if p.Val() != "{" {
 		return p.SyntaxErr("{")
@@ -250,7 +265,7 @@ func (p *parser) openCurlyBrace() error {
 // closeCurlyBrace expects the current token to be
 // a closing curly brace. This acts like an assertion
 // because it returns an error if the token is not
-// a closing curly brace. It does not advance the token.
+// a closing curly brace. It does NOT advance the token.
 func (p *parser) closeCurlyBrace() error {
 	if p.Val() != "}" {
 		return p.SyntaxErr("}")

config/parse/parsing_test.go

@@ -57,7 +57,7 @@ func TestStandardAddress(t *testing.T) {
 	}
 }
 
-func TestParseOne(t *testing.T) {
+func TestParseOneAndImport(t *testing.T) {
 	setupParseTests()
 
 	testParseOne := func(input string) (multiServerBlock, error) {
@@ -218,6 +218,23 @@ func TestParseOne(t *testing.T) {
 		}},
 
 		{``, false, []address{}, map[string]int{}},
+
+		{`localhost
+		  dir1 arg1
+		  import import_test1.txt`, false, []address{
+			{"localhost", ""},
+		}, map[string]int{
+			"dir1": 2,
+			"dir2": 3,
+			"dir3": 1,
+		}},
+
+		{`import import_test2.txt`, false, []address{
+			{"host1", ""},
+		}, map[string]int{
+			"dir1": 1,
+			"dir2": 2,
+		}},
 	} {
 		result, err := testParseOne(test.input)
 

dist/CHANGES.txt

@@ -1,5 +1,11 @@
 CHANGES
 
+<master>
+- errors: Error log now includes timestamp with each entry
+- gzip: Default filtering is by extension (fixes bug); removed MIME type filter
+- import: Fixed; works inside and outside server blocks
+- templates: Restricted or missing files result in proper 403 or 404 error
+
 
 0.7.2 (July 1, 2015)
 - Custom builds through caddyserver.com - extend Caddy by writing addons