reverseproxy: Add support for specifying IDs in Caddyfile

Adds one more debug log

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
-        go: [ '1.15', '1.16' ]
+        go: [ '1.16', '1.17' ]
 
         # Set some variables per OS, usable via ${{ matrix.VAR }}
         # CADDY_BIN_PATH: the path to the compiled Caddy binary, for artifact publishing
@@ -156,6 +156,7 @@ jobs:
     steps:
       - name: checkout
         uses: actions/checkout@v2
+      
       - uses: goreleaser/goreleaser-action@v2
         with:
           version: latest

.github/workflows/cross-build.yml

@@ -16,7 +16,7 @@ jobs:
       fail-fast: false
       matrix:
         goos: ['android', 'linux', 'solaris', 'illumos', 'dragonfly', 'freebsd', 'openbsd', 'plan9', 'windows', 'darwin', 'netbsd']
-        go: [ '1.15', '1.16' ]
+        go: [ '1.17' ]
     runs-on: ubuntu-latest
     continue-on-error: true
     steps:

.github/workflows/release.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        go: [ '1.16' ]
+        go: [ '1.17' ]
     runs-on: ${{ matrix.os }}
 
     steps:

README.md

@@ -75,7 +75,7 @@ For other install options, see https://caddyserver.com/docs/install.
 
 Requirements:
 
-- [Go 1.15 or newer](https://golang.org/dl/)
+- [Go 1.16 or newer](https://golang.org/dl/)
 
 ### For development
  

admin.go

@@ -109,6 +109,12 @@ type ConfigSettings struct {
 	//
 	// EXPERIMENTAL: Subject to change.
 	LoadRaw json.RawMessage `json:"load,omitempty" caddy:"namespace=caddy.config_loaders inline_key=module"`
+
+	// The interval to pull config. With a non-zero value, will pull config
+	// from config loader (eg. a http loader) with given interval.
+	//
+	// EXPERIMENTAL: Subject to change.
+	LoadInterval Duration `json:"load_interval,omitempty"`
 }
 
 // IdentityConfig configures management of this server's identity. An identity
@@ -329,6 +335,7 @@ func replaceLocalAdminServer(cfg *Config) error {
 		return err
 	}
 
+	serverMu.Lock()
 	localAdminServer = &http.Server{
 		Addr:              addr.String(), // for logging purposes only
 		Handler:           handler,
@@ -337,10 +344,14 @@ func replaceLocalAdminServer(cfg *Config) error {
 		IdleTimeout:       60 * time.Second,
 		MaxHeaderBytes:    1024 * 64,
 	}
+	serverMu.Unlock()
 
 	adminLogger := Log().Named("admin")
 	go func() {
-		if err := localAdminServer.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
+		serverMu.Lock()
+		server := localAdminServer
+		serverMu.Unlock()
+		if err := server.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
 			adminLogger.Error("admin server shutdown for unknown reason", zap.Error(err))
 		}
 	}()
@@ -468,6 +479,7 @@ func replaceRemoteAdminServer(ctx Context, cfg *Config) error {
 		return err
 	}
 
+	serverMu.Lock()
 	// create secure HTTP server
 	remoteAdminServer = &http.Server{
 		Addr:              addr.String(), // for logging purposes only
@@ -479,6 +491,7 @@ func replaceRemoteAdminServer(ctx Context, cfg *Config) error {
 		MaxHeaderBytes:    1024 * 64,
 		ErrorLog:          serverLogger,
 	}
+	serverMu.Unlock()
 
 	// start listener
 	ln, err := Listen(addr.Network, addr.JoinHostPort(0))
@@ -488,7 +501,10 @@ func replaceRemoteAdminServer(ctx Context, cfg *Config) error {
 	ln = tls.NewListener(ln, tlsConfig)
 
 	go func() {
-		if err := remoteAdminServer.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
+		serverMu.Lock()
+		server := remoteAdminServer
+		serverMu.Unlock()
+		if err := server.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
 			remoteLogger.Error("admin remote server shutdown for unknown reason", zap.Error(err))
 		}
 	}()
@@ -1223,6 +1239,7 @@ var bufPool = sync.Pool{
 
 // keep a reference to admin endpoint singletons while they're active
 var (
+	serverMu                            sync.Mutex
 	localAdminServer, remoteAdminServer *http.Server
 	identityCertCache                   *certmagic.Cache
 )

admin_test.go

@@ -17,9 +17,28 @@ package caddy
 import (
 	"encoding/json"
 	"reflect"
+	"sync"
 	"testing"
 )
 
+var testCfg = []byte(`{
+			"apps": {
+				"http": {
+					"servers": {
+						"myserver": {
+							"listen": ["tcp/localhost:8080-8084"],
+							"read_timeout": "30s"
+						},
+						"yourserver": {
+							"listen": ["127.0.0.1:5000"],
+							"read_header_timeout": "15s"
+						}
+					}
+				}
+			}
+		}
+		`)
+
 func TestUnsyncedConfigAccess(t *testing.T) {
 	// each test is performed in sequence, so
 	// each change builds on the previous ones;
@@ -108,25 +127,24 @@ func TestUnsyncedConfigAccess(t *testing.T) {
 	}
 }
 
+// TestLoadConcurrent exercises Load under concurrent conditions
+// and is most useful under test with `-race` enabled.
+func TestLoadConcurrent(t *testing.T) {
+	var wg sync.WaitGroup
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		go func() {
+			_ = Load(testCfg, true)
+			wg.Done()
+		}()
+	}
+
+	wg.Wait()
+}
+
 func BenchmarkLoad(b *testing.B) {
 	for i := 0; i < b.N; i++ {
-		cfg := []byte(`{
+		Load(testCfg, true)
-			"apps": {
-				"http": {
-					"servers": {
-						"myserver": {
-							"listen": ["tcp/localhost:8080-8084"],
-							"read_timeout": "30s"
-						},
-						"yourserver": {
-							"listen": ["127.0.0.1:5000"],
-							"read_header_timeout": "15s"
-						}
-					}
-				}
-			}
-		}
-		`)
-		Load(cfg, true)
 	}
 }

caddy.go

@@ -268,8 +268,9 @@ func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
 		newCfg != nil &&
 		newCfg.Admin != nil &&
 		newCfg.Admin.Config != nil &&
-		newCfg.Admin.Config.LoadRaw != nil {
+		newCfg.Admin.Config.LoadRaw != nil &&
-		return fmt.Errorf("recursive config loading detected: pulled configs cannot pull other configs")
+		newCfg.Admin.Config.LoadInterval <= 0 {
+		return fmt.Errorf("recursive config loading detected: pulled configs cannot pull other configs without positive load_interval")
 	}
 
 	// run the new config and start all its apps
@@ -480,23 +481,42 @@ func finishSettingUp(ctx Context, cfg *Config) error {
 		if err != nil {
 			return fmt.Errorf("loading config loader module: %s", err)
 		}
-		loadedConfig, err := val.(ConfigLoader).LoadConfig(ctx)
+		runLoadedConfig := func(config []byte) {
-		if err != nil {
+			Log().Info("applying dynamically-loaded config", zap.String("loader_module", val.(Module).CaddyModule().ID.Name()), zap.Int("pull_interval", int(cfg.Admin.Config.LoadInterval)))
-			return fmt.Errorf("loading dynamic config from %T: %v", val, err)
-		}
-
-		// do this in a goroutine so current config can finish being loaded; otherwise deadlock
-		go func() {
-			Log().Info("applying dynamically-loaded config", zap.String("loader_module", val.(Module).CaddyModule().ID.Name()))
 			currentCfgMu.Lock()
-			err := unsyncedDecodeAndRun(loadedConfig, false)
+			err := unsyncedDecodeAndRun(config, false)
 			currentCfgMu.Unlock()
 			if err == nil {
 				Log().Info("dynamically-loaded config applied successfully")
 			} else {
 				Log().Error("running dynamically-loaded config failed", zap.Error(err))
 			}
-		}()
+		}
+		if cfg.Admin.Config.LoadInterval > 0 {
+			go func() {
+				select {
+				// if LoadInterval is positive, will wait for the interval and then run with new config
+				case <-time.After(time.Duration(cfg.Admin.Config.LoadInterval)):
+					loadedConfig, err := val.(ConfigLoader).LoadConfig(ctx)
+					if err != nil {
+						Log().Error("loading dynamic config failed", zap.Error(err))
+						return
+					}
+					runLoadedConfig(loadedConfig)
+				case <-ctx.Done():
+					return
+				}
+			}()
+		} else {
+			// if no LoadInterval is provided, will load config synchronously
+			loadedConfig, err := val.(ConfigLoader).LoadConfig(ctx)
+			if err != nil {
+				return fmt.Errorf("loading dynamic config from %T: %v", val, err)
+			}
+			// do this in a goroutine so current config can finish being loaded; otherwise deadlock
+			go runLoadedConfig(loadedConfig)
+		}
+
 	}
 
 	return nil

caddyconfig/caddyfile/dispenser.go

@@ -345,13 +345,13 @@ func (d *Dispenser) EOFErr() error {
 
 // Err generates a custom parse-time error with a message of msg.
 func (d *Dispenser) Err(msg string) error {
-	msg = fmt.Sprintf("%s:%d - Error during parsing: %s", d.File(), d.Line(), msg)
+	return d.Errf(msg)
-	return errors.New(msg)
 }
 
 // Errf is like Err, but for formatted error messages
 func (d *Dispenser) Errf(format string, args ...interface{}) error {
-	return d.Err(fmt.Sprintf(format, args...))
+	err := fmt.Errorf(format, args...)
+	return fmt.Errorf("%s:%d - Error during parsing: %w", d.File(), d.Line(), err)
 }
 
 // Delete deletes the current token and returns the updated slice

caddyconfig/caddyfile/dispenser_test.go

@@ -15,6 +15,7 @@
 package caddyfile
 
 import (
+	"errors"
 	"reflect"
 	"strings"
 	"testing"
@@ -303,4 +304,10 @@ func TestDispenser_ArgErr_Err(t *testing.T) {
 	if !strings.Contains(err.Error(), "foobar") {
 		t.Errorf("Expected error message with custom message in it ('foobar'); got '%v'", err)
 	}
+
+	var ErrBarIsFull = errors.New("bar is full")
+	bookingError := d.Errf("unable to reserve: %w", ErrBarIsFull)
+	if !errors.Is(bookingError, ErrBarIsFull) {
+		t.Errorf("Errf(): should be able to unwrap the error chain")
+	}
 }

caddyconfig/caddyfile/parse.go

@@ -211,6 +211,12 @@ func (p *parser) addresses() error {
 			if expectingAnother {
 				return p.Errf("Expected another address but had '%s' - check for extra comma", tkn)
 			}
+			// Mark this server block as being defined with braces.
+			// This is used to provide a better error message when
+			// the user may have tried to define two server blocks
+			// without having used braces, which are required in
+			// that case.
+			p.block.HasBraces = true
 			break
 		}
 
@@ -229,6 +235,13 @@ func (p *parser) addresses() error {
 				expectingAnother = false // but we may still see another one on this line
 			}
 
+			// If there's a comma here, it's probably because they didn't use a space
+			// between their two domains, e.g. "foo.com,bar.com", which would not be
+			// parsed as two separate site addresses.
+			if strings.Contains(tkn, ",") {
+				return p.Errf("Site addresses cannot contain a comma ',': '%s' - put a space after the comma to separate site addresses", tkn)
+			}
+
 			p.block.Keys = append(p.block.Keys, tkn)
 		}
 
@@ -564,8 +577,9 @@ func (p *parser) snippetTokens() ([]Token, error) {
 // head of the server block with tokens, which are
 // grouped by segments.
 type ServerBlock struct {
-	Keys     []string
+	HasBraces bool
-	Segments []Segment
+	Keys      []string
+	Segments  []Segment
 }
 
 // DispenseDirective returns a dispenser that contains

caddyconfig/httpcaddyfile/directives.go

@@ -44,9 +44,9 @@ var directiveOrder = []string{
 	"request_body",
 
 	"redir",
-	"rewrite",
 
 	// URI manipulation
+	"rewrite",
 	"uri",
 	"try_files",
 
@@ -54,23 +54,23 @@ var directiveOrder = []string{
 	"basicauth",
 	"request_header",
 	"encode",
+	"push",
 	"templates",
 
 	// special routing & dispatching directives
 	"handle",
 	"handle_path",
 	"route",
-	"push",
 
 	// handlers that typically respond to requests
+	"abort",
+	"error",
 	"respond",
 	"metrics",
 	"reverse_proxy",
 	"php_fastcgi",
 	"file_server",
 	"acme_server",
-	"abort",
-	"error",
 }
 
 // directiveIsOrdered returns true if dir is
@@ -329,7 +329,7 @@ func parseSegmentAsConfig(h Helper) ([]ConfigValue, error) {
 			dir := seg.Directive()
 			dirFunc, ok := registeredDirectives[dir]
 			if !ok {
-				return nil, h.Errf("unrecognized directive: %s", dir)
+				return nil, h.Errf("unrecognized directive: %s - are you sure your Caddyfile structure (nesting and braces) is correct?", dir)
 			}
 
 			subHelper := h

caddyconfig/httpcaddyfile/httptype.go

@@ -113,6 +113,7 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		"{tls_client_serial}", "{http.request.tls.client.serial}",
 		"{tls_client_subject}", "{http.request.tls.client.subject}",
 		"{tls_client_certificate_pem}", "{http.request.tls.client.certificate_pem}",
+		"{upstream_hostport}", "{http.reverse_proxy.upstream.hostport}",
 	)
 
 	// these are placeholders that allow a user-defined final
@@ -169,7 +170,11 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 			dirFunc, ok := registeredDirectives[dir]
 			if !ok {
 				tkn := segment[0]
-				return nil, warnings, fmt.Errorf("%s:%d: unrecognized directive: %s", tkn.File, tkn.Line, dir)
+				message := "%s:%d: unrecognized directive: %s"
+				if !sb.block.HasBraces {
+					message += "\nDid you mean to define a second site? If so, you must use curly braces around each site to separate their configurations."
+				}
+				return nil, warnings, fmt.Errorf(message, tkn.File, tkn.Line, dir)
 			}
 
 			h := Helper{
@@ -522,6 +527,16 @@ func (st *ServerType) serversFromPairings(
 			}
 		}
 
+		// if needed, the ServerLogConfig is initialized beforehand so
+		// that all server blocks can populate it with data, even when not
+		// coming with a log directive
+		for _, sblock := range p.serverBlocks {
+			if len(sblock.pile["custom_log"]) != 0 {
+				srv.Logs = new(caddyhttp.ServerLogConfig)
+				break
+			}
+		}
+
 		// create a subroute for each site in the server block
 		for _, sblock := range p.serverBlocks {
 			matcherSetsEnc, err := st.compileEncodedMatcherSets(sblock)
@@ -636,9 +651,6 @@ func (st *ServerType) serversFromPairings(
 			sblockLogHosts := sblock.hostsFromKeys(true)
 			for _, cval := range sblock.pile["custom_log"] {
 				ncl := cval.Value.(namedCustomLog)
-				if srv.Logs == nil {
-					srv.Logs = new(caddyhttp.ServerLogConfig)
-				}
 				if sblock.hasHostCatchAllKey() {
 					// all requests for hosts not able to be listed should use
 					// this log because it's a catch-all-hosts server block

caddyconfig/httpcaddyfile/tlsapp.go

@@ -211,7 +211,7 @@ func (st ServerType) buildTLSApp(
 					// it that we would need to check here) since the hostname is known at handshake;
 					// and it is unexpected to switch to internal issuer when the user wants to get
 					// regular certificates on-demand for a class of certs like *.*.tld.
-					if !certmagic.SubjectIsIP(s) && !certmagic.SubjectIsInternal(s) && (strings.Count(s, "*.") < 2 || ap.OnDemand) {
+					if subjectQualifiesForPublicCert(ap, s) {
 						external = append(external, s)
 					} else {
 						internal = append(internal, s)
@@ -324,8 +324,12 @@ func (st ServerType) buildTLSApp(
 		globalPreferredChains := options["preferred_chains"]
 		hasGlobalACMEDefaults := globalEmail != nil || globalACMECA != nil || globalACMECARoot != nil || globalACMEDNS != nil || globalACMEEAB != nil || globalPreferredChains != nil
 		if hasGlobalACMEDefaults {
-			for _, ap := range tlsApp.Automation.Policies {
+			// for _, ap := range tlsApp.Automation.Policies {
-				if len(ap.Issuers) == 0 {
+			for i := 0; i < len(tlsApp.Automation.Policies); i++ {
+				ap := tlsApp.Automation.Policies[i]
+				if len(ap.Issuers) == 0 && automationPolicyHasAllPublicNames(ap) {
+					// for public names, create default issuers which will later be filled in with configured global defaults
+					// (internal names will implicitly use the internal issuer at auto-https time)
 					ap.Issuers = caddytls.DefaultIssuers()
 
 					// if a specific endpoint is configured, can't use multiple default issuers
@@ -494,16 +498,23 @@ func consolidateAutomationPolicies(aps []*caddytls.AutomationPolicy) []*caddytls
 	})
 
 	emptyAPCount := 0
+	origLenAPs := len(aps)
 	// compute the number of empty policies (disregarding subjects) - see #4128
 	emptyAP := new(caddytls.AutomationPolicy)
 	for i := 0; i < len(aps); i++ {
 		emptyAP.Subjects = aps[i].Subjects
 		if reflect.DeepEqual(aps[i], emptyAP) {
 			emptyAPCount++
+			if !automationPolicyHasAllPublicNames(aps[i]) {
+				// if this automation policy has internal names, we might as well remove it
+				// so auto-https can implicitly use the internal issuer
+				aps = append(aps[:i], aps[i+1:]...)
+				i--
+			}
 		}
 	}
 	// If all policies are empty, we can return nil, as there is no need to set any policy
-	if emptyAPCount == len(aps) {
+	if emptyAPCount == origLenAPs {
 		return nil
 	}
 
@@ -601,3 +612,21 @@ func automationPolicyShadows(i int, aps []*caddytls.AutomationPolicy) int {
 	}
 	return -1
 }
+
+// subjectQualifiesForPublicCert is like certmagic.SubjectQualifiesForPublicCert() except
+// that this allows domains with multiple wildcard levels like '*.*.example.com' to qualify
+// if the automation policy has OnDemand enabled (i.e. this function is more lenient).
+func subjectQualifiesForPublicCert(ap *caddytls.AutomationPolicy, subj string) bool {
+	return !certmagic.SubjectIsIP(subj) &&
+		!certmagic.SubjectIsInternal(subj) &&
+		(strings.Count(subj, "*.") < 2 || ap.OnDemand)
+}
+
+func automationPolicyHasAllPublicNames(ap *caddytls.AutomationPolicy) bool {
+	for _, subj := range ap.Subjects {
+		if !subjectQualifiesForPublicCert(ap, subj) {
+			return false
+		}
+	}
+	return true
+}

caddytest/integration/autohttps_test.go

@@ -103,3 +103,23 @@ func TestAutoHTTPRedirectsInsertedBeforeUserDefinedCatchAll(t *testing.T) {
 	tester.AssertGetResponse("http://foo.localhost:9080/", 200, "Foo")
 	tester.AssertGetResponse("http://baz.localhost:9080/", 200, "Baz")
 }
+
+func TestAutoHTTPRedirectsInsertedBeforeUserDefinedCatchAllWithNoExplicitHTTPSite(t *testing.T) {
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+	{
+		http_port     9080
+		https_port    9443
+		local_certs
+	}
+	http://:9080 {
+		respond "Foo"
+	}
+	bar.localhost {
+		respond "Bar"
+	}
+  `, "caddyfile")
+	tester.AssertRedirect("http://bar.localhost:9080/", "https://bar.localhost/", http.StatusPermanentRedirect)
+	tester.AssertGetResponse("http://foo.localhost:9080/", 200, "Foo")
+	tester.AssertGetResponse("http://baz.localhost:9080/", 200, "Foo")
+}

caddytest/integration/caddyfile_adapt/error_example.txt

@@ -0,0 +1,138 @@
+example.com {
+	root * /srv
+
+	# Trigger errors for certain paths
+	error /private* "Unauthorized" 403
+	error /hidden* "Not found" 404
+
+	# Handle the error by serving an HTML page 
+	handle_errors {
+		rewrite * /{http.error.status_code}.html
+		file_server
+	}
+
+	file_server
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "vars",
+													"root": "/srv"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Unauthorized",
+													"handler": "error",
+													"status_code": 403
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/private*"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Not found",
+													"handler": "error",
+													"status_code": 404
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/hidden*"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"handler": "file_server",
+													"hide": [
+														"./Caddyfile"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"errors": {
+						"routes": [
+							{
+								"match": [
+									{
+										"host": [
+											"example.com"
+										]
+									}
+								],
+								"handle": [
+									{
+										"handler": "subroute",
+										"routes": [
+											{
+												"group": "group0",
+												"handle": [
+													{
+														"handler": "rewrite",
+														"uri": "/{http.error.status_code}.html"
+													}
+												]
+											},
+											{
+												"handle": [
+													{
+														"handler": "file_server",
+														"hide": [
+															"./Caddyfile"
+														]
+													}
+												]
+											}
+										]
+									}
+								],
+								"terminal": true
+							}
+						]
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/file_server_disable_canonical_uris.txt

@@ -0,0 +1,32 @@
+:80
+
+file_server {
+	disable_canonical_uris
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"canonical_uris": false,
+									"handler": "file_server",
+									"hide": [
+										"./Caddyfile"
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options_preferred_chains.txt

@@ -2,7 +2,7 @@
 	preferred_chains smallest
 }
 
-localhost
+example.com
 ----------
 {
 	"apps": {
@@ -17,7 +17,7 @@ localhost
 							"match": [
 								{
 									"host": [
-										"localhost"
+										"example.com"
 									]
 								}
 							],
@@ -32,7 +32,7 @@ localhost
 				"policies": [
 					{
 						"subjects": [
-							"localhost"
+							"example.com"
 						],
 						"issuers": [
 							{

caddytest/integration/caddyfile_adapt/log_skip_hosts.txt

@@ -0,0 +1,75 @@
+one.example.com {
+	log
+}
+
+two.example.com {
+}
+
+three.example.com {
+}
+
+example.com {
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"three.example.com"
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"one.example.com"
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"two.example.com"
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"logs": {
+						"skip_hosts": [
+							"three.example.com",
+							"two.example.com",
+							"example.com"
+						]
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/php_fastcgi_handle_response.txt

@@ -0,0 +1,145 @@
+:8881 {
+	php_fastcgi app:9000 {
+		env FOO bar
+
+		@error status 4xx
+		handle_response @error {
+			root * /errors
+			rewrite * /{http.reverse_proxy.status_code}.html
+			file_server
+		}
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8881"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"file": {
+										"try_files": [
+											"{http.request.uri.path}/index.php"
+										]
+									},
+									"not": [
+										{
+											"path": [
+												"*/"
+											]
+										}
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "static_response",
+									"headers": {
+										"Location": [
+											"{http.request.uri.path}/"
+										]
+									},
+									"status_code": 308
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"file": {
+										"try_files": [
+											"{http.request.uri.path}",
+											"{http.request.uri.path}/index.php",
+											"index.php"
+										],
+										"split_path": [
+											".php"
+										]
+									}
+								}
+							],
+							"handle": [
+								{
+									"handler": "rewrite",
+									"uri": "{http.matchers.file.relative}"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"path": [
+										"*.php"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handle_response": [
+										{
+											"match": {
+												"status_code": [
+													4
+												]
+											},
+											"routes": [
+												{
+													"handle": [
+														{
+															"handler": "vars",
+															"root": "/errors"
+														}
+													]
+												},
+												{
+													"group": "group0",
+													"handle": [
+														{
+															"handler": "rewrite",
+															"uri": "/{http.reverse_proxy.status_code}.html"
+														}
+													]
+												},
+												{
+													"handle": [
+														{
+															"handler": "file_server",
+															"hide": [
+																"./Caddyfile"
+															]
+														}
+													]
+												}
+											]
+										}
+									],
+									"handler": "reverse_proxy",
+									"transport": {
+										"env": {
+											"FOO": "bar"
+										},
+										"protocol": "fastcgi",
+										"split_path": [
+											".php"
+										]
+									},
+									"upstreams": [
+										{
+											"dial": "app:9000"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/reverse_proxy_ids.txt

@@ -0,0 +1,46 @@
+:8884
+
+reverse_proxy one|http://localhost two|http://localhost {
+	to three|srv+http://localhost four|srv+http://localhost
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "reverse_proxy",
+									"upstreams": [
+										{
+											"dial": "localhost:80",
+											"id": "one"
+										},
+										{
+											"dial": "localhost:80",
+											"id": "two"
+										},
+										{
+											"id": "three",
+											"lookup_srv": "localhost"
+										},
+										{
+											"id": "four",
+											"lookup_srv": "localhost"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_automation_policies_global_email_localhost.txt

@@ -0,0 +1,66 @@
+{
+	email foo@bar
+}
+
+localhost {
+}
+
+example.com {
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"example.com"
+						],
+						"issuers": [
+							{
+								"email": "foo@bar",
+								"module": "acme"
+							},
+							{
+								"email": "foo@bar",
+								"module": "zerossl"
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/reverseproxy_test.go

@@ -371,7 +371,7 @@ func TestReverseProxyHealthCheck(t *testing.T) {
 		reverse_proxy {
 			to localhost:2020
 	
-			health_path /health
+			health_uri /health
 			health_port 2021
 			health_interval 2s
 			health_timeout 5s
@@ -426,7 +426,7 @@ func TestReverseProxyHealthCheckUnixSocket(t *testing.T) {
 		reverse_proxy {
 			to unix/%s
 	
-			health_path /health
+			health_uri /health
 			health_port 2021
 			health_interval 2s
 			health_timeout 5s
@@ -436,3 +436,57 @@ func TestReverseProxyHealthCheckUnixSocket(t *testing.T) {
 
 	tester.AssertGetResponse("http://localhost:9080/", 200, "Hello, World!")
 }
+
+func TestReverseProxyHealthCheckUnixSocketWithoutPort(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.SkipNow()
+	}
+	tester := caddytest.NewTester(t)
+	f, err := ioutil.TempFile("", "*.sock")
+	if err != nil {
+		t.Errorf("failed to create TempFile: %s", err)
+		return
+	}
+	// a hack to get a file name within a valid path to use as socket
+	socketName := f.Name()
+	os.Remove(f.Name())
+
+	server := http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			if strings.HasPrefix(req.URL.Path, "/health") {
+				w.Write([]byte("ok"))
+				return
+			}
+			w.Write([]byte("Hello, World!"))
+		}),
+	}
+
+	unixListener, err := net.Listen("unix", socketName)
+	if err != nil {
+		t.Errorf("failed to listen on the socket: %s", err)
+		return
+	}
+	go server.Serve(unixListener)
+	t.Cleanup(func() {
+		server.Close()
+	})
+	runtime.Gosched() // Allow other goroutines to run
+
+	tester.InitServer(fmt.Sprintf(`
+	{
+		http_port     9080
+		https_port    9443
+	}
+	http://localhost:9080 {
+		reverse_proxy {
+			to unix/%s
+	
+			health_uri /health
+			health_interval 2s
+			health_timeout 5s
+		}
+	}
+	`, socketName), "caddyfile")
+
+	tester.AssertGetResponse("http://localhost:9080/", 200, "Hello, World!")
+}

cmd/commandfuncs.go

@@ -19,16 +19,15 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"os/exec"
-	"reflect"
 	"runtime"
 	"runtime/debug"
 	"sort"
@@ -121,7 +120,7 @@ func cmdStart(fl Flags) (int, error) {
 		for {
 			conn, err := ln.Accept()
 			if err != nil {
-				if !strings.Contains(err.Error(), "use of closed network connection") {
+				if !errors.Is(err, net.ErrClosed) {
 					log.Println(err)
 				}
 				break
@@ -332,7 +331,7 @@ func cmdReload(fl Flags) (int, error) {
 }
 
 func cmdVersion(_ Flags) (int, error) {
-	fmt.Println(caddyVersion())
+	fmt.Println(CaddyVersion())
 	return caddy.ExitCodeSuccess, nil
 }
 
@@ -570,151 +569,6 @@ func cmdFmt(fl Flags) (int, error) {
 	return caddy.ExitCodeSuccess, nil
 }
 
-func cmdUpgrade(_ Flags) (int, error) {
-	l := caddy.Log()
-
-	thisExecPath, err := os.Executable()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("determining current executable path: %v", err)
-	}
-	thisExecStat, err := os.Stat(thisExecPath)
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("retrieving current executable permission bits: %v", err)
-	}
-	l.Info("this executable will be replaced", zap.String("path", thisExecPath))
-
-	// get the list of nonstandard plugins
-	_, nonstandard, _, err := getModules()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err)
-	}
-	pluginPkgs := make(map[string]struct{})
-	for _, mod := range nonstandard {
-		if mod.goModule.Replace != nil {
-			return caddy.ExitCodeFailedStartup, fmt.Errorf("cannot auto-upgrade when Go module has been replaced: %s => %s",
-				mod.goModule.Path, mod.goModule.Replace.Path)
-		}
-		l.Info("found non-standard module",
-			zap.String("id", mod.caddyModuleID),
-			zap.String("package", mod.goModule.Path))
-		pluginPkgs[mod.goModule.Path] = struct{}{}
-	}
-
-	// build the request URL to download this custom build
-	qs := url.Values{
-		"os":   {runtime.GOOS},
-		"arch": {runtime.GOARCH},
-	}
-	for pkg := range pluginPkgs {
-		qs.Add("p", pkg)
-	}
-	urlStr := fmt.Sprintf("https://caddyserver.com/api/download?%s", qs.Encode())
-
-	// initiate the build
-	l.Info("requesting build",
-		zap.String("os", qs.Get("os")),
-		zap.String("arch", qs.Get("arch")),
-		zap.Strings("packages", qs["p"]))
-	resp, err := http.Get(urlStr)
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("secure request failed: %v", err)
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		var details struct {
-			StatusCode int `json:"status_code"`
-			Error      struct {
-				Message string `json:"message"`
-				ID      string `json:"id"`
-			} `json:"error"`
-		}
-		err2 := json.NewDecoder(resp.Body).Decode(&details)
-		if err2 != nil {
-			return caddy.ExitCodeFailedStartup, fmt.Errorf("download and error decoding failed: HTTP %d: %v", resp.StatusCode, err2)
-		}
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download failed: HTTP %d: %s (id=%s)", resp.StatusCode, details.Error.Message, details.Error.ID)
-	}
-
-	// back up the current binary, in case something goes wrong we can replace it
-	backupExecPath := thisExecPath + ".tmp"
-	l.Info("build acquired; backing up current executable",
-		zap.String("current_path", thisExecPath),
-		zap.String("backup_path", backupExecPath))
-	err = os.Rename(thisExecPath, backupExecPath)
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("backing up current binary: %v", err)
-	}
-	defer func() {
-		if err != nil {
-			err2 := os.Rename(backupExecPath, thisExecPath)
-			if err2 != nil {
-				l.Error("restoring original executable failed; will need to be restored manually",
-					zap.String("backup_path", backupExecPath),
-					zap.String("original_path", thisExecPath),
-					zap.Error(err2))
-			}
-		}
-	}()
-
-	// download the file; do this in a closure to close reliably before we execute it
-	writeFile := func() error {
-		destFile, err := os.OpenFile(thisExecPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, thisExecStat.Mode())
-		if err != nil {
-			return fmt.Errorf("unable to open destination file: %v", err)
-		}
-		defer destFile.Close()
-
-		l.Info("downloading binary", zap.String("source", urlStr), zap.String("destination", thisExecPath))
-
-		_, err = io.Copy(destFile, resp.Body)
-		if err != nil {
-			return fmt.Errorf("unable to download file: %v", err)
-		}
-
-		err = destFile.Sync()
-		if err != nil {
-			return fmt.Errorf("syncing downloaded file to device: %v", err)
-		}
-
-		return nil
-	}
-	err = writeFile()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, err
-	}
-
-	l.Info("download successful; displaying new binary details", zap.String("location", thisExecPath))
-
-	// use the new binary to print out version and module info
-	fmt.Print("\nModule versions:\n\n")
-	cmd := exec.Command(thisExecPath, "list-modules", "--versions")
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	err = cmd.Run()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err)
-	}
-	fmt.Println("\nVersion:")
-	cmd = exec.Command(thisExecPath, "version")
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	err = cmd.Run()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err)
-	}
-	fmt.Println()
-
-	// clean up the backup file
-	err = os.Remove(backupExecPath)
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to clean up backup binary: %v", err)
-	}
-
-	l.Info("upgrade successful; please restart any running Caddy instances", zap.String("executable", thisExecPath))
-
-	return caddy.ExitCodeSuccess, nil
-}
-
 func cmdHelp(fl Flags) (int, error) {
 	const fullDocs = `Full documentation is available at:
 https://caddyserver.com/docs/command-line`
@@ -779,56 +633,6 @@ commands:
 	return caddy.ExitCodeSuccess, nil
 }
 
-func getModules() (standard, nonstandard, unknown []moduleInfo, err error) {
-	bi, ok := debug.ReadBuildInfo()
-	if !ok {
-		err = fmt.Errorf("no build info")
-		return
-	}
-
-	for _, modID := range caddy.Modules() {
-		modInfo, err := caddy.GetModule(modID)
-		if err != nil {
-			// that's weird, shouldn't happen
-			unknown = append(unknown, moduleInfo{caddyModuleID: modID, err: err})
-			continue
-		}
-
-		// to get the Caddy plugin's version info, we need to know
-		// the package that the Caddy module's value comes from; we
-		// can use reflection but we need a non-pointer value (I'm
-		// not sure why), and since New() should return a pointer
-		// value, we need to dereference it first
-		iface := interface{}(modInfo.New())
-		if rv := reflect.ValueOf(iface); rv.Kind() == reflect.Ptr {
-			iface = reflect.New(reflect.TypeOf(iface).Elem()).Elem().Interface()
-		}
-		modPkgPath := reflect.TypeOf(iface).PkgPath()
-
-		// now we find the Go module that the Caddy module's package
-		// belongs to; we assume the Caddy module package path will
-		// be prefixed by its Go module path, and we will choose the
-		// longest matching prefix in case there are nested modules
-		var matched *debug.Module
-		for _, dep := range bi.Deps {
-			if strings.HasPrefix(modPkgPath, dep.Path) {
-				if matched == nil || len(dep.Path) > len(matched.Path) {
-					matched = dep
-				}
-			}
-		}
-
-		caddyModGoMod := moduleInfo{caddyModuleID: modID, goModule: matched}
-
-		if strings.HasPrefix(modPkgPath, caddy.ImportPath) {
-			standard = append(standard, caddyModGoMod)
-		} else {
-			nonstandard = append(nonstandard, caddyModGoMod)
-		}
-	}
-	return
-}
-
 // apiRequest makes an API request to the endpoint adminAddr with the
 // given HTTP method and request URI. If body is non-nil, it will be
 // assumed to be Content-Type application/json.

cmd/commands.go

@@ -61,6 +61,12 @@ type Command struct {
 // any error that occurred.
 type CommandFunc func(Flags) (int, error)
 
+// Commands returns a list of commands initialised by
+// RegisterCommand
+func Commands() map[string]Command {
+	return commands
+}
+
 var commands = make(map[string]Command)
 
 func init() {
@@ -291,6 +297,30 @@ Downloads an updated Caddy binary with the same modules/plugins at the
 latest versions. EXPERIMENTAL: May be changed or removed.`,
 	})
 
+	RegisterCommand(Command{
+		Name:  "add-package",
+		Func:  cmdAddPackage,
+		Usage: "<packages...>",
+		Short: "Adds Caddy packages (EXPERIMENTAL)",
+		Long: `
+Downloads an updated Caddy binary with the specified packages (module/plugin)
+added. Retains existing packages. Returns an error if the any of packages are 
+already included. EXPERIMENTAL: May be changed or removed.
+`,
+	})
+
+	RegisterCommand(Command{
+		Name:  "remove-package",
+		Func:  cmdRemovePackage,
+		Usage: "<packages...>",
+		Short: "Removes Caddy packages (EXPERIMENTAL)",
+		Long: `
+Downloads an updated Caddy binaries without the specified packages (module/plugin). 
+Returns an error if any of the packages are not included. 
+EXPERIMENTAL: May be changed or removed.
+`,
+	})
+
 }
 
 // RegisterCommand registers the command cmd.

cmd/main.go

@@ -361,6 +361,11 @@ func loadEnvFromFile(envFile string) error {
 		}
 	}
 
+	// Update the storage paths to ensure they have the proper
+	// value after loading a specified env file.
+	caddy.ConfigAutosavePath = filepath.Join(caddy.AppConfigDir(), "autosave.json")
+	caddy.DefaultStorage = &certmagic.FileStorage{Path: caddy.AppDataDir()}
+
 	return nil
 }
 
@@ -415,7 +420,7 @@ func printEnvironment() {
 	fmt.Printf("caddy.AppDataDir=%s\n", caddy.AppDataDir())
 	fmt.Printf("caddy.AppConfigDir=%s\n", caddy.AppConfigDir())
 	fmt.Printf("caddy.ConfigAutosavePath=%s\n", caddy.ConfigAutosavePath)
-	fmt.Printf("caddy.Version=%s\n", caddyVersion())
+	fmt.Printf("caddy.Version=%s\n", CaddyVersion())
 	fmt.Printf("runtime.GOOS=%s\n", runtime.GOOS)
 	fmt.Printf("runtime.GOARCH=%s\n", runtime.GOARCH)
 	fmt.Printf("runtime.Compiler=%s\n", runtime.Compiler)
@@ -432,8 +437,8 @@ func printEnvironment() {
 	}
 }
 
-// caddyVersion returns a detailed version string, if available.
+// CaddyVersion returns a detailed version string, if available.
-func caddyVersion() string {
+func CaddyVersion() string {
 	goModule := caddy.GoModule()
 	ver := goModule.Version
 	if goModule.Sum != "" {

cmd/packagesfuncs.go

@@ -0,0 +1,306 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddycmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"reflect"
+	"runtime"
+	"runtime/debug"
+	"strings"
+
+	"github.com/caddyserver/caddy/v2"
+	"go.uber.org/zap"
+)
+
+func cmdUpgrade(_ Flags) (int, error) {
+	_, nonstandard, _, err := getModules()
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err)
+	}
+	pluginPkgs, err := getPluginPackages(nonstandard)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	return upgradeBuild(pluginPkgs)
+}
+
+func cmdAddPackage(fl Flags) (int, error) {
+	if len(fl.Args()) == 0 {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("at least one package name must be specified")
+	}
+	_, nonstandard, _, err := getModules()
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err)
+	}
+	pluginPkgs, err := getPluginPackages(nonstandard)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	for _, arg := range fl.Args() {
+		if _, ok := pluginPkgs[arg]; ok {
+			return caddy.ExitCodeFailedStartup, fmt.Errorf("package is already added")
+		}
+		pluginPkgs[arg] = struct{}{}
+	}
+
+	return upgradeBuild(pluginPkgs)
+}
+
+func cmdRemovePackage(fl Flags) (int, error) {
+	if len(fl.Args()) == 0 {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("at least one package name must be specified")
+	}
+	_, nonstandard, _, err := getModules()
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err)
+	}
+	pluginPkgs, err := getPluginPackages(nonstandard)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	for _, arg := range fl.Args() {
+		if _, ok := pluginPkgs[arg]; !ok {
+			// package does not exist
+			return caddy.ExitCodeFailedStartup, fmt.Errorf("package is not added")
+		}
+		delete(pluginPkgs, arg)
+	}
+
+	return upgradeBuild(pluginPkgs)
+}
+
+func upgradeBuild(pluginPkgs map[string]struct{}) (int, error) {
+	l := caddy.Log()
+
+	thisExecPath, err := os.Executable()
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("determining current executable path: %v", err)
+	}
+	thisExecStat, err := os.Stat(thisExecPath)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("retrieving current executable permission bits: %v", err)
+	}
+	l.Info("this executable will be replaced", zap.String("path", thisExecPath))
+
+	// build the request URL to download this custom build
+	qs := url.Values{
+		"os":   {runtime.GOOS},
+		"arch": {runtime.GOARCH},
+	}
+	for pkg := range pluginPkgs {
+		qs.Add("p", pkg)
+	}
+
+	// initiate the build
+	resp, err := downloadBuild(qs)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("download failed: %v", err)
+	}
+	defer resp.Body.Close()
+
+	// back up the current binary, in case something goes wrong we can replace it
+	backupExecPath := thisExecPath + ".tmp"
+	l.Info("build acquired; backing up current executable",
+		zap.String("current_path", thisExecPath),
+		zap.String("backup_path", backupExecPath))
+	err = os.Rename(thisExecPath, backupExecPath)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("backing up current binary: %v", err)
+	}
+	defer func() {
+		if err != nil {
+			err2 := os.Rename(backupExecPath, thisExecPath)
+			if err2 != nil {
+				l.Error("restoring original executable failed; will need to be restored manually",
+					zap.String("backup_path", backupExecPath),
+					zap.String("original_path", thisExecPath),
+					zap.Error(err2))
+			}
+		}
+	}()
+
+	// download the file; do this in a closure to close reliably before we execute it
+	err = writeCaddyBinary(thisExecPath, &resp.Body, thisExecStat)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	l.Info("download successful; displaying new binary details", zap.String("location", thisExecPath))
+
+	// use the new binary to print out version and module info
+	fmt.Print("\nModule versions:\n\n")
+	if err = listModules(thisExecPath); err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err)
+	}
+	fmt.Println("\nVersion:")
+	if err = showVersion(thisExecPath); err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err)
+	}
+	fmt.Println()
+
+	// clean up the backup file
+	if err = os.Remove(backupExecPath); err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to clean up backup binary: %v", err)
+	}
+	l.Info("upgrade successful; please restart any running Caddy instances", zap.String("executable", thisExecPath))
+
+	return caddy.ExitCodeSuccess, nil
+}
+
+func getModules() (standard, nonstandard, unknown []moduleInfo, err error) {
+	bi, ok := debug.ReadBuildInfo()
+	if !ok {
+		err = fmt.Errorf("no build info")
+		return
+	}
+
+	for _, modID := range caddy.Modules() {
+		modInfo, err := caddy.GetModule(modID)
+		if err != nil {
+			// that's weird, shouldn't happen
+			unknown = append(unknown, moduleInfo{caddyModuleID: modID, err: err})
+			continue
+		}
+
+		// to get the Caddy plugin's version info, we need to know
+		// the package that the Caddy module's value comes from; we
+		// can use reflection but we need a non-pointer value (I'm
+		// not sure why), and since New() should return a pointer
+		// value, we need to dereference it first
+		iface := interface{}(modInfo.New())
+		if rv := reflect.ValueOf(iface); rv.Kind() == reflect.Ptr {
+			iface = reflect.New(reflect.TypeOf(iface).Elem()).Elem().Interface()
+		}
+		modPkgPath := reflect.TypeOf(iface).PkgPath()
+
+		// now we find the Go module that the Caddy module's package
+		// belongs to; we assume the Caddy module package path will
+		// be prefixed by its Go module path, and we will choose the
+		// longest matching prefix in case there are nested modules
+		var matched *debug.Module
+		for _, dep := range bi.Deps {
+			if strings.HasPrefix(modPkgPath, dep.Path) {
+				if matched == nil || len(dep.Path) > len(matched.Path) {
+					matched = dep
+				}
+			}
+		}
+
+		caddyModGoMod := moduleInfo{caddyModuleID: modID, goModule: matched}
+
+		if strings.HasPrefix(modPkgPath, caddy.ImportPath) {
+			standard = append(standard, caddyModGoMod)
+		} else {
+			nonstandard = append(nonstandard, caddyModGoMod)
+		}
+	}
+	return
+}
+
+func listModules(path string) error {
+	cmd := exec.Command(path, "list-modules", "--versions")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("download succeeded, but unable to execute: %v", err)
+	}
+	return nil
+}
+
+func showVersion(path string) error {
+	cmd := exec.Command(path, "version")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("download succeeded, but unable to execute: %v", err)
+	}
+	return nil
+}
+
+func downloadBuild(qs url.Values) (*http.Response, error) {
+	l := caddy.Log()
+	l.Info("requesting build",
+		zap.String("os", qs.Get("os")),
+		zap.String("arch", qs.Get("arch")),
+		zap.Strings("packages", qs["p"]))
+	resp, err := http.Get(fmt.Sprintf("%s?%s", downloadPath, qs.Encode()))
+	if err != nil {
+		return nil, fmt.Errorf("secure request failed: %v", err)
+	}
+	if resp.StatusCode >= 400 {
+		var details struct {
+			StatusCode int `json:"status_code"`
+			Error      struct {
+				Message string `json:"message"`
+				ID      string `json:"id"`
+			} `json:"error"`
+		}
+		err2 := json.NewDecoder(resp.Body).Decode(&details)
+		if err2 != nil {
+			return nil, fmt.Errorf("download and error decoding failed: HTTP %d: %v", resp.StatusCode, err2)
+		}
+		return nil, fmt.Errorf("download failed: HTTP %d: %s (id=%s)", resp.StatusCode, details.Error.Message, details.Error.ID)
+	}
+	return resp, nil
+}
+
+func getPluginPackages(modules []moduleInfo) (map[string]struct{}, error) {
+	pluginPkgs := make(map[string]struct{})
+	for _, mod := range modules {
+		if mod.goModule.Replace != nil {
+			return nil, fmt.Errorf("cannot auto-upgrade when Go module has been replaced: %s => %s",
+				mod.goModule.Path, mod.goModule.Replace.Path)
+		}
+		pluginPkgs[mod.goModule.Path] = struct{}{}
+	}
+	return pluginPkgs, nil
+}
+
+func writeCaddyBinary(path string, body *io.ReadCloser, fileInfo os.FileInfo) error {
+	l := caddy.Log()
+	destFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileInfo.Mode())
+	if err != nil {
+		return fmt.Errorf("unable to open destination file: %v", err)
+	}
+	defer destFile.Close()
+
+	l.Info("downloading binary", zap.String("destination", path))
+
+	_, err = io.Copy(destFile, *body)
+	if err != nil {
+		return fmt.Errorf("unable to download file: %v", err)
+	}
+
+	err = destFile.Sync()
+	if err != nil {
+		return fmt.Errorf("syncing downloaded file to device: %v", err)
+	}
+
+	return nil
+}
+
+const downloadPath = "https://caddyserver.com/api/download"

go.mod

@@ -1,35 +1,35 @@
 module github.com/caddyserver/caddy/v2
 
-go 1.15
+go 1.16
 
 require (
 	github.com/Masterminds/sprig/v3 v3.2.2
-	github.com/alecthomas/chroma v0.9.1
+	github.com/alecthomas/chroma v0.9.2
 	github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b
-	github.com/caddyserver/certmagic v0.14.0
+	github.com/caddyserver/certmagic v0.14.5
 	github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/google/cel-go v0.7.3
-	github.com/google/uuid v1.2.0
+	github.com/google/uuid v1.3.0
-	github.com/klauspost/compress v1.13.0
+	github.com/klauspost/compress v1.13.4
-	github.com/klauspost/cpuid/v2 v2.0.6
+	github.com/klauspost/cpuid/v2 v2.0.9
-	github.com/lucas-clemente/quic-go v0.21.0
+	github.com/lucas-clemente/quic-go v0.23.0
-	github.com/mholt/acmez v0.1.3
+	github.com/mholt/acmez v1.0.0
 	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/naoina/toml v0.1.1
-	github.com/prometheus/client_golang v1.10.1-0.20210603120351-253906201bda
+	github.com/prometheus/client_golang v1.11.0
-	github.com/smallstep/certificates v0.15.15
+	github.com/smallstep/certificates v0.16.4
-	github.com/smallstep/cli v0.15.16
+	github.com/smallstep/cli v0.16.1
-	github.com/smallstep/nosql v0.3.6
+	github.com/smallstep/nosql v0.3.8
 	github.com/smallstep/truststore v0.9.6
-	github.com/yuin/goldmark v1.3.7
+	github.com/yuin/goldmark v1.4.0
 	github.com/yuin/goldmark-highlighting v0.0.0-20210516132338-9216f9c5aa01
-	go.uber.org/zap v1.17.0
+	go.uber.org/zap v1.19.0
-	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
+	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e
-	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
+	golang.org/x/net v0.0.0-20210614182718-04defd469f4e
 	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
-	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c
+	google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08
-	google.golang.org/protobuf v1.26.0
+	google.golang.org/protobuf v1.27.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v2 v2.4.0
 )

modules/caddyhttp/encode/encode.go

@@ -182,6 +182,7 @@ type responseWriter struct {
 	buf          *bytes.Buffer
 	config       *Encode
 	statusCode   int
+	wroteHeader  bool
 }
 
 // WriteHeader stores the status to write when the time comes
@@ -195,6 +196,19 @@ func (enc *Encode) Match(rw *responseWriter) bool {
 	return enc.Matcher.Match(rw.statusCode, rw.Header())
 }
 
+// Flush implements http.Flusher. It delays the actual Flush of the underlying ResponseWriterWrapper
+// until headers were written.
+func (rw *responseWriter) Flush() {
+	if !rw.wroteHeader {
+		// flushing the underlying ResponseWriter will write header and status code,
+		// but we need to delay that until we can determine if we must encode and
+		// therefore add the Content-Encoding header; this happens in the first call
+		// to rw.Write (see bug in #4314)
+		return
+	}
+	rw.ResponseWriterWrapper.Flush()
+}
+
 // Write writes to the response. If the response qualifies,
 // it is encoded using the encoder, which is initialized
 // if not done so already.
@@ -225,6 +239,7 @@ func (rw *responseWriter) Write(p []byte) (int, error) {
 	if rw.statusCode > 0 {
 		rw.ResponseWriter.WriteHeader(rw.statusCode)
 		rw.statusCode = 0
+		rw.wroteHeader = true
 	}
 
 	switch {
@@ -271,6 +286,7 @@ func (rw *responseWriter) Close() error {
 		// that rely on If-None-Match, for example
 		rw.ResponseWriter.WriteHeader(rw.statusCode)
 		rw.statusCode = 0
+		rw.wroteHeader = true
 	}
 	if rw.w != nil {
 		err2 := rw.w.Close()

modules/caddyhttp/encode/gzip/gzip.go

@@ -15,7 +15,6 @@
 package caddygzip
 
 import (
-	"compress/flate"
 	"fmt"
 	"strconv"
 
@@ -68,11 +67,11 @@ func (g *Gzip) Provision(ctx caddy.Context) error {
 
 // Validate validates g's configuration.
 func (g Gzip) Validate() error {
-	if g.Level < flate.NoCompression {
+	if g.Level < gzip.StatelessCompression {
-		return fmt.Errorf("quality too low; must be >= %d", flate.NoCompression)
+		return fmt.Errorf("quality too low; must be >= %d", gzip.StatelessCompression)
 	}
-	if g.Level > flate.BestCompression {
+	if g.Level > gzip.BestCompression {
-		return fmt.Errorf("quality too high; must be <= %d", flate.BestCompression)
+		return fmt.Errorf("quality too high; must be <= %d", gzip.BestCompression)
 	}
 	return nil
 }

modules/caddyhttp/encode/zstd/zstd.go

@@ -47,7 +47,10 @@ func (Zstd) AcceptEncoding() string { return "zstd" }
 
 // NewEncoder returns a new gzip writer.
 func (z Zstd) NewEncoder() encode.Encoder {
-	writer, _ := zstd.NewWriter(nil)
+	// The default of 8MB for the window is
+	// too large for many clients, so we limit
+	// it to 128K to lighten their load.
+	writer, _ := zstd.NewWriter(nil, zstd.WithWindowSize(128<<10), zstd.WithEncoderConcurrency(1), zstd.WithZeroFrames(true))
 	return writer
 }
 

modules/caddyhttp/fileserver/browsetplcontext.go

@@ -264,7 +264,7 @@ func (l byTime) Less(i, j int) bool { return l.Items[i].ModTime.Before(l.Items[j
 
 const (
 	sortByName         = "name"
-	sortByNameDirFirst = "name_dir_first"
+	sortByNameDirFirst = "namedirfirst"
 	sortBySize         = "size"
 	sortByTime         = "time"
 )

modules/caddyhttp/fileserver/caddyfile.go

@@ -41,6 +41,7 @@ func init() {
 //        browse        [<template_file>]
 //        precompressed <formats...>
 //        status        <status>
+//        disable_canonical_uris
 //    }
 //
 func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
@@ -112,6 +113,13 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 				}
 				fsrv.StatusCode = caddyhttp.WeakString(h.Val())
 
+			case "disable_canonical_uris":
+				if h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				falseBool := false
+				fsrv.CanonicalURIs = &falseBool
+
 			default:
 				return nil, h.Errf("unknown subdirective '%s'", h.Val())
 			}

modules/caddyhttp/fileserver/staticfiles.go

@@ -70,6 +70,10 @@ type FileServer struct {
 
 	// Use redirects to enforce trailing slashes for directories, or to
 	// remove trailing slash from URIs for files. Default is true.
+	//
+	// Canonicalization will not happen if the last element of the request's
+	// path (the filename) is changed in an internal rewrite, to avoid
+	// clobbering the explicit rewrite with implicit behavior.
 	CanonicalURIs *bool `json:"canonical_uris,omitempty"`
 
 	// Override the status code written when successfully serving a file.

modules/caddyhttp/matchers.go

@@ -82,7 +82,19 @@ type (
 	// MatchMethod matches requests by the method.
 	MatchMethod []string
 
-	// MatchQuery matches requests by URI's query string.
+	// MatchQuery matches requests by the URI's query string. It takes a JSON object
+	// keyed by the query keys, with an array of string values to match for that key.
+	// Query key matches are exact, but wildcards may be used for value matches. Both
+	// keys and values may be placeholders.
+	// An example of the structure to match `?key=value&topic=api&query=something` is:
+	//
+	// ```json
+	// {
+	// 	"key": ["value"],
+	//	"topic": ["api"],
+	//	"query": ["*"]
+	// }
+	// ```
 	MatchQuery url.Values
 
 	// MatchHeader matches requests by header fields. It performs fast,
@@ -667,7 +679,7 @@ func (MatchProtocol) CaddyModule() caddy.ModuleInfo {
 func (m MatchProtocol) Match(r *http.Request) bool {
 	switch string(m) {
 	case "grpc":
-		return r.Header.Get("content-type") == "application/grpc"
+		return strings.HasPrefix(r.Header.Get("content-type"), "application/grpc")
 	case "https":
 		return r.TLS != nil
 	case "http":

modules/caddyhttp/reverseproxy/admin.go

@@ -34,7 +34,8 @@ type adminUpstreams struct{}
 
 // upstreamResults holds the status of a particular upstream
 type upstreamStatus struct {
-	Address     string `json:"address"`
+	ID          string `json:"id"`
+	Address     string `json:"address"` // Address is deprecated, should be removed in a future release.
 	Healthy     bool   `json:"healthy"`
 	NumRequests int    `json:"num_requests"`
 	Fails       int    `json:"fails"`
@@ -78,7 +79,7 @@ func (adminUpstreams) handleUpstreams(w http.ResponseWriter, r *http.Request) er
 	// Iterate over the upstream pool (needs to be fast)
 	var rangeErr error
 	hosts.Range(func(key, val interface{}) bool {
-		address, ok := key.(string)
+		id, ok := key.(string)
 		if !ok {
 			rangeErr = caddy.APIError{
 				HTTPStatus: http.StatusInternalServerError,
@@ -97,7 +98,8 @@ func (adminUpstreams) handleUpstreams(w http.ResponseWriter, r *http.Request) er
 		}
 
 		results = append(results, upstreamStatus{
-			Address:     address,
+			ID:          id,
+			Address:     id,
 			Healthy:     !upstream.Unhealthy(),
 			NumRequests: upstream.NumRequests(),
 			Fails:       upstream.Fails(),

modules/caddyhttp/reverseproxy/caddyfile.go

@@ -61,7 +61,7 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 //         lb_try_interval <interval>
 //
 //         # active health checking
-//         health_path <path>
+//         health_uri  <uri>
 //         health_port <port>
 //         health_interval <interval>
 //         health_timeout <duration>
@@ -219,6 +219,12 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	// treated as a SRV-based upstream, and any port will be
 	// dropped.
 	appendUpstream := func(address string) error {
+		var id string
+		if strings.Contains(address, "|") {
+			parts := strings.SplitN(address, "|", 2)
+			id = parts[0]
+			address = parts[1]
+		}
 		isSRV := strings.HasPrefix(address, "srv+")
 		if isSRV {
 			address = strings.TrimPrefix(address, "srv+")
@@ -231,9 +237,9 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 			if host, _, err := net.SplitHostPort(dialAddr); err == nil {
 				dialAddr = host
 			}
-			h.Upstreams = append(h.Upstreams, &Upstream{LookupSRV: dialAddr})
+			h.Upstreams = append(h.Upstreams, &Upstream{ID: id, LookupSRV: dialAddr})
 		} else {
-			h.Upstreams = append(h.Upstreams, &Upstream{Dial: dialAddr})
+			h.Upstreams = append(h.Upstreams, &Upstream{ID: id, Dial: dialAddr})
 		}
 		return nil
 	}

modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go

@@ -192,7 +192,7 @@ func parsePHPFastCGI(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error
 	// NOTE: we delete the tokens as we go so that the reverse_proxy
 	// unmarshal doesn't see these subdirectives which it cannot handle
 	for dispenser.Next() {
-		for dispenser.NextBlock(0) {
+		for dispenser.NextBlock(0) && dispenser.Nesting() == 1 {
 			switch dispenser.Val() {
 			case "root":
 				if !dispenser.NextArg() {

modules/caddyhttp/reverseproxy/healthchecks.go

@@ -189,13 +189,14 @@ func (h *Handler) doActiveHealthCheckForAllHosts() {
 				return
 			}
 			hostAddr := addr.JoinHostPort(0)
+			dialAddr := hostAddr
 			if addr.IsUnixNetwork() {
 				// this will be used as the Host portion of a http.Request URL, and
 				// paths to socket files would produce an error when creating URL,
 				// so use a fake Host value instead; unix sockets are usually local
 				hostAddr = "localhost"
 			}
-			err = h.doActiveHealthCheck(DialInfo{Network: addr.Network, Address: hostAddr}, hostAddr, upstream.Host)
+			err = h.doActiveHealthCheck(DialInfo{Network: addr.Network, Address: dialAddr}, hostAddr, upstream.Host)
 			if err != nil {
 				h.HealthChecks.Active.logger.Error("active health check failed",
 					zap.String("address", hostAddr),

modules/caddyhttp/reverseproxy/hosts.go

@@ -65,6 +65,12 @@ type UpstreamPool []*Upstream
 type Upstream struct {
 	Host `json:"-"`
 
+	// The unique ID for this upstream, to disambiguate multiple
+	// upstreams with the same Dial address. This is optional,
+	// and only necessary if the upstream states need to be
+	// separate, such as having different health checking policies.
+	ID string `json:"id,omitempty"`
+
 	// The [network address](/docs/conventions#network-addresses)
 	// to dial to connect to the upstream. Must represent precisely
 	// one socket (i.e. no port ranges). A valid network address
@@ -98,6 +104,9 @@ type Upstream struct {
 }
 
 func (u Upstream) String() string {
+	if u.ID != "" {
+		return u.ID
+	}
 	if u.LookupSRV != "" {
 		return u.LookupSRV
 	}

modules/caddyhttp/reverseproxy/reverseproxy.go

@@ -23,6 +23,7 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"net/textproto"
 	"net/url"
 	"regexp"
 	"strconv"
@@ -80,10 +81,13 @@ type Handler struct {
 	// Upstreams is the list of backends to proxy to.
 	Upstreams UpstreamPool `json:"upstreams,omitempty"`
 
-	// Adjusts how often to flush the response buffer. A
+	// Adjusts how often to flush the response buffer. By default,
-	// negative value disables response buffering.
+	// no periodic flushing is done. A negative value disables
-	// TODO: figure out good defaults and write docs for this
+	// response buffering, and flushes immediately after each
-	// (see https://github.com/caddyserver/caddy/issues/1460)
+	// write to the client. This option is ignored when the upstream's
+	// response is recognized as a streaming response, or if its
+	// content length is -1; for such responses, writes are flushed
+	// to the client immediately.
 	FlushInterval caddy.Duration `json:"flush_interval,omitempty"`
 
 	// Headers manipulates headers between Caddy and the backend.
@@ -528,13 +532,19 @@ func (h Handler) prepareRequest(req *http.Request) error {
 		// If we aren't the first proxy retain prior
 		// X-Forwarded-For information as a comma+space
 		// separated list and fold multiple headers into one.
-		if prior, ok := req.Header["X-Forwarded-For"]; ok {
+		prior, ok := req.Header["X-Forwarded-For"]
+		omit := ok && prior == nil // Issue 38079: nil now means don't populate the header
+		if len(prior) > 0 {
 			clientIP = strings.Join(prior, ", ") + ", " + clientIP
 		}
-		req.Header.Set("X-Forwarded-For", clientIP)
+		if !omit {
+			req.Header.Set("X-Forwarded-For", clientIP)
+		}
 	}
 
-	if req.Header.Get("X-Forwarded-Proto") == "" {
+	prior, ok := req.Header["X-Forwarded-Proto"]
+	omit := ok && prior == nil
+	if len(prior) == 0 && !omit {
 		// set X-Forwarded-Proto; many backend apps expect this too
 		proto := "https"
 		if req.TLS == nil {
@@ -681,15 +691,6 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, repl *
 	}
 
 	rw.WriteHeader(res.StatusCode)
-
-	// some apps need the response headers before starting to stream content with http2,
-	// so it's important to explicitly flush the headers to the client before streaming the data.
-	// (see https://github.com/caddyserver/caddy/issues/3556 for use case and nuances)
-	if h.isBidirectionalStream(req, res) {
-		if wf, ok := rw.(http.Flusher); ok {
-			wf.Flush()
-		}
-	}
 	err = h.copyResponse(rw, res.Body, h.flushInterval(req, res))
 	res.Body.Close() // close now, instead of defer, to populate res.Trailer
 	if err != nil {
@@ -827,10 +828,10 @@ func upgradeType(h http.Header) string {
 // removeConnectionHeaders removes hop-by-hop headers listed in the "Connection" header of h.
 // See RFC 7230, section 6.1
 func removeConnectionHeaders(h http.Header) {
-	if c := h.Get("Connection"); c != "" {
+	for _, f := range h["Connection"] {
-		for _, f := range strings.Split(c, ",") {
+		for _, sf := range strings.Split(f, ",") {
-			if f = strings.TrimSpace(f); f != "" {
+			if sf = textproto.TrimString(sf); sf != "" {
-				h.Del(f)
+				h.Del(sf)
 			}
 		}
 	}

modules/caddyhttp/reverseproxy/streaming.go

@@ -32,6 +32,9 @@ import (
 func (h Handler) handleUpgradeResponse(logger *zap.Logger, rw http.ResponseWriter, req *http.Request, res *http.Response) {
 	reqUpType := upgradeType(req.Header)
 	resUpType := upgradeType(res.Header)
+	// TODO: Update to use "net/http/internal/ascii" once we bumped
+	// the minimum Go version to 1.17.
+	// See https://github.com/golang/go/commit/5c489514bc5e61ad9b5b07bd7d8ec65d66a0512a
 	if reqUpType != resUpType {
 		h.logger.Debug("backend tried to switch to unexpected protocol via Upgrade header",
 			zap.String("backend_upgrade", resUpType),
@@ -39,8 +42,6 @@ func (h Handler) handleUpgradeResponse(logger *zap.Logger, rw http.ResponseWrite
 		return
 	}
 
-	copyHeader(res.Header, rw.Header())
-
 	hj, ok := rw.(http.Hijacker)
 	if !ok {
 		h.logger.Sugar().Errorf("can't switch protocols using non-Hijacker ResponseWriter type %T", rw)
@@ -78,6 +79,9 @@ func (h Handler) handleUpgradeResponse(logger *zap.Logger, rw http.ResponseWrite
 		logger.Debug("connection closed", zap.Duration("duration", time.Since(start)))
 	}()
 
+	copyHeader(rw.Header(), res.Header)
+
+	res.Header = rw.Header()
 	res.Body = nil // so res.Write only writes the headers; we have res.Body in backConn above
 	if err := res.Write(brw); err != nil {
 		h.logger.Debug("response write", zap.Error(err))
@@ -107,13 +111,16 @@ func (h Handler) flushInterval(req *http.Request, res *http.Response) time.Durat
 		return -1 // negative means immediately
 	}
 
+	// We might have the case of streaming for which Content-Length might be unset.
+	if res.ContentLength == -1 {
+		return -1
+	}
+
 	// for h2 and h2c upstream streaming data to client (issues #3556 and #3606)
 	if h.isBidirectionalStream(req, res) {
 		return -1
 	}
 
-	// TODO: more specific cases? e.g. res.ContentLength == -1? (this TODO is from the std lib, but
-	// strangely similar to our isBidirectionalStream function that we implemented ourselves)
 	return time.Duration(h.FlushInterval)
 }
 
@@ -142,6 +149,11 @@ func (h Handler) copyResponse(dst io.Writer, src io.Reader, flushInterval time.D
 				latency: flushInterval,
 			}
 			defer mlw.stop()
+
+			// set up initial timer so headers get flushed even if body writes are delayed
+			mlw.flushPending = true
+			mlw.t = time.AfterFunc(flushInterval, mlw.delayedFlush)
+
 			dst = mlw
 		}
 	}

modules/caddyhttp/server.go

@@ -184,8 +184,11 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				log = logger.Error
 			}
 
+			userID, _ := repl.GetString("http.auth.user.id")
+
 			log("handled request",
 				zap.String("common_log", repl.ReplaceAll(commonLogFormat, commonLogEmptyValue)),
+				zap.String("user_id", userID),
 				zap.Duration("duration", duration),
 				zap.Int("size", wrec.Size()),
 				zap.Int("status", wrec.Status()),
@@ -379,7 +382,9 @@ func (s *Server) hasTLSClientAuth() bool {
 // that it is after any other host matcher but before any "catch-all"
 // route without a host matcher.
 func (s *Server) findLastRouteWithHostMatcher() int {
+	foundHostMatcher := false
 	lastIndex := len(s.Routes)
+
 	for i, route := range s.Routes {
 		// since we want to break out of an inner loop, use a closure
 		// to allow us to use 'return' when we found a host matcher
@@ -388,6 +393,7 @@ func (s *Server) findLastRouteWithHostMatcher() int {
 				for _, matcher := range sets {
 					switch matcher.(type) {
 					case *MatchHost:
+						foundHostMatcher = true
 						return true
 					}
 				}
@@ -401,6 +407,14 @@ func (s *Server) findLastRouteWithHostMatcher() int {
 			lastIndex = i + 1
 		}
 	}
+
+	// If we didn't actually find a host matcher, return 0
+	// because that means every defined route was a "catch-all".
+	// See https://caddy.community/t/how-to-set-priority-in-caddyfile/13002/8
+	if !foundHostMatcher {
+		return 0
+	}
+
 	return lastIndex
 }
 

modules/caddypki/acmeserver/acmeserver.go

@@ -50,17 +50,16 @@ type Handler struct {
 
 	// The hostname or IP address by which ACME clients
 	// will access the server. This is used to populate
-	// the ACME directory endpoint. Default: localhost.
+	// the ACME directory endpoint. If not set, the Host
+	// header of the request will be used.
 	// COMPATIBILITY NOTE / TODO: This property may go away in the
-	// future, as it is currently only required due to
+	// future. Do not rely on this property long-term; check release notes.
-	// limitations in the underlying library. Do not rely
-	// on this property long-term; check release notes.
 	Host string `json:"host,omitempty"`
 
 	// The path prefix under which to serve all ACME
 	// endpoints. All other requests will not be served
 	// by this handler and will be passed through to
-	// the next one. Default: "/acme/"
+	// the next one. Default: "/acme/".
 	// COMPATIBILITY NOTE / TODO: This property may go away in the
 	// future, as it is currently only required due to
 	// limitations in the underlying library. Do not rely
@@ -93,9 +92,6 @@ func (ash *Handler) Provision(ctx caddy.Context) error {
 	if ash.CA == "" {
 		ash.CA = caddypki.DefaultCAID
 	}
-	if ash.Host == "" {
-		ash.Host = defaultHost
-	}
 	if ash.PathPrefix == "" {
 		ash.PathPrefix = defaultPathPrefix
 	}
@@ -150,9 +146,9 @@ func (ash *Handler) Provision(ctx caddy.Context) error {
 	// create the router for the ACME endpoints
 	acmeRouterHandler := acmeAPI.NewHandler(acmeAPI.HandlerOptions{
 		CA:     auth,
-		DB:     acmeDB,         // stores all the server state
+		DB:     acmeDB,                            // stores all the server state
-		DNS:    ash.Host,       // used for directory links; TODO: not needed (follow-up upstream with step-ca)
+		DNS:    ash.Host,                          // used for directory links
-		Prefix: ash.PathPrefix, // used for directory links
+		Prefix: strings.Trim(ash.PathPrefix, "/"), // used for directory links
 	})
 
 	// extract its http.Handler so we can use it directly
@@ -219,10 +215,7 @@ func (ash Handler) openDatabase() (*db.AuthDB, error) {
 	return database.(databaseCloser).DB, err
 }
 
-const (
+const defaultPathPrefix = "/acme/"
-	defaultHost       = "localhost"
-	defaultPathPrefix = "/acme/"
-)
 
 var keyCleaner = regexp.MustCompile(`[^\w.-_]`)
 var databasePool = caddy.NewUsagePool()

modules/caddytls/automation.go

@@ -89,10 +89,6 @@ type AutomationPolicy struct {
 	// zerossl.
 	IssuersRaw []json.RawMessage `json:"issuers,omitempty" caddy:"namespace=tls.issuance inline_key=module"`
 
-	// DEPRECATED: Use `issuers` instead (November 2020). This field will
-	// be removed in the future.
-	IssuerRaw json.RawMessage `json:"issuer,omitempty" caddy:"namespace=tls.issuance inline_key=module"`
-
 	// If true, certificates will be requested with MustStaple. Not all
 	// CAs support this, and there are potentially serious consequences
 	// of enabling this feature without proper threat modeling.
@@ -180,12 +176,6 @@ func (ap *AutomationPolicy) Provision(tlsApp *TLS) error {
 		}
 	}
 
-	// TODO: IssuerRaw field deprecated as of November 2020 - remove this shim after deprecation is complete
-	if ap.IssuerRaw != nil {
-		tlsApp.logger.Warn("the 'issuer' field is deprecated and will be removed in the future; use 'issuers' instead; your issuer has been appended automatically for now")
-		ap.IssuersRaw = append(ap.IssuersRaw, ap.IssuerRaw)
-	}
-
 	// load and provision any explicitly-configured issuer modules
 	if ap.IssuersRaw != nil {
 		val, err := tlsApp.ctx.LoadModule(ap, "IssuersRaw")

modules/logging/encoders.go

@@ -135,6 +135,7 @@ func (SingleFieldEncoder) CaddyModule() caddy.ModuleInfo {
 
 // Provision sets up the encoder.
 func (se *SingleFieldEncoder) Provision(ctx caddy.Context) error {
+	caddy.Log().Named("caddy.logging.encoders.single_field").Warn("the 'single_field' encoder is deprecated and will be removed soon!")
 	if se.FallbackRaw != nil {
 		val, err := ctx.LoadModule(se, "FallbackRaw")
 		if err != nil {
@@ -307,6 +308,8 @@ func (lec *LogEncoderConfig) ZapcoreEncoderConfig() zapcore.EncoderConfig {
 			timeFormat = "2006/01/02 15:04:05.000"
 		case "wall_nano":
 			timeFormat = "2006/01/02 15:04:05.000000000"
+		case "common_log":
+			timeFormat = "02/Jan/2006:15:04:05 -0700"
 		}
 		timeFormatter = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) {
 			encoder.AppendString(ts.UTC().Format(timeFormat))

modules/logging/filters.go

@@ -188,9 +188,11 @@ func (m IPMaskFilter) Filter(in zapcore.Field) zapcore.Field {
 // Interface guards
 var (
 	_ LogFieldFilter = (*DeleteFilter)(nil)
+	_ LogFieldFilter = (*ReplaceFilter)(nil)
 	_ LogFieldFilter = (*IPMaskFilter)(nil)
 
 	_ caddyfile.Unmarshaler = (*DeleteFilter)(nil)
+	_ caddyfile.Unmarshaler = (*ReplaceFilter)(nil)
 	_ caddyfile.Unmarshaler = (*IPMaskFilter)(nil)
 
 	_ caddy.Provisioner = (*IPMaskFilter)(nil)

replacer.go

@@ -301,6 +301,10 @@ func globalDefaultReplacements(key string) (interface{}, bool) {
 		return nowFunc().Format("02/Jan/2006:15:04:05 -0700"), true
 	case "time.now.year":
 		return strconv.Itoa(nowFunc().Year()), true
+	case "time.now.unix":
+		return strconv.FormatInt(nowFunc().Unix(), 10), true
+	case "time.now.unix_ms":
+		return strconv.FormatInt(nowFunc().UnixNano()/int64(time.Millisecond), 10), true
 	}
 
 	return nil, false