Also reject null byte

* Add renegotiation option in reverseproxy tls client

* Update modules/caddyhttp/reverseproxy/httptransport.go

Co-authored-by: Matt Holt <mholt@users.noreply.github.com>

.editorconfig

@@ -0,0 +1,5 @@
+[*]
+end_of_line = lf
+
+[caddytest/integration/caddyfile_adapt/*.txt]
+indent_style = tab

.github/workflows/ci.yml

@@ -19,12 +19,20 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
-        go: [ '1.16', '1.17' ]
+        go: [ '1.17', '1.18' ]
+
+        include:
+        # Set the minimum Go patch version for the given Go minor
+        # Usable via ${{ matrix.GO_SEMVER }}
+        - go: '1.17'
+          GO_SEMVER: '~1.17.9'
+
+        - go: '1.18'
+          GO_SEMVER: '~1.18.1'
 
         # Set some variables per OS, usable via ${{ matrix.VAR }}
         # CADDY_BIN_PATH: the path to the compiled Caddy binary, for artifact publishing
         # SUCCESS: the typical value for $? per OS (Windows/pwsh returns 'True')
-        include:
         - os: ubuntu-latest
           CADDY_BIN_PATH: ./cmd/caddy/caddy
           SUCCESS: 0
@@ -41,12 +49,13 @@ jobs:
 
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
-        go-version: ${{ matrix.go }}
+        go-version: ${{ matrix.GO_SEMVER }}
+        check-latest: true
 
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # These tools would be useful if we later decide to reinvestigate
     # publishing test/coverage reports to some tool for easier consumption
@@ -69,12 +78,20 @@ jobs:
         printf "Git version: $(git version)\n\n"
         # Calculate the short SHA1 hash of the git commit
         echo "::set-output name=short_sha::$(git rev-parse --short HEAD)"
-        echo "::set-output name=go_cache::$(go env GOCACHE)"
 
     - name: Cache the build cache
       uses: actions/cache@v2
       with:
-        path: ${{ steps.vars.outputs.go_cache }}
+        # In order:
+        # * Module download cache
+        # * Build cache (Linux)
+        # * Build cache (Mac)
+        # * Build cache (Windows)
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
+          ~/Library/Caches/go-build
+          ~\AppData\Local\go-build
         key: ${{ runner.os }}-${{ matrix.go }}-go-ci-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-${{ matrix.go }}-go-ci
@@ -130,7 +147,7 @@ jobs:
     continue-on-error: true  # August 2020: s390x VM is down due to weather and power issues
     steps:
       - name: Checkout code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Run Tests
         run: |
           mkdir -p ~/.ssh && echo -e "${SSH_KEY//_/\\n}" > ~/.ssh/id_ecdsa && chmod og-rwx ~/.ssh/id_ecdsa
@@ -155,7 +172,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       
       - uses: goreleaser/goreleaser-action@v2
         with:

.github/workflows/cross-build.yml

@@ -16,14 +16,22 @@ jobs:
       fail-fast: false
       matrix:
         goos: ['android', 'linux', 'solaris', 'illumos', 'dragonfly', 'freebsd', 'openbsd', 'plan9', 'windows', 'darwin', 'netbsd']
-        go: [ '1.17' ]
+        go: [ '1.18' ]
+
+        include:
+        # Set the minimum Go patch version for the given Go minor
+        # Usable via ${{ matrix.GO_SEMVER }}
+        - go: '1.18'
+          GO_SEMVER: '~1.18.1'
+
     runs-on: ubuntu-latest
     continue-on-error: true
     steps:
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
-          go-version: ${{ matrix.go }}
+          go-version: ${{ matrix.GO_SEMVER }}
+          check-latest: true
 
       - name: Print Go version and environment
         id: vars
@@ -34,18 +42,22 @@ jobs:
           go env
           printf "\n\nSystem environment:\n\n"
           env
-          echo "::set-output name=go_cache::$(go env GOCACHE)"
 
       - name: Cache the build cache
         uses: actions/cache@v2
         with:
-          path: ${{ steps.vars.outputs.go_cache }}
+          # In order:
+          # * Module download cache
+          # * Build cache (Linux)
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
           key: cross-build-go${{ matrix.go }}-${{ matrix.goos }}-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             cross-build-go${{ matrix.go }}-${{ matrix.goos }}
 
       - name: Checkout code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Run Build
         env:

.github/workflows/lint.yml

@@ -16,10 +16,15 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
         with:
-          version: v1.31
+          go-version: '~1.17.9'
+          check-latest: true
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.44
           # Optional: show only new issues if it's a pull request. The default value is `false`.
           # only-new-issues: true

.github/workflows/release.yml

@@ -11,22 +11,30 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        go: [ '1.17' ]
+        go: [ '1.18' ]
+
+        include:
+        # Set the minimum Go patch version for the given Go minor
+        # Usable via ${{ matrix.GO_SEMVER }}
+        - go: '1.18'
+          GO_SEMVER: '~1.18.1'
+
     runs-on: ${{ matrix.os }}
 
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
-        go-version: ${{ matrix.go }}
+        go-version: ${{ matrix.GO_SEMVER }}
+        check-latest: true
 
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
 
     # Force fetch upstream tags -- because 65 minutes
-    # tl;dr: actions/checkout@v2 runs this line:
+    # tl;dr: actions/checkout@v3 runs this line:
     #   git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +ebc278ec98bb24f2852b61fde2a9bf2e3d83818b:refs/tags/
     # which makes its own local lightweight tag, losing all the annotations in the process. Our earlier script ran:
     #   git fetch --prune --unshallow
@@ -48,7 +56,6 @@ jobs:
         env
         echo "::set-output name=version_tag::${GITHUB_REF/refs\/tags\//}"
         echo "::set-output name=short_sha::$(git rev-parse --short HEAD)"
-        echo "::set-output name=go_cache::$(go env GOCACHE)"
 
         # Add "pip install" CLI tools to PATH
         echo ~/.local/bin >> $GITHUB_PATH
@@ -83,7 +90,12 @@ jobs:
     - name: Cache the build cache
       uses: actions/cache@v2
       with:
-        path: ${{ steps.vars.outputs.go_cache }}
+        # In order:
+        # * Module download cache
+        # * Build cache (Linux)
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
         key: ${{ runner.os }}-go${{ matrix.go }}-release-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go${{ matrix.go }}-release
@@ -99,7 +111,7 @@ jobs:
         TAG: ${{ steps.vars.outputs.version_tag }}
 
     # Only publish on non-special tags (e.g. non-beta)
-    # We will continue to push to Gemfury for the forseeable future, although
+    # We will continue to push to Gemfury for the foreseeable future, although
     # Cloudsmith is probably better, to not break things for existing users of Gemfury.
     # See https://gemfury.com/caddy/deb:caddy
     - name: Publish .deb to Gemfury

.gitignore

@@ -1,6 +1,7 @@
 _gitignore/
 *.log
 Caddyfile
+Caddyfile.*
 !caddyfile/
 
 # artifacts from pprof tooling

.goreleaser.yml

@@ -6,8 +6,7 @@ before:
     # subsequently causes gorleaser to refuse running.
     - mkdir -p caddy-build
     - cp cmd/caddy/main.go caddy-build/main.go
-    - cp ./go.mod caddy-build/go.mod
-    - sed -i.bkp 's|github.com/caddyserver/caddy/v2|caddy|g' ./caddy-build/go.mod
+    - /bin/sh -c 'cd ./caddy-build && go mod init caddy'
     # GoReleaser doesn't seem to offer {{.Tag}} at this stage, so we have to embed it into the env
     # so we run: TAG=$(git describe --abbrev=0) goreleaser release --rm-dist --skip-publish --skip-validate
     - go mod edit -require=github.com/caddyserver/caddy/v2@{{.Env.TAG}} ./caddy-build/go.mod
@@ -36,9 +35,9 @@ builds:
   - s390x
   - ppc64le
   goarm:
-  - 5
-  - 6
-  - 7
+  - "5"
+  - "6"
+  - "7"
   ignore:
     - goos: darwin
       goarch: arm
@@ -56,7 +55,7 @@ builds:
       goarch: s390x
     - goos: freebsd
       goarch: arm
-      goarm: 5
+      goarm: "5"
   flags:
   - -trimpath
   ldflags:

README.md

@@ -75,7 +75,7 @@ For other install options, see https://caddyserver.com/docs/install.
 
 Requirements:
 
-- [Go 1.16 or newer](https://golang.org/dl/)
+- [Go 1.17 or newer](https://golang.org/dl/)
 
 ### For development
  

admin.go

@@ -42,6 +42,7 @@ import (
 	"github.com/caddyserver/certmagic"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 )
 
 // AdminConfig configures Caddy's API endpoint, which is used
@@ -91,6 +92,10 @@ type AdminConfig struct {
 	//
 	// EXPERIMENTAL: This feature is subject to change.
 	Remote *RemoteAdmin `json:"remote,omitempty"`
+
+	// Holds onto the routers so that we can later provision them
+	// if they require provisioning.
+	routers []AdminRouter
 }
 
 // ConfigSettings configures the management of configuration.
@@ -100,20 +105,26 @@ type ConfigSettings struct {
 	// are not persisted; only configs that are pushed to Caddy get persisted.
 	Persist *bool `json:"persist,omitempty"`
 
-	// Loads a configuration to use. This is helpful if your configs are
-	// managed elsewhere, and you want Caddy to pull its config dynamically
+	// Loads a new configuration. This is helpful if your configs are
+	// managed elsewhere and you want Caddy to pull its config dynamically
 	// when it starts. The pulled config completely replaces the current
 	// one, just like any other config load. It is an error if a pulled
-	// config is configured to pull another config.
+	// config is configured to pull another config without a load_delay,
+	// as this creates a tight loop.
 	//
 	// EXPERIMENTAL: Subject to change.
 	LoadRaw json.RawMessage `json:"load,omitempty" caddy:"namespace=caddy.config_loaders inline_key=module"`
 
-	// The interval to pull config. With a non-zero value, will pull config
-	// from config loader (eg. a http loader) with given interval.
+	// The duration after which to load config. If set, config will be pulled
+	// from the config loader after this duration. A delay is required if a
+	// dynamically-loaded config is configured to load yet another config. To
+	// load configs on a regular interval, ensure this value is set the same
+	// on all loaded configs; it can also be variable if needed, and to stop
+	// the loop, simply remove dynamic config loading from the next-loaded
+	// config.
 	//
 	// EXPERIMENTAL: Subject to change.
-	LoadInterval Duration `json:"load_interval,omitempty"`
+	LoadDelay Duration `json:"load_delay,omitempty"`
 }
 
 // IdentityConfig configures management of this server's identity. An identity
@@ -183,7 +194,7 @@ type AdminPermissions struct {
 
 // newAdminHandler reads admin's config and returns an http.Handler suitable
 // for use in an admin endpoint server, which will be listening on listenAddr.
-func (admin AdminConfig) newAdminHandler(addr NetworkAddress, remote bool) adminHandler {
+func (admin *AdminConfig) newAdminHandler(addr NetworkAddress, remote bool) adminHandler {
 	muxWrap := adminHandler{mux: http.NewServeMux()}
 
 	// secure the local or remote endpoint respectively
@@ -192,6 +203,7 @@ func (admin AdminConfig) newAdminHandler(addr NetworkAddress, remote bool) admin
 	} else {
 		muxWrap.enforceHost = !addr.isWildcardInterface()
 		muxWrap.allowedOrigins = admin.allowedOrigins(addr)
+		muxWrap.enforceOrigin = admin.EnforceOrigin
 	}
 
 	addRouteWithMetrics := func(pattern string, handlerLabel string, h http.Handler) {
@@ -242,17 +254,39 @@ func (admin AdminConfig) newAdminHandler(addr NetworkAddress, remote bool) admin
 		for _, route := range router.Routes() {
 			addRoute(route.Pattern, handlerLabel, route.Handler)
 		}
+		admin.routers = append(admin.routers, router)
 	}
 
 	return muxWrap
 }
 
+// provisionAdminRouters provisions all the router modules
+// in the admin.api namespace that need provisioning.
+func (admin *AdminConfig) provisionAdminRouters(ctx Context) error {
+	for _, router := range admin.routers {
+		provisioner, ok := router.(Provisioner)
+		if !ok {
+			continue
+		}
+
+		err := provisioner.Provision(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	// We no longer need the routers once provisioned, allow for GC
+	admin.routers = nil
+
+	return nil
+}
+
 // allowedOrigins returns a list of origins that are allowed.
 // If admin.Origins is nil (null), the provided listen address
 // will be used as the default origin. If admin.Origins is
 // empty, no origins will be allowed, effectively bricking the
 // endpoint for non-unix-socket endpoints, but whatever.
-func (admin AdminConfig) allowedOrigins(addr NetworkAddress) []string {
+func (admin AdminConfig) allowedOrigins(addr NetworkAddress) []*url.URL {
 	uniqueOrigins := make(map[string]struct{})
 	for _, o := range admin.Origins {
 		uniqueOrigins[o] = struct{}{}
@@ -276,8 +310,23 @@ func (admin AdminConfig) allowedOrigins(addr NetworkAddress) []string {
 			uniqueOrigins[addr.JoinHostPort(0)] = struct{}{}
 		}
 	}
-	allowed := make([]string, 0, len(uniqueOrigins))
-	for origin := range uniqueOrigins {
+	allowed := make([]*url.URL, 0, len(uniqueOrigins))
+	for originStr := range uniqueOrigins {
+		var origin *url.URL
+		if strings.Contains(originStr, "://") {
+			var err error
+			origin, err = url.Parse(originStr)
+			if err != nil {
+				continue
+			}
+			origin.Path = ""
+			origin.RawPath = ""
+			origin.Fragment = ""
+			origin.RawFragment = ""
+			origin.RawQuery = ""
+		} else {
+			origin = &url.URL{Host: originStr}
+		}
 		allowed = append(allowed, origin)
 	}
 	return allowed
@@ -309,25 +358,26 @@ func replaceLocalAdminServer(cfg *Config) error {
 		}
 	}()
 
-	// always get a valid admin config
-	adminConfig := DefaultAdminConfig
-	if cfg != nil && cfg.Admin != nil {
-		adminConfig = cfg.Admin
+	// set a default if admin wasn't otherwise configured
+	if cfg.Admin == nil {
+		cfg.Admin = &AdminConfig{
+			Listen: DefaultAdminListen,
+		}
 	}
 
 	// if new admin endpoint is to be disabled, we're done
-	if adminConfig.Disabled {
+	if cfg.Admin.Disabled {
 		Log().Named("admin").Warn("admin endpoint disabled")
 		return nil
 	}
 
 	// extract a singular listener address
-	addr, err := parseAdminListenAddr(adminConfig.Listen, DefaultAdminListen)
+	addr, err := parseAdminListenAddr(cfg.Admin.Listen, DefaultAdminListen)
 	if err != nil {
 		return err
 	}
 
-	handler := adminConfig.newAdminHandler(addr, false)
+	handler := cfg.Admin.newAdminHandler(addr, false)
 
 	ln, err := Listen(addr.Network, addr.JoinHostPort(0))
 	if err != nil {
@@ -357,8 +407,8 @@ func replaceLocalAdminServer(cfg *Config) error {
 
 	adminLogger.Info("admin endpoint started",
 		zap.String("address", addr.String()),
-		zap.Bool("enforce_origin", adminConfig.EnforceOrigin),
-		zap.Strings("origins", handler.allowedOrigins))
+		zap.Bool("enforce_origin", cfg.Admin.EnforceOrigin),
+		zap.Array("origins", loggableURLArray(handler.allowedOrigins)))
 
 	if !handler.enforceHost {
 		adminLogger.Warn("admin endpoint on open interface; host checking disabled",
@@ -466,6 +516,9 @@ func replaceRemoteAdminServer(ctx Context, cfg *Config) error {
 	}
 
 	// create TLS config that will enforce mutual authentication
+	if identityCertCache == nil {
+		return fmt.Errorf("cannot enable remote admin without a certificate cache; configure identity management to initialize a certificate cache")
+	}
 	cmCfg := cfg.Admin.Identity.certmagicConfig(remoteLogger, false)
 	tlsConfig := cmCfg.TLSConfig()
 	tlsConfig.NextProtos = nil // this server does not solve ACME challenges
@@ -647,10 +700,10 @@ type AdminRoute struct {
 type adminHandler struct {
 	mux *http.ServeMux
 
-	// security for local/plaintext) endpoint, on by default
+	// security for local/plaintext endpoint
 	enforceOrigin  bool
 	enforceHost    bool
-	allowedOrigins []string
+	allowedOrigins []*url.URL
 
 	// security for remote/encrypted endpoint
 	remoteControl *RemoteAdmin
@@ -659,11 +712,17 @@ type adminHandler struct {
 // ServeHTTP is the external entry point for API requests.
 // It will only be called once per request.
 func (h adminHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ip, port, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		ip = r.RemoteAddr
+		port = ""
+	}
 	log := Log().Named("admin.api").With(
 		zap.String("method", r.Method),
 		zap.String("host", r.Host),
 		zap.String("uri", r.RequestURI),
-		zap.String("remote_addr", r.RemoteAddr),
+		zap.String("remote_ip", ip),
+		zap.String("remote_port", port),
 		zap.Reflect("headers", r.Header),
 	)
 	if r.TLS != nil {
@@ -770,8 +829,8 @@ func (h adminHandler) handleError(w http.ResponseWriter, r *http.Request, err er
 // rebinding attacks.
 func (h adminHandler) checkHost(r *http.Request) error {
 	var allowed bool
-	for _, allowedHost := range h.allowedOrigins {
-		if r.Host == allowedHost {
+	for _, allowedOrigin := range h.allowedOrigins {
+		if r.Host == allowedOrigin.Host {
 			allowed = true
 			break
 		}
@@ -790,43 +849,45 @@ func (h adminHandler) checkHost(r *http.Request) error {
 // sites from issuing requests to our listener. It
 // returns the origin that was obtained from r.
 func (h adminHandler) checkOrigin(r *http.Request) (string, error) {
-	origin := h.getOriginHost(r)
-	if origin == "" {
-		return origin, APIError{
+	originStr, origin := h.getOrigin(r)
+	if origin == nil {
+		return "", APIError{
 			HTTPStatus: http.StatusForbidden,
-			Err:        fmt.Errorf("missing required Origin header"),
+			Err:        fmt.Errorf("required Origin header is missing or invalid"),
 		}
 	}
 	if !h.originAllowed(origin) {
-		return origin, APIError{
+		return "", APIError{
 			HTTPStatus: http.StatusForbidden,
-			Err:        fmt.Errorf("client is not allowed to access from origin %s", origin),
+			Err:        fmt.Errorf("client is not allowed to access from origin '%s'", originStr),
 		}
 	}
-	return origin, nil
+	return origin.String(), nil
 }
 
-func (h adminHandler) getOriginHost(r *http.Request) string {
+func (h adminHandler) getOrigin(r *http.Request) (string, *url.URL) {
 	origin := r.Header.Get("Origin")
 	if origin == "" {
 		origin = r.Header.Get("Referer")
 	}
 	originURL, err := url.Parse(origin)
-	if err == nil && originURL.Host != "" {
-		origin = originURL.Host
+	if err != nil {
+		return origin, nil
 	}
-	return origin
+	originURL.Path = ""
+	originURL.RawPath = ""
+	originURL.Fragment = ""
+	originURL.RawFragment = ""
+	originURL.RawQuery = ""
+	return origin, originURL
 }
 
-func (h adminHandler) originAllowed(origin string) bool {
+func (h adminHandler) originAllowed(origin *url.URL) bool {
 	for _, allowedOrigin := range h.allowedOrigins {
-		originCopy := origin
-		if !strings.Contains(allowedOrigin, "://") {
-			// no scheme specified, so allow both
-			originCopy = strings.TrimPrefix(originCopy, "http://")
-			originCopy = strings.TrimPrefix(originCopy, "https://")
+		if allowedOrigin.Scheme != "" && origin.Scheme != allowedOrigin.Scheme {
+			continue
 		}
-		if originCopy == allowedOrigin {
+		if origin.Host == allowedOrigin.Host {
 			return true
 		}
 	}
@@ -877,7 +938,7 @@ func handleConfig(w http.ResponseWriter, r *http.Request) error {
 		forceReload := r.Header.Get("Cache-Control") == "must-revalidate"
 
 		err := changeConfig(r.Method, r.URL.Path, body, forceReload)
-		if err != nil {
+		if err != nil && !errors.Is(err, errSameConfig) {
 			return err
 		}
 
@@ -896,10 +957,16 @@ func handleConfigID(w http.ResponseWriter, r *http.Request) error {
 
 	parts := strings.Split(idPath, "/")
 	if len(parts) < 3 || parts[2] == "" {
-		return fmt.Errorf("request path is missing object ID")
+		return APIError{
+			HTTPStatus: http.StatusBadRequest,
+			Err:        fmt.Errorf("request path is missing object ID"),
+		}
 	}
 	if parts[0] != "" || parts[1] != "id" {
-		return fmt.Errorf("malformed object path")
+		return APIError{
+			HTTPStatus: http.StatusBadRequest,
+			Err:        fmt.Errorf("malformed object path"),
+		}
 	}
 	id := parts[2]
 
@@ -908,7 +975,10 @@ func handleConfigID(w http.ResponseWriter, r *http.Request) error {
 	expanded, ok := rawCfgIndex[id]
 	defer currentCfgMu.RUnlock()
 	if !ok {
-		return fmt.Errorf("unknown object ID '%s'", id)
+		return APIError{
+			HTTPStatus: http.StatusNotFound,
+			Err:        fmt.Errorf("unknown object ID '%s'", id),
+		}
 	}
 
 	// piece the full URL path back together
@@ -930,7 +1000,7 @@ func handleStop(w http.ResponseWriter, r *http.Request) error {
 		Log().Error("unable to notify stopping to service manager", zap.Error(err))
 	}
 
-	exitProcess(Log().Named("admin.api"))
+	exitProcess(context.Background(), Log().Named("admin.api"))
 	return nil
 }
 
@@ -1180,6 +1250,18 @@ func decodeBase64DERCert(certStr string) (*x509.Certificate, error) {
 	return x509.ParseCertificate(derBytes)
 }
 
+type loggableURLArray []*url.URL
+
+func (ua loggableURLArray) MarshalLogArray(enc zapcore.ArrayEncoder) error {
+	if ua == nil {
+		return nil
+	}
+	for _, u := range ua {
+		enc.AppendString(u.String())
+	}
+	return nil
+}
+
 var (
 	// DefaultAdminListen is the address for the local admin
 	// listener, if none is specified at startup.
@@ -1189,12 +1271,6 @@ var (
 	// (TLS-authenticated) admin listener, if enabled and not
 	// specified otherwise.
 	DefaultRemoteAdminListen = ":2021"
-
-	// DefaultAdminConfig is the default configuration
-	// for the local administration endpoint.
-	DefaultAdminConfig = &AdminConfig{
-		Listen: DefaultAdminListen,
-	}
 )
 
 // PIDFile writes a pidfile to the file at filename. It

caddy.go

@@ -18,6 +18,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -110,14 +111,19 @@ func Load(cfgJSON []byte, forceReload bool) error {
 		}
 	}()
 
-	return changeConfig(http.MethodPost, "/"+rawConfigKey, cfgJSON, forceReload)
+	err := changeConfig(http.MethodPost, "/"+rawConfigKey, cfgJSON, forceReload)
+	if errors.Is(err, errSameConfig) {
+		err = nil // not really an error
+	}
+	return err
 }
 
 // changeConfig changes the current config (rawCfg) according to the
 // method, traversed via the given path, and uses the given input as
 // the new value (if applicable; i.e. "DELETE" doesn't have an input).
 // If the resulting config is the same as the previous, no reload will
-// occur unless forceReload is true. This function is safe for
+// occur unless forceReload is true. If the config is unchanged and not
+// forcefully reloaded, then errConfigUnchanged This function is safe for
 // concurrent use.
 func changeConfig(method, path string, input []byte, forceReload bool) error {
 	switch method {
@@ -148,8 +154,8 @@ func changeConfig(method, path string, input []byte, forceReload bool) error {
 
 	// if nothing changed, no need to do a whole reload unless the client forces it
 	if !forceReload && bytes.Equal(rawCfgJSON, newCfg) {
-		Log().Named("admin.api").Info("config is unchanged")
-		return nil
+		Log().Info("config is unchanged")
+		return errSameConfig
 	}
 
 	// find any IDs in this config and index them
@@ -268,8 +274,8 @@ func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
 		newCfg.Admin != nil &&
 		newCfg.Admin.Config != nil &&
 		newCfg.Admin.Config.LoadRaw != nil &&
-		newCfg.Admin.Config.LoadInterval <= 0 {
-		return fmt.Errorf("recursive config loading detected: pulled configs cannot pull other configs without positive load_interval")
+		newCfg.Admin.Config.LoadDelay <= 0 {
+		return fmt.Errorf("recursive config loading detected: pulled configs cannot pull other configs without positive load_delay")
 	}
 
 	// run the new config and start all its apps
@@ -427,9 +433,16 @@ func run(newCfg *Config, start bool) error {
 		return nil
 	}
 
+	// Provision any admin routers which may need to access
+	// some of the other apps at runtime
+	err = newCfg.Admin.provisionAdminRouters(ctx)
+	if err != nil {
+		return err
+	}
+
 	// Start
 	err = func() error {
-		var started []string
+		started := make([]string, 0, len(newCfg.apps))
 		for name, a := range newCfg.apps {
 			err := a.Start()
 			if err != nil {
@@ -480,49 +493,74 @@ func finishSettingUp(ctx Context, cfg *Config) error {
 		if err != nil {
 			return fmt.Errorf("loading config loader module: %s", err)
 		}
-		runLoadedConfig := func(config []byte) {
-			Log().Info("applying dynamically-loaded config", zap.String("loader_module", val.(Module).CaddyModule().ID.Name()), zap.Int("pull_interval", int(cfg.Admin.Config.LoadInterval)))
-			currentCfgMu.Lock()
-			err := unsyncedDecodeAndRun(config, false)
-			currentCfgMu.Unlock()
-			if err == nil {
-				Log().Info("dynamically-loaded config applied successfully")
-			} else {
-				Log().Error("running dynamically-loaded config failed", zap.Error(err))
+
+		logger := Log().Named("config_loader").With(
+			zap.String("module", val.(Module).CaddyModule().ID.Name()),
+			zap.Int("load_delay", int(cfg.Admin.Config.LoadDelay)))
+
+		runLoadedConfig := func(config []byte) error {
+			logger.Info("applying dynamically-loaded config")
+			err := changeConfig(http.MethodPost, "/"+rawConfigKey, config, false)
+			if errors.Is(err, errSameConfig) {
+				return err
 			}
+			if err != nil {
+				logger.Error("failed to run dynamically-loaded config", zap.Error(err))
+				return err
+			}
+			logger.Info("successfully applied dynamically-loaded config")
+			return nil
 		}
-		if cfg.Admin.Config.LoadInterval > 0 {
+
+		if cfg.Admin.Config.LoadDelay > 0 {
 			go func() {
-				select {
-				// if LoadInterval is positive, will wait for the interval and then run with new config
-				case <-time.After(time.Duration(cfg.Admin.Config.LoadInterval)):
-					loadedConfig, err := val.(ConfigLoader).LoadConfig(ctx)
-					if err != nil {
-						Log().Error("loading dynamic config failed", zap.Error(err))
-						return
+				// the loop is here to iterate ONLY if there is an error, a no-op config load,
+				// or an unchanged config; in which case we simply wait the delay and try again
+				for {
+					timer := time.NewTimer(time.Duration(cfg.Admin.Config.LoadDelay))
+					select {
+					case <-timer.C:
+						loadedConfig, err := val.(ConfigLoader).LoadConfig(ctx)
+						if err != nil {
+							logger.Error("failed loading dynamic config; will retry", zap.Error(err))
+							continue
+						}
+						if loadedConfig == nil {
+							logger.Info("dynamically-loaded config was nil; will retry")
+							continue
+						}
+						err = runLoadedConfig(loadedConfig)
+						if errors.Is(err, errSameConfig) {
+							logger.Info("dynamically-loaded config was unchanged; will retry")
+							continue
+						}
+					case <-ctx.Done():
+						if !timer.Stop() {
+							<-timer.C
+						}
+						logger.Info("stopping dynamic config loading")
 					}
-					runLoadedConfig(loadedConfig)
-				case <-ctx.Done():
-					return
+					break
 				}
 			}()
 		} else {
-			// if no LoadInterval is provided, will load config synchronously
+			// if no LoadDelay is provided, will load config synchronously
 			loadedConfig, err := val.(ConfigLoader).LoadConfig(ctx)
 			if err != nil {
 				return fmt.Errorf("loading dynamic config from %T: %v", val, err)
 			}
 			// do this in a goroutine so current config can finish being loaded; otherwise deadlock
-			go runLoadedConfig(loadedConfig)
+			go func() { _ = runLoadedConfig(loadedConfig) }()
 		}
-
 	}
 
 	return nil
 }
 
-// ConfigLoader is a type that can load a Caddy config. The
-// returned config must be valid Caddy JSON.
+// ConfigLoader is a type that can load a Caddy config. If
+// the return value is non-nil, it must be valid Caddy JSON;
+// if nil or with non-nil error, it is considered to be a
+// no-op load and may be retried later.
 type ConfigLoader interface {
 	LoadConfig(Context) ([]byte, error)
 }
@@ -583,7 +621,7 @@ func Validate(cfg *Config) error {
 // PID file, and shuts down admin endpoint(s) in a goroutine.
 // Errors are logged along the way, and an appropriate exit
 // code is emitted.
-func exitProcess(logger *zap.Logger) {
+func exitProcess(ctx context.Context, logger *zap.Logger) {
 	if logger == nil {
 		logger = Log()
 	}
@@ -598,7 +636,7 @@ func exitProcess(logger *zap.Logger) {
 	}
 
 	// clean up certmagic locks
-	certmagic.CleanUpOwnLocks(logger)
+	certmagic.CleanUpOwnLocks(ctx, logger)
 
 	// remove pidfile
 	if pidfile != "" {
@@ -774,5 +812,10 @@ var (
 	rawCfgIndex map[string]string
 )
 
+// errSameConfig is returned if the new config is the same
+// as the old one. This isn't usually an actual, actionable
+// error; it's mostly a sentinel value.
+var errSameConfig = errors.New("config is unchanged")
+
 // ImportPath is the package import path for Caddy core.
 const ImportPath = "github.com/caddyserver/caddy/v2"

caddyconfig/caddyfile/adapter.go

@@ -88,7 +88,7 @@ func formattingDifference(filename string, body []byte) (caddyconfig.Warning, bo
 	return caddyconfig.Warning{
 		File:    filename,
 		Line:    line,
-		Message: "input is not formatted with 'caddy fmt'",
+		Message: "Caddyfile input is not formatted; run the 'caddy fmt' command to fix inconsistencies",
 	}, true
 }
 

caddyconfig/caddyfile/dispenser.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"strconv"
 	"strings"
 )
 
@@ -201,6 +202,43 @@ func (d *Dispenser) Val() string {
 	return d.tokens[d.cursor].Text
 }
 
+// ValRaw gets the raw text of the current token (including quotes).
+// If there is no token loaded, it returns empty string.
+func (d *Dispenser) ValRaw() string {
+	if d.cursor < 0 || d.cursor >= len(d.tokens) {
+		return ""
+	}
+	quote := d.tokens[d.cursor].wasQuoted
+	if quote > 0 {
+		return string(quote) + d.tokens[d.cursor].Text + string(quote) // string literal
+	}
+	return d.tokens[d.cursor].Text
+}
+
+// ScalarVal gets value of the current token, converted to the closest
+// scalar type. If there is no token loaded, it returns nil.
+func (d *Dispenser) ScalarVal() interface{} {
+	if d.cursor < 0 || d.cursor >= len(d.tokens) {
+		return nil
+	}
+	quote := d.tokens[d.cursor].wasQuoted
+	text := d.tokens[d.cursor].Text
+
+	if quote > 0 {
+		return text // string literal
+	}
+	if num, err := strconv.Atoi(text); err == nil {
+		return num
+	}
+	if num, err := strconv.ParseFloat(text, 64); err == nil {
+		return num
+	}
+	if bool, err := strconv.ParseBool(text); err == nil {
+		return bool
+	}
+	return text
+}
+
 // Line gets the line number of the current token.
 // If there is no token loaded, it returns 0.
 func (d *Dispenser) Line() int {
@@ -249,6 +287,19 @@ func (d *Dispenser) AllArgs(targets ...*string) bool {
 	return true
 }
 
+// CountRemainingArgs counts the amount of remaining arguments
+// (tokens on the same line) without consuming the tokens.
+func (d *Dispenser) CountRemainingArgs() int {
+	count := 0
+	for d.NextArg() {
+		count++
+	}
+	for i := 0; i < count; i++ {
+		d.Prev()
+	}
+	return count
+}
+
 // RemainingArgs loads any more arguments (tokens on the same line)
 // into a slice and returns them. Open curly brace tokens also indicate
 // the end of arguments, and the curly brace is not included in
@@ -261,6 +312,18 @@ func (d *Dispenser) RemainingArgs() []string {
 	return args
 }
 
+// RemainingArgsRaw loads any more arguments (tokens on the same line,
+// retaining quotes) into a slice and returns them. Open curly brace
+// tokens also indicate the end of arguments, and the curly brace is
+// not included in the return value nor is it loaded.
+func (d *Dispenser) RemainingArgsRaw() []string {
+	var args []string
+	for d.NextArg() {
+		args = append(args, d.ValRaw())
+	}
+	return args
+}
+
 // NewFromNextSegment returns a new dispenser with a copy of
 // the tokens from the current token until the end of the
 // "directive" whether that be to the end of the line or
@@ -350,7 +413,11 @@ func (d *Dispenser) Err(msg string) error {
 
 // Errf is like Err, but for formatted error messages
 func (d *Dispenser) Errf(format string, args ...interface{}) error {
-	err := fmt.Errorf(format, args...)
+	return d.WrapErr(fmt.Errorf(format, args...))
+}
+
+// WrapErr takes an existing error and adds the Caddyfile file and line number.
+func (d *Dispenser) WrapErr(err error) error {
 	return fmt.Errorf("%s:%d - Error during parsing: %w", d.File(), d.Line(), err)
 }
 
@@ -391,6 +458,60 @@ func (d *Dispenser) isNewLine() bool {
 	if d.cursor > len(d.tokens)-1 {
 		return false
 	}
-	return d.tokens[d.cursor-1].File != d.tokens[d.cursor].File ||
-		d.tokens[d.cursor-1].Line+d.numLineBreaks(d.cursor-1) < d.tokens[d.cursor].Line
+
+	prev := d.tokens[d.cursor-1]
+	curr := d.tokens[d.cursor]
+
+	// If the previous token is from a different file,
+	// we can assume it's from a different line
+	if prev.File != curr.File {
+		return true
+	}
+
+	// The previous token may contain line breaks if
+	// it was quoted and spanned multiple lines. e.g:
+	//
+	// dir "foo
+	//   bar
+	//   baz"
+	prevLineBreaks := d.numLineBreaks(d.cursor - 1)
+
+	// If the previous token (incl line breaks) ends
+	// on a line earlier than the current token,
+	// then the current token is on a new line
+	return prev.Line+prevLineBreaks < curr.Line
+}
+
+// isNextOnNewLine determines whether the current token is on a different
+// line (higher line number) than the next token. It handles imported
+// tokens correctly. If there isn't a next token, it returns true.
+func (d *Dispenser) isNextOnNewLine() bool {
+	if d.cursor < 0 {
+		return false
+	}
+	if d.cursor >= len(d.tokens)-1 {
+		return true
+	}
+
+	curr := d.tokens[d.cursor]
+	next := d.tokens[d.cursor+1]
+
+	// If the next token is from a different file,
+	// we can assume it's from a different line
+	if curr.File != next.File {
+		return true
+	}
+
+	// The current token may contain line breaks if
+	// it was quoted and spanned multiple lines. e.g:
+	//
+	// dir "foo
+	//   bar
+	//   baz"
+	currLineBreaks := d.numLineBreaks(d.cursor)
+
+	// If the current token (incl line breaks) ends
+	// on a line earlier than the next token,
+	// then the next token is on a new line
+	return curr.Line+currLineBreaks < next.Line
 }

caddyconfig/caddyfile/formatter_fuzz.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build gofuzz
 // +build gofuzz
 
 package caddyfile

caddyconfig/caddyfile/formatter_test.go

@@ -179,6 +179,11 @@ d {
 	{$F}
 }`,
 		},
+		{
+			description: "env var placeholders with port",
+			input:       `:{$PORT}`,
+			expect:      `:{$PORT}`,
+		},
 		{
 			description: "comments",
 			input: `#a "\n"

caddyconfig/caddyfile/lexer.go

@@ -38,6 +38,7 @@ type (
 		File        string
 		Line        int
 		Text        string
+		wasQuoted   rune // enclosing quote character, if any
 		inSnippet   bool
 		snippetName string
 	}
@@ -78,8 +79,9 @@ func (l *lexer) next() bool {
 	var val []rune
 	var comment, quoted, btQuoted, escaped bool
 
-	makeToken := func() bool {
+	makeToken := func(quoted rune) bool {
 		l.token.Text = string(val)
+		l.token.wasQuoted = quoted
 		return true
 	}
 
@@ -87,7 +89,7 @@ func (l *lexer) next() bool {
 		ch, _, err := l.reader.ReadRune()
 		if err != nil {
 			if len(val) > 0 {
-				return makeToken()
+				return makeToken(0)
 			}
 			if err == io.EOF {
 				return false
@@ -110,10 +112,10 @@ func (l *lexer) next() bool {
 				escaped = false
 			} else {
 				if quoted && ch == '"' {
-					return makeToken()
+					return makeToken('"')
 				}
 				if btQuoted && ch == '`' {
-					return makeToken()
+					return makeToken('`')
 				}
 			}
 			if ch == '\n' {
@@ -139,7 +141,7 @@ func (l *lexer) next() bool {
 				comment = false
 			}
 			if len(val) > 0 {
-				return makeToken()
+				return makeToken(0)
 			}
 			continue
 		}

caddyconfig/caddyfile/lexer_fuzz.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build gofuzz
 // +build gofuzz
 
 package caddyfile

caddyconfig/caddyfile/parse.go

@@ -18,13 +18,13 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/caddyserver/caddy/v2"
+	"go.uber.org/zap"
 )
 
 // Parse parses the input just enough to group tokens, in
@@ -37,7 +37,13 @@ import (
 // Environment variables in {$ENVIRONMENT_VARIABLE} notation
 // will be replaced before parsing begins.
 func Parse(filename string, input []byte) ([]ServerBlock, error) {
-	tokens, err := allTokens(filename, input)
+	// unfortunately, we must copy the input because parsing must
+	// remain a read-only operation, but we have to expand environment
+	// variables before we parse, which changes the underlying array (#4422)
+	inputCopy := make([]byte, len(input))
+	copy(inputCopy, input)
+
+	tokens, err := allTokens(filename, inputCopy)
 	if err != nil {
 		return nil, err
 	}
@@ -51,7 +57,23 @@ func Parse(filename string, input []byte) ([]ServerBlock, error) {
 	return p.parseAll()
 }
 
+// allTokens lexes the entire input, but does not parse it.
+// It returns all the tokens from the input, unstructured
+// and in order. It may mutate input as it expands env vars.
+func allTokens(filename string, input []byte) ([]Token, error) {
+	inputCopy, err := replaceEnvVars(input)
+	if err != nil {
+		return nil, err
+	}
+	tokens, err := Tokenize(inputCopy, filename)
+	if err != nil {
+		return nil, err
+	}
+	return tokens, nil
+}
+
 // replaceEnvVars replaces all occurrences of environment variables.
+// It mutates the underlying array and returns the updated slice.
 func replaceEnvVars(input []byte) ([]byte, error) {
 	var offset int
 	for {
@@ -96,21 +118,6 @@ func replaceEnvVars(input []byte) ([]byte, error) {
 	return input, nil
 }
 
-// allTokens lexes the entire input, but does not parse it.
-// It returns all the tokens from the input, unstructured
-// and in order.
-func allTokens(filename string, input []byte) ([]Token, error) {
-	input, err := replaceEnvVars(input)
-	if err != nil {
-		return nil, err
-	}
-	tokens, err := Tokenize(input, filename)
-	if err != nil {
-		return nil, err
-	}
-	return tokens, nil
-}
-
 type parser struct {
 	*Dispenser
 	block           ServerBlock // current server block being parsed
@@ -386,7 +393,7 @@ func (p *parser) doImport() error {
 		}
 		if len(matches) == 0 {
 			if strings.ContainsAny(globPattern, "*?[]") {
-				log.Printf("[WARNING] No files matching import glob pattern: %s", importPattern)
+				caddy.Log().Warn("No files matching import glob pattern", zap.String("pattern", importPattern))
 			} else {
 				return p.Errf("File to import not found: %s", importPattern)
 			}
@@ -487,6 +494,13 @@ func (p *parser) directive() error {
 	for p.Next() {
 		if p.Val() == "{" {
 			p.nesting++
+			if !p.isNextOnNewLine() && p.Token().wasQuoted == 0 {
+				return p.Err("Unexpected next token after '{' on same line")
+			}
+		} else if p.Val() == "{}" {
+			if p.isNextOnNewLine() && p.Token().wasQuoted == 0 {
+				return p.Err("Unexpected '{}' at end of line")
+			}
 		} else if p.isNewLine() && p.nesting == 0 {
 			p.cursor-- // read too far
 			break

caddyconfig/caddyfile/parse_test.go

@@ -191,6 +191,20 @@ func TestParseOneAndImport(t *testing.T) {
 
 		{``, false, []string{}, []int{}},
 
+		// Unexpected next token after '{' on same line
+		{`localhost
+		  dir1 { a b }`, true, []string{"localhost"}, []int{}},
+		// Workaround with quotes
+		{`localhost
+		  dir1 "{" a b "}"`, false, []string{"localhost"}, []int{5}},
+
+		// Unexpected '{}' at end of line
+		{`localhost
+		  dir1 {}`, true, []string{"localhost"}, []int{}},
+		// Workaround with quotes
+		{`localhost
+		  dir1 "{}"`, false, []string{"localhost"}, []int{2}},
+
 		// import with args
 		{`import testdata/import_args0.txt a`, false, []string{"a"}, []int{}},
 		{`import testdata/import_args1.txt a b`, false, []string{"a", "b"}, []int{}},

caddyconfig/httpcaddyfile/addresses.go

@@ -102,12 +102,20 @@ func (st *ServerType) mapAddressToServerBlocks(originalServerBlocks []serverBloc
 			}
 		}
 
+		// make a slice of the map keys so we can iterate in sorted order
+		addrs := make([]string, 0, len(addrToKeys))
+		for k := range addrToKeys {
+			addrs = append(addrs, k)
+		}
+		sort.Strings(addrs)
+
 		// now that we know which addresses serve which keys of this
 		// server block, we iterate that mapping and create a list of
 		// new server blocks for each address where the keys of the
 		// server block are only the ones which use the address; but
 		// the contents (tokens) are of course the same
-		for addr, keys := range addrToKeys {
+		for _, addr := range addrs {
+			keys := addrToKeys[addr]
 			// parse keys so that we only have to do it once
 			parsedKeys := make([]Address, 0, len(keys))
 			for _, key := range keys {
@@ -161,6 +169,7 @@ func (st *ServerType) consolidateAddrMappings(addrToServerBlocks map[string][]se
 				delete(addrToServerBlocks, otherAddr)
 			}
 		}
+		sort.Strings(a.addresses)
 
 		sbaddrs = append(sbaddrs, a)
 	}
@@ -208,12 +217,16 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 	}
 
 	// the bind directive specifies hosts, but is optional
-	lnHosts := make([]string, 0, len(sblock.pile))
+	lnHosts := make([]string, 0, len(sblock.pile["bind"]))
 	for _, cfgVal := range sblock.pile["bind"] {
 		lnHosts = append(lnHosts, cfgVal.Value.([]string)...)
 	}
 	if len(lnHosts) == 0 {
-		lnHosts = []string{""}
+		if defaultBind, ok := options["default_bind"].([]string); ok {
+			lnHosts = defaultBind
+		} else {
+			lnHosts = []string{""}
+		}
 	}
 
 	// use a map to prevent duplication
@@ -223,7 +236,7 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 		if err == nil && addr.IsUnixNetwork() {
 			listeners[host] = struct{}{}
 		} else {
-			listeners[net.JoinHostPort(host, lnPort)] = struct{}{}
+			listeners[host+":"+lnPort] = struct{}{}
 		}
 	}
 
@@ -232,6 +245,7 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 	for lnStr := range listeners {
 		listenersList = append(listenersList, lnStr)
 	}
+	sort.Strings(listenersList)
 
 	return listenersList, nil
 }

caddyconfig/httpcaddyfile/addresses_fuzz.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build gofuzz
 // +build gofuzz
 
 package httpcaddyfile

caddyconfig/httpcaddyfile/builtins.go

@@ -39,6 +39,7 @@ func init() {
 	RegisterDirective("bind", parseBind)
 	RegisterDirective("tls", parseTLS)
 	RegisterHandlerDirective("root", parseRoot)
+	RegisterHandlerDirective("vars", parseVars)
 	RegisterHandlerDirective("redir", parseRedir)
 	RegisterHandlerDirective("respond", parseRespond)
 	RegisterHandlerDirective("abort", parseAbort)
@@ -82,6 +83,7 @@ func parseBind(h Helper) ([]ConfigValue, error) {
 //         on_demand
 //         eab    <key_id> <mac_key>
 //         issuer <module_name> [...]
+//         get_certificate <module_name> [...]
 //     }
 //
 func parseTLS(h Helper) ([]ConfigValue, error) {
@@ -93,6 +95,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 	var keyType string
 	var internalIssuer *caddytls.InternalIssuer
 	var issuers []certmagic.Issuer
+	var certManagers []certmagic.Manager
 	var onDemand bool
 
 	for h.Next() {
@@ -307,6 +310,22 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 				}
 				issuers = append(issuers, issuer)
 
+			case "get_certificate":
+				if !h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				modName := h.Val()
+				modID := "tls.get_certificate." + modName
+				unm, err := caddyfile.UnmarshalModule(h.Dispenser, modID)
+				if err != nil {
+					return nil, err
+				}
+				certManager, ok := unm.(certmagic.Manager)
+				if !ok {
+					return nil, h.Errf("module %s (%T) is not a certmagic.CertificateManager", modID, unm)
+				}
+				certManagers = append(certManagers, certManager)
+
 			case "dns":
 				if !h.NextArg() {
 					return nil, h.ArgErr()
@@ -344,6 +363,22 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 				}
 				acmeIssuer.Challenges.DNS.Resolvers = args
 
+			case "dns_challenge_override_domain":
+				arg := h.RemainingArgs()
+				if len(arg) != 1 {
+					return nil, h.ArgErr()
+				}
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
+				if acmeIssuer.Challenges == nil {
+					acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
+				}
+				if acmeIssuer.Challenges.DNS == nil {
+					acmeIssuer.Challenges.DNS = new(caddytls.DNSChallengeConfig)
+				}
+				acmeIssuer.Challenges.DNS.OverrideDomain = arg[0]
+
 			case "ca_root":
 				arg := h.RemainingArgs()
 				if len(arg) != 1 {
@@ -453,6 +488,12 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 			Value: true,
 		})
 	}
+	for _, certManager := range certManagers {
+		configVals = append(configVals, ConfigValue{
+			Class: "tls.cert_manager",
+			Value: certManager,
+		})
+	}
 
 	// custom certificate selection
 	if len(certSelector.AnyTag) > 0 {
@@ -490,6 +531,13 @@ func parseRoot(h Helper) (caddyhttp.MiddlewareHandler, error) {
 	return caddyhttp.VarsMiddleware{"root": root}, nil
 }
 
+// parseVars parses the vars directive. See its UnmarshalCaddyfile method for syntax.
+func parseVars(h Helper) (caddyhttp.MiddlewareHandler, error) {
+	v := new(caddyhttp.VarsMiddleware)
+	err := v.UnmarshalCaddyfile(h.Dispenser)
+	return v, err
+}
+
 // parseRedir parses the redir directive. Syntax:
 //
 //     redir [<matcher>] <to> [<code>]
@@ -532,12 +580,24 @@ func parseRedir(h Helper) (caddyhttp.MiddlewareHandler, error) {
 		body = fmt.Sprintf(metaRedir, safeTo, safeTo, safeTo, safeTo)
 		code = "302"
 	default:
+		// Allow placeholders for the code
+		if strings.HasPrefix(code, "{") {
+			break
+		}
+		// Try to validate as an integer otherwise
 		codeInt, err := strconv.Atoi(code)
 		if err != nil {
 			return nil, h.Errf("Not a supported redir code type or not valid integer: '%s'", code)
 		}
-		if codeInt < 300 || codeInt > 399 {
-			return nil, h.Errf("Redir code not in the 3xx range: '%v'", codeInt)
+		// Sometimes, a 401 with Location header is desirable because
+		// requests made with XHR will "eat" the 3xx redirect; so if
+		// the intent was to redirect to an auth page, a 3xx won't
+		// work. Responding with 401 allows JS code to read the
+		// Location header and do a window.location redirect manually.
+		// see https://stackoverflow.com/a/2573589/846934
+		// see https://github.com/oauth2-proxy/oauth2-proxy/issues/1522
+		if codeInt < 300 || (codeInt > 399 && codeInt != 401) {
+			return nil, h.Errf("Redir code not in the 3xx range or 401: '%v'", codeInt)
 		}
 	}
 

caddyconfig/httpcaddyfile/builtins_test.go

@@ -37,8 +37,7 @@ func TestLogDirectiveSyntax(t *testing.T) {
 					format filter {
 						wrap console
 						fields {
-							common_log delete
-							request>remote_addr ip_mask {
+							request>remote_ip ip_mask {
 								ipv4 24
 								ipv6 32
 							}
@@ -47,7 +46,7 @@ func TestLogDirectiveSyntax(t *testing.T) {
 				}
 			}
 			`,
-			output:      `{"logging":{"logs":{"default":{"exclude":["http.log.access.log0"]},"log0":{"encoder":{"fields":{"common_log":{"filter":"delete"},"request\u003eremote_addr":{"filter":"ip_mask","ipv4_cidr":24,"ipv6_cidr":32}},"format":"filter","wrap":{"format":"console"}},"include":["http.log.access.log0"]}}},"apps":{"http":{"servers":{"srv0":{"listen":[":8080"],"logs":{"default_logger_name":"log0"}}}}}}`,
+			output:      `{"logging":{"logs":{"default":{"exclude":["http.log.access.log0"]},"log0":{"encoder":{"fields":{"request\u003eremote_ip":{"filter":"ip_mask","ipv4_cidr":24,"ipv6_cidr":32}},"format":"filter","wrap":{"format":"console"}},"include":["http.log.access.log0"]}}},"apps":{"http":{"servers":{"srv0":{"listen":[":8080"],"logs":{"default_logger_name":"log0"}}}}}}`,
 			expectError: false,
 		},
 		{
@@ -149,6 +148,27 @@ func TestRedirDirectiveSyntax(t *testing.T) {
 			}`,
 			expectError: false,
 		},
+		{
+			// this is now allowed so a Location header
+			// can be written and consumed by JS
+			// in the case of XHR requests
+			input: `:8080 {
+				redir * :8081 401
+			}`,
+			expectError: false,
+		},
+		{
+			input: `:8080 {
+				redir * :8081 402
+			}`,
+			expectError: true,
+		},
+		{
+			input: `:8080 {
+				redir * :8081 {http.reverse_proxy.status_code}
+			}`,
+			expectError: false,
+		},
 		{
 			input: `:8080 {
 				redir /old.html /new.html htlm
@@ -161,12 +181,6 @@ func TestRedirDirectiveSyntax(t *testing.T) {
 			}`,
 			expectError: true,
 		},
-		{
-			input: `:8080 {
-				redir * :8081 400
-			}`,
-			expectError: true,
-		},
 		{
 			input: `:8080 {
 				redir * :8081 temp

caddyconfig/httpcaddyfile/directives.go

@@ -37,21 +37,27 @@ import (
 // The header directive goes second so that headers
 // can be manipulated before doing redirects.
 var directiveOrder = []string{
+	"tracing",
+
 	"map",
+	"vars",
 	"root",
 
 	"header",
+	"copy_response_headers", // only in reverse_proxy's handle_response
 	"request_body",
 
 	"redir",
 
-	// URI manipulation
+	// incoming request manipulation
+	"method",
 	"rewrite",
 	"uri",
 	"try_files",
 
 	// middleware handlers; some wrap responses
 	"basicauth",
+	"forward_auth",
 	"request_header",
 	"encode",
 	"push",
@@ -65,6 +71,7 @@ var directiveOrder = []string{
 	// handlers that typically respond to requests
 	"abort",
 	"error",
+	"copy_response", // only in reverse_proxy's handle_response
 	"respond",
 	"metrics",
 	"reverse_proxy",
@@ -340,6 +347,9 @@ func parseSegmentAsConfig(h Helper) ([]ConfigValue, error) {
 			if err != nil {
 				return nil, h.Errf("parsing caddyfile tokens for '%s': %v", dir, err)
 			}
+
+			dir = normalizeDirectiveName(dir)
+
 			for _, result := range results {
 				result.directive = dir
 				allResults = append(allResults, result)
@@ -415,14 +425,29 @@ func sortRoutes(routes []ConfigValue) {
 			jPathLen = len(jPM[0])
 		}
 
-		// if both directives have no path matcher, use whichever one
-		// has any kind of matcher defined first.
-		if iPathLen == 0 && jPathLen == 0 {
-			return len(iRoute.MatcherSetsRaw) > 0 && len(jRoute.MatcherSetsRaw) == 0
-		}
+		// some directives involve setting values which can overwrite
+		// eachother, so it makes most sense to reverse the order so
+		// that the lease specific matcher is first; everything else
+		// has most-specific matcher first
+		if iDir == "vars" {
+			// if both directives have no path matcher, use whichever one
+			// has no matcher first.
+			if iPathLen == 0 && jPathLen == 0 {
+				return len(iRoute.MatcherSetsRaw) == 0 && len(jRoute.MatcherSetsRaw) > 0
+			}
 
-		// sort with the most-specific (longest) path first
-		return iPathLen > jPathLen
+			// sort with the least-specific (shortest) path first
+			return iPathLen < jPathLen
+		} else {
+			// if both directives have no path matcher, use whichever one
+			// has any kind of matcher defined first.
+			if iPathLen == 0 && jPathLen == 0 {
+				return len(iRoute.MatcherSetsRaw) > 0 && len(jRoute.MatcherSetsRaw) == 0
+			}
+
+			// sort with the most-specific (longest) path first
+			return iPathLen > jPathLen
+		}
 	})
 }
 
@@ -485,7 +510,7 @@ func (sb serverBlock) hostsFromKeysNotHTTP(httpPort string) []string {
 		if addr.Host == "" {
 			continue
 		}
-		if addr.Scheme != "http" || addr.Port != httpPort {
+		if addr.Scheme != "http" && addr.Port != httpPort {
 			hostMap[addr.Host] = struct{}{}
 		}
 	}
@@ -510,6 +535,17 @@ func (sb serverBlock) hasHostCatchAllKey() bool {
 	return false
 }
 
+// isAllHTTP returns true if all sb keys explicitly specify
+// the http:// scheme
+func (sb serverBlock) isAllHTTP() bool {
+	for _, addr := range sb.keys {
+		if addr.Scheme != "http" {
+			return false
+		}
+	}
+	return true
+}
+
 type (
 	// UnmarshalFunc is a function which can unmarshal Caddyfile
 	// tokens into zero or more config values using a Helper type.

caddyconfig/httpcaddyfile/httptype.go

@@ -17,7 +17,6 @@ package httpcaddyfile
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"reflect"
 	"regexp"
 	"sort"
@@ -30,6 +29,7 @@ import (
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddypki"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
+	"go.uber.org/zap"
 )
 
 func init() {
@@ -88,34 +88,10 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		return nil, warnings, err
 	}
 
-	// replace shorthand placeholders (which are
-	// convenient when writing a Caddyfile) with
-	// their actual placeholder identifiers or
-	// variable names
-	replacer := strings.NewReplacer(
-		"{dir}", "{http.request.uri.path.dir}",
-		"{file}", "{http.request.uri.path.file}",
-		"{host}", "{http.request.host}",
-		"{hostport}", "{http.request.hostport}",
-		"{port}", "{http.request.port}",
-		"{method}", "{http.request.method}",
-		"{path}", "{http.request.uri.path}",
-		"{query}", "{http.request.uri.query}",
-		"{remote}", "{http.request.remote}",
-		"{remote_host}", "{http.request.remote.host}",
-		"{remote_port}", "{http.request.remote.port}",
-		"{scheme}", "{http.request.scheme}",
-		"{uri}", "{http.request.uri}",
-		"{tls_cipher}", "{http.request.tls.cipher_suite}",
-		"{tls_version}", "{http.request.tls.version}",
-		"{tls_client_fingerprint}", "{http.request.tls.client.fingerprint}",
-		"{tls_client_issuer}", "{http.request.tls.client.issuer}",
-		"{tls_client_serial}", "{http.request.tls.client.serial}",
-		"{tls_client_subject}", "{http.request.tls.client.subject}",
-		"{tls_client_certificate_pem}", "{http.request.tls.client.certificate_pem}",
-		"{tls_client_certificate_der_base64}", "{http.request.tls.client.certificate_der_base64}",
-		"{upstream_hostport}", "{http.reverse_proxy.upstream.hostport}",
-	)
+	// replace shorthand placeholders (which are convenient
+	// when writing a Caddyfile) with their actual placeholder
+	// identifiers or variable names
+	replacer := strings.NewReplacer(placeholderShorthands()...)
 
 	// these are placeholders that allow a user-defined final
 	// parameters, but we still want to provide a shorthand
@@ -129,6 +105,9 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		{regexp.MustCompile(`{header\.([\w-]*)}`), "{http.request.header.$1}"},
 		{regexp.MustCompile(`{path\.([\w-]*)}`), "{http.request.uri.path.$1}"},
 		{regexp.MustCompile(`{re\.([\w-]*)\.([\w-]*)}`), "{http.regexp.$1.$2}"},
+		{regexp.MustCompile(`{vars\.([\w-]*)}`), "{http.vars.$1}"},
+		{regexp.MustCompile(`{rp\.([\w-\.]*)}`), "{http.reverse_proxy.$1}"},
+		{regexp.MustCompile(`{err\.([\w-\.]*)}`), "{http.error.$1}"},
 	}
 
 	for _, sb := range originalServerBlocks {
@@ -193,13 +172,7 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 				return nil, warnings, fmt.Errorf("parsing caddyfile tokens for '%s': %v", dir, err)
 			}
 
-			// As a special case, we want "handle_path" to be sorted
-			// at the same level as "handle", so we force them to use
-			// the same directive name after their parsing is complete.
-			// See https://github.com/caddyserver/caddy/issues/3675#issuecomment-678042377
-			if dir == "handle_path" {
-				dir = "handle"
-			}
+			dir = normalizeDirectiveName(dir)
 
 			for _, result := range results {
 				result.directive = dir
@@ -259,20 +232,13 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		}
 		customLogs = append(customLogs, ncl)
 	}
+
 	// Apply global log options, when set
 	if options["log"] != nil {
 		for _, logValue := range options["log"].([]ConfigValue) {
 			addCustomLog(logValue.Value.(namedCustomLog))
 		}
 	}
-	// Apply server-specific log options
-	for _, p := range pairings {
-		for _, sb := range p.serverBlocks {
-			for _, clVal := range sb.pile["custom_log"] {
-				addCustomLog(clVal.Value.(namedCustomLog))
-			}
-		}
-	}
 
 	if !hasDefaultLog {
 		// if the default log was not customized, ensure we
@@ -285,6 +251,15 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		}
 	}
 
+	// Apply server-specific log options
+	for _, p := range pairings {
+		for _, sb := range p.serverBlocks {
+			for _, clVal := range sb.pile["custom_log"] {
+				addCustomLog(clVal.Value.(namedCustomLog))
+			}
+		}
+	}
+
 	// annnd the top-level config, then we're done!
 	cfg := &caddy.Config{AppsRaw: make(caddy.ModuleMap)}
 
@@ -452,17 +427,29 @@ func (st *ServerType) serversFromPairings(
 		// handle the auto_https global option
 		if autoHTTPS != "on" {
 			srv.AutoHTTPS = new(caddyhttp.AutoHTTPSConfig)
-			if autoHTTPS == "off" {
+			switch autoHTTPS {
+			case "off":
 				srv.AutoHTTPS.Disabled = true
-			}
-			if autoHTTPS == "disable_redirects" {
+			case "disable_redirects":
 				srv.AutoHTTPS.DisableRedir = true
-			}
-			if autoHTTPS == "ignore_loaded_certs" {
+			case "disable_certs":
+				srv.AutoHTTPS.DisableCerts = true
+			case "ignore_loaded_certs":
 				srv.AutoHTTPS.IgnoreLoadedCerts = true
 			}
 		}
 
+		// Using paths in site addresses is deprecated
+		// See ParseAddress() where parsing should later reject paths
+		// See https://github.com/caddyserver/caddy/pull/4728 for a full explanation
+		for _, sblock := range p.serverBlocks {
+			for _, addr := range sblock.keys {
+				if addr.Path != "" {
+					caddy.Log().Named("caddyfile").Warn("Using a path in a site address is deprecated; please use the 'handle' directive instead", zap.String("address", addr.String()))
+				}
+			}
+		}
+
 		// sort server blocks by their keys; this is important because
 		// only the first matching site should be evaluated, and we should
 		// attempt to match most specific site first (host and path), in
@@ -550,7 +537,7 @@ func (st *ServerType) serversFromPairings(
 			// emit warnings if user put unspecified IP addresses; they probably want the bind directive
 			for _, h := range hosts {
 				if h == "0.0.0.0" || h == "::" {
-					log.Printf("[WARNING] Site block has unspecified IP address %s which only matches requests having that Host header; you probably want the 'bind' directive to configure the socket", h)
+					caddy.Log().Named("caddyfile").Warn("Site block has an unspecified IP address which only matches requests having that Host header; you probably want the 'bind' directive to configure the socket", zap.String("address", h))
 				}
 			}
 
@@ -586,7 +573,7 @@ func (st *ServerType) serversFromPairings(
 			}
 
 			for _, addr := range sblock.keys {
-				// if server only uses HTTPS port, auto-HTTPS will not apply
+				// if server only uses HTTP port, auto-HTTPS will not apply
 				if listenersUseAnyPortOtherThan(srv.Listen, httpPort) {
 					// exclude any hosts that were defined explicitly with "http://"
 					// in the key from automated cert management (issue #2998)
@@ -1061,6 +1048,19 @@ func buildSubroute(routes []ConfigValue, groupCounter counter) (*caddyhttp.Subro
 	return subroute, nil
 }
 
+// normalizeDirectiveName ensures directives that should be sorted
+// at the same level are named the same before sorting happens.
+func normalizeDirectiveName(directive string) string {
+	// As a special case, we want "handle_path" to be sorted
+	// at the same level as "handle", so we force them to use
+	// the same directive name after their parsing is complete.
+	// See https://github.com/caddyserver/caddy/issues/3675#issuecomment-678042377
+	if directive == "handle_path" {
+		directive = "handle"
+	}
+	return directive
+}
+
 // consolidateRoutes combines routes with the same properties
 // (same matchers, same Terminal and Group settings) for a
 // cleaner overall output.
@@ -1242,6 +1242,58 @@ func encodeMatcherSet(matchers map[string]caddyhttp.RequestMatcher) (caddy.Modul
 	return msEncoded, nil
 }
 
+// placeholderShorthands returns a slice of old-new string pairs,
+// where the left of the pair is a placeholder shorthand that may
+// be used in the Caddyfile, and the right is the replacement.
+func placeholderShorthands() []string {
+	return []string{
+		"{dir}", "{http.request.uri.path.dir}",
+		"{file}", "{http.request.uri.path.file}",
+		"{host}", "{http.request.host}",
+		"{hostport}", "{http.request.hostport}",
+		"{port}", "{http.request.port}",
+		"{method}", "{http.request.method}",
+		"{path}", "{http.request.uri.path}",
+		"{query}", "{http.request.uri.query}",
+		"{remote}", "{http.request.remote}",
+		"{remote_host}", "{http.request.remote.host}",
+		"{remote_port}", "{http.request.remote.port}",
+		"{scheme}", "{http.request.scheme}",
+		"{uri}", "{http.request.uri}",
+		"{tls_cipher}", "{http.request.tls.cipher_suite}",
+		"{tls_version}", "{http.request.tls.version}",
+		"{tls_client_fingerprint}", "{http.request.tls.client.fingerprint}",
+		"{tls_client_issuer}", "{http.request.tls.client.issuer}",
+		"{tls_client_serial}", "{http.request.tls.client.serial}",
+		"{tls_client_subject}", "{http.request.tls.client.subject}",
+		"{tls_client_certificate_pem}", "{http.request.tls.client.certificate_pem}",
+		"{tls_client_certificate_der_base64}", "{http.request.tls.client.certificate_der_base64}",
+		"{upstream_hostport}", "{http.reverse_proxy.upstream.hostport}",
+	}
+}
+
+// WasReplacedPlaceholderShorthand checks if a token string was
+// likely a replaced shorthand of the known Caddyfile placeholder
+// replacement outputs. Useful to prevent some user-defined map
+// output destinations from overlapping with one of the
+// predefined shorthands.
+func WasReplacedPlaceholderShorthand(token string) string {
+	prev := ""
+	for i, item := range placeholderShorthands() {
+		// only look at every 2nd item, which is the replacement
+		if i%2 == 0 {
+			prev = item
+			continue
+		}
+		if strings.Trim(token, "{}") == strings.Trim(item, "{}") {
+			// we return the original shorthand so it
+			// can be used for an error message
+			return prev
+		}
+	}
+	return ""
+}
+
 // tryInt tries to convert val to an integer. If it fails,
 // it downgrades the error to a warning and returns 0.
 func tryInt(val interface{}, warnings *[]caddyconfig.Warning) int {

caddyconfig/httpcaddyfile/options.go

@@ -29,11 +29,13 @@ func init() {
 	RegisterGlobalOption("debug", parseOptTrue)
 	RegisterGlobalOption("http_port", parseOptHTTPPort)
 	RegisterGlobalOption("https_port", parseOptHTTPSPort)
+	RegisterGlobalOption("default_bind", parseOptStringList)
 	RegisterGlobalOption("grace_period", parseOptDuration)
 	RegisterGlobalOption("default_sni", parseOptSingleString)
 	RegisterGlobalOption("order", parseOptOrder)
 	RegisterGlobalOption("storage", parseOptStorage)
 	RegisterGlobalOption("storage_clean_interval", parseOptDuration)
+	RegisterGlobalOption("renew_interval", parseOptDuration)
 	RegisterGlobalOption("acme_ca", parseOptSingleString)
 	RegisterGlobalOption("acme_ca_root", parseOptSingleString)
 	RegisterGlobalOption("acme_dns", parseOptACMEDNS)
@@ -277,6 +279,15 @@ func parseOptSingleString(d *caddyfile.Dispenser, _ interface{}) (interface{}, e
 	return val, nil
 }
 
+func parseOptStringList(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+	d.Next() // consume parameter name
+	val := d.RemainingArgs()
+	if len(val) == 0 {
+		return "", d.ArgErr()
+	}
+	return val, nil
+}
+
 func parseOptAdmin(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
 	adminCfg := new(caddy.AdminConfig)
 	for d.Next() {
@@ -382,8 +393,8 @@ func parseOptAutoHTTPS(d *caddyfile.Dispenser, _ interface{}) (interface{}, erro
 	if d.Next() {
 		return "", d.ArgErr()
 	}
-	if val != "off" && val != "disable_redirects" && val != "ignore_loaded_certs" {
-		return "", d.Errf("auto_https must be one of 'off', 'disable_redirects' or 'ignore_loaded_certs'")
+	if val != "off" && val != "disable_redirects" && val != "disable_certs" && val != "ignore_loaded_certs" {
+		return "", d.Errf("auto_https must be one of 'off', 'disable_redirects', 'disable_certs', or 'ignore_loaded_certs'")
 	}
 	return val, nil
 }

caddyconfig/httpcaddyfile/pkiapp.go

@@ -16,23 +16,176 @@ package httpcaddyfile
 
 import (
 	"github.com/caddyserver/caddy/v2/caddyconfig"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddypki"
 )
 
+func init() {
+	RegisterGlobalOption("pki", parsePKIApp)
+}
+
+// parsePKIApp parses the global log option. Syntax:
+//
+//     pki {
+//         ca [<id>] {
+//             name            <name>
+//             root_cn         <name>
+//             intermediate_cn <name>
+//             root {
+//                 cert   <path>
+//                 key    <path>
+//                 format <format>
+//             }
+//             intermediate {
+//                 cert   <path>
+//                 key    <path>
+//                 format <format>
+//             }
+//         }
+//     }
+//
+// When the CA ID is unspecified, 'local' is assumed.
+//
+func parsePKIApp(d *caddyfile.Dispenser, existingVal interface{}) (interface{}, error) {
+	pki := &caddypki.PKI{CAs: make(map[string]*caddypki.CA)}
+
+	for d.Next() {
+		for nesting := d.Nesting(); d.NextBlock(nesting); {
+			switch d.Val() {
+			case "ca":
+				pkiCa := new(caddypki.CA)
+				if d.NextArg() {
+					pkiCa.ID = d.Val()
+					if d.NextArg() {
+						return nil, d.ArgErr()
+					}
+				}
+				if pkiCa.ID == "" {
+					pkiCa.ID = caddypki.DefaultCAID
+				}
+
+				for nesting := d.Nesting(); d.NextBlock(nesting); {
+					switch d.Val() {
+					case "name":
+						if !d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						pkiCa.Name = d.Val()
+
+					case "root_cn":
+						if !d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						pkiCa.RootCommonName = d.Val()
+
+					case "intermediate_cn":
+						if !d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						pkiCa.IntermediateCommonName = d.Val()
+
+					case "root":
+						if pkiCa.Root == nil {
+							pkiCa.Root = new(caddypki.KeyPair)
+						}
+						for nesting := d.Nesting(); d.NextBlock(nesting); {
+							switch d.Val() {
+							case "cert":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Root.Certificate = d.Val()
+
+							case "key":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Root.PrivateKey = d.Val()
+
+							case "format":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Root.Format = d.Val()
+
+							default:
+								return nil, d.Errf("unrecognized pki ca root option '%s'", d.Val())
+							}
+						}
+
+					case "intermediate":
+						if pkiCa.Intermediate == nil {
+							pkiCa.Intermediate = new(caddypki.KeyPair)
+						}
+						for nesting := d.Nesting(); d.NextBlock(nesting); {
+							switch d.Val() {
+							case "cert":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Intermediate.Certificate = d.Val()
+
+							case "key":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Intermediate.PrivateKey = d.Val()
+
+							case "format":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Intermediate.Format = d.Val()
+
+							default:
+								return nil, d.Errf("unrecognized pki ca intermediate option '%s'", d.Val())
+							}
+						}
+
+					default:
+						return nil, d.Errf("unrecognized pki ca option '%s'", d.Val())
+					}
+				}
+
+				pki.CAs[pkiCa.ID] = pkiCa
+
+			default:
+				return nil, d.Errf("unrecognized pki option '%s'", d.Val())
+			}
+		}
+	}
+
+	return pki, nil
+}
+
 func (st ServerType) buildPKIApp(
 	pairings []sbAddrAssociation,
 	options map[string]interface{},
 	warnings []caddyconfig.Warning,
 ) (*caddypki.PKI, []caddyconfig.Warning, error) {
 
-	pkiApp := &caddypki.PKI{CAs: make(map[string]*caddypki.CA)}
-
 	skipInstallTrust := false
 	if _, ok := options["skip_install_trust"]; ok {
 		skipInstallTrust = true
 	}
 	falseBool := false
 
+	// Load the PKI app configured via global options
+	var pkiApp *caddypki.PKI
+	unwrappedPki, ok := options["pki"].(*caddypki.PKI)
+	if ok {
+		pkiApp = unwrappedPki
+	} else {
+		pkiApp = &caddypki.PKI{CAs: make(map[string]*caddypki.CA)}
+	}
+	for _, ca := range pkiApp.CAs {
+		if skipInstallTrust {
+			ca.InstallTrust = &falseBool
+		}
+		pkiApp.CAs[ca.ID] = ca
+	}
+
+	// Add in the CAs configured via directives
 	for _, p := range pairings {
 		for _, sblock := range p.serverBlocks {
 			// find all the CAs that were defined and add them to the app config
@@ -42,7 +195,12 @@ func (st ServerType) buildPKIApp(
 				if skipInstallTrust {
 					ca.InstallTrust = &falseBool
 				}
-				pkiApp.CAs[ca.ID] = ca
+
+				// the CA might already exist from global options, so
+				// don't overwrite it in that case
+				if _, ok := pkiApp.CAs[ca.ID]; !ok {
+					pkiApp.CAs[ca.ID] = ca
+				}
 			}
 		}
 	}

caddyconfig/httpcaddyfile/serveroptions.go

@@ -33,15 +33,16 @@ type serverOptions struct {
 	ListenerAddress string
 
 	// These will all map 1:1 to the caddyhttp.Server struct
-	ListenerWrappersRaw []json.RawMessage
-	ReadTimeout         caddy.Duration
-	ReadHeaderTimeout   caddy.Duration
-	WriteTimeout        caddy.Duration
-	IdleTimeout         caddy.Duration
-	MaxHeaderBytes      int
-	AllowH2C            bool
-	ExperimentalHTTP3   bool
-	StrictSNIHost       *bool
+	ListenerWrappersRaw  []json.RawMessage
+	ReadTimeout          caddy.Duration
+	ReadHeaderTimeout    caddy.Duration
+	WriteTimeout         caddy.Duration
+	IdleTimeout          caddy.Duration
+	MaxHeaderBytes       int
+	AllowH2C             bool
+	ExperimentalHTTP3    bool
+	StrictSNIHost        *bool
+	ShouldLogCredentials bool
 }
 
 func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error) {
@@ -134,6 +135,12 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error
 				}
 				serverOpts.MaxHeaderBytes = int(size)
 
+			case "log_credentials":
+				if d.NextArg() {
+					return nil, d.ArgErr()
+				}
+				serverOpts.ShouldLogCredentials = true
+
 			case "protocol":
 				for nesting := d.Nesting(); d.NextBlock(nesting); {
 					switch d.Val() {
@@ -150,11 +157,14 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error
 						serverOpts.ExperimentalHTTP3 = true
 
 					case "strict_sni_host":
-						if d.NextArg() {
-							return nil, d.ArgErr()
+						if d.NextArg() && d.Val() != "insecure_off" && d.Val() != "on" {
+							return nil, d.Errf("strict_sni_host only supports 'on' or 'insecure_off', got '%s'", d.Val())
 						}
-						trueBool := true
-						serverOpts.StrictSNIHost = &trueBool
+						boolVal := true
+						if d.Val() == "insecure_off" {
+							boolVal = false
+						}
+						serverOpts.StrictSNIHost = &boolVal
 
 					default:
 						return nil, d.Errf("unrecognized protocol option '%s'", d.Val())
@@ -222,6 +232,12 @@ func applyServerOptions(
 		server.AllowH2C = opts.AllowH2C
 		server.ExperimentalHTTP3 = opts.ExperimentalHTTP3
 		server.StrictSNIHost = opts.StrictSNIHost
+		if opts.ShouldLogCredentials {
+			if server.Logs == nil {
+				server.Logs = &caddyhttp.ServerLogConfig{}
+			}
+			server.Logs.ShouldLogCredentials = opts.ShouldLogCredentials
+		}
 	}
 
 	return nil

caddyconfig/httpcaddyfile/tlsapp.go

@@ -101,6 +101,12 @@ func (st ServerType) buildTLSApp(
 		}
 
 		for _, sblock := range p.serverBlocks {
+			// check the scheme of all the site addresses,
+			// skip building AP if they all had http://
+			if sblock.isAllHTTP() {
+				continue
+			}
+
 			// get values that populate an automation policy for this block
 			ap, err := newBaseAutomationPolicy(options, warnings, true)
 			if err != nil {
@@ -133,6 +139,13 @@ func (st ServerType) buildTLSApp(
 				ap.Issuers = issuers
 			}
 
+			// certificate managers
+			if certManagerVals, ok := sblock.pile["tls.cert_manager"]; ok {
+				for _, certManager := range certManagerVals {
+					certGetterName := certManager.Value.(caddy.Module).CaddyModule().ID.Name()
+					ap.ManagersRaw = append(ap.ManagersRaw, caddyconfig.JSONModuleObject(certManager.Value, "via", certGetterName, &warnings))
+				}
+			}
 			// custom bind host
 			for _, cfgVal := range sblock.pile["bind"] {
 				for _, iss := range ap.Issuers {
@@ -286,6 +299,19 @@ func (st ServerType) buildTLSApp(
 		tlsApp.Automation.StorageCleanInterval = storageCleanInterval
 	}
 
+	// set the expired certificates renew interval if configured
+	if renewCheckInterval, ok := options["renew_interval"].(caddy.Duration); ok {
+		if tlsApp.Automation == nil {
+			tlsApp.Automation = new(caddytls.AutomationConfig)
+		}
+		tlsApp.Automation.RenewCheckInterval = renewCheckInterval
+	}
+
+	// set whether OCSP stapling should be disabled for manually-managed certificates
+	if ocspConfig, ok := options["ocsp_stapling"].(certmagic.OCSPConfig); ok {
+		tlsApp.DisableOCSPStapling = ocspConfig.DisableStapling
+	}
+
 	// if any hostnames appear on the same server block as a key with
 	// no host, they will not be used with route matchers because the
 	// hostless key matches all hosts, therefore, it wouldn't be
@@ -324,7 +350,6 @@ func (st ServerType) buildTLSApp(
 		globalPreferredChains := options["preferred_chains"]
 		hasGlobalACMEDefaults := globalEmail != nil || globalACMECA != nil || globalACMECARoot != nil || globalACMEDNS != nil || globalACMEEAB != nil || globalPreferredChains != nil
 		if hasGlobalACMEDefaults {
-			// for _, ap := range tlsApp.Automation.Policies {
 			for i := 0; i < len(tlsApp.Automation.Policies); i++ {
 				ap := tlsApp.Automation.Policies[i]
 				if len(ap.Issuers) == 0 && automationPolicyHasAllPublicNames(ap) {

caddyconfig/httploader.go

@@ -71,21 +71,28 @@ func (HTTPLoader) CaddyModule() caddy.ModuleInfo {
 
 // LoadConfig loads a Caddy config.
 func (hl HTTPLoader) LoadConfig(ctx caddy.Context) ([]byte, error) {
+	repl := caddy.NewReplacer()
+
 	client, err := hl.makeClient(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	method := hl.Method
+	method := repl.ReplaceAll(hl.Method, "")
 	if method == "" {
 		method = http.MethodGet
 	}
 
-	req, err := http.NewRequest(method, hl.URL, nil)
+	url := repl.ReplaceAll(hl.URL, "")
+	req, err := http.NewRequest(method, url, nil)
 	if err != nil {
 		return nil, err
 	}
-	req.Header = hl.Headers
+	for key, vals := range hl.Headers {
+		for _, val := range vals {
+			req.Header.Add(repl.ReplaceAll(key, ""), repl.ReplaceKnown(val, ""))
+		}
+	}
 
 	resp, err := client.Do(req)
 	if err != nil {

caddytest/integration/caddyfile_adapt/bind_ipv6.txt

@@ -0,0 +1,29 @@
+example.com {
+	bind tcp6/[::]
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						"tcp6/[::]:443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/expression_quotes.txt

@@ -0,0 +1,114 @@
+example.com
+
+@a expression {http.error.status_code} == 400
+abort @a
+
+@b expression {http.error.status_code} == "401"
+abort @b
+
+@c expression {http.error.status_code} == `402`
+abort @c
+
+@d expression "{http.error.status_code} == 403"
+abort @d
+
+@e expression `{http.error.status_code} == 404`
+abort @e
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == 400"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == \"401\""
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == `402`"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == 403"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == 404"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/file_server_pass_thru.txt

@@ -0,0 +1,32 @@
+:80
+
+file_server {
+	pass_thru
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "file_server",
+									"hide": [
+										"./Caddyfile"
+									],
+									"pass_thru": true
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/forward_auth_authelia.txt

@@ -0,0 +1,137 @@
+app.example.com {
+	forward_auth authelia:9091 {
+		uri /api/verify?rd=https://authelia.example.com
+		copy_headers Remote-User Remote-Groups Remote-Name Remote-Email
+	}
+
+	reverse_proxy backend:8080
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"app.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handle_response": [
+														{
+															"match": {
+																"status_code": [
+																	2
+																]
+															},
+															"routes": [
+																{
+																	"handle": [
+																		{
+																			"handler": "headers",
+																			"request": {
+																				"set": {
+																					"Remote-Email": [
+																						"{http.reverse_proxy.header.Remote-Email}"
+																					],
+																					"Remote-Groups": [
+																						"{http.reverse_proxy.header.Remote-Groups}"
+																					],
+																					"Remote-Name": [
+																						"{http.reverse_proxy.header.Remote-Name}"
+																					],
+																					"Remote-User": [
+																						"{http.reverse_proxy.header.Remote-User}"
+																					]
+																				}
+																			}
+																		}
+																	]
+																}
+															]
+														},
+														{
+															"routes": [
+																{
+																	"handle": [
+																		{
+																			"exclude": [
+																				"Connection",
+																				"Keep-Alive",
+																				"Te",
+																				"Trailers",
+																				"Transfer-Encoding",
+																				"Upgrade"
+																			],
+																			"handler": "copy_response_headers"
+																		}
+																	]
+																},
+																{
+																	"handle": [
+																		{
+																			"handler": "copy_response"
+																		}
+																	]
+																}
+															]
+														}
+													],
+													"handler": "reverse_proxy",
+													"headers": {
+														"request": {
+															"set": {
+																"X-Forwarded-Method": [
+																	"{http.request.method}"
+																],
+																"X-Forwarded-Uri": [
+																	"{http.request.uri}"
+																]
+															}
+														}
+													},
+													"rewrite": {
+														"method": "GET",
+														"uri": "/api/verify?rd=https://authelia.example.com"
+													},
+													"upstreams": [
+														{
+															"dial": "authelia:9091"
+														}
+													]
+												},
+												{
+													"handler": "reverse_proxy",
+													"upstreams": [
+														{
+															"dial": "backend:8080"
+														}
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options.txt

@@ -10,6 +10,7 @@
 	}
 	acme_ca https://example.com
 	acme_ca_root /path/to/ca.crt
+	ocsp_stapling off
 
 	email test@example.com
 	admin off
@@ -61,7 +62,8 @@
 								"module": "internal"
 							}
 						],
-						"key_type": "ed25519"
+						"key_type": "ed25519",
+						"disable_ocsp_stapling": true
 					}
 				],
 				"on_demand": {
@@ -71,7 +73,8 @@
 					},
 					"ask": "https://example.com"
 				}
-			}
+			},
+			"disable_ocsp_stapling": true
 		}
 	}
 }

caddytest/integration/caddyfile_adapt/global_options_acme.txt

@@ -21,6 +21,7 @@
 		burst 20
 	}
 	storage_clean_interval 7d
+	renew_interval 1d
 
 	key_type ed25519
 }
@@ -82,6 +83,7 @@
 					},
 					"ask": "https://example.com"
 				},
+				"renew_interval": 86400000000000,
 				"storage_clean_interval": 604800000000000
 			}
 		}

caddytest/integration/caddyfile_adapt/global_options_debug_with_access_log.txt

@@ -0,0 +1,45 @@
+{
+	debug
+}
+
+:8881 {
+	log {
+		format console
+	}
+}
+----------
+{
+	"logging": {
+		"logs": {
+			"default": {
+				"level": "DEBUG",
+				"exclude": [
+					"http.log.access.log0"
+				]
+			},
+			"log0": {
+				"encoder": {
+					"format": "console"
+				},
+				"level": "DEBUG",
+				"include": [
+					"http.log.access.log0"
+				]
+			}
+		}
+	},
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8881"
+					],
+					"logs": {
+						"default_logger_name": "log0"
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options_default_bind.txt

@@ -0,0 +1,54 @@
+{
+	default_bind tcp4/0.0.0.0 tcp6/[::]
+}
+
+example.com {
+}
+
+example.org:12345 {
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						"tcp4/0.0.0.0:12345",
+						"tcp6/[::]:12345"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.org"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				},
+				"srv1": {
+					"listen": [
+						"tcp4/0.0.0.0:443",
+						"tcp6/[::]:443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options_log_custom.txt

@@ -3,8 +3,7 @@
 		format filter {
 			wrap console
 			fields {
-				common_log delete
-				request>remote_addr ip_mask {
+				request>remote_ip ip_mask {
 					ipv4 24
 					ipv6 32
 				}
@@ -19,10 +18,7 @@
 			"custom-logger": {
 				"encoder": {
 					"fields": {
-						"common_log": {
-							"filter": "delete"
-						},
-						"request\u003eremote_addr": {
+						"request\u003eremote_ip": {
 							"filter": "ip_mask",
 							"ipv4_cidr": 24,
 							"ipv6_cidr": 32

caddytest/integration/caddyfile_adapt/global_options_skip_install_trust.txt

@@ -1,10 +1,44 @@
 {
 	skip_install_trust
+	pki {
+		ca {
+			name "Local"
+			root_cn "Custom Local Root Name"
+			intermediate_cn "Custom Local Intermediate Name"
+			root {
+				cert /path/to/cert.pem
+				key /path/to/key.pem
+				format pem_file
+			}
+			intermediate {
+				cert /path/to/cert.pem
+				key /path/to/key.pem
+				format pem_file
+			}
+		}
+		ca foo {
+			name "Foo"
+			root_cn "Custom Foo Root Name"
+			intermediate_cn "Custom Foo Intermediate Name"
+		}
+	}
 }
 
 a.example.com {
 	tls internal
 }
+
+acme.example.com {
+	acme_server {
+		ca foo
+	}
+}
+
+acme-bar.example.com {
+	acme_server {
+		ca bar
+	}
+}
 ----------
 {
 	"apps": {
@@ -15,6 +49,56 @@ a.example.com {
 						":443"
 					],
 					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"acme-bar.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"ca": "bar",
+													"handler": "acme_server"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"acme.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"ca": "foo",
+													"handler": "acme_server"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						},
 						{
 							"match": [
 								{
@@ -31,14 +115,42 @@ a.example.com {
 		},
 		"pki": {
 			"certificate_authorities": {
-				"local": {
+				"bar": {
 					"install_trust": false
+				},
+				"foo": {
+					"name": "Foo",
+					"root_common_name": "Custom Foo Root Name",
+					"intermediate_common_name": "Custom Foo Intermediate Name",
+					"install_trust": false
+				},
+				"local": {
+					"name": "Local",
+					"root_common_name": "Custom Local Root Name",
+					"intermediate_common_name": "Custom Local Intermediate Name",
+					"install_trust": false,
+					"root": {
+						"certificate": "/path/to/cert.pem",
+						"private_key": "/path/to/key.pem",
+						"format": "pem_file"
+					},
+					"intermediate": {
+						"certificate": "/path/to/cert.pem",
+						"private_key": "/path/to/key.pem",
+						"format": "pem_file"
+					}
 				}
 			}
 		},
 		"tls": {
 			"automation": {
 				"policies": [
+					{
+						"subjects": [
+							"acme-bar.example.com",
+							"acme.example.com"
+						]
+					},
 					{
 						"subjects": [
 							"a.example.com"

caddytest/integration/caddyfile_adapt/global_server_options_multi.txt

@@ -3,6 +3,9 @@
 		timeouts {
 			idle 90s
 		}
+		protocol {
+			strict_sni_host insecure_off
+		}
 	}
 	servers :80 {
 		timeouts {
@@ -13,6 +16,9 @@
 		timeouts {
 			idle 30s
 		}
+		protocol {
+			strict_sni_host
+		}
 	}
 }
 
@@ -46,7 +52,8 @@ http://bar.com {
 							],
 							"terminal": true
 						}
-					]
+					],
+					"strict_sni_host": true
 				},
 				"srv1": {
 					"listen": [
@@ -70,7 +77,8 @@ http://bar.com {
 					"listen": [
 						":8080"
 					],
-					"idle_timeout": 90000000000
+					"idle_timeout": 90000000000,
+					"strict_sni_host": false
 				}
 			}
 		}

caddytest/integration/caddyfile_adapt/global_server_options_single.txt

@@ -1,6 +1,7 @@
 {
 	servers {
 		listener_wrappers {
+			http_redirect
 			tls
 		}
 		timeouts {
@@ -10,6 +11,7 @@
 			idle 30s
 		}
 		max_header_size 100MB
+		log_credentials
 		protocol {
 			allow_h2c
 			experimental_http3
@@ -31,6 +33,9 @@ foo.com {
 						":443"
 					],
 					"listener_wrappers": [
+						{
+							"wrapper": "http_redirect"
+						},
 						{
 							"wrapper": "tls"
 						}
@@ -53,6 +58,9 @@ foo.com {
 						}
 					],
 					"strict_sni_host": true,
+					"logs": {
+						"should_log_credentials": true
+					},
 					"experimental_http3": true,
 					"allow_h2c": true
 				}

caddytest/integration/caddyfile_adapt/header.txt

@@ -13,6 +13,10 @@
 	header @images {
 		Cache-Control "public, max-age=3600, stale-while-revalidate=86400"
 	}
+	header {
+		+Link "Foo"
+		+Link "Bar"
+	}
 }
 ----------
 {
@@ -121,6 +125,17 @@
 											]
 										}
 									}
+								},
+								{
+									"handler": "headers",
+									"response": {
+										"add": {
+											"Link": [
+												"Foo",
+												"Bar"
+											]
+										}
+									}
 								}
 							]
 						}

caddytest/integration/caddyfile_adapt/log_filters.txt

@@ -5,12 +5,24 @@ log {
 	format filter {
 		wrap console
 		fields {
+			uri query {
+				replace foo REDACTED
+				delete bar
+				hash baz
+			}
 			request>headers>Authorization replace REDACTED
 			request>headers>Server delete
-			request>remote_addr ip_mask {
+			request>headers>Cookie cookie {
+				replace foo REDACTED
+				delete bar
+				hash baz
+			}
+			request>remote_ip ip_mask {
 				ipv4 24
 				ipv6 32
 			}
+			request>headers>Regexp regexp secret REDACTED
+			request>headers>Hash hash
 		}
 	}
 }
@@ -33,13 +45,57 @@ log {
 							"filter": "replace",
 							"value": "REDACTED"
 						},
+						"request\u003eheaders\u003eCookie": {
+							"actions": [
+								{
+									"name": "foo",
+									"type": "replace",
+									"value": "REDACTED"
+								},
+								{
+									"name": "bar",
+									"type": "delete"
+								},
+								{
+									"name": "baz",
+									"type": "hash"
+								}
+							],
+							"filter": "cookie"
+						},
+						"request\u003eheaders\u003eHash": {
+							"filter": "hash"
+						},
+						"request\u003eheaders\u003eRegexp": {
+							"filter": "regexp",
+							"regexp": "secret",
+							"value": "REDACTED"
+						},
 						"request\u003eheaders\u003eServer": {
 							"filter": "delete"
 						},
-						"request\u003eremote_addr": {
+						"request\u003eremote_ip": {
 							"filter": "ip_mask",
 							"ipv4_cidr": 24,
 							"ipv6_cidr": 32
+						},
+						"uri": {
+							"actions": [
+								{
+									"parameter": "foo",
+									"type": "replace",
+									"value": "REDACTED"
+								},
+								{
+									"parameter": "bar",
+									"type": "delete"
+								},
+								{
+									"parameter": "baz",
+									"type": "hash"
+								}
+							],
+							"filter": "query"
 						}
 					},
 					"format": "filter",

caddytest/integration/caddyfile_adapt/log_roll_days.txt

@@ -3,6 +3,8 @@
 log {
 	output file /var/log/access.log {
 		roll_size 1gb
+		roll_uncompressed
+		roll_local_time
 		roll_keep 5
 		roll_keep_for 90d
 	}
@@ -20,8 +22,10 @@ log {
 				"writer": {
 					"filename": "/var/log/access.log",
 					"output": "file",
+					"roll_gzip": false,
 					"roll_keep": 5,
 					"roll_keep_days": 90,
+					"roll_local_time": true,
 					"roll_size_mb": 954
 				},
 				"include": [

caddytest/integration/caddyfile_adapt/map_and_vars_with_raw_types.txt

@@ -0,0 +1,126 @@
+example.com
+
+map {host}             {my_placeholder}  {magic_number} {
+	# Should output boolean "true" and an integer
+	example.com        true              3
+
+	# Should output a string and null
+	foo.example.com    "string value"
+
+	# Should output two strings (quoted int)
+	(.*)\.example.com  "${1} subdomain"  "5"
+
+	# Should output null and a string (quoted int)
+	~.*\.net$          -                 `7`
+
+	# Should output a float and the string "false"
+	~.*\.xyz$          123.456           "false"
+
+	# Should output two strings, second being escaped quote
+	default            "unknown domain"  \"""
+}
+
+vars foo bar
+vars {
+	abc true
+	def 1
+	ghi 2.3
+	jkl "mn op"
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"defaults": [
+														"unknown domain",
+														"\""
+													],
+													"destinations": [
+														"{my_placeholder}",
+														"{magic_number}"
+													],
+													"handler": "map",
+													"mappings": [
+														{
+															"input": "example.com",
+															"outputs": [
+																true,
+																3
+															]
+														},
+														{
+															"input": "foo.example.com",
+															"outputs": [
+																"string value",
+																null
+															]
+														},
+														{
+															"input": "(.*)\\.example.com",
+															"outputs": [
+																"${1} subdomain",
+																"5"
+															]
+														},
+														{
+															"input_regexp": ".*\\.net$",
+															"outputs": [
+																null,
+																"7"
+															]
+														},
+														{
+															"input_regexp": ".*\\.xyz$",
+															"outputs": [
+																123.456,
+																"false"
+															]
+														}
+													],
+													"source": "{http.request.host}"
+												},
+												{
+													"foo": "bar",
+													"handler": "vars"
+												},
+												{
+													"abc": true,
+													"def": 1,
+													"ghi": 2.3,
+													"handler": "vars",
+													"jkl": "mn op"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/matcher_syntax.txt

@@ -37,6 +37,9 @@
 		header Bar foo
 	}
 	respond @matcher9 "header matcher with null field matcher"
+
+	@matcher10 remote_ip private_ranges
+	respond @matcher10 "remote_ip matcher with private ranges"
 }
 ----------
 {
@@ -101,7 +104,9 @@
 							"match": [
 								{
 									"vars": {
-										"{http.request.uri}": "/vars-matcher"
+										"{http.request.uri}": [
+											"/vars-matcher"
+										]
 									}
 								}
 							],
@@ -207,6 +212,28 @@
 									"handler": "static_response"
 								}
 							]
+						},
+						{
+							"match": [
+								{
+									"remote_ip": {
+										"ranges": [
+											"192.168.0.0/16",
+											"172.16.0.0/12",
+											"10.0.0.0/8",
+											"127.0.0.1/8",
+											"fd00::/8",
+											"::1"
+										]
+									}
+								}
+							],
+							"handle": [
+								{
+									"body": "remote_ip matcher with private ranges",
+									"handler": "static_response"
+								}
+							]
 						}
 					]
 				}

caddytest/integration/caddyfile_adapt/method_directive.txt

@@ -0,0 +1,27 @@
+:8080 {
+	method FOO
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8080"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "rewrite",
+									"method": "FOO"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/php_fastcgi_handle_response.txt

@@ -1,145 +1,145 @@
-:8881 {
-	php_fastcgi app:9000 {
-		env FOO bar
-
-		@error status 4xx
-		handle_response @error {
-			root * /errors
-			rewrite * /{http.reverse_proxy.status_code}.html
-			file_server
-		}
-	}
-}
-----------
-{
-	"apps": {
-		"http": {
-			"servers": {
-				"srv0": {
-					"listen": [
-						":8881"
-					],
-					"routes": [
-						{
-							"match": [
-								{
-									"file": {
-										"try_files": [
-											"{http.request.uri.path}/index.php"
-										]
-									},
-									"not": [
-										{
-											"path": [
-												"*/"
-											]
-										}
-									]
-								}
-							],
-							"handle": [
-								{
-									"handler": "static_response",
-									"headers": {
-										"Location": [
-											"{http.request.uri.path}/"
-										]
-									},
-									"status_code": 308
-								}
-							]
-						},
-						{
-							"match": [
-								{
-									"file": {
-										"try_files": [
-											"{http.request.uri.path}",
-											"{http.request.uri.path}/index.php",
-											"index.php"
-										],
-										"split_path": [
-											".php"
-										]
-									}
-								}
-							],
-							"handle": [
-								{
-									"handler": "rewrite",
-									"uri": "{http.matchers.file.relative}"
-								}
-							]
-						},
-						{
-							"match": [
-								{
-									"path": [
-										"*.php"
-									]
-								}
-							],
-							"handle": [
-								{
-									"handle_response": [
-										{
-											"match": {
-												"status_code": [
-													4
-												]
-											},
-											"routes": [
-												{
-													"handle": [
-														{
-															"handler": "vars",
-															"root": "/errors"
-														}
-													]
-												},
-												{
-													"group": "group0",
-													"handle": [
-														{
-															"handler": "rewrite",
-															"uri": "/{http.reverse_proxy.status_code}.html"
-														}
-													]
-												},
-												{
-													"handle": [
-														{
-															"handler": "file_server",
-															"hide": [
-																"./Caddyfile"
-															]
-														}
-													]
-												}
-											]
-										}
-									],
-									"handler": "reverse_proxy",
-									"transport": {
-										"env": {
-											"FOO": "bar"
-										},
-										"protocol": "fastcgi",
-										"split_path": [
-											".php"
-										]
-									},
-									"upstreams": [
-										{
-											"dial": "app:9000"
-										}
-									]
-								}
-							]
-						}
-					]
-				}
-			}
-		}
-	}
+:8881 {
+	php_fastcgi app:9000 {
+		env FOO bar
+
+		@error status 4xx
+		handle_response @error {
+			root * /errors
+			rewrite * /{http.reverse_proxy.status_code}.html
+			file_server
+		}
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8881"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"file": {
+										"try_files": [
+											"{http.request.uri.path}/index.php"
+										]
+									},
+									"not": [
+										{
+											"path": [
+												"*/"
+											]
+										}
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "static_response",
+									"headers": {
+										"Location": [
+											"{http.request.uri.path}/"
+										]
+									},
+									"status_code": 308
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"file": {
+										"try_files": [
+											"{http.request.uri.path}",
+											"{http.request.uri.path}/index.php",
+											"index.php"
+										],
+										"split_path": [
+											".php"
+										]
+									}
+								}
+							],
+							"handle": [
+								{
+									"handler": "rewrite",
+									"uri": "{http.matchers.file.relative}"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"path": [
+										"*.php"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handle_response": [
+										{
+											"match": {
+												"status_code": [
+													4
+												]
+											},
+											"routes": [
+												{
+													"handle": [
+														{
+															"handler": "vars",
+															"root": "/errors"
+														}
+													]
+												},
+												{
+													"group": "group0",
+													"handle": [
+														{
+															"handler": "rewrite",
+															"uri": "/{http.reverse_proxy.status_code}.html"
+														}
+													]
+												},
+												{
+													"handle": [
+														{
+															"handler": "file_server",
+															"hide": [
+																"./Caddyfile"
+															]
+														}
+													]
+												}
+											]
+										}
+									],
+									"handler": "reverse_proxy",
+									"transport": {
+										"env": {
+											"FOO": "bar"
+										},
+										"protocol": "fastcgi",
+										"split_path": [
+											".php"
+										]
+									},
+									"upstreams": [
+										{
+											"dial": "app:9000"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
 }

caddytest/integration/caddyfile_adapt/php_fastcgi_try_files_override.txt

@@ -1,124 +1,124 @@
-:8884
-
-php_fastcgi localhost:9000 {
-	# some php_fastcgi-specific subdirectives
-	split .php .php5
-	env VAR1 value1
-	env VAR2 value2
-	root /var/www
-	try_files {path} {path}/index.php =404
-	dial_timeout 3s
-	read_timeout 10s
-	write_timeout 20s
-
-	# passed through to reverse_proxy (directive order doesn't matter!)
-	lb_policy random
-}
-----------
-{
-	"apps": {
-		"http": {
-			"servers": {
-				"srv0": {
-					"listen": [
-						":8884"
-					],
-					"routes": [
-						{
-							"match": [
-								{
-									"file": {
-										"try_files": [
-											"{http.request.uri.path}/index.php"
-										]
-									},
-									"not": [
-										{
-											"path": [
-												"*/"
-											]
-										}
-									]
-								}
-							],
-							"handle": [
-								{
-									"handler": "static_response",
-									"headers": {
-										"Location": [
-											"{http.request.uri.path}/"
-										]
-									},
-									"status_code": 308
-								}
-							]
-						},
-						{
-							"match": [
-								{
-									"file": {
-										"try_files": [
-											"{http.request.uri.path}",
-											"{http.request.uri.path}/index.php",
-											"=404"
-										],
-										"split_path": [
-											".php",
-											".php5"
-										]
-									}
-								}
-							],
-							"handle": [
-								{
-									"handler": "rewrite",
-									"uri": "{http.matchers.file.relative}"
-								}
-							]
-						},
-						{
-							"match": [
-								{
-									"path": [
-										"*.php",
-										"*.php5"
-									]
-								}
-							],
-							"handle": [
-								{
-									"handler": "reverse_proxy",
-									"load_balancing": {
-										"selection_policy": {
-											"policy": "random"
-										}
-									},
-									"transport": {
-										"dial_timeout": 3000000000,
-										"env": {
-											"VAR1": "value1",
-											"VAR2": "value2"
-										},
-										"protocol": "fastcgi",
-										"read_timeout": 10000000000,
-										"root": "/var/www",
-										"split_path": [
-											".php",
-											".php5"
-										],
-										"write_timeout": 20000000000
-									},
-									"upstreams": [
-										{
-											"dial": "localhost:9000"
-										}
-									]
-								}
-							]
-						}
-					]
-				}
-			}
-		}
-	}
-}
+:8884
+
+php_fastcgi localhost:9000 {
+	# some php_fastcgi-specific subdirectives
+	split .php .php5
+	env VAR1 value1
+	env VAR2 value2
+	root /var/www
+	try_files {path} {path}/index.php =404
+	dial_timeout 3s
+	read_timeout 10s
+	write_timeout 20s
+
+	# passed through to reverse_proxy (directive order doesn't matter!)
+	lb_policy random
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"file": {
+										"try_files": [
+											"{http.request.uri.path}/index.php"
+										]
+									},
+									"not": [
+										{
+											"path": [
+												"*/"
+											]
+										}
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "static_response",
+									"headers": {
+										"Location": [
+											"{http.request.uri.path}/"
+										]
+									},
+									"status_code": 308
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"file": {
+										"try_files": [
+											"{http.request.uri.path}",
+											"{http.request.uri.path}/index.php",
+											"=404"
+										],
+										"split_path": [
+											".php",
+											".php5"
+										]
+									}
+								}
+							],
+							"handle": [
+								{
+									"handler": "rewrite",
+									"uri": "{http.matchers.file.relative}"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"path": [
+										"*.php",
+										"*.php5"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "reverse_proxy",
+									"load_balancing": {
+										"selection_policy": {
+											"policy": "random"
+										}
+									},
+									"transport": {
+										"dial_timeout": 3000000000,
+										"env": {
+											"VAR1": "value1",
+											"VAR2": "value2"
+										},
+										"protocol": "fastcgi",
+										"read_timeout": 10000000000,
+										"root": "/var/www",
+										"split_path": [
+											".php",
+											".php5"
+										],
+										"write_timeout": 20000000000
+									},
+									"upstreams": [
+										{
+											"dial": "localhost:9000"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/reverse_proxy_dynamic_upstreams.txt

@@ -0,0 +1,116 @@
+:8884 {
+	reverse_proxy {
+		dynamic a foo 9000
+	}
+
+	reverse_proxy {
+		dynamic a {
+			name foo
+			port 9000
+			refresh 5m
+			resolvers 8.8.8.8 8.8.4.4
+			dial_timeout 2s
+			dial_fallback_delay 300ms
+		}
+	}
+}
+
+:8885 {
+	reverse_proxy {
+		dynamic srv _api._tcp.example.com
+	}
+
+	reverse_proxy {
+		dynamic srv {
+			service api
+			proto tcp
+			name example.com
+			refresh 5m
+			resolvers 8.8.8.8 8.8.4.4
+			dial_timeout 1s
+			dial_fallback_delay -1s
+		}
+	}
+}
+
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"dynamic_upstreams": {
+										"name": "foo",
+										"port": "9000",
+										"source": "a"
+									},
+									"handler": "reverse_proxy"
+								},
+								{
+									"dynamic_upstreams": {
+										"dial_fallback_delay": 300000000,
+										"dial_timeout": 2000000000,
+										"name": "foo",
+										"port": "9000",
+										"refresh": 300000000000,
+										"resolver": {
+											"addresses": [
+												"8.8.8.8",
+												"8.8.4.4"
+											]
+										},
+										"source": "a"
+									},
+									"handler": "reverse_proxy"
+								}
+							]
+						}
+					]
+				},
+				"srv1": {
+					"listen": [
+						":8885"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"dynamic_upstreams": {
+										"name": "_api._tcp.example.com",
+										"source": "srv"
+									},
+									"handler": "reverse_proxy"
+								},
+								{
+									"dynamic_upstreams": {
+										"dial_fallback_delay": -1000000000,
+										"dial_timeout": 1000000000,
+										"name": "example.com",
+										"proto": "tcp",
+										"refresh": 300000000000,
+										"resolver": {
+											"addresses": [
+												"8.8.8.8",
+												"8.8.4.4"
+											]
+										},
+										"service": "api",
+										"source": "srv"
+									},
+									"handler": "reverse_proxy"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/reverse_proxy_handle_response.txt

@@ -1,6 +1,14 @@
 :8884
 
 reverse_proxy 127.0.0.1:65535 {
+	@500 status 500
+	replace_status @500 400
+
+	@all status 2xx 3xx 4xx 5xx
+	replace_status @all {http.error.status_code}
+
+	replace_status {http.error.status_code}
+
 	@accel header X-Accel-Redirect *
 	handle_response @accel {
 		respond "Header X-Accel-Redirect!"
@@ -39,8 +47,19 @@ reverse_proxy 127.0.0.1:65535 {
 		respond "Headers Foo, Bar AND statuses 401, 403 and 404!"
 	}
 
-	@changeStatus status 500
-	handle_response @changeStatus 400
+	@200 status 200
+	handle_response @200 {
+		copy_response_headers {
+			include Foo Bar
+		}
+		respond "Copied headers from the response"
+	}
+
+	@201 status 201
+	handle_response @201 {
+		header Foo "Copying the response"
+		copy_response 404
+	}
 }
 ----------
 {
@@ -56,6 +75,25 @@ reverse_proxy 127.0.0.1:65535 {
 							"handle": [
 								{
 									"handle_response": [
+										{
+											"match": {
+												"status_code": [
+													500
+												]
+											},
+											"status_code": 400
+										},
+										{
+											"match": {
+												"status_code": [
+													2,
+													3,
+													4,
+													5
+												]
+											},
+											"status_code": "{http.error.status_code}"
+										},
 										{
 											"match": {
 												"headers": {
@@ -158,10 +196,56 @@ reverse_proxy 127.0.0.1:65535 {
 										{
 											"match": {
 												"status_code": [
-													500
+													200
 												]
 											},
-											"status_code": 400
+											"routes": [
+												{
+													"handle": [
+														{
+															"handler": "copy_response_headers",
+															"include": [
+																"Foo",
+																"Bar"
+															]
+														},
+														{
+															"body": "Copied headers from the response",
+															"handler": "static_response"
+														}
+													]
+												}
+											]
+										},
+										{
+											"match": {
+												"status_code": [
+													201
+												]
+											},
+											"routes": [
+												{
+													"handle": [
+														{
+															"handler": "headers",
+															"response": {
+																"set": {
+																	"Foo": [
+																		"Copying the response"
+																	]
+																}
+															}
+														},
+														{
+															"handler": "copy_response",
+															"status_code": 404
+														}
+													]
+												}
+											]
+										},
+										{
+											"status_code": "{http.error.status_code}"
 										},
 										{
 											"routes": [

caddytest/integration/caddyfile_adapt/reverse_proxy_health_headers.txt

@@ -7,6 +7,7 @@ reverse_proxy 127.0.0.1:65535 {
 		X-Header-Keys VbG4NZwWnipo 335Q9/MhqcNU3s2TO
 		X-Empty-Value
 	}
+	health_uri /health
 }
 ----------
 {
@@ -38,7 +39,8 @@ reverse_proxy 127.0.0.1:65535 {
 													"VbG4NZwWnipo",
 													"335Q9/MhqcNU3s2TO"
 												]
-											}
+											},
+											"uri": "/health"
 										}
 									},
 									"upstreams": [

caddytest/integration/caddyfile_adapt/reverse_proxy_options.txt

@@ -1,11 +1,11 @@
 
 https://example.com {
-	reverse_proxy /path http://localhost:54321 {
-		header_up Host {host}
-		header_up X-Real-IP {remote}
-		header_up X-Forwarded-For {remote}
-		header_up X-Forwarded-Port {server_port}
-		header_up X-Forwarded-Proto "http"
+	reverse_proxy /path https://localhost:54321 {
+		header_up Host {upstream_hostport}
+		header_up Foo bar
+
+		method GET
+		rewrite /rewritten?uri={uri}
 
 		buffer_requests
 
@@ -17,11 +17,14 @@ https://example.com {
 			dial_fallback_delay 5s
 			response_header_timeout 8s
 			expect_continue_timeout 9s
+			resolvers 8.8.8.8 8.8.4.4
 
 			versions h2c 2
 			compression off
 			max_conns_per_host 5
 			keepalive_idle_conns_per_host 2
+			keepalive_interval 30s
+			renegotiation freely
 		}
 	}
 }
@@ -56,37 +59,42 @@ https://example.com {
 													"headers": {
 														"request": {
 															"set": {
+																"Foo": [
+																	"bar"
+																],
 																"Host": [
-																	"{http.request.host}"
-																],
-																"X-Forwarded-For": [
-																	"{http.request.remote}"
-																],
-																"X-Forwarded-Port": [
-																	"{server_port}"
-																],
-																"X-Forwarded-Proto": [
-																	"http"
-																],
-																"X-Real-Ip": [
-																	"{http.request.remote}"
+																	"{http.reverse_proxy.upstream.hostport}"
 																]
 															}
 														}
 													},
+													"rewrite": {
+														"method": "GET",
+														"uri": "/rewritten?uri={http.request.uri}"
+													},
 													"transport": {
 														"compression": false,
 														"dial_fallback_delay": 5000000000,
 														"dial_timeout": 3000000000,
 														"expect_continue_timeout": 9000000000,
 														"keep_alive": {
-															"max_idle_conns_per_host": 2
+															"max_idle_conns_per_host": 2,
+															"probe_interval": 30000000000
 														},
 														"max_conns_per_host": 5,
 														"max_response_header_size": 30000000,
 														"protocol": "http",
 														"read_buffer_size": 10000000,
+														"resolver": {
+															"addresses": [
+																"8.8.8.8",
+																"8.8.4.4"
+															]
+														},
 														"response_header_timeout": 8000000000,
+														"tls": {
+															"renegotiation": "freely"
+														},
 														"versions": [
 															"h2c",
 															"2"

caddytest/integration/caddyfile_adapt/reverse_proxy_trusted_proxies.txt

@@ -0,0 +1,56 @@
+:8884
+
+reverse_proxy 127.0.0.1:65535 {
+	trusted_proxies 127.0.0.1
+}
+
+reverse_proxy 127.0.0.1:65535 {
+	trusted_proxies private_ranges
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "reverse_proxy",
+									"trusted_proxies": [
+										"127.0.0.1"
+									],
+									"upstreams": [
+										{
+											"dial": "127.0.0.1:65535"
+										}
+									]
+								},
+								{
+									"handler": "reverse_proxy",
+									"trusted_proxies": [
+										"192.168.0.0/16",
+										"172.16.0.0/12",
+										"10.0.0.0/8",
+										"127.0.0.1/8",
+										"fd00::/8",
+										"::1"
+									],
+									"upstreams": [
+										{
+											"dial": "127.0.0.1:65535"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/sort_directives_within_handle.txt

@@ -0,0 +1,133 @@
+*.example.com {
+	@foo host foo.example.com
+	handle @foo {
+		handle_path /strip* {
+			respond "this should be first"
+		}
+		handle {
+			respond "this should be second"
+		}
+	}
+	handle {
+		respond "this should be last"
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"*.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"group": "group5",
+											"handle": [
+												{
+													"handler": "subroute",
+													"routes": [
+														{
+															"group": "group2",
+															"handle": [
+																{
+																	"handler": "subroute",
+																	"routes": [
+																		{
+																			"handle": [
+																				{
+																					"handler": "rewrite",
+																					"strip_path_prefix": "/strip"
+																				}
+																			]
+																		},
+																		{
+																			"handle": [
+																				{
+																					"body": "this should be first",
+																					"handler": "static_response"
+																				}
+																			]
+																		}
+																	]
+																}
+															],
+															"match": [
+																{
+																	"path": [
+																		"/strip*"
+																	]
+																}
+															]
+														},
+														{
+															"group": "group2",
+															"handle": [
+																{
+																	"handler": "subroute",
+																	"routes": [
+																		{
+																			"handle": [
+																				{
+																					"body": "this should be second",
+																					"handler": "static_response"
+																				}
+																			]
+																		}
+																	]
+																}
+															]
+														}
+													]
+												}
+											],
+											"match": [
+												{
+													"host": [
+														"foo.example.com"
+													]
+												}
+											]
+										},
+										{
+											"group": "group5",
+											"handle": [
+												{
+													"handler": "subroute",
+													"routes": [
+														{
+															"handle": [
+																{
+																	"body": "this should be last",
+																	"handler": "static_response"
+																}
+															]
+														}
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/sort_vars_in_reverse.txt

@@ -0,0 +1,59 @@
+:80
+
+vars /foobar foo last
+vars /foo foo middle
+vars * foo first
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"foo": "first",
+									"handler": "vars"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"path": [
+										"/foo"
+									]
+								}
+							],
+							"handle": [
+								{
+									"foo": "middle",
+									"handler": "vars"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"path": [
+										"/foobar"
+									]
+								}
+							],
+							"handle": [
+								{
+									"foo": "last",
+									"handler": "vars"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_automation_policies_9.txt

@@ -0,0 +1,56 @@
+# example from issue #4640
+http://foo:8447, http://127.0.0.1:8447 {
+	reverse_proxy 127.0.0.1:8080
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8447"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"foo",
+										"127.0.0.1"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "reverse_proxy",
+													"upstreams": [
+														{
+															"dial": "127.0.0.1:8080"
+														}
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"automatic_https": {
+						"skip": [
+							"foo",
+							"127.0.0.1"
+						]
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_internal_options.txt

@@ -0,0 +1,54 @@
+a.example.com {
+	tls {
+		issuer internal {
+			ca foo
+			lifetime 24h
+			sign_with_root
+		}
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"a.example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"a.example.com"
+						],
+						"issuers": [
+							{
+								"ca": "foo",
+								"lifetime": 86400000000000,
+								"module": "internal",
+								"sign_with_root": true
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_propagation_timeout.txt

@@ -3,7 +3,12 @@ localhost
 respond "hello from localhost"
 tls {
 	issuer acme {
-		propagation_timeout "10m0s"
+		propagation_delay 5m10s
+		propagation_timeout 10m20s
+	}
+	issuer zerossl {
+		propagation_delay 5m30s
+		propagation_timeout -1
 	}
 }
 ----------
@@ -56,10 +61,20 @@ tls {
 							{
 								"challenges": {
 									"dns": {
-										"propagation_timeout": 600000000000
+										"propagation_delay": 310000000000,
+										"propagation_timeout": 620000000000
 									}
 								},
 								"module": "acme"
+							},
+							{
+								"challenges": {
+									"dns": {
+										"propagation_delay": 330000000000,
+										"propagation_timeout": -1
+									}
+								},
+								"module": "zerossl"
 							}
 						]
 					}

caddytest/integration/caddyfile_adapt/tracing.txt

@@ -0,0 +1,36 @@
+:80 {
+	tracing /myhandler {
+		span my-span
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"path": [
+										"/myhandler"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "tracing",
+									"span": "my-span"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_test.go

@@ -101,3 +101,27 @@ func TestReadCookie(t *testing.T) {
 	// act and assert
 	tester.AssertGetResponse("http://localhost:9080/cookie.html", 200, "<h2>Cookie.ClientName caddytest</h2>")
 }
+
+func TestReplIndex(t *testing.T) {
+
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+  {
+    http_port     9080
+    https_port    9443
+  }
+
+  localhost:9080 {
+    templates {
+      root testdata
+    }
+    file_server {
+      root testdata
+      index "index.{host}.html"
+    }
+  }
+  `, "caddyfile")
+
+	// act and assert
+	tester.AssertGetResponse("http://localhost:9080/", 200, "")
+}

cmd/commandfuncs.go

@@ -32,6 +32,7 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/aryann/difflib"
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
@@ -202,7 +203,7 @@ func cmdRun(fl Flags) (int, error) {
 	// we don't use 'else' here since this value might have been changed in 'if' block; i.e. not mutually exclusive
 	var configFile string
 	if !runCmdResumeFlag {
-		config, configFile, err = loadConfig(runCmdConfigFlag, runCmdConfigAdapterFlag)
+		config, configFile, err = LoadConfig(runCmdConfigFlag, runCmdConfigAdapterFlag)
 		if err != nil {
 			return caddy.ExitCodeFailedStartup, err
 		}
@@ -275,25 +276,33 @@ func cmdRun(fl Flags) (int, error) {
 }
 
 func cmdStop(fl Flags) (int, error) {
-	stopCmdAddrFlag := fl.String("address")
+	addrFlag := fl.String("address")
+	configFlag := fl.String("config")
+	configAdapterFlag := fl.String("adapter")
 
-	err := apiRequest(stopCmdAddrFlag, http.MethodPost, "/stop", nil, nil)
+	adminAddr, err := DetermineAdminAPIAddress(addrFlag, configFlag, configAdapterFlag)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("couldn't determine admin API address: %v", err)
+	}
+
+	resp, err := AdminAPIRequest(adminAddr, http.MethodPost, "/stop", nil, nil)
 	if err != nil {
 		caddy.Log().Warn("failed using API to stop instance", zap.Error(err))
 		return caddy.ExitCodeFailedStartup, err
 	}
+	defer resp.Body.Close()
 
 	return caddy.ExitCodeSuccess, nil
 }
 
 func cmdReload(fl Flags) (int, error) {
-	reloadCmdConfigFlag := fl.String("config")
-	reloadCmdConfigAdapterFlag := fl.String("adapter")
-	reloadCmdAddrFlag := fl.String("address")
-	reloadCmdForceFlag := fl.Bool("force")
+	configFlag := fl.String("config")
+	configAdapterFlag := fl.String("adapter")
+	addrFlag := fl.String("address")
+	forceFlag := fl.Bool("force")
 
 	// get the config in caddy's native format
-	config, configFile, err := loadConfig(reloadCmdConfigFlag, reloadCmdConfigAdapterFlag)
+	config, configFile, err := LoadConfig(configFlag, configAdapterFlag)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, err
 	}
@@ -301,30 +310,22 @@ func cmdReload(fl Flags) (int, error) {
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("no config file to load")
 	}
 
-	// get the address of the admin listener; use flag if specified
-	adminAddr := reloadCmdAddrFlag
-	if adminAddr == "" && len(config) > 0 {
-		var tmpStruct struct {
-			Admin caddy.AdminConfig `json:"admin"`
-		}
-		err = json.Unmarshal(config, &tmpStruct)
-		if err != nil {
-			return caddy.ExitCodeFailedStartup,
-				fmt.Errorf("unmarshaling admin listener address from config: %v", err)
-		}
-		adminAddr = tmpStruct.Admin.Listen
+	adminAddr, err := DetermineAdminAPIAddress(addrFlag, configFlag, configAdapterFlag)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("couldn't determine admin API address: %v", err)
 	}
 
 	// optionally force a config reload
 	headers := make(http.Header)
-	if reloadCmdForceFlag {
+	if forceFlag {
 		headers.Set("Cache-Control", "must-revalidate")
 	}
 
-	err = apiRequest(adminAddr, http.MethodPost, "/load", headers, bytes.NewReader(config))
+	resp, err := AdminAPIRequest(adminAddr, http.MethodPost, "/load", headers, bytes.NewReader(config))
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("sending configuration to instance: %v", err)
 	}
+	defer resp.Body.Close()
 
 	return caddy.ExitCodeSuccess, nil
 }
@@ -495,7 +496,9 @@ func cmdAdaptConfig(fl Flags) (int, error) {
 		if warn.Directive != "" {
 			msg = fmt.Sprintf("%s: %s", warn.Directive, warn.Message)
 		}
-		fmt.Fprintf(os.Stderr, "[WARNING][%s] %s:%d: %s\n", adaptCmdAdapterFlag, warn.File, warn.Line, msg)
+		caddy.Log().Named(adaptCmdAdapterFlag).Warn(msg,
+			zap.String("file", warn.File),
+			zap.Int("line", warn.Line))
 	}
 
 	// validate output if requested
@@ -518,7 +521,7 @@ func cmdValidateConfig(fl Flags) (int, error) {
 	validateCmdConfigFlag := fl.String("config")
 	validateCmdAdapterFlag := fl.String("adapter")
 
-	input, _, err := loadConfig(validateCmdConfigFlag, validateCmdAdapterFlag)
+	input, _, err := LoadConfig(validateCmdConfigFlag, validateCmdAdapterFlag)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, err
 	}
@@ -567,7 +570,21 @@ func cmdFmt(fl Flags) (int, error) {
 
 	if fl.Bool("overwrite") {
 		if err := os.WriteFile(formatCmdConfigFile, output, 0600); err != nil {
-			return caddy.ExitCodeFailedStartup, nil
+			return caddy.ExitCodeFailedStartup, fmt.Errorf("overwriting formatted file: %v", err)
+		}
+	} else if fl.Bool("diff") {
+		diff := difflib.Diff(
+			strings.Split(string(input), "\n"),
+			strings.Split(string(output), "\n"))
+		for _, d := range diff {
+			switch d.Delta {
+			case difflib.Common:
+				fmt.Printf("  %s\n", d.Payload)
+			case difflib.LeftOnly:
+				fmt.Printf("- %s\n", d.Payload)
+			case difflib.RightOnly:
+				fmt.Printf("+ %s\n", d.Payload)
+			}
 		}
 	} else {
 		fmt.Print(string(output))
@@ -640,27 +657,25 @@ commands:
 	return caddy.ExitCodeSuccess, nil
 }
 
-// apiRequest makes an API request to the endpoint adminAddr with the
-// given HTTP method and request URI. If body is non-nil, it will be
-// assumed to be Content-Type application/json.
-func apiRequest(adminAddr, method, uri string, headers http.Header, body io.Reader) error {
-	// parse the admin address
-	if adminAddr == "" {
-		adminAddr = caddy.DefaultAdminListen
-	}
+// AdminAPIRequest makes an API request according to the CLI flags given,
+// with the given HTTP method and request URI. If body is non-nil, it will
+// be assumed to be Content-Type application/json. The caller should close
+// the response body. Should only be used by Caddy CLI commands which
+// need to interact with a running instance of Caddy via the admin API.
+func AdminAPIRequest(adminAddr, method, uri string, headers http.Header, body io.Reader) (*http.Response, error) {
 	parsedAddr, err := caddy.ParseNetworkAddress(adminAddr)
 	if err != nil || parsedAddr.PortRangeSize() > 1 {
-		return fmt.Errorf("invalid admin address %s: %v", adminAddr, err)
+		return nil, fmt.Errorf("invalid admin address %s: %v", adminAddr, err)
 	}
-	origin := parsedAddr.JoinHostPort(0)
+	origin := "http://" + parsedAddr.JoinHostPort(0)
 	if parsedAddr.IsUnixNetwork() {
-		origin = "unixsocket" // hack so that http.NewRequest() is happy
+		origin = "http://unixsocket" // hack so that http.NewRequest() is happy
 	}
 
 	// form the request
-	req, err := http.NewRequest(method, "http://"+origin+uri, body)
+	req, err := http.NewRequest(method, origin+uri, body)
 	if err != nil {
-		return fmt.Errorf("making request: %v", err)
+		return nil, fmt.Errorf("making request: %v", err)
 	}
 	if parsedAddr.IsUnixNetwork() {
 		// When listening on a unix socket, the admin endpoint doesn't
@@ -700,20 +715,60 @@ func apiRequest(adminAddr, method, uri string, headers http.Header, body io.Read
 
 	resp, err := client.Do(req)
 	if err != nil {
-		return fmt.Errorf("performing request: %v", err)
+		return nil, fmt.Errorf("performing request: %v", err)
 	}
-	defer resp.Body.Close()
 
 	// if it didn't work, let the user know
 	if resp.StatusCode >= 400 {
 		respBody, err := io.ReadAll(io.LimitReader(resp.Body, 1024*10))
 		if err != nil {
-			return fmt.Errorf("HTTP %d: reading error message: %v", resp.StatusCode, err)
+			return nil, fmt.Errorf("HTTP %d: reading error message: %v", resp.StatusCode, err)
 		}
-		return fmt.Errorf("caddy responded with error: HTTP %d: %s", resp.StatusCode, respBody)
+		return nil, fmt.Errorf("caddy responded with error: HTTP %d: %s", resp.StatusCode, respBody)
 	}
 
-	return nil
+	return resp, nil
+}
+
+// DetermineAdminAPIAddress determines which admin API endpoint address should
+// be used based on the inputs. By priority: if `address` is specified, then
+// it is returned; if `configFile` (and `configAdapter`) are specified, then that
+// config will be loaded to find the admin address; otherwise, the default
+// admin listen address will be returned.
+func DetermineAdminAPIAddress(address, configFile, configAdapter string) (string, error) {
+	// Prefer the address if specified and non-empty
+	if address != "" {
+		return address, nil
+	}
+
+	// Try to load the config from file if specified, with the given adapter name
+	if configFile != "" {
+		// get the config in caddy's native format
+		config, loadedConfigFile, err := LoadConfig(configFile, configAdapter)
+		if err != nil {
+			return "", err
+		}
+		if loadedConfigFile == "" {
+			return "", fmt.Errorf("no config file to load")
+		}
+
+		// get the address of the admin listener if set
+		if len(config) > 0 {
+			var tmpStruct struct {
+				Admin caddy.AdminConfig `json:"admin"`
+			}
+			err = json.Unmarshal(config, &tmpStruct)
+			if err != nil {
+				return "", fmt.Errorf("unmarshaling admin listener address from config: %v", err)
+			}
+			if tmpStruct.Admin.Listen != "" {
+				return tmpStruct.Admin.Listen, nil
+			}
+		}
+	}
+
+	// Fallback to the default listen address otherwise
+	return caddy.DefaultAdminListen, nil
 }
 
 type moduleInfo struct {

cmd/commands.go

@@ -156,16 +156,19 @@ development environment.`,
 	RegisterCommand(Command{
 		Name:  "stop",
 		Func:  cmdStop,
+		Usage: "[--address <interface>] [--config <path> [--adapter <name>]]",
 		Short: "Gracefully stops a started Caddy process",
 		Long: `
 Stops the background Caddy process as gracefully as possible.
 
 It requires that the admin API is enabled and accessible, since it will
-use the API's /stop endpoint. The address of this request can be
-customized using the --address flag if it is not the default.`,
+use the API's /stop endpoint. The address of this request can be customized
+using the --address flag, or from the given --config, if not the default.`,
 		Flags: func() *flag.FlagSet {
 			fs := flag.NewFlagSet("stop", flag.ExitOnError)
 			fs.String("address", "", "The address to use to reach the admin API endpoint, if not the default")
+			fs.String("config", "", "Configuration file to use to parse the admin address, if --address is not used")
+			fs.String("adapter", "", "Name of config adapter to apply (when --config is used)")
 			return fs
 		}(),
 	})
@@ -279,12 +282,18 @@ human readability. It prints the result to stdout.
 If --overwrite is specified, the output will be written to the config file
 directly instead of printing it.
 
+If --diff is specified, the output will be compared against the input, and
+lines will be prefixed with '-' and '+' where they differ. Note that
+unchanged lines are prefixed with two spaces for alignment, and that this
+is not a valid patch format.
+
 If you wish you use stdin instead of a regular file, use - as the path.
 When reading from stdin, the --overwrite flag has no effect: the result
 is always printed to stdout.`,
 		Flags: func() *flag.FlagSet {
 			fs := flag.NewFlagSet("fmt", flag.ExitOnError)
 			fs.Bool("overwrite", false, "Overwrite the input file with the results")
+			fs.Bool("diff", false, "Print the differences between the input file and the formatted output")
 			return fs
 		}(),
 	})

cmd/main.go

@@ -103,15 +103,15 @@ func handlePingbackConn(conn net.Conn, expect []byte) error {
 	return nil
 }
 
-// loadConfig loads the config from configFile and adapts it
+// LoadConfig loads the config from configFile and adapts it
 // using adapterName. If adapterName is specified, configFile
 // must be also. If no configFile is specified, it tries
 // loading a default config file. The lack of a config file is
 // not treated as an error, but false will be returned if
 // there is no config available. It prints any warnings to stderr,
 // and returns the resulting JSON config bytes along with
-// whether a config file was loaded or not.
-func loadConfig(configFile, adapterName string) ([]byte, string, error) {
+// the name of the loaded config file (if any).
+func LoadConfig(configFile, adapterName string) ([]byte, string, error) {
 	// specifying an adapter without a config file is ambiguous
 	if adapterName != "" && configFile == "" {
 		return nil, "", fmt.Errorf("cannot adapt config without config file (use --config)")
@@ -262,7 +262,7 @@ func watchConfigFile(filename, adapterName string) {
 		lastModified = info.ModTime()
 
 		// load the contents of the file
-		config, _, err := loadConfig(filename, adapterName)
+		config, _, err := LoadConfig(filename, adapterName)
 		if err != nil {
 			logger().Error("unable to load latest config", zap.Error(err))
 			continue
@@ -368,42 +368,68 @@ func loadEnvFromFile(envFile string) error {
 	return nil
 }
 
+// parseEnvFile parses an env file from KEY=VALUE format.
+// It's pretty naive. Limited value quotation is supported,
+// but variable and command expansions are not supported.
 func parseEnvFile(envInput io.Reader) (map[string]string, error) {
 	envMap := make(map[string]string)
 
 	scanner := bufio.NewScanner(envInput)
-	var line string
-	lineNumber := 0
+	var lineNumber int
 
 	for scanner.Scan() {
-		line = strings.TrimSpace(scanner.Text())
+		line := strings.TrimSpace(scanner.Text())
 		lineNumber++
 
-		// skip lines starting with comment
-		if strings.HasPrefix(line, "#") {
-			continue
-		}
-
-		// skip empty line
-		if len(line) == 0 {
+		// skip empty lines and lines starting with comment
+		if line == "" || strings.HasPrefix(line, "#") {
 			continue
 		}
 
+		// split line into key and value
 		fields := strings.SplitN(line, "=", 2)
 		if len(fields) != 2 {
 			return nil, fmt.Errorf("can't parse line %d; line should be in KEY=VALUE format", lineNumber)
 		}
+		key, val := fields[0], fields[1]
 
-		if strings.Contains(fields[0], " ") {
-			return nil, fmt.Errorf("bad key on line %d: contains whitespace", lineNumber)
-		}
-
-		key := fields[0]
-		val := fields[1]
+		// sometimes keys are prefixed by "export " so file can be sourced in bash; ignore it here
+		key = strings.TrimPrefix(key, "export ")
 
+		// validate key and value
 		if key == "" {
 			return nil, fmt.Errorf("missing or empty key on line %d", lineNumber)
 		}
+		if strings.Contains(key, " ") {
+			return nil, fmt.Errorf("invalid key on line %d: contains whitespace: %s", lineNumber, key)
+		}
+		if strings.HasPrefix(val, " ") || strings.HasPrefix(val, "\t") {
+			return nil, fmt.Errorf("invalid value on line %d: whitespace before value: '%s'", lineNumber, val)
+		}
+
+		// remove any trailing comment after value
+		if commentStart := strings.Index(val, "#"); commentStart > 0 {
+			before := val[commentStart-1]
+			if before == '\t' || before == ' ' {
+				val = strings.TrimRight(val[:commentStart], " \t")
+			}
+		}
+
+		// quoted value: support newlines
+		if strings.HasPrefix(val, `"`) {
+			for !(strings.HasSuffix(line, `"`) && !strings.HasSuffix(line, `\"`)) {
+				val = strings.ReplaceAll(val, `\"`, `"`)
+				if !scanner.Scan() {
+					break
+				}
+				lineNumber++
+				line = strings.ReplaceAll(scanner.Text(), `\"`, `"`)
+				val += "\n" + line
+			}
+			val = strings.TrimPrefix(val, `"`)
+			val = strings.TrimSuffix(val, `"`)
+		}
+
 		envMap[key] = val
 	}
 

cmd/main_test.go

@@ -0,0 +1,170 @@
+package caddycmd
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestParseEnvFile(t *testing.T) {
+	for i, tc := range []struct {
+		input     string
+		expect    map[string]string
+		shouldErr bool
+	}{
+		{
+			input: `KEY=value`,
+			expect: map[string]string{
+				"KEY": "value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				INVALID KEY=asdf
+				OTHER_KEY=Some Value
+			`,
+			shouldErr: true,
+		},
+		{
+			input: `
+				KEY=value
+				SIMPLE_QUOTED="quoted value"
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":           "value",
+				"SIMPLE_QUOTED": "quoted value",
+				"OTHER_KEY":     "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				NEWLINES="foo
+	bar"
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"NEWLINES":  "foo\n\tbar",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				ESCAPED="\"escaped quotes\"
+here"
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"ESCAPED":   "\"escaped quotes\"\nhere",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				export KEY=value
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				=value
+				OTHER_KEY=Some Value
+			`,
+			shouldErr: true,
+		},
+		{
+			input: `
+				EMPTY=
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"EMPTY":     "",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				EMPTY=""
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"EMPTY":     "",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				#OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY": "value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				COMMENT=foo bar  # some comment here
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"COMMENT":   "foo bar",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				WHITESPACE=   foo 
+				OTHER_KEY=Some Value
+			`,
+			shouldErr: true,
+		},
+		{
+			input: `
+				KEY=value
+				WHITESPACE="   foo bar "
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":        "value",
+				"WHITESPACE": "   foo bar ",
+				"OTHER_KEY":  "Some Value",
+			},
+		},
+	} {
+		actual, err := parseEnvFile(strings.NewReader(tc.input))
+		if err != nil && !tc.shouldErr {
+			t.Errorf("Test %d: Got error but shouldn't have: %v", i, err)
+		}
+		if err == nil && tc.shouldErr {
+			t.Errorf("Test %d: Did not get error but should have", i)
+		}
+		if tc.shouldErr {
+			continue
+		}
+		if !reflect.DeepEqual(tc.expect, actual) {
+			t.Errorf("Test %d: Expected %v but got %v", i, tc.expect, actual)
+		}
+	}
+}

context.go

@@ -423,6 +423,17 @@ func (ctx Context) App(name string) (interface{}, error) {
 	return modVal, nil
 }
 
+// AppIsConfigured returns whether an app named name has been
+// configured. Can be called before calling App() to avoid
+// instantiating an empty app when that's not desirable.
+func (ctx Context) AppIsConfigured(name string) bool {
+	if _, ok := ctx.cfg.apps[name]; ok {
+		return true
+	}
+	appRaw := ctx.cfg.AppsRaw[name]
+	return appRaw != nil
+}
+
 // Storage returns the configured Caddy storage implementation.
 func (ctx Context) Storage() certmagic.Storage {
 	return ctx.cfg.storage

duration_fuzz.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build gofuzz
 // +build gofuzz
 
 package caddy

go.mod

@@ -1,39 +1,132 @@
 module github.com/caddyserver/caddy/v2
 
-go 1.16
+go 1.17
 
 require (
+	github.com/BurntSushi/toml v1.1.0
 	github.com/Masterminds/sprig/v3 v3.2.2
-	github.com/alecthomas/chroma v0.9.2
+	github.com/alecthomas/chroma v0.10.0
 	github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b
-	github.com/caddyserver/certmagic v0.15.2
+	github.com/caddyserver/certmagic v0.16.1
 	github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/google/cel-go v0.7.3
 	github.com/google/uuid v1.3.0
-	github.com/klauspost/compress v1.13.6
-	github.com/klauspost/cpuid/v2 v2.0.9
-	github.com/lucas-clemente/quic-go v0.23.0
-	github.com/mholt/acmez v1.0.1
-	github.com/naoina/go-stringutil v0.1.0 // indirect
-	github.com/naoina/toml v0.1.1
-	github.com/prometheus/client_golang v1.11.0
-	github.com/smallstep/certificates v0.17.5-0.20211008195551-04fe3126bebf
-	github.com/smallstep/cli v0.17.6
-	github.com/smallstep/nosql v0.3.8
-	github.com/smallstep/truststore v0.9.6
-	github.com/yuin/goldmark v1.4.1
-	github.com/yuin/goldmark-highlighting v0.0.0-20210516132338-9216f9c5aa01
-	go.uber.org/zap v1.19.0
-	golang.org/x/crypto v0.0.0-20210915214749-c084706c2272
-	golang.org/x/net v0.0.0-20210913180222-943fd674d43e
-	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
-	google.golang.org/genproto v0.0.0-20210719143636-1d5a45f8e492
+	github.com/klauspost/compress v1.15.4
+	github.com/klauspost/cpuid/v2 v2.0.12
+	github.com/lucas-clemente/quic-go v0.27.1
+	github.com/mholt/acmez v1.0.2
+	github.com/prometheus/client_golang v1.12.1
+	github.com/smallstep/certificates v0.19.0
+	github.com/smallstep/cli v0.18.0
+	github.com/smallstep/nosql v0.4.0
+	github.com/smallstep/truststore v0.11.0
+	github.com/tailscale/tscert v0.0.0-20220316030059-54bbcb9f74e2
+	github.com/yuin/goldmark v1.4.12
+	github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.29.0
+	go.opentelemetry.io/otel v1.4.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.4.0
+	go.opentelemetry.io/otel/sdk v1.4.0
+	go.uber.org/zap v1.21.0
+	golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+	google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
-	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
-// avoid license conflict from juju/ansiterm until https://github.com/manifoldco/promptui/pull/181
-// is merged or other dependency in path currently in violation fixes compliance
-replace github.com/manifoldco/promptui => github.com/nguyer/promptui v0.8.1-0.20210517132806-70ccd4709797
+require (
+	filippo.io/edwards25519 v1.0.0-rc.1 // indirect
+	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.1.1 // indirect
+	github.com/antlr/antlr4 v0.0.0-20200503195918-621b933c7a7f // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cheekybits/genny v1.0.0 // indirect
+	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/dgraph-io/badger v1.6.2 // indirect
+	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
+	github.com/dgraph-io/ristretto v0.0.4-0.20200906165740-41ebdbffecfd // indirect
+	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
+	github.com/dlclark/regexp2 v1.4.0 // indirect
+	github.com/felixge/httpsnoop v1.0.2 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-kit/kit v0.10.0 // indirect
+	github.com/go-logfmt/logfmt v0.5.0 // indirect
+	github.com/go-logr/logr v1.2.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/huandu/xstrings v1.3.2 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.10.1 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/pgtype v1.9.0 // indirect
+	github.com/jackc/pgx/v4 v4.14.0 // indirect
+	github.com/libdns/libdns v0.2.1 // indirect
+	github.com/manifoldco/promptui v0.9.0 // indirect
+	github.com/marten-seemann/qpack v0.2.1 // indirect
+	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
+	github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.13 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
+	github.com/micromdm/scep/v2 v2.1.0 // indirect
+	github.com/miekg/dns v1.1.46 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/go-ps v1.0.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/onsi/ginkgo v1.16.4 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/rs/xid v1.2.1 // indirect
+	github.com/russross/blackfriday/v2 v2.0.1 // indirect
+	github.com/shopspring/decimal v1.2.0 // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/slackhq/nebula v1.5.2 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
+	github.com/urfave/cli v1.22.5 // indirect
+	go.etcd.io/bbolt v1.3.6 // indirect
+	go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.0 // indirect
+	go.opentelemetry.io/otel/internal/metric v0.27.0 // indirect
+	go.opentelemetry.io/otel/metric v0.27.0 // indirect
+	go.opentelemetry.io/otel/trace v1.4.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.12.0 // indirect
+	go.step.sm/cli-utils v0.7.0 // indirect
+	go.step.sm/crypto v0.16.1 // indirect
+	go.step.sm/linkedca v0.15.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
+	golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect
+	golang.org/x/tools v0.1.7 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/grpc v1.44.0 // indirect
+	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	howett.net/plist v1.0.0 // indirect
+)

internal/metrics/metrics.go

@@ -0,0 +1,39 @@
+package metrics
+
+import (
+	"net/http"
+	"strconv"
+)
+
+func SanitizeCode(s int) string {
+	switch s {
+	case 0, 200:
+		return "200"
+	default:
+		return strconv.Itoa(s)
+	}
+}
+
+// Only support the list of "regular" HTTP methods, see
+// https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods
+var methodMap = map[string]string{
+	"GET": http.MethodGet, "get": http.MethodGet,
+	"HEAD": http.MethodHead, "head": http.MethodHead,
+	"PUT": http.MethodPut, "put": http.MethodPut,
+	"POST": http.MethodPost, "post": http.MethodPost,
+	"DELETE": http.MethodDelete, "delete": http.MethodDelete,
+	"CONNECT": http.MethodConnect, "connect": http.MethodConnect,
+	"OPTIONS": http.MethodOptions, "options": http.MethodOptions,
+	"TRACE": http.MethodTrace, "trace": http.MethodTrace,
+	"PATCH": http.MethodPatch, "patch": http.MethodPatch,
+}
+
+// SanitizeMethod sanitizes the method for use as a metric label. This helps
+// prevent high cardinality on the method label. The name is always upper case.
+func SanitizeMethod(m string) string {
+	if m, ok := methodMap[m]; ok {
+		return m
+	}
+
+	return "OTHER"
+}

internal/metrics/metrics_test.go

@@ -0,0 +1,28 @@
+package metrics
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestSanitizeMethod(t *testing.T) {
+	tests := []struct {
+		method   string
+		expected string
+	}{
+		{method: "get", expected: "GET"},
+		{method: "POST", expected: "POST"},
+		{method: "OPTIONS", expected: "OPTIONS"},
+		{method: "connect", expected: "CONNECT"},
+		{method: "trace", expected: "TRACE"},
+		{method: "UNKNOWN", expected: "OTHER"},
+		{method: strings.Repeat("ohno", 9999), expected: "OTHER"},
+	}
+
+	for _, d := range tests {
+		actual := SanitizeMethod(d.method)
+		if actual != d.expected {
+			t.Errorf("Not same: expected %#v, but got %#v", d.expected, actual)
+		}
+	}
+}

listeners.go

@@ -15,219 +15,225 @@
 package caddy
 
 import (
+	"context"
+	"crypto/tls"
+	"errors"
 	"fmt"
-	"log"
 	"net"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"syscall"
 	"time"
+
+	"github.com/lucas-clemente/quic-go"
+	"github.com/lucas-clemente/quic-go/http3"
 )
 
-// Listen returns a listener suitable for use in a Caddy module.
-// Always be sure to close listeners when you are done with them.
+// Listen is like net.Listen, except Caddy's listeners can overlap
+// each other: multiple listeners may be created on the same socket
+// at the same time. This is useful because during config changes,
+// the new config is started while the old config is still running.
+// When Caddy listeners are closed, the closing logic is virtualized
+// so the underlying socket isn't actually closed until all uses of
+// the socket have been finished. Always be sure to close listeners
+// when you are done with them, just like normal listeners.
 func Listen(network, addr string) (net.Listener, error) {
 	lnKey := network + "/" + addr
 
-	listenersMu.Lock()
-	defer listenersMu.Unlock()
-
-	// if listener already exists, increment usage counter, then return listener
-	if lnGlobal, ok := listeners[lnKey]; ok {
-		atomic.AddInt32(&lnGlobal.usage, 1)
-		return &fakeCloseListener{
-			usage:      &lnGlobal.usage,
-			deadline:   &lnGlobal.deadline,
-			deadlineMu: &lnGlobal.deadlineMu,
-			key:        lnKey,
-			Listener:   lnGlobal.ln,
-		}, nil
-	}
-
-	// or, create new one and save it
-	ln, err := net.Listen(network, addr)
+	sharedLn, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
+		ln, err := net.Listen(network, addr)
+		if err != nil {
+			// https://github.com/caddyserver/caddy/pull/4534
+			if isUnixNetwork(network) && isListenBindAddressAlreadyInUseError(err) {
+				return nil, fmt.Errorf("%w: this can happen if Caddy was forcefully killed", err)
+			}
+			return nil, err
+		}
+		return &sharedListener{Listener: ln, key: lnKey}, nil
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	// make sure to start its usage counter at 1
-	lnGlobal := &globalListener{usage: 1, ln: ln}
-	listeners[lnKey] = lnGlobal
-
-	return &fakeCloseListener{
-		usage:      &lnGlobal.usage,
-		deadline:   &lnGlobal.deadline,
-		deadlineMu: &lnGlobal.deadlineMu,
-		key:        lnKey,
-		Listener:   ln,
-	}, nil
+	return &fakeCloseListener{sharedListener: sharedLn.(*sharedListener)}, nil
 }
 
 // ListenPacket returns a net.PacketConn suitable for use in a Caddy module.
+// It is like Listen except for PacketConns.
 // Always be sure to close the PacketConn when you are done.
 func ListenPacket(network, addr string) (net.PacketConn, error) {
 	lnKey := network + "/" + addr
 
-	listenersMu.Lock()
-	defer listenersMu.Unlock()
-
-	// if listener already exists, increment usage counter, then return listener
-	if lnGlobal, ok := listeners[lnKey]; ok {
-		atomic.AddInt32(&lnGlobal.usage, 1)
-		log.Printf("[DEBUG] %s: Usage counter should not go above 2 or maybe 3, is now: %d", lnKey, atomic.LoadInt32(&lnGlobal.usage)) // TODO: remove
-		return &fakeClosePacketConn{usage: &lnGlobal.usage, key: lnKey, PacketConn: lnGlobal.pc}, nil
-	}
-
-	// or, create new one and save it
-	pc, err := net.ListenPacket(network, addr)
+	sharedPc, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
+		pc, err := net.ListenPacket(network, addr)
+		if err != nil {
+			// https://github.com/caddyserver/caddy/pull/4534
+			if isUnixNetwork(network) && isListenBindAddressAlreadyInUseError(err) {
+				return nil, fmt.Errorf("%w: this can happen if Caddy was forcefully killed", err)
+			}
+			return nil, err
+		}
+		return &sharedPacketConn{PacketConn: pc, key: lnKey}, nil
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	// make sure to start its usage counter at 1
-	lnGlobal := &globalListener{usage: 1, pc: pc}
-	listeners[lnKey] = lnGlobal
-
-	return &fakeClosePacketConn{usage: &lnGlobal.usage, key: lnKey, PacketConn: pc}, nil
+	return &fakeClosePacketConn{sharedPacketConn: sharedPc.(*sharedPacketConn)}, nil
 }
 
-// fakeCloseListener's Close() method is a no-op. This allows
-// stopping servers that are using the listener without giving
-// up the socket; thus, servers become hot-swappable while the
-// listener remains running. Listeners should be re-wrapped in
-// a new fakeCloseListener each time the listener is reused.
-// Other than the 'closed' field (which pertains to this value
-// only), the other fields in this struct should be pointers to
-// the associated globalListener's struct fields (except 'key'
-// which is there for read-only purposes, so it can be a copy).
+// ListenQUIC returns a quic.EarlyListener suitable for use in a Caddy module.
+// Note that the context passed to Accept is currently ignored, so using
+// a context other than context.Background is meaningless.
+func ListenQUIC(addr string, tlsConf *tls.Config) (quic.EarlyListener, error) {
+	lnKey := "quic/" + addr
+
+	sharedEl, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
+		el, err := quic.ListenAddrEarly(addr, http3.ConfigureTLSConfig(tlsConf), &quic.Config{})
+		if err != nil {
+			return nil, err
+		}
+		return &sharedQuicListener{EarlyListener: el, key: lnKey}, nil
+	})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	return &fakeCloseQuicListener{
+		sharedQuicListener: sharedEl.(*sharedQuicListener),
+		context:            ctx, contextCancel: cancel,
+	}, err
+}
+
+// fakeCloseListener is a private wrapper over a listener that
+// is shared. The state of fakeCloseListener is not shared.
+// This allows one user of a socket to "close" the listener
+// while in reality the socket stays open for other users of
+// the listener. In this way, servers become hot-swappable
+// while the listener remains running. Listeners should be
+// re-wrapped in a new fakeCloseListener each time the listener
+// is reused. This type is atomic and values must not be copied.
 type fakeCloseListener struct {
-	closed       int32       // accessed atomically; belongs to this struct only
-	usage        *int32      // accessed atomically; global
-	deadline     *bool       // protected by deadlineMu; global
-	deadlineMu   *sync.Mutex // global
-	key          string      // global, but read-only, so can be copy
-	net.Listener             // global
+	closed          int32 // accessed atomically; belongs to this struct only
+	*sharedListener       // embedded, so we also become a net.Listener
 }
 
-// Accept accepts connections until Close() is called.
 func (fcl *fakeCloseListener) Accept() (net.Conn, error) {
 	// if the listener is already "closed", return error
 	if atomic.LoadInt32(&fcl.closed) == 1 {
-		return nil, fcl.fakeClosedErr()
+		return nil, fakeClosedErr(fcl)
 	}
 
-	// wrap underlying accept
-	conn, err := fcl.Listener.Accept()
+	// call underlying accept
+	conn, err := fcl.sharedListener.Accept()
 	if err == nil {
 		return conn, nil
 	}
 
-	// accept returned with error
-	// TODO: This may be better as a condition variable so the deadline is cleared only once?
-	fcl.deadlineMu.Lock()
-	if *fcl.deadline {
-		switch ln := fcl.Listener.(type) {
-		case *net.TCPListener:
-			_ = ln.SetDeadline(time.Time{})
-		case *net.UnixListener:
-			_ = ln.SetDeadline(time.Time{})
-		}
-		*fcl.deadline = false
-	}
-	fcl.deadlineMu.Unlock()
-
+	// since Accept() returned an error, it may be because our reference to
+	// the listener (this fakeCloseListener) may have been closed, i.e. the
+	// server is shutting down; in that case, we need to clear the deadline
+	// that we set when Close() was called, and return a non-temporary and
+	// non-timeout error value to the caller, masking the "true" error, so
+	// that server loops / goroutines won't retry, linger, and leak
 	if atomic.LoadInt32(&fcl.closed) == 1 {
-		// if we canceled the Accept() by setting a deadline
-		// on the listener, we need to make sure any callers of
-		// Accept() think the listener was actually closed;
-		// if we return the timeout error instead, callers might
-		// simply retry, leaking goroutines for longer
+		// we dereference the sharedListener explicitly even though it's embedded
+		// so that it's clear in the code that side-effects are shared with other
+		// users of this listener, not just our own reference to it; we also don't
+		// do anything with the error because all we could do is log it, but we
+		// expliclty assign it to nothing so we don't forget it's there if needed
+		_ = fcl.sharedListener.clearDeadline()
+
 		if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
-			return nil, fcl.fakeClosedErr()
+			return nil, fakeClosedErr(fcl)
 		}
 	}
 
 	return nil, err
 }
 
-// Close stops accepting new connections without
-// closing the underlying listener, unless no one
-// else is using it.
+// Close stops accepting new connections without closing the
+// underlying listener. The underlying listener is only closed
+// if the caller is the last known user of the socket.
 func (fcl *fakeCloseListener) Close() error {
 	if atomic.CompareAndSwapInt32(&fcl.closed, 0, 1) {
-		// unfortunately, there is no way to cancel any
-		// currently-blocking calls to Accept() that are
-		// awaiting connections since we're not actually
-		// closing the listener; so we cheat by setting
-		// a deadline in the past, which forces it to
-		// time out; note that this only works for
-		// certain types of listeners...
-		fcl.deadlineMu.Lock()
-		if !*fcl.deadline {
-			switch ln := fcl.Listener.(type) {
-			case *net.TCPListener:
-				_ = ln.SetDeadline(time.Now().Add(-1 * time.Minute))
-			case *net.UnixListener:
-				_ = ln.SetDeadline(time.Now().Add(-1 * time.Minute))
-			}
-			*fcl.deadline = true
-		}
-		fcl.deadlineMu.Unlock()
-
-		// since we're no longer using this listener,
-		// decrement the usage counter and, if no one
-		// else is using it, close underlying listener
-		if atomic.AddInt32(fcl.usage, -1) == 0 {
-			listenersMu.Lock()
-			delete(listeners, fcl.key)
-			listenersMu.Unlock()
-			err := fcl.Listener.Close()
-			if err != nil {
-				return err
-			}
-		}
-
+		// There are two ways I know of to get an Accept()
+		// function to return to the server loop that called
+		// it: close the listener, or set a deadline in the
+		// past. Obviously, we can't close the socket yet
+		// since others may be using it (hence this whole
+		// file). But we can set the deadline in the past,
+		// and this is kind of cheating, but it works, and
+		// it apparently even works on Windows.
+		_ = fcl.sharedListener.setDeadline()
+		_, _ = listenerPool.Delete(fcl.sharedListener.key)
 	}
-
 	return nil
 }
 
-func (fcl *fakeCloseListener) fakeClosedErr() error {
+type fakeCloseQuicListener struct {
+	closed              int32 // accessed atomically; belongs to this struct only
+	*sharedQuicListener       // embedded, so we also become a quic.EarlyListener
+	context             context.Context
+	contextCancel       context.CancelFunc
+}
+
+// Currently Accept ignores the passed context, however a situation where
+// someone would need a hotswappable QUIC-only (not http3, since it uses context.Background here)
+// server on which Accept would be called with non-empty contexts
+// (mind that the default net listeners' Accept doesn't take a context argument)
+// sounds way too rare for us to sacrifice efficiency here.
+func (fcql *fakeCloseQuicListener) Accept(_ context.Context) (quic.EarlyConnection, error) {
+	conn, err := fcql.sharedQuicListener.Accept(fcql.context)
+	if err == nil {
+		return conn, nil
+	}
+
+	// if the listener is "closed", return a fake closed error instead
+	if atomic.LoadInt32(&fcql.closed) == 1 && errors.Is(err, context.Canceled) {
+		return nil, fakeClosedErr(fcql)
+	}
+	return nil, err
+}
+
+func (fcql *fakeCloseQuicListener) Close() error {
+	if atomic.CompareAndSwapInt32(&fcql.closed, 0, 1) {
+		fcql.contextCancel()
+		_, _ = listenerPool.Delete(fcql.sharedQuicListener.key)
+	}
+	return nil
+}
+
+// fakeClosedErr returns an error value that is not temporary
+// nor a timeout, suitable for making the caller think the
+// listener is actually closed
+func fakeClosedErr(l interface{ Addr() net.Addr }) error {
 	return &net.OpError{
 		Op:   "accept",
-		Net:  fcl.Listener.Addr().Network(),
-		Addr: fcl.Listener.Addr(),
+		Net:  l.Addr().Network(),
+		Addr: l.Addr(),
 		Err:  errFakeClosed,
 	}
 }
 
+// ErrFakeClosed is the underlying error value returned by
+// fakeCloseListener.Accept() after Close() has been called,
+// indicating that it is pretending to be closed so that the
+// server using it can terminate, while the underlying
+// socket is actually left open.
+var errFakeClosed = fmt.Errorf("listener 'closed' 😉")
+
+// fakeClosePacketConn is like fakeCloseListener, but for PacketConns.
 type fakeClosePacketConn struct {
-	closed int32  // accessed atomically
-	usage  *int32 // accessed atomically
-	key    string
-	net.PacketConn
+	closed            int32 // accessed atomically; belongs to this struct only
+	*sharedPacketConn       // embedded, so we also become a net.PacketConn
 }
 
 func (fcpc *fakeClosePacketConn) Close() error {
-	log.Println("[DEBUG] Fake-closing underlying packet conn") // TODO: remove this
-
 	if atomic.CompareAndSwapInt32(&fcpc.closed, 0, 1) {
-		// since we're no longer using this listener,
-		// decrement the usage counter and, if no one
-		// else is using it, close underlying listener
-		if atomic.AddInt32(fcpc.usage, -1) == 0 {
-			listenersMu.Lock()
-			delete(listeners, fcpc.key)
-			listenersMu.Unlock()
-			err := fcpc.PacketConn.Close()
-			if err != nil {
-				return err
-			}
-		}
+		_, _ = listenerPool.Delete(fcpc.sharedPacketConn.key)
 	}
-
 	return nil
 }
 
@@ -249,28 +255,75 @@ func (fcpc fakeClosePacketConn) SyscallConn() (syscall.RawConn, error) {
 	return nil, fmt.Errorf("SyscallConn() not implemented for %T", fcpc.PacketConn)
 }
 
-// ErrFakeClosed is the underlying error value returned by
-// fakeCloseListener.Accept() after Close() has been called,
-// indicating that it is pretending to be closed so that the
-// server using it can terminate, while the underlying
-// socket is actually left open.
-var errFakeClosed = fmt.Errorf("listener 'closed' 😉")
-
-// globalListener keeps global state for a listener
-// that may be shared by multiple servers. In other
-// words, values in this struct exist only once and
-// all other uses of these values point to the ones
-// in this struct. In particular, the usage count
-// (how many callers are using the listener), the
-// actual listener, and synchronization of the
-// listener's deadline changes are singular, global
-// values that must not be copied.
-type globalListener struct {
-	usage      int32 // accessed atomically
-	deadline   bool
+// sharedListener is a wrapper over an underlying listener. The listener
+// and the other fields on the struct are shared state that is synchronized,
+// so sharedListener structs must never be copied (always use a pointer).
+type sharedListener struct {
+	net.Listener
+	key        string // uniquely identifies this listener
+	deadline   bool   // whether a deadline is currently set
 	deadlineMu sync.Mutex
-	ln         net.Listener
-	pc         net.PacketConn
+}
+
+func (sl *sharedListener) clearDeadline() error {
+	var err error
+	sl.deadlineMu.Lock()
+	if sl.deadline {
+		switch ln := sl.Listener.(type) {
+		case *net.TCPListener:
+			err = ln.SetDeadline(time.Time{})
+		case *net.UnixListener:
+			err = ln.SetDeadline(time.Time{})
+		}
+		sl.deadline = false
+	}
+	sl.deadlineMu.Unlock()
+	return err
+}
+
+func (sl *sharedListener) setDeadline() error {
+	timeInPast := time.Now().Add(-1 * time.Minute)
+	var err error
+	sl.deadlineMu.Lock()
+	if !sl.deadline {
+		switch ln := sl.Listener.(type) {
+		case *net.TCPListener:
+			err = ln.SetDeadline(timeInPast)
+		case *net.UnixListener:
+			err = ln.SetDeadline(timeInPast)
+		}
+		sl.deadline = true
+	}
+	sl.deadlineMu.Unlock()
+	return err
+}
+
+// Destruct is called by the UsagePool when the listener is
+// finally not being used anymore. It closes the socket.
+func (sl *sharedListener) Destruct() error {
+	return sl.Listener.Close()
+}
+
+// sharedQuicListener is like sharedListener, but for quic.EarlyListeners.
+type sharedQuicListener struct {
+	quic.EarlyListener
+	key string
+}
+
+// Destruct closes the underlying QUIC listener.
+func (sql *sharedQuicListener) Destruct() error {
+	return sql.EarlyListener.Close()
+}
+
+// sharedPacketConn is like sharedListener, but for net.PacketConns.
+type sharedPacketConn struct {
+	net.PacketConn
+	key string
+}
+
+// Destruct closes the underlying socket.
+func (spc *sharedPacketConn) Destruct() error {
+	return spc.PacketConn.Close()
 }
 
 // NetworkAddress contains the individual components
@@ -349,6 +402,20 @@ func isUnixNetwork(netw string) bool {
 	return netw == "unix" || netw == "unixgram" || netw == "unixpacket"
 }
 
+func isListenBindAddressAlreadyInUseError(err error) bool {
+	switch networkOperationError := err.(type) {
+	case *net.OpError:
+		switch syscallError := networkOperationError.Err.(type) {
+		case *os.SyscallError:
+			if syscallError.Syscall == "bind" {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
 // ParseNetworkAddress parses addr into its individual
 // components. The input string is expected to be of
 // the form "network/host:port-range" where any part is
@@ -445,10 +512,8 @@ type ListenerWrapper interface {
 	WrapListener(net.Listener) net.Listener
 }
 
-var (
-	listeners   = make(map[string]*globalListener)
-	listenersMu sync.Mutex
-)
+// listenerPool stores and allows reuse of active listeners.
+var listenerPool = NewUsagePool()
 
 const maxPortSpan = 65535
 

listeners_fuzz.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build gofuzz
 // +build gofuzz
 
 package caddy

logging.go

@@ -661,9 +661,15 @@ func newDefaultProductionLog() (*defaultCustomLog, error) {
 
 	cl.buildCore()
 
+	logger := zap.New(cl.core)
+
+	// capture logs from other libraries which
+	// may not be using zap logging directly
+	_ = zap.RedirectStdLog(logger)
+
 	return &defaultCustomLog{
 		CustomLog: cl,
-		logger:    zap.New(cl.core),
+		logger:    logger,
 	}, nil
 }
 

metrics.go

@@ -2,9 +2,8 @@ package caddy
 
 import (
 	"net/http"
-	"strconv"
-	"strings"
 
+	"github.com/caddyserver/caddy/v2/internal/metrics"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -46,8 +45,8 @@ func instrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler)
 		d := newDelegator(w)
 		next.ServeHTTP(d, r)
 		counter.With(prometheus.Labels{
-			"code":   sanitizeCode(d.status),
-			"method": strings.ToUpper(r.Method),
+			"code":   metrics.SanitizeCode(d.status),
+			"method": metrics.SanitizeMethod(r.Method),
 		}).Inc()
 	})
 }
@@ -67,12 +66,3 @@ func (d *delegator) WriteHeader(code int) {
 	d.status = code
 	d.ResponseWriter.WriteHeader(code)
 }
-
-func sanitizeCode(s int) string {
-	switch s {
-	case 0, 200:
-		return "200"
-	default:
-		return strconv.Itoa(s)
-	}
-}

modules/caddyhttp/app.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"net"
 	"net/http"
 	"strconv"
 	"time"
@@ -50,6 +49,8 @@ func init() {
 // `{http.request.body}` | The request body (⚠️ inefficient; use only for debugging)
 // `{http.request.cookie.*}` | HTTP request cookie
 // `{http.request.duration}` | Time up to now spent handling the request (after decoding headers from client)
+// `{http.request.duration_ms}` | Same as 'duration', but in milliseconds.
+// `{http.request.uuid}` | The request unique identifier
 // `{http.request.header.*}` | Specific request header field
 // `{http.request.host.labels.*}` | Request host labels (0-based from right); e.g. for foo.example.com: 0=com, 1=example, 2=foo
 // `{http.request.host}` | The host part of the request's Host header
@@ -110,14 +111,15 @@ type App struct {
 	// be forcefully closed.
 	GracePeriod caddy.Duration `json:"grace_period,omitempty"`
 
+	Strict *StrictOptions `json:"strict,omitempty"`
+
 	// Servers is the list of servers, keyed by arbitrary names chosen
 	// at your discretion for your own convenience; the keys do not
 	// affect functionality.
 	Servers map[string]*Server `json:"servers,omitempty"`
 
-	servers     []*http.Server
-	h3servers   []*http3.Server
-	h3listeners []net.PacketConn
+	servers   []*http.Server
+	h3servers []*http3.Server
 
 	ctx    caddy.Context
 	logger *zap.Logger
@@ -127,6 +129,13 @@ type App struct {
 	allCertDomains []string
 }
 
+type StrictOptions struct {
+	Disabled            bool `json:"disable,omitempty"`
+	LenientQueryStrings bool `json:"lenient_query_strings,omitempty"`
+	LenientPaths        bool `json:"lenient_paths,omitempty"`
+	LenientHeaders      bool `json:"lenient_headers,omitempty"`
+}
+
 // CaddyModule returns the Caddy module information.
 func (App) CaddyModule() caddy.ModuleInfo {
 	return caddy.ModuleInfo{
@@ -162,6 +171,7 @@ func (app *App) Provision(ctx caddy.Context) error {
 		srv.tlsApp = app.tlsApp
 		srv.logger = app.logger.Named("log")
 		srv.errorLogger = app.logger.Named("log.error")
+		srv.strict = app.Strict
 
 		// only enable access logs if configured
 		if srv.Logs != nil {
@@ -352,9 +362,9 @@ func (app *App) Start() error {
 						app.logger.Info("enabling experimental HTTP/3 listener",
 							zap.String("addr", hostport),
 						)
-						h3ln, err := caddy.ListenPacket("udp", hostport)
+						h3ln, err := caddy.ListenQUIC(hostport, tlsCfg)
 						if err != nil {
-							return fmt.Errorf("getting HTTP/3 UDP listener: %v", err)
+							return fmt.Errorf("getting HTTP/3 QUIC listener: %v", err)
 						}
 						h3srv := &http3.Server{
 							Server: &http.Server{
@@ -365,9 +375,8 @@ func (app *App) Start() error {
 							},
 						}
 						//nolint:errcheck
-						go h3srv.Serve(h3ln)
+						go h3srv.ServeListener(h3ln)
 						app.h3servers = append(app.h3servers, h3srv)
-						app.h3listeners = append(app.h3listeners, h3ln)
 						srv.h3server = h3srv
 					}
 					/////////
@@ -425,13 +434,6 @@ func (app *App) Stop() error {
 		}
 	}
 
-	// close the http3 servers; it's unclear whether the bug reported in
-	// https://github.com/caddyserver/caddy/pull/2727#issuecomment-526856566
-	// was ever truly fixed, since it seemed racey/nondeterministic; but
-	// recent tests in 2020 were unable to replicate the issue again after
-	// repeated attempts (the bug manifested after a config reload; i.e.
-	// reusing a http3 server or listener was problematic), but it seems
-	// to be working fine now
 	for _, s := range app.h3servers {
 		// TODO: CloseGracefully, once implemented upstream
 		// (see https://github.com/lucas-clemente/quic-go/issues/2103)
@@ -440,17 +442,6 @@ func (app *App) Stop() error {
 			return err
 		}
 	}
-
-	// closing an http3.Server does not close their underlying listeners
-	// since apparently the listener can be used both by servers and
-	// clients at the same time; so we need to manually call Close()
-	// on the underlying h3 listeners (see lucas-clemente/quic-go#2103)
-	for _, pc := range app.h3listeners {
-		err := pc.Close()
-		if err != nil {
-			return err
-		}
-	}
 	return nil
 }
 

modules/caddyhttp/autohttps.go

@@ -31,13 +31,20 @@ import (
 // HTTPS is enabled automatically and by default when
 // qualifying hostnames are available from the config.
 type AutoHTTPSConfig struct {
-	// If true, automatic HTTPS will be entirely disabled.
+	// If true, automatic HTTPS will be entirely disabled,
+	// including certificate management and redirects.
 	Disabled bool `json:"disable,omitempty"`
 
 	// If true, only automatic HTTP->HTTPS redirects will
-	// be disabled.
+	// be disabled, but other auto-HTTPS features will
+	// remain enabled.
 	DisableRedir bool `json:"disable_redirects,omitempty"`
 
+	// If true, automatic certificate management will be
+	// disabled, but other auto-HTTPS features will
+	// remain enabled.
+	DisableCerts bool `json:"disable_certificates,omitempty"`
+
 	// Hosts/domain names listed here will not be included
 	// in automatic HTTPS (they will not have certificates
 	// loaded nor redirects applied).
@@ -104,12 +111,13 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 			srv.AutoHTTPS = new(AutoHTTPSConfig)
 		}
 		if srv.AutoHTTPS.Disabled {
+			app.logger.Warn("automatic HTTPS is completely disabled for server", zap.String("server_name", srvName))
 			continue
 		}
 
 		// skip if all listeners use the HTTP port
 		if !srv.listenersUseAnyPortOtherThan(app.httpPort()) {
-			app.logger.Info("server is listening only on the HTTP port, so no automatic HTTPS will be applied to this server",
+			app.logger.Warn("server is listening only on the HTTP port, so no automatic HTTPS will be applied to this server",
 				zap.String("server_name", srvName),
 				zap.Int("http_port", app.httpPort()),
 			)
@@ -166,30 +174,40 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 
 		// for all the hostnames we found, filter them so we have
 		// a deduplicated list of names for which to obtain certs
-		for d := range serverDomainSet {
-			if certmagic.SubjectQualifiesForCert(d) &&
-				!srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.SkipCerts) {
-				// if a certificate for this name is already loaded,
-				// don't obtain another one for it, unless we are
-				// supposed to ignore loaded certificates
-				if !srv.AutoHTTPS.IgnoreLoadedCerts &&
-					len(app.tlsApp.AllMatchingCertificates(d)) > 0 {
-					app.logger.Info("skipping automatic certificate management because one or more matching certificates are already loaded",
-						zap.String("domain", d),
-						zap.String("server_name", srvName),
-					)
+		// (only if cert management not disabled for this server)
+		if srv.AutoHTTPS.DisableCerts {
+			app.logger.Warn("skipping automated certificate management for server because it is disabled", zap.String("server_name", srvName))
+		} else {
+			for d := range serverDomainSet {
+				// the implicit Tailscale manager module will get its own certs at run-time
+				if isTailscaleDomain(d) {
 					continue
 				}
 
-				// most clients don't accept wildcards like *.tld... we
-				// can handle that, but as a courtesy, warn the user
-				if strings.Contains(d, "*") &&
-					strings.Count(strings.Trim(d, "."), ".") == 1 {
-					app.logger.Warn("most clients do not trust second-level wildcard certificates (*.tld)",
-						zap.String("domain", d))
-				}
+				if certmagic.SubjectQualifiesForCert(d) &&
+					!srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.SkipCerts) {
+					// if a certificate for this name is already loaded,
+					// don't obtain another one for it, unless we are
+					// supposed to ignore loaded certificates
+					if !srv.AutoHTTPS.IgnoreLoadedCerts &&
+						len(app.tlsApp.AllMatchingCertificates(d)) > 0 {
+						app.logger.Info("skipping automatic certificate management because one or more matching certificates are already loaded",
+							zap.String("domain", d),
+							zap.String("server_name", srvName),
+						)
+						continue
+					}
 
-				uniqueDomainsForCerts[d] = struct{}{}
+					// most clients don't accept wildcards like *.tld... we
+					// can handle that, but as a courtesy, warn the user
+					if strings.Contains(d, "*") &&
+						strings.Count(strings.Trim(d, "."), ".") == 1 {
+						app.logger.Warn("most clients do not trust second-level wildcard certificates (*.tld)",
+							zap.String("domain", d))
+					}
+
+					uniqueDomainsForCerts[d] = struct{}{}
+				}
 			}
 		}
 
@@ -200,19 +218,22 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 
 		// nothing left to do if auto redirects are disabled
 		if srv.AutoHTTPS.DisableRedir {
+			app.logger.Warn("automatic HTTP->HTTPS redirects are disabled", zap.String("server_name", srvName))
 			continue
 		}
 
-		app.logger.Info("enabling automatic HTTP->HTTPS redirects",
-			zap.String("server_name", srvName),
-		)
+		app.logger.Info("enabling automatic HTTP->HTTPS redirects", zap.String("server_name", srvName))
 
 		// create HTTP->HTTPS redirects
-		for _, addr := range srv.Listen {
+		for _, listenAddr := range srv.Listen {
 			// figure out the address we will redirect to...
-			addr, err := caddy.ParseNetworkAddress(addr)
+			addr, err := caddy.ParseNetworkAddress(listenAddr)
 			if err != nil {
-				return fmt.Errorf("%s: invalid listener address: %v", srvName, addr)
+				msg := "%s: invalid listener address: %v"
+				if strings.Count(listenAddr, ":") > 1 {
+					msg = msg + ", there are too many colons, so the port is ambiguous. Did you mean to wrap the IPv6 address with [] brackets?"
+				}
+				return fmt.Errorf(msg, srvName, listenAddr)
 			}
 
 			// this address might not have a hostname, i.e. might be a
@@ -232,7 +253,7 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 				// port, we'll have to choose one, so prefer the HTTPS port
 				if _, ok := redirDomains[d]; !ok ||
 					addr.StartPort == uint(app.httpsPort()) {
-					redirDomains[d] = append(redirDomains[d], addr)
+					redirDomains[d] = []caddy.NetworkAddress{addr}
 				}
 			}
 		}
@@ -418,7 +439,7 @@ func (app *App) makeRedirRoute(redirToPort uint, matcherSet MatcherSet) Route {
 	}
 }
 
-// createAutomationPolicy ensures that automated certificates for this
+// createAutomationPolicies ensures that automated certificates for this
 // app are managed properly. This adds up to two automation policies:
 // one for the public names, and one for the internal names. If a catch-all
 // automation policy exists, it will be shallow-copied and used as the
@@ -459,6 +480,22 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []stri
 			}
 		}
 
+		// if no external managers were configured, enable
+		// implicit Tailscale support for convenience
+		if ap.Managers == nil {
+			ts, err := implicitTailscale(ctx)
+			if err != nil {
+				return err
+			}
+			ap.Managers = []certmagic.Manager{ts}
+
+			// must reprovision the automation policy so that the underlying
+			// CertMagic config knows about the updated Managers
+			if err := ap.Provision(app.tlsApp); err != nil {
+				return fmt.Errorf("re-provisioning automation policy: %v", err)
+			}
+		}
+
 		// while we're here, is this the catch-all/base policy?
 		if !foundBasePolicy && len(ap.Subjects) == 0 {
 			basePolicy = ap
@@ -467,10 +504,19 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []stri
 	}
 
 	if basePolicy == nil {
-		// no base policy found, we will make one!
+		// no base policy found; we will make one
 		basePolicy = new(caddytls.AutomationPolicy)
 	}
 
+	if basePolicy.Managers == nil {
+		// add implicit Tailscale integration, for harmless convenience
+		ts, err := implicitTailscale(ctx)
+		if err != nil {
+			return err
+		}
+		basePolicy.Managers = []certmagic.Manager{ts}
+	}
+
 	// if the basePolicy has an existing ACMEIssuer (particularly to
 	// include any type that embeds/wraps an ACMEIssuer), let's use it
 	// (I guess we just use the first one?), otherwise we'll make one
@@ -482,8 +528,8 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []stri
 		}
 	}
 	if baseACMEIssuer == nil {
-		// note that this happens if basePolicy.Issuer is nil
-		// OR if it is not nil but is not an ACMEIssuer
+		// note that this happens if basePolicy.Issuers is empty
+		// OR if it is not empty but does not have not an ACMEIssuer
 		baseACMEIssuer = new(caddytls.ACMEIssuer)
 	}
 
@@ -653,4 +699,15 @@ func (app *App) automaticHTTPSPhase2() error {
 	return nil
 }
 
+// implicitTailscale returns a new and provisioned Tailscale module configured to be optional.
+func implicitTailscale(ctx caddy.Context) (caddytls.Tailscale, error) {
+	ts := caddytls.Tailscale{Optional: true}
+	err := ts.Provision(ctx)
+	return ts, err
+}
+
+func isTailscaleDomain(name string) bool {
+	return strings.HasSuffix(strings.ToLower(name), ".ts.net")
+}
+
 type acmeCapable interface{ GetACMEIssuer() *caddytls.ACMEIssuer }

modules/caddyhttp/caddyhttp.go

@@ -20,7 +20,6 @@ import (
 	"io"
 	"net"
 	"net/http"
-	"net/url"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -228,7 +227,6 @@ func StatusCodeMatches(actual, configured int) bool {
 // never be outside of root. The resulting path can be used
 // with the local file system.
 func SanitizedPathJoin(root, reqPath string) string {
-	reqPath, _ = url.PathUnescape(reqPath)
 	if root == "" {
 		root = "."
 	}

modules/caddyhttp/celmatcher.go

@@ -33,6 +33,7 @@ import (
 	"github.com/google/cel-go/common/types/traits"
 	"github.com/google/cel-go/ext"
 	"github.com/google/cel-go/interpreter/functions"
+	"go.uber.org/zap"
 	exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
 	"google.golang.org/protobuf/proto"
 )
@@ -62,6 +63,8 @@ type MatchExpression struct {
 	expandedExpr string
 	prg          cel.Program
 	ta           ref.TypeAdapter
+
+	log *zap.Logger
 }
 
 // CaddyModule returns the Caddy module information.
@@ -83,7 +86,9 @@ func (m *MatchExpression) UnmarshalJSON(data []byte) error {
 }
 
 // Provision sets ups m.
-func (m *MatchExpression) Provision(_ caddy.Context) error {
+func (m *MatchExpression) Provision(ctx caddy.Context) error {
+	m.log = ctx.Logger(m)
+
 	// replace placeholders with a function call - this is just some
 	// light (and possibly naïve) syntactic sugar
 	m.expandedExpr = placeholderRegexp.ReplaceAllString(m.Expr, placeholderExpansion)
@@ -137,9 +142,13 @@ func (m *MatchExpression) Provision(_ caddy.Context) error {
 
 // Match returns true if r matches m.
 func (m MatchExpression) Match(r *http.Request) bool {
-	out, _, _ := m.prg.Eval(map[string]interface{}{
+	out, _, err := m.prg.Eval(map[string]interface{}{
 		"request": celHTTPRequest{r},
 	})
+	if err != nil {
+		m.log.Error("evaluating expression", zap.Error(err))
+		return false
+	}
 	if outBool, ok := out.Value().(bool); ok {
 		return outBool
 	}
@@ -150,7 +159,11 @@ func (m MatchExpression) Match(r *http.Request) bool {
 // UnmarshalCaddyfile implements caddyfile.Unmarshaler.
 func (m *MatchExpression) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	for d.Next() {
-		m.Expr = strings.Join(d.RemainingArgs(), " ")
+		if d.CountRemainingArgs() > 1 {
+			m.Expr = strings.Join(d.RemainingArgsRaw(), " ")
+		} else {
+			m.Expr = d.Val()
+		}
 	}
 	return nil
 }

modules/caddyhttp/celmatcher_test.go

@@ -116,7 +116,7 @@ eqp31wM9il1n+guTNyxJd+FzVAH+hCZE5K+tCgVDdVFUlDEHHbS/wqb2PSIoouLV
 			}
 
 			if tt.expression.Match(req) != tt.wantResult {
-				t.Errorf("MatchExpression.Match() expected to return '%t', for expression : '%s'", tt.wantResult, tt.expression)
+				t.Errorf("MatchExpression.Match() expected to return '%t', for expression : '%s'", tt.wantResult, tt.expression.Expr)
 			}
 
 		})

modules/caddyhttp/errors.go

@@ -82,6 +82,9 @@ func (e HandlerError) Error() string {
 	return strings.TrimSpace(s)
 }
 
+// Unwrap returns the underlying error value. See the `errors` package for info.
+func (e HandlerError) Unwrap() error { return e.Err }
+
 // randString returns a string of n random characters.
 // It is not even remotely secure OR a proper distribution.
 // But it's good enough for some things. It excludes certain

modules/caddyhttp/fileserver/browse.go

@@ -16,6 +16,7 @@ package fileserver
 
 import (
 	"bytes"
+	_ "embed"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -31,6 +32,9 @@ import (
 	"go.uber.org/zap"
 )
 
+//go:embed browse.html
+var defaultBrowseTemplate string
+
 // Browse configures directory browsing.
 type Browse struct {
 	// Use this template file instead of the default browse template.
@@ -56,8 +60,11 @@ func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter,
 	// original URI is necessary because that's the URI the browser knows,
 	// we don't want to redirect from internally-rewritten URIs.)
 	// See https://github.com/caddyserver/caddy/issues/4205.
+	// We also redirect if the path is empty, because this implies the path
+	// prefix was fully stripped away by a `handle_path` handler for example.
+	// See https://github.com/caddyserver/caddy/issues/4466.
 	origReq := r.Context().Value(caddyhttp.OriginalRequestCtxKey).(http.Request)
-	if path.Base(origReq.URL.Path) == path.Base(r.URL.Path) {
+	if r.URL.Path == "" || path.Base(origReq.URL.Path) == path.Base(r.URL.Path) {
 		if !strings.HasSuffix(origReq.URL.Path, "/") {
 			fsrv.logger.Debug("redirecting to trailing slash to preserve hrefs", zap.String("request_path", r.URL.Path))
 			origReq.URL.Path += "/"
@@ -137,12 +144,7 @@ func (fsrv *FileServer) loadDirectoryContents(dir *os.File, root, urlPath string
 	// user can presumably browse "up" to parent folder if path is longer than "/"
 	canGoUp := len(urlPath) > 1
 
-	l, err := fsrv.directoryListing(files, canGoUp, root, urlPath, repl)
-	if err != nil {
-		return browseTemplateContext{}, err
-	}
-
-	return l, nil
+	return fsrv.directoryListing(files, canGoUp, root, urlPath, repl), nil
 }
 
 // browseApplyQueryParams applies query parameters to the listing.

modules/caddyhttp/fileserver/browse.html

@@ -1,20 +1,4 @@
-// Copyright 2015 Matthew Holt and The Caddy Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package fileserver
-
-const defaultBrowseTemplate = `<!DOCTYPE html>
+<!DOCTYPE html>
 <html>
 	<head>
 		<title>{{html .Name}}</title>
@@ -40,7 +24,7 @@ h1 a:hover {
 }
 
 a:visited {
-  color: #800080;
+	color: #800080;
 }
 
 header,
@@ -477,4 +461,4 @@ footer {
 			timeList.forEach(localizeDatetime);
 		</script>
 	</body>
-</html>`
+</html>

modules/caddyhttp/fileserver/browsetplcontext.go

@@ -24,10 +24,11 @@ import (
 	"time"
 
 	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/dustin/go-humanize"
 )
 
-func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, root, urlPath string, repl *caddy.Replacer) (browseTemplateContext, error) {
+func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, root, urlPath string, repl *caddy.Replacer) browseTemplateContext {
 	filesToHide := fsrv.transformHidePaths(repl)
 
 	var dirCount, fileCount int
@@ -42,26 +43,30 @@ func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, root
 
 		isDir := f.IsDir() || isSymlinkTargetDir(f, root, urlPath)
 
-		u := url.URL{Path: url.PathEscape(name)}
-
 		// add the slash after the escape of path to avoid escaping the slash as well
 		if isDir {
-			u.Path += "/"
+			name += "/"
 			dirCount++
 		} else {
 			fileCount++
 		}
 
-		fileIsSymlink := isSymlink(f)
 		size := f.Size()
+		fileIsSymlink := isSymlink(f)
 		if fileIsSymlink {
-			info, err := os.Stat(name)
-			if err != nil {
-				return browseTemplateContext{}, err
+			path := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, f.Name()))
+			fileInfo, err := os.Stat(path)
+			if err == nil {
+				size = fileInfo.Size()
 			}
-			size = info.Size()
+			// An error most likely means the symlink target doesn't exist,
+			// which isn't entirely unusual and shouldn't fail the listing.
+			// In this case, just use the size of the symlink itself, which
+			// was already set above.
 		}
 
+		u := url.URL{Path: "./" + name} // prepend with "./" to fix paths with ':' in the name
+
 		fileInfos = append(fileInfos, fileInfo{
 			IsDir:     isDir,
 			IsSymlink: fileIsSymlink,
@@ -72,15 +77,15 @@ func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, root
 			Mode:      f.Mode(),
 		})
 	}
-
+	name, _ := url.PathUnescape(urlPath)
 	return browseTemplateContext{
-		Name:     path.Base(urlPath),
+		Name:     path.Base(name),
 		Path:     urlPath,
 		CanGoUp:  canGoUp,
 		Items:    fileInfos,
 		NumDirs:  dirCount,
 		NumFiles: fileCount,
-	}, nil
+	}
 }
 
 // browseTemplateContext provides the template context for directory listings.
@@ -128,13 +133,16 @@ func (l browseTemplateContext) Breadcrumbs() []crumb {
 	if lpath[len(lpath)-1] == '/' {
 		lpath = lpath[:len(lpath)-1]
 	}
-
 	parts := strings.Split(lpath, "/")
 	result := make([]crumb, len(parts))
 	for i, p := range parts {
 		if i == 0 && p == "" {
 			p = "/"
 		}
+		// the directory name could include an encoded slash in its path,
+		// so the item name should be unescaped in the loop rather than unescaping the
+		// entire path outside the loop.
+		p, _ = url.PathUnescape(p)
 		lnk := strings.Repeat("../", len(parts)-i-1)
 		result[i] = crumb{Link: lnk, Text: p}
 	}

modules/caddyhttp/fileserver/browsetplcontext_test.go

@@ -36,6 +36,19 @@ func TestBreadcrumbs(t *testing.T) {
 			{Link: "../", Text: "quux"},
 			{Link: "", Text: "corge"},
 		}},
+		{"/مجلد/", []crumb{
+			{Link: "../", Text: "/"},
+			{Link: "", Text: "مجلد"},
+		}},
+		{"/مجلد-1/مجلد-2", []crumb{
+			{Link: "../../", Text: "/"},
+			{Link: "../", Text: "مجلد-1"},
+			{Link: "", Text: "مجلد-2"},
+		}},
+		{"/مجلد%2F1", []crumb{
+			{Link: "../", Text: "/"},
+			{Link: "", Text: "مجلد/1"},
+		}},
 	}
 
 	for _, d := range testdata {

modules/caddyhttp/fileserver/caddyfile.go

@@ -120,6 +120,12 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 				falseBool := false
 				fsrv.CanonicalURIs = &falseBool
 
+			case "pass_thru":
+				if h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				fsrv.PassThru = true
+
 			default:
 				return nil, h.Errf("unknown subdirective '%s'", h.Val())
 			}

modules/caddyhttp/fileserver/command.go

@@ -129,7 +129,7 @@ func cmdFileServer(fs caddycmd.Flags) (int, error) {
 		return caddy.ExitCodeFailedStartup, err
 	}
 
-	log.Printf("Caddy 2 serving static files on %s", listen)
+	log.Printf("Caddy serving static files on %s", listen)
 
 	select {}
 }

modules/caddyhttp/fileserver/staticfiles.go

@@ -46,7 +46,7 @@ type FileServer struct {
 	Root string `json:"root,omitempty"`
 
 	// A list of files or folders to hide; the file server will pretend as if
-	// they don't exist. Accepts globular patterns like "*.ext" or "/foo/*/bar"
+	// they don't exist. Accepts globular patterns like `*.ext` or `/foo/*/bar`
 	// as well as placeholders. Because site roots can be dynamic, this list
 	// uses file system paths, not request paths. To clarify, the base of
 	// relative paths is the current working directory, NOT the site root.
@@ -200,6 +200,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 	var implicitIndexFile bool
 	if info.IsDir() && len(fsrv.IndexNames) > 0 {
 		for _, indexPage := range fsrv.IndexNames {
+			indexPage := repl.ReplaceAll(indexPage, "")
 			indexPath := caddyhttp.SanitizedPathJoin(filename, indexPage)
 			if fileHidden(indexPath, filesToHide) {
 				// pretend this file doesn't exist

modules/caddyhttp/headers/caddyfile.go

@@ -222,7 +222,7 @@ func applyHeaderOp(ops *HeaderOps, respHeaderOps *RespHeaderOps, field, value, r
 		if ops.Add == nil {
 			ops.Add = make(http.Header)
 		}
-		ops.Add.Set(field[1:], value)
+		ops.Add.Add(field[1:], value)
 
 	case strings.HasPrefix(field, "-"): // delete
 		ops.Delete = append(ops.Delete, field[1:])

modules/caddyhttp/httpredirectlistener.go

@@ -0,0 +1,156 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddyhttp
+
+import (
+	"bufio"
+	"fmt"
+	"net"
+	"net/http"
+	"sync"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+)
+
+func init() {
+	caddy.RegisterModule(HTTPRedirectListenerWrapper{})
+}
+
+// HTTPRedirectListenerWrapper provides HTTP->HTTPS redirects for
+// connections that come on the TLS port as an HTTP request,
+// by detecting using the first few bytes that it's not a TLS
+// handshake, but instead an HTTP request.
+//
+// This is especially useful when using a non-standard HTTPS port.
+// A user may simply type the address in their browser without the
+// https:// scheme, which would cause the browser to attempt the
+// connection over HTTP, but this would cause a "Client sent an
+// HTTP request to an HTTPS server" error response.
+//
+// This listener wrapper must be placed BEFORE the "tls" listener
+// wrapper, for it to work properly.
+type HTTPRedirectListenerWrapper struct{}
+
+func (HTTPRedirectListenerWrapper) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "caddy.listeners.http_redirect",
+		New: func() caddy.Module { return new(HTTPRedirectListenerWrapper) },
+	}
+}
+
+func (h *HTTPRedirectListenerWrapper) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	return nil
+}
+
+func (h *HTTPRedirectListenerWrapper) WrapListener(l net.Listener) net.Listener {
+	return &httpRedirectListener{l}
+}
+
+// httpRedirectListener is listener that checks the first few bytes
+// of the request when the server is intended to accept HTTPS requests,
+// to respond to an HTTP request with a redirect.
+type httpRedirectListener struct {
+	net.Listener
+}
+
+// Accept waits for and returns the next connection to the listener,
+// wrapping it with a httpRedirectConn.
+func (l *httpRedirectListener) Accept() (net.Conn, error) {
+	c, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+
+	return &httpRedirectConn{
+		Conn: c,
+		r:    bufio.NewReader(c),
+	}, nil
+}
+
+type httpRedirectConn struct {
+	net.Conn
+	once sync.Once
+	r    *bufio.Reader
+}
+
+// Read tries to peek at the first few bytes of the request, and if we get
+// an error reading the headers, and that error was due to the bytes looking
+// like an HTTP request, then we perform a HTTP->HTTPS redirect on the same
+// port as the original connection.
+func (c *httpRedirectConn) Read(p []byte) (int, error) {
+	var errReturn error
+	c.once.Do(func() {
+		firstBytes, err := c.r.Peek(5)
+		if err != nil {
+			return
+		}
+
+		// If the request doesn't look like HTTP, then it's probably
+		// TLS bytes and we don't need to do anything.
+		if !firstBytesLookLikeHTTP(firstBytes) {
+			return
+		}
+
+		// Parse the HTTP request, so we can get the Host and URL to redirect to.
+		req, err := http.ReadRequest(c.r)
+		if err != nil {
+			return
+		}
+
+		// Build the redirect response, using the same Host and URL,
+		// but replacing the scheme with https.
+		headers := make(http.Header)
+		headers.Add("Location", "https://"+req.Host+req.URL.String())
+		resp := &http.Response{
+			Proto:      "HTTP/1.0",
+			Status:     "308 Permanent Redirect",
+			StatusCode: 308,
+			ProtoMajor: 1,
+			ProtoMinor: 0,
+			Header:     headers,
+		}
+
+		err = resp.Write(c.Conn)
+		if err != nil {
+			errReturn = fmt.Errorf("couldn't write HTTP->HTTPS redirect")
+			return
+		}
+
+		errReturn = fmt.Errorf("redirected HTTP request on HTTPS port")
+		c.Conn.Close()
+	})
+
+	if errReturn != nil {
+		return 0, errReturn
+	}
+
+	return c.r.Read(p)
+}
+
+// firstBytesLookLikeHTTP reports whether a TLS record header
+// looks like it might've been a misdirected plaintext HTTP request.
+func firstBytesLookLikeHTTP(hdr []byte) bool {
+	switch string(hdr[:5]) {
+	case "GET /", "HEAD ", "POST ", "PUT /", "OPTIO":
+		return true
+	}
+	return false
+}
+
+var (
+	_ caddy.ListenerWrapper = (*HTTPRedirectListenerWrapper)(nil)
+	_ caddyfile.Unmarshaler = (*HTTPRedirectListenerWrapper)(nil)
+)

modules/caddyhttp/map/caddyfile.go

@@ -56,6 +56,11 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 		if len(handler.Destinations) == 0 {
 			return nil, h.Err("missing destination argument(s)")
 		}
+		for _, dest := range handler.Destinations {
+			if shorthand := httpcaddyfile.WasReplacedPlaceholderShorthand(dest); shorthand != "" {
+				return nil, h.Errf("destination %s conflicts with a Caddyfile placeholder shorthand", shorthand)
+			}
+		}
 
 		// mappings
 		for h.NextBlock(0) {
@@ -74,11 +79,12 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 			// every other line maps one input to one or more outputs
 			in := h.Val()
 			var outs []interface{}
-			for _, out := range h.RemainingArgs() {
-				if out == "-" {
+			for h.NextArg() {
+				val := h.ScalarVal()
+				if val == "-" {
 					outs = append(outs, nil)
 				} else {
-					outs = append(outs, out)
+					outs = append(outs, val)
 				}
 			}
 

modules/caddyhttp/map/map.go

@@ -145,6 +145,10 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhtt
 				return string(result), true
 			}
 			if input == m.Input {
+				if outputStr, ok := output.(string); ok {
+					// NOTE: if the output has a placeholder that has the same key as the input, this is infinite recursion
+					return repl.ReplaceAll(outputStr, ""), true
+				}
 				return output, true
 			}
 		}