Also reject null byte

* Add renegotiation option in reverseproxy tls client

* Update modules/caddyhttp/reverseproxy/httptransport.go

Co-authored-by: Matt Holt <mholt@users.noreply.github.com>

.editorconfig

@@ -0,0 +1,5 @@
+[*]
+end_of_line = lf
+
+[caddytest/integration/caddyfile_adapt/*.txt]
+indent_style = tab

.github/SECURITY.md

@@ -48,7 +48,7 @@ We consider publicly-registered domain names to be public information. This nece
 
 It will speed things up if you suggest a working patch, such as a code diff, and explain why and how it works. Reports that are not actionable, do not contain enough information, are too pushy/demanding, or are not able to convince us that it is a viable and practical attack on the web server itself may be deferred to a later time or possibly ignored, depending on available resources. Priority will be given to credible, responsible reports that are constructive, specific, and actionable. (We get a lot of invalid reports.) Thank you for understanding.
 
-When you are ready, please email Matt Holt (the author) directly: matt [at] lightcodelabs [dot com].
+When you are ready, please email Matt Holt (the author) directly: matt at dyanim dot com.
 
 Please don't encrypt the email body. It only makes the process more complicated.
 

.github/workflows/ci.yml

@@ -19,12 +19,20 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
-        go: [ '1.16', '1.17' ]
+        go: [ '1.17', '1.18' ]
+
+        include:
+        # Set the minimum Go patch version for the given Go minor
+        # Usable via ${{ matrix.GO_SEMVER }}
+        - go: '1.17'
+          GO_SEMVER: '~1.17.9'
+
+        - go: '1.18'
+          GO_SEMVER: '~1.18.1'
 
         # Set some variables per OS, usable via ${{ matrix.VAR }}
         # CADDY_BIN_PATH: the path to the compiled Caddy binary, for artifact publishing
         # SUCCESS: the typical value for $? per OS (Windows/pwsh returns 'True')
-        include:
         - os: ubuntu-latest
           CADDY_BIN_PATH: ./cmd/caddy/caddy
           SUCCESS: 0
@@ -41,12 +49,13 @@ jobs:
 
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
-        go-version: ${{ matrix.go }}
+        go-version: ${{ matrix.GO_SEMVER }}
+        check-latest: true
 
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # These tools would be useful if we later decide to reinvestigate
     # publishing test/coverage reports to some tool for easier consumption
@@ -69,12 +78,20 @@ jobs:
         printf "Git version: $(git version)\n\n"
         # Calculate the short SHA1 hash of the git commit
         echo "::set-output name=short_sha::$(git rev-parse --short HEAD)"
-        echo "::set-output name=go_cache::$(go env GOCACHE)"
 
     - name: Cache the build cache
       uses: actions/cache@v2
       with:
-        path: ${{ steps.vars.outputs.go_cache }}
+        # In order:
+        # * Module download cache
+        # * Build cache (Linux)
+        # * Build cache (Mac)
+        # * Build cache (Windows)
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
+          ~/Library/Caches/go-build
+          ~\AppData\Local\go-build
         key: ${{ runner.os }}-${{ matrix.go }}-go-ci-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-${{ matrix.go }}-go-ci
@@ -130,7 +147,7 @@ jobs:
     continue-on-error: true  # August 2020: s390x VM is down due to weather and power issues
     steps:
       - name: Checkout code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Run Tests
         run: |
           mkdir -p ~/.ssh && echo -e "${SSH_KEY//_/\\n}" > ~/.ssh/id_ecdsa && chmod og-rwx ~/.ssh/id_ecdsa
@@ -155,7 +172,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       
       - uses: goreleaser/goreleaser-action@v2
         with:

.github/workflows/cross-build.yml

@@ -16,14 +16,22 @@ jobs:
       fail-fast: false
       matrix:
         goos: ['android', 'linux', 'solaris', 'illumos', 'dragonfly', 'freebsd', 'openbsd', 'plan9', 'windows', 'darwin', 'netbsd']
-        go: [ '1.17' ]
+        go: [ '1.18' ]
+
+        include:
+        # Set the minimum Go patch version for the given Go minor
+        # Usable via ${{ matrix.GO_SEMVER }}
+        - go: '1.18'
+          GO_SEMVER: '~1.18.1'
+
     runs-on: ubuntu-latest
     continue-on-error: true
     steps:
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
-          go-version: ${{ matrix.go }}
+          go-version: ${{ matrix.GO_SEMVER }}
+          check-latest: true
 
       - name: Print Go version and environment
         id: vars
@@ -34,18 +42,22 @@ jobs:
           go env
           printf "\n\nSystem environment:\n\n"
           env
-          echo "::set-output name=go_cache::$(go env GOCACHE)"
 
       - name: Cache the build cache
         uses: actions/cache@v2
         with:
-          path: ${{ steps.vars.outputs.go_cache }}
+          # In order:
+          # * Module download cache
+          # * Build cache (Linux)
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
           key: cross-build-go${{ matrix.go }}-${{ matrix.goos }}-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             cross-build-go${{ matrix.go }}-${{ matrix.goos }}
 
       - name: Checkout code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Run Build
         env:

.github/workflows/lint.yml

@@ -16,10 +16,15 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
         with:
-          version: v1.31
+          go-version: '~1.17.9'
+          check-latest: true
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.44
           # Optional: show only new issues if it's a pull request. The default value is `false`.
           # only-new-issues: true

.github/workflows/release.yml

@@ -11,22 +11,30 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        go: [ '1.17' ]
+        go: [ '1.18' ]
+
+        include:
+        # Set the minimum Go patch version for the given Go minor
+        # Usable via ${{ matrix.GO_SEMVER }}
+        - go: '1.18'
+          GO_SEMVER: '~1.18.1'
+
     runs-on: ${{ matrix.os }}
 
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
-        go-version: ${{ matrix.go }}
+        go-version: ${{ matrix.GO_SEMVER }}
+        check-latest: true
 
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
 
     # Force fetch upstream tags -- because 65 minutes
-    # tl;dr: actions/checkout@v2 runs this line:
+    # tl;dr: actions/checkout@v3 runs this line:
     #   git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +ebc278ec98bb24f2852b61fde2a9bf2e3d83818b:refs/tags/
     # which makes its own local lightweight tag, losing all the annotations in the process. Our earlier script ran:
     #   git fetch --prune --unshallow
@@ -48,7 +56,6 @@ jobs:
         env
         echo "::set-output name=version_tag::${GITHUB_REF/refs\/tags\//}"
         echo "::set-output name=short_sha::$(git rev-parse --short HEAD)"
-        echo "::set-output name=go_cache::$(go env GOCACHE)"
 
         # Add "pip install" CLI tools to PATH
         echo ~/.local/bin >> $GITHUB_PATH
@@ -83,7 +90,12 @@ jobs:
     - name: Cache the build cache
       uses: actions/cache@v2
       with:
-        path: ${{ steps.vars.outputs.go_cache }}
+        # In order:
+        # * Module download cache
+        # * Build cache (Linux)
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
         key: ${{ runner.os }}-go${{ matrix.go }}-release-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go${{ matrix.go }}-release
@@ -99,7 +111,7 @@ jobs:
         TAG: ${{ steps.vars.outputs.version_tag }}
 
     # Only publish on non-special tags (e.g. non-beta)
-    # We will continue to push to Gemfury for the forseeable future, although
+    # We will continue to push to Gemfury for the foreseeable future, although
     # Cloudsmith is probably better, to not break things for existing users of Gemfury.
     # See https://gemfury.com/caddy/deb:caddy
     - name: Publish .deb to Gemfury

.gitignore

@@ -1,6 +1,7 @@
 _gitignore/
 *.log
 Caddyfile
+Caddyfile.*
 !caddyfile/
 
 # artifacts from pprof tooling

.golangci.yml

@@ -1,6 +1,6 @@
 linters-settings:
   errcheck:
-    ignore: fmt:.*,io/ioutil:^Read.*,go.uber.org/zap/zapcore:^Add.*
+    ignore: fmt:.*,go.uber.org/zap/zapcore:^Add.*
     ignoretests: true
 
 linters:

.goreleaser.yml

@@ -6,8 +6,7 @@ before:
     # subsequently causes gorleaser to refuse running.
     - mkdir -p caddy-build
     - cp cmd/caddy/main.go caddy-build/main.go
-    - cp ./go.mod caddy-build/go.mod
-    - sed -i.bkp 's|github.com/caddyserver/caddy/v2|caddy|g' ./caddy-build/go.mod
+    - /bin/sh -c 'cd ./caddy-build && go mod init caddy'
     # GoReleaser doesn't seem to offer {{.Tag}} at this stage, so we have to embed it into the env
     # so we run: TAG=$(git describe --abbrev=0) goreleaser release --rm-dist --skip-publish --skip-validate
     - go mod edit -require=github.com/caddyserver/caddy/v2@{{.Env.TAG}} ./caddy-build/go.mod
@@ -36,9 +35,9 @@ builds:
   - s390x
   - ppc64le
   goarm:
-  - 5
-  - 6
-  - 7
+  - "5"
+  - "6"
+  - "7"
   ignore:
     - goos: darwin
       goarch: arm
@@ -56,7 +55,7 @@ builds:
       goarch: s390x
     - goos: freebsd
       goarch: arm
-      goarm: 5
+      goarm: "5"
   flags:
   - -trimpath
   ldflags:
@@ -75,7 +74,7 @@ nfpms:
   - id: default
     package_name: caddy
 
-    vendor: Light Code Labs
+    vendor: Dyanim
     homepage: https://caddyserver.com
     maintainer: Matthew Holt <mholt@users.noreply.github.com>
     description: |

README.md

@@ -75,7 +75,7 @@ For other install options, see https://caddyserver.com/docs/install.
 
 Requirements:
 
-- [Go 1.16 or newer](https://golang.org/dl/)
+- [Go 1.17 or newer](https://golang.org/dl/)
 
 ### For development
  
@@ -176,6 +176,8 @@ Please use our [issue tracker](https://github.com/caddyserver/caddy/issues) only
 
 ## About
 
+Matthew Holt began developing Caddy in 2014 while studying computer science at Brigham Young University. (The name "Caddy" was chosen because this software helps with the tedious, mundane tasks of serving the Web, and is also a single place for multiple things to be organized together.) It soon became the first web server to use HTTPS automatically and by default, and now has hundreds of contributors and has served trillions of HTTPS requests.
+
 **The name "Caddy" is trademarked.** The name of the software is "Caddy", not "Caddy Server" or "CaddyServer". Please call it "Caddy" or, if you wish to clarify, "the Caddy web server". Caddy is a registered trademark of Stack Holdings GmbH.
 
 - _Project on Twitter: [@caddyserver](https://twitter.com/caddyserver)_

admin.go

@@ -26,7 +26,6 @@ import (
 	"expvar"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/pprof"
@@ -43,6 +42,7 @@ import (
 	"github.com/caddyserver/certmagic"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 )
 
 // AdminConfig configures Caddy's API endpoint, which is used
@@ -92,6 +92,10 @@ type AdminConfig struct {
 	//
 	// EXPERIMENTAL: This feature is subject to change.
 	Remote *RemoteAdmin `json:"remote,omitempty"`
+
+	// Holds onto the routers so that we can later provision them
+	// if they require provisioning.
+	routers []AdminRouter
 }
 
 // ConfigSettings configures the management of configuration.
@@ -101,20 +105,26 @@ type ConfigSettings struct {
 	// are not persisted; only configs that are pushed to Caddy get persisted.
 	Persist *bool `json:"persist,omitempty"`
 
-	// Loads a configuration to use. This is helpful if your configs are
-	// managed elsewhere, and you want Caddy to pull its config dynamically
+	// Loads a new configuration. This is helpful if your configs are
+	// managed elsewhere and you want Caddy to pull its config dynamically
 	// when it starts. The pulled config completely replaces the current
 	// one, just like any other config load. It is an error if a pulled
-	// config is configured to pull another config.
+	// config is configured to pull another config without a load_delay,
+	// as this creates a tight loop.
 	//
 	// EXPERIMENTAL: Subject to change.
 	LoadRaw json.RawMessage `json:"load,omitempty" caddy:"namespace=caddy.config_loaders inline_key=module"`
 
-	// The interval to pull config. With a non-zero value, will pull config
-	// from config loader (eg. a http loader) with given interval.
+	// The duration after which to load config. If set, config will be pulled
+	// from the config loader after this duration. A delay is required if a
+	// dynamically-loaded config is configured to load yet another config. To
+	// load configs on a regular interval, ensure this value is set the same
+	// on all loaded configs; it can also be variable if needed, and to stop
+	// the loop, simply remove dynamic config loading from the next-loaded
+	// config.
 	//
 	// EXPERIMENTAL: Subject to change.
-	LoadInterval Duration `json:"load_interval,omitempty"`
+	LoadDelay Duration `json:"load_delay,omitempty"`
 }
 
 // IdentityConfig configures management of this server's identity. An identity
@@ -184,7 +194,7 @@ type AdminPermissions struct {
 
 // newAdminHandler reads admin's config and returns an http.Handler suitable
 // for use in an admin endpoint server, which will be listening on listenAddr.
-func (admin AdminConfig) newAdminHandler(addr NetworkAddress, remote bool) adminHandler {
+func (admin *AdminConfig) newAdminHandler(addr NetworkAddress, remote bool) adminHandler {
 	muxWrap := adminHandler{mux: http.NewServeMux()}
 
 	// secure the local or remote endpoint respectively
@@ -193,6 +203,7 @@ func (admin AdminConfig) newAdminHandler(addr NetworkAddress, remote bool) admin
 	} else {
 		muxWrap.enforceHost = !addr.isWildcardInterface()
 		muxWrap.allowedOrigins = admin.allowedOrigins(addr)
+		muxWrap.enforceOrigin = admin.EnforceOrigin
 	}
 
 	addRouteWithMetrics := func(pattern string, handlerLabel string, h http.Handler) {
@@ -243,17 +254,39 @@ func (admin AdminConfig) newAdminHandler(addr NetworkAddress, remote bool) admin
 		for _, route := range router.Routes() {
 			addRoute(route.Pattern, handlerLabel, route.Handler)
 		}
+		admin.routers = append(admin.routers, router)
 	}
 
 	return muxWrap
 }
 
+// provisionAdminRouters provisions all the router modules
+// in the admin.api namespace that need provisioning.
+func (admin *AdminConfig) provisionAdminRouters(ctx Context) error {
+	for _, router := range admin.routers {
+		provisioner, ok := router.(Provisioner)
+		if !ok {
+			continue
+		}
+
+		err := provisioner.Provision(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	// We no longer need the routers once provisioned, allow for GC
+	admin.routers = nil
+
+	return nil
+}
+
 // allowedOrigins returns a list of origins that are allowed.
 // If admin.Origins is nil (null), the provided listen address
 // will be used as the default origin. If admin.Origins is
 // empty, no origins will be allowed, effectively bricking the
 // endpoint for non-unix-socket endpoints, but whatever.
-func (admin AdminConfig) allowedOrigins(addr NetworkAddress) []string {
+func (admin AdminConfig) allowedOrigins(addr NetworkAddress) []*url.URL {
 	uniqueOrigins := make(map[string]struct{})
 	for _, o := range admin.Origins {
 		uniqueOrigins[o] = struct{}{}
@@ -277,8 +310,23 @@ func (admin AdminConfig) allowedOrigins(addr NetworkAddress) []string {
 			uniqueOrigins[addr.JoinHostPort(0)] = struct{}{}
 		}
 	}
-	allowed := make([]string, 0, len(uniqueOrigins))
-	for origin := range uniqueOrigins {
+	allowed := make([]*url.URL, 0, len(uniqueOrigins))
+	for originStr := range uniqueOrigins {
+		var origin *url.URL
+		if strings.Contains(originStr, "://") {
+			var err error
+			origin, err = url.Parse(originStr)
+			if err != nil {
+				continue
+			}
+			origin.Path = ""
+			origin.RawPath = ""
+			origin.Fragment = ""
+			origin.RawFragment = ""
+			origin.RawQuery = ""
+		} else {
+			origin = &url.URL{Host: originStr}
+		}
 		allowed = append(allowed, origin)
 	}
 	return allowed
@@ -310,25 +358,26 @@ func replaceLocalAdminServer(cfg *Config) error {
 		}
 	}()
 
-	// always get a valid admin config
-	adminConfig := DefaultAdminConfig
-	if cfg != nil && cfg.Admin != nil {
-		adminConfig = cfg.Admin
+	// set a default if admin wasn't otherwise configured
+	if cfg.Admin == nil {
+		cfg.Admin = &AdminConfig{
+			Listen: DefaultAdminListen,
+		}
 	}
 
 	// if new admin endpoint is to be disabled, we're done
-	if adminConfig.Disabled {
+	if cfg.Admin.Disabled {
 		Log().Named("admin").Warn("admin endpoint disabled")
 		return nil
 	}
 
 	// extract a singular listener address
-	addr, err := parseAdminListenAddr(adminConfig.Listen, DefaultAdminListen)
+	addr, err := parseAdminListenAddr(cfg.Admin.Listen, DefaultAdminListen)
 	if err != nil {
 		return err
 	}
 
-	handler := adminConfig.newAdminHandler(addr, false)
+	handler := cfg.Admin.newAdminHandler(addr, false)
 
 	ln, err := Listen(addr.Network, addr.JoinHostPort(0))
 	if err != nil {
@@ -358,8 +407,8 @@ func replaceLocalAdminServer(cfg *Config) error {
 
 	adminLogger.Info("admin endpoint started",
 		zap.String("address", addr.String()),
-		zap.Bool("enforce_origin", adminConfig.EnforceOrigin),
-		zap.Strings("origins", handler.allowedOrigins))
+		zap.Bool("enforce_origin", cfg.Admin.EnforceOrigin),
+		zap.Array("origins", loggableURLArray(handler.allowedOrigins)))
 
 	if !handler.enforceHost {
 		adminLogger.Warn("admin endpoint on open interface; host checking disabled",
@@ -467,6 +516,9 @@ func replaceRemoteAdminServer(ctx Context, cfg *Config) error {
 	}
 
 	// create TLS config that will enforce mutual authentication
+	if identityCertCache == nil {
+		return fmt.Errorf("cannot enable remote admin without a certificate cache; configure identity management to initialize a certificate cache")
+	}
 	cmCfg := cfg.Admin.Identity.certmagicConfig(remoteLogger, false)
 	tlsConfig := cmCfg.TLSConfig()
 	tlsConfig.NextProtos = nil // this server does not solve ACME challenges
@@ -648,10 +700,10 @@ type AdminRoute struct {
 type adminHandler struct {
 	mux *http.ServeMux
 
-	// security for local/plaintext) endpoint, on by default
+	// security for local/plaintext endpoint
 	enforceOrigin  bool
 	enforceHost    bool
-	allowedOrigins []string
+	allowedOrigins []*url.URL
 
 	// security for remote/encrypted endpoint
 	remoteControl *RemoteAdmin
@@ -660,11 +712,17 @@ type adminHandler struct {
 // ServeHTTP is the external entry point for API requests.
 // It will only be called once per request.
 func (h adminHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ip, port, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		ip = r.RemoteAddr
+		port = ""
+	}
 	log := Log().Named("admin.api").With(
 		zap.String("method", r.Method),
 		zap.String("host", r.Host),
 		zap.String("uri", r.RequestURI),
-		zap.String("remote_addr", r.RemoteAddr),
+		zap.String("remote_ip", ip),
+		zap.String("remote_port", port),
 		zap.Reflect("headers", r.Header),
 	)
 	if r.TLS != nil {
@@ -771,8 +829,8 @@ func (h adminHandler) handleError(w http.ResponseWriter, r *http.Request, err er
 // rebinding attacks.
 func (h adminHandler) checkHost(r *http.Request) error {
 	var allowed bool
-	for _, allowedHost := range h.allowedOrigins {
-		if r.Host == allowedHost {
+	for _, allowedOrigin := range h.allowedOrigins {
+		if r.Host == allowedOrigin.Host {
 			allowed = true
 			break
 		}
@@ -791,43 +849,45 @@ func (h adminHandler) checkHost(r *http.Request) error {
 // sites from issuing requests to our listener. It
 // returns the origin that was obtained from r.
 func (h adminHandler) checkOrigin(r *http.Request) (string, error) {
-	origin := h.getOriginHost(r)
-	if origin == "" {
-		return origin, APIError{
+	originStr, origin := h.getOrigin(r)
+	if origin == nil {
+		return "", APIError{
 			HTTPStatus: http.StatusForbidden,
-			Err:        fmt.Errorf("missing required Origin header"),
+			Err:        fmt.Errorf("required Origin header is missing or invalid"),
 		}
 	}
 	if !h.originAllowed(origin) {
-		return origin, APIError{
+		return "", APIError{
 			HTTPStatus: http.StatusForbidden,
-			Err:        fmt.Errorf("client is not allowed to access from origin %s", origin),
+			Err:        fmt.Errorf("client is not allowed to access from origin '%s'", originStr),
 		}
 	}
-	return origin, nil
+	return origin.String(), nil
 }
 
-func (h adminHandler) getOriginHost(r *http.Request) string {
+func (h adminHandler) getOrigin(r *http.Request) (string, *url.URL) {
 	origin := r.Header.Get("Origin")
 	if origin == "" {
 		origin = r.Header.Get("Referer")
 	}
 	originURL, err := url.Parse(origin)
-	if err == nil && originURL.Host != "" {
-		origin = originURL.Host
+	if err != nil {
+		return origin, nil
 	}
-	return origin
+	originURL.Path = ""
+	originURL.RawPath = ""
+	originURL.Fragment = ""
+	originURL.RawFragment = ""
+	originURL.RawQuery = ""
+	return origin, originURL
 }
 
-func (h adminHandler) originAllowed(origin string) bool {
+func (h adminHandler) originAllowed(origin *url.URL) bool {
 	for _, allowedOrigin := range h.allowedOrigins {
-		originCopy := origin
-		if !strings.Contains(allowedOrigin, "://") {
-			// no scheme specified, so allow both
-			originCopy = strings.TrimPrefix(originCopy, "http://")
-			originCopy = strings.TrimPrefix(originCopy, "https://")
+		if allowedOrigin.Scheme != "" && origin.Scheme != allowedOrigin.Scheme {
+			continue
 		}
-		if originCopy == allowedOrigin {
+		if origin.Host == allowedOrigin.Host {
 			return true
 		}
 	}
@@ -878,7 +938,7 @@ func handleConfig(w http.ResponseWriter, r *http.Request) error {
 		forceReload := r.Header.Get("Cache-Control") == "must-revalidate"
 
 		err := changeConfig(r.Method, r.URL.Path, body, forceReload)
-		if err != nil {
+		if err != nil && !errors.Is(err, errSameConfig) {
 			return err
 		}
 
@@ -897,10 +957,16 @@ func handleConfigID(w http.ResponseWriter, r *http.Request) error {
 
 	parts := strings.Split(idPath, "/")
 	if len(parts) < 3 || parts[2] == "" {
-		return fmt.Errorf("request path is missing object ID")
+		return APIError{
+			HTTPStatus: http.StatusBadRequest,
+			Err:        fmt.Errorf("request path is missing object ID"),
+		}
 	}
 	if parts[0] != "" || parts[1] != "id" {
-		return fmt.Errorf("malformed object path")
+		return APIError{
+			HTTPStatus: http.StatusBadRequest,
+			Err:        fmt.Errorf("malformed object path"),
+		}
 	}
 	id := parts[2]
 
@@ -909,7 +975,10 @@ func handleConfigID(w http.ResponseWriter, r *http.Request) error {
 	expanded, ok := rawCfgIndex[id]
 	defer currentCfgMu.RUnlock()
 	if !ok {
-		return fmt.Errorf("unknown object ID '%s'", id)
+		return APIError{
+			HTTPStatus: http.StatusNotFound,
+			Err:        fmt.Errorf("unknown object ID '%s'", id),
+		}
 	}
 
 	// piece the full URL path back together
@@ -931,7 +1000,7 @@ func handleStop(w http.ResponseWriter, r *http.Request) error {
 		Log().Error("unable to notify stopping to service manager", zap.Error(err))
 	}
 
-	exitProcess(Log().Named("admin.api"))
+	exitProcess(context.Background(), Log().Named("admin.api"))
 	return nil
 }
 
@@ -1181,6 +1250,18 @@ func decodeBase64DERCert(certStr string) (*x509.Certificate, error) {
 	return x509.ParseCertificate(derBytes)
 }
 
+type loggableURLArray []*url.URL
+
+func (ua loggableURLArray) MarshalLogArray(enc zapcore.ArrayEncoder) error {
+	if ua == nil {
+		return nil
+	}
+	for _, u := range ua {
+		enc.AppendString(u.String())
+	}
+	return nil
+}
+
 var (
 	// DefaultAdminListen is the address for the local admin
 	// listener, if none is specified at startup.
@@ -1190,19 +1271,13 @@ var (
 	// (TLS-authenticated) admin listener, if enabled and not
 	// specified otherwise.
 	DefaultRemoteAdminListen = ":2021"
-
-	// DefaultAdminConfig is the default configuration
-	// for the local administration endpoint.
-	DefaultAdminConfig = &AdminConfig{
-		Listen: DefaultAdminListen,
-	}
 )
 
 // PIDFile writes a pidfile to the file at filename. It
 // will get deleted before the process gracefully exits.
 func PIDFile(filename string) error {
 	pid := []byte(strconv.Itoa(os.Getpid()) + "\n")
-	err := ioutil.WriteFile(filename, pid, 0600)
+	err := os.WriteFile(filename, pid, 0600)
 	if err != nil {
 		return err
 	}

caddy.go

@@ -18,9 +18,9 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -111,14 +111,19 @@ func Load(cfgJSON []byte, forceReload bool) error {
 		}
 	}()
 
-	return changeConfig(http.MethodPost, "/"+rawConfigKey, cfgJSON, forceReload)
+	err := changeConfig(http.MethodPost, "/"+rawConfigKey, cfgJSON, forceReload)
+	if errors.Is(err, errSameConfig) {
+		err = nil // not really an error
+	}
+	return err
 }
 
 // changeConfig changes the current config (rawCfg) according to the
 // method, traversed via the given path, and uses the given input as
 // the new value (if applicable; i.e. "DELETE" doesn't have an input).
 // If the resulting config is the same as the previous, no reload will
-// occur unless forceReload is true. This function is safe for
+// occur unless forceReload is true. If the config is unchanged and not
+// forcefully reloaded, then errConfigUnchanged This function is safe for
 // concurrent use.
 func changeConfig(method, path string, input []byte, forceReload bool) error {
 	switch method {
@@ -149,8 +154,8 @@ func changeConfig(method, path string, input []byte, forceReload bool) error {
 
 	// if nothing changed, no need to do a whole reload unless the client forces it
 	if !forceReload && bytes.Equal(rawCfgJSON, newCfg) {
-		Log().Named("admin.api").Info("config is unchanged")
-		return nil
+		Log().Info("config is unchanged")
+		return errSameConfig
 	}
 
 	// find any IDs in this config and index them
@@ -269,8 +274,8 @@ func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
 		newCfg.Admin != nil &&
 		newCfg.Admin.Config != nil &&
 		newCfg.Admin.Config.LoadRaw != nil &&
-		newCfg.Admin.Config.LoadInterval <= 0 {
-		return fmt.Errorf("recursive config loading detected: pulled configs cannot pull other configs without positive load_interval")
+		newCfg.Admin.Config.LoadDelay <= 0 {
+		return fmt.Errorf("recursive config loading detected: pulled configs cannot pull other configs without positive load_delay")
 	}
 
 	// run the new config and start all its apps
@@ -300,7 +305,7 @@ func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
 				zap.String("dir", dir),
 				zap.Error(err))
 		} else {
-			err := ioutil.WriteFile(ConfigAutosavePath, cfgJSON, 0600)
+			err := os.WriteFile(ConfigAutosavePath, cfgJSON, 0600)
 			if err == nil {
 				Log().Info("autosaved config (load with --resume flag)", zap.String("file", ConfigAutosavePath))
 			} else {
@@ -428,9 +433,16 @@ func run(newCfg *Config, start bool) error {
 		return nil
 	}
 
+	// Provision any admin routers which may need to access
+	// some of the other apps at runtime
+	err = newCfg.Admin.provisionAdminRouters(ctx)
+	if err != nil {
+		return err
+	}
+
 	// Start
 	err = func() error {
-		var started []string
+		started := make([]string, 0, len(newCfg.apps))
 		for name, a := range newCfg.apps {
 			err := a.Start()
 			if err != nil {
@@ -481,49 +493,74 @@ func finishSettingUp(ctx Context, cfg *Config) error {
 		if err != nil {
 			return fmt.Errorf("loading config loader module: %s", err)
 		}
-		runLoadedConfig := func(config []byte) {
-			Log().Info("applying dynamically-loaded config", zap.String("loader_module", val.(Module).CaddyModule().ID.Name()), zap.Int("pull_interval", int(cfg.Admin.Config.LoadInterval)))
-			currentCfgMu.Lock()
-			err := unsyncedDecodeAndRun(config, false)
-			currentCfgMu.Unlock()
-			if err == nil {
-				Log().Info("dynamically-loaded config applied successfully")
-			} else {
-				Log().Error("running dynamically-loaded config failed", zap.Error(err))
+
+		logger := Log().Named("config_loader").With(
+			zap.String("module", val.(Module).CaddyModule().ID.Name()),
+			zap.Int("load_delay", int(cfg.Admin.Config.LoadDelay)))
+
+		runLoadedConfig := func(config []byte) error {
+			logger.Info("applying dynamically-loaded config")
+			err := changeConfig(http.MethodPost, "/"+rawConfigKey, config, false)
+			if errors.Is(err, errSameConfig) {
+				return err
 			}
+			if err != nil {
+				logger.Error("failed to run dynamically-loaded config", zap.Error(err))
+				return err
+			}
+			logger.Info("successfully applied dynamically-loaded config")
+			return nil
 		}
-		if cfg.Admin.Config.LoadInterval > 0 {
+
+		if cfg.Admin.Config.LoadDelay > 0 {
 			go func() {
-				select {
-				// if LoadInterval is positive, will wait for the interval and then run with new config
-				case <-time.After(time.Duration(cfg.Admin.Config.LoadInterval)):
-					loadedConfig, err := val.(ConfigLoader).LoadConfig(ctx)
-					if err != nil {
-						Log().Error("loading dynamic config failed", zap.Error(err))
-						return
+				// the loop is here to iterate ONLY if there is an error, a no-op config load,
+				// or an unchanged config; in which case we simply wait the delay and try again
+				for {
+					timer := time.NewTimer(time.Duration(cfg.Admin.Config.LoadDelay))
+					select {
+					case <-timer.C:
+						loadedConfig, err := val.(ConfigLoader).LoadConfig(ctx)
+						if err != nil {
+							logger.Error("failed loading dynamic config; will retry", zap.Error(err))
+							continue
+						}
+						if loadedConfig == nil {
+							logger.Info("dynamically-loaded config was nil; will retry")
+							continue
+						}
+						err = runLoadedConfig(loadedConfig)
+						if errors.Is(err, errSameConfig) {
+							logger.Info("dynamically-loaded config was unchanged; will retry")
+							continue
+						}
+					case <-ctx.Done():
+						if !timer.Stop() {
+							<-timer.C
+						}
+						logger.Info("stopping dynamic config loading")
 					}
-					runLoadedConfig(loadedConfig)
-				case <-ctx.Done():
-					return
+					break
 				}
 			}()
 		} else {
-			// if no LoadInterval is provided, will load config synchronously
+			// if no LoadDelay is provided, will load config synchronously
 			loadedConfig, err := val.(ConfigLoader).LoadConfig(ctx)
 			if err != nil {
 				return fmt.Errorf("loading dynamic config from %T: %v", val, err)
 			}
 			// do this in a goroutine so current config can finish being loaded; otherwise deadlock
-			go runLoadedConfig(loadedConfig)
+			go func() { _ = runLoadedConfig(loadedConfig) }()
 		}
-
 	}
 
 	return nil
 }
 
-// ConfigLoader is a type that can load a Caddy config. The
-// returned config must be valid Caddy JSON.
+// ConfigLoader is a type that can load a Caddy config. If
+// the return value is non-nil, it must be valid Caddy JSON;
+// if nil or with non-nil error, it is considered to be a
+// no-op load and may be retried later.
 type ConfigLoader interface {
 	LoadConfig(Context) ([]byte, error)
 }
@@ -584,7 +621,7 @@ func Validate(cfg *Config) error {
 // PID file, and shuts down admin endpoint(s) in a goroutine.
 // Errors are logged along the way, and an appropriate exit
 // code is emitted.
-func exitProcess(logger *zap.Logger) {
+func exitProcess(ctx context.Context, logger *zap.Logger) {
 	if logger == nil {
 		logger = Log()
 	}
@@ -599,7 +636,7 @@ func exitProcess(logger *zap.Logger) {
 	}
 
 	// clean up certmagic locks
-	certmagic.CleanUpOwnLocks(logger)
+	certmagic.CleanUpOwnLocks(ctx, logger)
 
 	// remove pidfile
 	if pidfile != "" {
@@ -700,13 +737,13 @@ func ParseDuration(s string) (time.Duration, error) {
 // have its own unique ID.
 func InstanceID() (uuid.UUID, error) {
 	uuidFilePath := filepath.Join(AppDataDir(), "instance.uuid")
-	uuidFileBytes, err := ioutil.ReadFile(uuidFilePath)
+	uuidFileBytes, err := os.ReadFile(uuidFilePath)
 	if os.IsNotExist(err) {
 		uuid, err := uuid.NewRandom()
 		if err != nil {
 			return uuid, err
 		}
-		err = ioutil.WriteFile(uuidFilePath, []byte(uuid.String()), 0600)
+		err = os.WriteFile(uuidFilePath, []byte(uuid.String()), 0600)
 		return uuid, err
 	} else if err != nil {
 		return [16]byte{}, err
@@ -775,5 +812,10 @@ var (
 	rawCfgIndex map[string]string
 )
 
+// errSameConfig is returned if the new config is the same
+// as the old one. This isn't usually an actual, actionable
+// error; it's mostly a sentinel value.
+var errSameConfig = errors.New("config is unchanged")
+
 // ImportPath is the package import path for Caddy core.
 const ImportPath = "github.com/caddyserver/caddy/v2"

caddyconfig/caddyfile/adapter.go

@@ -88,7 +88,7 @@ func formattingDifference(filename string, body []byte) (caddyconfig.Warning, bo
 	return caddyconfig.Warning{
 		File:    filename,
 		Line:    line,
-		Message: "input is not formatted with 'caddy fmt'",
+		Message: "Caddyfile input is not formatted; run the 'caddy fmt' command to fix inconsistencies",
 	}, true
 }
 

caddyconfig/caddyfile/dispenser.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"strconv"
 	"strings"
 )
 
@@ -201,6 +202,43 @@ func (d *Dispenser) Val() string {
 	return d.tokens[d.cursor].Text
 }
 
+// ValRaw gets the raw text of the current token (including quotes).
+// If there is no token loaded, it returns empty string.
+func (d *Dispenser) ValRaw() string {
+	if d.cursor < 0 || d.cursor >= len(d.tokens) {
+		return ""
+	}
+	quote := d.tokens[d.cursor].wasQuoted
+	if quote > 0 {
+		return string(quote) + d.tokens[d.cursor].Text + string(quote) // string literal
+	}
+	return d.tokens[d.cursor].Text
+}
+
+// ScalarVal gets value of the current token, converted to the closest
+// scalar type. If there is no token loaded, it returns nil.
+func (d *Dispenser) ScalarVal() interface{} {
+	if d.cursor < 0 || d.cursor >= len(d.tokens) {
+		return nil
+	}
+	quote := d.tokens[d.cursor].wasQuoted
+	text := d.tokens[d.cursor].Text
+
+	if quote > 0 {
+		return text // string literal
+	}
+	if num, err := strconv.Atoi(text); err == nil {
+		return num
+	}
+	if num, err := strconv.ParseFloat(text, 64); err == nil {
+		return num
+	}
+	if bool, err := strconv.ParseBool(text); err == nil {
+		return bool
+	}
+	return text
+}
+
 // Line gets the line number of the current token.
 // If there is no token loaded, it returns 0.
 func (d *Dispenser) Line() int {
@@ -249,6 +287,19 @@ func (d *Dispenser) AllArgs(targets ...*string) bool {
 	return true
 }
 
+// CountRemainingArgs counts the amount of remaining arguments
+// (tokens on the same line) without consuming the tokens.
+func (d *Dispenser) CountRemainingArgs() int {
+	count := 0
+	for d.NextArg() {
+		count++
+	}
+	for i := 0; i < count; i++ {
+		d.Prev()
+	}
+	return count
+}
+
 // RemainingArgs loads any more arguments (tokens on the same line)
 // into a slice and returns them. Open curly brace tokens also indicate
 // the end of arguments, and the curly brace is not included in
@@ -261,6 +312,18 @@ func (d *Dispenser) RemainingArgs() []string {
 	return args
 }
 
+// RemainingArgsRaw loads any more arguments (tokens on the same line,
+// retaining quotes) into a slice and returns them. Open curly brace
+// tokens also indicate the end of arguments, and the curly brace is
+// not included in the return value nor is it loaded.
+func (d *Dispenser) RemainingArgsRaw() []string {
+	var args []string
+	for d.NextArg() {
+		args = append(args, d.ValRaw())
+	}
+	return args
+}
+
 // NewFromNextSegment returns a new dispenser with a copy of
 // the tokens from the current token until the end of the
 // "directive" whether that be to the end of the line or
@@ -350,7 +413,11 @@ func (d *Dispenser) Err(msg string) error {
 
 // Errf is like Err, but for formatted error messages
 func (d *Dispenser) Errf(format string, args ...interface{}) error {
-	err := fmt.Errorf(format, args...)
+	return d.WrapErr(fmt.Errorf(format, args...))
+}
+
+// WrapErr takes an existing error and adds the Caddyfile file and line number.
+func (d *Dispenser) WrapErr(err error) error {
 	return fmt.Errorf("%s:%d - Error during parsing: %w", d.File(), d.Line(), err)
 }
 
@@ -391,6 +458,60 @@ func (d *Dispenser) isNewLine() bool {
 	if d.cursor > len(d.tokens)-1 {
 		return false
 	}
-	return d.tokens[d.cursor-1].File != d.tokens[d.cursor].File ||
-		d.tokens[d.cursor-1].Line+d.numLineBreaks(d.cursor-1) < d.tokens[d.cursor].Line
+
+	prev := d.tokens[d.cursor-1]
+	curr := d.tokens[d.cursor]
+
+	// If the previous token is from a different file,
+	// we can assume it's from a different line
+	if prev.File != curr.File {
+		return true
+	}
+
+	// The previous token may contain line breaks if
+	// it was quoted and spanned multiple lines. e.g:
+	//
+	// dir "foo
+	//   bar
+	//   baz"
+	prevLineBreaks := d.numLineBreaks(d.cursor - 1)
+
+	// If the previous token (incl line breaks) ends
+	// on a line earlier than the current token,
+	// then the current token is on a new line
+	return prev.Line+prevLineBreaks < curr.Line
+}
+
+// isNextOnNewLine determines whether the current token is on a different
+// line (higher line number) than the next token. It handles imported
+// tokens correctly. If there isn't a next token, it returns true.
+func (d *Dispenser) isNextOnNewLine() bool {
+	if d.cursor < 0 {
+		return false
+	}
+	if d.cursor >= len(d.tokens)-1 {
+		return true
+	}
+
+	curr := d.tokens[d.cursor]
+	next := d.tokens[d.cursor+1]
+
+	// If the next token is from a different file,
+	// we can assume it's from a different line
+	if curr.File != next.File {
+		return true
+	}
+
+	// The current token may contain line breaks if
+	// it was quoted and spanned multiple lines. e.g:
+	//
+	// dir "foo
+	//   bar
+	//   baz"
+	currLineBreaks := d.numLineBreaks(d.cursor)
+
+	// If the current token (incl line breaks) ends
+	// on a line earlier than the next token,
+	// then the next token is on a new line
+	return curr.Line+currLineBreaks < next.Line
 }

caddyconfig/caddyfile/formatter_fuzz.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build gofuzz
 // +build gofuzz
 
 package caddyfile

caddyconfig/caddyfile/formatter_test.go

@@ -179,6 +179,11 @@ d {
 	{$F}
 }`,
 		},
+		{
+			description: "env var placeholders with port",
+			input:       `:{$PORT}`,
+			expect:      `:{$PORT}`,
+		},
 		{
 			description: "comments",
 			input: `#a "\n"

caddyconfig/caddyfile/lexer.go

@@ -1,4 +1,4 @@
-// Copyright 2015 Light Code Labs, LLC
+// Copyright 2015 Matthew Holt and The Caddy Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -38,6 +38,7 @@ type (
 		File        string
 		Line        int
 		Text        string
+		wasQuoted   rune // enclosing quote character, if any
 		inSnippet   bool
 		snippetName string
 	}
@@ -78,8 +79,9 @@ func (l *lexer) next() bool {
 	var val []rune
 	var comment, quoted, btQuoted, escaped bool
 
-	makeToken := func() bool {
+	makeToken := func(quoted rune) bool {
 		l.token.Text = string(val)
+		l.token.wasQuoted = quoted
 		return true
 	}
 
@@ -87,7 +89,7 @@ func (l *lexer) next() bool {
 		ch, _, err := l.reader.ReadRune()
 		if err != nil {
 			if len(val) > 0 {
-				return makeToken()
+				return makeToken(0)
 			}
 			if err == io.EOF {
 				return false
@@ -110,10 +112,10 @@ func (l *lexer) next() bool {
 				escaped = false
 			} else {
 				if quoted && ch == '"' {
-					return makeToken()
+					return makeToken('"')
 				}
 				if btQuoted && ch == '`' {
-					return makeToken()
+					return makeToken('`')
 				}
 			}
 			if ch == '\n' {
@@ -139,7 +141,7 @@ func (l *lexer) next() bool {
 				comment = false
 			}
 			if len(val) > 0 {
-				return makeToken()
+				return makeToken(0)
 			}
 			continue
 		}

caddyconfig/caddyfile/lexer_fuzz.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build gofuzz
 // +build gofuzz
 
 package caddyfile

caddyconfig/caddyfile/lexer_test.go

@@ -1,4 +1,4 @@
-// Copyright 2015 Light Code Labs, LLC
+// Copyright 2015 Matthew Holt and The Caddy Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

caddyconfig/caddyfile/parse.go

@@ -1,4 +1,4 @@
-// Copyright 2015 Light Code Labs, LLC
+// Copyright 2015 Matthew Holt and The Caddy Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,14 +17,14 @@ package caddyfile
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
-	"log"
+	"io"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/caddyserver/caddy/v2"
+	"go.uber.org/zap"
 )
 
 // Parse parses the input just enough to group tokens, in
@@ -37,7 +37,13 @@ import (
 // Environment variables in {$ENVIRONMENT_VARIABLE} notation
 // will be replaced before parsing begins.
 func Parse(filename string, input []byte) ([]ServerBlock, error) {
-	tokens, err := allTokens(filename, input)
+	// unfortunately, we must copy the input because parsing must
+	// remain a read-only operation, but we have to expand environment
+	// variables before we parse, which changes the underlying array (#4422)
+	inputCopy := make([]byte, len(input))
+	copy(inputCopy, input)
+
+	tokens, err := allTokens(filename, inputCopy)
 	if err != nil {
 		return nil, err
 	}
@@ -51,7 +57,23 @@ func Parse(filename string, input []byte) ([]ServerBlock, error) {
 	return p.parseAll()
 }
 
+// allTokens lexes the entire input, but does not parse it.
+// It returns all the tokens from the input, unstructured
+// and in order. It may mutate input as it expands env vars.
+func allTokens(filename string, input []byte) ([]Token, error) {
+	inputCopy, err := replaceEnvVars(input)
+	if err != nil {
+		return nil, err
+	}
+	tokens, err := Tokenize(inputCopy, filename)
+	if err != nil {
+		return nil, err
+	}
+	return tokens, nil
+}
+
 // replaceEnvVars replaces all occurrences of environment variables.
+// It mutates the underlying array and returns the updated slice.
 func replaceEnvVars(input []byte) ([]byte, error) {
 	var offset int
 	for {
@@ -96,21 +118,6 @@ func replaceEnvVars(input []byte) ([]byte, error) {
 	return input, nil
 }
 
-// allTokens lexes the entire input, but does not parse it.
-// It returns all the tokens from the input, unstructured
-// and in order.
-func allTokens(filename string, input []byte) ([]Token, error) {
-	input, err := replaceEnvVars(input)
-	if err != nil {
-		return nil, err
-	}
-	tokens, err := Tokenize(input, filename)
-	if err != nil {
-		return nil, err
-	}
-	return tokens, nil
-}
-
 type parser struct {
 	*Dispenser
 	block           ServerBlock // current server block being parsed
@@ -386,7 +393,7 @@ func (p *parser) doImport() error {
 		}
 		if len(matches) == 0 {
 			if strings.ContainsAny(globPattern, "*?[]") {
-				log.Printf("[WARNING] No files matching import glob pattern: %s", importPattern)
+				caddy.Log().Warn("No files matching import glob pattern", zap.String("pattern", importPattern))
 			} else {
 				return p.Errf("File to import not found: %s", importPattern)
 			}
@@ -447,7 +454,7 @@ func (p *parser) doSingleImport(importFile string) ([]Token, error) {
 		return nil, p.Errf("Could not import %s: is a directory", importFile)
 	}
 
-	input, err := ioutil.ReadAll(file)
+	input, err := io.ReadAll(file)
 	if err != nil {
 		return nil, p.Errf("Could not read imported file %s: %v", importFile, err)
 	}
@@ -487,6 +494,13 @@ func (p *parser) directive() error {
 	for p.Next() {
 		if p.Val() == "{" {
 			p.nesting++
+			if !p.isNextOnNewLine() && p.Token().wasQuoted == 0 {
+				return p.Err("Unexpected next token after '{' on same line")
+			}
+		} else if p.Val() == "{}" {
+			if p.isNextOnNewLine() && p.Token().wasQuoted == 0 {
+				return p.Err("Unexpected '{}' at end of line")
+			}
 		} else if p.isNewLine() && p.nesting == 0 {
 			p.cursor-- // read too far
 			break

caddyconfig/caddyfile/parse_test.go

@@ -1,4 +1,4 @@
-// Copyright 2015 Light Code Labs, LLC
+// Copyright 2015 Matthew Holt and The Caddy Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,7 +16,6 @@ package caddyfile
 
 import (
 	"bytes"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
@@ -192,6 +191,20 @@ func TestParseOneAndImport(t *testing.T) {
 
 		{``, false, []string{}, []int{}},
 
+		// Unexpected next token after '{' on same line
+		{`localhost
+		  dir1 { a b }`, true, []string{"localhost"}, []int{}},
+		// Workaround with quotes
+		{`localhost
+		  dir1 "{" a b "}"`, false, []string{"localhost"}, []int{5}},
+
+		// Unexpected '{}' at end of line
+		{`localhost
+		  dir1 {}`, true, []string{"localhost"}, []int{}},
+		// Workaround with quotes
+		{`localhost
+		  dir1 "{}"`, false, []string{"localhost"}, []int{2}},
+
 		// import with args
 		{`import testdata/import_args0.txt a`, false, []string{"a"}, []int{}},
 		{`import testdata/import_args1.txt a b`, false, []string{"a", "b"}, []int{}},
@@ -280,7 +293,7 @@ func TestRecursiveImport(t *testing.T) {
 	}
 
 	// test relative recursive import
-	err = ioutil.WriteFile(recursiveFile1, []byte(
+	err = os.WriteFile(recursiveFile1, []byte(
 		`localhost
 		dir1
 		import recursive_import_test2`), 0644)
@@ -289,7 +302,7 @@ func TestRecursiveImport(t *testing.T) {
 	}
 	defer os.Remove(recursiveFile1)
 
-	err = ioutil.WriteFile(recursiveFile2, []byte("dir2 1"), 0644)
+	err = os.WriteFile(recursiveFile2, []byte("dir2 1"), 0644)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -314,7 +327,7 @@ func TestRecursiveImport(t *testing.T) {
 	}
 
 	// test absolute recursive import
-	err = ioutil.WriteFile(recursiveFile1, []byte(
+	err = os.WriteFile(recursiveFile1, []byte(
 		`localhost
 		dir1
 		import `+recursiveFile2), 0644)
@@ -370,7 +383,7 @@ func TestDirectiveImport(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = ioutil.WriteFile(directiveFile, []byte(`prop1 1
+	err = os.WriteFile(directiveFile, []byte(`prop1 1
 	prop2 2`), 0644)
 	if err != nil {
 		t.Fatal(err)
@@ -633,7 +646,7 @@ func TestSnippets(t *testing.T) {
 }
 
 func writeStringToTempFileOrDie(t *testing.T, str string) (pathToFile string) {
-	file, err := ioutil.TempFile("", t.Name())
+	file, err := os.CreateTemp("", t.Name())
 	if err != nil {
 		panic(err) // get a stack trace so we know where this was called from.
 	}

caddyconfig/httpcaddyfile/addresses.go

@@ -102,12 +102,20 @@ func (st *ServerType) mapAddressToServerBlocks(originalServerBlocks []serverBloc
 			}
 		}
 
+		// make a slice of the map keys so we can iterate in sorted order
+		addrs := make([]string, 0, len(addrToKeys))
+		for k := range addrToKeys {
+			addrs = append(addrs, k)
+		}
+		sort.Strings(addrs)
+
 		// now that we know which addresses serve which keys of this
 		// server block, we iterate that mapping and create a list of
 		// new server blocks for each address where the keys of the
 		// server block are only the ones which use the address; but
 		// the contents (tokens) are of course the same
-		for addr, keys := range addrToKeys {
+		for _, addr := range addrs {
+			keys := addrToKeys[addr]
 			// parse keys so that we only have to do it once
 			parsedKeys := make([]Address, 0, len(keys))
 			for _, key := range keys {
@@ -161,6 +169,7 @@ func (st *ServerType) consolidateAddrMappings(addrToServerBlocks map[string][]se
 				delete(addrToServerBlocks, otherAddr)
 			}
 		}
+		sort.Strings(a.addresses)
 
 		sbaddrs = append(sbaddrs, a)
 	}
@@ -208,12 +217,16 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 	}
 
 	// the bind directive specifies hosts, but is optional
-	lnHosts := make([]string, 0, len(sblock.pile))
+	lnHosts := make([]string, 0, len(sblock.pile["bind"]))
 	for _, cfgVal := range sblock.pile["bind"] {
 		lnHosts = append(lnHosts, cfgVal.Value.([]string)...)
 	}
 	if len(lnHosts) == 0 {
-		lnHosts = []string{""}
+		if defaultBind, ok := options["default_bind"].([]string); ok {
+			lnHosts = defaultBind
+		} else {
+			lnHosts = []string{""}
+		}
 	}
 
 	// use a map to prevent duplication
@@ -223,7 +236,7 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 		if err == nil && addr.IsUnixNetwork() {
 			listeners[host] = struct{}{}
 		} else {
-			listeners[net.JoinHostPort(host, lnPort)] = struct{}{}
+			listeners[host+":"+lnPort] = struct{}{}
 		}
 	}
 
@@ -232,6 +245,7 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 	for lnStr := range listeners {
 		listenersList = append(listenersList, lnStr)
 	}
+	sort.Strings(listenersList)
 
 	return listenersList, nil
 }
@@ -337,7 +351,9 @@ func (a Address) Normalize() Address {
 	// ensure host is normalized if it's an IP address
 	host := strings.TrimSpace(a.Host)
 	if ip := net.ParseIP(host); ip != nil {
-		host = ip.String()
+		if ipv6 := ip.To16(); ipv6 != nil && ipv6.DefaultMask() == nil {
+			host = ipv6.String()
+		}
 	}
 
 	return Address{
@@ -349,28 +365,6 @@ func (a Address) Normalize() Address {
 	}
 }
 
-// Key returns a string form of a, much like String() does, but this
-// method doesn't add anything default that wasn't in the original.
-func (a Address) Key() string {
-	res := ""
-	if a.Scheme != "" {
-		res += a.Scheme + "://"
-	}
-	if a.Host != "" {
-		res += a.Host
-	}
-	// insert port only if the original has its own explicit port
-	if a.Port != "" &&
-		len(a.Original) >= len(res) &&
-		strings.HasPrefix(a.Original[len(res):], ":"+a.Port) {
-		res += ":" + a.Port
-	}
-	if a.Path != "" {
-		res += a.Path
-	}
-	return res
-}
-
 // lowerExceptPlaceholders lowercases s except within
 // placeholders (substrings in non-escaped '{ }' spans).
 // See https://github.com/caddyserver/caddy/issues/3264

caddyconfig/httpcaddyfile/addresses_fuzz.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build gofuzz
 // +build gofuzz
 
 package httpcaddyfile

caddyconfig/httpcaddyfile/addresses_test.go

@@ -106,67 +106,128 @@ func TestAddressString(t *testing.T) {
 func TestKeyNormalization(t *testing.T) {
 	testCases := []struct {
 		input  string
-		expect string
+		expect Address
 	}{
 		{
-			input:  "example.com",
-			expect: "example.com",
+			input: "example.com",
+			expect: Address{
+				Host: "example.com",
+			},
 		},
 		{
-			input:  "http://host:1234/path",
-			expect: "http://host:1234/path",
+			input: "http://host:1234/path",
+			expect: Address{
+				Scheme: "http",
+				Host:   "host",
+				Port:   "1234",
+				Path:   "/path",
+			},
 		},
 		{
-			input:  "HTTP://A/ABCDEF",
-			expect: "http://a/ABCDEF",
+			input: "HTTP://A/ABCDEF",
+			expect: Address{
+				Scheme: "http",
+				Host:   "a",
+				Path:   "/ABCDEF",
+			},
 		},
 		{
-			input:  "A/ABCDEF",
-			expect: "a/ABCDEF",
+			input: "A/ABCDEF",
+			expect: Address{
+				Host: "a",
+				Path: "/ABCDEF",
+			},
 		},
 		{
-			input:  "A:2015/Path",
-			expect: "a:2015/Path",
+			input: "A:2015/Path",
+			expect: Address{
+				Host: "a",
+				Port: "2015",
+				Path: "/Path",
+			},
 		},
 		{
-			input:  "sub.{env.MY_DOMAIN}",
-			expect: "sub.{env.MY_DOMAIN}",
+			input: "sub.{env.MY_DOMAIN}",
+			expect: Address{
+				Host: "sub.{env.MY_DOMAIN}",
+			},
 		},
 		{
-			input:  "sub.ExAmPle",
-			expect: "sub.example",
+			input: "sub.ExAmPle",
+			expect: Address{
+				Host: "sub.example",
+			},
 		},
 		{
-			input:  "sub.\\{env.MY_DOMAIN\\}",
-			expect: "sub.\\{env.my_domain\\}",
+			input: "sub.\\{env.MY_DOMAIN\\}",
+			expect: Address{
+				Host: "sub.\\{env.my_domain\\}",
+			},
 		},
 		{
-			input:  "sub.{env.MY_DOMAIN}.com",
-			expect: "sub.{env.MY_DOMAIN}.com",
+			input: "sub.{env.MY_DOMAIN}.com",
+			expect: Address{
+				Host: "sub.{env.MY_DOMAIN}.com",
+			},
 		},
 		{
-			input:  ":80",
-			expect: ":80",
+			input: ":80",
+			expect: Address{
+				Port: "80",
+			},
 		},
 		{
-			input:  ":443",
-			expect: ":443",
+			input: ":443",
+			expect: Address{
+				Port: "443",
+			},
 		},
 		{
-			input:  ":1234",
-			expect: ":1234",
+			input: ":1234",
+			expect: Address{
+				Port: "1234",
+			},
 		},
 		{
 			input:  "",
-			expect: "",
+			expect: Address{},
 		},
 		{
 			input:  ":",
-			expect: "",
+			expect: Address{},
 		},
 		{
-			input:  "[::]",
-			expect: "::",
+			input: "[::]",
+			expect: Address{
+				Host: "::",
+			},
+		},
+		{
+			input: "127.0.0.1",
+			expect: Address{
+				Host: "127.0.0.1",
+			},
+		},
+		{
+			input: "[2001:db8:85a3:8d3:1319:8a2e:370:7348]:1234",
+			expect: Address{
+				Host: "2001:db8:85a3:8d3:1319:8a2e:370:7348",
+				Port: "1234",
+			},
+		},
+		{
+			// IPv4 address in IPv6 form (#4381)
+			input: "[::ffff:cff4:e77d]:1234",
+			expect: Address{
+				Host: "::ffff:cff4:e77d",
+				Port: "1234",
+			},
+		},
+		{
+			input: "::ffff:cff4:e77d",
+			expect: Address{
+				Host: "::ffff:cff4:e77d",
+			},
 		},
 	}
 	for i, tc := range testCases {
@@ -175,9 +236,18 @@ func TestKeyNormalization(t *testing.T) {
 			t.Errorf("Test %d: Parsing address '%s': %v", i, tc.input, err)
 			continue
 		}
-		if actual := addr.Normalize().Key(); actual != tc.expect {
-			t.Errorf("Test %d: Input '%s': Expected '%s' but got '%s'", i, tc.input, tc.expect, actual)
+		actual := addr.Normalize()
+		if actual.Scheme != tc.expect.Scheme {
+			t.Errorf("Test %d: Input '%s': Expected Scheme='%s' but got Scheme='%s'", i, tc.input, tc.expect.Scheme, actual.Scheme)
+		}
+		if actual.Host != tc.expect.Host {
+			t.Errorf("Test %d: Input '%s': Expected Host='%s' but got Host='%s'", i, tc.input, tc.expect.Host, actual.Host)
+		}
+		if actual.Port != tc.expect.Port {
+			t.Errorf("Test %d: Input '%s': Expected Port='%s' but got Port='%s'", i, tc.input, tc.expect.Port, actual.Port)
+		}
+		if actual.Path != tc.expect.Path {
+			t.Errorf("Test %d: Input '%s': Expected Path='%s' but got Path='%s'", i, tc.input, tc.expect.Path, actual.Path)
 		}
-
 	}
 }

caddyconfig/httpcaddyfile/builtins.go

@@ -19,8 +19,8 @@ import (
 	"encoding/pem"
 	"fmt"
 	"html"
-	"io/ioutil"
 	"net/http"
+	"os"
 	"reflect"
 	"strconv"
 	"strings"
@@ -39,6 +39,7 @@ func init() {
 	RegisterDirective("bind", parseBind)
 	RegisterDirective("tls", parseTLS)
 	RegisterHandlerDirective("root", parseRoot)
+	RegisterHandlerDirective("vars", parseVars)
 	RegisterHandlerDirective("redir", parseRedir)
 	RegisterHandlerDirective("respond", parseRespond)
 	RegisterHandlerDirective("abort", parseAbort)
@@ -82,6 +83,7 @@ func parseBind(h Helper) ([]ConfigValue, error) {
 //         on_demand
 //         eab    <key_id> <mac_key>
 //         issuer <module_name> [...]
+//         get_certificate <module_name> [...]
 //     }
 //
 func parseTLS(h Helper) ([]ConfigValue, error) {
@@ -93,6 +95,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 	var keyType string
 	var internalIssuer *caddytls.InternalIssuer
 	var issuers []certmagic.Issuer
+	var certManagers []certmagic.Manager
 	var onDemand bool
 
 	for h.Next() {
@@ -230,7 +233,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 							return nil, h.ArgErr()
 						}
 						filename := h.Val()
-						certDataPEM, err := ioutil.ReadFile(filename)
+						certDataPEM, err := os.ReadFile(filename)
 						if err != nil {
 							return nil, err
 						}
@@ -307,6 +310,22 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 				}
 				issuers = append(issuers, issuer)
 
+			case "get_certificate":
+				if !h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				modName := h.Val()
+				modID := "tls.get_certificate." + modName
+				unm, err := caddyfile.UnmarshalModule(h.Dispenser, modID)
+				if err != nil {
+					return nil, err
+				}
+				certManager, ok := unm.(certmagic.Manager)
+				if !ok {
+					return nil, h.Errf("module %s (%T) is not a certmagic.CertificateManager", modID, unm)
+				}
+				certManagers = append(certManagers, certManager)
+
 			case "dns":
 				if !h.NextArg() {
 					return nil, h.ArgErr()
@@ -344,6 +363,22 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 				}
 				acmeIssuer.Challenges.DNS.Resolvers = args
 
+			case "dns_challenge_override_domain":
+				arg := h.RemainingArgs()
+				if len(arg) != 1 {
+					return nil, h.ArgErr()
+				}
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
+				if acmeIssuer.Challenges == nil {
+					acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
+				}
+				if acmeIssuer.Challenges.DNS == nil {
+					acmeIssuer.Challenges.DNS = new(caddytls.DNSChallengeConfig)
+				}
+				acmeIssuer.Challenges.DNS.OverrideDomain = arg[0]
+
 			case "ca_root":
 				arg := h.RemainingArgs()
 				if len(arg) != 1 {
@@ -453,6 +488,12 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 			Value: true,
 		})
 	}
+	for _, certManager := range certManagers {
+		configVals = append(configVals, ConfigValue{
+			Class: "tls.cert_manager",
+			Value: certManager,
+		})
+	}
 
 	// custom certificate selection
 	if len(certSelector.AnyTag) > 0 {
@@ -490,6 +531,13 @@ func parseRoot(h Helper) (caddyhttp.MiddlewareHandler, error) {
 	return caddyhttp.VarsMiddleware{"root": root}, nil
 }
 
+// parseVars parses the vars directive. See its UnmarshalCaddyfile method for syntax.
+func parseVars(h Helper) (caddyhttp.MiddlewareHandler, error) {
+	v := new(caddyhttp.VarsMiddleware)
+	err := v.UnmarshalCaddyfile(h.Dispenser)
+	return v, err
+}
+
 // parseRedir parses the redir directive. Syntax:
 //
 //     redir [<matcher>] <to> [<code>]
@@ -532,12 +580,24 @@ func parseRedir(h Helper) (caddyhttp.MiddlewareHandler, error) {
 		body = fmt.Sprintf(metaRedir, safeTo, safeTo, safeTo, safeTo)
 		code = "302"
 	default:
+		// Allow placeholders for the code
+		if strings.HasPrefix(code, "{") {
+			break
+		}
+		// Try to validate as an integer otherwise
 		codeInt, err := strconv.Atoi(code)
 		if err != nil {
 			return nil, h.Errf("Not a supported redir code type or not valid integer: '%s'", code)
 		}
-		if codeInt < 300 || codeInt > 399 {
-			return nil, h.Errf("Redir code not in the 3xx range: '%v'", codeInt)
+		// Sometimes, a 401 with Location header is desirable because
+		// requests made with XHR will "eat" the 3xx redirect; so if
+		// the intent was to redirect to an auth page, a 3xx won't
+		// work. Responding with 401 allows JS code to read the
+		// Location header and do a window.location redirect manually.
+		// see https://stackoverflow.com/a/2573589/846934
+		// see https://github.com/oauth2-proxy/oauth2-proxy/issues/1522
+		if codeInt < 300 || (codeInt > 399 && codeInt != 401) {
+			return nil, h.Errf("Redir code not in the 3xx range or 401: '%v'", codeInt)
 		}
 	}
 

caddyconfig/httpcaddyfile/builtins_test.go

@@ -37,8 +37,7 @@ func TestLogDirectiveSyntax(t *testing.T) {
 					format filter {
 						wrap console
 						fields {
-							common_log delete
-							request>remote_addr ip_mask {
+							request>remote_ip ip_mask {
 								ipv4 24
 								ipv6 32
 							}
@@ -47,7 +46,7 @@ func TestLogDirectiveSyntax(t *testing.T) {
 				}
 			}
 			`,
-			output:      `{"logging":{"logs":{"default":{"exclude":["http.log.access.log0"]},"log0":{"encoder":{"fields":{"common_log":{"filter":"delete"},"request\u003eremote_addr":{"filter":"ip_mask","ipv4_cidr":24,"ipv6_cidr":32}},"format":"filter","wrap":{"format":"console"}},"include":["http.log.access.log0"]}}},"apps":{"http":{"servers":{"srv0":{"listen":[":8080"],"logs":{"default_logger_name":"log0"}}}}}}`,
+			output:      `{"logging":{"logs":{"default":{"exclude":["http.log.access.log0"]},"log0":{"encoder":{"fields":{"request\u003eremote_ip":{"filter":"ip_mask","ipv4_cidr":24,"ipv6_cidr":32}},"format":"filter","wrap":{"format":"console"}},"include":["http.log.access.log0"]}}},"apps":{"http":{"servers":{"srv0":{"listen":[":8080"],"logs":{"default_logger_name":"log0"}}}}}}`,
 			expectError: false,
 		},
 		{
@@ -149,6 +148,27 @@ func TestRedirDirectiveSyntax(t *testing.T) {
 			}`,
 			expectError: false,
 		},
+		{
+			// this is now allowed so a Location header
+			// can be written and consumed by JS
+			// in the case of XHR requests
+			input: `:8080 {
+				redir * :8081 401
+			}`,
+			expectError: false,
+		},
+		{
+			input: `:8080 {
+				redir * :8081 402
+			}`,
+			expectError: true,
+		},
+		{
+			input: `:8080 {
+				redir * :8081 {http.reverse_proxy.status_code}
+			}`,
+			expectError: false,
+		},
 		{
 			input: `:8080 {
 				redir /old.html /new.html htlm
@@ -161,12 +181,6 @@ func TestRedirDirectiveSyntax(t *testing.T) {
 			}`,
 			expectError: true,
 		},
-		{
-			input: `:8080 {
-				redir * :8081 400
-			}`,
-			expectError: true,
-		},
 		{
 			input: `:8080 {
 				redir * :8081 temp

caddyconfig/httpcaddyfile/directives.go

@@ -37,21 +37,27 @@ import (
 // The header directive goes second so that headers
 // can be manipulated before doing redirects.
 var directiveOrder = []string{
+	"tracing",
+
 	"map",
+	"vars",
 	"root",
 
 	"header",
+	"copy_response_headers", // only in reverse_proxy's handle_response
 	"request_body",
 
 	"redir",
 
-	// URI manipulation
+	// incoming request manipulation
+	"method",
 	"rewrite",
 	"uri",
 	"try_files",
 
 	// middleware handlers; some wrap responses
 	"basicauth",
+	"forward_auth",
 	"request_header",
 	"encode",
 	"push",
@@ -65,6 +71,7 @@ var directiveOrder = []string{
 	// handlers that typically respond to requests
 	"abort",
 	"error",
+	"copy_response", // only in reverse_proxy's handle_response
 	"respond",
 	"metrics",
 	"reverse_proxy",
@@ -340,6 +347,9 @@ func parseSegmentAsConfig(h Helper) ([]ConfigValue, error) {
 			if err != nil {
 				return nil, h.Errf("parsing caddyfile tokens for '%s': %v", dir, err)
 			}
+
+			dir = normalizeDirectiveName(dir)
+
 			for _, result := range results {
 				result.directive = dir
 				allResults = append(allResults, result)
@@ -415,14 +425,29 @@ func sortRoutes(routes []ConfigValue) {
 			jPathLen = len(jPM[0])
 		}
 
-		// if both directives have no path matcher, use whichever one
-		// has any kind of matcher defined first.
-		if iPathLen == 0 && jPathLen == 0 {
-			return len(iRoute.MatcherSetsRaw) > 0 && len(jRoute.MatcherSetsRaw) == 0
-		}
+		// some directives involve setting values which can overwrite
+		// eachother, so it makes most sense to reverse the order so
+		// that the lease specific matcher is first; everything else
+		// has most-specific matcher first
+		if iDir == "vars" {
+			// if both directives have no path matcher, use whichever one
+			// has no matcher first.
+			if iPathLen == 0 && jPathLen == 0 {
+				return len(iRoute.MatcherSetsRaw) == 0 && len(jRoute.MatcherSetsRaw) > 0
+			}
 
-		// sort with the most-specific (longest) path first
-		return iPathLen > jPathLen
+			// sort with the least-specific (shortest) path first
+			return iPathLen < jPathLen
+		} else {
+			// if both directives have no path matcher, use whichever one
+			// has any kind of matcher defined first.
+			if iPathLen == 0 && jPathLen == 0 {
+				return len(iRoute.MatcherSetsRaw) > 0 && len(jRoute.MatcherSetsRaw) == 0
+			}
+
+			// sort with the most-specific (longest) path first
+			return iPathLen > jPathLen
+		}
 	})
 }
 
@@ -510,6 +535,17 @@ func (sb serverBlock) hasHostCatchAllKey() bool {
 	return false
 }
 
+// isAllHTTP returns true if all sb keys explicitly specify
+// the http:// scheme
+func (sb serverBlock) isAllHTTP() bool {
+	for _, addr := range sb.keys {
+		if addr.Scheme != "http" {
+			return false
+		}
+	}
+	return true
+}
+
 type (
 	// UnmarshalFunc is a function which can unmarshal Caddyfile
 	// tokens into zero or more config values using a Helper type.

caddyconfig/httpcaddyfile/httptype.go

@@ -17,7 +17,6 @@ package httpcaddyfile
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"reflect"
 	"regexp"
 	"sort"
@@ -30,6 +29,7 @@ import (
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddypki"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
+	"go.uber.org/zap"
 )
 
 func init() {
@@ -88,33 +88,10 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		return nil, warnings, err
 	}
 
-	// replace shorthand placeholders (which are
-	// convenient when writing a Caddyfile) with
-	// their actual placeholder identifiers or
-	// variable names
-	replacer := strings.NewReplacer(
-		"{dir}", "{http.request.uri.path.dir}",
-		"{file}", "{http.request.uri.path.file}",
-		"{host}", "{http.request.host}",
-		"{hostport}", "{http.request.hostport}",
-		"{port}", "{http.request.port}",
-		"{method}", "{http.request.method}",
-		"{path}", "{http.request.uri.path}",
-		"{query}", "{http.request.uri.query}",
-		"{remote}", "{http.request.remote}",
-		"{remote_host}", "{http.request.remote.host}",
-		"{remote_port}", "{http.request.remote.port}",
-		"{scheme}", "{http.request.scheme}",
-		"{uri}", "{http.request.uri}",
-		"{tls_cipher}", "{http.request.tls.cipher_suite}",
-		"{tls_version}", "{http.request.tls.version}",
-		"{tls_client_fingerprint}", "{http.request.tls.client.fingerprint}",
-		"{tls_client_issuer}", "{http.request.tls.client.issuer}",
-		"{tls_client_serial}", "{http.request.tls.client.serial}",
-		"{tls_client_subject}", "{http.request.tls.client.subject}",
-		"{tls_client_certificate_pem}", "{http.request.tls.client.certificate_pem}",
-		"{upstream_hostport}", "{http.reverse_proxy.upstream.hostport}",
-	)
+	// replace shorthand placeholders (which are convenient
+	// when writing a Caddyfile) with their actual placeholder
+	// identifiers or variable names
+	replacer := strings.NewReplacer(placeholderShorthands()...)
 
 	// these are placeholders that allow a user-defined final
 	// parameters, but we still want to provide a shorthand
@@ -128,6 +105,9 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		{regexp.MustCompile(`{header\.([\w-]*)}`), "{http.request.header.$1}"},
 		{regexp.MustCompile(`{path\.([\w-]*)}`), "{http.request.uri.path.$1}"},
 		{regexp.MustCompile(`{re\.([\w-]*)\.([\w-]*)}`), "{http.regexp.$1.$2}"},
+		{regexp.MustCompile(`{vars\.([\w-]*)}`), "{http.vars.$1}"},
+		{regexp.MustCompile(`{rp\.([\w-\.]*)}`), "{http.reverse_proxy.$1}"},
+		{regexp.MustCompile(`{err\.([\w-\.]*)}`), "{http.error.$1}"},
 	}
 
 	for _, sb := range originalServerBlocks {
@@ -192,13 +172,7 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 				return nil, warnings, fmt.Errorf("parsing caddyfile tokens for '%s': %v", dir, err)
 			}
 
-			// As a special case, we want "handle_path" to be sorted
-			// at the same level as "handle", so we force them to use
-			// the same directive name after their parsing is complete.
-			// See https://github.com/caddyserver/caddy/issues/3675#issuecomment-678042377
-			if dir == "handle_path" {
-				dir = "handle"
-			}
+			dir = normalizeDirectiveName(dir)
 
 			for _, result := range results {
 				result.directive = dir
@@ -258,20 +232,13 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		}
 		customLogs = append(customLogs, ncl)
 	}
+
 	// Apply global log options, when set
 	if options["log"] != nil {
 		for _, logValue := range options["log"].([]ConfigValue) {
 			addCustomLog(logValue.Value.(namedCustomLog))
 		}
 	}
-	// Apply server-specific log options
-	for _, p := range pairings {
-		for _, sb := range p.serverBlocks {
-			for _, clVal := range sb.pile["custom_log"] {
-				addCustomLog(clVal.Value.(namedCustomLog))
-			}
-		}
-	}
 
 	if !hasDefaultLog {
 		// if the default log was not customized, ensure we
@@ -284,6 +251,15 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		}
 	}
 
+	// Apply server-specific log options
+	for _, p := range pairings {
+		for _, sb := range p.serverBlocks {
+			for _, clVal := range sb.pile["custom_log"] {
+				addCustomLog(clVal.Value.(namedCustomLog))
+			}
+		}
+	}
+
 	// annnd the top-level config, then we're done!
 	cfg := &caddy.Config{AppsRaw: make(caddy.ModuleMap)}
 
@@ -451,17 +427,29 @@ func (st *ServerType) serversFromPairings(
 		// handle the auto_https global option
 		if autoHTTPS != "on" {
 			srv.AutoHTTPS = new(caddyhttp.AutoHTTPSConfig)
-			if autoHTTPS == "off" {
+			switch autoHTTPS {
+			case "off":
 				srv.AutoHTTPS.Disabled = true
-			}
-			if autoHTTPS == "disable_redirects" {
+			case "disable_redirects":
 				srv.AutoHTTPS.DisableRedir = true
-			}
-			if autoHTTPS == "ignore_loaded_certs" {
+			case "disable_certs":
+				srv.AutoHTTPS.DisableCerts = true
+			case "ignore_loaded_certs":
 				srv.AutoHTTPS.IgnoreLoadedCerts = true
 			}
 		}
 
+		// Using paths in site addresses is deprecated
+		// See ParseAddress() where parsing should later reject paths
+		// See https://github.com/caddyserver/caddy/pull/4728 for a full explanation
+		for _, sblock := range p.serverBlocks {
+			for _, addr := range sblock.keys {
+				if addr.Path != "" {
+					caddy.Log().Named("caddyfile").Warn("Using a path in a site address is deprecated; please use the 'handle' directive instead", zap.String("address", addr.String()))
+				}
+			}
+		}
+
 		// sort server blocks by their keys; this is important because
 		// only the first matching site should be evaluated, and we should
 		// attempt to match most specific site first (host and path), in
@@ -549,7 +537,7 @@ func (st *ServerType) serversFromPairings(
 			// emit warnings if user put unspecified IP addresses; they probably want the bind directive
 			for _, h := range hosts {
 				if h == "0.0.0.0" || h == "::" {
-					log.Printf("[WARNING] Site block has unspecified IP address %s which only matches requests having that Host header; you probably want the 'bind' directive to configure the socket", h)
+					caddy.Log().Named("caddyfile").Warn("Site block has an unspecified IP address which only matches requests having that Host header; you probably want the 'bind' directive to configure the socket", zap.String("address", h))
 				}
 			}
 
@@ -585,7 +573,7 @@ func (st *ServerType) serversFromPairings(
 			}
 
 			for _, addr := range sblock.keys {
-				// if server only uses HTTPS port, auto-HTTPS will not apply
+				// if server only uses HTTP port, auto-HTTPS will not apply
 				if listenersUseAnyPortOtherThan(srv.Listen, httpPort) {
 					// exclude any hosts that were defined explicitly with "http://"
 					// in the key from automated cert management (issue #2998)
@@ -1060,6 +1048,19 @@ func buildSubroute(routes []ConfigValue, groupCounter counter) (*caddyhttp.Subro
 	return subroute, nil
 }
 
+// normalizeDirectiveName ensures directives that should be sorted
+// at the same level are named the same before sorting happens.
+func normalizeDirectiveName(directive string) string {
+	// As a special case, we want "handle_path" to be sorted
+	// at the same level as "handle", so we force them to use
+	// the same directive name after their parsing is complete.
+	// See https://github.com/caddyserver/caddy/issues/3675#issuecomment-678042377
+	if directive == "handle_path" {
+		directive = "handle"
+	}
+	return directive
+}
+
 // consolidateRoutes combines routes with the same properties
 // (same matchers, same Terminal and Group settings) for a
 // cleaner overall output.
@@ -1241,6 +1242,58 @@ func encodeMatcherSet(matchers map[string]caddyhttp.RequestMatcher) (caddy.Modul
 	return msEncoded, nil
 }
 
+// placeholderShorthands returns a slice of old-new string pairs,
+// where the left of the pair is a placeholder shorthand that may
+// be used in the Caddyfile, and the right is the replacement.
+func placeholderShorthands() []string {
+	return []string{
+		"{dir}", "{http.request.uri.path.dir}",
+		"{file}", "{http.request.uri.path.file}",
+		"{host}", "{http.request.host}",
+		"{hostport}", "{http.request.hostport}",
+		"{port}", "{http.request.port}",
+		"{method}", "{http.request.method}",
+		"{path}", "{http.request.uri.path}",
+		"{query}", "{http.request.uri.query}",
+		"{remote}", "{http.request.remote}",
+		"{remote_host}", "{http.request.remote.host}",
+		"{remote_port}", "{http.request.remote.port}",
+		"{scheme}", "{http.request.scheme}",
+		"{uri}", "{http.request.uri}",
+		"{tls_cipher}", "{http.request.tls.cipher_suite}",
+		"{tls_version}", "{http.request.tls.version}",
+		"{tls_client_fingerprint}", "{http.request.tls.client.fingerprint}",
+		"{tls_client_issuer}", "{http.request.tls.client.issuer}",
+		"{tls_client_serial}", "{http.request.tls.client.serial}",
+		"{tls_client_subject}", "{http.request.tls.client.subject}",
+		"{tls_client_certificate_pem}", "{http.request.tls.client.certificate_pem}",
+		"{tls_client_certificate_der_base64}", "{http.request.tls.client.certificate_der_base64}",
+		"{upstream_hostport}", "{http.reverse_proxy.upstream.hostport}",
+	}
+}
+
+// WasReplacedPlaceholderShorthand checks if a token string was
+// likely a replaced shorthand of the known Caddyfile placeholder
+// replacement outputs. Useful to prevent some user-defined map
+// output destinations from overlapping with one of the
+// predefined shorthands.
+func WasReplacedPlaceholderShorthand(token string) string {
+	prev := ""
+	for i, item := range placeholderShorthands() {
+		// only look at every 2nd item, which is the replacement
+		if i%2 == 0 {
+			prev = item
+			continue
+		}
+		if strings.Trim(token, "{}") == strings.Trim(item, "{}") {
+			// we return the original shorthand so it
+			// can be used for an error message
+			return prev
+		}
+	}
+	return ""
+}
+
 // tryInt tries to convert val to an integer. If it fails,
 // it downgrades the error to a warning and returns 0.
 func tryInt(val interface{}, warnings *[]caddyconfig.Warning) int {

caddyconfig/httpcaddyfile/options.go

@@ -29,11 +29,13 @@ func init() {
 	RegisterGlobalOption("debug", parseOptTrue)
 	RegisterGlobalOption("http_port", parseOptHTTPPort)
 	RegisterGlobalOption("https_port", parseOptHTTPSPort)
+	RegisterGlobalOption("default_bind", parseOptStringList)
 	RegisterGlobalOption("grace_period", parseOptDuration)
 	RegisterGlobalOption("default_sni", parseOptSingleString)
 	RegisterGlobalOption("order", parseOptOrder)
 	RegisterGlobalOption("storage", parseOptStorage)
 	RegisterGlobalOption("storage_clean_interval", parseOptDuration)
+	RegisterGlobalOption("renew_interval", parseOptDuration)
 	RegisterGlobalOption("acme_ca", parseOptSingleString)
 	RegisterGlobalOption("acme_ca_root", parseOptSingleString)
 	RegisterGlobalOption("acme_dns", parseOptACMEDNS)
@@ -277,6 +279,15 @@ func parseOptSingleString(d *caddyfile.Dispenser, _ interface{}) (interface{}, e
 	return val, nil
 }
 
+func parseOptStringList(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+	d.Next() // consume parameter name
+	val := d.RemainingArgs()
+	if len(val) == 0 {
+		return "", d.ArgErr()
+	}
+	return val, nil
+}
+
 func parseOptAdmin(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
 	adminCfg := new(caddy.AdminConfig)
 	for d.Next() {
@@ -382,8 +393,8 @@ func parseOptAutoHTTPS(d *caddyfile.Dispenser, _ interface{}) (interface{}, erro
 	if d.Next() {
 		return "", d.ArgErr()
 	}
-	if val != "off" && val != "disable_redirects" && val != "ignore_loaded_certs" {
-		return "", d.Errf("auto_https must be one of 'off', 'disable_redirects' or 'ignore_loaded_certs'")
+	if val != "off" && val != "disable_redirects" && val != "disable_certs" && val != "ignore_loaded_certs" {
+		return "", d.Errf("auto_https must be one of 'off', 'disable_redirects', 'disable_certs', or 'ignore_loaded_certs'")
 	}
 	return val, nil
 }

caddyconfig/httpcaddyfile/pkiapp.go

@@ -16,23 +16,176 @@ package httpcaddyfile
 
 import (
 	"github.com/caddyserver/caddy/v2/caddyconfig"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddypki"
 )
 
+func init() {
+	RegisterGlobalOption("pki", parsePKIApp)
+}
+
+// parsePKIApp parses the global log option. Syntax:
+//
+//     pki {
+//         ca [<id>] {
+//             name            <name>
+//             root_cn         <name>
+//             intermediate_cn <name>
+//             root {
+//                 cert   <path>
+//                 key    <path>
+//                 format <format>
+//             }
+//             intermediate {
+//                 cert   <path>
+//                 key    <path>
+//                 format <format>
+//             }
+//         }
+//     }
+//
+// When the CA ID is unspecified, 'local' is assumed.
+//
+func parsePKIApp(d *caddyfile.Dispenser, existingVal interface{}) (interface{}, error) {
+	pki := &caddypki.PKI{CAs: make(map[string]*caddypki.CA)}
+
+	for d.Next() {
+		for nesting := d.Nesting(); d.NextBlock(nesting); {
+			switch d.Val() {
+			case "ca":
+				pkiCa := new(caddypki.CA)
+				if d.NextArg() {
+					pkiCa.ID = d.Val()
+					if d.NextArg() {
+						return nil, d.ArgErr()
+					}
+				}
+				if pkiCa.ID == "" {
+					pkiCa.ID = caddypki.DefaultCAID
+				}
+
+				for nesting := d.Nesting(); d.NextBlock(nesting); {
+					switch d.Val() {
+					case "name":
+						if !d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						pkiCa.Name = d.Val()
+
+					case "root_cn":
+						if !d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						pkiCa.RootCommonName = d.Val()
+
+					case "intermediate_cn":
+						if !d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						pkiCa.IntermediateCommonName = d.Val()
+
+					case "root":
+						if pkiCa.Root == nil {
+							pkiCa.Root = new(caddypki.KeyPair)
+						}
+						for nesting := d.Nesting(); d.NextBlock(nesting); {
+							switch d.Val() {
+							case "cert":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Root.Certificate = d.Val()
+
+							case "key":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Root.PrivateKey = d.Val()
+
+							case "format":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Root.Format = d.Val()
+
+							default:
+								return nil, d.Errf("unrecognized pki ca root option '%s'", d.Val())
+							}
+						}
+
+					case "intermediate":
+						if pkiCa.Intermediate == nil {
+							pkiCa.Intermediate = new(caddypki.KeyPair)
+						}
+						for nesting := d.Nesting(); d.NextBlock(nesting); {
+							switch d.Val() {
+							case "cert":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Intermediate.Certificate = d.Val()
+
+							case "key":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Intermediate.PrivateKey = d.Val()
+
+							case "format":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Intermediate.Format = d.Val()
+
+							default:
+								return nil, d.Errf("unrecognized pki ca intermediate option '%s'", d.Val())
+							}
+						}
+
+					default:
+						return nil, d.Errf("unrecognized pki ca option '%s'", d.Val())
+					}
+				}
+
+				pki.CAs[pkiCa.ID] = pkiCa
+
+			default:
+				return nil, d.Errf("unrecognized pki option '%s'", d.Val())
+			}
+		}
+	}
+
+	return pki, nil
+}
+
 func (st ServerType) buildPKIApp(
 	pairings []sbAddrAssociation,
 	options map[string]interface{},
 	warnings []caddyconfig.Warning,
 ) (*caddypki.PKI, []caddyconfig.Warning, error) {
 
-	pkiApp := &caddypki.PKI{CAs: make(map[string]*caddypki.CA)}
-
 	skipInstallTrust := false
 	if _, ok := options["skip_install_trust"]; ok {
 		skipInstallTrust = true
 	}
 	falseBool := false
 
+	// Load the PKI app configured via global options
+	var pkiApp *caddypki.PKI
+	unwrappedPki, ok := options["pki"].(*caddypki.PKI)
+	if ok {
+		pkiApp = unwrappedPki
+	} else {
+		pkiApp = &caddypki.PKI{CAs: make(map[string]*caddypki.CA)}
+	}
+	for _, ca := range pkiApp.CAs {
+		if skipInstallTrust {
+			ca.InstallTrust = &falseBool
+		}
+		pkiApp.CAs[ca.ID] = ca
+	}
+
+	// Add in the CAs configured via directives
 	for _, p := range pairings {
 		for _, sblock := range p.serverBlocks {
 			// find all the CAs that were defined and add them to the app config
@@ -42,7 +195,12 @@ func (st ServerType) buildPKIApp(
 				if skipInstallTrust {
 					ca.InstallTrust = &falseBool
 				}
-				pkiApp.CAs[ca.ID] = ca
+
+				// the CA might already exist from global options, so
+				// don't overwrite it in that case
+				if _, ok := pkiApp.CAs[ca.ID]; !ok {
+					pkiApp.CAs[ca.ID] = ca
+				}
 			}
 		}
 	}

caddyconfig/httpcaddyfile/serveroptions.go

@@ -33,15 +33,16 @@ type serverOptions struct {
 	ListenerAddress string
 
 	// These will all map 1:1 to the caddyhttp.Server struct
-	ListenerWrappersRaw []json.RawMessage
-	ReadTimeout         caddy.Duration
-	ReadHeaderTimeout   caddy.Duration
-	WriteTimeout        caddy.Duration
-	IdleTimeout         caddy.Duration
-	MaxHeaderBytes      int
-	AllowH2C            bool
-	ExperimentalHTTP3   bool
-	StrictSNIHost       *bool
+	ListenerWrappersRaw  []json.RawMessage
+	ReadTimeout          caddy.Duration
+	ReadHeaderTimeout    caddy.Duration
+	WriteTimeout         caddy.Duration
+	IdleTimeout          caddy.Duration
+	MaxHeaderBytes       int
+	AllowH2C             bool
+	ExperimentalHTTP3    bool
+	StrictSNIHost        *bool
+	ShouldLogCredentials bool
 }
 
 func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error) {
@@ -134,6 +135,12 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error
 				}
 				serverOpts.MaxHeaderBytes = int(size)
 
+			case "log_credentials":
+				if d.NextArg() {
+					return nil, d.ArgErr()
+				}
+				serverOpts.ShouldLogCredentials = true
+
 			case "protocol":
 				for nesting := d.Nesting(); d.NextBlock(nesting); {
 					switch d.Val() {
@@ -150,11 +157,14 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error
 						serverOpts.ExperimentalHTTP3 = true
 
 					case "strict_sni_host":
-						if d.NextArg() {
-							return nil, d.ArgErr()
+						if d.NextArg() && d.Val() != "insecure_off" && d.Val() != "on" {
+							return nil, d.Errf("strict_sni_host only supports 'on' or 'insecure_off', got '%s'", d.Val())
 						}
-						trueBool := true
-						serverOpts.StrictSNIHost = &trueBool
+						boolVal := true
+						if d.Val() == "insecure_off" {
+							boolVal = false
+						}
+						serverOpts.StrictSNIHost = &boolVal
 
 					default:
 						return nil, d.Errf("unrecognized protocol option '%s'", d.Val())
@@ -222,6 +232,12 @@ func applyServerOptions(
 		server.AllowH2C = opts.AllowH2C
 		server.ExperimentalHTTP3 = opts.ExperimentalHTTP3
 		server.StrictSNIHost = opts.StrictSNIHost
+		if opts.ShouldLogCredentials {
+			if server.Logs == nil {
+				server.Logs = &caddyhttp.ServerLogConfig{}
+			}
+			server.Logs.ShouldLogCredentials = opts.ShouldLogCredentials
+		}
 	}
 
 	return nil

caddyconfig/httpcaddyfile/tlsapp.go

@@ -101,6 +101,12 @@ func (st ServerType) buildTLSApp(
 		}
 
 		for _, sblock := range p.serverBlocks {
+			// check the scheme of all the site addresses,
+			// skip building AP if they all had http://
+			if sblock.isAllHTTP() {
+				continue
+			}
+
 			// get values that populate an automation policy for this block
 			ap, err := newBaseAutomationPolicy(options, warnings, true)
 			if err != nil {
@@ -133,6 +139,13 @@ func (st ServerType) buildTLSApp(
 				ap.Issuers = issuers
 			}
 
+			// certificate managers
+			if certManagerVals, ok := sblock.pile["tls.cert_manager"]; ok {
+				for _, certManager := range certManagerVals {
+					certGetterName := certManager.Value.(caddy.Module).CaddyModule().ID.Name()
+					ap.ManagersRaw = append(ap.ManagersRaw, caddyconfig.JSONModuleObject(certManager.Value, "via", certGetterName, &warnings))
+				}
+			}
 			// custom bind host
 			for _, cfgVal := range sblock.pile["bind"] {
 				for _, iss := range ap.Issuers {
@@ -286,6 +299,19 @@ func (st ServerType) buildTLSApp(
 		tlsApp.Automation.StorageCleanInterval = storageCleanInterval
 	}
 
+	// set the expired certificates renew interval if configured
+	if renewCheckInterval, ok := options["renew_interval"].(caddy.Duration); ok {
+		if tlsApp.Automation == nil {
+			tlsApp.Automation = new(caddytls.AutomationConfig)
+		}
+		tlsApp.Automation.RenewCheckInterval = renewCheckInterval
+	}
+
+	// set whether OCSP stapling should be disabled for manually-managed certificates
+	if ocspConfig, ok := options["ocsp_stapling"].(certmagic.OCSPConfig); ok {
+		tlsApp.DisableOCSPStapling = ocspConfig.DisableStapling
+	}
+
 	// if any hostnames appear on the same server block as a key with
 	// no host, they will not be used with route matchers because the
 	// hostless key matches all hosts, therefore, it wouldn't be
@@ -324,7 +350,6 @@ func (st ServerType) buildTLSApp(
 		globalPreferredChains := options["preferred_chains"]
 		hasGlobalACMEDefaults := globalEmail != nil || globalACMECA != nil || globalACMECARoot != nil || globalACMEDNS != nil || globalACMEEAB != nil || globalPreferredChains != nil
 		if hasGlobalACMEDefaults {
-			// for _, ap := range tlsApp.Automation.Policies {
 			for i := 0; i < len(tlsApp.Automation.Policies); i++ {
 				ap := tlsApp.Automation.Policies[i]
 				if len(ap.Issuers) == 0 && automationPolicyHasAllPublicNames(ap) {

caddyconfig/httploader.go

@@ -1,11 +1,26 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package caddyconfig
 
 import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
+	"os"
 	"time"
 
 	"github.com/caddyserver/caddy/v2"
@@ -56,21 +71,28 @@ func (HTTPLoader) CaddyModule() caddy.ModuleInfo {
 
 // LoadConfig loads a Caddy config.
 func (hl HTTPLoader) LoadConfig(ctx caddy.Context) ([]byte, error) {
+	repl := caddy.NewReplacer()
+
 	client, err := hl.makeClient(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	method := hl.Method
+	method := repl.ReplaceAll(hl.Method, "")
 	if method == "" {
 		method = http.MethodGet
 	}
 
-	req, err := http.NewRequest(method, hl.URL, nil)
+	url := repl.ReplaceAll(hl.URL, "")
+	req, err := http.NewRequest(method, url, nil)
 	if err != nil {
 		return nil, err
 	}
-	req.Header = hl.Headers
+	for key, vals := range hl.Headers {
+		for _, val := range vals {
+			req.Header.Add(repl.ReplaceAll(key, ""), repl.ReplaceKnown(val, ""))
+		}
+	}
 
 	resp, err := client.Do(req)
 	if err != nil {
@@ -81,7 +103,7 @@ func (hl HTTPLoader) LoadConfig(ctx caddy.Context) ([]byte, error) {
 		return nil, fmt.Errorf("server responded with HTTP %d", resp.StatusCode)
 	}
 
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, err
 	}
@@ -130,7 +152,7 @@ func (hl HTTPLoader) makeClient(ctx caddy.Context) (*http.Client, error) {
 		if len(hl.TLS.RootCAPEMFiles) > 0 {
 			rootPool := x509.NewCertPool()
 			for _, pemFile := range hl.TLS.RootCAPEMFiles {
-				pemData, err := ioutil.ReadFile(pemFile)
+				pemData, err := os.ReadFile(pemFile)
 				if err != nil {
 					return nil, fmt.Errorf("failed reading ca cert: %v", err)
 				}

caddytest/caddytest.go

@@ -7,7 +7,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"net"
 	"net/http"
@@ -129,7 +129,7 @@ func (tc *Tester) initServer(rawConfig string, configType string) error {
 				return
 			}
 			defer res.Body.Close()
-			body, _ := ioutil.ReadAll(res.Body)
+			body, _ := io.ReadAll(res.Body)
 
 			var out bytes.Buffer
 			_ = json.Indent(&out, body, "", "  ")
@@ -162,7 +162,7 @@ func (tc *Tester) initServer(rawConfig string, configType string) error {
 	timeElapsed(start, "caddytest: config load time")
 
 	defer res.Body.Close()
-	body, err := ioutil.ReadAll(res.Body)
+	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		tc.t.Errorf("unable to read response. %s", err)
 		return err
@@ -202,7 +202,7 @@ func (tc *Tester) ensureConfigRunning(rawConfig string, configType string) error
 			return nil
 		}
 		defer resp.Body.Close()
-		actualBytes, err := ioutil.ReadAll(resp.Body)
+		actualBytes, err := io.ReadAll(resp.Body)
 		if err != nil {
 			return nil
 		}
@@ -471,7 +471,7 @@ func (tc *Tester) AssertResponse(req *http.Request, expectedStatusCode int, expe
 	resp := tc.AssertResponseCode(req, expectedStatusCode)
 
 	defer resp.Body.Close()
-	bytes, err := ioutil.ReadAll(resp.Body)
+	bytes, err := io.ReadAll(resp.Body)
 	if err != nil {
 		tc.t.Fatalf("unable to read the response body %s", err)
 	}

caddytest/integration/caddyfile_adapt/bind_ipv6.txt

@@ -0,0 +1,29 @@
+example.com {
+	bind tcp6/[::]
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						"tcp6/[::]:443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/expression_quotes.txt

@@ -0,0 +1,114 @@
+example.com
+
+@a expression {http.error.status_code} == 400
+abort @a
+
+@b expression {http.error.status_code} == "401"
+abort @b
+
+@c expression {http.error.status_code} == `402`
+abort @c
+
+@d expression "{http.error.status_code} == 403"
+abort @d
+
+@e expression `{http.error.status_code} == 404`
+abort @e
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == 400"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == \"401\""
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == `402`"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == 403"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == 404"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/file_server_pass_thru.txt

@@ -0,0 +1,32 @@
+:80
+
+file_server {
+	pass_thru
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "file_server",
+									"hide": [
+										"./Caddyfile"
+									],
+									"pass_thru": true
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/forward_auth_authelia.txt

@@ -0,0 +1,137 @@
+app.example.com {
+	forward_auth authelia:9091 {
+		uri /api/verify?rd=https://authelia.example.com
+		copy_headers Remote-User Remote-Groups Remote-Name Remote-Email
+	}
+
+	reverse_proxy backend:8080
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"app.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handle_response": [
+														{
+															"match": {
+																"status_code": [
+																	2
+																]
+															},
+															"routes": [
+																{
+																	"handle": [
+																		{
+																			"handler": "headers",
+																			"request": {
+																				"set": {
+																					"Remote-Email": [
+																						"{http.reverse_proxy.header.Remote-Email}"
+																					],
+																					"Remote-Groups": [
+																						"{http.reverse_proxy.header.Remote-Groups}"
+																					],
+																					"Remote-Name": [
+																						"{http.reverse_proxy.header.Remote-Name}"
+																					],
+																					"Remote-User": [
+																						"{http.reverse_proxy.header.Remote-User}"
+																					]
+																				}
+																			}
+																		}
+																	]
+																}
+															]
+														},
+														{
+															"routes": [
+																{
+																	"handle": [
+																		{
+																			"exclude": [
+																				"Connection",
+																				"Keep-Alive",
+																				"Te",
+																				"Trailers",
+																				"Transfer-Encoding",
+																				"Upgrade"
+																			],
+																			"handler": "copy_response_headers"
+																		}
+																	]
+																},
+																{
+																	"handle": [
+																		{
+																			"handler": "copy_response"
+																		}
+																	]
+																}
+															]
+														}
+													],
+													"handler": "reverse_proxy",
+													"headers": {
+														"request": {
+															"set": {
+																"X-Forwarded-Method": [
+																	"{http.request.method}"
+																],
+																"X-Forwarded-Uri": [
+																	"{http.request.uri}"
+																]
+															}
+														}
+													},
+													"rewrite": {
+														"method": "GET",
+														"uri": "/api/verify?rd=https://authelia.example.com"
+													},
+													"upstreams": [
+														{
+															"dial": "authelia:9091"
+														}
+													]
+												},
+												{
+													"handler": "reverse_proxy",
+													"upstreams": [
+														{
+															"dial": "backend:8080"
+														}
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options.txt

@@ -10,6 +10,7 @@
 	}
 	acme_ca https://example.com
 	acme_ca_root /path/to/ca.crt
+	ocsp_stapling off
 
 	email test@example.com
 	admin off
@@ -61,7 +62,8 @@
 								"module": "internal"
 							}
 						],
-						"key_type": "ed25519"
+						"key_type": "ed25519",
+						"disable_ocsp_stapling": true
 					}
 				],
 				"on_demand": {
@@ -71,7 +73,8 @@
 					},
 					"ask": "https://example.com"
 				}
-			}
+			},
+			"disable_ocsp_stapling": true
 		}
 	}
 }

caddytest/integration/caddyfile_adapt/global_options_acme.txt

@@ -21,6 +21,7 @@
 		burst 20
 	}
 	storage_clean_interval 7d
+	renew_interval 1d
 
 	key_type ed25519
 }
@@ -82,6 +83,7 @@
 					},
 					"ask": "https://example.com"
 				},
+				"renew_interval": 86400000000000,
 				"storage_clean_interval": 604800000000000
 			}
 		}

caddytest/integration/caddyfile_adapt/global_options_debug_with_access_log.txt

@@ -0,0 +1,45 @@
+{
+	debug
+}
+
+:8881 {
+	log {
+		format console
+	}
+}
+----------
+{
+	"logging": {
+		"logs": {
+			"default": {
+				"level": "DEBUG",
+				"exclude": [
+					"http.log.access.log0"
+				]
+			},
+			"log0": {
+				"encoder": {
+					"format": "console"
+				},
+				"level": "DEBUG",
+				"include": [
+					"http.log.access.log0"
+				]
+			}
+		}
+	},
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8881"
+					],
+					"logs": {
+						"default_logger_name": "log0"
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options_default_bind.txt

@@ -0,0 +1,54 @@
+{
+	default_bind tcp4/0.0.0.0 tcp6/[::]
+}
+
+example.com {
+}
+
+example.org:12345 {
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						"tcp4/0.0.0.0:12345",
+						"tcp6/[::]:12345"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.org"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				},
+				"srv1": {
+					"listen": [
+						"tcp4/0.0.0.0:443",
+						"tcp6/[::]:443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options_log_custom.txt

@@ -3,8 +3,7 @@
 		format filter {
 			wrap console
 			fields {
-				common_log delete
-				request>remote_addr ip_mask {
+				request>remote_ip ip_mask {
 					ipv4 24
 					ipv6 32
 				}
@@ -19,10 +18,7 @@
 			"custom-logger": {
 				"encoder": {
 					"fields": {
-						"common_log": {
-							"filter": "delete"
-						},
-						"request\u003eremote_addr": {
+						"request\u003eremote_ip": {
 							"filter": "ip_mask",
 							"ipv4_cidr": 24,
 							"ipv6_cidr": 32

caddytest/integration/caddyfile_adapt/global_options_skip_install_trust.txt

@@ -1,10 +1,44 @@
 {
 	skip_install_trust
+	pki {
+		ca {
+			name "Local"
+			root_cn "Custom Local Root Name"
+			intermediate_cn "Custom Local Intermediate Name"
+			root {
+				cert /path/to/cert.pem
+				key /path/to/key.pem
+				format pem_file
+			}
+			intermediate {
+				cert /path/to/cert.pem
+				key /path/to/key.pem
+				format pem_file
+			}
+		}
+		ca foo {
+			name "Foo"
+			root_cn "Custom Foo Root Name"
+			intermediate_cn "Custom Foo Intermediate Name"
+		}
+	}
 }
 
 a.example.com {
 	tls internal
 }
+
+acme.example.com {
+	acme_server {
+		ca foo
+	}
+}
+
+acme-bar.example.com {
+	acme_server {
+		ca bar
+	}
+}
 ----------
 {
 	"apps": {
@@ -15,6 +49,56 @@ a.example.com {
 						":443"
 					],
 					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"acme-bar.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"ca": "bar",
+													"handler": "acme_server"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"acme.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"ca": "foo",
+													"handler": "acme_server"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						},
 						{
 							"match": [
 								{
@@ -31,14 +115,42 @@ a.example.com {
 		},
 		"pki": {
 			"certificate_authorities": {
-				"local": {
+				"bar": {
 					"install_trust": false
+				},
+				"foo": {
+					"name": "Foo",
+					"root_common_name": "Custom Foo Root Name",
+					"intermediate_common_name": "Custom Foo Intermediate Name",
+					"install_trust": false
+				},
+				"local": {
+					"name": "Local",
+					"root_common_name": "Custom Local Root Name",
+					"intermediate_common_name": "Custom Local Intermediate Name",
+					"install_trust": false,
+					"root": {
+						"certificate": "/path/to/cert.pem",
+						"private_key": "/path/to/key.pem",
+						"format": "pem_file"
+					},
+					"intermediate": {
+						"certificate": "/path/to/cert.pem",
+						"private_key": "/path/to/key.pem",
+						"format": "pem_file"
+					}
 				}
 			}
 		},
 		"tls": {
 			"automation": {
 				"policies": [
+					{
+						"subjects": [
+							"acme-bar.example.com",
+							"acme.example.com"
+						]
+					},
 					{
 						"subjects": [
 							"a.example.com"

caddytest/integration/caddyfile_adapt/global_server_options_multi.txt

@@ -3,6 +3,9 @@
 		timeouts {
 			idle 90s
 		}
+		protocol {
+			strict_sni_host insecure_off
+		}
 	}
 	servers :80 {
 		timeouts {
@@ -13,6 +16,9 @@
 		timeouts {
 			idle 30s
 		}
+		protocol {
+			strict_sni_host
+		}
 	}
 }
 
@@ -46,7 +52,8 @@ http://bar.com {
 							],
 							"terminal": true
 						}
-					]
+					],
+					"strict_sni_host": true
 				},
 				"srv1": {
 					"listen": [
@@ -70,7 +77,8 @@ http://bar.com {
 					"listen": [
 						":8080"
 					],
-					"idle_timeout": 90000000000
+					"idle_timeout": 90000000000,
+					"strict_sni_host": false
 				}
 			}
 		}

caddytest/integration/caddyfile_adapt/global_server_options_single.txt

@@ -1,6 +1,7 @@
 {
 	servers {
 		listener_wrappers {
+			http_redirect
 			tls
 		}
 		timeouts {
@@ -10,6 +11,7 @@
 			idle 30s
 		}
 		max_header_size 100MB
+		log_credentials
 		protocol {
 			allow_h2c
 			experimental_http3
@@ -31,6 +33,9 @@ foo.com {
 						":443"
 					],
 					"listener_wrappers": [
+						{
+							"wrapper": "http_redirect"
+						},
 						{
 							"wrapper": "tls"
 						}
@@ -53,6 +58,9 @@ foo.com {
 						}
 					],
 					"strict_sni_host": true,
+					"logs": {
+						"should_log_credentials": true
+					},
 					"experimental_http3": true,
 					"allow_h2c": true
 				}

caddytest/integration/caddyfile_adapt/header.txt

@@ -13,6 +13,10 @@
 	header @images {
 		Cache-Control "public, max-age=3600, stale-while-revalidate=86400"
 	}
+	header {
+		+Link "Foo"
+		+Link "Bar"
+	}
 }
 ----------
 {
@@ -121,6 +125,17 @@
 											]
 										}
 									}
+								},
+								{
+									"handler": "headers",
+									"response": {
+										"add": {
+											"Link": [
+												"Foo",
+												"Bar"
+											]
+										}
+									}
 								}
 							]
 						}

caddytest/integration/caddyfile_adapt/log_filters.txt

@@ -5,12 +5,24 @@ log {
 	format filter {
 		wrap console
 		fields {
+			uri query {
+				replace foo REDACTED
+				delete bar
+				hash baz
+			}
 			request>headers>Authorization replace REDACTED
 			request>headers>Server delete
-			request>remote_addr ip_mask {
+			request>headers>Cookie cookie {
+				replace foo REDACTED
+				delete bar
+				hash baz
+			}
+			request>remote_ip ip_mask {
 				ipv4 24
 				ipv6 32
 			}
+			request>headers>Regexp regexp secret REDACTED
+			request>headers>Hash hash
 		}
 	}
 }
@@ -33,13 +45,57 @@ log {
 							"filter": "replace",
 							"value": "REDACTED"
 						},
+						"request\u003eheaders\u003eCookie": {
+							"actions": [
+								{
+									"name": "foo",
+									"type": "replace",
+									"value": "REDACTED"
+								},
+								{
+									"name": "bar",
+									"type": "delete"
+								},
+								{
+									"name": "baz",
+									"type": "hash"
+								}
+							],
+							"filter": "cookie"
+						},
+						"request\u003eheaders\u003eHash": {
+							"filter": "hash"
+						},
+						"request\u003eheaders\u003eRegexp": {
+							"filter": "regexp",
+							"regexp": "secret",
+							"value": "REDACTED"
+						},
 						"request\u003eheaders\u003eServer": {
 							"filter": "delete"
 						},
-						"request\u003eremote_addr": {
+						"request\u003eremote_ip": {
 							"filter": "ip_mask",
 							"ipv4_cidr": 24,
 							"ipv6_cidr": 32
+						},
+						"uri": {
+							"actions": [
+								{
+									"parameter": "foo",
+									"type": "replace",
+									"value": "REDACTED"
+								},
+								{
+									"parameter": "bar",
+									"type": "delete"
+								},
+								{
+									"parameter": "baz",
+									"type": "hash"
+								}
+							],
+							"filter": "query"
 						}
 					},
 					"format": "filter",

caddytest/integration/caddyfile_adapt/log_roll_days.txt

@@ -3,6 +3,8 @@
 log {
 	output file /var/log/access.log {
 		roll_size 1gb
+		roll_uncompressed
+		roll_local_time
 		roll_keep 5
 		roll_keep_for 90d
 	}
@@ -20,8 +22,10 @@ log {
 				"writer": {
 					"filename": "/var/log/access.log",
 					"output": "file",
+					"roll_gzip": false,
 					"roll_keep": 5,
 					"roll_keep_days": 90,
+					"roll_local_time": true,
 					"roll_size_mb": 954
 				},
 				"include": [

caddytest/integration/caddyfile_adapt/map_and_vars_with_raw_types.txt

@@ -0,0 +1,126 @@
+example.com
+
+map {host}             {my_placeholder}  {magic_number} {
+	# Should output boolean "true" and an integer
+	example.com        true              3
+
+	# Should output a string and null
+	foo.example.com    "string value"
+
+	# Should output two strings (quoted int)
+	(.*)\.example.com  "${1} subdomain"  "5"
+
+	# Should output null and a string (quoted int)
+	~.*\.net$          -                 `7`
+
+	# Should output a float and the string "false"
+	~.*\.xyz$          123.456           "false"
+
+	# Should output two strings, second being escaped quote
+	default            "unknown domain"  \"""
+}
+
+vars foo bar
+vars {
+	abc true
+	def 1
+	ghi 2.3
+	jkl "mn op"
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"defaults": [
+														"unknown domain",
+														"\""
+													],
+													"destinations": [
+														"{my_placeholder}",
+														"{magic_number}"
+													],
+													"handler": "map",
+													"mappings": [
+														{
+															"input": "example.com",
+															"outputs": [
+																true,
+																3
+															]
+														},
+														{
+															"input": "foo.example.com",
+															"outputs": [
+																"string value",
+																null
+															]
+														},
+														{
+															"input": "(.*)\\.example.com",
+															"outputs": [
+																"${1} subdomain",
+																"5"
+															]
+														},
+														{
+															"input_regexp": ".*\\.net$",
+															"outputs": [
+																null,
+																"7"
+															]
+														},
+														{
+															"input_regexp": ".*\\.xyz$",
+															"outputs": [
+																123.456,
+																"false"
+															]
+														}
+													],
+													"source": "{http.request.host}"
+												},
+												{
+													"foo": "bar",
+													"handler": "vars"
+												},
+												{
+													"abc": true,
+													"def": 1,
+													"ghi": 2.3,
+													"handler": "vars",
+													"jkl": "mn op"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/matcher_syntax.txt

@@ -37,6 +37,9 @@
 		header Bar foo
 	}
 	respond @matcher9 "header matcher with null field matcher"
+
+	@matcher10 remote_ip private_ranges
+	respond @matcher10 "remote_ip matcher with private ranges"
 }
 ----------
 {
@@ -101,7 +104,9 @@
 							"match": [
 								{
 									"vars": {
-										"{http.request.uri}": "/vars-matcher"
+										"{http.request.uri}": [
+											"/vars-matcher"
+										]
 									}
 								}
 							],
@@ -207,6 +212,28 @@
 									"handler": "static_response"
 								}
 							]
+						},
+						{
+							"match": [
+								{
+									"remote_ip": {
+										"ranges": [
+											"192.168.0.0/16",
+											"172.16.0.0/12",
+											"10.0.0.0/8",
+											"127.0.0.1/8",
+											"fd00::/8",
+											"::1"
+										]
+									}
+								}
+							],
+							"handle": [
+								{
+									"body": "remote_ip matcher with private ranges",
+									"handler": "static_response"
+								}
+							]
 						}
 					]
 				}

caddytest/integration/caddyfile_adapt/method_directive.txt

@@ -0,0 +1,27 @@
+:8080 {
+	method FOO
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8080"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "rewrite",
+									"method": "FOO"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/php_fastcgi_handle_response.txt

@@ -1,145 +1,145 @@
-:8881 {
-	php_fastcgi app:9000 {
-		env FOO bar
-
-		@error status 4xx
-		handle_response @error {
-			root * /errors
-			rewrite * /{http.reverse_proxy.status_code}.html
-			file_server
-		}
-	}
-}
-----------
-{
-	"apps": {
-		"http": {
-			"servers": {
-				"srv0": {
-					"listen": [
-						":8881"
-					],
-					"routes": [
-						{
-							"match": [
-								{
-									"file": {
-										"try_files": [
-											"{http.request.uri.path}/index.php"
-										]
-									},
-									"not": [
-										{
-											"path": [
-												"*/"
-											]
-										}
-									]
-								}
-							],
-							"handle": [
-								{
-									"handler": "static_response",
-									"headers": {
-										"Location": [
-											"{http.request.uri.path}/"
-										]
-									},
-									"status_code": 308
-								}
-							]
-						},
-						{
-							"match": [
-								{
-									"file": {
-										"try_files": [
-											"{http.request.uri.path}",
-											"{http.request.uri.path}/index.php",
-											"index.php"
-										],
-										"split_path": [
-											".php"
-										]
-									}
-								}
-							],
-							"handle": [
-								{
-									"handler": "rewrite",
-									"uri": "{http.matchers.file.relative}"
-								}
-							]
-						},
-						{
-							"match": [
-								{
-									"path": [
-										"*.php"
-									]
-								}
-							],
-							"handle": [
-								{
-									"handle_response": [
-										{
-											"match": {
-												"status_code": [
-													4
-												]
-											},
-											"routes": [
-												{
-													"handle": [
-														{
-															"handler": "vars",
-															"root": "/errors"
-														}
-													]
-												},
-												{
-													"group": "group0",
-													"handle": [
-														{
-															"handler": "rewrite",
-															"uri": "/{http.reverse_proxy.status_code}.html"
-														}
-													]
-												},
-												{
-													"handle": [
-														{
-															"handler": "file_server",
-															"hide": [
-																"./Caddyfile"
-															]
-														}
-													]
-												}
-											]
-										}
-									],
-									"handler": "reverse_proxy",
-									"transport": {
-										"env": {
-											"FOO": "bar"
-										},
-										"protocol": "fastcgi",
-										"split_path": [
-											".php"
-										]
-									},
-									"upstreams": [
-										{
-											"dial": "app:9000"
-										}
-									]
-								}
-							]
-						}
-					]
-				}
-			}
-		}
-	}
+:8881 {
+	php_fastcgi app:9000 {
+		env FOO bar
+
+		@error status 4xx
+		handle_response @error {
+			root * /errors
+			rewrite * /{http.reverse_proxy.status_code}.html
+			file_server
+		}
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8881"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"file": {
+										"try_files": [
+											"{http.request.uri.path}/index.php"
+										]
+									},
+									"not": [
+										{
+											"path": [
+												"*/"
+											]
+										}
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "static_response",
+									"headers": {
+										"Location": [
+											"{http.request.uri.path}/"
+										]
+									},
+									"status_code": 308
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"file": {
+										"try_files": [
+											"{http.request.uri.path}",
+											"{http.request.uri.path}/index.php",
+											"index.php"
+										],
+										"split_path": [
+											".php"
+										]
+									}
+								}
+							],
+							"handle": [
+								{
+									"handler": "rewrite",
+									"uri": "{http.matchers.file.relative}"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"path": [
+										"*.php"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handle_response": [
+										{
+											"match": {
+												"status_code": [
+													4
+												]
+											},
+											"routes": [
+												{
+													"handle": [
+														{
+															"handler": "vars",
+															"root": "/errors"
+														}
+													]
+												},
+												{
+													"group": "group0",
+													"handle": [
+														{
+															"handler": "rewrite",
+															"uri": "/{http.reverse_proxy.status_code}.html"
+														}
+													]
+												},
+												{
+													"handle": [
+														{
+															"handler": "file_server",
+															"hide": [
+																"./Caddyfile"
+															]
+														}
+													]
+												}
+											]
+										}
+									],
+									"handler": "reverse_proxy",
+									"transport": {
+										"env": {
+											"FOO": "bar"
+										},
+										"protocol": "fastcgi",
+										"split_path": [
+											".php"
+										]
+									},
+									"upstreams": [
+										{
+											"dial": "app:9000"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
 }

caddytest/integration/caddyfile_adapt/php_fastcgi_try_files_override.txt

@@ -0,0 +1,124 @@
+:8884
+
+php_fastcgi localhost:9000 {
+	# some php_fastcgi-specific subdirectives
+	split .php .php5
+	env VAR1 value1
+	env VAR2 value2
+	root /var/www
+	try_files {path} {path}/index.php =404
+	dial_timeout 3s
+	read_timeout 10s
+	write_timeout 20s
+
+	# passed through to reverse_proxy (directive order doesn't matter!)
+	lb_policy random
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"file": {
+										"try_files": [
+											"{http.request.uri.path}/index.php"
+										]
+									},
+									"not": [
+										{
+											"path": [
+												"*/"
+											]
+										}
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "static_response",
+									"headers": {
+										"Location": [
+											"{http.request.uri.path}/"
+										]
+									},
+									"status_code": 308
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"file": {
+										"try_files": [
+											"{http.request.uri.path}",
+											"{http.request.uri.path}/index.php",
+											"=404"
+										],
+										"split_path": [
+											".php",
+											".php5"
+										]
+									}
+								}
+							],
+							"handle": [
+								{
+									"handler": "rewrite",
+									"uri": "{http.matchers.file.relative}"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"path": [
+										"*.php",
+										"*.php5"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "reverse_proxy",
+									"load_balancing": {
+										"selection_policy": {
+											"policy": "random"
+										}
+									},
+									"transport": {
+										"dial_timeout": 3000000000,
+										"env": {
+											"VAR1": "value1",
+											"VAR2": "value2"
+										},
+										"protocol": "fastcgi",
+										"read_timeout": 10000000000,
+										"root": "/var/www",
+										"split_path": [
+											".php",
+											".php5"
+										],
+										"write_timeout": 20000000000
+									},
+									"upstreams": [
+										{
+											"dial": "localhost:9000"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/reverse_proxy_dynamic_upstreams.txt

@@ -0,0 +1,116 @@
+:8884 {
+	reverse_proxy {
+		dynamic a foo 9000
+	}
+
+	reverse_proxy {
+		dynamic a {
+			name foo
+			port 9000
+			refresh 5m
+			resolvers 8.8.8.8 8.8.4.4
+			dial_timeout 2s
+			dial_fallback_delay 300ms
+		}
+	}
+}
+
+:8885 {
+	reverse_proxy {
+		dynamic srv _api._tcp.example.com
+	}
+
+	reverse_proxy {
+		dynamic srv {
+			service api
+			proto tcp
+			name example.com
+			refresh 5m
+			resolvers 8.8.8.8 8.8.4.4
+			dial_timeout 1s
+			dial_fallback_delay -1s
+		}
+	}
+}
+
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"dynamic_upstreams": {
+										"name": "foo",
+										"port": "9000",
+										"source": "a"
+									},
+									"handler": "reverse_proxy"
+								},
+								{
+									"dynamic_upstreams": {
+										"dial_fallback_delay": 300000000,
+										"dial_timeout": 2000000000,
+										"name": "foo",
+										"port": "9000",
+										"refresh": 300000000000,
+										"resolver": {
+											"addresses": [
+												"8.8.8.8",
+												"8.8.4.4"
+											]
+										},
+										"source": "a"
+									},
+									"handler": "reverse_proxy"
+								}
+							]
+						}
+					]
+				},
+				"srv1": {
+					"listen": [
+						":8885"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"dynamic_upstreams": {
+										"name": "_api._tcp.example.com",
+										"source": "srv"
+									},
+									"handler": "reverse_proxy"
+								},
+								{
+									"dynamic_upstreams": {
+										"dial_fallback_delay": -1000000000,
+										"dial_timeout": 1000000000,
+										"name": "example.com",
+										"proto": "tcp",
+										"refresh": 300000000000,
+										"resolver": {
+											"addresses": [
+												"8.8.8.8",
+												"8.8.4.4"
+											]
+										},
+										"service": "api",
+										"source": "srv"
+									},
+									"handler": "reverse_proxy"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/reverse_proxy_handle_response.txt

@@ -1,6 +1,14 @@
 :8884
 
 reverse_proxy 127.0.0.1:65535 {
+	@500 status 500
+	replace_status @500 400
+
+	@all status 2xx 3xx 4xx 5xx
+	replace_status @all {http.error.status_code}
+
+	replace_status {http.error.status_code}
+
 	@accel header X-Accel-Redirect *
 	handle_response @accel {
 		respond "Header X-Accel-Redirect!"
@@ -39,8 +47,19 @@ reverse_proxy 127.0.0.1:65535 {
 		respond "Headers Foo, Bar AND statuses 401, 403 and 404!"
 	}
 
-	@changeStatus status 500
-	handle_response @changeStatus 400
+	@200 status 200
+	handle_response @200 {
+		copy_response_headers {
+			include Foo Bar
+		}
+		respond "Copied headers from the response"
+	}
+
+	@201 status 201
+	handle_response @201 {
+		header Foo "Copying the response"
+		copy_response 404
+	}
 }
 ----------
 {
@@ -56,6 +75,25 @@ reverse_proxy 127.0.0.1:65535 {
 							"handle": [
 								{
 									"handle_response": [
+										{
+											"match": {
+												"status_code": [
+													500
+												]
+											},
+											"status_code": 400
+										},
+										{
+											"match": {
+												"status_code": [
+													2,
+													3,
+													4,
+													5
+												]
+											},
+											"status_code": "{http.error.status_code}"
+										},
 										{
 											"match": {
 												"headers": {
@@ -158,10 +196,56 @@ reverse_proxy 127.0.0.1:65535 {
 										{
 											"match": {
 												"status_code": [
-													500
+													200
 												]
 											},
-											"status_code": 400
+											"routes": [
+												{
+													"handle": [
+														{
+															"handler": "copy_response_headers",
+															"include": [
+																"Foo",
+																"Bar"
+															]
+														},
+														{
+															"body": "Copied headers from the response",
+															"handler": "static_response"
+														}
+													]
+												}
+											]
+										},
+										{
+											"match": {
+												"status_code": [
+													201
+												]
+											},
+											"routes": [
+												{
+													"handle": [
+														{
+															"handler": "headers",
+															"response": {
+																"set": {
+																	"Foo": [
+																		"Copying the response"
+																	]
+																}
+															}
+														},
+														{
+															"handler": "copy_response",
+															"status_code": 404
+														}
+													]
+												}
+											]
+										},
+										{
+											"status_code": "{http.error.status_code}"
 										},
 										{
 											"routes": [

caddytest/integration/caddyfile_adapt/reverse_proxy_health_headers.txt

@@ -7,6 +7,7 @@ reverse_proxy 127.0.0.1:65535 {
 		X-Header-Keys VbG4NZwWnipo 335Q9/MhqcNU3s2TO
 		X-Empty-Value
 	}
+	health_uri /health
 }
 ----------
 {
@@ -38,7 +39,8 @@ reverse_proxy 127.0.0.1:65535 {
 													"VbG4NZwWnipo",
 													"335Q9/MhqcNU3s2TO"
 												]
-											}
+											},
+											"uri": "/health"
 										}
 									},
 									"upstreams": [

caddytest/integration/caddyfile_adapt/reverse_proxy_ids.txt

@@ -1,46 +0,0 @@
-:8884
-
-reverse_proxy one|http://localhost two|http://localhost {
-	to three|srv+http://localhost four|srv+http://localhost
-}
-----------
-{
-	"apps": {
-		"http": {
-			"servers": {
-				"srv0": {
-					"listen": [
-						":8884"
-					],
-					"routes": [
-						{
-							"handle": [
-								{
-									"handler": "reverse_proxy",
-									"upstreams": [
-										{
-											"dial": "localhost:80",
-											"id": "one"
-										},
-										{
-											"dial": "localhost:80",
-											"id": "two"
-										},
-										{
-											"id": "three",
-											"lookup_srv": "localhost"
-										},
-										{
-											"id": "four",
-											"lookup_srv": "localhost"
-										}
-									]
-								}
-							]
-						}
-					]
-				}
-			}
-		}
-	}
-}

caddytest/integration/caddyfile_adapt/reverse_proxy_options.txt

@@ -1,11 +1,11 @@
 
 https://example.com {
-	reverse_proxy /path http://localhost:54321 {
-		header_up Host {host}
-		header_up X-Real-IP {remote}
-		header_up X-Forwarded-For {remote}
-		header_up X-Forwarded-Port {server_port}
-		header_up X-Forwarded-Proto "http"
+	reverse_proxy /path https://localhost:54321 {
+		header_up Host {upstream_hostport}
+		header_up Foo bar
+
+		method GET
+		rewrite /rewritten?uri={uri}
 
 		buffer_requests
 
@@ -17,11 +17,14 @@ https://example.com {
 			dial_fallback_delay 5s
 			response_header_timeout 8s
 			expect_continue_timeout 9s
+			resolvers 8.8.8.8 8.8.4.4
 
 			versions h2c 2
 			compression off
 			max_conns_per_host 5
 			keepalive_idle_conns_per_host 2
+			keepalive_interval 30s
+			renegotiation freely
 		}
 	}
 }
@@ -56,37 +59,42 @@ https://example.com {
 													"headers": {
 														"request": {
 															"set": {
+																"Foo": [
+																	"bar"
+																],
 																"Host": [
-																	"{http.request.host}"
-																],
-																"X-Forwarded-For": [
-																	"{http.request.remote}"
-																],
-																"X-Forwarded-Port": [
-																	"{server_port}"
-																],
-																"X-Forwarded-Proto": [
-																	"http"
-																],
-																"X-Real-Ip": [
-																	"{http.request.remote}"
+																	"{http.reverse_proxy.upstream.hostport}"
 																]
 															}
 														}
 													},
+													"rewrite": {
+														"method": "GET",
+														"uri": "/rewritten?uri={http.request.uri}"
+													},
 													"transport": {
 														"compression": false,
 														"dial_fallback_delay": 5000000000,
 														"dial_timeout": 3000000000,
 														"expect_continue_timeout": 9000000000,
 														"keep_alive": {
-															"max_idle_conns_per_host": 2
+															"max_idle_conns_per_host": 2,
+															"probe_interval": 30000000000
 														},
 														"max_conns_per_host": 5,
 														"max_response_header_size": 30000000,
 														"protocol": "http",
 														"read_buffer_size": 10000000,
+														"resolver": {
+															"addresses": [
+																"8.8.8.8",
+																"8.8.4.4"
+															]
+														},
 														"response_header_timeout": 8000000000,
+														"tls": {
+															"renegotiation": "freely"
+														},
 														"versions": [
 															"h2c",
 															"2"

caddytest/integration/caddyfile_adapt/reverse_proxy_trusted_proxies.txt

@@ -0,0 +1,56 @@
+:8884
+
+reverse_proxy 127.0.0.1:65535 {
+	trusted_proxies 127.0.0.1
+}
+
+reverse_proxy 127.0.0.1:65535 {
+	trusted_proxies private_ranges
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "reverse_proxy",
+									"trusted_proxies": [
+										"127.0.0.1"
+									],
+									"upstreams": [
+										{
+											"dial": "127.0.0.1:65535"
+										}
+									]
+								},
+								{
+									"handler": "reverse_proxy",
+									"trusted_proxies": [
+										"192.168.0.0/16",
+										"172.16.0.0/12",
+										"10.0.0.0/8",
+										"127.0.0.1/8",
+										"fd00::/8",
+										"::1"
+									],
+									"upstreams": [
+										{
+											"dial": "127.0.0.1:65535"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/sort_directives_within_handle.txt

@@ -0,0 +1,133 @@
+*.example.com {
+	@foo host foo.example.com
+	handle @foo {
+		handle_path /strip* {
+			respond "this should be first"
+		}
+		handle {
+			respond "this should be second"
+		}
+	}
+	handle {
+		respond "this should be last"
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"*.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"group": "group5",
+											"handle": [
+												{
+													"handler": "subroute",
+													"routes": [
+														{
+															"group": "group2",
+															"handle": [
+																{
+																	"handler": "subroute",
+																	"routes": [
+																		{
+																			"handle": [
+																				{
+																					"handler": "rewrite",
+																					"strip_path_prefix": "/strip"
+																				}
+																			]
+																		},
+																		{
+																			"handle": [
+																				{
+																					"body": "this should be first",
+																					"handler": "static_response"
+																				}
+																			]
+																		}
+																	]
+																}
+															],
+															"match": [
+																{
+																	"path": [
+																		"/strip*"
+																	]
+																}
+															]
+														},
+														{
+															"group": "group2",
+															"handle": [
+																{
+																	"handler": "subroute",
+																	"routes": [
+																		{
+																			"handle": [
+																				{
+																					"body": "this should be second",
+																					"handler": "static_response"
+																				}
+																			]
+																		}
+																	]
+																}
+															]
+														}
+													]
+												}
+											],
+											"match": [
+												{
+													"host": [
+														"foo.example.com"
+													]
+												}
+											]
+										},
+										{
+											"group": "group5",
+											"handle": [
+												{
+													"handler": "subroute",
+													"routes": [
+														{
+															"handle": [
+																{
+																	"body": "this should be last",
+																	"handler": "static_response"
+																}
+															]
+														}
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/sort_vars_in_reverse.txt

@@ -0,0 +1,59 @@
+:80
+
+vars /foobar foo last
+vars /foo foo middle
+vars * foo first
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"foo": "first",
+									"handler": "vars"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"path": [
+										"/foo"
+									]
+								}
+							],
+							"handle": [
+								{
+									"foo": "middle",
+									"handler": "vars"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"path": [
+										"/foobar"
+									]
+								}
+							],
+							"handle": [
+								{
+									"foo": "last",
+									"handler": "vars"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_automation_policies_8.txt

@@ -0,0 +1,98 @@
+# (this Caddyfile is contrived, but based on issues #4176 and #4198)
+
+http://example.com {
+}
+
+https://example.com {
+	tls abc@example.com
+}
+
+http://localhost:8081 {
+}
+
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				},
+				"srv1": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				},
+				"srv2": {
+					"listen": [
+						":8081"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"automatic_https": {
+						"skip": [
+							"localhost"
+						]
+					}
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"example.com"
+						],
+						"issuers": [
+							{
+								"email": "abc@example.com",
+								"module": "acme"
+							},
+							{
+								"email": "abc@example.com",
+								"module": "zerossl"
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_automation_policies_9.txt

@@ -0,0 +1,56 @@
+# example from issue #4640
+http://foo:8447, http://127.0.0.1:8447 {
+	reverse_proxy 127.0.0.1:8080
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8447"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"foo",
+										"127.0.0.1"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "reverse_proxy",
+													"upstreams": [
+														{
+															"dial": "127.0.0.1:8080"
+														}
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"automatic_https": {
+						"skip": [
+							"foo",
+							"127.0.0.1"
+						]
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_internal_options.txt

@@ -0,0 +1,54 @@
+a.example.com {
+	tls {
+		issuer internal {
+			ca foo
+			lifetime 24h
+			sign_with_root
+		}
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"a.example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"a.example.com"
+						],
+						"issuers": [
+							{
+								"ca": "foo",
+								"lifetime": 86400000000000,
+								"module": "internal",
+								"sign_with_root": true
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_propagation_timeout.txt

@@ -3,7 +3,12 @@ localhost
 respond "hello from localhost"
 tls {
 	issuer acme {
-		propagation_timeout "10m0s"
+		propagation_delay 5m10s
+		propagation_timeout 10m20s
+	}
+	issuer zerossl {
+		propagation_delay 5m30s
+		propagation_timeout -1
 	}
 }
 ----------
@@ -56,10 +61,20 @@ tls {
 							{
 								"challenges": {
 									"dns": {
-										"propagation_timeout": 600000000000
+										"propagation_delay": 310000000000,
+										"propagation_timeout": 620000000000
 									}
 								},
 								"module": "acme"
+							},
+							{
+								"challenges": {
+									"dns": {
+										"propagation_delay": 330000000000,
+										"propagation_timeout": -1
+									}
+								},
+								"module": "zerossl"
 							}
 						]
 					}

caddytest/integration/caddyfile_adapt/tracing.txt

@@ -0,0 +1,36 @@
+:80 {
+	tracing /myhandler {
+		span my-span
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"path": [
+										"/myhandler"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "tracing",
+									"span": "my-span"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt_test.go

@@ -3,7 +3,7 @@ package integration
 import (
 	jsonMod "encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -14,7 +14,7 @@ import (
 
 func TestCaddyfileAdaptToJSON(t *testing.T) {
 	// load the list of test files from the dir
-	files, err := ioutil.ReadDir("./caddyfile_adapt")
+	files, err := os.ReadDir("./caddyfile_adapt")
 	if err != nil {
 		t.Errorf("failed to read caddyfile_adapt dir: %s", err)
 	}
@@ -29,7 +29,7 @@ func TestCaddyfileAdaptToJSON(t *testing.T) {
 
 		// read the test file
 		filename := f.Name()
-		data, err := ioutil.ReadFile("./caddyfile_adapt/" + filename)
+		data, err := os.ReadFile("./caddyfile_adapt/" + filename)
 		if err != nil {
 			t.Errorf("failed to read %s dir: %s", filename, err)
 		}

caddytest/integration/caddyfile_test.go

@@ -101,3 +101,27 @@ func TestReadCookie(t *testing.T) {
 	// act and assert
 	tester.AssertGetResponse("http://localhost:9080/cookie.html", 200, "<h2>Cookie.ClientName caddytest</h2>")
 }
+
+func TestReplIndex(t *testing.T) {
+
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+  {
+    http_port     9080
+    https_port    9443
+  }
+
+  localhost:9080 {
+    templates {
+      root testdata
+    }
+    file_server {
+      root testdata
+      index "index.{host}.html"
+    }
+  }
+  `, "caddyfile")
+
+	// act and assert
+	tester.AssertGetResponse("http://localhost:9080/", 200, "")
+}

caddytest/integration/reverseproxy_test.go

@@ -2,7 +2,6 @@ package integration
 
 import (
 	"fmt"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
@@ -85,7 +84,7 @@ func TestDialWithPlaceholderUnix(t *testing.T) {
 		t.SkipNow()
 	}
 
-	f, err := ioutil.TempFile("", "*.sock")
+	f, err := os.CreateTemp("", "*.sock")
 	if err != nil {
 		t.Errorf("failed to create TempFile: %s", err)
 		return
@@ -387,7 +386,7 @@ func TestReverseProxyHealthCheckUnixSocket(t *testing.T) {
 		t.SkipNow()
 	}
 	tester := caddytest.NewTester(t)
-	f, err := ioutil.TempFile("", "*.sock")
+	f, err := os.CreateTemp("", "*.sock")
 	if err != nil {
 		t.Errorf("failed to create TempFile: %s", err)
 		return
@@ -442,7 +441,7 @@ func TestReverseProxyHealthCheckUnixSocketWithoutPort(t *testing.T) {
 		t.SkipNow()
 	}
 	tester := caddytest.NewTester(t)
-	f, err := ioutil.TempFile("", "*.sock")
+	f, err := os.CreateTemp("", "*.sock")
 	if err != nil {
 		t.Errorf("failed to create TempFile: %s", err)
 		return

caddytest/integration/stream_test.go

@@ -6,7 +6,6 @@ import (
 	"crypto/rand"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -110,7 +109,7 @@ func TestH2ToH2CStream(t *testing.T) {
 	r, w := io.Pipe()
 	req := &http.Request{
 		Method: "PUT",
-		Body:   ioutil.NopCloser(r),
+		Body:   io.NopCloser(r),
 		URL: &url.URL{
 			Scheme: "https",
 			Host:   "127.0.0.1:9443",
@@ -134,7 +133,7 @@ func TestH2ToH2CStream(t *testing.T) {
 	}()
 
 	defer resp.Body.Close()
-	bytes, err := ioutil.ReadAll(resp.Body)
+	bytes, err := io.ReadAll(resp.Body)
 	if err != nil {
 		t.Fatalf("unable to read the response body %s", err)
 	}
@@ -319,7 +318,7 @@ func TestH2ToH1ChunkedResponse(t *testing.T) {
 	r, w := io.Pipe()
 	req := &http.Request{
 		Method: "PUT",
-		Body:   ioutil.NopCloser(r),
+		Body:   io.NopCloser(r),
 		URL: &url.URL{
 			Scheme: "https",
 			Host:   "127.0.0.1:9443",
@@ -342,7 +341,7 @@ func TestH2ToH1ChunkedResponse(t *testing.T) {
 	}
 
 	defer resp.Body.Close()
-	bytes, err := ioutil.ReadAll(resp.Body)
+	bytes, err := io.ReadAll(resp.Body)
 	if err != nil {
 		t.Fatalf("unable to read the response body %s", err)
 	}
@@ -370,7 +369,7 @@ func testH2ToH1ChunkedResponseServeH1(t *testing.T) *http.Server {
 		}
 
 		defer r.Body.Close()
-		bytes, err := ioutil.ReadAll(r.Body)
+		bytes, err := io.ReadAll(r.Body)
 		if err != nil {
 			t.Fatalf("unable to read the response body %s", err)
 		}

cmd/commandfuncs.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
@@ -33,6 +32,7 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/aryann/difflib"
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
@@ -181,7 +181,7 @@ func cmdRun(fl Flags) (int, error) {
 	var config []byte
 	var err error
 	if runCmdResumeFlag {
-		config, err = ioutil.ReadFile(caddy.ConfigAutosavePath)
+		config, err = os.ReadFile(caddy.ConfigAutosavePath)
 		if os.IsNotExist(err) {
 			// not a bad error; just can't resume if autosave file doesn't exist
 			caddy.Log().Info("no autosave file exists", zap.String("autosave_file", caddy.ConfigAutosavePath))
@@ -203,7 +203,7 @@ func cmdRun(fl Flags) (int, error) {
 	// we don't use 'else' here since this value might have been changed in 'if' block; i.e. not mutually exclusive
 	var configFile string
 	if !runCmdResumeFlag {
-		config, configFile, err = loadConfig(runCmdConfigFlag, runCmdConfigAdapterFlag)
+		config, configFile, err = LoadConfig(runCmdConfigFlag, runCmdConfigAdapterFlag)
 		if err != nil {
 			return caddy.ExitCodeFailedStartup, err
 		}
@@ -219,7 +219,7 @@ func cmdRun(fl Flags) (int, error) {
 	// if we are to report to another process the successful start
 	// of the server, do so now by echoing back contents of stdin
 	if runCmdPingbackFlag != "" {
-		confirmationBytes, err := ioutil.ReadAll(os.Stdin)
+		confirmationBytes, err := io.ReadAll(os.Stdin)
 		if err != nil {
 			return caddy.ExitCodeFailedStartup,
 				fmt.Errorf("reading confirmation bytes from stdin: %v", err)
@@ -276,25 +276,33 @@ func cmdRun(fl Flags) (int, error) {
 }
 
 func cmdStop(fl Flags) (int, error) {
-	stopCmdAddrFlag := fl.String("address")
+	addrFlag := fl.String("address")
+	configFlag := fl.String("config")
+	configAdapterFlag := fl.String("adapter")
 
-	err := apiRequest(stopCmdAddrFlag, http.MethodPost, "/stop", nil, nil)
+	adminAddr, err := DetermineAdminAPIAddress(addrFlag, configFlag, configAdapterFlag)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("couldn't determine admin API address: %v", err)
+	}
+
+	resp, err := AdminAPIRequest(adminAddr, http.MethodPost, "/stop", nil, nil)
 	if err != nil {
 		caddy.Log().Warn("failed using API to stop instance", zap.Error(err))
 		return caddy.ExitCodeFailedStartup, err
 	}
+	defer resp.Body.Close()
 
 	return caddy.ExitCodeSuccess, nil
 }
 
 func cmdReload(fl Flags) (int, error) {
-	reloadCmdConfigFlag := fl.String("config")
-	reloadCmdConfigAdapterFlag := fl.String("adapter")
-	reloadCmdAddrFlag := fl.String("address")
-	reloadCmdForceFlag := fl.Bool("force")
+	configFlag := fl.String("config")
+	configAdapterFlag := fl.String("adapter")
+	addrFlag := fl.String("address")
+	forceFlag := fl.Bool("force")
 
 	// get the config in caddy's native format
-	config, configFile, err := loadConfig(reloadCmdConfigFlag, reloadCmdConfigAdapterFlag)
+	config, configFile, err := LoadConfig(configFlag, configAdapterFlag)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, err
 	}
@@ -302,30 +310,22 @@ func cmdReload(fl Flags) (int, error) {
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("no config file to load")
 	}
 
-	// get the address of the admin listener; use flag if specified
-	adminAddr := reloadCmdAddrFlag
-	if adminAddr == "" && len(config) > 0 {
-		var tmpStruct struct {
-			Admin caddy.AdminConfig `json:"admin"`
-		}
-		err = json.Unmarshal(config, &tmpStruct)
-		if err != nil {
-			return caddy.ExitCodeFailedStartup,
-				fmt.Errorf("unmarshaling admin listener address from config: %v", err)
-		}
-		adminAddr = tmpStruct.Admin.Listen
+	adminAddr, err := DetermineAdminAPIAddress(addrFlag, configFlag, configAdapterFlag)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("couldn't determine admin API address: %v", err)
 	}
 
 	// optionally force a config reload
 	headers := make(http.Header)
-	if reloadCmdForceFlag {
+	if forceFlag {
 		headers.Set("Cache-Control", "must-revalidate")
 	}
 
-	err = apiRequest(adminAddr, http.MethodPost, "/load", headers, bytes.NewReader(config))
+	resp, err := AdminAPIRequest(adminAddr, http.MethodPost, "/load", headers, bytes.NewReader(config))
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("sending configuration to instance: %v", err)
 	}
+	defer resp.Body.Close()
 
 	return caddy.ExitCodeSuccess, nil
 }
@@ -361,6 +361,7 @@ func cmdBuildInfo(fl Flags) (int, error) {
 func cmdListModules(fl Flags) (int, error) {
 	packages := fl.Bool("packages")
 	versions := fl.Bool("versions")
+	skipStandard := fl.Bool("skip-standard")
 
 	printModuleInfo := func(mi moduleInfo) {
 		fmt.Print(mi.caddyModuleID)
@@ -389,14 +390,19 @@ func cmdListModules(fl Flags) (int, error) {
 		return caddy.ExitCodeSuccess, nil
 	}
 
-	if len(standard) > 0 {
-		for _, mod := range standard {
-			printModuleInfo(mod)
-		}
-	}
-	fmt.Printf("\n  Standard modules: %d\n", len(standard))
-	if len(nonstandard) > 0 {
+	// Standard modules (always shipped with Caddy)
+	if !skipStandard {
 		if len(standard) > 0 {
+			for _, mod := range standard {
+				printModuleInfo(mod)
+			}
+		}
+		fmt.Printf("\n  Standard modules: %d\n", len(standard))
+	}
+
+	// Non-standard modules (third party plugins)
+	if len(nonstandard) > 0 {
+		if len(standard) > 0 && !skipStandard {
 			fmt.Println()
 		}
 		for _, mod := range nonstandard {
@@ -404,8 +410,10 @@ func cmdListModules(fl Flags) (int, error) {
 		}
 	}
 	fmt.Printf("\n  Non-standard modules: %d\n", len(nonstandard))
+
+	// Unknown modules (couldn't get Caddy module info)
 	if len(unknown) > 0 {
-		if len(standard) > 0 || len(nonstandard) > 0 {
+		if (len(standard) > 0 && !skipStandard) || len(nonstandard) > 0 {
 			fmt.Println()
 		}
 		for _, mod := range unknown {
@@ -457,7 +465,7 @@ func cmdAdaptConfig(fl Flags) (int, error) {
 			fmt.Errorf("unrecognized config adapter: %s", adaptCmdAdapterFlag)
 	}
 
-	input, err := ioutil.ReadFile(adaptCmdInputFlag)
+	input, err := os.ReadFile(adaptCmdInputFlag)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup,
 			fmt.Errorf("reading input file: %v", err)
@@ -488,7 +496,9 @@ func cmdAdaptConfig(fl Flags) (int, error) {
 		if warn.Directive != "" {
 			msg = fmt.Sprintf("%s: %s", warn.Directive, warn.Message)
 		}
-		fmt.Fprintf(os.Stderr, "[WARNING][%s] %s:%d: %s\n", adaptCmdAdapterFlag, warn.File, warn.Line, msg)
+		caddy.Log().Named(adaptCmdAdapterFlag).Warn(msg,
+			zap.String("file", warn.File),
+			zap.Int("line", warn.Line))
 	}
 
 	// validate output if requested
@@ -511,7 +521,7 @@ func cmdValidateConfig(fl Flags) (int, error) {
 	validateCmdConfigFlag := fl.String("config")
 	validateCmdAdapterFlag := fl.String("adapter")
 
-	input, _, err := loadConfig(validateCmdConfigFlag, validateCmdAdapterFlag)
+	input, _, err := LoadConfig(validateCmdConfigFlag, validateCmdAdapterFlag)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, err
 	}
@@ -541,7 +551,7 @@ func cmdFmt(fl Flags) (int, error) {
 
 	// as a special case, read from stdin if the file name is "-"
 	if formatCmdConfigFile == "-" {
-		input, err := ioutil.ReadAll(os.Stdin)
+		input, err := io.ReadAll(os.Stdin)
 		if err != nil {
 			return caddy.ExitCodeFailedStartup,
 				fmt.Errorf("reading stdin: %v", err)
@@ -550,7 +560,7 @@ func cmdFmt(fl Flags) (int, error) {
 		return caddy.ExitCodeSuccess, nil
 	}
 
-	input, err := ioutil.ReadFile(formatCmdConfigFile)
+	input, err := os.ReadFile(formatCmdConfigFile)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup,
 			fmt.Errorf("reading input file: %v", err)
@@ -559,8 +569,22 @@ func cmdFmt(fl Flags) (int, error) {
 	output := caddyfile.Format(input)
 
 	if fl.Bool("overwrite") {
-		if err := ioutil.WriteFile(formatCmdConfigFile, output, 0600); err != nil {
-			return caddy.ExitCodeFailedStartup, nil
+		if err := os.WriteFile(formatCmdConfigFile, output, 0600); err != nil {
+			return caddy.ExitCodeFailedStartup, fmt.Errorf("overwriting formatted file: %v", err)
+		}
+	} else if fl.Bool("diff") {
+		diff := difflib.Diff(
+			strings.Split(string(input), "\n"),
+			strings.Split(string(output), "\n"))
+		for _, d := range diff {
+			switch d.Delta {
+			case difflib.Common:
+				fmt.Printf("  %s\n", d.Payload)
+			case difflib.LeftOnly:
+				fmt.Printf("- %s\n", d.Payload)
+			case difflib.RightOnly:
+				fmt.Printf("+ %s\n", d.Payload)
+			}
 		}
 	} else {
 		fmt.Print(string(output))
@@ -633,27 +657,25 @@ commands:
 	return caddy.ExitCodeSuccess, nil
 }
 
-// apiRequest makes an API request to the endpoint adminAddr with the
-// given HTTP method and request URI. If body is non-nil, it will be
-// assumed to be Content-Type application/json.
-func apiRequest(adminAddr, method, uri string, headers http.Header, body io.Reader) error {
-	// parse the admin address
-	if adminAddr == "" {
-		adminAddr = caddy.DefaultAdminListen
-	}
+// AdminAPIRequest makes an API request according to the CLI flags given,
+// with the given HTTP method and request URI. If body is non-nil, it will
+// be assumed to be Content-Type application/json. The caller should close
+// the response body. Should only be used by Caddy CLI commands which
+// need to interact with a running instance of Caddy via the admin API.
+func AdminAPIRequest(adminAddr, method, uri string, headers http.Header, body io.Reader) (*http.Response, error) {
 	parsedAddr, err := caddy.ParseNetworkAddress(adminAddr)
 	if err != nil || parsedAddr.PortRangeSize() > 1 {
-		return fmt.Errorf("invalid admin address %s: %v", adminAddr, err)
+		return nil, fmt.Errorf("invalid admin address %s: %v", adminAddr, err)
 	}
-	origin := parsedAddr.JoinHostPort(0)
+	origin := "http://" + parsedAddr.JoinHostPort(0)
 	if parsedAddr.IsUnixNetwork() {
-		origin = "unixsocket" // hack so that http.NewRequest() is happy
+		origin = "http://unixsocket" // hack so that http.NewRequest() is happy
 	}
 
 	// form the request
-	req, err := http.NewRequest(method, "http://"+origin+uri, body)
+	req, err := http.NewRequest(method, origin+uri, body)
 	if err != nil {
-		return fmt.Errorf("making request: %v", err)
+		return nil, fmt.Errorf("making request: %v", err)
 	}
 	if parsedAddr.IsUnixNetwork() {
 		// When listening on a unix socket, the admin endpoint doesn't
@@ -693,20 +715,60 @@ func apiRequest(adminAddr, method, uri string, headers http.Header, body io.Read
 
 	resp, err := client.Do(req)
 	if err != nil {
-		return fmt.Errorf("performing request: %v", err)
+		return nil, fmt.Errorf("performing request: %v", err)
 	}
-	defer resp.Body.Close()
 
 	// if it didn't work, let the user know
 	if resp.StatusCode >= 400 {
-		respBody, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1024*10))
+		respBody, err := io.ReadAll(io.LimitReader(resp.Body, 1024*10))
 		if err != nil {
-			return fmt.Errorf("HTTP %d: reading error message: %v", resp.StatusCode, err)
+			return nil, fmt.Errorf("HTTP %d: reading error message: %v", resp.StatusCode, err)
 		}
-		return fmt.Errorf("caddy responded with error: HTTP %d: %s", resp.StatusCode, respBody)
+		return nil, fmt.Errorf("caddy responded with error: HTTP %d: %s", resp.StatusCode, respBody)
 	}
 
-	return nil
+	return resp, nil
+}
+
+// DetermineAdminAPIAddress determines which admin API endpoint address should
+// be used based on the inputs. By priority: if `address` is specified, then
+// it is returned; if `configFile` (and `configAdapter`) are specified, then that
+// config will be loaded to find the admin address; otherwise, the default
+// admin listen address will be returned.
+func DetermineAdminAPIAddress(address, configFile, configAdapter string) (string, error) {
+	// Prefer the address if specified and non-empty
+	if address != "" {
+		return address, nil
+	}
+
+	// Try to load the config from file if specified, with the given adapter name
+	if configFile != "" {
+		// get the config in caddy's native format
+		config, loadedConfigFile, err := LoadConfig(configFile, configAdapter)
+		if err != nil {
+			return "", err
+		}
+		if loadedConfigFile == "" {
+			return "", fmt.Errorf("no config file to load")
+		}
+
+		// get the address of the admin listener if set
+		if len(config) > 0 {
+			var tmpStruct struct {
+				Admin caddy.AdminConfig `json:"admin"`
+			}
+			err = json.Unmarshal(config, &tmpStruct)
+			if err != nil {
+				return "", fmt.Errorf("unmarshaling admin listener address from config: %v", err)
+			}
+			if tmpStruct.Admin.Listen != "" {
+				return tmpStruct.Admin.Listen, nil
+			}
+		}
+	}
+
+	// Fallback to the default listen address otherwise
+	return caddy.DefaultAdminListen, nil
 }
 
 type moduleInfo struct {

cmd/commands.go

@@ -156,16 +156,19 @@ development environment.`,
 	RegisterCommand(Command{
 		Name:  "stop",
 		Func:  cmdStop,
+		Usage: "[--address <interface>] [--config <path> [--adapter <name>]]",
 		Short: "Gracefully stops a started Caddy process",
 		Long: `
 Stops the background Caddy process as gracefully as possible.
 
 It requires that the admin API is enabled and accessible, since it will
-use the API's /stop endpoint. The address of this request can be
-customized using the --address flag if it is not the default.`,
+use the API's /stop endpoint. The address of this request can be customized
+using the --address flag, or from the given --config, if not the default.`,
 		Flags: func() *flag.FlagSet {
 			fs := flag.NewFlagSet("stop", flag.ExitOnError)
 			fs.String("address", "", "The address to use to reach the admin API endpoint, if not the default")
+			fs.String("config", "", "Configuration file to use to parse the admin address, if --address is not used")
+			fs.String("adapter", "", "Name of config adapter to apply (when --config is used)")
 			return fs
 		}(),
 	})
@@ -208,6 +211,7 @@ config file; otherwise the default is assumed.`,
 			fs := flag.NewFlagSet("list-modules", flag.ExitOnError)
 			fs.Bool("packages", false, "Print package paths")
 			fs.Bool("versions", false, "Print version information")
+			fs.Bool("skip-standard", false, "Skip printing standard modules")
 			return fs
 		}(),
 	})
@@ -259,7 +263,7 @@ Loads and provisions the provided config, but does not start running it.
 This reveals any errors with the configuration through the loading and
 provisioning stages.`,
 		Flags: func() *flag.FlagSet {
-			fs := flag.NewFlagSet("load", flag.ExitOnError)
+			fs := flag.NewFlagSet("validate", flag.ExitOnError)
 			fs.String("config", "", "Input configuration file")
 			fs.String("adapter", "", "Name of config adapter")
 			return fs
@@ -278,12 +282,18 @@ human readability. It prints the result to stdout.
 If --overwrite is specified, the output will be written to the config file
 directly instead of printing it.
 
+If --diff is specified, the output will be compared against the input, and
+lines will be prefixed with '-' and '+' where they differ. Note that
+unchanged lines are prefixed with two spaces for alignment, and that this
+is not a valid patch format.
+
 If you wish you use stdin instead of a regular file, use - as the path.
 When reading from stdin, the --overwrite flag has no effect: the result
 is always printed to stdout.`,
 		Flags: func() *flag.FlagSet {
-			fs := flag.NewFlagSet("format", flag.ExitOnError)
+			fs := flag.NewFlagSet("fmt", flag.ExitOnError)
 			fs.Bool("overwrite", false, "Overwrite the input file with the results")
+			fs.Bool("diff", false, "Print the differences between the input file and the formatted output")
 			return fs
 		}(),
 	})
@@ -295,6 +305,11 @@ is always printed to stdout.`,
 		Long: `
 Downloads an updated Caddy binary with the same modules/plugins at the
 latest versions. EXPERIMENTAL: May be changed or removed.`,
+		Flags: func() *flag.FlagSet {
+			fs := flag.NewFlagSet("upgrade", flag.ExitOnError)
+			fs.Bool("keep-backup", false, "Keep the backed up binary, instead of deleting it")
+			return fs
+		}(),
 	})
 
 	RegisterCommand(Command{
@@ -307,6 +322,11 @@ Downloads an updated Caddy binary with the specified packages (module/plugin)
 added. Retains existing packages. Returns an error if the any of packages are 
 already included. EXPERIMENTAL: May be changed or removed.
 `,
+		Flags: func() *flag.FlagSet {
+			fs := flag.NewFlagSet("add-package", flag.ExitOnError)
+			fs.Bool("keep-backup", false, "Keep the backed up binary, instead of deleting it")
+			return fs
+		}(),
 	})
 
 	RegisterCommand(Command{
@@ -319,6 +339,11 @@ Downloads an updated Caddy binaries without the specified packages (module/plugi
 Returns an error if any of the packages are not included. 
 EXPERIMENTAL: May be changed or removed.
 `,
+		Flags: func() *flag.FlagSet {
+			fs := flag.NewFlagSet("remove-package", flag.ExitOnError)
+			fs.Bool("keep-backup", false, "Keep the backed up binary, instead of deleting it")
+			return fs
+		}(),
 	})
 
 }

cmd/main.go

@@ -20,7 +20,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net"
 	"os"
@@ -94,7 +93,7 @@ func Main() {
 // the bytes in expect, or returns an error if it doesn't.
 func handlePingbackConn(conn net.Conn, expect []byte) error {
 	defer conn.Close()
-	confirmationBytes, err := ioutil.ReadAll(io.LimitReader(conn, 32))
+	confirmationBytes, err := io.ReadAll(io.LimitReader(conn, 32))
 	if err != nil {
 		return err
 	}
@@ -104,15 +103,15 @@ func handlePingbackConn(conn net.Conn, expect []byte) error {
 	return nil
 }
 
-// loadConfig loads the config from configFile and adapts it
+// LoadConfig loads the config from configFile and adapts it
 // using adapterName. If adapterName is specified, configFile
 // must be also. If no configFile is specified, it tries
 // loading a default config file. The lack of a config file is
 // not treated as an error, but false will be returned if
 // there is no config available. It prints any warnings to stderr,
 // and returns the resulting JSON config bytes along with
-// whether a config file was loaded or not.
-func loadConfig(configFile, adapterName string) ([]byte, string, error) {
+// the name of the loaded config file (if any).
+func LoadConfig(configFile, adapterName string) ([]byte, string, error) {
 	// specifying an adapter without a config file is ambiguous
 	if adapterName != "" && configFile == "" {
 		return nil, "", fmt.Errorf("cannot adapt config without config file (use --config)")
@@ -124,9 +123,9 @@ func loadConfig(configFile, adapterName string) ([]byte, string, error) {
 	var err error
 	if configFile != "" {
 		if configFile == "-" {
-			config, err = ioutil.ReadAll(os.Stdin)
+			config, err = io.ReadAll(os.Stdin)
 		} else {
-			config, err = ioutil.ReadFile(configFile)
+			config, err = os.ReadFile(configFile)
 		}
 		if err != nil {
 			return nil, "", fmt.Errorf("reading config file: %v", err)
@@ -140,7 +139,7 @@ func loadConfig(configFile, adapterName string) ([]byte, string, error) {
 		// plugged in, and if so, try using a default Caddyfile
 		cfgAdapter = caddyconfig.GetAdapter("caddyfile")
 		if cfgAdapter != nil {
-			config, err = ioutil.ReadFile("Caddyfile")
+			config, err = os.ReadFile("Caddyfile")
 			if os.IsNotExist(err) {
 				// okay, no default Caddyfile; pretend like this never happened
 				cfgAdapter = nil
@@ -263,7 +262,7 @@ func watchConfigFile(filename, adapterName string) {
 		lastModified = info.ModTime()
 
 		// load the contents of the file
-		config, _, err := loadConfig(filename, adapterName)
+		config, _, err := LoadConfig(filename, adapterName)
 		if err != nil {
 			logger().Error("unable to load latest config", zap.Error(err))
 			continue
@@ -369,42 +368,68 @@ func loadEnvFromFile(envFile string) error {
 	return nil
 }
 
+// parseEnvFile parses an env file from KEY=VALUE format.
+// It's pretty naive. Limited value quotation is supported,
+// but variable and command expansions are not supported.
 func parseEnvFile(envInput io.Reader) (map[string]string, error) {
 	envMap := make(map[string]string)
 
 	scanner := bufio.NewScanner(envInput)
-	var line string
-	lineNumber := 0
+	var lineNumber int
 
 	for scanner.Scan() {
-		line = strings.TrimSpace(scanner.Text())
+		line := strings.TrimSpace(scanner.Text())
 		lineNumber++
 
-		// skip lines starting with comment
-		if strings.HasPrefix(line, "#") {
-			continue
-		}
-
-		// skip empty line
-		if len(line) == 0 {
+		// skip empty lines and lines starting with comment
+		if line == "" || strings.HasPrefix(line, "#") {
 			continue
 		}
 
+		// split line into key and value
 		fields := strings.SplitN(line, "=", 2)
 		if len(fields) != 2 {
 			return nil, fmt.Errorf("can't parse line %d; line should be in KEY=VALUE format", lineNumber)
 		}
+		key, val := fields[0], fields[1]
 
-		if strings.Contains(fields[0], " ") {
-			return nil, fmt.Errorf("bad key on line %d: contains whitespace", lineNumber)
-		}
-
-		key := fields[0]
-		val := fields[1]
+		// sometimes keys are prefixed by "export " so file can be sourced in bash; ignore it here
+		key = strings.TrimPrefix(key, "export ")
 
+		// validate key and value
 		if key == "" {
 			return nil, fmt.Errorf("missing or empty key on line %d", lineNumber)
 		}
+		if strings.Contains(key, " ") {
+			return nil, fmt.Errorf("invalid key on line %d: contains whitespace: %s", lineNumber, key)
+		}
+		if strings.HasPrefix(val, " ") || strings.HasPrefix(val, "\t") {
+			return nil, fmt.Errorf("invalid value on line %d: whitespace before value: '%s'", lineNumber, val)
+		}
+
+		// remove any trailing comment after value
+		if commentStart := strings.Index(val, "#"); commentStart > 0 {
+			before := val[commentStart-1]
+			if before == '\t' || before == ' ' {
+				val = strings.TrimRight(val[:commentStart], " \t")
+			}
+		}
+
+		// quoted value: support newlines
+		if strings.HasPrefix(val, `"`) {
+			for !(strings.HasSuffix(line, `"`) && !strings.HasSuffix(line, `\"`)) {
+				val = strings.ReplaceAll(val, `\"`, `"`)
+				if !scanner.Scan() {
+					break
+				}
+				lineNumber++
+				line = strings.ReplaceAll(scanner.Text(), `\"`, `"`)
+				val += "\n" + line
+			}
+			val = strings.TrimPrefix(val, `"`)
+			val = strings.TrimSuffix(val, `"`)
+		}
+
 		envMap[key] = val
 	}
 

cmd/main_test.go

@@ -0,0 +1,170 @@
+package caddycmd
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestParseEnvFile(t *testing.T) {
+	for i, tc := range []struct {
+		input     string
+		expect    map[string]string
+		shouldErr bool
+	}{
+		{
+			input: `KEY=value`,
+			expect: map[string]string{
+				"KEY": "value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				INVALID KEY=asdf
+				OTHER_KEY=Some Value
+			`,
+			shouldErr: true,
+		},
+		{
+			input: `
+				KEY=value
+				SIMPLE_QUOTED="quoted value"
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":           "value",
+				"SIMPLE_QUOTED": "quoted value",
+				"OTHER_KEY":     "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				NEWLINES="foo
+	bar"
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"NEWLINES":  "foo\n\tbar",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				ESCAPED="\"escaped quotes\"
+here"
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"ESCAPED":   "\"escaped quotes\"\nhere",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				export KEY=value
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				=value
+				OTHER_KEY=Some Value
+			`,
+			shouldErr: true,
+		},
+		{
+			input: `
+				EMPTY=
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"EMPTY":     "",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				EMPTY=""
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"EMPTY":     "",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				#OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY": "value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				COMMENT=foo bar  # some comment here
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"COMMENT":   "foo bar",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				WHITESPACE=   foo 
+				OTHER_KEY=Some Value
+			`,
+			shouldErr: true,
+		},
+		{
+			input: `
+				KEY=value
+				WHITESPACE="   foo bar "
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":        "value",
+				"WHITESPACE": "   foo bar ",
+				"OTHER_KEY":  "Some Value",
+			},
+		},
+	} {
+		actual, err := parseEnvFile(strings.NewReader(tc.input))
+		if err != nil && !tc.shouldErr {
+			t.Errorf("Test %d: Got error but shouldn't have: %v", i, err)
+		}
+		if err == nil && tc.shouldErr {
+			t.Errorf("Test %d: Did not get error but should have", i)
+		}
+		if tc.shouldErr {
+			continue
+		}
+		if !reflect.DeepEqual(tc.expect, actual) {
+			t.Errorf("Test %d: Expected %v but got %v", i, tc.expect, actual)
+		}
+	}
+}

cmd/packagesfuncs.go

@@ -31,7 +31,7 @@ import (
 	"go.uber.org/zap"
 )
 
-func cmdUpgrade(_ Flags) (int, error) {
+func cmdUpgrade(fl Flags) (int, error) {
 	_, nonstandard, _, err := getModules()
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err)
@@ -41,7 +41,7 @@ func cmdUpgrade(_ Flags) (int, error) {
 		return caddy.ExitCodeFailedStartup, err
 	}
 
-	return upgradeBuild(pluginPkgs)
+	return upgradeBuild(pluginPkgs, fl)
 }
 
 func cmdAddPackage(fl Flags) (int, error) {
@@ -64,7 +64,7 @@ func cmdAddPackage(fl Flags) (int, error) {
 		pluginPkgs[arg] = struct{}{}
 	}
 
-	return upgradeBuild(pluginPkgs)
+	return upgradeBuild(pluginPkgs, fl)
 }
 
 func cmdRemovePackage(fl Flags) (int, error) {
@@ -88,10 +88,10 @@ func cmdRemovePackage(fl Flags) (int, error) {
 		delete(pluginPkgs, arg)
 	}
 
-	return upgradeBuild(pluginPkgs)
+	return upgradeBuild(pluginPkgs, fl)
 }
 
-func upgradeBuild(pluginPkgs map[string]struct{}) (int, error) {
+func upgradeBuild(pluginPkgs map[string]struct{}, fl Flags) (int, error) {
 	l := caddy.Log()
 
 	thisExecPath, err := os.Executable()
@@ -152,18 +152,23 @@ func upgradeBuild(pluginPkgs map[string]struct{}) (int, error) {
 	// use the new binary to print out version and module info
 	fmt.Print("\nModule versions:\n\n")
 	if err = listModules(thisExecPath); err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err)
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute 'caddy list-modules': %v", err)
 	}
 	fmt.Println("\nVersion:")
 	if err = showVersion(thisExecPath); err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err)
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute 'caddy version': %v", err)
 	}
 	fmt.Println()
 
 	// clean up the backup file
-	if err = os.Remove(backupExecPath); err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to clean up backup binary: %v", err)
+	if !fl.Bool("keep-backup") {
+		if err = removeCaddyBinary(backupExecPath); err != nil {
+			return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to clean up backup binary: %v", err)
+		}
+	} else {
+		l.Info("skipped cleaning up the backup file", zap.String("backup_path", backupExecPath))
 	}
+
 	l.Info("upgrade successful; please restart any running Caddy instances", zap.String("executable", thisExecPath))
 
 	return caddy.ExitCodeSuccess, nil
@@ -220,25 +225,17 @@ func getModules() (standard, nonstandard, unknown []moduleInfo, err error) {
 }
 
 func listModules(path string) error {
-	cmd := exec.Command(path, "list-modules", "--versions")
+	cmd := exec.Command(path, "list-modules", "--versions", "--skip-standard")
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	err := cmd.Run()
-	if err != nil {
-		return fmt.Errorf("download succeeded, but unable to execute: %v", err)
-	}
-	return nil
+	return cmd.Run()
 }
 
 func showVersion(path string) error {
 	cmd := exec.Command(path, "version")
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	err := cmd.Run()
-	if err != nil {
-		return fmt.Errorf("download succeeded, but unable to execute: %v", err)
-	}
-	return nil
+	return cmd.Run()
 }
 
 func downloadBuild(qs url.Values) (*http.Response, error) {

cmd/removebinary.go

@@ -0,0 +1,30 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !windows
+// +build !windows
+
+package caddycmd
+
+import (
+	"os"
+)
+
+// removeCaddyBinary removes the Caddy binary at the given path.
+//
+// On any non-Windows OS, this simply calls os.Remove, since they should
+// probably not exhibit any issue with processes deleting themselves.
+func removeCaddyBinary(path string) error {
+	return os.Remove(path)
+}

cmd/removebinary_windows.go

@@ -0,0 +1,36 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddycmd
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+// removeCaddyBinary removes the Caddy binary at the given path.
+//
+// On Windows, this uses a syscall to indirectly remove the file,
+// because otherwise we get an "Access is denied." error when trying
+// to delete the binary while Caddy is still running and performing
+// the upgrade. "cmd.exe /C" executes a command specified by the
+// following arguments, i.e. "del" which will run as a separate process,
+// which avoids the "Access is denied." error.
+func removeCaddyBinary(path string) error {
+	var sI syscall.StartupInfo
+	var pI syscall.ProcessInformation
+	argv := syscall.StringToUTF16Ptr(filepath.Join(os.Getenv("windir"), "system32", "cmd.exe") + " /C del " + path)
+	return syscall.CreateProcess(nil, argv, nil, nil, true, 0, nil, nil, &sI, &pI)
+}

context.go

@@ -423,6 +423,17 @@ func (ctx Context) App(name string) (interface{}, error) {
 	return modVal, nil
 }
 
+// AppIsConfigured returns whether an app named name has been
+// configured. Can be called before calling App() to avoid
+// instantiating an empty app when that's not desirable.
+func (ctx Context) AppIsConfigured(name string) bool {
+	if _, ok := ctx.cfg.apps[name]; ok {
+		return true
+	}
+	appRaw := ctx.cfg.AppsRaw[name]
+	return appRaw != nil
+}
+
 // Storage returns the configured Caddy storage implementation.
 func (ctx Context) Storage() certmagic.Storage {
 	return ctx.cfg.storage

duration_fuzz.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build gofuzz
 // +build gofuzz
 
 package caddy

go.mod

@@ -1,35 +1,132 @@
 module github.com/caddyserver/caddy/v2
 
-go 1.16
+go 1.17
 
 require (
+	github.com/BurntSushi/toml v1.1.0
 	github.com/Masterminds/sprig/v3 v3.2.2
-	github.com/alecthomas/chroma v0.9.2
+	github.com/alecthomas/chroma v0.10.0
 	github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b
-	github.com/caddyserver/certmagic v0.14.5
+	github.com/caddyserver/certmagic v0.16.1
 	github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/google/cel-go v0.7.3
 	github.com/google/uuid v1.3.0
-	github.com/klauspost/compress v1.13.4
-	github.com/klauspost/cpuid/v2 v2.0.9
-	github.com/lucas-clemente/quic-go v0.23.0
-	github.com/mholt/acmez v1.0.0
-	github.com/naoina/go-stringutil v0.1.0 // indirect
-	github.com/naoina/toml v0.1.1
-	github.com/prometheus/client_golang v1.11.0
-	github.com/smallstep/certificates v0.16.4
-	github.com/smallstep/cli v0.16.1
-	github.com/smallstep/nosql v0.3.8
-	github.com/smallstep/truststore v0.9.6
-	github.com/yuin/goldmark v1.4.0
-	github.com/yuin/goldmark-highlighting v0.0.0-20210516132338-9216f9c5aa01
-	go.uber.org/zap v1.19.0
-	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e
-	golang.org/x/net v0.0.0-20210614182718-04defd469f4e
-	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
-	google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08
+	github.com/klauspost/compress v1.15.4
+	github.com/klauspost/cpuid/v2 v2.0.12
+	github.com/lucas-clemente/quic-go v0.27.1
+	github.com/mholt/acmez v1.0.2
+	github.com/prometheus/client_golang v1.12.1
+	github.com/smallstep/certificates v0.19.0
+	github.com/smallstep/cli v0.18.0
+	github.com/smallstep/nosql v0.4.0
+	github.com/smallstep/truststore v0.11.0
+	github.com/tailscale/tscert v0.0.0-20220316030059-54bbcb9f74e2
+	github.com/yuin/goldmark v1.4.12
+	github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.29.0
+	go.opentelemetry.io/otel v1.4.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.4.0
+	go.opentelemetry.io/otel/sdk v1.4.0
+	go.uber.org/zap v1.21.0
+	golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+	google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
-	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
+	filippo.io/edwards25519 v1.0.0-rc.1 // indirect
+	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.1.1 // indirect
+	github.com/antlr/antlr4 v0.0.0-20200503195918-621b933c7a7f // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cheekybits/genny v1.0.0 // indirect
+	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/dgraph-io/badger v1.6.2 // indirect
+	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
+	github.com/dgraph-io/ristretto v0.0.4-0.20200906165740-41ebdbffecfd // indirect
+	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
+	github.com/dlclark/regexp2 v1.4.0 // indirect
+	github.com/felixge/httpsnoop v1.0.2 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-kit/kit v0.10.0 // indirect
+	github.com/go-logfmt/logfmt v0.5.0 // indirect
+	github.com/go-logr/logr v1.2.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/huandu/xstrings v1.3.2 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.10.1 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/pgtype v1.9.0 // indirect
+	github.com/jackc/pgx/v4 v4.14.0 // indirect
+	github.com/libdns/libdns v0.2.1 // indirect
+	github.com/manifoldco/promptui v0.9.0 // indirect
+	github.com/marten-seemann/qpack v0.2.1 // indirect
+	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
+	github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.13 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
+	github.com/micromdm/scep/v2 v2.1.0 // indirect
+	github.com/miekg/dns v1.1.46 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/go-ps v1.0.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/onsi/ginkgo v1.16.4 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/rs/xid v1.2.1 // indirect
+	github.com/russross/blackfriday/v2 v2.0.1 // indirect
+	github.com/shopspring/decimal v1.2.0 // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/slackhq/nebula v1.5.2 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
+	github.com/urfave/cli v1.22.5 // indirect
+	go.etcd.io/bbolt v1.3.6 // indirect
+	go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.0 // indirect
+	go.opentelemetry.io/otel/internal/metric v0.27.0 // indirect
+	go.opentelemetry.io/otel/metric v0.27.0 // indirect
+	go.opentelemetry.io/otel/trace v1.4.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.12.0 // indirect
+	go.step.sm/cli-utils v0.7.0 // indirect
+	go.step.sm/crypto v0.16.1 // indirect
+	go.step.sm/linkedca v0.15.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
+	golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect
+	golang.org/x/tools v0.1.7 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/grpc v1.44.0 // indirect
+	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	howett.net/plist v1.0.0 // indirect
 )

internal/metrics/metrics.go

@@ -0,0 +1,39 @@
+package metrics
+
+import (
+	"net/http"
+	"strconv"
+)
+
+func SanitizeCode(s int) string {
+	switch s {
+	case 0, 200:
+		return "200"
+	default:
+		return strconv.Itoa(s)
+	}
+}
+
+// Only support the list of "regular" HTTP methods, see
+// https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods
+var methodMap = map[string]string{
+	"GET": http.MethodGet, "get": http.MethodGet,
+	"HEAD": http.MethodHead, "head": http.MethodHead,
+	"PUT": http.MethodPut, "put": http.MethodPut,
+	"POST": http.MethodPost, "post": http.MethodPost,
+	"DELETE": http.MethodDelete, "delete": http.MethodDelete,
+	"CONNECT": http.MethodConnect, "connect": http.MethodConnect,
+	"OPTIONS": http.MethodOptions, "options": http.MethodOptions,
+	"TRACE": http.MethodTrace, "trace": http.MethodTrace,
+	"PATCH": http.MethodPatch, "patch": http.MethodPatch,
+}
+
+// SanitizeMethod sanitizes the method for use as a metric label. This helps
+// prevent high cardinality on the method label. The name is always upper case.
+func SanitizeMethod(m string) string {
+	if m, ok := methodMap[m]; ok {
+		return m
+	}
+
+	return "OTHER"
+}

internal/metrics/metrics_test.go

@@ -0,0 +1,28 @@
+package metrics
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestSanitizeMethod(t *testing.T) {
+	tests := []struct {
+		method   string
+		expected string
+	}{
+		{method: "get", expected: "GET"},
+		{method: "POST", expected: "POST"},
+		{method: "OPTIONS", expected: "OPTIONS"},
+		{method: "connect", expected: "CONNECT"},
+		{method: "trace", expected: "TRACE"},
+		{method: "UNKNOWN", expected: "OTHER"},
+		{method: strings.Repeat("ohno", 9999), expected: "OTHER"},
+	}
+
+	for _, d := range tests {
+		actual := SanitizeMethod(d.method)
+		if actual != d.expected {
+			t.Errorf("Not same: expected %#v, but got %#v", d.expected, actual)
+		}
+	}
+}

listeners.go

@@ -15,219 +15,225 @@
 package caddy
 
 import (
+	"context"
+	"crypto/tls"
+	"errors"
 	"fmt"
-	"log"
 	"net"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"syscall"
 	"time"
+
+	"github.com/lucas-clemente/quic-go"
+	"github.com/lucas-clemente/quic-go/http3"
 )
 
-// Listen returns a listener suitable for use in a Caddy module.
-// Always be sure to close listeners when you are done with them.
+// Listen is like net.Listen, except Caddy's listeners can overlap
+// each other: multiple listeners may be created on the same socket
+// at the same time. This is useful because during config changes,
+// the new config is started while the old config is still running.
+// When Caddy listeners are closed, the closing logic is virtualized
+// so the underlying socket isn't actually closed until all uses of
+// the socket have been finished. Always be sure to close listeners
+// when you are done with them, just like normal listeners.
 func Listen(network, addr string) (net.Listener, error) {
 	lnKey := network + "/" + addr
 
-	listenersMu.Lock()
-	defer listenersMu.Unlock()
-
-	// if listener already exists, increment usage counter, then return listener
-	if lnGlobal, ok := listeners[lnKey]; ok {
-		atomic.AddInt32(&lnGlobal.usage, 1)
-		return &fakeCloseListener{
-			usage:      &lnGlobal.usage,
-			deadline:   &lnGlobal.deadline,
-			deadlineMu: &lnGlobal.deadlineMu,
-			key:        lnKey,
-			Listener:   lnGlobal.ln,
-		}, nil
-	}
-
-	// or, create new one and save it
-	ln, err := net.Listen(network, addr)
+	sharedLn, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
+		ln, err := net.Listen(network, addr)
+		if err != nil {
+			// https://github.com/caddyserver/caddy/pull/4534
+			if isUnixNetwork(network) && isListenBindAddressAlreadyInUseError(err) {
+				return nil, fmt.Errorf("%w: this can happen if Caddy was forcefully killed", err)
+			}
+			return nil, err
+		}
+		return &sharedListener{Listener: ln, key: lnKey}, nil
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	// make sure to start its usage counter at 1
-	lnGlobal := &globalListener{usage: 1, ln: ln}
-	listeners[lnKey] = lnGlobal
-
-	return &fakeCloseListener{
-		usage:      &lnGlobal.usage,
-		deadline:   &lnGlobal.deadline,
-		deadlineMu: &lnGlobal.deadlineMu,
-		key:        lnKey,
-		Listener:   ln,
-	}, nil
+	return &fakeCloseListener{sharedListener: sharedLn.(*sharedListener)}, nil
 }
 
 // ListenPacket returns a net.PacketConn suitable for use in a Caddy module.
+// It is like Listen except for PacketConns.
 // Always be sure to close the PacketConn when you are done.
 func ListenPacket(network, addr string) (net.PacketConn, error) {
 	lnKey := network + "/" + addr
 
-	listenersMu.Lock()
-	defer listenersMu.Unlock()
-
-	// if listener already exists, increment usage counter, then return listener
-	if lnGlobal, ok := listeners[lnKey]; ok {
-		atomic.AddInt32(&lnGlobal.usage, 1)
-		log.Printf("[DEBUG] %s: Usage counter should not go above 2 or maybe 3, is now: %d", lnKey, atomic.LoadInt32(&lnGlobal.usage)) // TODO: remove
-		return &fakeClosePacketConn{usage: &lnGlobal.usage, key: lnKey, PacketConn: lnGlobal.pc}, nil
-	}
-
-	// or, create new one and save it
-	pc, err := net.ListenPacket(network, addr)
+	sharedPc, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
+		pc, err := net.ListenPacket(network, addr)
+		if err != nil {
+			// https://github.com/caddyserver/caddy/pull/4534
+			if isUnixNetwork(network) && isListenBindAddressAlreadyInUseError(err) {
+				return nil, fmt.Errorf("%w: this can happen if Caddy was forcefully killed", err)
+			}
+			return nil, err
+		}
+		return &sharedPacketConn{PacketConn: pc, key: lnKey}, nil
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	// make sure to start its usage counter at 1
-	lnGlobal := &globalListener{usage: 1, pc: pc}
-	listeners[lnKey] = lnGlobal
-
-	return &fakeClosePacketConn{usage: &lnGlobal.usage, key: lnKey, PacketConn: pc}, nil
+	return &fakeClosePacketConn{sharedPacketConn: sharedPc.(*sharedPacketConn)}, nil
 }
 
-// fakeCloseListener's Close() method is a no-op. This allows
-// stopping servers that are using the listener without giving
-// up the socket; thus, servers become hot-swappable while the
-// listener remains running. Listeners should be re-wrapped in
-// a new fakeCloseListener each time the listener is reused.
-// Other than the 'closed' field (which pertains to this value
-// only), the other fields in this struct should be pointers to
-// the associated globalListener's struct fields (except 'key'
-// which is there for read-only purposes, so it can be a copy).
+// ListenQUIC returns a quic.EarlyListener suitable for use in a Caddy module.
+// Note that the context passed to Accept is currently ignored, so using
+// a context other than context.Background is meaningless.
+func ListenQUIC(addr string, tlsConf *tls.Config) (quic.EarlyListener, error) {
+	lnKey := "quic/" + addr
+
+	sharedEl, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
+		el, err := quic.ListenAddrEarly(addr, http3.ConfigureTLSConfig(tlsConf), &quic.Config{})
+		if err != nil {
+			return nil, err
+		}
+		return &sharedQuicListener{EarlyListener: el, key: lnKey}, nil
+	})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	return &fakeCloseQuicListener{
+		sharedQuicListener: sharedEl.(*sharedQuicListener),
+		context:            ctx, contextCancel: cancel,
+	}, err
+}
+
+// fakeCloseListener is a private wrapper over a listener that
+// is shared. The state of fakeCloseListener is not shared.
+// This allows one user of a socket to "close" the listener
+// while in reality the socket stays open for other users of
+// the listener. In this way, servers become hot-swappable
+// while the listener remains running. Listeners should be
+// re-wrapped in a new fakeCloseListener each time the listener
+// is reused. This type is atomic and values must not be copied.
 type fakeCloseListener struct {
-	closed       int32       // accessed atomically; belongs to this struct only
-	usage        *int32      // accessed atomically; global
-	deadline     *bool       // protected by deadlineMu; global
-	deadlineMu   *sync.Mutex // global
-	key          string      // global, but read-only, so can be copy
-	net.Listener             // global
+	closed          int32 // accessed atomically; belongs to this struct only
+	*sharedListener       // embedded, so we also become a net.Listener
 }
 
-// Accept accepts connections until Close() is called.
 func (fcl *fakeCloseListener) Accept() (net.Conn, error) {
 	// if the listener is already "closed", return error
 	if atomic.LoadInt32(&fcl.closed) == 1 {
-		return nil, fcl.fakeClosedErr()
+		return nil, fakeClosedErr(fcl)
 	}
 
-	// wrap underlying accept
-	conn, err := fcl.Listener.Accept()
+	// call underlying accept
+	conn, err := fcl.sharedListener.Accept()
 	if err == nil {
 		return conn, nil
 	}
 
-	// accept returned with error
-	// TODO: This may be better as a condition variable so the deadline is cleared only once?
-	fcl.deadlineMu.Lock()
-	if *fcl.deadline {
-		switch ln := fcl.Listener.(type) {
-		case *net.TCPListener:
-			_ = ln.SetDeadline(time.Time{})
-		case *net.UnixListener:
-			_ = ln.SetDeadline(time.Time{})
-		}
-		*fcl.deadline = false
-	}
-	fcl.deadlineMu.Unlock()
-
+	// since Accept() returned an error, it may be because our reference to
+	// the listener (this fakeCloseListener) may have been closed, i.e. the
+	// server is shutting down; in that case, we need to clear the deadline
+	// that we set when Close() was called, and return a non-temporary and
+	// non-timeout error value to the caller, masking the "true" error, so
+	// that server loops / goroutines won't retry, linger, and leak
 	if atomic.LoadInt32(&fcl.closed) == 1 {
-		// if we canceled the Accept() by setting a deadline
-		// on the listener, we need to make sure any callers of
-		// Accept() think the listener was actually closed;
-		// if we return the timeout error instead, callers might
-		// simply retry, leaking goroutines for longer
+		// we dereference the sharedListener explicitly even though it's embedded
+		// so that it's clear in the code that side-effects are shared with other
+		// users of this listener, not just our own reference to it; we also don't
+		// do anything with the error because all we could do is log it, but we
+		// expliclty assign it to nothing so we don't forget it's there if needed
+		_ = fcl.sharedListener.clearDeadline()
+
 		if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
-			return nil, fcl.fakeClosedErr()
+			return nil, fakeClosedErr(fcl)
 		}
 	}
 
 	return nil, err
 }
 
-// Close stops accepting new connections without
-// closing the underlying listener, unless no one
-// else is using it.
+// Close stops accepting new connections without closing the
+// underlying listener. The underlying listener is only closed
+// if the caller is the last known user of the socket.
 func (fcl *fakeCloseListener) Close() error {
 	if atomic.CompareAndSwapInt32(&fcl.closed, 0, 1) {
-		// unfortunately, there is no way to cancel any
-		// currently-blocking calls to Accept() that are
-		// awaiting connections since we're not actually
-		// closing the listener; so we cheat by setting
-		// a deadline in the past, which forces it to
-		// time out; note that this only works for
-		// certain types of listeners...
-		fcl.deadlineMu.Lock()
-		if !*fcl.deadline {
-			switch ln := fcl.Listener.(type) {
-			case *net.TCPListener:
-				_ = ln.SetDeadline(time.Now().Add(-1 * time.Minute))
-			case *net.UnixListener:
-				_ = ln.SetDeadline(time.Now().Add(-1 * time.Minute))
-			}
-			*fcl.deadline = true
-		}
-		fcl.deadlineMu.Unlock()
-
-		// since we're no longer using this listener,
-		// decrement the usage counter and, if no one
-		// else is using it, close underlying listener
-		if atomic.AddInt32(fcl.usage, -1) == 0 {
-			listenersMu.Lock()
-			delete(listeners, fcl.key)
-			listenersMu.Unlock()
-			err := fcl.Listener.Close()
-			if err != nil {
-				return err
-			}
-		}
-
+		// There are two ways I know of to get an Accept()
+		// function to return to the server loop that called
+		// it: close the listener, or set a deadline in the
+		// past. Obviously, we can't close the socket yet
+		// since others may be using it (hence this whole
+		// file). But we can set the deadline in the past,
+		// and this is kind of cheating, but it works, and
+		// it apparently even works on Windows.
+		_ = fcl.sharedListener.setDeadline()
+		_, _ = listenerPool.Delete(fcl.sharedListener.key)
 	}
-
 	return nil
 }
 
-func (fcl *fakeCloseListener) fakeClosedErr() error {
+type fakeCloseQuicListener struct {
+	closed              int32 // accessed atomically; belongs to this struct only
+	*sharedQuicListener       // embedded, so we also become a quic.EarlyListener
+	context             context.Context
+	contextCancel       context.CancelFunc
+}
+
+// Currently Accept ignores the passed context, however a situation where
+// someone would need a hotswappable QUIC-only (not http3, since it uses context.Background here)
+// server on which Accept would be called with non-empty contexts
+// (mind that the default net listeners' Accept doesn't take a context argument)
+// sounds way too rare for us to sacrifice efficiency here.
+func (fcql *fakeCloseQuicListener) Accept(_ context.Context) (quic.EarlyConnection, error) {
+	conn, err := fcql.sharedQuicListener.Accept(fcql.context)
+	if err == nil {
+		return conn, nil
+	}
+
+	// if the listener is "closed", return a fake closed error instead
+	if atomic.LoadInt32(&fcql.closed) == 1 && errors.Is(err, context.Canceled) {
+		return nil, fakeClosedErr(fcql)
+	}
+	return nil, err
+}
+
+func (fcql *fakeCloseQuicListener) Close() error {
+	if atomic.CompareAndSwapInt32(&fcql.closed, 0, 1) {
+		fcql.contextCancel()
+		_, _ = listenerPool.Delete(fcql.sharedQuicListener.key)
+	}
+	return nil
+}
+
+// fakeClosedErr returns an error value that is not temporary
+// nor a timeout, suitable for making the caller think the
+// listener is actually closed
+func fakeClosedErr(l interface{ Addr() net.Addr }) error {
 	return &net.OpError{
 		Op:   "accept",
-		Net:  fcl.Listener.Addr().Network(),
-		Addr: fcl.Listener.Addr(),
+		Net:  l.Addr().Network(),
+		Addr: l.Addr(),
 		Err:  errFakeClosed,
 	}
 }
 
+// ErrFakeClosed is the underlying error value returned by
+// fakeCloseListener.Accept() after Close() has been called,
+// indicating that it is pretending to be closed so that the
+// server using it can terminate, while the underlying
+// socket is actually left open.
+var errFakeClosed = fmt.Errorf("listener 'closed' 😉")
+
+// fakeClosePacketConn is like fakeCloseListener, but for PacketConns.
 type fakeClosePacketConn struct {
-	closed int32  // accessed atomically
-	usage  *int32 // accessed atomically
-	key    string
-	net.PacketConn
+	closed            int32 // accessed atomically; belongs to this struct only
+	*sharedPacketConn       // embedded, so we also become a net.PacketConn
 }
 
 func (fcpc *fakeClosePacketConn) Close() error {
-	log.Println("[DEBUG] Fake-closing underlying packet conn") // TODO: remove this
-
 	if atomic.CompareAndSwapInt32(&fcpc.closed, 0, 1) {
-		// since we're no longer using this listener,
-		// decrement the usage counter and, if no one
-		// else is using it, close underlying listener
-		if atomic.AddInt32(fcpc.usage, -1) == 0 {
-			listenersMu.Lock()
-			delete(listeners, fcpc.key)
-			listenersMu.Unlock()
-			err := fcpc.PacketConn.Close()
-			if err != nil {
-				return err
-			}
-		}
+		_, _ = listenerPool.Delete(fcpc.sharedPacketConn.key)
 	}
-
 	return nil
 }
 
@@ -249,28 +255,75 @@ func (fcpc fakeClosePacketConn) SyscallConn() (syscall.RawConn, error) {
 	return nil, fmt.Errorf("SyscallConn() not implemented for %T", fcpc.PacketConn)
 }
 
-// ErrFakeClosed is the underlying error value returned by
-// fakeCloseListener.Accept() after Close() has been called,
-// indicating that it is pretending to be closed so that the
-// server using it can terminate, while the underlying
-// socket is actually left open.
-var errFakeClosed = fmt.Errorf("listener 'closed' 😉")
-
-// globalListener keeps global state for a listener
-// that may be shared by multiple servers. In other
-// words, values in this struct exist only once and
-// all other uses of these values point to the ones
-// in this struct. In particular, the usage count
-// (how many callers are using the listener), the
-// actual listener, and synchronization of the
-// listener's deadline changes are singular, global
-// values that must not be copied.
-type globalListener struct {
-	usage      int32 // accessed atomically
-	deadline   bool
+// sharedListener is a wrapper over an underlying listener. The listener
+// and the other fields on the struct are shared state that is synchronized,
+// so sharedListener structs must never be copied (always use a pointer).
+type sharedListener struct {
+	net.Listener
+	key        string // uniquely identifies this listener
+	deadline   bool   // whether a deadline is currently set
 	deadlineMu sync.Mutex
-	ln         net.Listener
-	pc         net.PacketConn
+}
+
+func (sl *sharedListener) clearDeadline() error {
+	var err error
+	sl.deadlineMu.Lock()
+	if sl.deadline {
+		switch ln := sl.Listener.(type) {
+		case *net.TCPListener:
+			err = ln.SetDeadline(time.Time{})
+		case *net.UnixListener:
+			err = ln.SetDeadline(time.Time{})
+		}
+		sl.deadline = false
+	}
+	sl.deadlineMu.Unlock()
+	return err
+}
+
+func (sl *sharedListener) setDeadline() error {
+	timeInPast := time.Now().Add(-1 * time.Minute)
+	var err error
+	sl.deadlineMu.Lock()
+	if !sl.deadline {
+		switch ln := sl.Listener.(type) {
+		case *net.TCPListener:
+			err = ln.SetDeadline(timeInPast)
+		case *net.UnixListener:
+			err = ln.SetDeadline(timeInPast)
+		}
+		sl.deadline = true
+	}
+	sl.deadlineMu.Unlock()
+	return err
+}
+
+// Destruct is called by the UsagePool when the listener is
+// finally not being used anymore. It closes the socket.
+func (sl *sharedListener) Destruct() error {
+	return sl.Listener.Close()
+}
+
+// sharedQuicListener is like sharedListener, but for quic.EarlyListeners.
+type sharedQuicListener struct {
+	quic.EarlyListener
+	key string
+}
+
+// Destruct closes the underlying QUIC listener.
+func (sql *sharedQuicListener) Destruct() error {
+	return sql.EarlyListener.Close()
+}
+
+// sharedPacketConn is like sharedListener, but for net.PacketConns.
+type sharedPacketConn struct {
+	net.PacketConn
+	key string
+}
+
+// Destruct closes the underlying socket.
+func (spc *sharedPacketConn) Destruct() error {
+	return spc.PacketConn.Close()
 }
 
 // NetworkAddress contains the individual components
@@ -349,6 +402,20 @@ func isUnixNetwork(netw string) bool {
 	return netw == "unix" || netw == "unixgram" || netw == "unixpacket"
 }
 
+func isListenBindAddressAlreadyInUseError(err error) bool {
+	switch networkOperationError := err.(type) {
+	case *net.OpError:
+		switch syscallError := networkOperationError.Err.(type) {
+		case *os.SyscallError:
+			if syscallError.Syscall == "bind" {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
 // ParseNetworkAddress parses addr into its individual
 // components. The input string is expected to be of
 // the form "network/host:port-range" where any part is
@@ -445,10 +512,8 @@ type ListenerWrapper interface {
 	WrapListener(net.Listener) net.Listener
 }
 
-var (
-	listeners   = make(map[string]*globalListener)
-	listenersMu sync.Mutex
-)
+// listenerPool stores and allows reuse of active listeners.
+var listenerPool = NewUsagePool()
 
 const maxPortSpan = 65535
 

listeners_fuzz.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build gofuzz
 // +build gofuzz
 
 package caddy

logging.go

@@ -18,7 +18,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"strings"
@@ -630,9 +629,9 @@ func (StderrWriter) OpenWriter() (io.WriteCloser, error) {
 	return notClosable{os.Stderr}, nil
 }
 
-// OpenWriter returns ioutil.Discard that can't be closed.
+// OpenWriter returns io.Discard that can't be closed.
 func (DiscardWriter) OpenWriter() (io.WriteCloser, error) {
-	return notClosable{ioutil.Discard}, nil
+	return notClosable{io.Discard}, nil
 }
 
 // notClosable is an io.WriteCloser that can't be closed.
@@ -662,9 +661,15 @@ func newDefaultProductionLog() (*defaultCustomLog, error) {
 
 	cl.buildCore()
 
+	logger := zap.New(cl.core)
+
+	// capture logs from other libraries which
+	// may not be using zap logging directly
+	_ = zap.RedirectStdLog(logger)
+
 	return &defaultCustomLog{
 		CustomLog: cl,
-		logger:    zap.New(cl.core),
+		logger:    logger,
 	}, nil
 }
 

metrics.go

@@ -2,9 +2,8 @@ package caddy
 
 import (
 	"net/http"
-	"strconv"
-	"strings"
 
+	"github.com/caddyserver/caddy/v2/internal/metrics"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -46,8 +45,8 @@ func instrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler)
 		d := newDelegator(w)
 		next.ServeHTTP(d, r)
 		counter.With(prometheus.Labels{
-			"code":   sanitizeCode(d.status),
-			"method": strings.ToUpper(r.Method),
+			"code":   metrics.SanitizeCode(d.status),
+			"method": metrics.SanitizeMethod(r.Method),
 		}).Inc()
 	})
 }
@@ -67,12 +66,3 @@ func (d *delegator) WriteHeader(code int) {
 	d.status = code
 	d.ResponseWriter.WriteHeader(code)
 }
-
-func sanitizeCode(s int) string {
-	switch s {
-	case 0, 200:
-		return "200"
-	default:
-		return strconv.Itoa(s)
-	}
-}

modules/caddyhttp/app.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"net"
 	"net/http"
 	"strconv"
 	"time"
@@ -50,6 +49,8 @@ func init() {
 // `{http.request.body}` | The request body (⚠️ inefficient; use only for debugging)
 // `{http.request.cookie.*}` | HTTP request cookie
 // `{http.request.duration}` | Time up to now spent handling the request (after decoding headers from client)
+// `{http.request.duration_ms}` | Same as 'duration', but in milliseconds.
+// `{http.request.uuid}` | The request unique identifier
 // `{http.request.header.*}` | Specific request header field
 // `{http.request.host.labels.*}` | Request host labels (0-based from right); e.g. for foo.example.com: 0=com, 1=example, 2=foo
 // `{http.request.host}` | The host part of the request's Host header
@@ -77,6 +78,7 @@ func init() {
 // `{http.request.tls.client.public_key}` | The public key of the client certificate.
 // `{http.request.tls.client.public_key_sha256}` | The SHA256 checksum of the client's public key.
 // `{http.request.tls.client.certificate_pem}` | The PEM-encoded value of the certificate.
+// `{http.request.tls.client.certificate_der_base64}` | The base64-encoded value of the certificate.
 // `{http.request.tls.client.issuer}` | The issuer DN of the client certificate
 // `{http.request.tls.client.serial}` | The serial number of the client certificate
 // `{http.request.tls.client.subject}` | The subject DN of the client certificate
@@ -109,14 +111,15 @@ type App struct {
 	// be forcefully closed.
 	GracePeriod caddy.Duration `json:"grace_period,omitempty"`
 
+	Strict *StrictOptions `json:"strict,omitempty"`
+
 	// Servers is the list of servers, keyed by arbitrary names chosen
 	// at your discretion for your own convenience; the keys do not
 	// affect functionality.
 	Servers map[string]*Server `json:"servers,omitempty"`
 
-	servers     []*http.Server
-	h3servers   []*http3.Server
-	h3listeners []net.PacketConn
+	servers   []*http.Server
+	h3servers []*http3.Server
 
 	ctx    caddy.Context
 	logger *zap.Logger
@@ -126,6 +129,13 @@ type App struct {
 	allCertDomains []string
 }
 
+type StrictOptions struct {
+	Disabled            bool `json:"disable,omitempty"`
+	LenientQueryStrings bool `json:"lenient_query_strings,omitempty"`
+	LenientPaths        bool `json:"lenient_paths,omitempty"`
+	LenientHeaders      bool `json:"lenient_headers,omitempty"`
+}
+
 // CaddyModule returns the Caddy module information.
 func (App) CaddyModule() caddy.ModuleInfo {
 	return caddy.ModuleInfo{
@@ -161,6 +171,7 @@ func (app *App) Provision(ctx caddy.Context) error {
 		srv.tlsApp = app.tlsApp
 		srv.logger = app.logger.Named("log")
 		srv.errorLogger = app.logger.Named("log.error")
+		srv.strict = app.Strict
 
 		// only enable access logs if configured
 		if srv.Logs != nil {
@@ -351,9 +362,9 @@ func (app *App) Start() error {
 						app.logger.Info("enabling experimental HTTP/3 listener",
 							zap.String("addr", hostport),
 						)
-						h3ln, err := caddy.ListenPacket("udp", hostport)
+						h3ln, err := caddy.ListenQUIC(hostport, tlsCfg)
 						if err != nil {
-							return fmt.Errorf("getting HTTP/3 UDP listener: %v", err)
+							return fmt.Errorf("getting HTTP/3 QUIC listener: %v", err)
 						}
 						h3srv := &http3.Server{
 							Server: &http.Server{
@@ -364,9 +375,8 @@ func (app *App) Start() error {
 							},
 						}
 						//nolint:errcheck
-						go h3srv.Serve(h3ln)
+						go h3srv.ServeListener(h3ln)
 						app.h3servers = append(app.h3servers, h3srv)
-						app.h3listeners = append(app.h3listeners, h3ln)
 						srv.h3server = h3srv
 					}
 					/////////
@@ -424,13 +434,6 @@ func (app *App) Stop() error {
 		}
 	}
 
-	// close the http3 servers; it's unclear whether the bug reported in
-	// https://github.com/caddyserver/caddy/pull/2727#issuecomment-526856566
-	// was ever truly fixed, since it seemed racey/nondeterministic; but
-	// recent tests in 2020 were unable to replicate the issue again after
-	// repeated attempts (the bug manifested after a config reload; i.e.
-	// reusing a http3 server or listener was problematic), but it seems
-	// to be working fine now
 	for _, s := range app.h3servers {
 		// TODO: CloseGracefully, once implemented upstream
 		// (see https://github.com/lucas-clemente/quic-go/issues/2103)
@@ -439,17 +442,6 @@ func (app *App) Stop() error {
 			return err
 		}
 	}
-
-	// closing an http3.Server does not close their underlying listeners
-	// since apparently the listener can be used both by servers and
-	// clients at the same time; so we need to manually call Close()
-	// on the underlying h3 listeners (see lucas-clemente/quic-go#2103)
-	for _, pc := range app.h3listeners {
-		err := pc.Close()
-		if err != nil {
-			return err
-		}
-	}
 	return nil
 }
 

modules/caddyhttp/autohttps.go

@@ -31,13 +31,20 @@ import (
 // HTTPS is enabled automatically and by default when
 // qualifying hostnames are available from the config.
 type AutoHTTPSConfig struct {
-	// If true, automatic HTTPS will be entirely disabled.
+	// If true, automatic HTTPS will be entirely disabled,
+	// including certificate management and redirects.
 	Disabled bool `json:"disable,omitempty"`
 
 	// If true, only automatic HTTP->HTTPS redirects will
-	// be disabled.
+	// be disabled, but other auto-HTTPS features will
+	// remain enabled.
 	DisableRedir bool `json:"disable_redirects,omitempty"`
 
+	// If true, automatic certificate management will be
+	// disabled, but other auto-HTTPS features will
+	// remain enabled.
+	DisableCerts bool `json:"disable_certificates,omitempty"`
+
 	// Hosts/domain names listed here will not be included
 	// in automatic HTTPS (they will not have certificates
 	// loaded nor redirects applied).
@@ -104,12 +111,13 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 			srv.AutoHTTPS = new(AutoHTTPSConfig)
 		}
 		if srv.AutoHTTPS.Disabled {
+			app.logger.Warn("automatic HTTPS is completely disabled for server", zap.String("server_name", srvName))
 			continue
 		}
 
 		// skip if all listeners use the HTTP port
 		if !srv.listenersUseAnyPortOtherThan(app.httpPort()) {
-			app.logger.Info("server is listening only on the HTTP port, so no automatic HTTPS will be applied to this server",
+			app.logger.Warn("server is listening only on the HTTP port, so no automatic HTTPS will be applied to this server",
 				zap.String("server_name", srvName),
 				zap.Int("http_port", app.httpPort()),
 			)
@@ -166,30 +174,40 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 
 		// for all the hostnames we found, filter them so we have
 		// a deduplicated list of names for which to obtain certs
-		for d := range serverDomainSet {
-			if certmagic.SubjectQualifiesForCert(d) &&
-				!srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.SkipCerts) {
-				// if a certificate for this name is already loaded,
-				// don't obtain another one for it, unless we are
-				// supposed to ignore loaded certificates
-				if !srv.AutoHTTPS.IgnoreLoadedCerts &&
-					len(app.tlsApp.AllMatchingCertificates(d)) > 0 {
-					app.logger.Info("skipping automatic certificate management because one or more matching certificates are already loaded",
-						zap.String("domain", d),
-						zap.String("server_name", srvName),
-					)
+		// (only if cert management not disabled for this server)
+		if srv.AutoHTTPS.DisableCerts {
+			app.logger.Warn("skipping automated certificate management for server because it is disabled", zap.String("server_name", srvName))
+		} else {
+			for d := range serverDomainSet {
+				// the implicit Tailscale manager module will get its own certs at run-time
+				if isTailscaleDomain(d) {
 					continue
 				}
 
-				// most clients don't accept wildcards like *.tld... we
-				// can handle that, but as a courtesy, warn the user
-				if strings.Contains(d, "*") &&
-					strings.Count(strings.Trim(d, "."), ".") == 1 {
-					app.logger.Warn("most clients do not trust second-level wildcard certificates (*.tld)",
-						zap.String("domain", d))
-				}
+				if certmagic.SubjectQualifiesForCert(d) &&
+					!srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.SkipCerts) {
+					// if a certificate for this name is already loaded,
+					// don't obtain another one for it, unless we are
+					// supposed to ignore loaded certificates
+					if !srv.AutoHTTPS.IgnoreLoadedCerts &&
+						len(app.tlsApp.AllMatchingCertificates(d)) > 0 {
+						app.logger.Info("skipping automatic certificate management because one or more matching certificates are already loaded",
+							zap.String("domain", d),
+							zap.String("server_name", srvName),
+						)
+						continue
+					}
 
-				uniqueDomainsForCerts[d] = struct{}{}
+					// most clients don't accept wildcards like *.tld... we
+					// can handle that, but as a courtesy, warn the user
+					if strings.Contains(d, "*") &&
+						strings.Count(strings.Trim(d, "."), ".") == 1 {
+						app.logger.Warn("most clients do not trust second-level wildcard certificates (*.tld)",
+							zap.String("domain", d))
+					}
+
+					uniqueDomainsForCerts[d] = struct{}{}
+				}
 			}
 		}
 
@@ -200,19 +218,22 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 
 		// nothing left to do if auto redirects are disabled
 		if srv.AutoHTTPS.DisableRedir {
+			app.logger.Warn("automatic HTTP->HTTPS redirects are disabled", zap.String("server_name", srvName))
 			continue
 		}
 
-		app.logger.Info("enabling automatic HTTP->HTTPS redirects",
-			zap.String("server_name", srvName),
-		)
+		app.logger.Info("enabling automatic HTTP->HTTPS redirects", zap.String("server_name", srvName))
 
 		// create HTTP->HTTPS redirects
-		for _, addr := range srv.Listen {
+		for _, listenAddr := range srv.Listen {
 			// figure out the address we will redirect to...
-			addr, err := caddy.ParseNetworkAddress(addr)
+			addr, err := caddy.ParseNetworkAddress(listenAddr)
 			if err != nil {
-				return fmt.Errorf("%s: invalid listener address: %v", srvName, addr)
+				msg := "%s: invalid listener address: %v"
+				if strings.Count(listenAddr, ":") > 1 {
+					msg = msg + ", there are too many colons, so the port is ambiguous. Did you mean to wrap the IPv6 address with [] brackets?"
+				}
+				return fmt.Errorf(msg, srvName, listenAddr)
 			}
 
 			// this address might not have a hostname, i.e. might be a
@@ -232,7 +253,7 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 				// port, we'll have to choose one, so prefer the HTTPS port
 				if _, ok := redirDomains[d]; !ok ||
 					addr.StartPort == uint(app.httpsPort()) {
-					redirDomains[d] = append(redirDomains[d], addr)
+					redirDomains[d] = []caddy.NetworkAddress{addr}
 				}
 			}
 		}
@@ -418,7 +439,7 @@ func (app *App) makeRedirRoute(redirToPort uint, matcherSet MatcherSet) Route {
 	}
 }
 
-// createAutomationPolicy ensures that automated certificates for this
+// createAutomationPolicies ensures that automated certificates for this
 // app are managed properly. This adds up to two automation policies:
 // one for the public names, and one for the internal names. If a catch-all
 // automation policy exists, it will be shallow-copied and used as the
@@ -459,6 +480,22 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []stri
 			}
 		}
 
+		// if no external managers were configured, enable
+		// implicit Tailscale support for convenience
+		if ap.Managers == nil {
+			ts, err := implicitTailscale(ctx)
+			if err != nil {
+				return err
+			}
+			ap.Managers = []certmagic.Manager{ts}
+
+			// must reprovision the automation policy so that the underlying
+			// CertMagic config knows about the updated Managers
+			if err := ap.Provision(app.tlsApp); err != nil {
+				return fmt.Errorf("re-provisioning automation policy: %v", err)
+			}
+		}
+
 		// while we're here, is this the catch-all/base policy?
 		if !foundBasePolicy && len(ap.Subjects) == 0 {
 			basePolicy = ap
@@ -467,10 +504,19 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []stri
 	}
 
 	if basePolicy == nil {
-		// no base policy found, we will make one!
+		// no base policy found; we will make one
 		basePolicy = new(caddytls.AutomationPolicy)
 	}
 
+	if basePolicy.Managers == nil {
+		// add implicit Tailscale integration, for harmless convenience
+		ts, err := implicitTailscale(ctx)
+		if err != nil {
+			return err
+		}
+		basePolicy.Managers = []certmagic.Manager{ts}
+	}
+
 	// if the basePolicy has an existing ACMEIssuer (particularly to
 	// include any type that embeds/wraps an ACMEIssuer), let's use it
 	// (I guess we just use the first one?), otherwise we'll make one
@@ -482,8 +528,8 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []stri
 		}
 	}
 	if baseACMEIssuer == nil {
-		// note that this happens if basePolicy.Issuer is nil
-		// OR if it is not nil but is not an ACMEIssuer
+		// note that this happens if basePolicy.Issuers is empty
+		// OR if it is not empty but does not have not an ACMEIssuer
 		baseACMEIssuer = new(caddytls.ACMEIssuer)
 	}
 
@@ -653,4 +699,15 @@ func (app *App) automaticHTTPSPhase2() error {
 	return nil
 }
 
+// implicitTailscale returns a new and provisioned Tailscale module configured to be optional.
+func implicitTailscale(ctx caddy.Context) (caddytls.Tailscale, error) {
+	ts := caddytls.Tailscale{Optional: true}
+	err := ts.Provision(ctx)
+	return ts, err
+}
+
+func isTailscaleDomain(name string) bool {
+	return strings.HasSuffix(strings.ToLower(name), ".ts.net")
+}
+
 type acmeCapable interface{ GetACMEIssuer() *caddytls.ACMEIssuer }

modules/caddyhttp/celmatcher.go

@@ -33,6 +33,7 @@ import (
 	"github.com/google/cel-go/common/types/traits"
 	"github.com/google/cel-go/ext"
 	"github.com/google/cel-go/interpreter/functions"
+	"go.uber.org/zap"
 	exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
 	"google.golang.org/protobuf/proto"
 )
@@ -62,6 +63,8 @@ type MatchExpression struct {
 	expandedExpr string
 	prg          cel.Program
 	ta           ref.TypeAdapter
+
+	log *zap.Logger
 }
 
 // CaddyModule returns the Caddy module information.
@@ -83,7 +86,9 @@ func (m *MatchExpression) UnmarshalJSON(data []byte) error {
 }
 
 // Provision sets ups m.
-func (m *MatchExpression) Provision(_ caddy.Context) error {
+func (m *MatchExpression) Provision(ctx caddy.Context) error {
+	m.log = ctx.Logger(m)
+
 	// replace placeholders with a function call - this is just some
 	// light (and possibly naïve) syntactic sugar
 	m.expandedExpr = placeholderRegexp.ReplaceAllString(m.Expr, placeholderExpansion)
@@ -137,9 +142,13 @@ func (m *MatchExpression) Provision(_ caddy.Context) error {
 
 // Match returns true if r matches m.
 func (m MatchExpression) Match(r *http.Request) bool {
-	out, _, _ := m.prg.Eval(map[string]interface{}{
+	out, _, err := m.prg.Eval(map[string]interface{}{
 		"request": celHTTPRequest{r},
 	})
+	if err != nil {
+		m.log.Error("evaluating expression", zap.Error(err))
+		return false
+	}
 	if outBool, ok := out.Value().(bool); ok {
 		return outBool
 	}
@@ -150,7 +159,11 @@ func (m MatchExpression) Match(r *http.Request) bool {
 // UnmarshalCaddyfile implements caddyfile.Unmarshaler.
 func (m *MatchExpression) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	for d.Next() {
-		m.Expr = strings.Join(d.RemainingArgs(), " ")
+		if d.CountRemainingArgs() > 1 {
+			m.Expr = strings.Join(d.RemainingArgsRaw(), " ")
+		} else {
+			m.Expr = d.Val()
+		}
 	}
 	return nil
 }

modules/caddyhttp/celmatcher_test.go

@@ -116,7 +116,7 @@ eqp31wM9il1n+guTNyxJd+FzVAH+hCZE5K+tCgVDdVFUlDEHHbS/wqb2PSIoouLV
 			}
 
 			if tt.expression.Match(req) != tt.wantResult {
-				t.Errorf("MatchExpression.Match() expected to return '%t', for expression : '%s'", tt.wantResult, tt.expression)
+				t.Errorf("MatchExpression.Match() expected to return '%t', for expression : '%s'", tt.wantResult, tt.expression.Expr)
 			}
 
 		})

modules/caddyhttp/errors.go

@@ -82,6 +82,9 @@ func (e HandlerError) Error() string {
 	return strings.TrimSpace(s)
 }
 
+// Unwrap returns the underlying error value. See the `errors` package for info.
+func (e HandlerError) Unwrap() error { return e.Err }
+
 // randString returns a string of n random characters.
 // It is not even remotely secure OR a proper distribution.
 // But it's good enough for some things. It excludes certain