core: Variadic Context.Logger(); soft deprecation

Ideally I'd just remove the parameter to caddy.Context.Logger(), but
this would break most Caddy plugins.

Instead, I'm making it variadic and marking it as partially deprecated.
In the future, I might completely remove the parameter once most
plugins have updated.

.gitattributes

@@ -0,0 +1 @@
+*.go text eol=lf

.github/workflows/ci.yml

@@ -19,16 +19,16 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
-        go: [ '1.17', '1.18' ]
+        go: [ '1.18', '1.19' ]
 
         include:
         # Set the minimum Go patch version for the given Go minor
         # Usable via ${{ matrix.GO_SEMVER }}
-        - go: '1.17'
-          GO_SEMVER: '~1.17.9'
-
         - go: '1.18'
-          GO_SEMVER: '~1.18.1'
+          GO_SEMVER: '~1.18.4'
+
+        - go: '1.19'
+          GO_SEMVER: '~1.19.0'
 
         # Set some variables per OS, usable via ${{ matrix.VAR }}
         # CADDY_BIN_PATH: the path to the compiled Caddy binary, for artifact publishing
@@ -78,12 +78,20 @@ jobs:
         printf "Git version: $(git version)\n\n"
         # Calculate the short SHA1 hash of the git commit
         echo "::set-output name=short_sha::$(git rev-parse --short HEAD)"
-        echo "::set-output name=go_cache::$(go env GOCACHE)"
 
     - name: Cache the build cache
       uses: actions/cache@v2
       with:
-        path: ${{ steps.vars.outputs.go_cache }}
+        # In order:
+        # * Module download cache
+        # * Build cache (Linux)
+        # * Build cache (Mac)
+        # * Build cache (Windows)
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
+          ~/Library/Caches/go-build
+          ~\AppData\Local\go-build
         key: ${{ runner.os }}-${{ matrix.go }}-go-ci-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-${{ matrix.go }}-go-ci

.github/workflows/cross-build.yml

@@ -16,13 +16,13 @@ jobs:
       fail-fast: false
       matrix:
         goos: ['android', 'linux', 'solaris', 'illumos', 'dragonfly', 'freebsd', 'openbsd', 'plan9', 'windows', 'darwin', 'netbsd']
-        go: [ '1.18' ]
+        go: [ '1.19' ]
 
         include:
         # Set the minimum Go patch version for the given Go minor
         # Usable via ${{ matrix.GO_SEMVER }}
-        - go: '1.18'
-          GO_SEMVER: '~1.18.1'
+        - go: '1.19'
+          GO_SEMVER: '~1.19.0'
 
     runs-on: ubuntu-latest
     continue-on-error: true
@@ -42,12 +42,16 @@ jobs:
           go env
           printf "\n\nSystem environment:\n\n"
           env
-          echo "::set-output name=go_cache::$(go env GOCACHE)"
 
       - name: Cache the build cache
         uses: actions/cache@v2
         with:
-          path: ${{ steps.vars.outputs.go_cache }}
+          # In order:
+          # * Module download cache
+          # * Build cache (Linux)
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
           key: cross-build-go${{ matrix.go }}-${{ matrix.goos }}-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             cross-build-go${{ matrix.go }}-${{ matrix.goos }}

.github/workflows/lint.yml

@@ -14,17 +14,22 @@ jobs:
   # From https://github.com/golangci/golangci-lint-action
   golangci:
     name: lint
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: '~1.17.9'
+          go-version: '~1.18.4'
           check-latest: true
 
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.44
+          version: v1.47
+          # Windows times out frequently after about 5m50s if we don't set a longer timeout.
+          args: --timeout 10m
           # Optional: show only new issues if it's a pull request. The default value is `false`.
           # only-new-issues: true

.github/workflows/release.yml

@@ -11,15 +11,22 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        go: [ '1.18' ]
+        go: [ '1.19' ]
 
         include:
         # Set the minimum Go patch version for the given Go minor
         # Usable via ${{ matrix.GO_SEMVER }}
-        - go: '1.18'
-          GO_SEMVER: '~1.18.1'
+        - go: '1.19'
+          GO_SEMVER: '~1.19.0'
 
     runs-on: ${{ matrix.os }}
+    # https://github.com/sigstore/cosign/issues/1258#issuecomment-1002251233
+    # https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect#adding-permissions-settings
+    permissions:
+      id-token: write
+      # https://docs.github.com/en/rest/overview/permissions-required-for-github-apps#permission-on-contents
+      # "Releases" is part of `contents`, so it needs the `write`
+      contents: write
 
     steps:
     - name: Install Go
@@ -56,7 +63,6 @@ jobs:
         env
         echo "::set-output name=version_tag::${GITHUB_REF/refs\/tags\//}"
         echo "::set-output name=short_sha::$(git rev-parse --short HEAD)"
-        echo "::set-output name=go_cache::$(go env GOCACHE)"
 
         # Add "pip install" CLI tools to PATH
         echo ~/.local/bin >> $GITHUB_PATH
@@ -91,11 +97,23 @@ jobs:
     - name: Cache the build cache
       uses: actions/cache@v2
       with:
-        path: ${{ steps.vars.outputs.go_cache }}
+        # In order:
+        # * Module download cache
+        # * Build cache (Linux)
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
         key: ${{ runner.os }}-go${{ matrix.go }}-release-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go${{ matrix.go }}-release
-
+    - name: Install Cosign
+      uses: sigstore/cosign-installer@main
+    - name: Cosign version
+      run: cosign version
+    - name: Install Syft
+      uses: anchore/sbom-action/download-syft@main
+    - name: Syft version
+      run: syft version
     # GoReleaser will take care of publishing those artifacts into the release
     - name: Run GoReleaser
       uses: goreleaser/goreleaser-action@v2
@@ -105,6 +123,7 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         TAG: ${{ steps.vars.outputs.version_tag }}
+        COSIGN_EXPERIMENTAL: 1
 
     # Only publish on non-special tags (e.g. non-beta)
     # We will continue to push to Gemfury for the foreseeable future, although

.goreleaser.yml

@@ -14,7 +14,11 @@ before:
     # run `go mod tidy`. The `/bin/sh -c '...'` is because goreleaser can't find cd in PATH without shell invocation.
     - /bin/sh -c 'cd ./caddy-build && go mod tidy'
     - git clone --depth 1 https://github.com/caddyserver/dist caddy-dist
+    - mkdir -p caddy-dist/man
     - go mod download
+    - go run cmd/caddy/main.go manpage --directory ./caddy-dist/man
+    - gzip -r ./caddy-dist/man/
+    - /bin/sh -c 'go run cmd/caddy/main.go completion bash > ./caddy-dist/scripts/bash-completion'
 
 builds:
 - env:
@@ -58,9 +62,22 @@ builds:
       goarm: "5"
   flags:
   - -trimpath
+  - -mod=readonly
   ldflags:
   - -s -w
-
+signs:
+  - cmd: cosign
+    signature: "${artifact}.sig"
+    certificate: '{{ trimsuffix .Env.artifact ".tar.gz" }}.pem'
+    args: ["sign-blob", "--output-signature=${signature}", "--output-certificate", "${certificate}", "${artifact}"]
+    artifacts: all
+sboms:
+  - artifacts: binary
+    # defaults to
+    # documents:
+    # - "{{ .Binary }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}.sbom"
+    cmd: syft
+    args: ["$artifact", "--file", "${document}", "--output", "cyclonedx-json"]
 archives:
   - format_overrides:
       - goos: windows
@@ -96,13 +113,16 @@ nfpms:
       - src: ./caddy-dist/welcome/index.html
         dst: /usr/share/caddy/index.html
       
-      - src: ./caddy-dist/scripts/completions/bash-completion
+      - src: ./caddy-dist/scripts/bash-completion
         dst: /etc/bash_completion.d/caddy
     
       - src: ./caddy-dist/config/Caddyfile
         dst: /etc/caddy/Caddyfile
         type: config
 
+      - src: ./caddy-dist/man/*
+        dst: /usr/share/man/man8/
+
     scripts:
       postinstall: ./caddy-dist/scripts/postinstall.sh
       preremove: ./caddy-dist/scripts/preremove.sh

README.md

@@ -57,25 +57,25 @@
 	- Multi-issuer fallback
 - **Stays up when other servers go down** due to TLS/OCSP/certificate-related issues
 - **Production-ready** after serving trillions of requests and managing millions of TLS certificates
-- **Scales to tens of thousands of sites** ... and probably more
-- **HTTP/1.1, HTTP/2, and experimental HTTP/3** support
+- **Scales to hundreds of thousands of sites** as proven in production
+- **HTTP/1.1, HTTP/2, and HTTP/3** supported all by default
 - **Highly extensible** [modular architecture](https://caddyserver.com/docs/architecture) lets Caddy do anything without bloat
 - **Runs anywhere** with **no external dependencies** (not even libc)
 - Written in Go, a language with higher **memory safety guarantees** than other servers
 - Actually **fun to use**
-- So, so much more to [discover](https://caddyserver.com/v2)
+- So much more to [discover](https://caddyserver.com/v2)
 
 ## Install
 
-The simplest, cross-platform way is to download from [GitHub Releases](https://github.com/caddyserver/caddy/releases) and place the executable file in your PATH.
+The simplest, cross-platform way to get started is to download Caddy from [GitHub Releases](https://github.com/caddyserver/caddy/releases) and place the executable file in your PATH.
 
-For other install options, see https://caddyserver.com/docs/install.
+See [our online documentation](https://caddyserver.com/docs/install) for other install instructions.
 
 ## Build from source
 
 Requirements:
 
-- [Go 1.17 or newer](https://golang.org/dl/)
+- [Go 1.18 or newer](https://golang.org/dl/)
 
 ### For development
  
@@ -164,9 +164,9 @@ The docs are also open source. You can contribute to them here: https://github.c
 
 ## Getting help
 
-- We **strongly recommend** that all professionals or companies using Caddy get a support contract through [Ardan Labs](https://www.ardanlabs.com/my/contact-us?dd=caddy) before help is needed.
+- We advise companies using Caddy to secure a support contract through [Ardan Labs](https://www.ardanlabs.com/my/contact-us?dd=caddy) before help is needed.
 
-- A [sponsorship](https://github.com/sponsors/mholt) goes a long way! If Caddy is benefitting your company, please consider a sponsorship! This not only helps fund full-time work to ensure the longevity of the project, it's also a great look for your company to your customers and potential customers!
+- A [sponsorship](https://github.com/sponsors/mholt) goes a long way! We can offer private help to sponsors. If Caddy is benefitting your company, please consider a sponsorship. This not only helps fund full-time work to ensure the longevity of the project, it provides your company the resources, support, and discounts you need; along with being a great look for your company to your customers and potential customers!
 
 - Individuals can exchange help for free on our community forum at https://caddy.community. Remember that people give help out of their spare time and good will. The best way to get help is to give it first!
 

admin.go

@@ -25,6 +25,8 @@ import (
 	"errors"
 	"expvar"
 	"fmt"
+	"hash"
+	"hash/fnv"
 	"io"
 	"net"
 	"net/http"
@@ -38,7 +40,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/caddyserver/caddy/v2/notify"
 	"github.com/caddyserver/certmagic"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
@@ -338,17 +339,19 @@ func (admin AdminConfig) allowedOrigins(addr NetworkAddress) []*url.URL {
 // that there is always an admin server (unless it is explicitly
 // configured to be disabled).
 func replaceLocalAdminServer(cfg *Config) error {
-	// always be sure to close down the old admin endpoint
+	// always* be sure to close down the old admin endpoint
 	// as gracefully as possible, even if the new one is
 	// disabled -- careful to use reference to the current
 	// (old) admin endpoint since it will be different
 	// when the function returns
+	// (* except if the new one fails to start)
 	oldAdminServer := localAdminServer
+	var err error
 	defer func() {
 		// do the shutdown asynchronously so that any
 		// current API request gets a response; this
 		// goroutine may last a few seconds
-		if oldAdminServer != nil {
+		if oldAdminServer != nil && err == nil {
 			go func(oldAdminServer *http.Server) {
 				err := stopAdminServer(oldAdminServer)
 				if err != nil {
@@ -439,7 +442,7 @@ func manageIdentity(ctx Context, cfg *Config) error {
 		if err != nil {
 			return fmt.Errorf("loading identity issuer modules: %s", err)
 		}
-		for _, issVal := range val.([]interface{}) {
+		for _, issVal := range val.([]any) {
 			cfg.Admin.Identity.issuers = append(cfg.Admin.Identity.issuers, issVal.(certmagic.Issuer))
 		}
 	}
@@ -894,16 +897,36 @@ func (h adminHandler) originAllowed(origin *url.URL) bool {
 	return false
 }
 
+// etagHasher returns a the hasher we used on the config to both
+// produce and verify ETags.
+func etagHasher() hash.Hash32 { return fnv.New32a() }
+
+// makeEtag returns an Etag header value (including quotes) for
+// the given config path and hash of contents at that path.
+func makeEtag(path string, hash hash.Hash) string {
+	return fmt.Sprintf(`"%s %x"`, path, hash.Sum(nil))
+}
+
 func handleConfig(w http.ResponseWriter, r *http.Request) error {
 	switch r.Method {
 	case http.MethodGet:
 		w.Header().Set("Content-Type", "application/json")
+		// Set the ETag as a trailer header.
+		// The alternative is to write the config to a buffer, and
+		// then hash that.
+		w.Header().Set("Trailer", "ETag")
 
-		err := readConfig(r.URL.Path, w)
+		hash := etagHasher()
+		configWriter := io.MultiWriter(w, hash)
+		err := readConfig(r.URL.Path, configWriter)
 		if err != nil {
 			return APIError{HTTPStatus: http.StatusBadRequest, Err: err}
 		}
 
+		// we could consider setting up a sync.Pool for the summed
+		// hashes to reduce GC pressure.
+		w.Header().Set("Etag", makeEtag(r.URL.Path, hash))
+
 		return nil
 
 	case http.MethodPost,
@@ -937,7 +960,7 @@ func handleConfig(w http.ResponseWriter, r *http.Request) error {
 
 		forceReload := r.Header.Get("Cache-Control") == "must-revalidate"
 
-		err := changeConfig(r.Method, r.URL.Path, body, forceReload)
+		err := changeConfig(r.Method, r.URL.Path, body, r.Header.Get("If-Match"), forceReload)
 		if err != nil && !errors.Is(err, errSameConfig) {
 			return err
 		}
@@ -971,9 +994,9 @@ func handleConfigID(w http.ResponseWriter, r *http.Request) error {
 	id := parts[2]
 
 	// map the ID to the expanded path
-	currentCfgMu.RLock()
+	currentCtxMu.RLock()
 	expanded, ok := rawCfgIndex[id]
-	defer currentCfgMu.RUnlock()
+	defer currentCtxMu.RUnlock()
 	if !ok {
 		return APIError{
 			HTTPStatus: http.StatusNotFound,
@@ -996,10 +1019,6 @@ func handleStop(w http.ResponseWriter, r *http.Request) error {
 		}
 	}
 
-	if err := notify.NotifyStopping(); err != nil {
-		Log().Error("unable to notify stopping to service manager", zap.Error(err))
-	}
-
 	exitProcess(context.Background(), Log().Named("admin.api"))
 	return nil
 }
@@ -1008,11 +1027,11 @@ func handleStop(w http.ResponseWriter, r *http.Request) error {
 // the operation at path according to method, using body and out as
 // needed. This is a low-level, unsynchronized function; most callers
 // will want to use changeConfig or readConfig instead. This requires a
-// read or write lock on currentCfgMu, depending on method (GET needs
+// read or write lock on currentCtxMu, depending on method (GET needs
 // only a read lock; all others need a write lock).
 func unsyncedConfigAccess(method, path string, body []byte, out io.Writer) error {
 	var err error
-	var val interface{}
+	var val any
 
 	// if there is a request body, decode it into the
 	// variable that will be set in the config according
@@ -1049,16 +1068,16 @@ func unsyncedConfigAccess(method, path string, body []byte, out io.Writer) error
 		parts = parts[:len(parts)-1]
 	}
 
-	var ptr interface{} = rawCfg
+	var ptr any = rawCfg
 
 traverseLoop:
 	for i, part := range parts {
 		switch v := ptr.(type) {
-		case map[string]interface{}:
+		case map[string]any:
 			// if the next part enters a slice, and the slice is our destination,
 			// handle it specially (because appending to the slice copies the slice
 			// header, which does not replace the original one like we want)
-			if arr, ok := v[part].([]interface{}); ok && i == len(parts)-2 {
+			if arr, ok := v[part].([]any); ok && i == len(parts)-2 {
 				var idx int
 				if method != http.MethodPost {
 					idxStr := parts[len(parts)-1]
@@ -1080,7 +1099,7 @@ traverseLoop:
 					}
 				case http.MethodPost:
 					if ellipses {
-						valArray, ok := val.([]interface{})
+						valArray, ok := val.([]any)
 						if !ok {
 							return fmt.Errorf("final element is not an array")
 						}
@@ -1115,9 +1134,9 @@ traverseLoop:
 				case http.MethodPost:
 					// if the part is an existing list, POST appends to
 					// it, otherwise it just sets or creates the value
-					if arr, ok := v[part].([]interface{}); ok {
+					if arr, ok := v[part].([]any); ok {
 						if ellipses {
-							valArray, ok := val.([]interface{})
+							valArray, ok := val.([]any)
 							if !ok {
 								return fmt.Errorf("final element is not an array")
 							}
@@ -1148,12 +1167,12 @@ traverseLoop:
 				// might not exist yet; that's OK but we need to make them as
 				// we go, while we still have a pointer from the level above
 				if v[part] == nil && method == http.MethodPut {
-					v[part] = make(map[string]interface{})
+					v[part] = make(map[string]any)
 				}
 				ptr = v[part]
 			}
 
-		case []interface{}:
+		case []any:
 			partInt, err := strconv.Atoi(part)
 			if err != nil {
 				return fmt.Errorf("[/%s] invalid array index '%s': %v",
@@ -1175,7 +1194,7 @@ traverseLoop:
 
 // RemoveMetaFields removes meta fields like "@id" from a JSON message
 // by using a simple regular expression. (An alternate way to do this
-// would be to delete them from the raw, map[string]interface{}
+// would be to delete them from the raw, map[string]any
 // representation as they are indexed, then iterate the index we made
 // and add them back after encoding as JSON, but this is simpler.)
 func RemoveMetaFields(rawJSON []byte) []byte {
@@ -1307,7 +1326,7 @@ const (
 )
 
 var bufPool = sync.Pool{
-	New: func() interface{} {
+	New: func() any {
 		return new(bytes.Buffer)
 	},
 }

admin_test.go

@@ -16,6 +16,8 @@ package caddy
 
 import (
 	"encoding/json"
+	"fmt"
+	"net/http"
 	"reflect"
 	"sync"
 	"testing"
@@ -113,7 +115,7 @@ func TestUnsyncedConfigAccess(t *testing.T) {
 		}
 
 		// decode the expected config so we can do a convenient DeepEqual
-		var expectedDecoded interface{}
+		var expectedDecoded any
 		err = json.Unmarshal([]byte(tc.expect), &expectedDecoded)
 		if err != nil {
 			t.Fatalf("Test %d: Unmarshaling expected config: %v", i, err)
@@ -139,10 +141,57 @@ func TestLoadConcurrent(t *testing.T) {
 			wg.Done()
 		}()
 	}
-
 	wg.Wait()
 }
 
+type fooModule struct {
+	IntField int
+	StrField string
+}
+
+func (fooModule) CaddyModule() ModuleInfo {
+	return ModuleInfo{
+		ID:  "foo",
+		New: func() Module { return new(fooModule) },
+	}
+}
+func (fooModule) Start() error { return nil }
+func (fooModule) Stop() error  { return nil }
+
+func TestETags(t *testing.T) {
+	RegisterModule(fooModule{})
+
+	if err := Load([]byte(`{"apps": {"foo": {"strField": "abc", "intField": 0}}}`), true); err != nil {
+		t.Fatalf("loading: %s", err)
+	}
+
+	const key = "/" + rawConfigKey + "/apps/foo"
+
+	// try update the config with the wrong etag
+	err := changeConfig(http.MethodPost, key, []byte(`{"strField": "abc", "intField": 1}}`), fmt.Sprintf(`"/%s not_an_etag"`, rawConfigKey), false)
+	if apiErr, ok := err.(APIError); !ok || apiErr.HTTPStatus != http.StatusPreconditionFailed {
+		t.Fatalf("expected precondition failed; got %v", err)
+	}
+
+	// get the etag
+	hash := etagHasher()
+	if err := readConfig(key, hash); err != nil {
+		t.Fatalf("reading: %s", err)
+	}
+
+	// do the same update with the correct key
+	err = changeConfig(http.MethodPost, key, []byte(`{"strField": "abc", "intField": 1}`), makeEtag(key, hash), false)
+	if err != nil {
+		t.Fatalf("expected update to work; got %v", err)
+	}
+
+	// now try another update. The hash should no longer match and we should get precondition failed
+	err = changeConfig(http.MethodPost, key, []byte(`{"strField": "abc", "intField": 2}`), makeEtag(key, hash), false)
+	if apiErr, ok := err.(APIError); !ok || apiErr.HTTPStatus != http.StatusPreconditionFailed {
+		t.Fatalf("expected precondition failed; got %v", err)
+	}
+}
+
 func BenchmarkLoad(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		Load(testCfg, true)

caddy.go

@@ -17,6 +17,7 @@ package caddy
 import (
 	"bytes"
 	"context"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -101,20 +102,32 @@ func Run(cfg *Config) error {
 // if it is different from the current config or
 // forceReload is true.
 func Load(cfgJSON []byte, forceReload bool) error {
-	if err := notify.NotifyReloading(); err != nil {
-		Log().Error("unable to notify reloading to service manager", zap.Error(err))
+	if err := notify.Reloading(); err != nil {
+		Log().Error("unable to notify service manager of reloading state", zap.Error(err))
 	}
 
+	// after reload, notify system of success or, if
+	// failure, update with status (error message)
+	var err error
 	defer func() {
-		if err := notify.NotifyReadiness(); err != nil {
-			Log().Error("unable to notify readiness to service manager", zap.Error(err))
+		if err != nil {
+			if notifyErr := notify.Error(err, 0); notifyErr != nil {
+				Log().Error("unable to notify to service manager of reload error",
+					zap.Error(notifyErr),
+					zap.String("reload_err", err.Error()))
+			}
+			return
+		}
+		if err := notify.Ready(); err != nil {
+			Log().Error("unable to notify to service manager of ready state", zap.Error(err))
 		}
 	}()
 
-	err := changeConfig(http.MethodPost, "/"+rawConfigKey, cfgJSON, forceReload)
+	err = changeConfig(http.MethodPost, "/"+rawConfigKey, cfgJSON, "", forceReload)
 	if errors.Is(err, errSameConfig) {
 		err = nil // not really an error
 	}
+
 	return err
 }
 
@@ -125,7 +138,14 @@ func Load(cfgJSON []byte, forceReload bool) error {
 // occur unless forceReload is true. If the config is unchanged and not
 // forcefully reloaded, then errConfigUnchanged This function is safe for
 // concurrent use.
-func changeConfig(method, path string, input []byte, forceReload bool) error {
+// The ifMatchHeader can optionally be given a string of the format:
+//
+//	"<path> <hash>"
+//
+// where <path> is the absolute path in the config and <hash> is the expected hash of
+// the config at that path. If the hash in the ifMatchHeader doesn't match
+// the hash of the config, then an APIError with status 412 will be returned.
+func changeConfig(method, path string, input []byte, ifMatchHeader string, forceReload bool) error {
 	switch method {
 	case http.MethodGet,
 		http.MethodHead,
@@ -135,8 +155,42 @@ func changeConfig(method, path string, input []byte, forceReload bool) error {
 		return fmt.Errorf("method not allowed")
 	}
 
-	currentCfgMu.Lock()
-	defer currentCfgMu.Unlock()
+	currentCtxMu.Lock()
+	defer currentCtxMu.Unlock()
+
+	if ifMatchHeader != "" {
+		// expect the first and last character to be quotes
+		if len(ifMatchHeader) < 2 || ifMatchHeader[0] != '"' || ifMatchHeader[len(ifMatchHeader)-1] != '"' {
+			return APIError{
+				HTTPStatus: http.StatusBadRequest,
+				Err:        fmt.Errorf("malformed If-Match header; expect quoted string"),
+			}
+		}
+
+		// read out the parts
+		parts := strings.Fields(ifMatchHeader[1 : len(ifMatchHeader)-1])
+		if len(parts) != 2 {
+			return APIError{
+				HTTPStatus: http.StatusBadRequest,
+				Err:        fmt.Errorf("malformed If-Match header; expect format \"<path> <hash>\""),
+			}
+		}
+
+		// get the current hash of the config
+		// at the given path
+		hash := etagHasher()
+		err := unsyncedConfigAccess(http.MethodGet, parts[0], nil, hash)
+		if err != nil {
+			return err
+		}
+
+		if hex.EncodeToString(hash.Sum(nil)) != parts[1] {
+			return APIError{
+				HTTPStatus: http.StatusPreconditionFailed,
+				Err:        fmt.Errorf("If-Match header did not match current config hash"),
+			}
+		}
+	}
 
 	err := unsyncedConfigAccess(method, path, input, nil)
 	if err != nil {
@@ -177,7 +231,7 @@ func changeConfig(method, path string, input []byte, forceReload bool) error {
 			// with what caddy is still running; we need to
 			// unmarshal it again because it's likely that
 			// pointers deep in our rawCfg map were modified
-			var oldCfg interface{}
+			var oldCfg any
 			err2 := json.Unmarshal(rawCfgJSON, &oldCfg)
 			if err2 != nil {
 				err = fmt.Errorf("%v; additionally, restoring old config: %v", err, err2)
@@ -202,18 +256,18 @@ func changeConfig(method, path string, input []byte, forceReload bool) error {
 // readConfig traverses the current config to path
 // and writes its JSON encoding to out.
 func readConfig(path string, out io.Writer) error {
-	currentCfgMu.RLock()
-	defer currentCfgMu.RUnlock()
+	currentCtxMu.RLock()
+	defer currentCtxMu.RUnlock()
 	return unsyncedConfigAccess(http.MethodGet, path, nil, out)
 }
 
 // indexConfigObjects recursively searches ptr for object fields named
 // "@id" and maps that ID value to the full configPath in the index.
 // This function is NOT safe for concurrent access; obtain a write lock
-// on currentCfgMu.
-func indexConfigObjects(ptr interface{}, configPath string, index map[string]string) error {
+// on currentCtxMu.
+func indexConfigObjects(ptr any, configPath string, index map[string]string) error {
 	switch val := ptr.(type) {
-	case map[string]interface{}:
+	case map[string]any:
 		for k, v := range val {
 			if k == idKey {
 				switch idVal := v.(type) {
@@ -232,7 +286,7 @@ func indexConfigObjects(ptr interface{}, configPath string, index map[string]str
 				return err
 			}
 		}
-	case []interface{}:
+	case []any:
 		// traverse each element of the array recursively
 		for i := range val {
 			err := indexConfigObjects(val[i], path.Join(configPath, strconv.Itoa(i)), index)
@@ -250,7 +304,7 @@ func indexConfigObjects(ptr interface{}, configPath string, index map[string]str
 // it as the new config, replacing any other current config.
 // It does NOT update the raw config state, as this is a
 // lower-level function; most callers will want to use Load
-// instead. A write lock on currentCfgMu is required! If
+// instead. A write lock on currentCtxMu is required! If
 // allowPersist is false, it will not be persisted to disk,
 // even if it is configured to.
 func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
@@ -279,17 +333,17 @@ func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
 	}
 
 	// run the new config and start all its apps
-	err = run(newCfg, true)
+	ctx, err := run(newCfg, true)
 	if err != nil {
 		return err
 	}
 
-	// swap old config with the new one
-	oldCfg := currentCfg
-	currentCfg = newCfg
+	// swap old context (including its config) with the new one
+	oldCtx := currentCtx
+	currentCtx = ctx
 
 	// Stop, Cleanup each old app
-	unsyncedStop(oldCfg)
+	unsyncedStop(oldCtx)
 
 	// autosave a non-nil config, if not disabled
 	if allowPersist &&
@@ -333,7 +387,7 @@ func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
 // This is a low-level function; most callers
 // will want to use Run instead, which also
 // updates the config's raw state.
-func run(newCfg *Config, start bool) error {
+func run(newCfg *Config, start bool) (Context, error) {
 	// because we will need to roll back any state
 	// modifications if this function errors, we
 	// keep a single error value and scope all
@@ -364,8 +418,8 @@ func run(newCfg *Config, start bool) error {
 			cancel()
 
 			// also undo any other state changes we made
-			if currentCfg != nil {
-				certmagic.Default.Storage = currentCfg.storage
+			if currentCtx.cfg != nil {
+				certmagic.Default.Storage = currentCtx.cfg.storage
 			}
 		}
 	}()
@@ -377,14 +431,14 @@ func run(newCfg *Config, start bool) error {
 	}
 	err = newCfg.Logging.openLogs(ctx)
 	if err != nil {
-		return err
+		return ctx, err
 	}
 
 	// start the admin endpoint (and stop any prior one)
 	if start {
 		err = replaceLocalAdminServer(newCfg)
 		if err != nil {
-			return fmt.Errorf("starting caddy administration endpoint: %v", err)
+			return ctx, fmt.Errorf("starting caddy administration endpoint: %v", err)
 		}
 	}
 
@@ -413,7 +467,7 @@ func run(newCfg *Config, start bool) error {
 		return nil
 	}()
 	if err != nil {
-		return err
+		return ctx, err
 	}
 
 	// Load and Provision each app and their submodules
@@ -426,23 +480,23 @@ func run(newCfg *Config, start bool) error {
 		return nil
 	}()
 	if err != nil {
-		return err
+		return ctx, err
 	}
 
 	if !start {
-		return nil
+		return ctx, nil
 	}
 
 	// Provision any admin routers which may need to access
 	// some of the other apps at runtime
 	err = newCfg.Admin.provisionAdminRouters(ctx)
 	if err != nil {
-		return err
+		return ctx, err
 	}
 
 	// Start
 	err = func() error {
-		var started []string
+		started := make([]string, 0, len(newCfg.apps))
 		for name, a := range newCfg.apps {
 			err := a.Start()
 			if err != nil {
@@ -462,12 +516,12 @@ func run(newCfg *Config, start bool) error {
 		return nil
 	}()
 	if err != nil {
-		return err
+		return ctx, err
 	}
 
 	// now that the user's config is running, finish setting up anything else,
 	// such as remote admin endpoint, config loader, etc.
-	return finishSettingUp(ctx, newCfg)
+	return ctx, finishSettingUp(ctx, newCfg)
 }
 
 // finishSettingUp should be run after all apps have successfully started.
@@ -500,7 +554,7 @@ func finishSettingUp(ctx Context, cfg *Config) error {
 
 		runLoadedConfig := func(config []byte) error {
 			logger.Info("applying dynamically-loaded config")
-			err := changeConfig(http.MethodPost, "/"+rawConfigKey, config, false)
+			err := changeConfig(http.MethodPost, "/"+rawConfigKey, config, "", false)
 			if errors.Is(err, errSameConfig) {
 				return err
 			}
@@ -572,10 +626,10 @@ type ConfigLoader interface {
 // stop the others. Stop should only be called
 // if not replacing with a new config.
 func Stop() error {
-	currentCfgMu.Lock()
-	defer currentCfgMu.Unlock()
-	unsyncedStop(currentCfg)
-	currentCfg = nil
+	currentCtxMu.Lock()
+	defer currentCtxMu.Unlock()
+	unsyncedStop(currentCtx)
+	currentCtx = Context{}
 	rawCfgJSON = nil
 	rawCfgIndex = nil
 	rawCfg[rawConfigKey] = nil
@@ -588,13 +642,13 @@ func Stop() error {
 // it is logged and the function continues stopping
 // the next app. This function assumes all apps in
 // cfg were successfully started first.
-func unsyncedStop(cfg *Config) {
-	if cfg == nil {
+func unsyncedStop(ctx Context) {
+	if ctx.cfg == nil {
 		return
 	}
 
 	// stop each app
-	for name, a := range cfg.apps {
+	for name, a := range ctx.cfg.apps {
 		err := a.Stop()
 		if err != nil {
 			log.Printf("[ERROR] stop %s: %v", name, err)
@@ -602,13 +656,13 @@ func unsyncedStop(cfg *Config) {
 	}
 
 	// clean up all modules
-	cfg.cancelFunc()
+	ctx.cfg.cancelFunc()
 }
 
 // Validate loads, provisions, and validates
 // cfg, but does not start running it.
 func Validate(cfg *Config) error {
-	err := run(cfg, false)
+	_, err := run(cfg, false)
 	if err == nil {
 		cfg.cancelFunc() // call Cleanup on all modules
 	}
@@ -622,6 +676,10 @@ func Validate(cfg *Config) error {
 // Errors are logged along the way, and an appropriate exit
 // code is emitted.
 func exitProcess(ctx context.Context, logger *zap.Logger) {
+	if err := notify.Stopping(); err != nil {
+		Log().Error("unable to notify service manager of stopping state", zap.Error(err))
+	}
+
 	if logger == nil {
 		logger = Log()
 	}
@@ -705,8 +763,12 @@ func (d *Duration) UnmarshalJSON(b []byte) error {
 
 // ParseDuration parses a duration string, adding
 // support for the "d" unit meaning number of days,
-// where a day is assumed to be 24h.
+// where a day is assumed to be 24h. The maximum
+// input string length is 1024.
 func ParseDuration(s string) (time.Duration, error) {
+	if len(s) > 1024 {
+		return 0, fmt.Errorf("parsing duration: input string too long")
+	}
 	var inNumber bool
 	var numStart int
 	for i := 0; i < len(s); i++ {
@@ -751,36 +813,106 @@ func InstanceID() (uuid.UUID, error) {
 	return uuid.ParseBytes(uuidFileBytes)
 }
 
-// GoModule returns the build info of this Caddy
-// build from debug.BuildInfo (requires Go modules).
-// If no version information is available, a non-nil
-// value will still be returned, but with an
-// unknown version.
-func GoModule() *debug.Module {
-	var mod debug.Module
-	return goModule(&mod)
-}
-
-// goModule holds the actual implementation of GoModule.
-// Allocating debug.Module in GoModule() and passing a
-// reference to goModule enables mid-stack inlining.
-func goModule(mod *debug.Module) *debug.Module {
-	mod.Version = "unknown"
+// Version returns the Caddy version in a simple/short form, and
+// a full version string. The short form will not have spaces and
+// is intended for User-Agent strings and similar, but may be
+// omitting valuable information. Note that Caddy must be compiled
+// in a special way to properly embed complete version information.
+// First this function tries to get the version from the embedded
+// build info provided by go.mod dependencies; then it tries to
+// get info from embedded VCS information, which requires having
+// built Caddy from a git repository. If no version is available,
+// this function returns "(devel)" becaise Go uses that, but for
+// the simple form we change it to "unknown".
+//
+// See relevant Go issues: https://github.com/golang/go/issues/29228
+// and https://github.com/golang/go/issues/50603.
+//
+// This function is experimental and subject to change or removal.
+func Version() (simple, full string) {
+	// the currently-recommended way to build Caddy involves
+	// building it as a dependency so we can extract version
+	// information from go.mod tooling; once the upstream
+	// Go issues are fixed, we should just be able to use
+	// bi.Main... hopefully.
+	var module *debug.Module
 	bi, ok := debug.ReadBuildInfo()
 	if ok {
-		mod.Path = bi.Main.Path
-		// The recommended way to build Caddy involves
-		// creating a separate main module, which
-		// TODO: track related Go issue: https://github.com/golang/go/issues/29228
-		// once that issue is fixed, we should just be able to use bi.Main... hopefully.
+		// find the Caddy module in the dependency list
 		for _, dep := range bi.Deps {
 			if dep.Path == ImportPath {
-				return dep
+				module = dep
+				break
 			}
 		}
-		return &bi.Main
 	}
-	return mod
+	if module != nil {
+		simple, full = module.Version, module.Version
+		if module.Sum != "" {
+			full += " " + module.Sum
+		}
+		if module.Replace != nil {
+			full += " => " + module.Replace.Path
+			if module.Replace.Version != "" {
+				simple = module.Replace.Version + "_custom"
+				full += "@" + module.Replace.Version
+			}
+			if module.Replace.Sum != "" {
+				full += " " + module.Replace.Sum
+			}
+		}
+	}
+
+	if full == "" {
+		var vcsRevision string
+		var vcsTime time.Time
+		var vcsModified bool
+		for _, setting := range bi.Settings {
+			switch setting.Key {
+			case "vcs.revision":
+				vcsRevision = setting.Value
+			case "vcs.time":
+				vcsTime, _ = time.Parse(time.RFC3339, setting.Value)
+			case "vcs.modified":
+				vcsModified, _ = strconv.ParseBool(setting.Value)
+			}
+		}
+
+		if vcsRevision != "" {
+			var modified string
+			if vcsModified {
+				modified = "+modified"
+			}
+			full = fmt.Sprintf("%s%s (%s)", vcsRevision, modified, vcsTime.Format(time.RFC822))
+			simple = vcsRevision
+
+			// use short checksum for simple, if hex-only
+			if _, err := hex.DecodeString(simple); err == nil {
+				simple = simple[:8]
+			}
+
+			// append date to simple since it can be convenient
+			// to know the commit date as part of the version
+			if !vcsTime.IsZero() {
+				simple += "-" + vcsTime.Format("20060102")
+			}
+		}
+	}
+
+	if simple == "" || simple == "(devel)" {
+		simple = "unknown"
+	}
+
+	return
+}
+
+// ActiveContext returns the currently-active context.
+// This function is experimental and might be changed
+// or removed in the future.
+func ActiveContext() Context {
+	currentCtxMu.RLock()
+	defer currentCtxMu.RUnlock()
+	return currentCtx
 }
 
 // CtxKey is a value type for use with context.WithValue.
@@ -788,18 +920,21 @@ type CtxKey string
 
 // This group of variables pertains to the current configuration.
 var (
-	// currentCfgMu protects everything in this var block.
-	currentCfgMu sync.RWMutex
+	// currentCtxMu protects everything in this var block.
+	currentCtxMu sync.RWMutex
 
-	// currentCfg is the currently-running configuration.
-	currentCfg *Config
+	// currentCtx is the root context for the currently-running
+	// configuration, which can be accessed through this value.
+	// If the Config contained in this value is not nil, then
+	// a config is currently active/running.
+	currentCtx Context
 
 	// rawCfg is the current, generic-decoded configuration;
 	// we initialize it as a map with one field ("config")
 	// to maintain parity with the API endpoint and to avoid
 	// the special case of having to access/mutate the variable
 	// directly without traversing into it.
-	rawCfg = map[string]interface{}{
+	rawCfg = map[string]any{
 		rawConfigKey: nil,
 	}
 
@@ -818,4 +953,5 @@ var (
 var errSameConfig = errors.New("config is unchanged")
 
 // ImportPath is the package import path for Caddy core.
+// This identifier may be removed in the future.
 const ImportPath = "github.com/caddyserver/caddy/v2"

caddyconfig/caddyfile/adapter.go

@@ -29,12 +29,12 @@ type Adapter struct {
 }
 
 // Adapt converts the Caddyfile config in body to Caddy JSON.
-func (a Adapter) Adapt(body []byte, options map[string]interface{}) ([]byte, []caddyconfig.Warning, error) {
+func (a Adapter) Adapt(body []byte, options map[string]any) ([]byte, []caddyconfig.Warning, error) {
 	if a.ServerType == nil {
 		return nil, nil, fmt.Errorf("no server type")
 	}
 	if options == nil {
-		options = make(map[string]interface{})
+		options = make(map[string]any)
 	}
 
 	filename, _ := options["filename"].(string)
@@ -116,7 +116,7 @@ type ServerType interface {
 	// (e.g. CLI flags) and creates a Caddy
 	// config, along with any warnings or
 	// an error.
-	Setup([]ServerBlock, map[string]interface{}) (*caddy.Config, []caddyconfig.Warning, error)
+	Setup([]ServerBlock, map[string]any) (*caddy.Config, []caddyconfig.Warning, error)
 }
 
 // UnmarshalModule instantiates a module with the given ID and invokes

caddyconfig/caddyfile/dispenser.go

@@ -146,15 +146,15 @@ func (d *Dispenser) NextLine() bool {
 //
 // Proper use of this method looks like this:
 //
-//     for nesting := d.Nesting(); d.NextBlock(nesting); {
-//     }
+//	for nesting := d.Nesting(); d.NextBlock(nesting); {
+//	}
 //
 // However, in simple cases where it is known that the
 // Dispenser is new and has not already traversed state
 // by a loop over NextBlock(), this will do:
 //
-//     for d.NextBlock(0) {
-//     }
+//	for d.NextBlock(0) {
+//	}
 //
 // As with other token parsing logic, a loop over
 // NextBlock() should be contained within a loop over
@@ -217,7 +217,7 @@ func (d *Dispenser) ValRaw() string {
 
 // ScalarVal gets value of the current token, converted to the closest
 // scalar type. If there is no token loaded, it returns nil.
-func (d *Dispenser) ScalarVal() interface{} {
+func (d *Dispenser) ScalarVal() any {
 	if d.cursor < 0 || d.cursor >= len(d.tokens) {
 		return nil
 	}
@@ -412,7 +412,7 @@ func (d *Dispenser) Err(msg string) error {
 }
 
 // Errf is like Err, but for formatted error messages
-func (d *Dispenser) Errf(format string, args ...interface{}) error {
+func (d *Dispenser) Errf(format string, args ...any) error {
 	return d.WrapErr(fmt.Errorf(format, args...))
 }
 

caddyconfig/caddyfile/formatter.go

@@ -153,7 +153,10 @@ func Format(input []byte) []byte {
 			openBraceWritten = true
 			nextLine()
 			newLines = 0
-			nesting++
+			// prevent infinite nesting from ridiculous inputs (issue #4169)
+			if nesting < 10 {
+				nesting++
+			}
 		}
 
 		switch {

caddyconfig/caddyfile/formatter_fuzz.go

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 //go:build gofuzz
-// +build gofuzz
 
 package caddyfile
 

caddyconfig/caddyfile/lexer.go

@@ -191,3 +191,7 @@ func Tokenize(input []byte, filename string) ([]Token, error) {
 	}
 	return tokens, nil
 }
+
+func (t Token) Quoted() bool {
+	return t.wasQuoted > 0
+}

caddyconfig/caddyfile/lexer_fuzz.go

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 //go:build gofuzz
-// +build gofuzz
 
 package caddyfile
 

caddyconfig/configadapters.go

@@ -24,7 +24,7 @@ import (
 // Adapter is a type which can adapt a configuration to Caddy JSON.
 // It returns the results and any warnings, or an error.
 type Adapter interface {
-	Adapt(body []byte, options map[string]interface{}) ([]byte, []Warning, error)
+	Adapt(body []byte, options map[string]any) ([]byte, []Warning, error)
 }
 
 // Warning represents a warning or notice related to conversion.
@@ -48,7 +48,7 @@ func (w Warning) String() string {
 // are converted to warnings. This is convenient when filling config
 // structs that require a json.RawMessage, without having to worry
 // about errors.
-func JSON(val interface{}, warnings *[]Warning) json.RawMessage {
+func JSON(val any, warnings *[]Warning) json.RawMessage {
 	b, err := json.Marshal(val)
 	if err != nil {
 		if warnings != nil {
@@ -64,9 +64,9 @@ func JSON(val interface{}, warnings *[]Warning) json.RawMessage {
 // for encoding module values where the module name has to be described within
 // the object by a certain key; for example, `"handler": "file_server"` for a
 // file server HTTP handler (fieldName="handler" and fieldVal="file_server").
-// The val parameter must encode into a map[string]interface{} (i.e. it must be
+// The val parameter must encode into a map[string]any (i.e. it must be
 // a struct or map). Any errors are converted into warnings.
-func JSONModuleObject(val interface{}, fieldName, fieldVal string, warnings *[]Warning) json.RawMessage {
+func JSONModuleObject(val any, fieldName, fieldVal string, warnings *[]Warning) json.RawMessage {
 	// encode to a JSON object first
 	enc, err := json.Marshal(val)
 	if err != nil {
@@ -77,7 +77,7 @@ func JSONModuleObject(val interface{}, fieldName, fieldVal string, warnings *[]W
 	}
 
 	// then decode the object
-	var tmp map[string]interface{}
+	var tmp map[string]any
 	err = json.Unmarshal(enc, &tmp)
 	if err != nil {
 		if warnings != nil {

caddyconfig/httpcaddyfile/addresses.go

@@ -17,6 +17,7 @@ package httpcaddyfile
 import (
 	"fmt"
 	"net"
+	"net/netip"
 	"reflect"
 	"sort"
 	"strconv"
@@ -35,12 +36,12 @@ import (
 // server block that share the same address stay grouped together so the config
 // isn't repeated unnecessarily. For example, this Caddyfile:
 //
-// 	example.com {
-// 		bind 127.0.0.1
-// 	}
-// 	www.example.com, example.net/path, localhost:9999 {
-// 		bind 127.0.0.1 1.2.3.4
-// 	}
+//	example.com {
+//		bind 127.0.0.1
+//	}
+//	www.example.com, example.net/path, localhost:9999 {
+//		bind 127.0.0.1 1.2.3.4
+//	}
 //
 // has two server blocks to start with. But expressed in this Caddyfile are
 // actually 4 listener addresses: 127.0.0.1:443, 1.2.3.4:443, 127.0.0.1:9999,
@@ -76,7 +77,7 @@ import (
 // multiple addresses to the same lists of server blocks (a many:many mapping).
 // (Doing this is essentially a map-reduce technique.)
 func (st *ServerType) mapAddressToServerBlocks(originalServerBlocks []serverBlock,
-	options map[string]interface{}) (map[string][]serverBlock, error) {
+	options map[string]any) (map[string][]serverBlock, error) {
 	sbmap := make(map[string][]serverBlock)
 
 	for i, sblock := range originalServerBlocks {
@@ -102,12 +103,20 @@ func (st *ServerType) mapAddressToServerBlocks(originalServerBlocks []serverBloc
 			}
 		}
 
+		// make a slice of the map keys so we can iterate in sorted order
+		addrs := make([]string, 0, len(addrToKeys))
+		for k := range addrToKeys {
+			addrs = append(addrs, k)
+		}
+		sort.Strings(addrs)
+
 		// now that we know which addresses serve which keys of this
 		// server block, we iterate that mapping and create a list of
 		// new server blocks for each address where the keys of the
 		// server block are only the ones which use the address; but
 		// the contents (tokens) are of course the same
-		for addr, keys := range addrToKeys {
+		for _, addr := range addrs {
+			keys := addrToKeys[addr]
 			// parse keys so that we only have to do it once
 			parsedKeys := make([]Address, 0, len(keys))
 			for _, key := range keys {
@@ -161,6 +170,7 @@ func (st *ServerType) consolidateAddrMappings(addrToServerBlocks map[string][]se
 				delete(addrToServerBlocks, otherAddr)
 			}
 		}
+		sort.Strings(a.addresses)
 
 		sbaddrs = append(sbaddrs, a)
 	}
@@ -174,8 +184,10 @@ func (st *ServerType) consolidateAddrMappings(addrToServerBlocks map[string][]se
 	return sbaddrs
 }
 
+// listenerAddrsForServerBlockKey essentially converts the Caddyfile
+// site addresses to Caddy listener addresses for each server block.
 func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key string,
-	options map[string]interface{}) ([]string, error) {
+	options map[string]any) ([]string, error) {
 	addr, err := ParseAddress(key)
 	if err != nil {
 		return nil, fmt.Errorf("parsing key: %v", err)
@@ -207,14 +219,14 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 		return nil, fmt.Errorf("[%s] scheme and port violate convention", key)
 	}
 
-	// the bind directive specifies hosts, but is optional
-	lnHosts := make([]string, 0, len(sblock.pile))
+	// the bind directive specifies hosts (and potentially network), but is optional
+	lnHosts := make([]string, 0, len(sblock.pile["bind"]))
 	for _, cfgVal := range sblock.pile["bind"] {
 		lnHosts = append(lnHosts, cfgVal.Value.([]string)...)
 	}
 	if len(lnHosts) == 0 {
-		if defaultBind, ok := options["default_bind"].(string); ok {
-			lnHosts = []string{defaultBind}
+		if defaultBind, ok := options["default_bind"].([]string); ok {
+			lnHosts = defaultBind
 		} else {
 			lnHosts = []string{""}
 		}
@@ -222,13 +234,27 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 
 	// use a map to prevent duplication
 	listeners := make(map[string]struct{})
-	for _, host := range lnHosts {
-		addr, err := caddy.ParseNetworkAddress(host)
-		if err == nil && addr.IsUnixNetwork() {
-			listeners[host] = struct{}{}
-		} else {
-			listeners[host+":"+lnPort] = struct{}{}
+	for _, lnHost := range lnHosts {
+		// normally we would simply append the port,
+		// but if lnHost is IPv6, we need to ensure it
+		// is enclosed in [ ]; net.JoinHostPort does
+		// this for us, but lnHost might also have a
+		// network type in front (e.g. "tcp/") leading
+		// to "[tcp/::1]" which causes parsing failures
+		// later; what we need is "tcp/[::1]", so we have
+		// to split the network and host, then re-combine
+		network, host, ok := strings.Cut(lnHost, "/")
+		if !ok {
+			host = network
+			network = ""
 		}
+		host = strings.Trim(host, "[]") // IPv6
+		networkAddr := caddy.JoinNetworkAddress(network, host, lnPort)
+		addr, err := caddy.ParseNetworkAddress(networkAddr)
+		if err != nil {
+			return nil, fmt.Errorf("parsing network address: %v", err)
+		}
+		listeners[addr.String()] = struct{}{}
 	}
 
 	// now turn map into list
@@ -236,6 +262,7 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 	for lnStr := range listeners {
 		listenersList = append(listenersList, lnStr)
 	}
+	sort.Strings(listenersList)
 
 	return listenersList, nil
 }
@@ -340,9 +367,9 @@ func (a Address) Normalize() Address {
 
 	// ensure host is normalized if it's an IP address
 	host := strings.TrimSpace(a.Host)
-	if ip := net.ParseIP(host); ip != nil {
-		if ipv6 := ip.To16(); ipv6 != nil && ipv6.DefaultMask() == nil {
-			host = ipv6.String()
+	if ip, err := netip.ParseAddr(host); err == nil {
+		if ip.Is6() && !ip.Is4() && !ip.Is4In6() {
+			host = ip.String()
 		}
 	}
 

caddyconfig/httpcaddyfile/addresses_fuzz.go

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 //go:build gofuzz
-// +build gofuzz
 
 package httpcaddyfile
 

caddyconfig/httpcaddyfile/builtins.go

@@ -48,12 +48,12 @@ func init() {
 	RegisterHandlerDirective("handle", parseHandle)
 	RegisterDirective("handle_errors", parseHandleErrors)
 	RegisterDirective("log", parseLog)
+	RegisterHandlerDirective("skip_log", parseSkipLog)
 }
 
 // parseBind parses the bind directive. Syntax:
 //
-//     bind <addresses...>
-//
+//	bind <addresses...>
 func parseBind(h Helper) ([]ConfigValue, error) {
 	var lnHosts []string
 	for h.Next() {
@@ -64,28 +64,28 @@ func parseBind(h Helper) ([]ConfigValue, error) {
 
 // parseTLS parses the tls directive. Syntax:
 //
-//     tls [<email>|internal]|[<cert_file> <key_file>] {
-//         protocols <min> [<max>]
-//         ciphers   <cipher_suites...>
-//         curves    <curves...>
-//         client_auth {
-//             mode                   [request|require|verify_if_given|require_and_verify]
-//             trusted_ca_cert        <base64_der>
-//             trusted_ca_cert_file   <filename>
-//             trusted_leaf_cert      <base64_der>
-//             trusted_leaf_cert_file <filename>
-//         }
-//         alpn      <values...>
-//         load      <paths...>
-//         ca        <acme_ca_endpoint>
-//         ca_root   <pem_file>
-//         dns       <provider_name> [...]
-//         on_demand
-//         eab    <key_id> <mac_key>
-//         issuer <module_name> [...]
-//         get_certificate <module_name> [...]
-//     }
-//
+//	tls [<email>|internal]|[<cert_file> <key_file>] {
+//	    protocols <min> [<max>]
+//	    ciphers   <cipher_suites...>
+//	    curves    <curves...>
+//	    client_auth {
+//	        mode                   [request|require|verify_if_given|require_and_verify]
+//	        trusted_ca_cert        <base64_der>
+//	        trusted_ca_cert_file   <filename>
+//	        trusted_leaf_cert      <base64_der>
+//	        trusted_leaf_cert_file <filename>
+//	    }
+//	    alpn      <values...>
+//	    load      <paths...>
+//	    ca        <acme_ca_endpoint>
+//	    ca_root   <pem_file>
+//	    dns       <provider_name> [...]
+//	    on_demand
+//	    eab    <key_id> <mac_key>
+//	    issuer <module_name> [...]
+//	    get_certificate <module_name> [...]
+//	    insecure_secrets_log <log_file>
+//	}
 func parseTLS(h Helper) ([]ConfigValue, error) {
 	cp := new(caddytls.ConnectionPolicy)
 	var fileLoader caddytls.FileLoader
@@ -395,6 +395,12 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 				}
 				onDemand = true
 
+			case "insecure_secrets_log":
+				if !h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				cp.InsecureSecretsLog = h.Val()
+
 			default:
 				return nil, h.Errf("unknown subdirective: %s", h.Val())
 			}
@@ -515,8 +521,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 
 // parseRoot parses the root directive. Syntax:
 //
-//     root [<matcher>] <path>
-//
+//	root [<matcher>] <path>
 func parseRoot(h Helper) (caddyhttp.MiddlewareHandler, error) {
 	var root string
 	for h.Next() {
@@ -540,8 +545,13 @@ func parseVars(h Helper) (caddyhttp.MiddlewareHandler, error) {
 
 // parseRedir parses the redir directive. Syntax:
 //
-//     redir [<matcher>] <to> [<code>]
+//	redir [<matcher>] <to> [<code>]
 //
+// <code> can be "permanent" for 301, "temporary" for 302 (default),
+// a placeholder, or any number in the 3xx range or 401. The special
+// code "html" can be used to redirect only browser clients (will
+// respond with HTTP 200 and no Location header; redirect is performed
+// with JS and a meta tag).
 func parseRedir(h Helper) (caddyhttp.MiddlewareHandler, error) {
 	if !h.Next() {
 		return nil, h.ArgErr()
@@ -558,6 +568,7 @@ func parseRedir(h Helper) (caddyhttp.MiddlewareHandler, error) {
 	}
 
 	var body string
+	var hdr http.Header
 	switch code {
 	case "permanent":
 		code = "301"
@@ -578,7 +589,7 @@ func parseRedir(h Helper) (caddyhttp.MiddlewareHandler, error) {
 `
 		safeTo := html.EscapeString(to)
 		body = fmt.Sprintf(metaRedir, safeTo, safeTo, safeTo, safeTo)
-		code = "302"
+		code = "200" // don't redirect non-browser clients
 	default:
 		// Allow placeholders for the code
 		if strings.HasPrefix(code, "{") {
@@ -601,9 +612,14 @@ func parseRedir(h Helper) (caddyhttp.MiddlewareHandler, error) {
 		}
 	}
 
+	// don't redirect non-browser clients
+	if code != "200" {
+		hdr = http.Header{"Location": []string{to}}
+	}
+
 	return caddyhttp.StaticResponse{
 		StatusCode: caddyhttp.WeakString(code),
-		Headers:    http.Header{"Location": []string{to}},
+		Headers:    hdr,
 		Body:       body,
 	}, nil
 }
@@ -683,12 +699,11 @@ func parseHandleErrors(h Helper) ([]ConfigValue, error) {
 
 // parseLog parses the log directive. Syntax:
 //
-//     log {
-//         output <writer_module> ...
-//         format <encoder_module> ...
-//         level  <level>
-//     }
-//
+//	log {
+//	    output <writer_module> ...
+//	    format <encoder_module> ...
+//	    level  <level>
+//	}
 func parseLog(h Helper) ([]ConfigValue, error) {
 	return parseLogHelper(h, nil)
 }
@@ -847,3 +862,15 @@ func parseLogHelper(h Helper, globalLogNames map[string]struct{}) ([]ConfigValue
 	}
 	return configValues, nil
 }
+
+// parseSkipLog parses the skip_log directive. Syntax:
+//
+//	skip_log [<matcher>]
+func parseSkipLog(h Helper) (caddyhttp.MiddlewareHandler, error) {
+	for h.Next() {
+		if h.NextArg() {
+			return nil, h.ArgErr()
+		}
+	}
+	return caddyhttp.VarsMiddleware{"skip_log": true}, nil
+}

caddyconfig/httpcaddyfile/directives.go

@@ -42,6 +42,7 @@ var directiveOrder = []string{
 	"map",
 	"vars",
 	"root",
+	"skip_log",
 
 	"header",
 	"copy_response_headers", // only in reverse_proxy's handle_response
@@ -142,8 +143,8 @@ func RegisterGlobalOption(opt string, setupFunc UnmarshalGlobalFunc) {
 type Helper struct {
 	*caddyfile.Dispenser
 	// State stores intermediate variables during caddyfile adaptation.
-	State        map[string]interface{}
-	options      map[string]interface{}
+	State        map[string]any
+	options      map[string]any
 	warnings     *[]caddyconfig.Warning
 	matcherDefs  map[string]caddy.ModuleMap
 	parentBlock  caddyfile.ServerBlock
@@ -151,7 +152,7 @@ type Helper struct {
 }
 
 // Option gets the option keyed by name.
-func (h Helper) Option(name string) interface{} {
+func (h Helper) Option(name string) any {
 	return h.options[name]
 }
 
@@ -175,7 +176,7 @@ func (h Helper) Caddyfiles() []string {
 }
 
 // JSON converts val into JSON. Any errors are added to warnings.
-func (h Helper) JSON(val interface{}) json.RawMessage {
+func (h Helper) JSON(val any) json.RawMessage {
 	return caddyconfig.JSON(val, h.warnings)
 }
 
@@ -375,7 +376,7 @@ type ConfigValue struct {
 	// The value to be used when building the config.
 	// Generally its type is associated with the
 	// name of the Class.
-	Value interface{}
+	Value any
 
 	directive string
 }
@@ -406,7 +407,7 @@ func sortRoutes(routes []ConfigValue) {
 			return false
 		}
 
-		// decode the path matchers, if there is just one of them
+		// decode the path matchers if there is just one matcher set
 		var iPM, jPM caddyhttp.MatchPath
 		if len(iRoute.MatcherSetsRaw) == 1 {
 			_ = json.Unmarshal(iRoute.MatcherSetsRaw[0]["path"], &iPM)
@@ -415,38 +416,45 @@ func sortRoutes(routes []ConfigValue) {
 			_ = json.Unmarshal(jRoute.MatcherSetsRaw[0]["path"], &jPM)
 		}
 
-		// sort by longer path (more specific) first; missing path
-		// matchers or multi-matchers are treated as zero-length paths
+		// if there is only one path in the path matcher, sort by longer path
+		// (more specific) first; missing path matchers or multi-matchers are
+		// treated as zero-length paths
 		var iPathLen, jPathLen int
-		if len(iPM) > 0 {
+		if len(iPM) == 1 {
 			iPathLen = len(iPM[0])
 		}
-		if len(jPM) > 0 {
+		if len(jPM) == 1 {
 			jPathLen = len(jPM[0])
 		}
 
 		// some directives involve setting values which can overwrite
-		// eachother, so it makes most sense to reverse the order so
+		// each other, so it makes most sense to reverse the order so
 		// that the lease specific matcher is first; everything else
 		// has most-specific matcher first
 		if iDir == "vars" {
-			// if both directives have no path matcher, use whichever one
-			// has no matcher first.
-			if iPathLen == 0 && jPathLen == 0 {
-				return len(iRoute.MatcherSetsRaw) == 0 && len(jRoute.MatcherSetsRaw) > 0
+			// we can only confidently compare path lengths if both
+			// directives have a single path to match (issue #5037)
+			if iPathLen > 0 && jPathLen > 0 {
+				// sort least-specific (shortest) path first
+				return iPathLen < jPathLen
 			}
 
-			// sort with the least-specific (shortest) path first
-			return iPathLen < jPathLen
+			// if both directives don't have a single path to compare,
+			// sort whichever one has no matcher first; if both have
+			// no matcher, sort equally (stable sort preserves order)
+			return len(iRoute.MatcherSetsRaw) == 0 && len(jRoute.MatcherSetsRaw) > 0
 		} else {
-			// if both directives have no path matcher, use whichever one
-			// has any kind of matcher defined first.
-			if iPathLen == 0 && jPathLen == 0 {
-				return len(iRoute.MatcherSetsRaw) > 0 && len(jRoute.MatcherSetsRaw) == 0
+			// we can only confidently compare path lengths if both
+			// directives have a single path to match (issue #5037)
+			if iPathLen > 0 && jPathLen > 0 {
+				// sort most-specific (longest) path first
+				return iPathLen > jPathLen
 			}
 
-			// sort with the most-specific (longest) path first
-			return iPathLen > jPathLen
+			// if both directives don't have a single path to compare,
+			// sort whichever one has a matcher first; if both have
+			// a matcher, sort equally (stable sort preserves order)
+			return len(iRoute.MatcherSetsRaw) > 0 && len(jRoute.MatcherSetsRaw) == 0
 		}
 	})
 }
@@ -567,7 +575,7 @@ type (
 	// tokens from a global option. It is passed the tokens to parse and
 	// existing value from the previous instance of this global option
 	// (if any). It returns the value to associate with this global option.
-	UnmarshalGlobalFunc func(d *caddyfile.Dispenser, existingVal interface{}) (interface{}, error)
+	UnmarshalGlobalFunc func(d *caddyfile.Dispenser, existingVal any) (any, error)
 )
 
 var registeredDirectives = make(map[string]UnmarshalFunc)

caddyconfig/httpcaddyfile/httptype.go

@@ -53,27 +53,18 @@ type ServerType struct {
 
 // Setup makes a config from the tokens.
 func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
-	options map[string]interface{}) (*caddy.Config, []caddyconfig.Warning, error) {
+	options map[string]any) (*caddy.Config, []caddyconfig.Warning, error) {
 	var warnings []caddyconfig.Warning
 	gc := counter{new(int)}
-	state := make(map[string]interface{})
+	state := make(map[string]any)
 
-	// load all the server blocks and associate them with a "pile"
-	// of config values; also prohibit duplicate keys because they
-	// can make a config confusing if more than one server block is
-	// chosen to handle a request - we actually will make each
-	// server block's route terminal so that only one will run
-	sbKeys := make(map[string]struct{})
+	// load all the server blocks and associate them with a "pile" of config values
 	originalServerBlocks := make([]serverBlock, 0, len(inputServerBlocks))
-	for i, sblock := range inputServerBlocks {
+	for _, sblock := range inputServerBlocks {
 		for j, k := range sblock.Keys {
 			if j == 0 && strings.HasPrefix(k, "@") {
 				return nil, warnings, fmt.Errorf("cannot define a matcher outside of a site block: '%s'", k)
 			}
-			if _, ok := sbKeys[k]; ok {
-				return nil, warnings, fmt.Errorf("duplicate site address not allowed: '%s' in %v (site block %d, key %d)", k, sblock.Keys, i, j)
-			}
-			sbKeys[k] = struct{}{}
 		}
 		originalServerBlocks = append(originalServerBlocks, serverBlock{
 			block: sblock,
@@ -100,13 +91,17 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		search  *regexp.Regexp
 		replace string
 	}{
-		{regexp.MustCompile(`{query\.([\w-]*)}`), "{http.request.uri.query.$1}"},
-		{regexp.MustCompile(`{labels\.([\w-]*)}`), "{http.request.host.labels.$1}"},
 		{regexp.MustCompile(`{header\.([\w-]*)}`), "{http.request.header.$1}"},
+		{regexp.MustCompile(`{cookie\.([\w-]*)}`), "{http.request.cookie.$1}"},
+		{regexp.MustCompile(`{labels\.([\w-]*)}`), "{http.request.host.labels.$1}"},
 		{regexp.MustCompile(`{path\.([\w-]*)}`), "{http.request.uri.path.$1}"},
+		{regexp.MustCompile(`{file\.([\w-]*)}`), "{http.request.uri.path.file.$1}"},
+		{regexp.MustCompile(`{query\.([\w-]*)}`), "{http.request.uri.query.$1}"},
 		{regexp.MustCompile(`{re\.([\w-]*)\.([\w-]*)}`), "{http.regexp.$1.$2}"},
 		{regexp.MustCompile(`{vars\.([\w-]*)}`), "{http.vars.$1}"},
 		{regexp.MustCompile(`{rp\.([\w-\.]*)}`), "{http.reverse_proxy.$1}"},
+		{regexp.MustCompile(`{err\.([\w-\.]*)}`), "{http.error.$1}"},
+		{regexp.MustCompile(`{file_match\.([\w-]*)}`), "{http.matchers.file.$1}"},
 	}
 
 	for _, sb := range originalServerBlocks {
@@ -198,10 +193,11 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 
 	// now that each server is configured, make the HTTP app
 	httpApp := caddyhttp.App{
-		HTTPPort:    tryInt(options["http_port"], &warnings),
-		HTTPSPort:   tryInt(options["https_port"], &warnings),
-		GracePeriod: tryDuration(options["grace_period"], &warnings),
-		Servers:     servers,
+		HTTPPort:      tryInt(options["http_port"], &warnings),
+		HTTPSPort:     tryInt(options["https_port"], &warnings),
+		GracePeriod:   tryDuration(options["grace_period"], &warnings),
+		ShutdownDelay: tryDuration(options["shutdown_delay"], &warnings),
+		Servers:       servers,
 	}
 
 	// then make the TLS app
@@ -227,7 +223,7 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 			hasDefaultLog = true
 		}
 		if _, ok := options["debug"]; ok && ncl.log.Level == "" {
-			ncl.log.Level = "DEBUG"
+			ncl.log.Level = zap.DebugLevel.CapitalString()
 		}
 		customLogs = append(customLogs, ncl)
 	}
@@ -245,7 +241,7 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		if _, ok := options["debug"]; ok {
 			customLogs = append(customLogs, namedCustomLog{
 				name: "default",
-				log:  &caddy.CustomLog{Level: "DEBUG"},
+				log:  &caddy.CustomLog{Level: zap.DebugLevel.CapitalString()},
 			})
 		}
 	}
@@ -321,14 +317,14 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 // which is expected to be the first server block if it has zero
 // keys. It returns the updated list of server blocks with the
 // global options block removed, and updates options accordingly.
-func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options map[string]interface{}) ([]serverBlock, error) {
+func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options map[string]any) ([]serverBlock, error) {
 	if len(serverBlocks) == 0 || len(serverBlocks[0].block.Keys) > 0 {
 		return serverBlocks, nil
 	}
 
 	for _, segment := range serverBlocks[0].block.Segments {
 		opt := segment.Directive()
-		var val interface{}
+		var val any
 		var err error
 		disp := caddyfile.NewDispenser(segment)
 
@@ -398,7 +394,7 @@ func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options
 // to server blocks. Each pairing is essentially a server definition.
 func (st *ServerType) serversFromPairings(
 	pairings []sbAddrAssociation,
-	options map[string]interface{},
+	options map[string]any,
 	warnings *[]caddyconfig.Warning,
 	groupCounter counter,
 ) (map[string]*caddyhttp.Server, error) {
@@ -419,6 +415,23 @@ func (st *ServerType) serversFromPairings(
 	}
 
 	for i, p := range pairings {
+		// detect ambiguous site definitions: server blocks which
+		// have the same host bound to the same interface (listener
+		// address), otherwise their routes will improperly be added
+		// to the same server (see issue #4635)
+		for j, sblock1 := range p.serverBlocks {
+			for _, key := range sblock1.block.Keys {
+				for k, sblock2 := range p.serverBlocks {
+					if k == j {
+						continue
+					}
+					if sliceContains(sblock2.block.Keys, key) {
+						return nil, fmt.Errorf("ambiguous site definition: %s", key)
+					}
+				}
+			}
+		}
+
 		srv := &caddyhttp.Server{
 			Listen: p.addresses,
 		}
@@ -716,7 +729,7 @@ func (st *ServerType) serversFromPairings(
 	return servers, nil
 }
 
-func detectConflictingSchemes(srv *caddyhttp.Server, serverBlocks []serverBlock, options map[string]interface{}) error {
+func detectConflictingSchemes(srv *caddyhttp.Server, serverBlocks []serverBlock, options map[string]any) error {
 	httpPort := strconv.Itoa(caddyhttp.DefaultHTTPPort)
 	if hp, ok := options["http_port"].(int); ok {
 		httpPort = strconv.Itoa(hp)
@@ -942,7 +955,7 @@ func appendSubrouteToRouteList(routeList caddyhttp.RouteList,
 func buildSubroute(routes []ConfigValue, groupCounter counter) (*caddyhttp.Subroute, error) {
 	for _, val := range routes {
 		if !directiveIsOrdered(val.directive) {
-			return nil, fmt.Errorf("directive '%s' is not ordered, so it cannot be used here", val.directive)
+			return nil, fmt.Errorf("directive '%s' is not an ordered HTTP handler, so it cannot be used here", val.directive)
 		}
 	}
 
@@ -1190,6 +1203,7 @@ func (st *ServerType) compileEncodedMatcherSets(sblock serverBlock) ([]caddy.Mod
 
 func parseMatcherDefinitions(d *caddyfile.Dispenser, matchers map[string]caddy.ModuleMap) error {
 	for d.Next() {
+		// this is the "name" for "named matchers"
 		definitionName := d.Val()
 
 		if _, ok := matchers[definitionName]; ok {
@@ -1197,16 +1211,9 @@ func parseMatcherDefinitions(d *caddyfile.Dispenser, matchers map[string]caddy.M
 		}
 		matchers[definitionName] = make(caddy.ModuleMap)
 
-		// in case there are multiple instances of the same matcher, concatenate
-		// their tokens (we expect that UnmarshalCaddyfile should be able to
-		// handle more than one segment); otherwise, we'd overwrite other
-		// instances of the matcher in this set
-		tokensByMatcherName := make(map[string][]caddyfile.Token)
-		for nesting := d.Nesting(); d.NextArg() || d.NextBlock(nesting); {
-			matcherName := d.Val()
-			tokensByMatcherName[matcherName] = append(tokensByMatcherName[matcherName], d.NextSegment()...)
-		}
-		for matcherName, tokens := range tokensByMatcherName {
+		// given a matcher name and the tokens following it, parse
+		// the tokens as a matcher module and record it
+		makeMatcher := func(matcherName string, tokens []caddyfile.Token) error {
 			mod, err := caddy.GetModule("http.matchers." + matcherName)
 			if err != nil {
 				return fmt.Errorf("getting matcher module '%s': %v", matcherName, err)
@@ -1224,6 +1231,39 @@ func parseMatcherDefinitions(d *caddyfile.Dispenser, matchers map[string]caddy.M
 				return fmt.Errorf("matcher module '%s' is not a request matcher", matcherName)
 			}
 			matchers[definitionName][matcherName] = caddyconfig.JSON(rm, nil)
+			return nil
+		}
+
+		// if the next token is quoted, we can assume it's not a matcher name
+		// and that it's probably an 'expression' matcher
+		if d.NextArg() {
+			if d.Token().Quoted() {
+				err := makeMatcher("expression", []caddyfile.Token{d.Token()})
+				if err != nil {
+					return err
+				}
+				continue
+			}
+
+			// if it wasn't quoted, then we need to rewind after calling
+			// d.NextArg() so the below properly grabs the matcher name
+			d.Prev()
+		}
+
+		// in case there are multiple instances of the same matcher, concatenate
+		// their tokens (we expect that UnmarshalCaddyfile should be able to
+		// handle more than one segment); otherwise, we'd overwrite other
+		// instances of the matcher in this set
+		tokensByMatcherName := make(map[string][]caddyfile.Token)
+		for nesting := d.Nesting(); d.NextArg() || d.NextBlock(nesting); {
+			matcherName := d.Val()
+			tokensByMatcherName[matcherName] = append(tokensByMatcherName[matcherName], d.NextSegment()...)
+		}
+		for matcherName, tokens := range tokensByMatcherName {
+			err := makeMatcher(matcherName, tokens)
+			if err != nil {
+				return err
+			}
 		}
 	}
 	return nil
@@ -1295,7 +1335,7 @@ func WasReplacedPlaceholderShorthand(token string) string {
 
 // tryInt tries to convert val to an integer. If it fails,
 // it downgrades the error to a warning and returns 0.
-func tryInt(val interface{}, warnings *[]caddyconfig.Warning) int {
+func tryInt(val any, warnings *[]caddyconfig.Warning) int {
 	intVal, ok := val.(int)
 	if val != nil && !ok && warnings != nil {
 		*warnings = append(*warnings, caddyconfig.Warning{Message: "not an integer type"})
@@ -1303,7 +1343,7 @@ func tryInt(val interface{}, warnings *[]caddyconfig.Warning) int {
 	return intVal
 }
 
-func tryString(val interface{}, warnings *[]caddyconfig.Warning) string {
+func tryString(val any, warnings *[]caddyconfig.Warning) string {
 	stringVal, ok := val.(string)
 	if val != nil && !ok && warnings != nil {
 		*warnings = append(*warnings, caddyconfig.Warning{Message: "not a string type"})
@@ -1311,7 +1351,7 @@ func tryString(val interface{}, warnings *[]caddyconfig.Warning) string {
 	return stringVal
 }
 
-func tryDuration(val interface{}, warnings *[]caddyconfig.Warning) caddy.Duration {
+func tryDuration(val any, warnings *[]caddyconfig.Warning) caddy.Duration {
 	durationVal, ok := val.(caddy.Duration)
 	if val != nil && !ok && warnings != nil {
 		*warnings = append(*warnings, caddyconfig.Warning{Message: "not a duration type"})

caddyconfig/httpcaddyfile/options.go

@@ -29,13 +29,15 @@ func init() {
 	RegisterGlobalOption("debug", parseOptTrue)
 	RegisterGlobalOption("http_port", parseOptHTTPPort)
 	RegisterGlobalOption("https_port", parseOptHTTPSPort)
-	RegisterGlobalOption("default_bind", parseOptSingleString)
+	RegisterGlobalOption("default_bind", parseOptStringList)
 	RegisterGlobalOption("grace_period", parseOptDuration)
+	RegisterGlobalOption("shutdown_delay", parseOptDuration)
 	RegisterGlobalOption("default_sni", parseOptSingleString)
 	RegisterGlobalOption("order", parseOptOrder)
 	RegisterGlobalOption("storage", parseOptStorage)
 	RegisterGlobalOption("storage_clean_interval", parseOptDuration)
 	RegisterGlobalOption("renew_interval", parseOptDuration)
+	RegisterGlobalOption("ocsp_interval", parseOptDuration)
 	RegisterGlobalOption("acme_ca", parseOptSingleString)
 	RegisterGlobalOption("acme_ca_root", parseOptSingleString)
 	RegisterGlobalOption("acme_dns", parseOptACMEDNS)
@@ -54,9 +56,9 @@ func init() {
 	RegisterGlobalOption("preferred_chains", parseOptPreferredChains)
 }
 
-func parseOptTrue(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) { return true, nil }
+func parseOptTrue(d *caddyfile.Dispenser, _ any) (any, error) { return true, nil }
 
-func parseOptHTTPPort(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptHTTPPort(d *caddyfile.Dispenser, _ any) (any, error) {
 	var httpPort int
 	for d.Next() {
 		var httpPortStr string
@@ -72,7 +74,7 @@ func parseOptHTTPPort(d *caddyfile.Dispenser, _ interface{}) (interface{}, error
 	return httpPort, nil
 }
 
-func parseOptHTTPSPort(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptHTTPSPort(d *caddyfile.Dispenser, _ any) (any, error) {
 	var httpsPort int
 	for d.Next() {
 		var httpsPortStr string
@@ -88,7 +90,7 @@ func parseOptHTTPSPort(d *caddyfile.Dispenser, _ interface{}) (interface{}, erro
 	return httpsPort, nil
 }
 
-func parseOptOrder(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptOrder(d *caddyfile.Dispenser, _ any) (any, error) {
 	newOrder := directiveOrder
 
 	for d.Next() {
@@ -164,7 +166,7 @@ func parseOptOrder(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
 	return newOrder, nil
 }
 
-func parseOptStorage(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptStorage(d *caddyfile.Dispenser, _ any) (any, error) {
 	if !d.Next() { // consume option name
 		return nil, d.ArgErr()
 	}
@@ -183,7 +185,7 @@ func parseOptStorage(d *caddyfile.Dispenser, _ interface{}) (interface{}, error)
 	return storage, nil
 }
 
-func parseOptDuration(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptDuration(d *caddyfile.Dispenser, _ any) (any, error) {
 	if !d.Next() { // consume option name
 		return nil, d.ArgErr()
 	}
@@ -197,7 +199,7 @@ func parseOptDuration(d *caddyfile.Dispenser, _ interface{}) (interface{}, error
 	return caddy.Duration(dur), nil
 }
 
-func parseOptACMEDNS(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptACMEDNS(d *caddyfile.Dispenser, _ any) (any, error) {
 	if !d.Next() { // consume option name
 		return nil, d.ArgErr()
 	}
@@ -216,7 +218,7 @@ func parseOptACMEDNS(d *caddyfile.Dispenser, _ interface{}) (interface{}, error)
 	return prov, nil
 }
 
-func parseOptACMEEAB(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptACMEEAB(d *caddyfile.Dispenser, _ any) (any, error) {
 	eab := new(acme.EAB)
 	for d.Next() {
 		if d.NextArg() {
@@ -244,7 +246,7 @@ func parseOptACMEEAB(d *caddyfile.Dispenser, _ interface{}) (interface{}, error)
 	return eab, nil
 }
 
-func parseOptCertIssuer(d *caddyfile.Dispenser, existing interface{}) (interface{}, error) {
+func parseOptCertIssuer(d *caddyfile.Dispenser, existing any) (any, error) {
 	var issuers []certmagic.Issuer
 	if existing != nil {
 		issuers = existing.([]certmagic.Issuer)
@@ -267,7 +269,7 @@ func parseOptCertIssuer(d *caddyfile.Dispenser, existing interface{}) (interface
 	return issuers, nil
 }
 
-func parseOptSingleString(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptSingleString(d *caddyfile.Dispenser, _ any) (any, error) {
 	d.Next() // consume parameter name
 	if !d.Next() {
 		return "", d.ArgErr()
@@ -279,7 +281,16 @@ func parseOptSingleString(d *caddyfile.Dispenser, _ interface{}) (interface{}, e
 	return val, nil
 }
 
-func parseOptAdmin(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptStringList(d *caddyfile.Dispenser, _ any) (any, error) {
+	d.Next() // consume parameter name
+	val := d.RemainingArgs()
+	if len(val) == 0 {
+		return "", d.ArgErr()
+	}
+	return val, nil
+}
+
+func parseOptAdmin(d *caddyfile.Dispenser, _ any) (any, error) {
 	adminCfg := new(caddy.AdminConfig)
 	for d.Next() {
 		if d.NextArg() {
@@ -315,7 +326,7 @@ func parseOptAdmin(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
 	return adminCfg, nil
 }
 
-func parseOptOnDemand(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptOnDemand(d *caddyfile.Dispenser, _ any) (any, error) {
 	var ond *caddytls.OnDemandConfig
 	for d.Next() {
 		if d.NextArg() {
@@ -375,7 +386,7 @@ func parseOptOnDemand(d *caddyfile.Dispenser, _ interface{}) (interface{}, error
 	return ond, nil
 }
 
-func parseOptAutoHTTPS(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptAutoHTTPS(d *caddyfile.Dispenser, _ any) (any, error) {
 	d.Next() // consume parameter name
 	if !d.Next() {
 		return "", d.ArgErr()
@@ -390,11 +401,11 @@ func parseOptAutoHTTPS(d *caddyfile.Dispenser, _ interface{}) (interface{}, erro
 	return val, nil
 }
 
-func parseServerOptions(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseServerOptions(d *caddyfile.Dispenser, _ any) (any, error) {
 	return unmarshalCaddyfileServerOptions(d)
 }
 
-func parseOCSPStaplingOptions(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOCSPStaplingOptions(d *caddyfile.Dispenser, _ any) (any, error) {
 	d.Next() // consume option name
 	var val string
 	if !d.AllArgs(&val) {
@@ -410,18 +421,17 @@ func parseOCSPStaplingOptions(d *caddyfile.Dispenser, _ interface{}) (interface{
 
 // parseLogOptions parses the global log option. Syntax:
 //
-//     log [name] {
-//         output  <writer_module> ...
-//         format  <encoder_module> ...
-//         level   <level>
-//         include <namespaces...>
-//         exclude <namespaces...>
-//     }
+//	log [name] {
+//	    output  <writer_module> ...
+//	    format  <encoder_module> ...
+//	    level   <level>
+//	    include <namespaces...>
+//	    exclude <namespaces...>
+//	}
 //
 // When the name argument is unspecified, this directive modifies the default
 // logger.
-//
-func parseLogOptions(d *caddyfile.Dispenser, existingVal interface{}) (interface{}, error) {
+func parseLogOptions(d *caddyfile.Dispenser, existingVal any) (any, error) {
 	currentNames := make(map[string]struct{})
 	if existingVal != nil {
 		innerVals, ok := existingVal.([]ConfigValue)
@@ -456,7 +466,7 @@ func parseLogOptions(d *caddyfile.Dispenser, existingVal interface{}) (interface
 	return configValues, nil
 }
 
-func parseOptPreferredChains(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptPreferredChains(d *caddyfile.Dispenser, _ any) (any, error) {
 	d.Next()
 	return caddytls.ParseCaddyfilePreferredChainsOptions(d)
 }

caddyconfig/httpcaddyfile/pkiapp.go

@@ -45,8 +45,7 @@ func init() {
 //     }
 //
 // When the CA ID is unspecified, 'local' is assumed.
-//
-func parsePKIApp(d *caddyfile.Dispenser, existingVal interface{}) (interface{}, error) {
+func parsePKIApp(d *caddyfile.Dispenser, existingVal any) (any, error) {
 	pki := &caddypki.PKI{CAs: make(map[string]*caddypki.CA)}
 
 	for d.Next() {
@@ -160,7 +159,7 @@ func parsePKIApp(d *caddyfile.Dispenser, existingVal interface{}) (interface{},
 
 func (st ServerType) buildPKIApp(
 	pairings []sbAddrAssociation,
-	options map[string]interface{},
+	options map[string]any,
 	warnings []caddyconfig.Warning,
 ) (*caddypki.PKI, []caddyconfig.Warning, error) {
 

caddyconfig/httpcaddyfile/serveroptions.go

@@ -38,14 +38,15 @@ type serverOptions struct {
 	ReadHeaderTimeout    caddy.Duration
 	WriteTimeout         caddy.Duration
 	IdleTimeout          caddy.Duration
+	KeepAliveInterval    caddy.Duration
 	MaxHeaderBytes       int
-	AllowH2C             bool
-	ExperimentalHTTP3    bool
+	Protocols            []string
 	StrictSNIHost        *bool
 	ShouldLogCredentials bool
+	Metrics              *caddyhttp.Metrics
 }
 
-func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error) {
+func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (any, error) {
 	serverOpts := serverOptions{}
 	for d.Next() {
 		if d.NextArg() {
@@ -123,6 +124,15 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error
 						return nil, d.Errf("unrecognized timeouts option '%s'", d.Val())
 					}
 				}
+			case "keepalive_interval":
+				if !d.NextArg() {
+					return nil, d.ArgErr()
+				}
+				dur, err := caddy.ParseDuration(d.Val())
+				if err != nil {
+					return nil, d.Errf("parsing keepalive interval duration: %v", err)
+				}
+				serverOpts.KeepAliveInterval = caddy.Duration(dur)
 
 			case "max_header_size":
 				var sizeStr string
@@ -141,22 +151,60 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error
 				}
 				serverOpts.ShouldLogCredentials = true
 
+			case "protocols":
+				protos := d.RemainingArgs()
+				for _, proto := range protos {
+					if proto != "h1" && proto != "h2" && proto != "h2c" && proto != "h3" {
+						return nil, d.Errf("unknown protocol '%s': expected h1, h2, h2c, or h3", proto)
+					}
+					if sliceContains(serverOpts.Protocols, proto) {
+						return nil, d.Errf("protocol %s specified more than once", proto)
+					}
+					serverOpts.Protocols = append(serverOpts.Protocols, proto)
+				}
+				if d.NextBlock(0) {
+					return nil, d.ArgErr()
+				}
+
+			case "strict_sni_host":
+				if d.NextArg() && d.Val() != "insecure_off" && d.Val() != "on" {
+					return nil, d.Errf("strict_sni_host only supports 'on' or 'insecure_off', got '%s'", d.Val())
+				}
+				boolVal := true
+				if d.Val() == "insecure_off" {
+					boolVal = false
+				}
+				serverOpts.StrictSNIHost = &boolVal
+
+			case "metrics":
+				if d.NextArg() {
+					return nil, d.ArgErr()
+				}
+				if d.NextBlock(0) {
+					return nil, d.ArgErr()
+				}
+				serverOpts.Metrics = new(caddyhttp.Metrics)
+
+			// TODO: DEPRECATED. (August 2022)
 			case "protocol":
+				caddy.Log().Named("caddyfile").Warn("DEPRECATED: protocol sub-option will be removed soon")
+
 				for nesting := d.Nesting(); d.NextBlock(nesting); {
 					switch d.Val() {
 					case "allow_h2c":
-						if d.NextArg() {
-							return nil, d.ArgErr()
-						}
-						serverOpts.AllowH2C = true
+						caddy.Log().Named("caddyfile").Warn("DEPRECATED: allow_h2c will be removed soon; use protocols option instead")
 
-					case "experimental_http3":
 						if d.NextArg() {
 							return nil, d.ArgErr()
 						}
-						serverOpts.ExperimentalHTTP3 = true
+						if sliceContains(serverOpts.Protocols, "h2c") {
+							return nil, d.Errf("protocol h2c already specified")
+						}
+						serverOpts.Protocols = append(serverOpts.Protocols, "h2c")
 
 					case "strict_sni_host":
+						caddy.Log().Named("caddyfile").Warn("DEPRECATED: protocol > strict_sni_host in this position will be removed soon; move up to the servers block instead")
+
 						if d.NextArg() && d.Val() != "insecure_off" && d.Val() != "on" {
 							return nil, d.Errf("strict_sni_host only supports 'on' or 'insecure_off', got '%s'", d.Val())
 						}
@@ -182,20 +230,9 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error
 // applyServerOptions sets the server options on the appropriate servers
 func applyServerOptions(
 	servers map[string]*caddyhttp.Server,
-	options map[string]interface{},
+	options map[string]any,
 	warnings *[]caddyconfig.Warning,
 ) error {
-	// If experimental HTTP/3 is enabled, enable it on each server.
-	// We already know there won't be a conflict with serverOptions because
-	// we validated earlier that "experimental_http3" cannot be set at the same
-	// time as "servers"
-	if enableH3, ok := options["experimental_http3"].(bool); ok && enableH3 {
-		*warnings = append(*warnings, caddyconfig.Warning{Message: "the 'experimental_http3' global option is deprecated, please use the 'servers > protocol > experimental_http3' option instead"})
-		for _, srv := range servers {
-			srv.ExperimentalHTTP3 = true
-		}
-	}
-
 	serverOpts, ok := options["servers"].([]serverOptions)
 	if !ok {
 		return nil
@@ -228,10 +265,11 @@ func applyServerOptions(
 		server.ReadHeaderTimeout = opts.ReadHeaderTimeout
 		server.WriteTimeout = opts.WriteTimeout
 		server.IdleTimeout = opts.IdleTimeout
+		server.KeepAliveInterval = opts.KeepAliveInterval
 		server.MaxHeaderBytes = opts.MaxHeaderBytes
-		server.AllowH2C = opts.AllowH2C
-		server.ExperimentalHTTP3 = opts.ExperimentalHTTP3
+		server.Protocols = opts.Protocols
 		server.StrictSNIHost = opts.StrictSNIHost
+		server.Metrics = opts.Metrics
 		if opts.ShouldLogCredentials {
 			if server.Logs == nil {
 				server.Logs = &caddyhttp.ServerLogConfig{}

caddyconfig/httpcaddyfile/tlsapp.go

@@ -33,7 +33,7 @@ import (
 
 func (st ServerType) buildTLSApp(
 	pairings []sbAddrAssociation,
-	options map[string]interface{},
+	options map[string]any,
 	warnings []caddyconfig.Warning,
 ) (*caddytls.TLS, []caddyconfig.Warning, error) {
 
@@ -307,6 +307,14 @@ func (st ServerType) buildTLSApp(
 		tlsApp.Automation.RenewCheckInterval = renewCheckInterval
 	}
 
+	// set the OCSP check interval if configured
+	if ocspCheckInterval, ok := options["ocsp_interval"].(caddy.Duration); ok {
+		if tlsApp.Automation == nil {
+			tlsApp.Automation = new(caddytls.AutomationConfig)
+		}
+		tlsApp.Automation.OCSPCheckInterval = ocspCheckInterval
+	}
+
 	// set whether OCSP stapling should be disabled for manually-managed certificates
 	if ocspConfig, ok := options["ocsp_stapling"].(certmagic.OCSPConfig); ok {
 		tlsApp.DisableOCSPStapling = ocspConfig.DisableStapling
@@ -350,7 +358,6 @@ func (st ServerType) buildTLSApp(
 		globalPreferredChains := options["preferred_chains"]
 		hasGlobalACMEDefaults := globalEmail != nil || globalACMECA != nil || globalACMECARoot != nil || globalACMEDNS != nil || globalACMEEAB != nil || globalPreferredChains != nil
 		if hasGlobalACMEDefaults {
-			// for _, ap := range tlsApp.Automation.Policies {
 			for i := 0; i < len(tlsApp.Automation.Policies); i++ {
 				ap := tlsApp.Automation.Policies[i]
 				if len(ap.Issuers) == 0 && automationPolicyHasAllPublicNames(ap) {
@@ -421,7 +428,7 @@ func (st ServerType) buildTLSApp(
 
 type acmeCapable interface{ GetACMEIssuer() *caddytls.ACMEIssuer }
 
-func fillInGlobalACMEDefaults(issuer certmagic.Issuer, options map[string]interface{}) error {
+func fillInGlobalACMEDefaults(issuer certmagic.Issuer, options map[string]any) error {
 	acmeWrapper, ok := issuer.(acmeCapable)
 	if !ok {
 		return nil
@@ -468,7 +475,7 @@ func fillInGlobalACMEDefaults(issuer certmagic.Issuer, options map[string]interf
 // for any other automation policies. A nil policy (and no error) will be
 // returned if there are no default/global options. However, if always is
 // true, a non-nil value will always be returned (unless there is an error).
-func newBaseAutomationPolicy(options map[string]interface{}, warnings []caddyconfig.Warning, always bool) (*caddytls.AutomationPolicy, error) {
+func newBaseAutomationPolicy(options map[string]any, warnings []caddyconfig.Warning, always bool) (*caddytls.AutomationPolicy, error) {
 	issuers, hasIssuers := options["cert_issuer"]
 	_, hasLocalCerts := options["local_certs"]
 	keyType, hasKeyType := options["key_type"]

caddyconfig/httploader.go

@@ -113,7 +113,7 @@ func (hl HTTPLoader) LoadConfig(ctx caddy.Context) ([]byte, error) {
 		return nil, err
 	}
 	for _, warn := range warnings {
-		ctx.Logger(hl).Warn(warn.String())
+		ctx.Logger().Warn(warn.String())
 	}
 
 	return result, nil
@@ -129,7 +129,7 @@ func (hl HTTPLoader) makeClient(ctx caddy.Context) (*http.Client, error) {
 
 		// client authentication
 		if hl.TLS.UseServerIdentity {
-			certs, err := ctx.IdentityCredentials(ctx.Logger(hl))
+			certs, err := ctx.IdentityCredentials(ctx.Logger())
 			if err != nil {
 				return nil, fmt.Errorf("getting server identity credentials: %v", err)
 			}

caddyconfig/load.go

@@ -58,6 +58,10 @@ func (al adminLoad) Routes() []caddy.AdminRoute {
 			Pattern: "/load",
 			Handler: caddy.AdminHandlerFunc(al.handleLoad),
 		},
+		{
+			Pattern: "/adapt",
+			Handler: caddy.AdminHandlerFunc(al.handleAdapt),
+		},
 	}
 }
 
@@ -122,7 +126,48 @@ func (adminLoad) handleLoad(w http.ResponseWriter, r *http.Request) error {
 	return nil
 }
 
-// adaptByContentType adapts body to Caddy JSON using the adapter specified by contenType.
+// handleAdapt adapts the given Caddy config to JSON and responds with the result.
+func (adminLoad) handleAdapt(w http.ResponseWriter, r *http.Request) error {
+	if r.Method != http.MethodPost {
+		return caddy.APIError{
+			HTTPStatus: http.StatusMethodNotAllowed,
+			Err:        fmt.Errorf("method not allowed"),
+		}
+	}
+
+	buf := bufPool.Get().(*bytes.Buffer)
+	buf.Reset()
+	defer bufPool.Put(buf)
+
+	_, err := io.Copy(buf, r.Body)
+	if err != nil {
+		return caddy.APIError{
+			HTTPStatus: http.StatusBadRequest,
+			Err:        fmt.Errorf("reading request body: %v", err),
+		}
+	}
+
+	result, warnings, err := adaptByContentType(r.Header.Get("Content-Type"), buf.Bytes())
+	if err != nil {
+		return caddy.APIError{
+			HTTPStatus: http.StatusBadRequest,
+			Err:        err,
+		}
+	}
+
+	out := struct {
+		Warnings []Warning       `json:"warnings,omitempty"`
+		Result   json.RawMessage `json:"result"`
+	}{
+		Warnings: warnings,
+		Result:   result,
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	return json.NewEncoder(w).Encode(out)
+}
+
+// adaptByContentType adapts body to Caddy JSON using the adapter specified by contentType.
 // If contentType is empty or ends with "/json", the input will be returned, as a no-op.
 func adaptByContentType(contentType string, body []byte) ([]byte, []Warning, error) {
 	// assume JSON as the default
@@ -144,12 +189,11 @@ func adaptByContentType(contentType string, body []byte) ([]byte, []Warning, err
 	}
 
 	// adapter name should be suffix of MIME type
-	slashIdx := strings.Index(ct, "/")
-	if slashIdx < 0 {
+	_, adapterName, slashFound := strings.Cut(ct, "/")
+	if !slashFound {
 		return nil, nil, fmt.Errorf("malformed Content-Type")
 	}
 
-	adapterName := ct[slashIdx+1:]
 	cfgAdapter := GetAdapter(adapterName)
 	if cfgAdapter == nil {
 		return nil, nil, fmt.Errorf("unrecognized config adapter '%s'", adapterName)
@@ -164,7 +208,7 @@ func adaptByContentType(contentType string, body []byte) ([]byte, []Warning, err
 }
 
 var bufPool = sync.Pool{
-	New: func() interface{} {
+	New: func() any {
 		return new(bytes.Buffer)
 	},
 }

caddytest/caddytest.go

@@ -100,7 +100,7 @@ func (tc *Tester) InitServer(rawConfig string, configType string) {
 		tc.t.Fail()
 	}
 	if err := tc.ensureConfigRunning(rawConfig, configType); err != nil {
-		tc.t.Logf("failed ensurng config is running: %s", err)
+		tc.t.Logf("failed ensuring config is running: %s", err)
 		tc.t.Fail()
 	}
 }
@@ -186,7 +186,7 @@ func (tc *Tester) ensureConfigRunning(rawConfig string, configType string) error
 		expectedBytes, _, _ = adapter.Adapt([]byte(rawConfig), nil)
 	}
 
-	var expected interface{}
+	var expected any
 	err := json.Unmarshal(expectedBytes, &expected)
 	if err != nil {
 		return err
@@ -196,7 +196,7 @@ func (tc *Tester) ensureConfigRunning(rawConfig string, configType string) error
 		Timeout: Default.LoadRequestTimeout,
 	}
 
-	fetchConfig := func(client *http.Client) interface{} {
+	fetchConfig := func(client *http.Client) any {
 		resp, err := client.Get(fmt.Sprintf("http://localhost:%d/config/", Default.AdminPort))
 		if err != nil {
 			return nil
@@ -206,7 +206,7 @@ func (tc *Tester) ensureConfigRunning(rawConfig string, configType string) error
 		if err != nil {
 			return nil
 		}
-		var actual interface{}
+		var actual any
 		err = json.Unmarshal(actualBytes, &actual)
 		if err != nil {
 			return nil
@@ -371,7 +371,7 @@ func CompareAdapt(t *testing.T, filename, rawConfig string, adapterName string,
 		return false
 	}
 
-	options := make(map[string]interface{})
+	options := make(map[string]any)
 
 	result, warnings, err := cfgAdapter.Adapt([]byte(rawConfig), options)
 	if err != nil {

caddytest/integration/caddyfile_adapt/forward_auth_authelia.txt

@@ -63,32 +63,6 @@ app.example.com {
 																	]
 																}
 															]
-														},
-														{
-															"routes": [
-																{
-																	"handle": [
-																		{
-																			"exclude": [
-																				"Connection",
-																				"Keep-Alive",
-																				"Te",
-																				"Trailers",
-																				"Transfer-Encoding",
-																				"Upgrade"
-																			],
-																			"handler": "copy_response_headers"
-																		}
-																	]
-																},
-																{
-																	"handle": [
-																		{
-																			"handler": "copy_response"
-																		}
-																	]
-																}
-															]
 														}
 													],
 													"handler": "reverse_proxy",

caddytest/integration/caddyfile_adapt/forward_auth_rename_headers.txt

@@ -0,0 +1,90 @@
+:8881
+
+forward_auth localhost:9000 {
+	uri /auth
+	copy_headers A>1 B C>3 {
+		D
+		E>5
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8881"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handle_response": [
+										{
+											"match": {
+												"status_code": [
+													2
+												]
+											},
+											"routes": [
+												{
+													"handle": [
+														{
+															"handler": "headers",
+															"request": {
+																"set": {
+																	"1": [
+																		"{http.reverse_proxy.header.A}"
+																	],
+																	"3": [
+																		"{http.reverse_proxy.header.C}"
+																	],
+																	"5": [
+																		"{http.reverse_proxy.header.E}"
+																	],
+																	"B": [
+																		"{http.reverse_proxy.header.B}"
+																	],
+																	"D": [
+																		"{http.reverse_proxy.header.D}"
+																	]
+																}
+															}
+														}
+													]
+												}
+											]
+										}
+									],
+									"handler": "reverse_proxy",
+									"headers": {
+										"request": {
+											"set": {
+												"X-Forwarded-Method": [
+													"{http.request.method}"
+												],
+												"X-Forwarded-Uri": [
+													"{http.request.uri}"
+												]
+											}
+										}
+									},
+									"rewrite": {
+										"method": "GET",
+										"uri": "/auth"
+									},
+									"upstreams": [
+										{
+											"dial": "localhost:9000"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options.txt

@@ -3,6 +3,7 @@
 	http_port 8080
 	https_port 8443
 	grace_period 5s
+	shutdown_delay 10s
 	default_sni localhost
 	order root first
 	storage file_system {
@@ -45,6 +46,7 @@
 			"http_port": 8080,
 			"https_port": 8443,
 			"grace_period": 5000000000,
+			"shutdown_delay": 10000000000,
 			"servers": {
 				"srv0": {
 					"listen": [

caddytest/integration/caddyfile_adapt/global_options_acme.txt

@@ -22,6 +22,7 @@
 	}
 	storage_clean_interval 7d
 	renew_interval 1d
+	ocsp_interval 2d
 
 	key_type ed25519
 }
@@ -83,6 +84,7 @@
 					},
 					"ask": "https://example.com"
 				},
+				"ocsp_interval": 172800000000000,
 				"renew_interval": 86400000000000,
 				"storage_clean_interval": 604800000000000
 			}

caddytest/integration/caddyfile_adapt/global_options_default_bind.txt

@@ -1,5 +1,5 @@
 {
-	default_bind tcp4/0.0.0.0
+	default_bind tcp4/0.0.0.0 tcp6/[::]
 }
 
 example.com {
@@ -14,7 +14,8 @@ example.org:12345 {
 			"servers": {
 				"srv0": {
 					"listen": [
-						"tcp4/0.0.0.0:12345"
+						"tcp4/0.0.0.0:12345",
+						"tcp6/[::]:12345"
 					],
 					"routes": [
 						{
@@ -31,7 +32,8 @@ example.org:12345 {
 				},
 				"srv1": {
 					"listen": [
-						"tcp4/0.0.0.0:443"
+						"tcp4/0.0.0.0:443",
+						"tcp6/[::]:443"
 					],
 					"routes": [
 						{

caddytest/integration/caddyfile_adapt/global_server_options_single.txt

@@ -12,11 +12,8 @@
 		}
 		max_header_size 100MB
 		log_credentials
-		protocol {
-			allow_h2c
-			experimental_http3
-			strict_sni_host
-		}
+		strict_sni_host
+		protocols h1 h2 h2c h3
 	}
 }
 
@@ -61,8 +58,12 @@ foo.com {
 					"logs": {
 						"should_log_credentials": true
 					},
-					"experimental_http3": true,
-					"allow_h2c": true
+					"protocols": [
+						"h1",
+						"h2",
+						"h2c",
+						"h3"
+					]
 				}
 			}
 		}

caddytest/integration/caddyfile_adapt/log_except_catchall_blocks.txt

@@ -1,5 +1,7 @@
 http://localhost:2020 {
 	log
+	skip_log /first-hidden*
+	skip_log /second-hidden*
 	respond 200
 }
 
@@ -28,6 +30,36 @@ http://localhost:2020 {
 								{
 									"handler": "subroute",
 									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "vars",
+													"skip_log": true
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/second-hidden*"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"handler": "vars",
+													"skip_log": true
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/first-hidden*"
+													]
+												}
+											]
+										},
 										{
 											"handle": [
 												{

caddytest/integration/caddyfile_adapt/matcher_syntax.txt

@@ -19,27 +19,30 @@
 	@matcher6 vars_regexp "{http.request.uri}" `\.([a-f0-9]{6})\.(css|js)$`
 	respond @matcher6 "from vars_regexp matcher without name"
 
-	@matcher7 {
+	@matcher7 `path('/foo*') && method('GET')`
+	respond @matcher7 "inline expression matcher shortcut"
+
+	@matcher8 {
 		header Foo bar
 		header Foo foobar
 		header Bar foo
 	}
-	respond @matcher7 "header matcher merging values of the same field"
+	respond @matcher8 "header matcher merging values of the same field"
 
-	@matcher8 {
+	@matcher9 {
 		query foo=bar foo=baz bar=foo
 		query bar=baz
 	}
-	respond @matcher8 "query matcher merging pairs with the same keys"
+	respond @matcher9 "query matcher merging pairs with the same keys"
 
-	@matcher9 {
+	@matcher10 {
 		header !Foo
 		header Bar foo
 	}
-	respond @matcher9 "header matcher with null field matcher"
+	respond @matcher10 "header matcher with null field matcher"
 
-	@matcher10 remote_ip private_ranges
-	respond @matcher10 "remote_ip matcher with private ranges"
+	@matcher11 remote_ip private_ranges
+	respond @matcher11 "remote_ip matcher with private ranges"
 }
 ----------
 {
@@ -152,6 +155,19 @@
 								}
 							]
 						},
+						{
+							"match": [
+								{
+									"expression": "path('/foo*') \u0026\u0026 method('GET')"
+								}
+							],
+							"handle": [
+								{
+									"body": "inline expression matcher shortcut",
+									"handler": "static_response"
+								}
+							]
+						},
 						{
 							"match": [
 								{

caddytest/integration/caddyfile_adapt/reverse_proxy_h2c_shorthand.txt

@@ -1,6 +1,8 @@
 :8884
 
 reverse_proxy h2c://localhost:8080
+
+reverse_proxy unix+h2c//run/app.sock
 ----------
 {
 	"apps": {
@@ -27,6 +29,21 @@ reverse_proxy h2c://localhost:8080
 											"dial": "localhost:8080"
 										}
 									]
+								},
+								{
+									"handler": "reverse_proxy",
+									"transport": {
+										"protocol": "http",
+										"versions": [
+											"h2c",
+											"2"
+										]
+									},
+									"upstreams": [
+										{
+											"dial": "unix//run/app.sock"
+										}
+									]
 								}
 							]
 						}

caddytest/integration/caddyfile_adapt/reverse_proxy_load_balance.txt

@@ -0,0 +1,64 @@
+:8884
+
+reverse_proxy 127.0.0.1:65535 {
+	lb_policy first
+	lb_retries 5
+	lb_try_duration 10s
+	lb_try_interval 500ms
+	lb_retry_match {
+		path /foo*
+		method POST
+	}
+	lb_retry_match path /bar*
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "reverse_proxy",
+									"load_balancing": {
+										"retries": 5,
+										"retry_match": [
+											{
+												"method": [
+													"POST"
+												],
+												"path": [
+													"/foo*"
+												]
+											},
+											{
+												"path": [
+													"/bar*"
+												]
+											}
+										],
+										"selection_policy": {
+											"policy": "first"
+										},
+										"try_duration": 10000000000,
+										"try_interval": 500000000
+									},
+									"upstreams": [
+										{
+											"dial": "127.0.0.1:65535"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/reverse_proxy_options.txt

@@ -24,6 +24,9 @@ https://example.com {
 			max_conns_per_host 5
 			keepalive_idle_conns_per_host 2
 			keepalive_interval 30s
+			
+			tls_renegotiation freely
+			tls_except_ports 8181 8182
 		}
 	}
 }
@@ -91,7 +94,13 @@ https://example.com {
 															]
 														},
 														"response_header_timeout": 8000000000,
-														"tls": {},
+														"tls": {
+															"except_ports": [
+																"8181",
+																"8182"
+															],
+															"renegotiation": "freely"
+														},
 														"versions": [
 															"h2c",
 															"2"

caddytest/integration/caddyfile_test.go

@@ -68,7 +68,7 @@ func TestDuplicateHosts(t *testing.T) {
     }
     `,
 		"caddyfile",
-		"duplicate site address not allowed")
+		"ambiguous site definition")
 }
 
 func TestReadCookie(t *testing.T) {

caddytest/integration/map_test.go

@@ -60,7 +60,7 @@ func TestMapRespondWithDefault(t *testing.T) {
 	tester.AssertPostResponseBody("http://localhost:9080/version", []string{}, bytes.NewBuffer([]byte{}), 200, "hello from localhost unknown")
 }
 
-func TestMapAsJson(t *testing.T) {
+func TestMapAsJSON(t *testing.T) {
 	// arrange
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
@@ -85,7 +85,7 @@ func TestMapAsJson(t *testing.T) {
 													{
 														"handler": "map",
 														"source": "{http.request.method}",
-														"destinations": ["dest-name"],
+														"destinations": ["{dest-name}"],
 														"defaults": ["unknown"],
 														"mappings": [
 															{

caddytest/integration/stream_test.go

@@ -123,8 +123,8 @@ func TestH2ToH2CStream(t *testing.T) {
 	// Disable any compression method from server.
 	req.Header.Set("Accept-Encoding", "identity")
 
-	resp := tester.AssertResponseCode(req, 200)
-	if 200 != resp.StatusCode {
+	resp := tester.AssertResponseCode(req, http.StatusOK)
+	if resp.StatusCode != http.StatusOK {
 		return
 	}
 	go func() {
@@ -143,7 +143,6 @@ func TestH2ToH2CStream(t *testing.T) {
 	if !strings.Contains(body, expectedBody) {
 		t.Errorf("requesting \"%s\" expected response body \"%s\" but got \"%s\"", req.RequestURI, expectedBody, body)
 	}
-	return
 }
 
 func testH2ToH2CStreamServeH2C(t *testing.T) *http.Server {
@@ -335,8 +334,8 @@ func TestH2ToH1ChunkedResponse(t *testing.T) {
 		fmt.Fprint(w, expectedBody)
 		w.Close()
 	}()
-	resp := tester.AssertResponseCode(req, 200)
-	if 200 != resp.StatusCode {
+	resp := tester.AssertResponseCode(req, http.StatusOK)
+	if resp.StatusCode != http.StatusOK {
 		return
 	}
 
@@ -351,7 +350,6 @@ func TestH2ToH1ChunkedResponse(t *testing.T) {
 	if body != expectedBody {
 		t.Errorf("requesting \"%s\" expected response body \"%s\" but got \"%s\"", req.RequestURI, expectedBody, body)
 	}
-	return
 }
 
 func testH2ToH1ChunkedResponseServeH1(t *testing.T) *http.Server {

cmd/caddy/main.go

@@ -24,6 +24,8 @@
 //   3. Run `go mod init caddy`
 //   4. Run `go install` or `go build` - you now have a custom binary!
 //
+// Or you can use xcaddy which does it all for you as a command:
+// https://github.com/caddyserver/xcaddy
 package main
 
 import (

cmd/cobra.go

@@ -0,0 +1,120 @@
+package caddycmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var rootCmd = &cobra.Command{
+	Use: "caddy",
+	Long: `Caddy is an extensible server platform written in Go.
+
+At its core, Caddy merely manages configuration. Modules are plugged
+in statically at compile-time to provide useful functionality. Caddy's
+standard distribution includes common modules to serve HTTP, TLS,
+and PKI applications, including the automation of certificates.
+
+To run Caddy, use:
+
+	- 'caddy run' to run Caddy in the foreground (recommended).
+	- 'caddy start' to start Caddy in the background; only do this
+	  if you will be keeping the terminal window open until you run
+	  'caddy stop' to close the server.
+
+When Caddy is started, it opens a locally-bound administrative socket
+to which configuration can be POSTed via a restful HTTP API (see
+https://caddyserver.com/docs/api).
+
+Caddy's native configuration format is JSON. However, config adapters
+can be used to convert other config formats to JSON when Caddy receives
+its configuration. The Caddyfile is a built-in config adapter that is
+popular for hand-written configurations due to its straightforward
+syntax (see https://caddyserver.com/docs/caddyfile). Many third-party
+adapters are available (see https://caddyserver.com/docs/config-adapters).
+Use 'caddy adapt' to see how a config translates to JSON.
+
+For convenience, the CLI can act as an HTTP client to give Caddy its
+initial configuration for you. If a file named Caddyfile is in the
+current working directory, it will do this automatically. Otherwise,
+you can use the --config flag to specify the path to a config file.
+
+Some special-purpose subcommands build and load a configuration file
+for you directly from command line input; for example:
+
+	- caddy file-server
+	- caddy reverse-proxy
+	- caddy respond
+
+These commands disable the administration endpoint because their
+configuration is specified solely on the command line.
+
+In general, the most common way to run Caddy is simply:
+
+	$ caddy run
+
+Or, with a configuration file:
+
+	$ caddy run --config caddy.json
+
+If running interactively in a terminal, running Caddy in the
+background may be more convenient:
+
+	$ caddy start
+	...
+	$ caddy stop
+
+This allows you to run other commands while Caddy stays running.
+Be sure to stop Caddy before you close the terminal!
+
+Depending on the system, Caddy may need permission to bind to low
+ports. One way to do this on Linux is to use setcap:
+
+	$ sudo setcap cap_net_bind_service=+ep $(which caddy)
+
+Remember to run that command again after replacing the binary.
+
+See the Caddy website for tutorials, configuration structure,
+syntax, and module documentation: https://caddyserver.com/docs/
+
+Custom Caddy builds are available on the Caddy download page at:
+https://caddyserver.com/download
+
+The xcaddy command can be used to build Caddy from source with or
+without additional plugins: https://github.com/caddyserver/xcaddy
+
+Where possible, Caddy should be installed using officially-supported
+package installers: https://caddyserver.com/docs/install
+
+Instructions for running Caddy in production are also available:
+https://caddyserver.com/docs/running
+`,
+	Example: `  $ caddy run
+  $ caddy run --config caddy.json
+  $ caddy reload --config caddy.json
+  $ caddy stop`,
+
+	// kind of annoying to have all the help text printed out if
+	// caddy has an error provisioning its modules, for instance...
+	SilenceUsage: true,
+}
+
+const fullDocsFooter = `Full documentation is available at:
+https://caddyserver.com/docs/command-line`
+
+func init() {
+	rootCmd.SetHelpTemplate(rootCmd.HelpTemplate() + "\n" + fullDocsFooter)
+}
+
+func caddyCmdToCoral(caddyCmd Command) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   caddyCmd.Name,
+		Short: caddyCmd.Short,
+		Long:  caddyCmd.Long,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			fls := cmd.Flags()
+			_, err := caddyCmd.Func(Flags{fls})
+			return err
+		},
+	}
+	cmd.Flags().AddGoFlagSet(caddyCmd.Flags)
+	return cmd
+}

cmd/commandfuncs.go

@@ -29,7 +29,6 @@ import (
 	"os/exec"
 	"runtime"
 	"runtime/debug"
-	"sort"
 	"strings"
 
 	"github.com/aryann/difflib"
@@ -280,7 +279,7 @@ func cmdStop(fl Flags) (int, error) {
 	configFlag := fl.String("config")
 	configAdapterFlag := fl.String("adapter")
 
-	adminAddr, err := DetermineAdminAPIAddress(addrFlag, configFlag, configAdapterFlag)
+	adminAddr, err := DetermineAdminAPIAddress(addrFlag, nil, configFlag, configAdapterFlag)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("couldn't determine admin API address: %v", err)
 	}
@@ -310,7 +309,7 @@ func cmdReload(fl Flags) (int, error) {
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("no config file to load")
 	}
 
-	adminAddr, err := DetermineAdminAPIAddress(addrFlag, configFlag, configAdapterFlag)
+	adminAddr, err := DetermineAdminAPIAddress(addrFlag, config, configFlag, configAdapterFlag)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("couldn't determine admin API address: %v", err)
 	}
@@ -331,30 +330,17 @@ func cmdReload(fl Flags) (int, error) {
 }
 
 func cmdVersion(_ Flags) (int, error) {
-	fmt.Println(CaddyVersion())
+	_, full := caddy.Version()
+	fmt.Println(full)
 	return caddy.ExitCodeSuccess, nil
 }
 
-func cmdBuildInfo(fl Flags) (int, error) {
+func cmdBuildInfo(_ Flags) (int, error) {
 	bi, ok := debug.ReadBuildInfo()
 	if !ok {
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("no build information")
 	}
-
-	fmt.Printf("go_version: %s\n", runtime.Version())
-	fmt.Printf("go_os:      %s\n", runtime.GOOS)
-	fmt.Printf("go_arch:    %s\n", runtime.GOARCH)
-	fmt.Printf("path:       %s\n", bi.Path)
-	fmt.Printf("main:       %s %s %s\n", bi.Main.Path, bi.Main.Version, bi.Main.Sum)
-	fmt.Println("dependencies:")
-
-	for _, goMod := range bi.Deps {
-		fmt.Printf("%s %s %s", goMod.Path, goMod.Version, goMod.Sum)
-		if goMod.Replace != nil {
-			fmt.Printf(" => %s %s %s", goMod.Replace.Path, goMod.Replace.Version, goMod.Replace.Sum)
-		}
-		fmt.Println()
-	}
+	fmt.Println(bi)
 	return caddy.ExitCodeSuccess, nil
 }
 
@@ -471,7 +457,7 @@ func cmdAdaptConfig(fl Flags) (int, error) {
 			fmt.Errorf("reading input file: %v", err)
 	}
 
-	opts := map[string]interface{}{"filename": adaptCmdInputFlag}
+	opts := map[string]any{"filename": adaptCmdInputFlag}
 
 	adaptedConfig, warnings, err := cfgAdapter.Adapt(input, opts)
 	if err != nil {
@@ -593,70 +579,6 @@ func cmdFmt(fl Flags) (int, error) {
 	return caddy.ExitCodeSuccess, nil
 }
 
-func cmdHelp(fl Flags) (int, error) {
-	const fullDocs = `Full documentation is available at:
-https://caddyserver.com/docs/command-line`
-
-	args := fl.Args()
-	if len(args) == 0 {
-		s := `Caddy is an extensible server platform.
-
-usage:
-  caddy <command> [<args...>]
-
-commands:
-`
-		keys := make([]string, 0, len(commands))
-		for k := range commands {
-			keys = append(keys, k)
-		}
-		sort.Strings(keys)
-		for _, k := range keys {
-			cmd := commands[k]
-			short := strings.TrimSuffix(cmd.Short, ".")
-			s += fmt.Sprintf("  %-15s %s\n", cmd.Name, short)
-		}
-
-		s += "\nUse 'caddy help <command>' for more information about a command.\n"
-		s += "\n" + fullDocs + "\n"
-
-		fmt.Print(s)
-
-		return caddy.ExitCodeSuccess, nil
-	} else if len(args) > 1 {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("can only give help with one command")
-	}
-
-	subcommand, ok := commands[args[0]]
-	if !ok {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("unknown command: %s", args[0])
-	}
-
-	helpText := strings.TrimSpace(subcommand.Long)
-	if helpText == "" {
-		helpText = subcommand.Short
-		if !strings.HasSuffix(helpText, ".") {
-			helpText += "."
-		}
-	}
-
-	result := fmt.Sprintf("%s\n\nusage:\n  caddy %s %s\n",
-		helpText,
-		subcommand.Name,
-		strings.TrimSpace(subcommand.Usage),
-	)
-
-	if help := flagHelp(subcommand.Flags); help != "" {
-		result += fmt.Sprintf("\nflags:\n%s", help)
-	}
-
-	result += "\n" + fullDocs + "\n"
-
-	fmt.Print(result)
-
-	return caddy.ExitCodeSuccess, nil
-}
-
 // AdminAPIRequest makes an API request according to the CLI flags given,
 // with the given HTTP method and request URI. If body is non-nil, it will
 // be assumed to be Content-Type application/json. The caller should close
@@ -732,10 +654,11 @@ func AdminAPIRequest(adminAddr, method, uri string, headers http.Header, body io
 
 // DetermineAdminAPIAddress determines which admin API endpoint address should
 // be used based on the inputs. By priority: if `address` is specified, then
-// it is returned; if `configFile` (and `configAdapter`) are specified, then that
-// config will be loaded to find the admin address; otherwise, the default
-// admin listen address will be returned.
-func DetermineAdminAPIAddress(address, configFile, configAdapter string) (string, error) {
+// it is returned; if `config` is specified, then that config will be used for
+// finding the admin address; if `configFile` (and `configAdapter`) are specified,
+// then that config will be loaded to find the admin address; otherwise, the
+// default admin listen address will be returned.
+func DetermineAdminAPIAddress(address string, config []byte, configFile, configAdapter string) (string, error) {
 	// Prefer the address if specified and non-empty
 	if address != "" {
 		return address, nil
@@ -743,21 +666,29 @@ func DetermineAdminAPIAddress(address, configFile, configAdapter string) (string
 
 	// Try to load the config from file if specified, with the given adapter name
 	if configFile != "" {
-		// get the config in caddy's native format
-		config, loadedConfigFile, err := LoadConfig(configFile, configAdapter)
-		if err != nil {
-			return "", err
-		}
-		if loadedConfigFile == "" {
-			return "", fmt.Errorf("no config file to load")
+		var loadedConfigFile string
+		var err error
+
+		// use the provided loaded config if non-empty
+		// otherwise, load it from the specified file/adapter
+		loadedConfig := config
+		if len(loadedConfig) == 0 {
+			// get the config in caddy's native format
+			loadedConfig, loadedConfigFile, err = LoadConfig(configFile, configAdapter)
+			if err != nil {
+				return "", err
+			}
+			if loadedConfigFile == "" {
+				return "", fmt.Errorf("no config file to load; either use --config flag or ensure Caddyfile exists in current directory")
+			}
 		}
 
-		// get the address of the admin listener if set
-		if len(config) > 0 {
+		// get the address of the admin listener from the config
+		if len(loadedConfig) > 0 {
 			var tmpStruct struct {
 				Admin caddy.AdminConfig `json:"admin"`
 			}
-			err = json.Unmarshal(config, &tmpStruct)
+			err := json.Unmarshal(loadedConfig, &tmpStruct)
 			if err != nil {
 				return "", fmt.Errorf("unmarshaling admin listener address from config: %v", err)
 			}

cmd/commands.go

@@ -16,7 +16,14 @@ package caddycmd
 
 import (
 	"flag"
+	"fmt"
+	"os"
 	"regexp"
+	"strings"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/spf13/cobra"
+	"github.com/spf13/cobra/doc"
 )
 
 // Command represents a subcommand. Name, Func,
@@ -70,13 +77,6 @@ func Commands() map[string]Command {
 var commands = make(map[string]Command)
 
 func init() {
-	RegisterCommand(Command{
-		Name:  "help",
-		Func:  cmdHelp,
-		Usage: "<command>",
-		Short: "Shows help for a Caddy subcommand",
-	})
-
 	RegisterCommand(Command{
 		Name:  "start",
 		Func:  cmdStart,
@@ -137,8 +137,8 @@ The --resume flag will override the --config flag if there is a config auto-
 save file. It is not an error if --resume is used and no autosave file exists.
 
 If --watch is specified, the config file will be loaded automatically after
-changes. ⚠️ This is dangerous in production! Only use this option in a local
-development environment.`,
+changes. ⚠️ This can make unintentional config changes easier; only use this
+option in a local development environment.`,
 		Flags: func() *flag.FlagSet {
 			fs := flag.NewFlagSet("run", flag.ExitOnError)
 			fs.String("config", "", "Configuration file")
@@ -200,6 +200,19 @@ config file; otherwise the default is assumed.`,
 		Name:  "version",
 		Func:  cmdVersion,
 		Short: "Prints the version",
+		Long: `
+Prints the version of this Caddy binary.
+
+Version information must be embedded into the binary at compile-time in
+order for Caddy to display anything useful with this command. If Caddy
+is built from within a version control repository, the Go command will
+embed the revision hash if available. However, if Caddy is built in the
+way specified by our online documentation (or by using xcaddy), more
+detailed version information is printed as given by Go modules.
+
+For more details about the full version string, see the Go module
+documentation: https://go.dev/doc/modules/version-numbers
+`,
 	})
 
 	RegisterCommand(Command{
@@ -226,6 +239,24 @@ config file; otherwise the default is assumed.`,
 		Name:  "environ",
 		Func:  cmdEnviron,
 		Short: "Prints the environment",
+		Long: `
+Prints the environment as seen by this Caddy process.
+
+The environment includes variables set in the system. If your Caddy
+configuration uses environment variables (e.g. "{env.VARIABLE}") then
+this command can be useful for verifying that the variables will have
+the values you expect in your config.
+
+Note that environments may be different depending on how you run Caddy.
+Environments for Caddy instances started by service managers such as
+systemd are often different than the environment inherited from your
+shell or terminal.
+
+You can also print the environment the same time you use "caddy run"
+by adding the "--environ" flag.
+
+Environments may contain sensitive data.
+`,
 	})
 
 	RegisterCommand(Command{
@@ -346,16 +377,111 @@ EXPERIMENTAL: May be changed or removed.
 		}(),
 	})
 
+	RegisterCommand(Command{
+		Name: "manpage",
+		Func: func(fl Flags) (int, error) {
+			dir := strings.TrimSpace(fl.String("directory"))
+			if dir == "" {
+				return caddy.ExitCodeFailedQuit, fmt.Errorf("designated output directory and specified section are required")
+			}
+			if err := os.MkdirAll(dir, 0755); err != nil {
+				return caddy.ExitCodeFailedQuit, err
+			}
+			if err := doc.GenManTree(rootCmd, &doc.GenManHeader{
+				Title:   "Caddy",
+				Section: "8", // https://en.wikipedia.org/wiki/Man_page#Manual_sections
+			}, dir); err != nil {
+				return caddy.ExitCodeFailedQuit, err
+			}
+			return caddy.ExitCodeSuccess, nil
+		},
+		Usage: "--directory <path>",
+		Short: "Generates the manual pages for Caddy commands",
+		Long: `
+Generates the manual pages for Caddy commands into the designated directory
+tagged into section 8 (System Administration).
+
+The manual page files are generated into the directory specified by the
+argument of --directory. If the directory does not exist, it will be created.
+`,
+		Flags: func() *flag.FlagSet {
+			fs := flag.NewFlagSet("manpage", flag.ExitOnError)
+			fs.String("directory", "", "The output directory where the manpages are generated")
+			return fs
+		}(),
+	})
+
+	// source: https://github.com/spf13/cobra/blob/main/shell_completions.md
+	rootCmd.AddCommand(&cobra.Command{
+		Use:   "completion [bash|zsh|fish|powershell]",
+		Short: "Generate completion script",
+		Long: fmt.Sprintf(`To load completions:
+	
+	Bash:
+	
+	  $ source <(%[1]s completion bash)
+	
+	  # To load completions for each session, execute once:
+	  # Linux:
+	  $ %[1]s completion bash > /etc/bash_completion.d/%[1]s
+	  # macOS:
+	  $ %[1]s completion bash > $(brew --prefix)/etc/bash_completion.d/%[1]s
+	
+	Zsh:
+	
+	  # If shell completion is not already enabled in your environment,
+	  # you will need to enable it.  You can execute the following once:
+	
+	  $ echo "autoload -U compinit; compinit" >> ~/.zshrc
+	
+	  # To load completions for each session, execute once:
+	  $ %[1]s completion zsh > "${fpath[1]}/_%[1]s"
+	
+	  # You will need to start a new shell for this setup to take effect.
+	
+	fish:
+	
+	  $ %[1]s completion fish | source
+	
+	  # To load completions for each session, execute once:
+	  $ %[1]s completion fish > ~/.config/fish/completions/%[1]s.fish
+	
+	PowerShell:
+	
+	  PS> %[1]s completion powershell | Out-String | Invoke-Expression
+	
+	  # To load completions for every new session, run:
+	  PS> %[1]s completion powershell > %[1]s.ps1
+	  # and source this file from your PowerShell profile.
+	`, rootCmd.Root().Name()),
+		DisableFlagsInUseLine: true,
+		ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
+		Args:                  cobra.ExactValidArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			switch args[0] {
+			case "bash":
+				return cmd.Root().GenBashCompletion(os.Stdout)
+			case "zsh":
+				return cmd.Root().GenZshCompletion(os.Stdout)
+			case "fish":
+				return cmd.Root().GenFishCompletion(os.Stdout, true)
+			case "powershell":
+				return cmd.Root().GenPowerShellCompletionWithDesc(os.Stdout)
+			default:
+				return fmt.Errorf("unrecognized shell: %s", args[0])
+			}
+		},
+	})
 }
 
 // RegisterCommand registers the command cmd.
 // cmd.Name must be unique and conform to the
 // following format:
 //
-//    - lowercase
-//    - alphanumeric and hyphen characters only
-//    - cannot start or end with a hyphen
-//    - hyphen cannot be adjacent to another hyphen
+//   - lowercase
+//   - alphanumeric and hyphen characters only
+//   - cannot start or end with a hyphen
+//   - hyphen cannot be adjacent to another hyphen
 //
 // This function panics if the name is already registered,
 // if the name does not meet the described format, or if
@@ -378,7 +504,7 @@ func RegisterCommand(cmd Command) {
 	if !commandNameRegex.MatchString(cmd.Name) {
 		panic("invalid command name")
 	}
-	commands[cmd.Name] = cmd
+	rootCmd.AddCommand(caddyCmdToCoral(cmd))
 }
 
 var commandNameRegex = regexp.MustCompile(`^[a-z0-9]$|^([a-z0-9]+-?[a-z0-9]*)+[a-z0-9]$`)

cmd/main.go

@@ -33,60 +33,37 @@ import (
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"github.com/caddyserver/certmagic"
+	"github.com/spf13/pflag"
 	"go.uber.org/zap"
 )
 
 func init() {
 	// set a fitting User-Agent for ACME requests
-	goModule := caddy.GoModule()
-	cleanModVersion := strings.TrimPrefix(goModule.Version, "v")
-	certmagic.UserAgent = "Caddy/" + cleanModVersion
+	version, _ := caddy.Version()
+	cleanModVersion := strings.TrimPrefix(version, "v")
+	ua := "Caddy/" + cleanModVersion
+	if uaEnv, ok := os.LookupEnv("USERAGENT"); ok {
+		ua = uaEnv + " " + ua
+	}
+	certmagic.UserAgent = ua
 
 	// by using Caddy, user indicates agreement to CA terms
-	// (very important, or ACME account creation will fail!)
+	// (very important, as Caddy is often non-interactive
+	// and thus ACME account creation will fail!)
 	certmagic.DefaultACME.Agreed = true
 }
 
 // Main implements the main function of the caddy command.
 // Call this if Caddy is to be the main() of your program.
 func Main() {
-	switch len(os.Args) {
-	case 0:
+	if len(os.Args) == 0 {
 		fmt.Printf("[FATAL] no arguments provided by OS; args[0] must be command\n")
 		os.Exit(caddy.ExitCodeFailedStartup)
-	case 1:
-		os.Args = append(os.Args, "help")
 	}
 
-	subcommandName := os.Args[1]
-	subcommand, ok := commands[subcommandName]
-	if !ok {
-		if strings.HasPrefix(os.Args[1], "-") {
-			// user probably forgot to type the subcommand
-			fmt.Println("[ERROR] first argument must be a subcommand; see 'caddy help'")
-		} else {
-			fmt.Printf("[ERROR] '%s' is not a recognized subcommand; see 'caddy help'\n", os.Args[1])
-		}
-		os.Exit(caddy.ExitCodeFailedStartup)
+	if err := rootCmd.Execute(); err != nil {
+		os.Exit(1)
 	}
-
-	fs := subcommand.Flags
-	if fs == nil {
-		fs = flag.NewFlagSet(subcommand.Name, flag.ExitOnError)
-	}
-
-	err := fs.Parse(os.Args[2:])
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(caddy.ExitCodeFailedStartup)
-	}
-
-	exitCode, err := subcommand.Func(Flags{fs})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s: %v\n", subcommand.Name, err)
-	}
-
-	os.Exit(exitCode)
 }
 
 // handlePingbackConn reads from conn and ensures it matches
@@ -173,7 +150,7 @@ func LoadConfig(configFile, adapterName string) ([]byte, string, error) {
 
 	// adapt config
 	if cfgAdapter != nil {
-		adaptedConfig, warnings, err := cfgAdapter.Adapt(config, map[string]interface{}{
+		adaptedConfig, warnings, err := cfgAdapter.Adapt(config, map[string]any{
 			"filename": configFile,
 		})
 		if err != nil {
@@ -280,7 +257,7 @@ func watchConfigFile(filename, adapterName string) {
 // Flags wraps a FlagSet so that typed values
 // from flags can be easily retrieved.
 type Flags struct {
-	*flag.FlagSet
+	*pflag.FlagSet
 }
 
 // String returns the string representation of the
@@ -326,22 +303,6 @@ func (f Flags) Duration(name string) time.Duration {
 	return val
 }
 
-// flagHelp returns the help text for fs.
-func flagHelp(fs *flag.FlagSet) string {
-	if fs == nil {
-		return ""
-	}
-
-	// temporarily redirect output
-	out := fs.Output()
-	defer fs.SetOutput(out)
-
-	buf := new(bytes.Buffer)
-	fs.SetOutput(buf)
-	fs.PrintDefaults()
-	return buf.String()
-}
-
 func loadEnvFromFile(envFile string) error {
 	file, err := os.Open(envFile)
 	if err != nil {
@@ -387,11 +348,11 @@ func parseEnvFile(envInput io.Reader) (map[string]string, error) {
 		}
 
 		// split line into key and value
-		fields := strings.SplitN(line, "=", 2)
-		if len(fields) != 2 {
+		before, after, isCut := strings.Cut(line, "=")
+		if !isCut {
 			return nil, fmt.Errorf("can't parse line %d; line should be in KEY=VALUE format", lineNumber)
 		}
-		key, val := fields[0], fields[1]
+		key, val := before, after
 
 		// sometimes keys are prefixed by "export " so file can be sourced in bash; ignore it here
 		key = strings.TrimPrefix(key, "export ")
@@ -408,11 +369,8 @@ func parseEnvFile(envInput io.Reader) (map[string]string, error) {
 		}
 
 		// remove any trailing comment after value
-		if commentStart := strings.Index(val, "#"); commentStart > 0 {
-			before := val[commentStart-1]
-			if before == '\t' || before == ' ' {
-				val = strings.TrimRight(val[:commentStart], " \t")
-			}
+		if commentStart, _, found := strings.Cut(val, "#"); found {
+			val = strings.TrimRight(commentStart, " \t")
 		}
 
 		// quoted value: support newlines
@@ -441,11 +399,12 @@ func parseEnvFile(envInput io.Reader) (map[string]string, error) {
 }
 
 func printEnvironment() {
+	_, version := caddy.Version()
 	fmt.Printf("caddy.HomeDir=%s\n", caddy.HomeDir())
 	fmt.Printf("caddy.AppDataDir=%s\n", caddy.AppDataDir())
 	fmt.Printf("caddy.AppConfigDir=%s\n", caddy.AppConfigDir())
 	fmt.Printf("caddy.ConfigAutosavePath=%s\n", caddy.ConfigAutosavePath)
-	fmt.Printf("caddy.Version=%s\n", CaddyVersion())
+	fmt.Printf("caddy.Version=%s\n", version)
 	fmt.Printf("runtime.GOOS=%s\n", runtime.GOOS)
 	fmt.Printf("runtime.GOARCH=%s\n", runtime.GOARCH)
 	fmt.Printf("runtime.Compiler=%s\n", runtime.Compiler)
@@ -462,21 +421,15 @@ func printEnvironment() {
 	}
 }
 
-// CaddyVersion returns a detailed version string, if available.
-func CaddyVersion() string {
-	goModule := caddy.GoModule()
-	ver := goModule.Version
-	if goModule.Sum != "" {
-		ver += " " + goModule.Sum
-	}
-	if goModule.Replace != nil {
-		ver += " => " + goModule.Replace.Path
-		if goModule.Replace.Version != "" {
-			ver += "@" + goModule.Replace.Version
-		}
-		if goModule.Replace.Sum != "" {
-			ver += " " + goModule.Replace.Sum
-		}
-	}
-	return ver
+// StringSlice is a flag.Value that enables repeated use of a string flag.
+type StringSlice []string
+
+func (ss StringSlice) String() string { return "[" + strings.Join(ss, ", ") + "]" }
+
+func (ss *StringSlice) Set(value string) error {
+	*ss = append(*ss, value)
+	return nil
 }
+
+// Interface guard
+var _ flag.Value = (*StringSlice)(nil)

cmd/packagesfuncs.go

@@ -194,7 +194,7 @@ func getModules() (standard, nonstandard, unknown []moduleInfo, err error) {
 		// can use reflection but we need a non-pointer value (I'm
 		// not sure why), and since New() should return a pointer
 		// value, we need to dereference it first
-		iface := interface{}(modInfo.New())
+		iface := any(modInfo.New())
 		if rv := reflect.ValueOf(iface); rv.Kind() == reflect.Ptr {
 			iface = reflect.New(reflect.TypeOf(iface).Elem()).Elem().Interface()
 		}

cmd/removebinary.go

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 //go:build !windows
-// +build !windows
 
 package caddycmd
 

cmd/removebinary_windows.go

@@ -31,6 +31,9 @@ import (
 func removeCaddyBinary(path string) error {
 	var sI syscall.StartupInfo
 	var pI syscall.ProcessInformation
-	argv := syscall.StringToUTF16Ptr(filepath.Join(os.Getenv("windir"), "system32", "cmd.exe") + " /C del " + path)
+	argv, err := syscall.UTF16PtrFromString(filepath.Join(os.Getenv("windir"), "system32", "cmd.exe") + " /C del " + path)
+	if err != nil {
+		return err
+	}
 	return syscall.CreateProcess(nil, argv, nil, nil, true, 0, nil, nil, &sI, &pI)
 }

context.go

@@ -37,9 +37,10 @@ import (
 // not actually need to do this).
 type Context struct {
 	context.Context
-	moduleInstances map[string][]interface{}
+	moduleInstances map[string][]Module
 	cfg             *Config
 	cleanupFuncs    []func()
+	ancestry        []Module
 }
 
 // NewContext provides a new context derived from the given
@@ -51,7 +52,7 @@ type Context struct {
 // modules which are loaded will be properly unloaded.
 // See standard library context package's documentation.
 func NewContext(ctx Context) (Context, context.CancelFunc) {
-	newCtx := Context{moduleInstances: make(map[string][]interface{}), cfg: ctx.cfg}
+	newCtx := Context{moduleInstances: make(map[string][]Module), cfg: ctx.cfg}
 	c, cancel := context.WithCancel(ctx.Context)
 	wrappedCancel := func() {
 		cancel()
@@ -90,15 +91,15 @@ func (ctx *Context) OnCancel(f func()) {
 // ModuleMap may be used in place of map[string]json.RawMessage. The return value's
 // underlying type mirrors the input field's type:
 //
-//    json.RawMessage              => interface{}
-//    []json.RawMessage            => []interface{}
-//    [][]json.RawMessage          => [][]interface{}
-//    map[string]json.RawMessage   => map[string]interface{}
-//    []map[string]json.RawMessage => []map[string]interface{}
+//	json.RawMessage              => any
+//	[]json.RawMessage            => []any
+//	[][]json.RawMessage          => [][]any
+//	map[string]json.RawMessage   => map[string]any
+//	[]map[string]json.RawMessage => []map[string]any
 //
 // The field must have a "caddy" struct tag in this format:
 //
-//    caddy:"key1=val1 key2=val2"
+//	caddy:"key1=val1 key2=val2"
 //
 // To load modules, a "namespace" key is required. For example, to load modules
 // in the "http.handlers" namespace, you'd put: `namespace=http.handlers` in the
@@ -115,20 +116,20 @@ func (ctx *Context) OnCancel(f func()) {
 // meaning the key containing the module's name that is defined inline with the module
 // itself. You must specify the inline key in a struct tag, along with the namespace:
 //
-//    caddy:"namespace=http.handlers inline_key=handler"
+//	caddy:"namespace=http.handlers inline_key=handler"
 //
 // This will look for a key/value pair like `"handler": "..."` in the json.RawMessage
 // in order to know the module name.
 //
 // To make use of the loaded module(s) (the return value), you will probably want
-// to type-assert each interface{} value(s) to the types that are useful to you
+// to type-assert each 'any' value(s) to the types that are useful to you
 // and store them on the same struct. Storing them on the same struct makes for
 // easy garbage collection when your host module is no longer needed.
 //
 // Loaded modules have already been provisioned and validated. Upon returning
 // successfully, this method clears the json.RawMessage(s) in the field since
 // the raw JSON is no longer needed, and this allows the GC to free up memory.
-func (ctx Context) LoadModule(structPointer interface{}, fieldName string) (interface{}, error) {
+func (ctx Context) LoadModule(structPointer any, fieldName string) (any, error) {
 	val := reflect.ValueOf(structPointer).Elem().FieldByName(fieldName)
 	typ := val.Type()
 
@@ -148,7 +149,7 @@ func (ctx Context) LoadModule(structPointer interface{}, fieldName string) (inte
 	}
 	inlineModuleKey := opts["inline_key"]
 
-	var result interface{}
+	var result any
 
 	switch val.Kind() {
 	case reflect.Slice:
@@ -170,7 +171,7 @@ func (ctx Context) LoadModule(structPointer interface{}, fieldName string) (inte
 			if inlineModuleKey == "" {
 				panic("unable to determine module name without inline_key because type is not a ModuleMap")
 			}
-			var all []interface{}
+			var all []any
 			for i := 0; i < val.Len(); i++ {
 				val, err := ctx.loadModuleInline(inlineModuleKey, moduleNamespace, val.Index(i).Interface().(json.RawMessage))
 				if err != nil {
@@ -186,10 +187,10 @@ func (ctx Context) LoadModule(structPointer interface{}, fieldName string) (inte
 			if inlineModuleKey == "" {
 				panic("unable to determine module name without inline_key because type is not a ModuleMap")
 			}
-			var all [][]interface{}
+			var all [][]any
 			for i := 0; i < val.Len(); i++ {
 				innerVal := val.Index(i)
-				var allInner []interface{}
+				var allInner []any
 				for j := 0; j < innerVal.Len(); j++ {
 					innerInnerVal, err := ctx.loadModuleInline(inlineModuleKey, moduleNamespace, innerVal.Index(j).Interface().(json.RawMessage))
 					if err != nil {
@@ -204,7 +205,7 @@ func (ctx Context) LoadModule(structPointer interface{}, fieldName string) (inte
 		} else if isModuleMapType(typ.Elem()) {
 			// val is `[]map[string]json.RawMessage`
 
-			var all []map[string]interface{}
+			var all []map[string]any
 			for i := 0; i < val.Len(); i++ {
 				thisSet, err := ctx.loadModulesFromSomeMap(moduleNamespace, inlineModuleKey, val.Index(i))
 				if err != nil {
@@ -232,10 +233,10 @@ func (ctx Context) LoadModule(structPointer interface{}, fieldName string) (inte
 	return result, nil
 }
 
-// loadModulesFromSomeMap loads modules from val, which must be a type of map[string]interface{}.
+// loadModulesFromSomeMap loads modules from val, which must be a type of map[string]any.
 // Depending on inlineModuleKey, it will be interpreted as either a ModuleMap (key is the module
 // name) or as a regular map (key is not the module name, and module name is defined inline).
-func (ctx Context) loadModulesFromSomeMap(namespace, inlineModuleKey string, val reflect.Value) (map[string]interface{}, error) {
+func (ctx Context) loadModulesFromSomeMap(namespace, inlineModuleKey string, val reflect.Value) (map[string]any, error) {
 	// if no inline_key is specified, then val must be a ModuleMap,
 	// where the key is the module name
 	if inlineModuleKey == "" {
@@ -253,8 +254,8 @@ func (ctx Context) loadModulesFromSomeMap(namespace, inlineModuleKey string, val
 // loadModulesFromRegularMap loads modules from val, where val is a map[string]json.RawMessage.
 // Map keys are NOT interpreted as module names, so module names are still expected to appear
 // inline with the objects.
-func (ctx Context) loadModulesFromRegularMap(namespace, inlineModuleKey string, val reflect.Value) (map[string]interface{}, error) {
-	mods := make(map[string]interface{})
+func (ctx Context) loadModulesFromRegularMap(namespace, inlineModuleKey string, val reflect.Value) (map[string]any, error) {
+	mods := make(map[string]any)
 	iter := val.MapRange()
 	for iter.Next() {
 		k := iter.Key()
@@ -268,10 +269,10 @@ func (ctx Context) loadModulesFromRegularMap(namespace, inlineModuleKey string,
 	return mods, nil
 }
 
-// loadModuleMap loads modules from a ModuleMap, i.e. map[string]interface{}, where the key is the
+// loadModuleMap loads modules from a ModuleMap, i.e. map[string]any, where the key is the
 // module name. With a module map, module names do not need to be defined inline with their values.
-func (ctx Context) loadModuleMap(namespace string, val reflect.Value) (map[string]interface{}, error) {
-	all := make(map[string]interface{})
+func (ctx Context) loadModuleMap(namespace string, val reflect.Value) (map[string]any, error) {
+	all := make(map[string]any)
 	iter := val.MapRange()
 	for iter.Next() {
 		k := iter.Key().Interface().(string)
@@ -299,19 +300,19 @@ func (ctx Context) loadModuleMap(namespace string, val reflect.Value) (map[strin
 // directly by most modules. However, this method is useful when
 // dynamically loading/unloading modules in their own context,
 // like from embedded scripts, etc.
-func (ctx Context) LoadModuleByID(id string, rawMsg json.RawMessage) (interface{}, error) {
+func (ctx Context) LoadModuleByID(id string, rawMsg json.RawMessage) (any, error) {
 	modulesMu.RLock()
-	mod, ok := modules[id]
+	modInfo, ok := modules[id]
 	modulesMu.RUnlock()
 	if !ok {
 		return nil, fmt.Errorf("unknown module: %s", id)
 	}
 
-	if mod.New == nil {
-		return nil, fmt.Errorf("module '%s' has no constructor", mod.ID)
+	if modInfo.New == nil {
+		return nil, fmt.Errorf("module '%s' has no constructor", modInfo.ID)
 	}
 
-	val := mod.New().(interface{})
+	val := modInfo.New()
 
 	// value must be a pointer for unmarshaling into concrete type, even if
 	// the module's concrete type is a slice or map; New() *should* return
@@ -327,7 +328,7 @@ func (ctx Context) LoadModuleByID(id string, rawMsg json.RawMessage) (interface{
 	if len(rawMsg) > 0 {
 		err := strictUnmarshalJSON(rawMsg, &val)
 		if err != nil {
-			return nil, fmt.Errorf("decoding module config: %s: %v", mod, err)
+			return nil, fmt.Errorf("decoding module config: %s: %v", modInfo, err)
 		}
 	}
 
@@ -340,6 +341,8 @@ func (ctx Context) LoadModuleByID(id string, rawMsg json.RawMessage) (interface{
 		return nil, fmt.Errorf("module value cannot be null")
 	}
 
+	ctx.ancestry = append(ctx.ancestry, val)
+
 	if prov, ok := val.(Provisioner); ok {
 		err := prov.Provision(ctx)
 		if err != nil {
@@ -351,7 +354,7 @@ func (ctx Context) LoadModuleByID(id string, rawMsg json.RawMessage) (interface{
 					err = fmt.Errorf("%v; additionally, cleanup: %v", err, err2)
 				}
 			}
-			return nil, fmt.Errorf("provision %s: %v", mod, err)
+			return nil, fmt.Errorf("provision %s: %v", modInfo, err)
 		}
 	}
 
@@ -365,7 +368,7 @@ func (ctx Context) LoadModuleByID(id string, rawMsg json.RawMessage) (interface{
 					err = fmt.Errorf("%v; additionally, cleanup: %v", err, err2)
 				}
 			}
-			return nil, fmt.Errorf("%s: invalid configuration: %v", mod, err)
+			return nil, fmt.Errorf("%s: invalid configuration: %v", modInfo, err)
 		}
 	}
 
@@ -375,7 +378,7 @@ func (ctx Context) LoadModuleByID(id string, rawMsg json.RawMessage) (interface{
 }
 
 // loadModuleInline loads a module from a JSON raw message which decodes to
-// a map[string]interface{}, where one of the object keys is moduleNameKey
+// a map[string]any, where one of the object keys is moduleNameKey
 // and the corresponding value is the module name (as a string) which can
 // be found in the given scope. In other words, the module name is declared
 // in-line with the module itself.
@@ -385,7 +388,7 @@ func (ctx Context) LoadModuleByID(id string, rawMsg json.RawMessage) (interface{
 // multiple instances in the map or it appears in an array (where there are
 // no custom keys). In other words, the key containing the module name is
 // treated special/separate from all the other keys in the object.
-func (ctx Context) loadModuleInline(moduleNameKey, moduleScope string, raw json.RawMessage) (interface{}, error) {
+func (ctx Context) loadModuleInline(moduleNameKey, moduleScope string, raw json.RawMessage) (any, error) {
 	moduleName, raw, err := getModuleNameInline(moduleNameKey, raw)
 	if err != nil {
 		return nil, err
@@ -407,7 +410,7 @@ func (ctx Context) loadModuleInline(moduleNameKey, moduleScope string, raw json.
 // called during the Provision/Validate phase to reference a
 // module's own host app (since the parent app module is still
 // in the process of being provisioned, it is not yet ready).
-func (ctx Context) App(name string) (interface{}, error) {
+func (ctx Context) App(name string) (any, error) {
 	if app, ok := ctx.cfg.apps[name]; ok {
 		return app, nil
 	}
@@ -439,8 +442,27 @@ func (ctx Context) Storage() certmagic.Storage {
 	return ctx.cfg.storage
 }
 
-// Logger returns a logger that can be used by mod.
-func (ctx Context) Logger(mod Module) *zap.Logger {
+// Logger returns a logger that is intended for use by the most
+// recent module associated with the context. Callers should not
+// pass in any arguments unless they want to associate with a
+// different module; it panics if more than 1 value is passed in.
+//
+// Originally, this method's signature was `Logger(mod Module)`,
+// requiring that an instance of a Caddy module be passsed in.
+// However, that is no longer necessary, as the closest module
+// most recently associated with the context will be automatically
+// assumed. To prevent a sudden breaking change, this method's
+// signature has been changed to be variadic, but we may remove
+// the parameter altogether in the future. Callers should not
+// pass in any argument. If there is valid need to specify a
+// different module, please open an issue to discuss.
+//
+// PARTIALLY DEPRECATED: The Logger(module) form is deprecated and
+// may be removed in the future. Do not pass in any arguments.
+func (ctx Context) Logger(module ...Module) *zap.Logger {
+	if len(module) > 1 {
+		panic("more than 1 module passed in")
+	}
 	if ctx.cfg == nil {
 		// often the case in tests; just use a dev logger
 		l, err := zap.NewDevelopment()
@@ -449,5 +471,26 @@ func (ctx Context) Logger(mod Module) *zap.Logger {
 		}
 		return l
 	}
+	mod := ctx.Module()
+	if len(module) > 0 {
+		mod = module[0]
+	}
 	return ctx.cfg.Logging.Logger(mod)
 }
+
+// Modules returns the lineage of modules that this context provisioned,
+// with the most recent/current module being last in the list.
+func (ctx Context) Modules() []Module {
+	mods := make([]Module, len(ctx.ancestry))
+	copy(mods, ctx.ancestry)
+	return mods
+}
+
+// Module returns the current module, or the most recent one
+// provisioned by the context.
+func (ctx Context) Module() Module {
+	if len(ctx.ancestry) == 0 {
+		return nil
+	}
+	return ctx.ancestry[len(ctx.ancestry)-1]
+}

context_test.go

@@ -71,13 +71,13 @@ func ExampleContext_LoadModule_array() {
 		},
 	}
 
-	// since our input is []json.RawMessage, the output will be []interface{}
+	// since our input is []json.RawMessage, the output will be []any
 	mods, err := ctx.LoadModule(myStruct, "GuestModulesRaw")
 	if err != nil {
 		// you'd want to actually handle the error here
 		// return fmt.Errorf("loading guest modules: %v", err)
 	}
-	for _, mod := range mods.([]interface{}) {
+	for _, mod := range mods.([]any) {
 		myStruct.guestModules = append(myStruct.guestModules, mod.(io.Writer))
 	}
 
@@ -104,13 +104,13 @@ func ExampleContext_LoadModule_map() {
 		},
 	}
 
-	// since our input is map[string]json.RawMessage, the output will be map[string]interface{}
+	// since our input is map[string]json.RawMessage, the output will be map[string]any
 	mods, err := ctx.LoadModule(myStruct, "GuestModulesRaw")
 	if err != nil {
 		// you'd want to actually handle the error here
 		// return fmt.Errorf("loading guest modules: %v", err)
 	}
-	for modName, mod := range mods.(map[string]interface{}) {
+	for modName, mod := range mods.(map[string]any) {
 		myStruct.guestModules[modName] = mod.(io.Writer)
 	}
 

duration_fuzz.go

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 //go:build gofuzz
-// +build gofuzz
 
 package caddy
 

go.mod

@@ -1,41 +1,48 @@
 module github.com/caddyserver/caddy/v2
 
-go 1.17
+go 1.18
 
 require (
-	github.com/BurntSushi/toml v1.0.0
+	github.com/BurntSushi/toml v1.2.0
 	github.com/Masterminds/sprig/v3 v3.2.2
 	github.com/alecthomas/chroma v0.10.0
 	github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b
-	github.com/caddyserver/certmagic v0.16.1
+	github.com/caddyserver/certmagic v0.17.1
 	github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac
 	github.com/go-chi/chi v4.1.2+incompatible
-	github.com/google/cel-go v0.7.3
+	github.com/google/cel-go v0.12.4
 	github.com/google/uuid v1.3.0
-	github.com/klauspost/compress v1.15.0
-	github.com/klauspost/cpuid/v2 v2.0.11
-	github.com/lucas-clemente/quic-go v0.26.0
-	github.com/mholt/acmez v1.0.2
-	github.com/prometheus/client_golang v1.12.1
-	github.com/smallstep/certificates v0.19.0
-	github.com/smallstep/cli v0.18.0
+	github.com/klauspost/compress v1.15.9
+	github.com/klauspost/cpuid/v2 v2.1.0
+	github.com/lucas-clemente/quic-go v0.28.2-0.20220813150001-9957668d4301
+	github.com/mholt/acmez v1.0.4
+	github.com/prometheus/client_golang v1.12.2
+	github.com/smallstep/certificates v0.21.0
+	github.com/smallstep/cli v0.21.0
 	github.com/smallstep/nosql v0.4.0
-	github.com/smallstep/truststore v0.11.0
-	github.com/tailscale/tscert v0.0.0-20220125204807-4509a5fbaf74
-	github.com/yuin/goldmark v1.4.8
+	github.com/smallstep/truststore v0.12.0
+	github.com/spf13/cobra v1.1.3
+	github.com/spf13/pflag v1.0.5
+	github.com/tailscale/tscert v0.0.0-20220316030059-54bbcb9f74e2
+	github.com/yuin/goldmark v1.4.13
 	github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.29.0
-	go.opentelemetry.io/otel v1.4.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.34.0
+	go.opentelemetry.io/otel v1.9.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.4.0
 	go.opentelemetry.io/otel/sdk v1.4.0
 	go.uber.org/zap v1.21.0
-	golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
+	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
+	golang.org/x/net v0.0.0-20220812165438-1d4ff48094d1
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
-	google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf
-	google.golang.org/protobuf v1.27.1
+	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
-	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
+	github.com/golang/mock v1.6.0 // indirect
+	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 
 require (
@@ -43,24 +50,23 @@ require (
 	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.1.1 // indirect
-	github.com/antlr/antlr4 v0.0.0-20200503195918-621b933c7a7f // indirect
+	github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/cheekybits/genny v1.0.0 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/dgraph-io/badger v1.6.2 // indirect
 	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
 	github.com/dgraph-io/ristretto v0.0.4-0.20200906165740-41ebdbffecfd // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/dlclark/regexp2 v1.4.0 // indirect
-	github.com/felixge/httpsnoop v1.0.2 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-kit/kit v0.10.0 // indirect
 	github.com/go-logfmt/logfmt v0.5.0 // indirect
-	github.com/go-logr/logr v1.2.2 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-sql-driver/mysql v1.6.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
@@ -69,6 +75,7 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgconn v1.10.1 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
@@ -80,15 +87,14 @@ require (
 	github.com/libdns/libdns v0.2.1 // indirect
 	github.com/manifoldco/promptui v0.9.0 // indirect
 	github.com/marten-seemann/qpack v0.2.1 // indirect
-	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
-	github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
-	github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
+	github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.8 // indirect
 	github.com/mattn/go-isatty v0.0.13 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 	github.com/micromdm/scep/v2 v2.1.0 // indirect
-	github.com/miekg/dns v1.1.46 // indirect
+	github.com/miekg/dns v1.1.50 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-ps v1.0.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
@@ -99,7 +105,7 @@ require (
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rs/xid v1.2.1 // indirect
-	github.com/russross/blackfriday/v2 v2.0.1 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
@@ -111,21 +117,21 @@ require (
 	go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.0 // indirect
-	go.opentelemetry.io/otel/internal/metric v0.27.0 // indirect
-	go.opentelemetry.io/otel/metric v0.27.0 // indirect
-	go.opentelemetry.io/otel/trace v1.4.0 // indirect
+	go.opentelemetry.io/otel/metric v0.31.0 // indirect
+	go.opentelemetry.io/otel/trace v1.9.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.12.0 // indirect
-	go.step.sm/cli-utils v0.7.0 // indirect
-	go.step.sm/crypto v0.16.1 // indirect
-	go.step.sm/linkedca v0.15.0 // indirect
+	go.step.sm/cli-utils v0.7.3 // indirect
+	go.step.sm/crypto v0.16.2 // indirect
+	go.step.sm/linkedca v0.16.1 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/mod v0.4.2 // indirect
-	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
+	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10
 	golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect
-	golang.org/x/tools v0.1.7 // indirect
+	golang.org/x/tools v0.1.10 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/grpc v1.44.0 // indirect
+	google.golang.org/grpc v1.46.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	howett.net/plist v1.0.0 // indirect

listen.go

@@ -0,0 +1,172 @@
+//go:build !linux
+
+package caddy
+
+import (
+	"fmt"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"go.uber.org/zap"
+)
+
+func ListenTimeout(network, addr string, keepAlivePeriod time.Duration) (net.Listener, error) {
+	// check to see if plugin provides listener
+	if ln, err := getListenerFromPlugin(network, addr); err != nil || ln != nil {
+		return ln, err
+	}
+
+	lnKey := listenerKey(network, addr)
+
+	sharedLn, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
+		ln, err := net.Listen(network, addr)
+		if err != nil {
+			// https://github.com/caddyserver/caddy/pull/4534
+			if isUnixNetwork(network) && isListenBindAddressAlreadyInUseError(err) {
+				return nil, fmt.Errorf("%w: this can happen if Caddy was forcefully killed", err)
+			}
+			return nil, err
+		}
+		return &sharedListener{Listener: ln, key: lnKey}, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &fakeCloseListener{sharedListener: sharedLn.(*sharedListener), keepAlivePeriod: keepAlivePeriod}, nil
+}
+
+// fakeCloseListener is a private wrapper over a listener that
+// is shared. The state of fakeCloseListener is not shared.
+// This allows one user of a socket to "close" the listener
+// while in reality the socket stays open for other users of
+// the listener. In this way, servers become hot-swappable
+// while the listener remains running. Listeners should be
+// re-wrapped in a new fakeCloseListener each time the listener
+// is reused. This type is atomic and values must not be copied.
+type fakeCloseListener struct {
+	closed          int32 // accessed atomically; belongs to this struct only
+	*sharedListener       // embedded, so we also become a net.Listener
+	keepAlivePeriod time.Duration
+}
+
+type canSetKeepAlive interface {
+	SetKeepAlivePeriod(d time.Duration) error
+	SetKeepAlive(bool) error
+}
+
+func (fcl *fakeCloseListener) Accept() (net.Conn, error) {
+	// if the listener is already "closed", return error
+	if atomic.LoadInt32(&fcl.closed) == 1 {
+		return nil, fakeClosedErr(fcl)
+	}
+
+	// call underlying accept
+	conn, err := fcl.sharedListener.Accept()
+	if err == nil {
+		// if 0, do nothing, Go's default is already set
+		// and if the connection allows setting KeepAlive, set it
+		if tconn, ok := conn.(canSetKeepAlive); ok && fcl.keepAlivePeriod != 0 {
+			if fcl.keepAlivePeriod > 0 {
+				err = tconn.SetKeepAlivePeriod(fcl.keepAlivePeriod)
+			} else { // negative
+				err = tconn.SetKeepAlive(false)
+			}
+			if err != nil {
+				Log().With(zap.String("server", fcl.sharedListener.key)).Warn("unable to set keepalive for new connection:", zap.Error(err))
+			}
+		}
+		return conn, nil
+	}
+
+	// since Accept() returned an error, it may be because our reference to
+	// the listener (this fakeCloseListener) may have been closed, i.e. the
+	// server is shutting down; in that case, we need to clear the deadline
+	// that we set when Close() was called, and return a non-temporary and
+	// non-timeout error value to the caller, masking the "true" error, so
+	// that server loops / goroutines won't retry, linger, and leak
+	if atomic.LoadInt32(&fcl.closed) == 1 {
+		// we dereference the sharedListener explicitly even though it's embedded
+		// so that it's clear in the code that side-effects are shared with other
+		// users of this listener, not just our own reference to it; we also don't
+		// do anything with the error because all we could do is log it, but we
+		// expliclty assign it to nothing so we don't forget it's there if needed
+		_ = fcl.sharedListener.clearDeadline()
+
+		if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
+			return nil, fakeClosedErr(fcl)
+		}
+	}
+
+	return nil, err
+}
+
+// Close stops accepting new connections without closing the
+// underlying listener. The underlying listener is only closed
+// if the caller is the last known user of the socket.
+func (fcl *fakeCloseListener) Close() error {
+	if atomic.CompareAndSwapInt32(&fcl.closed, 0, 1) {
+		// There are two ways I know of to get an Accept()
+		// function to return to the server loop that called
+		// it: close the listener, or set a deadline in the
+		// past. Obviously, we can't close the socket yet
+		// since others may be using it (hence this whole
+		// file). But we can set the deadline in the past,
+		// and this is kind of cheating, but it works, and
+		// it apparently even works on Windows.
+		_ = fcl.sharedListener.setDeadline()
+		_, _ = listenerPool.Delete(fcl.sharedListener.key)
+	}
+	return nil
+}
+
+// sharedListener is a wrapper over an underlying listener. The listener
+// and the other fields on the struct are shared state that is synchronized,
+// so sharedListener structs must never be copied (always use a pointer).
+type sharedListener struct {
+	net.Listener
+	key        string // uniquely identifies this listener
+	deadline   bool   // whether a deadline is currently set
+	deadlineMu sync.Mutex
+}
+
+func (sl *sharedListener) clearDeadline() error {
+	var err error
+	sl.deadlineMu.Lock()
+	if sl.deadline {
+		switch ln := sl.Listener.(type) {
+		case *net.TCPListener:
+			err = ln.SetDeadline(time.Time{})
+		case *net.UnixListener:
+			err = ln.SetDeadline(time.Time{})
+		}
+		sl.deadline = false
+	}
+	sl.deadlineMu.Unlock()
+	return err
+}
+
+func (sl *sharedListener) setDeadline() error {
+	timeInPast := time.Now().Add(-1 * time.Minute)
+	var err error
+	sl.deadlineMu.Lock()
+	if !sl.deadline {
+		switch ln := sl.Listener.(type) {
+		case *net.TCPListener:
+			err = ln.SetDeadline(timeInPast)
+		case *net.UnixListener:
+			err = ln.SetDeadline(timeInPast)
+		}
+		sl.deadline = true
+	}
+	sl.deadlineMu.Unlock()
+	return err
+}
+
+// Destruct is called by the UsagePool when the listener is
+// finally not being used anymore. It closes the socket.
+func (sl *sharedListener) Destruct() error {
+	return sl.Listener.Close()
+}

listen_linux.go

@@ -0,0 +1,34 @@
+package caddy
+
+import (
+	"context"
+	"net"
+	"syscall"
+	"time"
+
+	"go.uber.org/zap"
+	"golang.org/x/sys/unix"
+)
+
+// ListenTimeout is the same as Listen, but with a configurable keep-alive timeout duration.
+func ListenTimeout(network, addr string, keepalivePeriod time.Duration) (net.Listener, error) {
+	// check to see if plugin provides listener
+	if ln, err := getListenerFromPlugin(network, addr); err != nil || ln != nil {
+		return ln, err
+	}
+
+	config := &net.ListenConfig{Control: reusePort, KeepAlive: keepalivePeriod}
+	return config.Listen(context.Background(), network, addr)
+}
+
+func reusePort(network, address string, conn syscall.RawConn) error {
+	return conn.Control(func(descriptor uintptr) {
+		if err := unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
+			Log().Error("setting SO_REUSEPORT",
+				zap.String("network", network),
+				zap.String("address", address),
+				zap.Uintptr("descriptor", descriptor),
+				zap.Error(err))
+		}
+	})
+}

listeners.go

@@ -20,16 +20,16 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"net/netip"
 	"os"
 	"strconv"
 	"strings"
-	"sync"
 	"sync/atomic"
 	"syscall"
-	"time"
 
 	"github.com/lucas-clemente/quic-go"
 	"github.com/lucas-clemente/quic-go/http3"
+	"go.uber.org/zap"
 )
 
 // Listen is like net.Listen, except Caddy's listeners can overlap
@@ -41,31 +41,30 @@ import (
 // the socket have been finished. Always be sure to close listeners
 // when you are done with them, just like normal listeners.
 func Listen(network, addr string) (net.Listener, error) {
-	lnKey := network + "/" + addr
+	// a 0 timeout means Go uses its default
+	return ListenTimeout(network, addr, 0)
+}
 
-	sharedLn, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
-		ln, err := net.Listen(network, addr)
-		if err != nil {
-			// https://github.com/caddyserver/caddy/pull/4534
-			if isUnixNetwork(network) && isListenBindAddressAlreadyInUseError(err) {
-				return nil, fmt.Errorf("%w: this can happen if Caddy was forcefully killed", err)
-			}
-			return nil, err
-		}
-		return &sharedListener{Listener: ln, key: lnKey}, nil
-	})
-	if err != nil {
-		return nil, err
+// getListenerFromPlugin returns a listener on the given network and address
+// if a plugin has registered the network name. It may return (nil, nil) if
+// no plugin can provide a listener.
+func getListenerFromPlugin(network, addr string) (net.Listener, error) {
+	network = strings.TrimSpace(strings.ToLower(network))
+
+	// get listener from plugin if network type is registered
+	if getListener, ok := networkTypes[network]; ok {
+		Log().Debug("getting listener from plugin", zap.String("network", network))
+		return getListener(network, addr)
 	}
 
-	return &fakeCloseListener{sharedListener: sharedLn.(*sharedListener)}, nil
+	return nil, nil
 }
 
 // ListenPacket returns a net.PacketConn suitable for use in a Caddy module.
 // It is like Listen except for PacketConns.
 // Always be sure to close the PacketConn when you are done.
 func ListenPacket(network, addr string) (net.PacketConn, error) {
-	lnKey := network + "/" + addr
+	lnKey := listenerKey(network, addr)
 
 	sharedPc, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
 		pc, err := net.ListenPacket(network, addr)
@@ -88,88 +87,45 @@ func ListenPacket(network, addr string) (net.PacketConn, error) {
 // ListenQUIC returns a quic.EarlyListener suitable for use in a Caddy module.
 // Note that the context passed to Accept is currently ignored, so using
 // a context other than context.Background is meaningless.
-func ListenQUIC(addr string, tlsConf *tls.Config) (quic.EarlyListener, error) {
-	lnKey := "quic/" + addr
+// This API is EXPERIMENTAL and may change.
+func ListenQUIC(addr string, tlsConf *tls.Config, activeRequests *int64) (quic.EarlyListener, error) {
+	lnKey := listenerKey("udp", addr)
 
 	sharedEl, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
-		el, err := quic.ListenAddrEarly(addr, http3.ConfigureTLSConfig(tlsConf), &quic.Config{})
+		el, err := quic.ListenAddrEarly(addr, http3.ConfigureTLSConfig(tlsConf), &quic.Config{
+			RequireAddressValidation: func(clientAddr net.Addr) bool {
+				var highLoad bool
+				if activeRequests != nil {
+					highLoad = atomic.LoadInt64(activeRequests) > 1000 // TODO: make tunable?
+				}
+				return highLoad
+			},
+		})
 		if err != nil {
 			return nil, err
 		}
 		return &sharedQuicListener{EarlyListener: el, key: lnKey}, nil
 	})
+	if err != nil {
+		return nil, err
+	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	return &fakeCloseQuicListener{
 		sharedQuicListener: sharedEl.(*sharedQuicListener),
-		context:            ctx, contextCancel: cancel,
-	}, err
+		context:            ctx,
+		contextCancel:      cancel,
+	}, nil
 }
 
-// fakeCloseListener is a private wrapper over a listener that
-// is shared. The state of fakeCloseListener is not shared.
-// This allows one user of a socket to "close" the listener
-// while in reality the socket stays open for other users of
-// the listener. In this way, servers become hot-swappable
-// while the listener remains running. Listeners should be
-// re-wrapped in a new fakeCloseListener each time the listener
-// is reused. This type is atomic and values must not be copied.
-type fakeCloseListener struct {
-	closed          int32 // accessed atomically; belongs to this struct only
-	*sharedListener       // embedded, so we also become a net.Listener
+// ListenerUsage returns the current usage count of the given listener address.
+func ListenerUsage(network, addr string) int {
+	count, _ := listenerPool.References(listenerKey(network, addr))
+	return count
 }
 
-func (fcl *fakeCloseListener) Accept() (net.Conn, error) {
-	// if the listener is already "closed", return error
-	if atomic.LoadInt32(&fcl.closed) == 1 {
-		return nil, fakeClosedErr(fcl)
-	}
-
-	// call underlying accept
-	conn, err := fcl.sharedListener.Accept()
-	if err == nil {
-		return conn, nil
-	}
-
-	// since Accept() returned an error, it may be because our reference to
-	// the listener (this fakeCloseListener) may have been closed, i.e. the
-	// server is shutting down; in that case, we need to clear the deadline
-	// that we set when Close() was called, and return a non-temporary and
-	// non-timeout error value to the caller, masking the "true" error, so
-	// that server loops / goroutines won't retry, linger, and leak
-	if atomic.LoadInt32(&fcl.closed) == 1 {
-		// we dereference the sharedListener explicitly even though it's embedded
-		// so that it's clear in the code that side-effects are shared with other
-		// users of this listener, not just our own reference to it; we also don't
-		// do anything with the error because all we could do is log it, but we
-		// expliclty assign it to nothing so we don't forget it's there if needed
-		_ = fcl.sharedListener.clearDeadline()
-
-		if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
-			return nil, fakeClosedErr(fcl)
-		}
-	}
-
-	return nil, err
-}
-
-// Close stops accepting new connections without closing the
-// underlying listener. The underlying listener is only closed
-// if the caller is the last known user of the socket.
-func (fcl *fakeCloseListener) Close() error {
-	if atomic.CompareAndSwapInt32(&fcl.closed, 0, 1) {
-		// There are two ways I know of to get an Accept()
-		// function to return to the server loop that called
-		// it: close the listener, or set a deadline in the
-		// past. Obviously, we can't close the socket yet
-		// since others may be using it (hence this whole
-		// file). But we can set the deadline in the past,
-		// and this is kind of cheating, but it works, and
-		// it apparently even works on Windows.
-		_ = fcl.sharedListener.setDeadline()
-		_, _ = listenerPool.Delete(fcl.sharedListener.key)
-	}
-	return nil
+func listenerKey(network, addr string) string {
+	return network + "/" + addr
 }
 
 type fakeCloseQuicListener struct {
@@ -184,7 +140,7 @@ type fakeCloseQuicListener struct {
 // server on which Accept would be called with non-empty contexts
 // (mind that the default net listeners' Accept doesn't take a context argument)
 // sounds way too rare for us to sacrifice efficiency here.
-func (fcql *fakeCloseQuicListener) Accept(_ context.Context) (quic.EarlySession, error) {
+func (fcql *fakeCloseQuicListener) Accept(_ context.Context) (quic.EarlyConnection, error) {
 	conn, err := fcql.sharedQuicListener.Accept(fcql.context)
 	if err == nil {
 		return conn, nil
@@ -255,55 +211,6 @@ func (fcpc fakeClosePacketConn) SyscallConn() (syscall.RawConn, error) {
 	return nil, fmt.Errorf("SyscallConn() not implemented for %T", fcpc.PacketConn)
 }
 
-// sharedListener is a wrapper over an underlying listener. The listener
-// and the other fields on the struct are shared state that is synchronized,
-// so sharedListener structs must never be copied (always use a pointer).
-type sharedListener struct {
-	net.Listener
-	key        string // uniquely identifies this listener
-	deadline   bool   // whether a deadline is currently set
-	deadlineMu sync.Mutex
-}
-
-func (sl *sharedListener) clearDeadline() error {
-	var err error
-	sl.deadlineMu.Lock()
-	if sl.deadline {
-		switch ln := sl.Listener.(type) {
-		case *net.TCPListener:
-			err = ln.SetDeadline(time.Time{})
-		case *net.UnixListener:
-			err = ln.SetDeadline(time.Time{})
-		}
-		sl.deadline = false
-	}
-	sl.deadlineMu.Unlock()
-	return err
-}
-
-func (sl *sharedListener) setDeadline() error {
-	timeInPast := time.Now().Add(-1 * time.Minute)
-	var err error
-	sl.deadlineMu.Lock()
-	if !sl.deadline {
-		switch ln := sl.Listener.(type) {
-		case *net.TCPListener:
-			err = ln.SetDeadline(timeInPast)
-		case *net.UnixListener:
-			err = ln.SetDeadline(timeInPast)
-		}
-		sl.deadline = true
-	}
-	sl.deadlineMu.Unlock()
-	return err
-}
-
-// Destruct is called by the UsagePool when the listener is
-// finally not being used anymore. It closes the socket.
-func (sl *sharedListener) Destruct() error {
-	return sl.Listener.Close()
-}
-
 // sharedQuicListener is like sharedListener, but for quic.EarlyListeners.
 type sharedQuicListener struct {
 	quic.EarlyListener
@@ -353,11 +260,25 @@ func (na NetworkAddress) JoinHostPort(offset uint) string {
 	return net.JoinHostPort(na.Host, strconv.Itoa(int(na.StartPort+offset)))
 }
 
+func (na NetworkAddress) Expand() []NetworkAddress {
+	size := na.PortRangeSize()
+	addrs := make([]NetworkAddress, size)
+	for portOffset := uint(0); portOffset < size; portOffset++ {
+		na2 := na
+		na2.StartPort, na2.EndPort = na.StartPort+portOffset, na.StartPort+portOffset
+		addrs[portOffset] = na2
+	}
+	return addrs
+}
+
 // PortRangeSize returns how many ports are in
 // pa's port range. Port ranges are inclusive,
 // so the size is the difference of start and
 // end ports plus one.
 func (na NetworkAddress) PortRangeSize() uint {
+	if na.EndPort < na.StartPort {
+		return 0
+	}
 	return (na.EndPort - na.StartPort) + 1
 }
 
@@ -368,7 +289,7 @@ func (na NetworkAddress) isLoopback() bool {
 	if na.Host == "localhost" {
 		return true
 	}
-	if ip := net.ParseIP(na.Host); ip != nil {
+	if ip, err := netip.ParseAddr(na.Host); err == nil {
 		return ip.IsLoopback()
 	}
 	return false
@@ -378,7 +299,7 @@ func (na NetworkAddress) isWildcardInterface() bool {
 	if na.Host == "" {
 		return true
 	}
-	if ip := net.ParseIP(na.Host); ip != nil {
+	if ip, err := netip.ParseAddr(na.Host); err == nil {
 		return ip.IsUnspecified()
 	}
 	return false
@@ -391,10 +312,13 @@ func (na NetworkAddress) port() string {
 	return fmt.Sprintf("%d-%d", na.StartPort, na.EndPort)
 }
 
-// String reconstructs the address string to the form expected
-// by ParseNetworkAddress(). If the address is a unix socket,
-// any non-zero port will be dropped.
+// String reconstructs the address string for human display.
+// The output can be parsed by ParseNetworkAddress(). If the
+// address is a unix socket, any non-zero port will be dropped.
 func (na NetworkAddress) String() string {
+	if na.Network == "tcp" && (na.Host != "" || na.port() != "") {
+		na.Network = "" // omit default network value for brevity
+	}
 	return JoinNetworkAddress(na.Network, na.Host, na.port())
 }
 
@@ -427,36 +351,38 @@ func isListenBindAddressAlreadyInUseError(err error) bool {
 func ParseNetworkAddress(addr string) (NetworkAddress, error) {
 	var host, port string
 	network, host, port, err := SplitNetworkAddress(addr)
-	if network == "" {
-		network = "tcp"
-	}
 	if err != nil {
 		return NetworkAddress{}, err
 	}
+	if network == "" {
+		network = "tcp"
+	}
 	if isUnixNetwork(network) {
 		return NetworkAddress{
 			Network: network,
 			Host:    host,
 		}, nil
 	}
-	ports := strings.SplitN(port, "-", 2)
-	if len(ports) == 1 {
-		ports = append(ports, ports[0])
-	}
 	var start, end uint64
-	start, err = strconv.ParseUint(ports[0], 10, 16)
-	if err != nil {
-		return NetworkAddress{}, fmt.Errorf("invalid start port: %v", err)
-	}
-	end, err = strconv.ParseUint(ports[1], 10, 16)
-	if err != nil {
-		return NetworkAddress{}, fmt.Errorf("invalid end port: %v", err)
-	}
-	if end < start {
-		return NetworkAddress{}, fmt.Errorf("end port must not be less than start port")
-	}
-	if (end - start) > maxPortSpan {
-		return NetworkAddress{}, fmt.Errorf("port range exceeds %d ports", maxPortSpan)
+	if port != "" {
+		before, after, found := strings.Cut(port, "-")
+		if !found {
+			after = before
+		}
+		start, err = strconv.ParseUint(before, 10, 16)
+		if err != nil {
+			return NetworkAddress{}, fmt.Errorf("invalid start port: %v", err)
+		}
+		end, err = strconv.ParseUint(after, 10, 16)
+		if err != nil {
+			return NetworkAddress{}, fmt.Errorf("invalid end port: %v", err)
+		}
+		if end < start {
+			return NetworkAddress{}, fmt.Errorf("end port must not be less than start port")
+		}
+		if (end - start) > maxPortSpan {
+			return NetworkAddress{}, fmt.Errorf("port range exceeds %d ports", maxPortSpan)
+		}
 	}
 	return NetworkAddress{
 		Network:   network,
@@ -469,15 +395,29 @@ func ParseNetworkAddress(addr string) (NetworkAddress, error) {
 // SplitNetworkAddress splits a into its network, host, and port components.
 // Note that port may be a port range (:X-Y), or omitted for unix sockets.
 func SplitNetworkAddress(a string) (network, host, port string, err error) {
-	if idx := strings.Index(a, "/"); idx >= 0 {
-		network = strings.ToLower(strings.TrimSpace(a[:idx]))
-		a = a[idx+1:]
+	beforeSlash, afterSlash, slashFound := strings.Cut(a, "/")
+	if slashFound {
+		network = strings.ToLower(strings.TrimSpace(beforeSlash))
+		a = afterSlash
 	}
 	if isUnixNetwork(network) {
 		host = a
 		return
 	}
 	host, port, err = net.SplitHostPort(a)
+	if err == nil || a == "" {
+		return
+	}
+	// in general, if there was an error, it was likely "missing port",
+	// so try adding a bogus port to take advantage of standard library's
+	// robust parser, then strip the artificial port before returning
+	// (don't overwrite original error though; might still be relevant)
+	var err2 error
+	host, port, err2 = net.SplitHostPort(a + ":0")
+	if err2 == nil {
+		err = nil
+		port = ""
+	}
 	return
 }
 
@@ -499,6 +439,35 @@ func JoinNetworkAddress(network, host, port string) string {
 	return a
 }
 
+// RegisterNetwork registers a network type with Caddy so that if a listener is
+// created for that network type, getListener will be invoked to get the listener.
+// This should be called during init() and will panic if the network type is standard
+// or reserved, or if it is already registered. EXPERIMENTAL and subject to change.
+func RegisterNetwork(network string, getListener ListenerFunc) {
+	network = strings.TrimSpace(strings.ToLower(network))
+
+	if network == "tcp" || network == "tcp4" || network == "tcp6" ||
+		network == "udp" || network == "udp4" || network == "udp6" ||
+		network == "unix" || network == "unixpacket" || network == "unixgram" ||
+		strings.HasPrefix("ip:", network) || strings.HasPrefix("ip4:", network) || strings.HasPrefix("ip6:", network) {
+		panic("network type " + network + " is reserved")
+	}
+
+	if _, ok := networkTypes[strings.ToLower(network)]; ok {
+		panic("network type " + network + " is already registered")
+	}
+
+	networkTypes[network] = getListener
+}
+
+// ListenerFunc is a function that can return a listener given a network and address.
+// The listeners must be capable of overlapping: with Caddy, new configs are loaded
+// before old ones are unloaded, so listeners may overlap briefly if the configs
+// both need the same listener. EXPERIMENTAL and subject to change.
+type ListenerFunc func(network, addr string) (net.Listener, error)
+
+var networkTypes = map[string]ListenerFunc{}
+
 // ListenerWrapper is a type that wraps a listener
 // so it can modify the input listener's methods.
 // Modules that implement this interface are found

listeners_fuzz.go

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 //go:build gofuzz
-// +build gofuzz
 
 package caddy
 

listeners_test.go

@@ -32,9 +32,24 @@ func TestSplitNetworkAddress(t *testing.T) {
 			expectErr: true,
 		},
 		{
-			input:     "foo",
+			input:      "foo",
+			expectHost: "foo",
+		},
+		{
+			input: ":", // empty host & empty port
+		},
+		{
+			input:     "::",
 			expectErr: true,
 		},
+		{
+			input:      "[::]",
+			expectHost: "::",
+		},
+		{
+			input:      ":1234",
+			expectPort: "1234",
+		},
 		{
 			input:      "foo:1234",
 			expectHost: "foo",
@@ -80,10 +95,10 @@ func TestSplitNetworkAddress(t *testing.T) {
 	} {
 		actualNetwork, actualHost, actualPort, err := SplitNetworkAddress(tc.input)
 		if tc.expectErr && err == nil {
-			t.Errorf("Test %d: Expected error but got: %v", i, err)
+			t.Errorf("Test %d: Expected error but got %v", i, err)
 		}
 		if !tc.expectErr && err != nil {
-			t.Errorf("Test %d: Expected no error but got: %v", i, err)
+			t.Errorf("Test %d: Expected no error but got %v", i, err)
 		}
 		if actualNetwork != tc.expectNetwork {
 			t.Errorf("Test %d: Expected network '%s' but got '%s'", i, tc.expectNetwork, actualNetwork)
@@ -169,8 +184,17 @@ func TestParseNetworkAddress(t *testing.T) {
 			expectErr: true,
 		},
 		{
-			input:     ":",
-			expectErr: true,
+			input: ":",
+			expectAddr: NetworkAddress{
+				Network: "tcp",
+			},
+		},
+		{
+			input: "[::]",
+			expectAddr: NetworkAddress{
+				Network: "tcp",
+				Host:    "::",
+			},
 		},
 		{
 			input: ":1234",
@@ -307,3 +331,85 @@ func TestJoinHostPort(t *testing.T) {
 		}
 	}
 }
+
+func TestExpand(t *testing.T) {
+	for i, tc := range []struct {
+		input  NetworkAddress
+		expect []NetworkAddress
+	}{
+		{
+			input: NetworkAddress{
+				Network:   "tcp",
+				Host:      "localhost",
+				StartPort: 2000,
+				EndPort:   2000,
+			},
+			expect: []NetworkAddress{
+				{
+					Network:   "tcp",
+					Host:      "localhost",
+					StartPort: 2000,
+					EndPort:   2000,
+				},
+			},
+		},
+		{
+			input: NetworkAddress{
+				Network:   "tcp",
+				Host:      "localhost",
+				StartPort: 2000,
+				EndPort:   2002,
+			},
+			expect: []NetworkAddress{
+				{
+					Network:   "tcp",
+					Host:      "localhost",
+					StartPort: 2000,
+					EndPort:   2000,
+				},
+				{
+					Network:   "tcp",
+					Host:      "localhost",
+					StartPort: 2001,
+					EndPort:   2001,
+				},
+				{
+					Network:   "tcp",
+					Host:      "localhost",
+					StartPort: 2002,
+					EndPort:   2002,
+				},
+			},
+		},
+		{
+			input: NetworkAddress{
+				Network:   "tcp",
+				Host:      "localhost",
+				StartPort: 2000,
+				EndPort:   1999,
+			},
+			expect: []NetworkAddress{},
+		},
+		{
+			input: NetworkAddress{
+				Network:   "unix",
+				Host:      "/foo/bar",
+				StartPort: 0,
+				EndPort:   0,
+			},
+			expect: []NetworkAddress{
+				{
+					Network:   "unix",
+					Host:      "/foo/bar",
+					StartPort: 0,
+					EndPort:   0,
+				},
+			},
+		},
+	} {
+		actual := tc.input.Expand()
+		if !reflect.DeepEqual(actual, tc.expect) {
+			t.Errorf("Test %d: Expected %+v but got %+v", i, tc.expect, actual)
+		}
+	}
+}

modules.go

@@ -44,7 +44,7 @@ import (
 // Provisioner, the Provision() method is called. 4) If the
 // module is a Validator, the Validate() method is called.
 // 5) The module will probably be type-asserted from
-// interface{} to some other, more useful interface expected
+// 'any' to some other, more useful interface expected
 // by the host module. For example, HTTP handler modules are
 // type-asserted as caddyhttp.MiddlewareHandler values.
 // 6) When a module's containing Context is canceled, if it is
@@ -172,7 +172,7 @@ func GetModule(name string) (ModuleInfo, error) {
 // GetModuleName returns a module's name (the last label of its ID)
 // from an instance of its value. If the value is not a module, an
 // empty string will be returned.
-func GetModuleName(instance interface{}) string {
+func GetModuleName(instance any) string {
 	var name string
 	if mod, ok := instance.(Module); ok {
 		name = mod.CaddyModule().ID.Name()
@@ -182,7 +182,7 @@ func GetModuleName(instance interface{}) string {
 
 // GetModuleID returns a module's ID from an instance of its value.
 // If the value is not a module, an empty string will be returned.
-func GetModuleID(instance interface{}) string {
+func GetModuleID(instance any) string {
 	var id string
 	if mod, ok := instance.(Module); ok {
 		id = string(mod.CaddyModule().ID)
@@ -259,7 +259,7 @@ func Modules() []string {
 // where raw must be a JSON encoding of a map. It returns that value,
 // along with the result of removing that key from raw.
 func getModuleNameInline(moduleNameKey string, raw json.RawMessage) (string, json.RawMessage, error) {
-	var tmp map[string]interface{}
+	var tmp map[string]any
 	err := json.Unmarshal(raw, &tmp)
 	if err != nil {
 		return "", nil, err
@@ -324,11 +324,11 @@ func ParseStructTag(tag string) (map[string]string, error) {
 		if pair == "" {
 			continue
 		}
-		parts := strings.SplitN(pair, "=", 2)
-		if len(parts) != 2 {
+		before, after, isCut := strings.Cut(pair, "=")
+		if !isCut {
 			return nil, fmt.Errorf("missing key in '%s' (pair %d)", pair, i)
 		}
-		results[parts[0]] = parts[1]
+		results[before] = after
 	}
 	return results, nil
 }
@@ -337,7 +337,7 @@ func ParseStructTag(tag string) (map[string]string, error) {
 // if any of the fields are unrecognized. Useful when decoding
 // module configurations, where you want to be more sure they're
 // correct.
-func strictUnmarshalJSON(data []byte, v interface{}) error {
+func strictUnmarshalJSON(data []byte, v any) error {
 	dec := json.NewDecoder(bytes.NewReader(data))
 	dec.DisallowUnknownFields()
 	return dec.Decode(v)

modules/caddyevents/app.go

@@ -0,0 +1,373 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddyevents
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/google/uuid"
+	"go.uber.org/zap"
+)
+
+func init() {
+	caddy.RegisterModule(App{})
+}
+
+// App implements a global eventing system within Caddy.
+// Modules can emit and subscribe to events, providing
+// hooks into deep parts of the code base that aren't
+// otherwise accessible. Events provide information about
+// what and when things are happening, and this facility
+// allows handlers to take action when events occur,
+// add information to the event's metadata, and even
+// control program flow in some cases.
+//
+// Events are propagated in a DOM-like fashion. An event
+// emitted from module `a.b.c` (the "origin") will first
+// invoke handlers listening to `a.b.c`, then `a.b`,
+// then `a`, then those listening regardless of origin.
+// If a handler returns the special error Aborted, then
+// propagation immediately stops and the event is marked
+// as aborted. Emitters may optionally choose to adjust
+// program flow based on an abort.
+//
+// Modules can subscribe to events by origin and/or name.
+// A handler is invoked only if it is subscribed to the
+// event by name and origin. Subscriptions should be
+// registered during the provisioning phase, before apps
+// are started.
+//
+// Event handlers are fired synchronously as part of the
+// regular flow of the program. This allows event handlers
+// to control the flow of the program if the origin permits
+// it and also allows handlers to convey new information
+// back into the origin module before it continues.
+// In essence, event handlers are similar to HTTP
+// middleware handlers.
+//
+// Event bindings/subscribers are unordered; i.e.
+// event handlers are invoked in an arbitrary order.
+// Event handlers should not rely on the logic of other
+// handlers to succeed.
+//
+// The entirety of this app module is EXPERIMENTAL and
+// subject to change. Pay attention to release notes.
+type App struct {
+	// Subscriptions bind handlers to one or more events
+	// either globally or scoped to specific modules or module
+	// namespaces.
+	Subscriptions []*Subscription `json:"subscriptions,omitempty"`
+
+	// Map of event name to map of module ID/namespace to handlers
+	subscriptions map[string]map[caddy.ModuleID][]Handler
+
+	logger  *zap.Logger
+	started bool
+}
+
+// Subscription represents binding of one or more handlers to
+// one or more events.
+type Subscription struct {
+	// The name(s) of the event(s) to bind to. Default: all events.
+	Events []string `json:"events,omitempty"`
+
+	// The ID or namespace of the module(s) from which events
+	// originate to listen to for events. Default: all modules.
+	//
+	// Events propagate up, so events emitted by module "a.b.c"
+	// will also trigger the event for "a.b" and "a". Thus, to
+	// receive all events from "a.b.c" and "a.b.d", for example,
+	// one can subscribe to either "a.b" or all of "a" entirely.
+	Modules []caddy.ModuleID `json:"modules,omitempty"`
+
+	// The event handler modules. These implement the actual
+	// behavior to invoke when an event occurs. At least one
+	// handler is required.
+	HandlersRaw []json.RawMessage `json:"handlers,omitempty" caddy:"namespace=events.handlers inline_key=handler"`
+
+	// The decoded handlers; Go code that is subscribing to
+	// an event should set this field directly; HandlersRaw
+	// is meant for JSON configuration to fill out this field.
+	Handlers []Handler `json:"-"`
+}
+
+// CaddyModule returns the Caddy module information.
+func (App) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "events",
+		New: func() caddy.Module { return new(App) },
+	}
+}
+
+// Provision sets up the app.
+func (app *App) Provision(ctx caddy.Context) error {
+	app.logger = ctx.Logger()
+	app.subscriptions = make(map[string]map[caddy.ModuleID][]Handler)
+
+	for _, sub := range app.Subscriptions {
+		if sub.HandlersRaw != nil {
+			handlersIface, err := ctx.LoadModule(sub, "HandlersRaw")
+			if err != nil {
+				return fmt.Errorf("loading event subscriber modules: %v", err)
+			}
+			for _, h := range handlersIface.([]any) {
+				sub.Handlers = append(sub.Handlers, h.(Handler))
+			}
+			if len(sub.Handlers) == 0 {
+				// pointless to bind without any handlers
+				return fmt.Errorf("no handlers defined")
+			}
+		}
+	}
+
+	return nil
+}
+
+// Start runs the app.
+func (app *App) Start() error {
+	for _, sub := range app.Subscriptions {
+		if err := app.Subscribe(sub); err != nil {
+			return err
+		}
+	}
+
+	app.started = true
+
+	return nil
+}
+
+// Stop gracefully shuts down the app.
+func (app *App) Stop() error {
+	return nil
+}
+
+// Subscribe binds one or more event handlers to one or more events
+// according to the subscription s. For now, subscriptions can only
+// be created during the provision phase; new bindings cannot be
+// created after the events app has started.
+func (app *App) Subscribe(s *Subscription) error {
+	if app.started {
+		return fmt.Errorf("events already started; new subscriptions closed")
+	}
+
+	// handle special case of catch-alls (omission of event name or module space implies all)
+	if len(s.Events) == 0 {
+		s.Events = []string{""}
+	}
+	if len(s.Modules) == 0 {
+		s.Modules = []caddy.ModuleID{""}
+	}
+
+	for _, eventName := range s.Events {
+		if app.subscriptions[eventName] == nil {
+			app.subscriptions[eventName] = make(map[caddy.ModuleID][]Handler)
+		}
+		for _, originModule := range s.Modules {
+			app.subscriptions[eventName][originModule] = append(app.subscriptions[eventName][originModule], s.Handlers...)
+		}
+	}
+
+	return nil
+}
+
+// On is syntactic sugar for Subscribe() that binds a single handler
+// to a single event from any module. If the eventName is empty string,
+// it counts for all events.
+func (app *App) On(eventName string, handler Handler) error {
+	return app.Subscribe(&Subscription{
+		Events:   []string{eventName},
+		Handlers: []Handler{handler},
+	})
+}
+
+// Emit creates and dispatches an event named eventName to all relevant handlers with
+// the metadata data. Events are emitted and propagated synchronously. The returned Event
+// value will have any additional information from the invoked handlers.
+func (app *App) Emit(ctx caddy.Context, eventName string, data map[string]any) Event {
+	logger := app.logger.With(zap.String("name", eventName))
+
+	id, err := uuid.NewRandom()
+	if err != nil {
+		logger.Error("failed generating new event ID", zap.Error(err))
+	}
+
+	eventName = strings.ToLower(eventName)
+
+	e := Event{
+		id:     id,
+		ts:     time.Now(),
+		name:   eventName,
+		origin: ctx.Module(),
+		data:   data,
+	}
+
+	logger = logger.With(
+		zap.String("id", e.id.String()),
+		zap.String("origin", e.origin.CaddyModule().String()))
+
+	// add event info to replacer, make sure it's in the context
+	repl, ok := ctx.Context.Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
+	if !ok {
+		repl = caddy.NewReplacer()
+		ctx.Context = context.WithValue(ctx.Context, caddy.ReplacerCtxKey, repl)
+	}
+	repl.Map(func(key string) (any, bool) {
+		switch key {
+		case "event":
+			return e, true
+		case "event.id":
+			return e.id, true
+		case "event.name":
+			return e.name, true
+		case "event.time":
+			return e.ts, true
+		case "event.time_unix":
+			return e.ts.UnixMilli(), true
+		case "event.module":
+			return e.origin.CaddyModule().ID, true
+		case "event.data":
+			return e.data, true
+		}
+
+		if strings.HasPrefix(key, "event.data.") {
+			key = strings.TrimPrefix(key, "event.data.")
+			if val, ok := data[key]; ok {
+				return val, true
+			}
+		}
+
+		return nil, false
+	})
+
+	logger.Debug("event", zap.Any("data", e.data))
+
+	// invoke handlers bound to the event by name and also all events; this for loop
+	// iterates twice at most: once for the event name, once for "" (all events)
+	for {
+		moduleID := e.origin.CaddyModule().ID
+
+		// implement propagation up the module tree (i.e. start with "a.b.c" then "a.b" then "a" then "")
+		for {
+			if app.subscriptions[eventName] == nil {
+				break // shortcut if event not bound at all
+			}
+
+			for _, handler := range app.subscriptions[eventName][moduleID] {
+				select {
+				case <-ctx.Done():
+					logger.Error("context canceled; event handling stopped")
+					return e
+				default:
+				}
+
+				if err := handler.Handle(ctx, e); err != nil {
+					aborted := errors.Is(err, ErrAborted)
+
+					logger.Error("handler error",
+						zap.Error(err),
+						zap.Bool("aborted", aborted))
+
+					if aborted {
+						e.Aborted = err
+						return e
+					}
+				}
+			}
+
+			if moduleID == "" {
+				break
+			}
+			lastDot := strings.LastIndex(string(moduleID), ".")
+			if lastDot < 0 {
+				moduleID = "" // include handlers bound to events regardless of module
+			} else {
+				moduleID = moduleID[:lastDot]
+			}
+		}
+
+		// include handlers listening to all events
+		if eventName == "" {
+			break
+		}
+		eventName = ""
+	}
+
+	return e
+}
+
+// Event represents something that has happened or is happening.
+type Event struct {
+	id     uuid.UUID
+	ts     time.Time
+	name   string
+	origin caddy.Module
+	data   map[string]any
+
+	// If non-nil, the event has been aborted, meaning
+	// propagation has stopped to other handlers and
+	// the code should stop what it was doing. Emitters
+	// may choose to use this as a signal to adjust their
+	// code path appropriately.
+	Aborted error
+}
+
+// CloudEvent exports event e as a structure that, when
+// serialized as JSON, is compatible with the
+// CloudEvents spec.
+func (e Event) CloudEvent() CloudEvent {
+	dataJSON, _ := json.Marshal(e.data)
+	return CloudEvent{
+		ID:              e.id.String(),
+		Source:          e.origin.CaddyModule().String(),
+		SpecVersion:     "1.0",
+		Type:            e.name,
+		Time:            e.ts,
+		DataContentType: "application/json",
+		Data:            dataJSON,
+	}
+}
+
+// CloudEvent is a JSON-serializable structure that
+// is compatible with the CloudEvents specification.
+// See https://cloudevents.io.
+type CloudEvent struct {
+	ID              string          `json:"id"`
+	Source          string          `json:"source"`
+	SpecVersion     string          `json:"specversion"`
+	Type            string          `json:"type"`
+	Time            time.Time       `json:"time"`
+	DataContentType string          `json:"datacontenttype,omitempty"`
+	Data            json.RawMessage `json:"data,omitempty"`
+}
+
+// ErrAborted cancels an event.
+var ErrAborted = errors.New("event aborted")
+
+// Handler is a type that can handle events.
+type Handler interface {
+	Handle(context.Context, Event) error
+}
+
+// Interface guards
+var (
+	_ caddy.App         = (*App)(nil)
+	_ caddy.Provisioner = (*App)(nil)
+)

modules/caddyevents/eventsconfig/caddyfile.go

@@ -0,0 +1,88 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package eventsconfig is for configuring caddyevents.App with the
+// Caddyfile. This code can't be in the caddyevents package because
+// the httpcaddyfile package imports caddyhttp, which imports
+// caddyevents: hence, it creates an import cycle.
+package eventsconfig
+
+import (
+	"encoding/json"
+
+	"github.com/caddyserver/caddy/v2/caddyconfig"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
+	"github.com/caddyserver/caddy/v2/modules/caddyevents"
+)
+
+func init() {
+	httpcaddyfile.RegisterGlobalOption("events", parseApp)
+}
+
+// parseApp configures the "events" global option from Caddyfile to set up the events app.
+// Syntax:
+//
+//	events {
+//		on <event> <handler_module...>
+//	}
+//
+// If <event> is *, then it will bind to all events.
+func parseApp(d *caddyfile.Dispenser, _ any) (any, error) {
+	app := new(caddyevents.App)
+
+	// consume the option name
+	if !d.Next() {
+		return nil, d.ArgErr()
+	}
+
+	// handle the block
+	for d.NextBlock(0) {
+		switch d.Val() {
+		case "on":
+			if !d.NextArg() {
+				return nil, d.ArgErr()
+			}
+			eventName := d.Val()
+			if eventName == "*" {
+				eventName = ""
+			}
+
+			if !d.NextArg() {
+				return nil, d.ArgErr()
+			}
+			handlerName := d.Val()
+			modID := "events.handlers." + handlerName
+			unm, err := caddyfile.UnmarshalModule(d, modID)
+			if err != nil {
+				return nil, err
+			}
+
+			app.Subscriptions = append(app.Subscriptions, &caddyevents.Subscription{
+				Events: []string{eventName},
+				HandlersRaw: []json.RawMessage{
+					caddyconfig.JSONModuleObject(unm, "handler", handlerName, nil),
+				},
+			})
+
+		default:
+			return nil, d.ArgErr()
+		}
+	}
+
+	return httpcaddyfile.App{
+		Name:  "events",
+		Value: caddyconfig.JSON(app, nil),
+	}, nil
+}

modules/caddyhttp/app.go

@@ -20,11 +20,12 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddyevents"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
-	"github.com/lucas-clemente/quic-go/http3"
 	"go.uber.org/zap"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
@@ -95,6 +96,8 @@ func init() {
 // `{http.request.uri}` | The full request URI
 // `{http.response.header.*}` | Specific response header field
 // `{http.vars.*}` | Custom variables in the HTTP handler chain
+// `{http.shutting_down}` | True if the HTTP app is shutting down
+// `{http.time_until_shutdown}` | Time until HTTP server shutdown, if scheduled
 type App struct {
 	// HTTPPort specifies the port to use for HTTP (as opposed to HTTPS),
 	// which is used when setting up HTTP->HTTPS redirects or ACME HTTP
@@ -107,18 +110,31 @@ type App struct {
 	HTTPSPort int `json:"https_port,omitempty"`
 
 	// GracePeriod is how long to wait for active connections when shutting
-	// down the server. Once the grace period is over, connections will
-	// be forcefully closed.
+	// down the servers. During the grace period, no new connections are
+	// accepted, idle connections are closed, and active connections will
+	// be given the full length of time to become idle and close.
+	// Once the grace period is over, connections will be forcefully closed.
+	// If zero, the grace period is eternal. Default: 0.
 	GracePeriod caddy.Duration `json:"grace_period,omitempty"`
 
+	// ShutdownDelay is how long to wait before initiating the grace
+	// period. When this app is stopping (e.g. during a config reload or
+	// process exit), all servers will be shut down. Normally this immediately
+	// initiates the grace period. However, if this delay is configured, servers
+	// will not be shut down until the delay is over. During this time, servers
+	// continue to function normally and allow new connections. At the end, the
+	// grace period will begin. This can be useful to allow downstream load
+	// balancers time to move this instance out of the rotation without hiccups.
+	//
+	// When shutdown has been scheduled, placeholders {http.shutting_down} (bool)
+	// and {http.time_until_shutdown} (duration) may be useful for health checks.
+	ShutdownDelay caddy.Duration `json:"shutdown_delay,omitempty"`
+
 	// Servers is the list of servers, keyed by arbitrary names chosen
 	// at your discretion for your own convenience; the keys do not
 	// affect functionality.
 	Servers map[string]*Server `json:"servers,omitempty"`
 
-	servers   []*http.Server
-	h3servers []*http3.Server
-
 	ctx    caddy.Context
 	logger *zap.Logger
 	tlsApp *caddytls.TLS
@@ -144,7 +160,12 @@ func (app *App) Provision(ctx caddy.Context) error {
 	}
 	app.tlsApp = tlsAppIface.(*caddytls.TLS)
 	app.ctx = ctx
-	app.logger = ctx.Logger(app)
+	app.logger = ctx.Logger()
+
+	eventsAppIface, err := ctx.App("events")
+	if err != nil {
+		return fmt.Errorf("getting events app: %v", err)
+	}
 
 	repl := caddy.NewReplacer()
 
@@ -157,17 +178,33 @@ func (app *App) Provision(ctx caddy.Context) error {
 	}
 
 	// prepare each server
+	oldContext := ctx.Context
 	for srvName, srv := range app.Servers {
+		ctx.Context = context.WithValue(oldContext, ServerCtxKey, srv)
 		srv.name = srvName
 		srv.tlsApp = app.tlsApp
+		srv.events = eventsAppIface.(*caddyevents.App)
+		srv.ctx = ctx
 		srv.logger = app.logger.Named("log")
 		srv.errorLogger = app.logger.Named("log.error")
+		srv.shutdownAtMu = new(sync.RWMutex)
 
 		// only enable access logs if configured
 		if srv.Logs != nil {
 			srv.accessLogger = app.logger.Named("log.access")
 		}
 
+		// the Go standard library does not let us serve only HTTP/2 using
+		// http.Server; we would probably need to write our own server
+		if !srv.protocol("h1") && (srv.protocol("h2") || srv.protocol("h2c")) {
+			return fmt.Errorf("server %s: cannot enable HTTP/2 or H2C without enabling HTTP/1.1; add h1 to protocols or remove h2/h2c", srvName)
+		}
+
+		// if no protocols configured explicitly, enable all except h2c
+		if len(srv.Protocols) == 0 {
+			srv.Protocols = []string{"h1", "h2", "h3"}
+		}
+
 		// if not explicitly configured by the user, disallow TLS
 		// client auth bypass (domain fronting) which could
 		// otherwise be exploited by sending an unprotected SNI
@@ -179,8 +216,7 @@ func (app *App) Provision(ctx caddy.Context) error {
 		// based on hostname
 		if srv.StrictSNIHost == nil && srv.hasTLSClientAuth() {
 			app.logger.Warn("enabling strict SNI-Host enforcement because TLS client auth is configured",
-				zap.String("server_id", srvName),
-			)
+				zap.String("server_id", srvName))
 			trueBool := true
 			srv.StrictSNIHost = &trueBool
 		}
@@ -189,8 +225,7 @@ func (app *App) Provision(ctx caddy.Context) error {
 		for i := range srv.Listen {
 			lnOut, err := repl.ReplaceOrErr(srv.Listen[i], true, true)
 			if err != nil {
-				return fmt.Errorf("server %s, listener %d: %v",
-					srvName, i, err)
+				return fmt.Errorf("server %s, listener %d: %v", srvName, i, err)
 			}
 			srv.Listen[i] = lnOut
 		}
@@ -202,7 +237,7 @@ func (app *App) Provision(ctx caddy.Context) error {
 				return fmt.Errorf("loading listener wrapper modules: %v", err)
 			}
 			var hasTLSPlaceholder bool
-			for i, val := range vals.([]interface{}) {
+			for i, val := range vals.([]any) {
 				if _, ok := val.(*tlsPlaceholderWrapper); ok {
 					if i == 0 {
 						// putting the tls placeholder wrapper first is nonsensical because
@@ -230,7 +265,7 @@ func (app *App) Provision(ctx caddy.Context) error {
 		// route handler so that important security checks are done, etc.
 		primaryRoute := emptyHandler
 		if srv.Routes != nil {
-			err := srv.Routes.ProvisionHandlers(ctx)
+			err := srv.Routes.ProvisionHandlers(ctx, srv.Metrics)
 			if err != nil {
 				return fmt.Errorf("server %s: setting up route handlers: %v", srvName, err)
 			}
@@ -260,7 +295,7 @@ func (app *App) Provision(ctx caddy.Context) error {
 			srv.IdleTimeout = defaultIdleTimeout
 		}
 	}
-
+	ctx.Context = oldContext
 	return nil
 }
 
@@ -298,7 +333,7 @@ func (app *App) Start() error {
 	}
 
 	for srvName, srv := range app.Servers {
-		s := &http.Server{
+		srv.server = &http.Server{
 			ReadTimeout:       time.Duration(srv.ReadTimeout),
 			ReadHeaderTimeout: time.Duration(srv.ReadHeaderTimeout),
 			WriteTimeout:      time.Duration(srv.WriteTimeout),
@@ -308,12 +343,38 @@ func (app *App) Start() error {
 			ErrorLog:          serverLogger,
 		}
 
-		// enable h2c if configured
-		if srv.AllowH2C {
+		// disable HTTP/2, which we enabled by default during provisioning
+		if !srv.protocol("h2") {
+			srv.server.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+			for _, cp := range srv.TLSConnPolicies {
+				// the TLSConfig was already provisioned, so... manually remove it
+				for i, np := range cp.TLSConfig.NextProtos {
+					if np == "h2" {
+						cp.TLSConfig.NextProtos = append(cp.TLSConfig.NextProtos[:i], cp.TLSConfig.NextProtos[i+1:]...)
+						break
+					}
+				}
+				// remove it from the parent connection policy too, just to keep things tidy
+				for i, alpn := range cp.ALPN {
+					if alpn == "h2" {
+						cp.ALPN = append(cp.ALPN[:i], cp.ALPN[i+1:]...)
+						break
+					}
+				}
+			}
+		}
+
+		// this TLS config is used by the std lib to choose the actual TLS config for connections
+		// by looking through the connection policies to find the first one that matches
+		tlsCfg := srv.TLSConnPolicies.TLSConfig(app.ctx)
+		srv.configureServer(srv.server)
+
+		// enable H2C if configured
+		if srv.protocol("h2c") {
 			h2server := &http2.Server{
 				IdleTimeout: time.Duration(srv.IdleTimeout),
 			}
-			s.Handler = h2c.NewHandler(srv, h2server)
+			srv.server.Handler = h2c.NewHandler(srv, h2server)
 		}
 
 		for _, lnAddr := range srv.Listen {
@@ -321,10 +382,12 @@ func (app *App) Start() error {
 			if err != nil {
 				return fmt.Errorf("%s: parsing listen address '%s': %v", srvName, lnAddr, err)
 			}
+			srv.addresses = append(srv.addresses, listenAddr)
+
 			for portOffset := uint(0); portOffset < listenAddr.PortRangeSize(); portOffset++ {
 				// create the listener for this socket
 				hostport := listenAddr.JoinHostPort(portOffset)
-				ln, err := caddy.Listen(listenAddr.Network, hostport)
+				ln, err := caddy.ListenTimeout(listenAddr.Network, hostport, time.Duration(srv.KeepAliveInterval))
 				if err != nil {
 					return fmt.Errorf("%s: listening on %s: %v", listenAddr.Network, hostport, err)
 				}
@@ -342,34 +405,16 @@ func (app *App) Start() error {
 				// enable TLS if there is a policy and if this is not the HTTP port
 				useTLS := len(srv.TLSConnPolicies) > 0 && int(listenAddr.StartPort+portOffset) != app.httpPort()
 				if useTLS {
-					// create TLS listener
-					tlsCfg := srv.TLSConnPolicies.TLSConfig(app.ctx)
+					// create TLS listener - this enables and terminates TLS
 					ln = tls.NewListener(ln, tlsCfg)
 
-					/////////
-					// TODO: HTTP/3 support is experimental for now
-					if srv.ExperimentalHTTP3 {
-						app.logger.Info("enabling experimental HTTP/3 listener",
-							zap.String("addr", hostport),
-						)
-						h3ln, err := caddy.ListenQUIC(hostport, tlsCfg)
-						if err != nil {
-							return fmt.Errorf("getting HTTP/3 QUIC listener: %v", err)
+					// enable HTTP/3 if configured
+					if srv.protocol("h3") {
+						app.logger.Info("enabling HTTP/3 listener", zap.String("addr", hostport))
+						if err := srv.serveHTTP3(hostport, tlsCfg); err != nil {
+							return err
 						}
-						h3srv := &http3.Server{
-							Server: &http.Server{
-								Addr:      hostport,
-								Handler:   srv,
-								TLSConfig: tlsCfg,
-								ErrorLog:  serverLogger,
-							},
-						}
-						//nolint:errcheck
-						go h3srv.ServeListener(h3ln)
-						app.h3servers = append(app.h3servers, h3srv)
-						srv.h3server = h3srv
 					}
-					/////////
 				}
 
 				// finish wrapping listener where we left off before TLS
@@ -388,15 +433,22 @@ func (app *App) Start() error {
 
 				app.logger.Debug("starting server loop",
 					zap.String("address", ln.Addr().String()),
-					zap.Bool("http3", srv.ExperimentalHTTP3),
 					zap.Bool("tls", useTLS),
-				)
+					zap.Bool("http3", srv.h3server != nil))
 
-				//nolint:errcheck
-				go s.Serve(ln)
-				app.servers = append(app.servers, s)
+				srv.listeners = append(srv.listeners, ln)
+
+				// enable HTTP/1 if configured
+				if srv.protocol("h1") {
+					//nolint:errcheck
+					go srv.server.Serve(ln)
+				}
 			}
 		}
+
+		srv.logger.Info("server running",
+			zap.String("name", srvName),
+			zap.Strings("protocols", srv.Protocols))
 	}
 
 	// finish automatic HTTPS by finally beginning
@@ -412,26 +464,65 @@ func (app *App) Start() error {
 // Stop gracefully shuts down the HTTP server.
 func (app *App) Stop() error {
 	ctx := context.Background()
+
+	// see if any listeners in our config will be closing or if they are continuing
+	// hrough a reload; because if any are closing, we will enforce shutdown delay
+	var delay bool
+	scheduledTime := time.Now().Add(time.Duration(app.ShutdownDelay))
+	if app.ShutdownDelay > 0 {
+		for _, server := range app.Servers {
+			for _, na := range server.addresses {
+				for _, addr := range na.Expand() {
+					if caddy.ListenerUsage(addr.Network, addr.JoinHostPort(0)) < 2 {
+						app.logger.Debug("listener closing and shutdown delay is configured", zap.String("address", addr.String()))
+						server.shutdownAtMu.Lock()
+						server.shutdownAt = scheduledTime
+						server.shutdownAtMu.Unlock()
+						delay = true
+					} else {
+						app.logger.Debug("shutdown delay configured but listener will remain open", zap.String("address", addr.String()))
+					}
+				}
+			}
+		}
+	}
+
+	// honor scheduled/delayed shutdown time
+	if delay {
+		app.logger.Debug("shutdown scheduled",
+			zap.Duration("delay_duration", time.Duration(app.ShutdownDelay)),
+			zap.Time("time", scheduledTime))
+		time.Sleep(time.Duration(app.ShutdownDelay))
+	}
+
+	// enforce grace period if configured
 	if app.GracePeriod > 0 {
 		var cancel context.CancelFunc
 		ctx, cancel = context.WithTimeout(ctx, time.Duration(app.GracePeriod))
 		defer cancel()
+		app.logger.Debug("servers shutting down; grace period initiated", zap.Duration("duration", time.Duration(app.GracePeriod)))
+	} else {
+		app.logger.Debug("servers shutting down with eternal grace period")
 	}
-	for _, s := range app.servers {
-		err := s.Shutdown(ctx)
-		if err != nil {
-			return err
+
+	// shut down servers
+	for _, server := range app.Servers {
+		if err := server.server.Shutdown(ctx); err != nil {
+			app.logger.Error("server shutdown",
+				zap.Error(err),
+				zap.Strings("addresses", server.Listen))
+		}
+
+		if server.h3server != nil {
+			// TODO: CloseGracefully, once implemented upstream (see https://github.com/lucas-clemente/quic-go/issues/2103)
+			if err := server.h3server.Close(); err != nil {
+				app.logger.Error("HTTP/3 server shutdown",
+					zap.Error(err),
+					zap.Strings("addresses", server.Listen))
+			}
 		}
 	}
 
-	for _, s := range app.h3servers {
-		// TODO: CloseGracefully, once implemented upstream
-		// (see https://github.com/lucas-clemente/quic-go/issues/2103)
-		err := s.Close()
-		if err != nil {
-			return err
-		}
-	}
 	return nil
 }
 

modules/caddyhttp/autohttps.go

@@ -93,6 +93,9 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 	// https://github.com/caddyserver/caddy/issues/3443)
 	redirDomains := make(map[string][]caddy.NetworkAddress)
 
+	// the log configuration for an HTTPS enabled server
+	var logCfg *ServerLogConfig
+
 	for srvName, srv := range app.Servers {
 		// as a prerequisite, provision route matchers; this is
 		// required for all routes on all servers, and must be
@@ -152,9 +155,7 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 								return fmt.Errorf("%s: route %d, matcher set %d, matcher %d, host matcher %d: %v",
 									srvName, routeIdx, matcherSetIdx, matcherIdx, hostMatcherIdx, err)
 							}
-							// only include domain if it's not explicitly skipped and it's not a Tailscale domain
-							// (the implicit Tailscale manager module will get those certs at run-time)
-							if !srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.Skip) && !isTailscaleDomain(d) {
+							if !srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.Skip) {
 								serverDomainSet[d] = struct{}{}
 							}
 						}
@@ -174,6 +175,13 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 			continue
 		}
 
+		// clone the logger so we can apply it to the HTTP server
+		// (not sure if necessary to clone it; but probably safer)
+		// (we choose one log cfg arbitrarily; not sure which is best)
+		if srv.Logs != nil {
+			logCfg = srv.Logs.clone()
+		}
+
 		// for all the hostnames we found, filter them so we have
 		// a deduplicated list of names for which to obtain certs
 		// (only if cert management not disabled for this server)
@@ -181,6 +189,11 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 			app.logger.Warn("skipping automated certificate management for server because it is disabled", zap.String("server_name", srvName))
 		} else {
 			for d := range serverDomainSet {
+				// the implicit Tailscale manager module will get its own certs at run-time
+				if isTailscaleDomain(d) {
+					continue
+				}
+
 				if certmagic.SubjectQualifiesForCert(d) &&
 					!srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.SkipCerts) {
 					// if a certificate for this name is already loaded,
@@ -397,6 +410,7 @@ redirServersLoop:
 		app.Servers["remaining_auto_https_redirects"] = &Server{
 			Listen: redirServerAddrsList,
 			Routes: appendCatchAll(redirRoutes),
+			Logs:   logCfg,
 		}
 	}
 
@@ -436,7 +450,7 @@ func (app *App) makeRedirRoute(redirToPort uint, matcherSet MatcherSet) Route {
 	}
 }
 
-// createAutomationPolicy ensures that automated certificates for this
+// createAutomationPolicies ensures that automated certificates for this
 // app are managed properly. This adds up to two automation policies:
 // one for the public names, and one for the internal names. If a catch-all
 // automation policy exists, it will be shallow-copied and used as the
@@ -485,6 +499,12 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []stri
 				return err
 			}
 			ap.Managers = []certmagic.Manager{ts}
+
+			// must reprovision the automation policy so that the underlying
+			// CertMagic config knows about the updated Managers
+			if err := ap.Provision(app.tlsApp); err != nil {
+				return fmt.Errorf("re-provisioning automation policy: %v", err)
+			}
 		}
 
 		// while we're here, is this the catch-all/base policy?
@@ -495,14 +515,17 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []stri
 	}
 
 	if basePolicy == nil {
-		// no base policy found, we will make one! (with implicit Tailscale integration)
+		// no base policy found; we will make one
+		basePolicy = new(caddytls.AutomationPolicy)
+	}
+
+	if basePolicy.Managers == nil {
+		// add implicit Tailscale integration, for harmless convenience
 		ts, err := implicitTailscale(ctx)
 		if err != nil {
 			return err
 		}
-		basePolicy = &caddytls.AutomationPolicy{
-			Managers: []certmagic.Manager{ts},
-		}
+		basePolicy.Managers = []certmagic.Manager{ts}
 	}
 
 	// if the basePolicy has an existing ACMEIssuer (particularly to

modules/caddyhttp/caddyauth/basicauth.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	weakrand "math/rand"
 	"net/http"
+	"strings"
 	"sync"
 	"time"
 
@@ -94,10 +95,7 @@ func (hba *HTTPBasicAuth) Provision(ctx caddy.Context) error {
 
 	// if supported, generate a fake password we can compare against if needed
 	if hasher, ok := hba.Hash.(Hasher); ok {
-		hba.fakePassword, err = hasher.Hash([]byte("antitiming"), []byte("fakesalt"))
-		if err != nil {
-			return fmt.Errorf("generating anti-timing password hash: %v", err)
-		}
+		hba.fakePassword = hasher.FakeHash()
 	}
 
 	repl := caddy.NewReplacer()
@@ -117,10 +115,19 @@ func (hba *HTTPBasicAuth) Provision(ctx caddy.Context) error {
 			return fmt.Errorf("account %d: username and password are required", i)
 		}
 
-		acct.password, err = base64.StdEncoding.DecodeString(acct.Password)
-		if err != nil {
-			return fmt.Errorf("base64-decoding password: %v", err)
+		// TODO: Remove support for redundantly-encoded b64-encoded hashes
+		// Passwords starting with '$' are likely in Modular Crypt Format,
+		// so we don't need to base64 decode them. But historically, we
+		// required redundant base64, so we try to decode it otherwise.
+		if strings.HasPrefix(acct.Password, "$") {
+			acct.password = []byte(acct.Password)
+		} else {
+			acct.password, err = base64.StdEncoding.DecodeString(acct.Password)
+			if err != nil {
+				return fmt.Errorf("base64-decoding password: %v", err)
+			}
 		}
+
 		if acct.Salt != "" {
 			acct.salt, err = base64.StdEncoding.DecodeString(acct.Salt)
 			if err != nil {
@@ -271,9 +278,11 @@ type Comparer interface {
 // that require a salt). Hashing modules which implement
 // this interface can be used with the hash-password
 // subcommand as well as benefitting from anti-timing
-// features.
+// features. A hasher also returns a fake hash which
+// can be used for timing side-channel mitigation.
 type Hasher interface {
 	Hash(plaintext, salt []byte) ([]byte, error)
+	FakeHash() []byte
 }
 
 // Account contains a username, password, and salt (if applicable).

modules/caddyhttp/caddyauth/caddyauth.go

@@ -56,13 +56,13 @@ func (Authentication) CaddyModule() caddy.ModuleInfo {
 
 // Provision sets up a.
 func (a *Authentication) Provision(ctx caddy.Context) error {
-	a.logger = ctx.Logger(a)
+	a.logger = ctx.Logger()
 	a.Providers = make(map[string]Authenticator)
 	mods, err := ctx.LoadModule(a, "ProvidersRaw")
 	if err != nil {
 		return fmt.Errorf("loading authentication providers: %v", err)
 	}
-	for modName, modIface := range mods.(map[string]interface{}) {
+	for modName, modIface := range mods.(map[string]any) {
 		a.Providers[modName] = modIface.(Authenticator)
 	}
 	return nil

modules/caddyhttp/caddyauth/command.go

@@ -42,11 +42,13 @@ hash is written to stdout as a base64 string.
 Caddy is attached to a controlling tty, the plaintext will
 not be echoed.
 
---algorithm may be bcrypt or scrypt. If script, the default
+--algorithm may be bcrypt or scrypt. If scrypt, the default
 parameters are used.
 
 Use the --salt flag for algorithms which require a salt to
 be provided (scrypt).
+
+Note that scrypt is deprecated. Please use 'bcrypt' instead.
 `,
 		Flags: func() *flag.FlagSet {
 			fs := flag.NewFlagSet("hash-password", flag.ExitOnError)
@@ -112,13 +114,16 @@ func cmdHashPassword(fs caddycmd.Flags) (int, error) {
 	}
 
 	var hash []byte
+	var hashString string
 	switch algorithm {
 	case "bcrypt":
 		hash, err = BcryptHash{}.Hash(plaintext, nil)
+		hashString = string(hash)
 	case "scrypt":
 		def := ScryptHash{}
 		def.SetDefaults()
 		hash, err = def.Hash(plaintext, salt)
+		hashString = base64.StdEncoding.EncodeToString(hash)
 	default:
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("unrecognized hash algorithm: %s", algorithm)
 	}
@@ -126,9 +131,7 @@ func cmdHashPassword(fs caddycmd.Flags) (int, error) {
 		return caddy.ExitCodeFailedStartup, err
 	}
 
-	hashBase64 := base64.StdEncoding.EncodeToString(hash)
-
-	fmt.Println(hashBase64)
+	fmt.Println(hashString)
 
 	return 0, nil
 }

modules/caddyhttp/caddyauth/hashes.go

@@ -16,6 +16,7 @@ package caddyauth
 
 import (
 	"crypto/subtle"
+	"encoding/base64"
 
 	"github.com/caddyserver/caddy/v2"
 	"golang.org/x/crypto/bcrypt"
@@ -55,7 +56,16 @@ func (BcryptHash) Hash(plaintext, _ []byte) ([]byte, error) {
 	return bcrypt.GenerateFromPassword(plaintext, 14)
 }
 
+// FakeHash returns a fake hash.
+func (BcryptHash) FakeHash() []byte {
+	// hashed with the following command:
+	// caddy hash-password --plaintext "antitiming" --algorithm "bcrypt"
+	return []byte("$2a$14$X3ulqf/iGxnf1k6oMZ.RZeJUoqI9PX2PM4rS5lkIKJXduLGXGPrt6")
+}
+
 // ScryptHash implements the scrypt KDF as a hash.
+//
+// DEPRECATED, please use 'bcrypt' instead.
 type ScryptHash struct {
 	// scrypt's N parameter. If unset or 0, a safe default is used.
 	N int `json:"N,omitempty"`
@@ -80,8 +90,9 @@ func (ScryptHash) CaddyModule() caddy.ModuleInfo {
 }
 
 // Provision sets up s.
-func (s *ScryptHash) Provision(_ caddy.Context) error {
+func (s *ScryptHash) Provision(ctx caddy.Context) error {
 	s.SetDefaults()
+	ctx.Logger().Warn("use of 'scrypt' is deprecated, please use 'bcrypt' instead")
 	return nil
 }
 
@@ -123,6 +134,14 @@ func (s ScryptHash) Hash(plaintext, salt []byte) ([]byte, error) {
 	return scrypt.Key(plaintext, salt, s.N, s.R, s.P, s.KeyLength)
 }
 
+// FakeHash returns a fake hash.
+func (ScryptHash) FakeHash() []byte {
+	// hashed with the following command:
+	// caddy hash-password --plaintext "antitiming" --salt "fakesalt" --algorithm "scrypt"
+	bytes, _ := base64.StdEncoding.DecodeString("kFbjiVemlwK/ZS0tS6/UQqEDeaNMigyCs48KEsGUse8=")
+	return bytes
+}
+
 func hashesMatch(pwdHash1, pwdHash2 []byte) bool {
 	return subtle.ConstantTimeCompare(pwdHash1, pwdHash2) == 1
 }

modules/caddyhttp/caddyhttp.go

@@ -20,6 +20,7 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -244,6 +245,40 @@ func SanitizedPathJoin(root, reqPath string) string {
 	return path
 }
 
+// CleanPath cleans path p according to path.Clean(), but only
+// merges repeated slashes if collapseSlashes is true, and always
+// preserves trailing slashes.
+func CleanPath(p string, collapseSlashes bool) string {
+	if collapseSlashes {
+		return cleanPath(p)
+	}
+
+	// insert an invalid/impossible URI character into each two consecutive
+	// slashes to expand empty path segments; then clean the path as usual,
+	// and then remove the remaining temporary characters.
+	const tmpCh = 0xff
+	var sb strings.Builder
+	for i, ch := range p {
+		if ch == '/' && i > 0 && p[i-1] == '/' {
+			sb.WriteByte(tmpCh)
+		}
+		sb.WriteRune(ch)
+	}
+	halfCleaned := cleanPath(sb.String())
+	halfCleaned = strings.ReplaceAll(halfCleaned, string([]byte{tmpCh}), "")
+
+	return halfCleaned
+}
+
+// cleanPath does path.Clean(p) but preserves any trailing slash.
+func cleanPath(p string) string {
+	cleaned := path.Clean(p)
+	if cleaned != "/" && strings.HasSuffix(p, "/") {
+		cleaned = cleaned + "/"
+	}
+	return cleaned
+}
+
 // tlsPlaceholderWrapper is a no-op listener wrapper that marks
 // where the TLS listener should be in a chain of listener wrappers.
 // It should only be used if another listener wrapper must be placed

modules/caddyhttp/caddyhttp_test.go

@@ -92,3 +92,60 @@ func TestSanitizedPathJoin(t *testing.T) {
 		}
 	}
 }
+
+func TestCleanPath(t *testing.T) {
+	for i, tc := range []struct {
+		input        string
+		mergeSlashes bool
+		expect       string
+	}{
+		{
+			input:  "/foo",
+			expect: "/foo",
+		},
+		{
+			input:  "/foo/",
+			expect: "/foo/",
+		},
+		{
+			input:  "//foo",
+			expect: "//foo",
+		},
+		{
+			input:        "//foo",
+			mergeSlashes: true,
+			expect:       "/foo",
+		},
+		{
+			input:        "/foo//bar/",
+			mergeSlashes: true,
+			expect:       "/foo/bar/",
+		},
+		{
+			input:  "/foo/./.././bar",
+			expect: "/bar",
+		},
+		{
+			input:  "/foo//./..//./bar",
+			expect: "/foo//bar",
+		},
+		{
+			input:  "/foo///./..//./bar",
+			expect: "/foo///bar",
+		},
+		{
+			input:  "/foo///./..//.",
+			expect: "/foo//",
+		},
+		{
+			input:  "/foo//./bar",
+			expect: "/foo//bar",
+		},
+	} {
+		actual := CleanPath(tc.input, tc.mergeSlashes)
+		if actual != tc.expect {
+			t.Errorf("Test %d [input='%s' mergeSlashes=%t]: Got '%s', expected '%s'",
+				i, tc.input, tc.mergeSlashes, actual, tc.expect)
+		}
+	}
+}

modules/caddyhttp/celmatcher.go

@@ -17,6 +17,7 @@ package caddyhttp
 import (
 	"crypto/x509/pkix"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"reflect"
@@ -27,14 +28,17 @@ import (
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/google/cel-go/cel"
-	"github.com/google/cel-go/checker/decls"
+	"github.com/google/cel-go/common"
+	"github.com/google/cel-go/common/operators"
 	"github.com/google/cel-go/common/types"
 	"github.com/google/cel-go/common/types/ref"
 	"github.com/google/cel-go/common/types/traits"
 	"github.com/google/cel-go/ext"
+	"github.com/google/cel-go/interpreter"
 	"github.com/google/cel-go/interpreter/functions"
+	"github.com/google/cel-go/parser"
+	"go.uber.org/zap"
 	exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
-	"google.golang.org/protobuf/proto"
 )
 
 func init() {
@@ -62,6 +66,8 @@ type MatchExpression struct {
 	expandedExpr string
 	prg          cel.Program
 	ta           ref.TypeAdapter
+
+	log *zap.Logger
 }
 
 // CaddyModule returns the Caddy module information.
@@ -83,7 +89,9 @@ func (m *MatchExpression) UnmarshalJSON(data []byte) error {
 }
 
 // Provision sets ups m.
-func (m *MatchExpression) Provision(_ caddy.Context) error {
+func (m *MatchExpression) Provision(ctx caddy.Context) error {
+	m.log = ctx.Logger()
+
 	// replace placeholders with a function call - this is just some
 	// light (and possibly naïve) syntactic sugar
 	m.expandedExpr = placeholderRegexp.ReplaceAllString(m.Expr, placeholderExpansion)
@@ -91,17 +99,40 @@ func (m *MatchExpression) Provision(_ caddy.Context) error {
 	// our type adapter expands CEL's standard type support
 	m.ta = celTypeAdapter{}
 
+	// initialize the CEL libraries from the Matcher implementations which
+	// have been configured to support CEL.
+	matcherLibProducers := []CELLibraryProducer{}
+	for _, info := range caddy.GetModules("http.matchers") {
+		p, ok := info.New().(CELLibraryProducer)
+		if ok {
+			matcherLibProducers = append(matcherLibProducers, p)
+		}
+	}
+	// Assemble the compilation and program options from the different library
+	// producers into a single cel.Library implementation.
+	matcherEnvOpts := []cel.EnvOption{}
+	matcherProgramOpts := []cel.ProgramOption{}
+	for _, producer := range matcherLibProducers {
+		l, err := producer.CELLibrary(ctx)
+		if err != nil {
+			return fmt.Errorf("error initializing CEL library for %T: %v", producer, err)
+		}
+		matcherEnvOpts = append(matcherEnvOpts, l.CompileOptions()...)
+		matcherProgramOpts = append(matcherProgramOpts, l.ProgramOptions()...)
+	}
+	matcherLib := cel.Lib(NewMatcherCELLibrary(matcherEnvOpts, matcherProgramOpts))
+
 	// create the CEL environment
 	env, err := cel.NewEnv(
-		cel.Declarations(
-			decls.NewVar("request", httpRequestObjectType),
-			decls.NewFunction(placeholderFuncName,
-				decls.NewOverload(placeholderFuncName+"_httpRequest_string",
-					[]*exprpb.Type{httpRequestObjectType, decls.String},
-					decls.Any)),
-		),
+		cel.Function(placeholderFuncName, cel.SingletonBinaryImpl(m.caddyPlaceholderFunc), cel.Overload(
+			placeholderFuncName+"_httpRequest_string",
+			[]*cel.Type{httpRequestObjectType, cel.StringType},
+			cel.AnyType,
+		)),
+		cel.Variable("request", httpRequestObjectType),
 		cel.CustomTypeAdapter(m.ta),
 		ext.Strings(),
+		matcherLib,
 	)
 	if err != nil {
 		return fmt.Errorf("setting up CEL environment: %v", err)
@@ -109,26 +140,18 @@ func (m *MatchExpression) Provision(_ caddy.Context) error {
 
 	// parse and type-check the expression
 	checked, issues := env.Compile(m.expandedExpr)
-	if issues != nil && issues.Err() != nil {
+	if issues.Err() != nil {
 		return fmt.Errorf("compiling CEL program: %s", issues.Err())
 	}
 
 	// request matching is a boolean operation, so we don't really know
 	// what to do if the expression returns a non-boolean type
-	if !proto.Equal(checked.ResultType(), decls.Bool) {
-		return fmt.Errorf("CEL request matcher expects return type of bool, not %s", checked.ResultType())
+	if checked.OutputType() != cel.BoolType {
+		return fmt.Errorf("CEL request matcher expects return type of bool, not %s", checked.OutputType())
 	}
 
 	// compile the "program"
-	m.prg, err = env.Program(checked,
-		cel.Functions(
-			&functions.Overload{
-				Operator: placeholderFuncName,
-				Binary:   m.caddyPlaceholderFunc,
-			},
-		),
-	)
-
+	m.prg, err = env.Program(checked, cel.EvalOptions(cel.OptOptimize))
 	if err != nil {
 		return fmt.Errorf("compiling CEL program: %s", err)
 	}
@@ -137,14 +160,17 @@ func (m *MatchExpression) Provision(_ caddy.Context) error {
 
 // Match returns true if r matches m.
 func (m MatchExpression) Match(r *http.Request) bool {
-	out, _, _ := m.prg.Eval(map[string]interface{}{
-		"request": celHTTPRequest{r},
-	})
+	celReq := celHTTPRequest{r}
+	out, _, err := m.prg.Eval(celReq)
+	if err != nil {
+		m.log.Error("evaluating expression", zap.Error(err))
+		SetVar(r.Context(), MatcherErrorVarKey, err)
+		return false
+	}
 	if outBool, ok := out.Value().(bool); ok {
 		return outBool
 	}
 	return false
-
 }
 
 // UnmarshalCaddyfile implements caddyfile.Unmarshaler.
@@ -166,13 +192,15 @@ func (m MatchExpression) caddyPlaceholderFunc(lhs, rhs ref.Val) ref.Val {
 	if !ok {
 		return types.NewErr(
 			"invalid request of type '%v' to "+placeholderFuncName+"(request, placeholderVarName)",
-			lhs.Type())
+			lhs.Type(),
+		)
 	}
 	phStr, ok := rhs.(types.String)
 	if !ok {
 		return types.NewErr(
 			"invalid placeholder variable name of type '%v' to "+placeholderFuncName+"(request, placeholderVarName)",
-			rhs.Type())
+			rhs.Type(),
+		)
 	}
 
 	repl := celReq.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
@@ -184,11 +212,24 @@ func (m MatchExpression) caddyPlaceholderFunc(lhs, rhs ref.Val) ref.Val {
 // httpRequestCELType is the type representation of a native HTTP request.
 var httpRequestCELType = types.NewTypeValue("http.Request", traits.ReceiverType)
 
-// cellHTTPRequest wraps an http.Request with
-// methods to satisfy the ref.Val interface.
+// celHTTPRequest wraps an http.Request with ref.Val interface methods.
+//
+// This type also implements the interpreter.Activation interface which
+// drops allocation costs for CEL expression evaluations by roughly half.
 type celHTTPRequest struct{ *http.Request }
 
-func (cr celHTTPRequest) ConvertToNative(typeDesc reflect.Type) (interface{}, error) {
+func (cr celHTTPRequest) ResolveName(name string) (any, bool) {
+	if name == "request" {
+		return cr, true
+	}
+	return nil, false
+}
+
+func (cr celHTTPRequest) Parent() interpreter.Activation {
+	return nil
+}
+
+func (cr celHTTPRequest) ConvertToNative(typeDesc reflect.Type) (any, error) {
 	return cr.Request, nil
 }
 func (celHTTPRequest) ConvertToType(typeVal ref.Type) ref.Val {
@@ -200,8 +241,8 @@ func (cr celHTTPRequest) Equal(other ref.Val) ref.Val {
 	}
 	return types.ValOrErr(other, "%v is not comparable type", other)
 }
-func (celHTTPRequest) Type() ref.Type        { return httpRequestCELType }
-func (cr celHTTPRequest) Value() interface{} { return cr }
+func (celHTTPRequest) Type() ref.Type { return httpRequestCELType }
+func (cr celHTTPRequest) Value() any  { return cr }
 
 var pkixNameCELType = types.NewTypeValue("pkix.Name", traits.ReceiverType)
 
@@ -209,7 +250,7 @@ var pkixNameCELType = types.NewTypeValue("pkix.Name", traits.ReceiverType)
 // methods to satisfy the ref.Val interface.
 type celPkixName struct{ *pkix.Name }
 
-func (pn celPkixName) ConvertToNative(typeDesc reflect.Type) (interface{}, error) {
+func (pn celPkixName) ConvertToNative(typeDesc reflect.Type) (any, error) {
 	return pn.Name, nil
 }
 func (celPkixName) ConvertToType(typeVal ref.Type) ref.Val {
@@ -221,13 +262,13 @@ func (pn celPkixName) Equal(other ref.Val) ref.Val {
 	}
 	return types.ValOrErr(other, "%v is not comparable type", other)
 }
-func (celPkixName) Type() ref.Type        { return pkixNameCELType }
-func (pn celPkixName) Value() interface{} { return pn }
+func (celPkixName) Type() ref.Type { return pkixNameCELType }
+func (pn celPkixName) Value() any  { return pn }
 
 // celTypeAdapter can adapt our custom types to a CEL value.
 type celTypeAdapter struct{}
 
-func (celTypeAdapter) NativeToValue(value interface{}) ref.Val {
+func (celTypeAdapter) NativeToValue(value any) ref.Val {
 	switch v := value.(type) {
 	case celHTTPRequest:
 		return v
@@ -241,15 +282,385 @@ func (celTypeAdapter) NativeToValue(value interface{}) ref.Val {
 	return types.DefaultTypeAdapter.NativeToValue(value)
 }
 
+// CELLibraryProducer provide CEL libraries that expose a Matcher
+// implementation as a first class function within the CEL expression
+// matcher.
+type CELLibraryProducer interface {
+	// CELLibrary creates a cel.Library which makes it possible to use the
+	// target object within CEL expression matchers.
+	CELLibrary(caddy.Context) (cel.Library, error)
+}
+
+// CELMatcherImpl creates a new cel.Library based on the following pieces of
+// data:
+//
+//   - macroName: the function name to be used within CEL. This will be a macro
+//     and not a function proper.
+//   - funcName: the function overload name generated by the CEL macro used to
+//     represent the matcher.
+//   - matcherDataTypes: the argument types to the macro.
+//   - fac: a matcherFactory implementation which converts from CEL constant
+//     values to a Matcher instance.
+//
+// Note, macro names and function names must not collide with other macros or
+// functions exposed within CEL expressions, or an error will be produced
+// during the expression matcher plan time.
+//
+// The existing CELMatcherImpl support methods are configured to support a
+// limited set of function signatures. For strong type validation you may need
+// to provide a custom macro which does a more detailed analysis of the CEL
+// literal provided to the macro as an argument.
+func CELMatcherImpl(macroName, funcName string, matcherDataTypes []*cel.Type, fac CELMatcherFactory) (cel.Library, error) {
+	requestType := cel.ObjectType("http.Request")
+	var macro parser.Macro
+	switch len(matcherDataTypes) {
+	case 1:
+		matcherDataType := matcherDataTypes[0]
+		switch matcherDataType.String() {
+		case "list(string)":
+			macro = parser.NewGlobalVarArgMacro(macroName, celMatcherStringListMacroExpander(funcName))
+		case cel.StringType.String():
+			macro = parser.NewGlobalMacro(macroName, 1, celMatcherStringMacroExpander(funcName))
+		case CELTypeJSON.String():
+			macro = parser.NewGlobalMacro(macroName, 1, celMatcherJSONMacroExpander(funcName))
+		default:
+			return nil, fmt.Errorf("unsupported matcher data type: %s", matcherDataType)
+		}
+	case 2:
+		if matcherDataTypes[0] == cel.StringType && matcherDataTypes[1] == cel.StringType {
+			macro = parser.NewGlobalMacro(macroName, 2, celMatcherStringListMacroExpander(funcName))
+			matcherDataTypes = []*cel.Type{cel.ListType(cel.StringType)}
+		} else {
+			return nil, fmt.Errorf("unsupported matcher data type: %s, %s", matcherDataTypes[0], matcherDataTypes[1])
+		}
+	case 3:
+		if matcherDataTypes[0] == cel.StringType && matcherDataTypes[1] == cel.StringType && matcherDataTypes[2] == cel.StringType {
+			macro = parser.NewGlobalMacro(macroName, 3, celMatcherStringListMacroExpander(funcName))
+			matcherDataTypes = []*cel.Type{cel.ListType(cel.StringType)}
+		} else {
+			return nil, fmt.Errorf("unsupported matcher data type: %s, %s, %s", matcherDataTypes[0], matcherDataTypes[1], matcherDataTypes[2])
+		}
+	}
+	envOptions := []cel.EnvOption{
+		cel.Macros(macro),
+		cel.Function(funcName,
+			cel.Overload(funcName, append([]*cel.Type{requestType}, matcherDataTypes...), cel.BoolType),
+			cel.SingletonBinaryImpl(CELMatcherRuntimeFunction(funcName, fac))),
+	}
+	programOptions := []cel.ProgramOption{
+		cel.CustomDecorator(CELMatcherDecorator(funcName, fac)),
+	}
+	return NewMatcherCELLibrary(envOptions, programOptions), nil
+}
+
+// CELMatcherFactory converts a constant CEL value into a RequestMatcher.
+type CELMatcherFactory func(data ref.Val) (RequestMatcher, error)
+
+// matcherCELLibrary is a simplistic configurable cel.Library implementation.
+type matcherCELLibary struct {
+	envOptions     []cel.EnvOption
+	programOptions []cel.ProgramOption
+}
+
+// NewMatcherCELLibrary creates a matcherLibrary from option setes.
+func NewMatcherCELLibrary(envOptions []cel.EnvOption, programOptions []cel.ProgramOption) cel.Library {
+	return &matcherCELLibary{
+		envOptions:     envOptions,
+		programOptions: programOptions,
+	}
+}
+
+func (lib *matcherCELLibary) CompileOptions() []cel.EnvOption {
+	return lib.envOptions
+}
+
+func (lib *matcherCELLibary) ProgramOptions() []cel.ProgramOption {
+	return lib.programOptions
+}
+
+// CELMatcherDecorator matches a call overload generated by a CEL macro
+// that takes a single argument, and optimizes the implementation to precompile
+// the matcher and return a function that references the precompiled and
+// provisioned matcher.
+func CELMatcherDecorator(funcName string, fac CELMatcherFactory) interpreter.InterpretableDecorator {
+	return func(i interpreter.Interpretable) (interpreter.Interpretable, error) {
+		call, ok := i.(interpreter.InterpretableCall)
+		if !ok {
+			return i, nil
+		}
+		if call.OverloadID() != funcName {
+			return i, nil
+		}
+		callArgs := call.Args()
+		reqAttr, ok := callArgs[0].(interpreter.InterpretableAttribute)
+		if !ok {
+			return nil, errors.New("missing 'request' argument")
+		}
+		nsAttr, ok := reqAttr.Attr().(interpreter.NamespacedAttribute)
+		if !ok {
+			return nil, errors.New("missing 'request' argument")
+		}
+		varNames := nsAttr.CandidateVariableNames()
+		if len(varNames) != 1 || len(varNames) == 1 && varNames[0] != "request" {
+			return nil, errors.New("missing 'request' argument")
+		}
+		matcherData, ok := callArgs[1].(interpreter.InterpretableConst)
+		if !ok {
+			// If the matcher arguments are not constant, then this means
+			// they contain a Caddy placeholder reference and the evaluation
+			// and matcher provisioning should be handled at dynamically.
+			return i, nil
+		}
+		matcher, err := fac(matcherData.Value())
+		if err != nil {
+			return nil, err
+		}
+		return interpreter.NewCall(
+			i.ID(), funcName, funcName+"_opt",
+			[]interpreter.Interpretable{reqAttr},
+			func(args ...ref.Val) ref.Val {
+				// The request value, guaranteed to be of type celHTTPRequest
+				celReq := args[0]
+				// If needed this call could be changed to convert the value
+				// to a *http.Request using CEL's ConvertToNative method.
+				httpReq := celReq.Value().(celHTTPRequest)
+				return types.Bool(matcher.Match(httpReq.Request))
+			},
+		), nil
+	}
+}
+
+// CELMatcherRuntimeFunction creates a function binding for when the input to the matcher
+// is dynamically resolved rather than a set of static constant values.
+func CELMatcherRuntimeFunction(funcName string, fac CELMatcherFactory) functions.BinaryOp {
+	return func(celReq, matcherData ref.Val) ref.Val {
+		matcher, err := fac(matcherData)
+		if err != nil {
+			return types.NewErr(err.Error())
+		}
+		httpReq := celReq.Value().(celHTTPRequest)
+		return types.Bool(matcher.Match(httpReq.Request))
+	}
+}
+
+// celMatcherStringListMacroExpander validates that the macro is called
+// with a variable number of string arguments (at least one).
+//
+// The arguments are collected into a single list argument the following
+// function call returned: <funcName>(request, [args])
+func celMatcherStringListMacroExpander(funcName string) parser.MacroExpander {
+	return func(eh parser.ExprHelper, target *exprpb.Expr, args []*exprpb.Expr) (*exprpb.Expr, *common.Error) {
+		matchArgs := []*exprpb.Expr{}
+		if len(args) == 0 {
+			return nil, &common.Error{
+				Message: "matcher requires at least one argument",
+			}
+		}
+		for _, arg := range args {
+			if isCELStringExpr(arg) {
+				matchArgs = append(matchArgs, arg)
+			} else {
+				return nil, &common.Error{
+					Location: eh.OffsetLocation(arg.GetId()),
+					Message:  "matcher arguments must be string constants",
+				}
+			}
+		}
+		return eh.GlobalCall(funcName, eh.Ident("request"), eh.NewList(matchArgs...)), nil
+	}
+}
+
+// celMatcherStringMacroExpander validates that the macro is called a single
+// string argument.
+//
+// The following function call is returned: <funcName>(request, arg)
+func celMatcherStringMacroExpander(funcName string) parser.MacroExpander {
+	return func(eh parser.ExprHelper, target *exprpb.Expr, args []*exprpb.Expr) (*exprpb.Expr, *common.Error) {
+		if len(args) != 1 {
+			return nil, &common.Error{
+				Message: "matcher requires one argument",
+			}
+		}
+		if isCELStringExpr(args[0]) {
+			return eh.GlobalCall(funcName, eh.Ident("request"), args[0]), nil
+		}
+		return nil, &common.Error{
+			Location: eh.OffsetLocation(args[0].GetId()),
+			Message:  "matcher argument must be a string literal",
+		}
+	}
+}
+
+// celMatcherStringMacroExpander validates that the macro is called a single
+// map literal argument.
+//
+// The following function call is returned: <funcName>(request, arg)
+func celMatcherJSONMacroExpander(funcName string) parser.MacroExpander {
+	return func(eh parser.ExprHelper, target *exprpb.Expr, args []*exprpb.Expr) (*exprpb.Expr, *common.Error) {
+		if len(args) != 1 {
+			return nil, &common.Error{
+				Message: "matcher requires a map literal argument",
+			}
+		}
+		arg := args[0]
+		switch arg.GetExprKind().(type) {
+		case *exprpb.Expr_StructExpr:
+			structExpr := arg.GetStructExpr()
+			if structExpr.GetMessageName() != "" {
+				return nil, &common.Error{
+					Location: eh.OffsetLocation(arg.GetId()),
+					Message: fmt.Sprintf(
+						"matcher input must be a map literal, not a %s",
+						structExpr.GetMessageName(),
+					),
+				}
+			}
+			for _, entry := range structExpr.GetEntries() {
+				isStringPlaceholder := isCELStringExpr(entry.GetMapKey())
+				if !isStringPlaceholder {
+					return nil, &common.Error{
+						Location: eh.OffsetLocation(entry.GetId()),
+						Message:  "matcher map keys must be string literals",
+					}
+				}
+				isStringListPlaceholder := isCELStringExpr(entry.GetValue()) ||
+					isCELStringListLiteral(entry.GetValue())
+				if !isStringListPlaceholder {
+					return nil, &common.Error{
+						Location: eh.OffsetLocation(entry.GetValue().GetId()),
+						Message:  "matcher map values must be string or list literals",
+					}
+				}
+			}
+			return eh.GlobalCall(funcName, eh.Ident("request"), arg), nil
+		}
+
+		return nil, &common.Error{
+			Location: eh.OffsetLocation(arg.GetId()),
+			Message:  "matcher requires a map literal argument",
+		}
+	}
+}
+
+// CELValueToMapStrList converts a CEL value to a map[string][]string
+//
+// Earlier validation stages should guarantee that the value has this type
+// at compile time, and that the runtime value type is map[string]any.
+// The reason for the slight difference in value type is that CEL allows for
+// map literals containing heterogeneous values, in this case string and list
+// of string.
+func CELValueToMapStrList(data ref.Val) (map[string][]string, error) {
+	mapStrType := reflect.TypeOf(map[string]any{})
+	mapStrRaw, err := data.ConvertToNative(mapStrType)
+	if err != nil {
+		return nil, err
+	}
+	mapStrIface := mapStrRaw.(map[string]any)
+	mapStrListStr := make(map[string][]string, len(mapStrIface))
+	for k, v := range mapStrIface {
+		switch val := v.(type) {
+		case string:
+			mapStrListStr[k] = []string{val}
+		case types.String:
+			mapStrListStr[k] = []string{string(val)}
+		case []string:
+			mapStrListStr[k] = val
+		case []ref.Val:
+			convVals := make([]string, len(val))
+			for i, elem := range val {
+				strVal, ok := elem.(types.String)
+				if !ok {
+					return nil, fmt.Errorf("unsupported value type in header match: %T", val)
+				}
+				convVals[i] = string(strVal)
+			}
+			mapStrListStr[k] = convVals
+		default:
+			return nil, fmt.Errorf("unsupported value type in header match: %T", val)
+		}
+	}
+	return mapStrListStr, nil
+}
+
+// isCELStringExpr indicates whether the expression is a supported string expression
+func isCELStringExpr(e *exprpb.Expr) bool {
+	return isCELStringLiteral(e) || isCELCaddyPlaceholderCall(e) || isCELConcatCall(e)
+}
+
+// isCELStringLiteral returns whether the expression is a CEL string literal.
+func isCELStringLiteral(e *exprpb.Expr) bool {
+	switch e.GetExprKind().(type) {
+	case *exprpb.Expr_ConstExpr:
+		constant := e.GetConstExpr()
+		switch constant.GetConstantKind().(type) {
+		case *exprpb.Constant_StringValue:
+			return true
+		}
+	}
+	return false
+}
+
+// isCELCaddyPlaceholderCall returns whether the expression is a caddy placeholder call.
+func isCELCaddyPlaceholderCall(e *exprpb.Expr) bool {
+	switch e.GetExprKind().(type) {
+	case *exprpb.Expr_CallExpr:
+		call := e.GetCallExpr()
+		if call.GetFunction() == "caddyPlaceholder" {
+			return true
+		}
+	}
+	return false
+}
+
+// isCELConcatCall tests whether the expression is a concat function (+) with string, placeholder, or
+// other concat call arguments.
+func isCELConcatCall(e *exprpb.Expr) bool {
+	switch e.GetExprKind().(type) {
+	case *exprpb.Expr_CallExpr:
+		call := e.GetCallExpr()
+		if call.GetTarget() != nil {
+			return false
+		}
+		if call.GetFunction() != operators.Add {
+			return false
+		}
+		for _, arg := range call.GetArgs() {
+			if !isCELStringExpr(arg) {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
+// isCELStringListLiteral returns whether the expression resolves to a list literal
+// containing only string constants or a placeholder call.
+func isCELStringListLiteral(e *exprpb.Expr) bool {
+	switch e.GetExprKind().(type) {
+	case *exprpb.Expr_ListExpr:
+		list := e.GetListExpr()
+		for _, elem := range list.GetElements() {
+			if !isCELStringExpr(elem) {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
 // Variables used for replacing Caddy placeholders in CEL
 // expressions with a proper CEL function call; this is
 // just for syntactic sugar.
 var (
-	placeholderRegexp    = regexp.MustCompile(`{([\w.-]+)}`)
+	placeholderRegexp    = regexp.MustCompile(`{([a-zA-Z][\w.-]+)}`)
 	placeholderExpansion = `caddyPlaceholder(request, "${1}")`
+
+	CELTypeJSON = cel.MapType(cel.StringType, cel.DynType)
 )
 
-var httpRequestObjectType = decls.NewObjectType("http.Request")
+var httpRequestObjectType = cel.ObjectType("http.Request")
 
 // The name of the CEL function which accesses Replacer values.
 const placeholderFuncName = "caddyPlaceholder"

modules/caddyhttp/celmatcher_test.go

@@ -19,12 +19,462 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
+	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	"github.com/caddyserver/caddy/v2"
 )
 
+var (
+	clientCert = []byte(`-----BEGIN CERTIFICATE-----
+MIIB9jCCAV+gAwIBAgIBAjANBgkqhkiG9w0BAQsFADAYMRYwFAYDVQQDDA1DYWRk
+eSBUZXN0IENBMB4XDTE4MDcyNDIxMzUwNVoXDTI4MDcyMTIxMzUwNVowHTEbMBkG
+A1UEAwwSY2xpZW50LmxvY2FsZG9tYWluMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB
+iQKBgQDFDEpzF0ew68teT3xDzcUxVFaTII+jXH1ftHXxxP4BEYBU4q90qzeKFneF
+z83I0nC0WAQ45ZwHfhLMYHFzHPdxr6+jkvKPASf0J2v2HDJuTM1bHBbik5Ls5eq+
+fVZDP8o/VHKSBKxNs8Goc2NTsr5b07QTIpkRStQK+RJALk4x9QIDAQABo0swSTAJ
+BgNVHRMEAjAAMAsGA1UdDwQEAwIHgDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
+AAEwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDQYJKoZIhvcNAQELBQADgYEANSjz2Sk+
+eqp31wM9il1n+guTNyxJd+FzVAH+hCZE5K+tCgVDdVFUlDEHHbS/wqb2PSIoouLV
+3Q9fgDkiUod+uIK0IynzIKvw+Cjg+3nx6NQ0IM0zo8c7v398RzB4apbXKZyeeqUH
+9fNwfEi+OoXR6s+upSKobCmLGLGi9Na5s5g=
+-----END CERTIFICATE-----`)
+
+	matcherTests = []struct {
+		name              string
+		expression        *MatchExpression
+		urlTarget         string
+		httpMethod        string
+		httpHeader        *http.Header
+		wantErr           bool
+		wantResult        bool
+		clientCertificate []byte
+	}{
+		{
+			name: "boolean matches succeed for placeholder http.request.tls.client.subject",
+			expression: &MatchExpression{
+				Expr: "{http.request.tls.client.subject} == 'CN=client.localdomain'",
+			},
+			clientCertificate: clientCert,
+			urlTarget:         "https://example.com/foo",
+			wantResult:        true,
+		},
+		{
+			name: "header matches (MatchHeader)",
+			expression: &MatchExpression{
+				Expr: `header({'Field': 'foo'})`,
+			},
+			urlTarget:  "https://example.com/foo",
+			httpHeader: &http.Header{"Field": []string{"foo", "bar"}},
+			wantResult: true,
+		},
+		{
+			name: "header error (MatchHeader)",
+			expression: &MatchExpression{
+				Expr: `header('foo')`,
+			},
+			urlTarget:  "https://example.com/foo",
+			httpHeader: &http.Header{"Field": []string{"foo", "bar"}},
+			wantErr:    true,
+		},
+		{
+			name: "header_regexp matches (MatchHeaderRE)",
+			expression: &MatchExpression{
+				Expr: `header_regexp('Field', 'fo{2}')`,
+			},
+			urlTarget:  "https://example.com/foo",
+			httpHeader: &http.Header{"Field": []string{"foo", "bar"}},
+			wantResult: true,
+		},
+		{
+			name: "header_regexp matches with name (MatchHeaderRE)",
+			expression: &MatchExpression{
+				Expr: `header_regexp('foo', 'Field', 'fo{2}')`,
+			},
+			urlTarget:  "https://example.com/foo",
+			httpHeader: &http.Header{"Field": []string{"foo", "bar"}},
+			wantResult: true,
+		},
+		{
+			name: "header_regexp does not match (MatchHeaderRE)",
+			expression: &MatchExpression{
+				Expr: `header_regexp('foo', 'Nope', 'fo{2}')`,
+			},
+			urlTarget:  "https://example.com/foo",
+			httpHeader: &http.Header{"Field": []string{"foo", "bar"}},
+			wantResult: false,
+		},
+		{
+			name: "header_regexp error (MatchHeaderRE)",
+			expression: &MatchExpression{
+				Expr: `header_regexp('foo')`,
+			},
+			urlTarget:  "https://example.com/foo",
+			httpHeader: &http.Header{"Field": []string{"foo", "bar"}},
+			wantErr:    true,
+		},
+		{
+			name: "host matches localhost (MatchHost)",
+			expression: &MatchExpression{
+				Expr: `host('localhost')`,
+			},
+			urlTarget:  "http://localhost",
+			wantResult: true,
+		},
+		{
+			name: "host matches (MatchHost)",
+			expression: &MatchExpression{
+				Expr: `host('*.example.com')`,
+			},
+			urlTarget:  "https://foo.example.com",
+			wantResult: true,
+		},
+		{
+			name: "host does not match (MatchHost)",
+			expression: &MatchExpression{
+				Expr: `host('example.net', '*.example.com')`,
+			},
+			urlTarget:  "https://foo.example.org",
+			wantResult: false,
+		},
+		{
+			name: "host error (MatchHost)",
+			expression: &MatchExpression{
+				Expr: `host(80)`,
+			},
+			urlTarget: "http://localhost:80",
+			wantErr:   true,
+		},
+		{
+			name: "method does not match (MatchMethod)",
+			expression: &MatchExpression{
+				Expr: `method('PUT')`,
+			},
+			urlTarget:  "https://foo.example.com",
+			httpMethod: "GET",
+			wantResult: false,
+		},
+		{
+			name: "method matches (MatchMethod)",
+			expression: &MatchExpression{
+				Expr: `method('DELETE', 'PUT', 'POST')`,
+			},
+			urlTarget:  "https://foo.example.com",
+			httpMethod: "PUT",
+			wantResult: true,
+		},
+		{
+			name: "method error not enough arguments (MatchMethod)",
+			expression: &MatchExpression{
+				Expr: `method()`,
+			},
+			urlTarget:  "https://foo.example.com",
+			httpMethod: "PUT",
+			wantErr:    true,
+		},
+		{
+			name: "path matches substring (MatchPath)",
+			expression: &MatchExpression{
+				Expr: `path('*substring*')`,
+			},
+			urlTarget:  "https://example.com/foo/substring/bar.txt",
+			wantResult: true,
+		},
+		{
+			name: "path does not match (MatchPath)",
+			expression: &MatchExpression{
+				Expr: `path('/foo')`,
+			},
+			urlTarget:  "https://example.com/foo/bar",
+			wantResult: false,
+		},
+		{
+			name: "path matches end url fragment (MatchPath)",
+			expression: &MatchExpression{
+				Expr: `path('/foo')`,
+			},
+			urlTarget:  "https://example.com/FOO",
+			wantResult: true,
+		},
+		{
+			name: "path matches end fragment with substring prefix (MatchPath)",
+			expression: &MatchExpression{
+				Expr: `path('/foo*')`,
+			},
+			urlTarget:  "https://example.com/FOOOOO",
+			wantResult: true,
+		},
+		{
+			name: "path matches one of multiple (MatchPath)",
+			expression: &MatchExpression{
+				Expr: `path('/foo', '/foo/*', '/bar', '/bar/*', '/baz', '/baz*')`,
+			},
+			urlTarget:  "https://example.com/foo",
+			wantResult: true,
+		},
+		{
+			name: "path_regexp with empty regex matches empty path (MatchPathRE)",
+			expression: &MatchExpression{
+				Expr: `path_regexp('')`,
+			},
+			urlTarget:  "https://example.com/",
+			wantResult: true,
+		},
+		{
+			name: "path_regexp with slash regex matches empty path (MatchPathRE)",
+			expression: &MatchExpression{
+				Expr: `path_regexp('/')`,
+			},
+			urlTarget:  "https://example.com/",
+			wantResult: true,
+		},
+		{
+			name: "path_regexp matches end url fragment (MatchPathRE)",
+			expression: &MatchExpression{
+				Expr: `path_regexp('^/foo')`,
+			},
+			urlTarget:  "https://example.com/foo/",
+			wantResult: true,
+		},
+		{
+			name: "path_regexp does not match fragment at end (MatchPathRE)",
+			expression: &MatchExpression{
+				Expr: `path_regexp('bar_at_start', '^/bar')`,
+			},
+			urlTarget:  "https://example.com/foo/bar",
+			wantResult: false,
+		},
+		{
+			name: "protocol matches (MatchProtocol)",
+			expression: &MatchExpression{
+				Expr: `protocol('HTTPs')`,
+			},
+			urlTarget:  "https://example.com",
+			wantResult: true,
+		},
+		{
+			name: "protocol does not match (MatchProtocol)",
+			expression: &MatchExpression{
+				Expr: `protocol('grpc')`,
+			},
+			urlTarget:  "https://example.com",
+			wantResult: false,
+		},
+		{
+			name: "protocol invocation error no args (MatchProtocol)",
+			expression: &MatchExpression{
+				Expr: `protocol()`,
+			},
+			urlTarget: "https://example.com",
+			wantErr:   true,
+		},
+		{
+			name: "protocol invocation error too many args (MatchProtocol)",
+			expression: &MatchExpression{
+				Expr: `protocol('grpc', 'https')`,
+			},
+			urlTarget: "https://example.com",
+			wantErr:   true,
+		},
+		{
+			name: "protocol invocation error wrong arg type (MatchProtocol)",
+			expression: &MatchExpression{
+				Expr: `protocol(true)`,
+			},
+			urlTarget: "https://example.com",
+			wantErr:   true,
+		},
+		{
+			name: "query does not match against a specific value (MatchQuery)",
+			expression: &MatchExpression{
+				Expr: `query({"debug": "1"})`,
+			},
+			urlTarget:  "https://example.com/foo",
+			wantResult: false,
+		},
+		{
+			name: "query matches against a specific value (MatchQuery)",
+			expression: &MatchExpression{
+				Expr: `query({"debug": "1"})`,
+			},
+			urlTarget:  "https://example.com/foo/?debug=1",
+			wantResult: true,
+		},
+		{
+			name: "query matches against multiple values (MatchQuery)",
+			expression: &MatchExpression{
+				Expr: `query({"debug": ["0", "1", {http.request.uri.query.debug}+"1"]})`,
+			},
+			urlTarget:  "https://example.com/foo/?debug=1",
+			wantResult: true,
+		},
+		{
+			name: "query matches against a wildcard (MatchQuery)",
+			expression: &MatchExpression{
+				Expr: `query({"debug": ["*"]})`,
+			},
+			urlTarget:  "https://example.com/foo/?debug=something",
+			wantResult: true,
+		},
+		{
+			name: "query matches against a placeholder value (MatchQuery)",
+			expression: &MatchExpression{
+				Expr: `query({"debug": {http.request.uri.query.debug}})`,
+			},
+			urlTarget:  "https://example.com/foo/?debug=1",
+			wantResult: true,
+		},
+		{
+			name: "query error bad map key type (MatchQuery)",
+			expression: &MatchExpression{
+				Expr: `query({1: "1"})`,
+			},
+			urlTarget: "https://example.com/foo",
+			wantErr:   true,
+		},
+		{
+			name: "query error typed struct instead of map (MatchQuery)",
+			expression: &MatchExpression{
+				Expr: `query(Message{field: "1"})`,
+			},
+			urlTarget: "https://example.com/foo",
+			wantErr:   true,
+		},
+		{
+			name: "query error bad map value type (MatchQuery)",
+			expression: &MatchExpression{
+				Expr: `query({"debug": 1})`,
+			},
+			urlTarget: "https://example.com/foo/?debug=1",
+			wantErr:   true,
+		},
+		{
+			name: "query error no args (MatchQuery)",
+			expression: &MatchExpression{
+				Expr: `query()`,
+			},
+			urlTarget: "https://example.com/foo/?debug=1",
+			wantErr:   true,
+		},
+		{
+			name: "remote_ip error no args (MatchRemoteIP)",
+			expression: &MatchExpression{
+				Expr: `remote_ip()`,
+			},
+			urlTarget: "https://example.com/foo",
+			wantErr:   true,
+		},
+		{
+			name: "remote_ip single IP match (MatchRemoteIP)",
+			expression: &MatchExpression{
+				Expr: `remote_ip('192.0.2.1')`,
+			},
+			urlTarget:  "https://example.com/foo",
+			wantResult: true,
+		},
+		{
+			name: "remote_ip forwarded (MatchRemoteIP)",
+			expression: &MatchExpression{
+				Expr: `remote_ip('forwarded', '192.0.2.1')`,
+			},
+			urlTarget:  "https://example.com/foo",
+			wantResult: true,
+		},
+		{
+			name: "remote_ip forwarded not first (MatchRemoteIP)",
+			expression: &MatchExpression{
+				Expr: `remote_ip('192.0.2.1', 'forwarded')`,
+			},
+			urlTarget: "https://example.com/foo",
+			wantErr:   true,
+		},
+	}
+)
+
+func TestMatchExpressionMatch(t *testing.T) {
+	for _, tst := range matcherTests {
+		tc := tst
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.expression.Provision(caddy.Context{})
+			if err != nil {
+				if !tc.wantErr {
+					t.Errorf("MatchExpression.Provision() error = %v, wantErr %v", err, tc.wantErr)
+				}
+				return
+			}
+
+			req := httptest.NewRequest(tc.httpMethod, tc.urlTarget, nil)
+			if tc.httpHeader != nil {
+				req.Header = *tc.httpHeader
+			}
+			repl := caddy.NewReplacer()
+			ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
+			req = req.WithContext(ctx)
+			addHTTPVarsToReplacer(repl, req, httptest.NewRecorder())
+
+			if tc.clientCertificate != nil {
+				block, _ := pem.Decode(clientCert)
+				if block == nil {
+					t.Fatalf("failed to decode PEM certificate")
+				}
+
+				cert, err := x509.ParseCertificate(block.Bytes)
+				if err != nil {
+					t.Fatalf("failed to decode PEM certificate: %v", err)
+				}
+
+				req.TLS = &tls.ConnectionState{
+					PeerCertificates: []*x509.Certificate{cert},
+				}
+			}
+
+			if tc.expression.Match(req) != tc.wantResult {
+				t.Errorf("MatchExpression.Match() expected to return '%t', for expression : '%s'", tc.wantResult, tc.expression.Expr)
+			}
+		})
+	}
+}
+
+func BenchmarkMatchExpressionMatch(b *testing.B) {
+	for _, tst := range matcherTests {
+		tc := tst
+		if tc.wantErr {
+			continue
+		}
+		b.Run(tst.name, func(b *testing.B) {
+			tc.expression.Provision(caddy.Context{})
+			req := httptest.NewRequest(tc.httpMethod, tc.urlTarget, nil)
+			if tc.httpHeader != nil {
+				req.Header = *tc.httpHeader
+			}
+			repl := caddy.NewReplacer()
+			ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
+			req = req.WithContext(ctx)
+			addHTTPVarsToReplacer(repl, req, httptest.NewRecorder())
+			if tc.clientCertificate != nil {
+				block, _ := pem.Decode(clientCert)
+				if block == nil {
+					b.Fatalf("failed to decode PEM certificate")
+				}
+
+				cert, err := x509.ParseCertificate(block.Bytes)
+				if err != nil {
+					b.Fatalf("failed to decode PEM certificate: %v", err)
+				}
+
+				req.TLS = &tls.ConnectionState{
+					PeerCertificates: []*x509.Certificate{cert},
+				}
+			}
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				tc.expression.Match(req)
+			}
+		})
+	}
+}
+
 func TestMatchExpressionProvision(t *testing.T) {
 	tests := []struct {
 		name       string
@@ -54,71 +504,3 @@ func TestMatchExpressionProvision(t *testing.T) {
 		})
 	}
 }
-
-func TestMatchExpressionMatch(t *testing.T) {
-
-	clientCert := []byte(`-----BEGIN CERTIFICATE-----
-MIIB9jCCAV+gAwIBAgIBAjANBgkqhkiG9w0BAQsFADAYMRYwFAYDVQQDDA1DYWRk
-eSBUZXN0IENBMB4XDTE4MDcyNDIxMzUwNVoXDTI4MDcyMTIxMzUwNVowHTEbMBkG
-A1UEAwwSY2xpZW50LmxvY2FsZG9tYWluMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB
-iQKBgQDFDEpzF0ew68teT3xDzcUxVFaTII+jXH1ftHXxxP4BEYBU4q90qzeKFneF
-z83I0nC0WAQ45ZwHfhLMYHFzHPdxr6+jkvKPASf0J2v2HDJuTM1bHBbik5Ls5eq+
-fVZDP8o/VHKSBKxNs8Goc2NTsr5b07QTIpkRStQK+RJALk4x9QIDAQABo0swSTAJ
-BgNVHRMEAjAAMAsGA1UdDwQEAwIHgDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
-AAEwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDQYJKoZIhvcNAQELBQADgYEANSjz2Sk+
-eqp31wM9il1n+guTNyxJd+FzVAH+hCZE5K+tCgVDdVFUlDEHHbS/wqb2PSIoouLV
-3Q9fgDkiUod+uIK0IynzIKvw+Cjg+3nx6NQ0IM0zo8c7v398RzB4apbXKZyeeqUH
-9fNwfEi+OoXR6s+upSKobCmLGLGi9Na5s5g=
------END CERTIFICATE-----`)
-
-	tests := []struct {
-		name              string
-		expression        *MatchExpression
-		wantErr           bool
-		wantResult        bool
-		clientCertificate []byte
-	}{
-		{
-			name: "boolean matches succeed for placeholder http.request.tls.client.subject",
-			expression: &MatchExpression{
-				Expr: "{http.request.tls.client.subject} == 'CN=client.localdomain'",
-			},
-			clientCertificate: clientCert,
-			wantResult:        true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := tt.expression.Provision(caddy.Context{}); (err != nil) != tt.wantErr {
-				t.Errorf("MatchExpression.Provision() error = %v, wantErr %v", err, tt.wantErr)
-			}
-
-			req := httptest.NewRequest("GET", "https://example.com/foo", nil)
-			repl := caddy.NewReplacer()
-			ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
-			req = req.WithContext(ctx)
-			addHTTPVarsToReplacer(repl, req, httptest.NewRecorder())
-
-			if tt.clientCertificate != nil {
-				block, _ := pem.Decode(clientCert)
-				if block == nil {
-					t.Fatalf("failed to decode PEM certificate")
-				}
-
-				cert, err := x509.ParseCertificate(block.Bytes)
-				if err != nil {
-					t.Fatalf("failed to decode PEM certificate: %v", err)
-				}
-
-				req.TLS = &tls.ConnectionState{
-					PeerCertificates: []*x509.Certificate{cert},
-				}
-			}
-
-			if tt.expression.Match(req) != tt.wantResult {
-				t.Errorf("MatchExpression.Match() expected to return '%t', for expression : '%s'", tt.wantResult, tt.expression)
-			}
-
-		})
-	}
-}

modules/caddyhttp/encode/encode.go

@@ -20,7 +20,6 @@
 package encode
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"math"
@@ -71,7 +70,7 @@ func (enc *Encode) Provision(ctx caddy.Context) error {
 	if err != nil {
 		return fmt.Errorf("loading encoder modules: %v", err)
 	}
-	for modName, modIface := range mods.(map[string]interface{}) {
+	for modName, modIface := range mods.(map[string]any) {
 		err = enc.addEncoding(modIface.(Encoding))
 		if err != nil {
 			return fmt.Errorf("adding encoding %s: %v", modName, err)
@@ -142,7 +141,7 @@ func (enc *Encode) addEncoding(e Encoding) error {
 		enc.writerPools = make(map[string]*sync.Pool)
 	}
 	enc.writerPools[ae] = &sync.Pool{
-		New: func() interface{} {
+		New: func() any {
 			return e.NewEncoder()
 		},
 	}
@@ -160,13 +159,12 @@ func (enc *Encode) openResponseWriter(encodingName string, w http.ResponseWriter
 // initResponseWriter initializes the responseWriter instance
 // allocated in openResponseWriter, enabling mid-stack inlining.
 func (enc *Encode) initResponseWriter(rw *responseWriter, encodingName string, wrappedRW http.ResponseWriter) *responseWriter {
-	buf := bufPool.Get().(*bytes.Buffer)
-	buf.Reset()
-
-	// The allocation of ResponseWriterWrapper might be optimized as well.
-	rw.ResponseWriterWrapper = &caddyhttp.ResponseWriterWrapper{ResponseWriter: wrappedRW}
+	if httpInterfaces, ok := wrappedRW.(caddyhttp.HTTPInterfaces); ok {
+		rw.HTTPInterfaces = httpInterfaces
+	} else {
+		rw.HTTPInterfaces = &caddyhttp.ResponseWriterWrapper{ResponseWriter: wrappedRW}
+	}
 	rw.encodingName = encodingName
-	rw.buf = buf
 	rw.config = enc
 
 	return rw
@@ -176,10 +174,9 @@ func (enc *Encode) initResponseWriter(rw *responseWriter, encodingName string, w
 // using the encoding represented by encodingName and
 // configured by config.
 type responseWriter struct {
-	*caddyhttp.ResponseWriterWrapper
+	caddyhttp.HTTPInterfaces
 	encodingName string
 	w            Encoder
-	buf          *bytes.Buffer
 	config       *Encode
 	statusCode   int
 	wroteHeader  bool
@@ -206,28 +203,33 @@ func (rw *responseWriter) Flush() {
 		// to rw.Write (see bug in #4314)
 		return
 	}
-	rw.ResponseWriterWrapper.Flush()
+	rw.HTTPInterfaces.Flush()
 }
 
 // Write writes to the response. If the response qualifies,
 // it is encoded using the encoder, which is initialized
 // if not done so already.
 func (rw *responseWriter) Write(p []byte) (int, error) {
-	var n, written int
-	var err error
+	// ignore zero data writes, probably head request
+	if len(p) == 0 {
+		return 0, nil
+	}
 
-	if rw.buf != nil && rw.config.MinLength > 0 {
-		written = rw.buf.Len()
-		_, err := rw.buf.Write(p)
-		if err != nil {
-			return 0, err
+	// sniff content-type and determine content-length
+	if !rw.wroteHeader && rw.config.MinLength > 0 {
+		var gtMinLength bool
+		if len(p) > rw.config.MinLength {
+			gtMinLength = true
+		} else if cl, err := strconv.Atoi(rw.Header().Get("Content-Length")); err == nil && cl > rw.config.MinLength {
+			gtMinLength = true
+		}
+
+		if gtMinLength {
+			if rw.Header().Get("Content-Type") == "" {
+				rw.Header().Set("Content-Type", http.DetectContentType(p))
+			}
+			rw.init()
 		}
-		rw.init()
-		p = rw.buf.Bytes()
-		defer func() {
-			bufPool.Put(rw.buf)
-			rw.buf = nil
-		}()
 	}
 
 	// before we write to the response, we need to make
@@ -236,63 +238,44 @@ func (rw *responseWriter) Write(p []byte) (int, error) {
 	// and if so, that means we haven't written the
 	// header OR the default status code will be written
 	// by the standard library
-	if rw.statusCode > 0 {
-		rw.ResponseWriter.WriteHeader(rw.statusCode)
-		rw.statusCode = 0
+	if !rw.wroteHeader {
+		if rw.statusCode != 0 {
+			rw.HTTPInterfaces.WriteHeader(rw.statusCode)
+		} else {
+			rw.HTTPInterfaces.WriteHeader(http.StatusOK)
+		}
 		rw.wroteHeader = true
 	}
 
-	switch {
-	case rw.w != nil:
-		n, err = rw.w.Write(p)
-	default:
-		n, err = rw.ResponseWriter.Write(p)
+	if rw.w != nil {
+		return rw.w.Write(p)
+	} else {
+		return rw.HTTPInterfaces.Write(p)
 	}
-	n -= written
-	if n < 0 {
-		n = 0
-	}
-	return n, err
 }
 
 // Close writes any remaining buffered response and
 // deallocates any active resources.
 func (rw *responseWriter) Close() error {
-	var err error
-	// only attempt to write the remaining buffered response
-	// if there are any bytes left to write; otherwise, if
-	// the handler above us returned an error without writing
-	// anything, we'd write to the response when we instead
-	// should simply let the error propagate back down; this
-	// is why the check for rw.buf.Len() > 0 is crucial
-	if rw.buf != nil && rw.buf.Len() > 0 {
-		rw.init()
-		p := rw.buf.Bytes()
-		defer func() {
-			bufPool.Put(rw.buf)
-			rw.buf = nil
-		}()
-		switch {
-		case rw.w != nil:
-			_, err = rw.w.Write(p)
-		default:
-			_, err = rw.ResponseWriter.Write(p)
+	// didn't write, probably head request
+	if !rw.wroteHeader {
+		cl, err := strconv.Atoi(rw.Header().Get("Content-Length"))
+		if err == nil && cl > rw.config.MinLength {
+			rw.init()
+		}
+
+		if rw.statusCode != 0 {
+			rw.HTTPInterfaces.WriteHeader(rw.statusCode)
+		} else {
+			rw.HTTPInterfaces.WriteHeader(http.StatusOK)
 		}
-	} else if rw.statusCode != 0 {
-		// it is possible that a body was not written, and
-		// a header was not even written yet, even though
-		// we are closing; ensure the proper status code is
-		// written exactly once, or we risk breaking requests
-		// that rely on If-None-Match, for example
-		rw.ResponseWriter.WriteHeader(rw.statusCode)
-		rw.statusCode = 0
 		rw.wroteHeader = true
 	}
+
+	var err error
 	if rw.w != nil {
-		err2 := rw.w.Close()
-		if err2 != nil && err == nil {
-			err = err2
-		}
+		err = rw.w.Close()
+		rw.w.Reset(nil)
 		rw.config.writerPools[rw.encodingName].Put(rw.w)
 		rw.w = nil
 	}
@@ -302,16 +285,15 @@ func (rw *responseWriter) Close() error {
 // init should be called before we write a response, if rw.buf has contents.
 func (rw *responseWriter) init() {
 	if rw.Header().Get("Content-Encoding") == "" &&
-		rw.buf.Len() >= rw.config.MinLength &&
 		rw.config.Match(rw) {
 
 		rw.w = rw.config.writerPools[rw.encodingName].Get().(Encoder)
-		rw.w.Reset(rw.ResponseWriter)
+		rw.w.Reset(rw.HTTPInterfaces)
 		rw.Header().Del("Content-Length") // https://github.com/golang/go/issues/14975
 		rw.Header().Set("Content-Encoding", rw.encodingName)
 		rw.Header().Add("Vary", "Accept-Encoding")
+		rw.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-encoded content
 	}
-	rw.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-encoded content
 }
 
 // AcceptedEncodings returns the list of encodings that the
@@ -417,12 +399,6 @@ type Precompressed interface {
 	Suffix() string
 }
 
-var bufPool = sync.Pool{
-	New: func() interface{} {
-		return new(bytes.Buffer)
-	},
-}
-
 // defaultMinLength is the minimum length at which to compress content.
 const defaultMinLength = 512
 

modules/caddyhttp/encode/zstd/zstd.go

@@ -45,7 +45,7 @@ func (z *Zstd) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 // used in the Accept-Encoding request headers.
 func (Zstd) AcceptEncoding() string { return "zstd" }
 
-// NewEncoder returns a new gzip writer.
+// NewEncoder returns a new Zstandard writer.
 func (z Zstd) NewEncoder() encode.Encoder {
 	// The default of 8MB for the window is
 	// too large for many clients, so we limit

modules/caddyhttp/fileserver/browse.go

@@ -19,6 +19,8 @@ import (
 	_ "embed"
 	"encoding/json"
 	"fmt"
+	"io"
+	"io/fs"
 	"net/http"
 	"os"
 	"path"
@@ -67,9 +69,7 @@ func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter,
 	if r.URL.Path == "" || path.Base(origReq.URL.Path) == path.Base(r.URL.Path) {
 		if !strings.HasSuffix(origReq.URL.Path, "/") {
 			fsrv.logger.Debug("redirecting to trailing slash to preserve hrefs", zap.String("request_path", r.URL.Path))
-			origReq.URL.Path += "/"
-			http.Redirect(w, r, origReq.URL.String(), http.StatusMovedPermanently)
-			return nil
+			return redirect(w, r, origReq.URL.Path+"/")
 		}
 	}
 
@@ -82,7 +82,7 @@ func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter,
 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
 
 	// calling path.Clean here prevents weird breadcrumbs when URL paths are sketchy like /%2e%2e%2f
-	listing, err := fsrv.loadDirectoryContents(dir, root, path.Clean(r.URL.Path), repl)
+	listing, err := fsrv.loadDirectoryContents(dir.(fs.ReadDirFile), root, path.Clean(r.URL.Path), repl)
 	switch {
 	case os.IsPermission(err):
 		return caddyhttp.Error(http.StatusForbidden, err)
@@ -95,6 +95,7 @@ func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter,
 	fsrv.browseApplyQueryParams(w, r, &listing)
 
 	buf := bufPool.Get().(*bytes.Buffer)
+	buf.Reset()
 	defer bufPool.Put(buf)
 
 	acceptHeader := strings.ToLower(strings.Join(r.Header["Accept"], ","))
@@ -135,9 +136,9 @@ func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter,
 	return nil
 }
 
-func (fsrv *FileServer) loadDirectoryContents(dir *os.File, root, urlPath string, repl *caddy.Replacer) (browseTemplateContext, error) {
-	files, err := dir.Readdir(-1)
-	if err != nil {
+func (fsrv *FileServer) loadDirectoryContents(dir fs.ReadDirFile, root, urlPath string, repl *caddy.Replacer) (browseTemplateContext, error) {
+	files, err := dir.ReadDir(10000) // TODO: this limit should probably be configurable
+	if err != nil && err != io.EOF {
 		return browseTemplateContext{}, err
 	}
 
@@ -203,25 +204,25 @@ func (fsrv *FileServer) makeBrowseTemplate(tplCtx *templateContext) (*template.T
 	return tpl, nil
 }
 
-// isSymlink return true if f is a symbolic link
-func isSymlink(f os.FileInfo) bool {
-	return f.Mode()&os.ModeSymlink != 0
-}
-
 // isSymlinkTargetDir returns true if f's symbolic link target
 // is a directory.
-func isSymlinkTargetDir(f os.FileInfo, root, urlPath string) bool {
+func (fsrv *FileServer) isSymlinkTargetDir(f fs.FileInfo, root, urlPath string) bool {
 	if !isSymlink(f) {
 		return false
 	}
 	target := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, f.Name()))
-	targetInfo, err := os.Stat(target)
+	targetInfo, err := fs.Stat(fsrv.fileSystem, target)
 	if err != nil {
 		return false
 	}
 	return targetInfo.IsDir()
 }
 
+// isSymlink return true if f is a symbolic link.
+func isSymlink(f fs.FileInfo) bool {
+	return f.Mode()&os.ModeSymlink != 0
+}
+
 // templateContext powers the context used when evaluating the browse template.
 // It combines browse-specific features with the standard templates handler
 // features.
@@ -232,7 +233,7 @@ type templateContext struct {
 
 // bufPool is used to increase the efficiency of file listings.
 var bufPool = sync.Pool{
-	New: func() interface{} {
+	New: func() any {
 		return new(bytes.Buffer)
 	},
 }

modules/caddyhttp/fileserver/browsetplcontext.go

@@ -15,6 +15,7 @@
 package fileserver
 
 import (
+	"io/fs"
 	"net/url"
 	"os"
 	"path"
@@ -26,22 +27,31 @@ import (
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/dustin/go-humanize"
+	"go.uber.org/zap"
 )
 
-func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, root, urlPath string, repl *caddy.Replacer) browseTemplateContext {
+func (fsrv *FileServer) directoryListing(entries []fs.DirEntry, canGoUp bool, root, urlPath string, repl *caddy.Replacer) browseTemplateContext {
 	filesToHide := fsrv.transformHidePaths(repl)
 
 	var dirCount, fileCount int
 	fileInfos := []fileInfo{}
 
-	for _, f := range files {
-		name := f.Name()
+	for _, entry := range entries {
+		name := entry.Name()
 
 		if fileHidden(name, filesToHide) {
 			continue
 		}
 
-		isDir := f.IsDir() || isSymlinkTargetDir(f, root, urlPath)
+		info, err := entry.Info()
+		if err != nil {
+			fsrv.logger.Error("could not get info about directory entry",
+				zap.String("name", entry.Name()),
+				zap.String("root", root))
+			continue
+		}
+
+		isDir := entry.IsDir() || fsrv.isSymlinkTargetDir(info, root, urlPath)
 
 		// add the slash after the escape of path to avoid escaping the slash as well
 		if isDir {
@@ -51,11 +61,11 @@ func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, root
 			fileCount++
 		}
 
-		size := f.Size()
-		fileIsSymlink := isSymlink(f)
+		size := info.Size()
+		fileIsSymlink := isSymlink(info)
 		if fileIsSymlink {
-			path := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, f.Name()))
-			fileInfo, err := os.Stat(path)
+			path := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, info.Name()))
+			fileInfo, err := fs.Stat(fsrv.fileSystem, path)
 			if err == nil {
 				size = fileInfo.Size()
 			}
@@ -73,8 +83,8 @@ func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, root
 			Name:      name,
 			Size:      size,
 			URL:       u.String(),
-			ModTime:   f.ModTime().UTC(),
-			Mode:      f.Mode(),
+			ModTime:   info.ModTime().UTC(),
+			Mode:      info.Mode(),
 		})
 	}
 	name, _ := url.PathUnescape(urlPath)

modules/caddyhttp/fileserver/caddyfile.go

@@ -15,11 +15,13 @@
 package fileserver
 
 import (
+	"io/fs"
 	"path/filepath"
 	"strings"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
@@ -34,16 +36,16 @@ func init() {
 // parseCaddyfile parses the file_server directive. It enables the static file
 // server and configures it with this syntax:
 //
-//    file_server [<matcher>] [browse] {
-//        root          <path>
-//        hide          <files...>
-//        index         <files...>
-//        browse        [<template_file>]
-//        precompressed <formats...>
-//        status        <status>
-//        disable_canonical_uris
-//    }
-//
+//	file_server [<matcher>] [browse] {
+//	    fs            <backend...>
+//	    root          <path>
+//	    hide          <files...>
+//	    index         <files...>
+//	    browse        [<template_file>]
+//	    precompressed <formats...>
+//	    status        <status>
+//	    disable_canonical_uris
+//	}
 func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
 	var fsrv FileServer
 
@@ -62,6 +64,25 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 
 		for h.NextBlock(0) {
 			switch h.Val() {
+			case "fs":
+				if !h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				if fsrv.FileSystemRaw != nil {
+					return nil, h.Err("file system module already specified")
+				}
+				name := h.Val()
+				modID := "caddy.fs." + name
+				unm, err := caddyfile.UnmarshalModule(h.Dispenser, modID)
+				if err != nil {
+					return nil, err
+				}
+				fsys, ok := unm.(fs.FS)
+				if !ok {
+					return nil, h.Errf("module %s (%T) is not a supported file system implementation (requires fs.FS)", modID, unm)
+				}
+				fsrv.FileSystemRaw = caddyconfig.JSONModuleObject(fsys, "backend", name, nil)
+
 			case "hide":
 				fsrv.Hide = h.RemainingArgs()
 				if len(fsrv.Hide) == 0 {
@@ -155,22 +176,23 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 // with a rewrite directive, so this is not a standard handler directive.
 // A try_files directive has this syntax (notice no matcher tokens accepted):
 //
-//    try_files <files...>
+//	try_files <files...> {
+//		policy first_exist|smallest_size|largest_size|most_recently_modified
+//	}
 //
 // and is basically shorthand for:
 //
-//    @try_files {
-//        file {
-//            try_files <files...>
-//        }
-//    }
-//    rewrite @try_files {http.matchers.file.relative}
+//	@try_files file {
+//		try_files <files...>
+//		policy first_exist|smallest_size|largest_size|most_recently_modified
+//	}
+//	rewrite @try_files {http.matchers.file.relative}
 //
 // This directive rewrites request paths only, preserving any other part
 // of the URI, unless the part is explicitly given in the file list. For
 // example, if any of the files in the list have a query string:
 //
-//    try_files {path} index.php?{query}&p={path}
+//	try_files {path} index.php?{query}&p={path}
 //
 // then the query string will not be treated as part of the file name; and
 // if that file matches, the given query string will replace any query string
@@ -185,6 +207,27 @@ func parseTryFiles(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error)
 		return nil, h.ArgErr()
 	}
 
+	// parse out the optional try policy
+	var tryPolicy string
+	for nesting := h.Nesting(); h.NextBlock(nesting); {
+		switch h.Val() {
+		case "policy":
+			if tryPolicy != "" {
+				return nil, h.Err("try policy already configured")
+			}
+			if !h.NextArg() {
+				return nil, h.ArgErr()
+			}
+			tryPolicy = h.Val()
+
+			switch tryPolicy {
+			case tryPolicyFirstExist, tryPolicyLargestSize, tryPolicySmallestSize, tryPolicyMostRecentlyMod:
+			default:
+				return nil, h.Errf("unrecognized try policy: %s", tryPolicy)
+			}
+		}
+	}
+
 	// makeRoute returns a route that tries the files listed in try
 	// and then rewrites to the matched file; userQueryString is
 	// appended to the rewrite rule.
@@ -193,7 +236,7 @@ func parseTryFiles(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error)
 			URI: "{http.matchers.file.relative}" + userQueryString,
 		}
 		matcherSet := caddy.ModuleMap{
-			"file": h.JSON(MatchFile{TryFiles: try}),
+			"file": h.JSON(MatchFile{TryFiles: try, TryPolicy: tryPolicy}),
 		}
 		return h.NewRoute(matcherSet, handler)
 	}

modules/caddyhttp/fileserver/command.go

@@ -27,6 +27,7 @@ import (
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	caddytpl "github.com/caddyserver/caddy/v2/modules/caddyhttp/templates"
 	"github.com/caddyserver/certmagic"
+	"go.uber.org/zap"
 )
 
 func init() {
@@ -70,6 +71,7 @@ func cmdFileServer(fs caddycmd.Flags) (int, error) {
 	browse := fs.Bool("browse")
 	templates := fs.Bool("templates")
 	accessLog := fs.Bool("access-log")
+	debug := fs.Bool("debug")
 
 	var handlers []json.RawMessage
 
@@ -117,13 +119,27 @@ func cmdFileServer(fs caddycmd.Flags) (int, error) {
 		Servers: map[string]*caddyhttp.Server{"static": server},
 	}
 
+	var false bool
 	cfg := &caddy.Config{
-		Admin: &caddy.AdminConfig{Disabled: true},
+		Admin: &caddy.AdminConfig{
+			Disabled: true,
+			Config: &caddy.ConfigSettings{
+				Persist: &false,
+			},
+		},
 		AppsRaw: caddy.ModuleMap{
 			"http": caddyconfig.JSON(httpApp, nil),
 		},
 	}
 
+	if debug {
+		cfg.Logging = &caddy.Logging{
+			Logs: map[string]*caddy.CustomLog{
+				"default": {Level: zap.DebugLevel.CapitalString()},
+			},
+		}
+	}
+
 	err := caddy.Run(cfg)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, err

modules/caddyhttp/fileserver/matcher.go

@@ -15,17 +15,27 @@
 package fileserver
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/fs"
 	"net/http"
 	"os"
 	"path"
+	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/common"
+	"github.com/google/cel-go/common/operators"
+	"github.com/google/cel-go/common/types/ref"
+	"github.com/google/cel-go/parser"
+	"go.uber.org/zap"
+	exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
 )
 
 func init() {
@@ -47,7 +57,15 @@ func init() {
 // the matched file is a directory, "file" otherwise.
 // - `{http.matchers.file.remainder}` Set to the remainder
 // of the path if the path was split by `split_path`.
+//
+// Even though file matching may depend on the OS path
+// separator, the placeholder values always use /.
 type MatchFile struct {
+	// The file system implementation to use. By default, the
+	// local disk file system will be used.
+	FileSystemRaw json.RawMessage `json:"file_system,omitempty" caddy:"namespace=caddy.fs inline_key=backend"`
+	fileSystem    fs.FS
+
 	// The root directory, used for creating absolute
 	// file paths, and required when working with
 	// relative paths; if not specified, `{http.vars.root}`
@@ -88,6 +106,8 @@ type MatchFile struct {
 	// Each delimiter must appear at the end of a URI path
 	// component in order to be used as a split delimiter.
 	SplitPath []string `json:"split_path,omitempty"`
+
+	logger *zap.Logger
 }
 
 // CaddyModule returns the Caddy module information.
@@ -100,12 +120,11 @@ func (MatchFile) CaddyModule() caddy.ModuleInfo {
 
 // UnmarshalCaddyfile sets up the matcher from Caddyfile tokens. Syntax:
 //
-//     file <files...> {
-//         root <path>
-//         try_files <files...>
-//         try_policy first_exist|smallest_size|largest_size|most_recently_modified
-//     }
-//
+//	file <files...> {
+//	    root      <path>
+//	    try_files <files...>
+//	    try_policy first_exist|smallest_size|largest_size|most_recently_modified
+//	}
 func (m *MatchFile) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	for d.Next() {
 		m.TryFiles = append(m.TryFiles, d.RemainingArgs()...)
@@ -139,11 +158,122 @@ func (m *MatchFile) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	return nil
 }
 
+// CELLibrary produces options that expose this matcher for use in CEL
+// expression matchers.
+//
+// Example:
+//
+//	expression file({'root': '/srv', 'try_files': [{http.request.uri.path}, '/index.php'], 'try_policy': 'first_exist', 'split_path': ['.php']})
+func (MatchFile) CELLibrary(ctx caddy.Context) (cel.Library, error) {
+	requestType := cel.ObjectType("http.Request")
+
+	matcherFactory := func(data ref.Val) (caddyhttp.RequestMatcher, error) {
+		values, err := caddyhttp.CELValueToMapStrList(data)
+		if err != nil {
+			return nil, err
+		}
+
+		var root string
+		if len(values["root"]) > 0 {
+			root = values["root"][0]
+		}
+
+		var try_policy string
+		if len(values["try_policy"]) > 0 {
+			root = values["try_policy"][0]
+		}
+
+		m := MatchFile{
+			Root:      root,
+			TryFiles:  values["try_files"],
+			TryPolicy: try_policy,
+			SplitPath: values["split_path"],
+		}
+
+		err = m.Provision(ctx)
+		return m, err
+	}
+
+	envOptions := []cel.EnvOption{
+		cel.Macros(parser.NewGlobalVarArgMacro("file", celFileMatcherMacroExpander())),
+		cel.Function("file", cel.Overload("file_request_map", []*cel.Type{requestType, caddyhttp.CELTypeJSON}, cel.BoolType)),
+		cel.Function("file_request_map",
+			cel.Overload("file_request_map", []*cel.Type{requestType, caddyhttp.CELTypeJSON}, cel.BoolType),
+			cel.SingletonBinaryImpl(caddyhttp.CELMatcherRuntimeFunction("file_request_map", matcherFactory))),
+	}
+
+	programOptions := []cel.ProgramOption{
+		cel.CustomDecorator(caddyhttp.CELMatcherDecorator("file_request_map", matcherFactory)),
+	}
+
+	return caddyhttp.NewMatcherCELLibrary(envOptions, programOptions), nil
+}
+
+func celFileMatcherMacroExpander() parser.MacroExpander {
+	return func(eh parser.ExprHelper, target *exprpb.Expr, args []*exprpb.Expr) (*exprpb.Expr, *common.Error) {
+		if len(args) == 0 {
+			return nil, &common.Error{
+				Message: "matcher requires at least one argument",
+			}
+		}
+		if len(args) == 1 {
+			arg := args[0]
+			if isCELStringLiteral(arg) || isCELCaddyPlaceholderCall(arg) {
+				return eh.GlobalCall("file",
+					eh.Ident("request"),
+					eh.NewMap(
+						eh.NewMapEntry(eh.LiteralString("try_files"), eh.NewList(arg)),
+					),
+				), nil
+			}
+			if isCELTryFilesLiteral(arg) {
+				return eh.GlobalCall("file", eh.Ident("request"), arg), nil
+			}
+			return nil, &common.Error{
+				Location: eh.OffsetLocation(arg.GetId()),
+				Message:  "matcher requires either a map or string literal argument",
+			}
+		}
+
+		for _, arg := range args {
+			if !(isCELStringLiteral(arg) || isCELCaddyPlaceholderCall(arg)) {
+				return nil, &common.Error{
+					Location: eh.OffsetLocation(arg.GetId()),
+					Message:  "matcher only supports repeated string literal arguments",
+				}
+			}
+		}
+		return eh.GlobalCall("file",
+			eh.Ident("request"),
+			eh.NewMap(
+				eh.NewMapEntry(
+					eh.LiteralString("try_files"), eh.NewList(args...),
+				),
+			),
+		), nil
+	}
+}
+
 // Provision sets up m's defaults.
-func (m *MatchFile) Provision(_ caddy.Context) error {
+func (m *MatchFile) Provision(ctx caddy.Context) error {
+	m.logger = ctx.Logger()
+
+	// establish the file system to use
+	if len(m.FileSystemRaw) > 0 {
+		mod, err := ctx.LoadModule(m, "FileSystemRaw")
+		if err != nil {
+			return fmt.Errorf("loading file system module: %v", err)
+		}
+		m.fileSystem = mod.(fs.FS)
+	}
+	if m.fileSystem == nil {
+		m.fileSystem = osFS{}
+	}
+
 	if m.Root == "" {
 		m.Root = "{http.vars.root}"
 	}
+
 	// if list of files to try was omitted entirely, assume URL path
 	// (use placeholder instead of r.URL.Path; see issue #4146)
 	if m.TryFiles == nil {
@@ -169,10 +299,10 @@ func (m MatchFile) Validate() error {
 // Match returns true if r matches m. Returns true
 // if a file was matched. If so, four placeholders
 // will be available:
-//    - http.matchers.file.relative
-//    - http.matchers.file.absolute
-//    - http.matchers.file.type
-//    - http.matchers.file.remainder
+//   - http.matchers.file.relative: Path to file relative to site root
+//   - http.matchers.file.absolute: Path to file including site root
+//   - http.matchers.file.type: file or directory
+//   - http.matchers.file.remainder: Portion remaining after splitting file path (if configured)
 func (m MatchFile) Match(r *http.Request) bool {
 	return m.selectFile(r)
 }
@@ -182,23 +312,80 @@ func (m MatchFile) Match(r *http.Request) bool {
 func (m MatchFile) selectFile(r *http.Request) (matched bool) {
 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
 
-	root := repl.ReplaceAll(m.Root, ".")
+	root := filepath.Clean(repl.ReplaceAll(m.Root, "."))
 
-	// common preparation of the file into parts
-	prepareFilePath := func(file string) (suffix, fullpath, remainder string) {
-		suffix, remainder = m.firstSplit(path.Clean(repl.ReplaceAll(file, "")))
-		if strings.HasSuffix(file, "/") {
-			suffix += "/"
-		}
-		fullpath = caddyhttp.SanitizedPathJoin(root, suffix)
-		return
+	type matchCandidate struct {
+		fullpath, relative, splitRemainder string
 	}
 
-	// sets up the placeholders for the matched file
-	setPlaceholders := func(info os.FileInfo, rel string, abs string, remainder string) {
-		repl.Set("http.matchers.file.relative", rel)
-		repl.Set("http.matchers.file.absolute", abs)
-		repl.Set("http.matchers.file.remainder", remainder)
+	// makeCandidates evaluates placeholders in file and expands any glob expressions
+	// to build a list of file candidates. Special glob characters are escaped in
+	// placeholder replacements so globs cannot be expanded from placeholders, and
+	// globs are not evaluated on Windows because of its path separator character:
+	// escaping is not supported so we can't safely glob on Windows, or we can't
+	// support placeholders on Windows (pick one). (Actually, evaluating untrusted
+	// globs is not the end of the world since the file server will still hide any
+	// hidden files, it just might lead to unexpected behavior.)
+	makeCandidates := func(file string) []matchCandidate {
+		// first, evaluate placeholders in the file pattern
+		expandedFile, err := repl.ReplaceFunc(file, func(variable string, val any) (any, error) {
+			if runtime.GOOS == "windows" {
+				return val, nil
+			}
+			switch v := val.(type) {
+			case string:
+				return globSafeRepl.Replace(v), nil
+			case fmt.Stringer:
+				return globSafeRepl.Replace(v.String()), nil
+			}
+			return val, nil
+		})
+		if err != nil {
+			m.logger.Error("evaluating placeholders", zap.Error(err))
+			expandedFile = file // "oh well," I guess?
+		}
+
+		// clean the path and split, if configured -- we must split before
+		// globbing so that the file system doesn't include the remainder
+		// ("afterSplit") in the filename; be sure to restore trailing slash
+		beforeSplit, afterSplit := m.firstSplit(path.Clean(expandedFile))
+		if strings.HasSuffix(file, "/") {
+			beforeSplit += "/"
+		}
+
+		// create the full path to the file by prepending the site root
+		fullPattern := caddyhttp.SanitizedPathJoin(root, beforeSplit)
+
+		// expand glob expressions, but not on Windows because Glob() doesn't
+		// support escaping on Windows due to path separator)
+		var globResults []string
+		if runtime.GOOS == "windows" {
+			globResults = []string{fullPattern} // precious Windows
+		} else {
+			globResults, err = fs.Glob(m.fileSystem, fullPattern)
+			if err != nil {
+				m.logger.Error("expanding glob", zap.Error(err))
+			}
+		}
+
+		// for each glob result, combine all the forms of the path
+		var candidates []matchCandidate
+		for _, result := range globResults {
+			candidates = append(candidates, matchCandidate{
+				fullpath:       result,
+				relative:       strings.TrimPrefix(result, root),
+				splitRemainder: afterSplit,
+			})
+		}
+
+		return candidates
+	}
+
+	// setPlaceholders creates the placeholders for the matched file
+	setPlaceholders := func(candidate matchCandidate, info fs.FileInfo) {
+		repl.Set("http.matchers.file.relative", filepath.ToSlash(candidate.relative))
+		repl.Set("http.matchers.file.absolute", filepath.ToSlash(candidate.fullpath))
+		repl.Set("http.matchers.file.remainder", filepath.ToSlash(candidate.splitRemainder))
 
 		fileType := "file"
 		if info.IsDir() {
@@ -207,76 +394,83 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
 		repl.Set("http.matchers.file.type", fileType)
 	}
 
+	// match file according to the configured policy
 	switch m.TryPolicy {
 	case "", tryPolicyFirstExist:
-		for _, f := range m.TryFiles {
-			if err := parseErrorCode(f); err != nil {
+		for _, pattern := range m.TryFiles {
+			if err := parseErrorCode(pattern); err != nil {
 				caddyhttp.SetVar(r.Context(), caddyhttp.MatcherErrorVarKey, err)
 				return
 			}
-			suffix, fullpath, remainder := prepareFilePath(f)
-			if info, exists := strictFileExists(fullpath); exists {
-				setPlaceholders(info, suffix, fullpath, remainder)
-				return true
+			candidates := makeCandidates(pattern)
+			for _, c := range candidates {
+				if info, exists := m.strictFileExists(c.fullpath); exists {
+					setPlaceholders(c, info)
+					return true
+				}
 			}
 		}
 
 	case tryPolicyLargestSize:
 		var largestSize int64
-		var largestFilename string
-		var largestSuffix string
-		var remainder string
-		var info os.FileInfo
-		for _, f := range m.TryFiles {
-			suffix, fullpath, splitRemainder := prepareFilePath(f)
-			info, err := os.Stat(fullpath)
-			if err == nil && info.Size() > largestSize {
-				largestSize = info.Size()
-				largestFilename = fullpath
-				largestSuffix = suffix
-				remainder = splitRemainder
+		var largest matchCandidate
+		var largestInfo os.FileInfo
+		for _, pattern := range m.TryFiles {
+			candidates := makeCandidates(pattern)
+			for _, c := range candidates {
+				info, err := fs.Stat(m.fileSystem, c.fullpath)
+				if err == nil && info.Size() > largestSize {
+					largestSize = info.Size()
+					largest = c
+					largestInfo = info
+				}
 			}
 		}
-		setPlaceholders(info, largestSuffix, largestFilename, remainder)
+		if largestInfo == nil {
+			return false
+		}
+		setPlaceholders(largest, largestInfo)
 		return true
 
 	case tryPolicySmallestSize:
 		var smallestSize int64
-		var smallestFilename string
-		var smallestSuffix string
-		var remainder string
-		var info os.FileInfo
-		for _, f := range m.TryFiles {
-			suffix, fullpath, splitRemainder := prepareFilePath(f)
-			info, err := os.Stat(fullpath)
-			if err == nil && (smallestSize == 0 || info.Size() < smallestSize) {
-				smallestSize = info.Size()
-				smallestFilename = fullpath
-				smallestSuffix = suffix
-				remainder = splitRemainder
+		var smallest matchCandidate
+		var smallestInfo os.FileInfo
+		for _, pattern := range m.TryFiles {
+			candidates := makeCandidates(pattern)
+			for _, c := range candidates {
+				info, err := fs.Stat(m.fileSystem, c.fullpath)
+				if err == nil && (smallestSize == 0 || info.Size() < smallestSize) {
+					smallestSize = info.Size()
+					smallest = c
+					smallestInfo = info
+				}
 			}
 		}
-		setPlaceholders(info, smallestSuffix, smallestFilename, remainder)
+		if smallestInfo == nil {
+			return false
+		}
+		setPlaceholders(smallest, smallestInfo)
 		return true
 
 	case tryPolicyMostRecentlyMod:
-		var recentDate time.Time
-		var recentFilename string
-		var recentSuffix string
-		var remainder string
-		var info os.FileInfo
-		for _, f := range m.TryFiles {
-			suffix, fullpath, splitRemainder := prepareFilePath(f)
-			info, err := os.Stat(fullpath)
-			if err == nil &&
-				(recentDate.IsZero() || info.ModTime().After(recentDate)) {
-				recentDate = info.ModTime()
-				recentFilename = fullpath
-				recentSuffix = suffix
-				remainder = splitRemainder
+		var recent matchCandidate
+		var recentInfo os.FileInfo
+		for _, pattern := range m.TryFiles {
+			candidates := makeCandidates(pattern)
+			for _, c := range candidates {
+				info, err := fs.Stat(m.fileSystem, c.fullpath)
+				if err == nil &&
+					(recentInfo == nil || info.ModTime().After(recentInfo.ModTime())) {
+					recent = c
+					recentInfo = info
+				}
 			}
 		}
-		setPlaceholders(info, recentSuffix, recentFilename, remainder)
+		if recentInfo == nil {
+			return false
+		}
+		setPlaceholders(recent, recentInfo)
 		return true
 	}
 
@@ -303,8 +497,8 @@ func parseErrorCode(input string) error {
 // the file must also be a directory; if it does
 // NOT end in a forward slash, the file must NOT
 // be a directory.
-func strictFileExists(file string) (os.FileInfo, bool) {
-	stat, err := os.Stat(file)
+func (m MatchFile) strictFileExists(file string) (os.FileInfo, bool) {
+	info, err := fs.Stat(m.fileSystem, file)
 	if err != nil {
 		// in reality, this can be any error
 		// such as permission or even obscure
@@ -319,11 +513,11 @@ func strictFileExists(file string) (os.FileInfo, bool) {
 	if strings.HasSuffix(file, separator) {
 		// by convention, file paths ending
 		// in a path separator must be a directory
-		return stat, stat.IsDir()
+		return info, info.IsDir()
 	}
 	// by convention, file paths NOT ending
 	// in a path separator must NOT be a directory
-	return stat, !stat.IsDir()
+	return info, !info.IsDir()
 }
 
 // firstSplit returns the first result where the path
@@ -359,6 +553,116 @@ func indexFold(haystack, needle string) int {
 	return -1
 }
 
+// isCELMapLiteral returns whether the expression resolves to a map literal containing
+// only string keys with or a placeholder call.
+func isCELTryFilesLiteral(e *exprpb.Expr) bool {
+	switch e.GetExprKind().(type) {
+	case *exprpb.Expr_StructExpr:
+		structExpr := e.GetStructExpr()
+		if structExpr.GetMessageName() != "" {
+			return false
+		}
+		for _, entry := range structExpr.GetEntries() {
+			mapKey := entry.GetMapKey()
+			mapVal := entry.GetValue()
+			if !isCELStringLiteral(mapKey) {
+				return false
+			}
+			mapKeyStr := mapKey.GetConstExpr().GetStringValue()
+			if mapKeyStr == "try_files" || mapKeyStr == "split_path" {
+				if !isCELStringListLiteral(mapVal) {
+					return false
+				}
+			} else if mapKeyStr == "try_policy" || mapKeyStr == "root" {
+				if !(isCELStringExpr(mapVal)) {
+					return false
+				}
+			} else {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
+// isCELStringExpr indicates whether the expression is a supported string expression
+func isCELStringExpr(e *exprpb.Expr) bool {
+	return isCELStringLiteral(e) || isCELCaddyPlaceholderCall(e) || isCELConcatCall(e)
+}
+
+// isCELStringLiteral returns whether the expression is a CEL string literal.
+func isCELStringLiteral(e *exprpb.Expr) bool {
+	switch e.GetExprKind().(type) {
+	case *exprpb.Expr_ConstExpr:
+		constant := e.GetConstExpr()
+		switch constant.GetConstantKind().(type) {
+		case *exprpb.Constant_StringValue:
+			return true
+		}
+	}
+	return false
+}
+
+// isCELCaddyPlaceholderCall returns whether the expression is a caddy placeholder call.
+func isCELCaddyPlaceholderCall(e *exprpb.Expr) bool {
+	switch e.GetExprKind().(type) {
+	case *exprpb.Expr_CallExpr:
+		call := e.GetCallExpr()
+		if call.GetFunction() == "caddyPlaceholder" {
+			return true
+		}
+	}
+	return false
+}
+
+// isCELConcatCall tests whether the expression is a concat function (+) with string, placeholder, or
+// other concat call arguments.
+func isCELConcatCall(e *exprpb.Expr) bool {
+	switch e.GetExprKind().(type) {
+	case *exprpb.Expr_CallExpr:
+		call := e.GetCallExpr()
+		if call.GetTarget() != nil {
+			return false
+		}
+		if call.GetFunction() != operators.Add {
+			return false
+		}
+		for _, arg := range call.GetArgs() {
+			if !isCELStringExpr(arg) {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
+// isCELStringListLiteral returns whether the expression resolves to a list literal
+// containing only string constants or a placeholder call.
+func isCELStringListLiteral(e *exprpb.Expr) bool {
+	switch e.GetExprKind().(type) {
+	case *exprpb.Expr_ListExpr:
+		list := e.GetListExpr()
+		for _, elem := range list.GetElements() {
+			if !isCELStringExpr(elem) {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
+// globSafeRepl replaces special glob characters with escaped
+// equivalents. Note that the filepath godoc states that
+// escaping is not done on Windows because of the separator.
+var globSafeRepl = strings.NewReplacer(
+	"*", "\\*",
+	"[", "\\[",
+	"?", "\\?",
+)
+
 const (
 	tryPolicyFirstExist      = "first_exist"
 	tryPolicyLargestSize     = "largest_size"
@@ -368,6 +672,7 @@ const (
 
 // Interface guards
 var (
-	_ caddy.Validator          = (*MatchFile)(nil)
-	_ caddyhttp.RequestMatcher = (*MatchFile)(nil)
+	_ caddy.Validator              = (*MatchFile)(nil)
+	_ caddyhttp.RequestMatcher     = (*MatchFile)(nil)
+	_ caddyhttp.CELLibraryProducer = (*MatchFile)(nil)
 )

modules/caddyhttp/fileserver/matcher_test.go

@@ -15,17 +15,19 @@
 package fileserver
 
 import (
+	"context"
 	"net/http"
+	"net/http/httptest"
 	"net/url"
 	"os"
 	"runtime"
 	"testing"
 
+	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 )
 
 func TestFileMatcher(t *testing.T) {
-
 	// Windows doesn't like colons in files names
 	isWindows := runtime.GOOS == "windows"
 	if !isWindows {
@@ -84,37 +86,38 @@ func TestFileMatcher(t *testing.T) {
 		},
 		{
 			path:         "ملف.txt", // the path file name is not escaped
-			expectedPath: "ملف.txt",
+			expectedPath: "/ملف.txt",
 			expectedType: "file",
 			matched:      true,
 		},
 		{
 			path:         url.PathEscape("ملف.txt"), // singly-escaped path
-			expectedPath: "ملف.txt",
+			expectedPath: "/ملف.txt",
 			expectedType: "file",
 			matched:      true,
 		},
 		{
 			path:         url.PathEscape(url.PathEscape("ملف.txt")), // doubly-escaped path
-			expectedPath: "%D9%85%D9%84%D9%81.txt",
+			expectedPath: "/%D9%85%D9%84%D9%81.txt",
 			expectedType: "file",
 			matched:      true,
 		},
 		{
 			path:         "./with:in-name.txt", // browsers send the request with the path as such
-			expectedPath: "with:in-name.txt",
+			expectedPath: "/with:in-name.txt",
 			expectedType: "file",
 			matched:      !isWindows,
 		},
 	} {
 		m := &MatchFile{
-			Root:     "./testdata",
-			TryFiles: []string{"{http.request.uri.path}", "{http.request.uri.path}/"},
+			fileSystem: osFS{},
+			Root:       "./testdata",
+			TryFiles:   []string{"{http.request.uri.path}", "{http.request.uri.path}/"},
 		}
 
 		u, err := url.Parse(tc.path)
 		if err != nil {
-			t.Fatalf("Test %d: parsing path: %v", i, err)
+			t.Errorf("Test %d: parsing path: %v", i, err)
 		}
 
 		req := &http.Request{URL: u}
@@ -122,24 +125,24 @@ func TestFileMatcher(t *testing.T) {
 
 		result := m.Match(req)
 		if result != tc.matched {
-			t.Fatalf("Test %d: expected match=%t, got %t", i, tc.matched, result)
+			t.Errorf("Test %d: expected match=%t, got %t", i, tc.matched, result)
 		}
 
 		rel, ok := repl.Get("http.matchers.file.relative")
 		if !ok && result {
-			t.Fatalf("Test %d: expected replacer value", i)
+			t.Errorf("Test %d: expected replacer value", i)
 		}
 		if !result {
 			continue
 		}
 
 		if rel != tc.expectedPath {
-			t.Fatalf("Test %d: actual path: %v, expected: %v", i, rel, tc.expectedPath)
+			t.Errorf("Test %d: actual path: %v, expected: %v", i, rel, tc.expectedPath)
 		}
 
 		fileType, _ := repl.Get("http.matchers.file.type")
 		if fileType != tc.expectedType {
-			t.Fatalf("Test %d: actual file type: %v, expected: %v", i, fileType, tc.expectedType)
+			t.Errorf("Test %d: actual file type: %v, expected: %v", i, fileType, tc.expectedType)
 		}
 	}
 }
@@ -210,14 +213,15 @@ func TestPHPFileMatcher(t *testing.T) {
 		},
 	} {
 		m := &MatchFile{
-			Root:      "./testdata",
-			TryFiles:  []string{"{http.request.uri.path}", "{http.request.uri.path}/index.php"},
-			SplitPath: []string{".php"},
+			fileSystem: osFS{},
+			Root:       "./testdata",
+			TryFiles:   []string{"{http.request.uri.path}", "{http.request.uri.path}/index.php"},
+			SplitPath:  []string{".php"},
 		}
 
 		u, err := url.Parse(tc.path)
 		if err != nil {
-			t.Fatalf("Test %d: parsing path: %v", i, err)
+			t.Errorf("Test %d: parsing path: %v", i, err)
 		}
 
 		req := &http.Request{URL: u}
@@ -225,24 +229,24 @@ func TestPHPFileMatcher(t *testing.T) {
 
 		result := m.Match(req)
 		if result != tc.matched {
-			t.Fatalf("Test %d: expected match=%t, got %t", i, tc.matched, result)
+			t.Errorf("Test %d: expected match=%t, got %t", i, tc.matched, result)
 		}
 
 		rel, ok := repl.Get("http.matchers.file.relative")
 		if !ok && result {
-			t.Fatalf("Test %d: expected replacer value", i)
+			t.Errorf("Test %d: expected replacer value", i)
 		}
 		if !result {
 			continue
 		}
 
 		if rel != tc.expectedPath {
-			t.Fatalf("Test %d: actual path: %v, expected: %v", i, rel, tc.expectedPath)
+			t.Errorf("Test %d: actual path: %v, expected: %v", i, rel, tc.expectedPath)
 		}
 
 		fileType, _ := repl.Get("http.matchers.file.type")
 		if fileType != tc.expectedType {
-			t.Fatalf("Test %d: actual file type: %v, expected: %v", i, fileType, tc.expectedType)
+			t.Errorf("Test %d: actual file type: %v, expected: %v", i, fileType, tc.expectedType)
 		}
 	}
 }
@@ -259,3 +263,109 @@ func TestFirstSplit(t *testing.T) {
 		t.Errorf("Expected remainder %s but got %s", expectedRemainder, remainder)
 	}
 }
+
+var (
+	expressionTests = []struct {
+		name              string
+		expression        *caddyhttp.MatchExpression
+		urlTarget         string
+		httpMethod        string
+		httpHeader        *http.Header
+		wantErr           bool
+		wantResult        bool
+		clientCertificate []byte
+	}{
+		{
+			name: "file error no args (MatchFile)",
+			expression: &caddyhttp.MatchExpression{
+				Expr: `file()`,
+			},
+			urlTarget: "https://example.com/foo",
+			wantErr:   true,
+		},
+		{
+			name: "file error bad try files (MatchFile)",
+			expression: &caddyhttp.MatchExpression{
+				Expr: `file({"try_file": ["bad_arg"]})`,
+			},
+			urlTarget: "https://example.com/foo",
+			wantErr:   true,
+		},
+		{
+			name: "file match short pattern index.php (MatchFile)",
+			expression: &caddyhttp.MatchExpression{
+				Expr: `file("index.php")`,
+			},
+			urlTarget:  "https://example.com/foo",
+			wantResult: true,
+		},
+		{
+			name: "file match short pattern foo.txt (MatchFile)",
+			expression: &caddyhttp.MatchExpression{
+				Expr: `file({http.request.uri.path})`,
+			},
+			urlTarget:  "https://example.com/foo.txt",
+			wantResult: true,
+		},
+		{
+			name: "file match index.php (MatchFile)",
+			expression: &caddyhttp.MatchExpression{
+				Expr: `file({"root": "./testdata", "try_files": [{http.request.uri.path}, "/index.php"]})`,
+			},
+			urlTarget:  "https://example.com/foo",
+			wantResult: true,
+		},
+		{
+			name: "file match long pattern foo.txt (MatchFile)",
+			expression: &caddyhttp.MatchExpression{
+				Expr: `file({"root": "./testdata", "try_files": [{http.request.uri.path}]})`,
+			},
+			urlTarget:  "https://example.com/foo.txt",
+			wantResult: true,
+		},
+		{
+			name: "file match long pattern foo.txt with concatenation (MatchFile)",
+			expression: &caddyhttp.MatchExpression{
+				Expr: `file({"root": ".", "try_files": ["./testdata" + {http.request.uri.path}]})`,
+			},
+			urlTarget:  "https://example.com/foo.txt",
+			wantResult: true,
+		},
+		{
+			name: "file not match long pattern (MatchFile)",
+			expression: &caddyhttp.MatchExpression{
+				Expr: `file({"root": "./testdata", "try_files": [{http.request.uri.path}]})`,
+			},
+			urlTarget:  "https://example.com/nopenope.txt",
+			wantResult: false,
+		},
+	}
+)
+
+func TestMatchExpressionMatch(t *testing.T) {
+	for _, tst := range expressionTests {
+		tc := tst
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.expression.Provision(caddy.Context{})
+			if err != nil {
+				if !tc.wantErr {
+					t.Errorf("MatchExpression.Provision() error = %v, wantErr %v", err, tc.wantErr)
+				}
+				return
+			}
+
+			req := httptest.NewRequest(tc.httpMethod, tc.urlTarget, nil)
+			if tc.httpHeader != nil {
+				req.Header = *tc.httpHeader
+			}
+			repl := caddyhttp.NewTestReplacer(req)
+			repl.Set("http.vars.root", "./testdata")
+			ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
+			req = req.WithContext(ctx)
+
+			if tc.expression.Match(req) != tc.wantResult {
+				t.Errorf("MatchExpression.Match() expected to return '%t', for expression : '%s'", tc.wantResult, tc.expression.Expr)
+			}
+		})
+	}
+}

modules/caddyhttp/fileserver/staticfiles.go

@@ -15,11 +15,14 @@
 package fileserver
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
+	"io/fs"
 	weakrand "math/rand"
 	"mime"
 	"net/http"
-	"net/url"
 	"os"
 	"path"
 	"path/filepath"
@@ -39,10 +42,63 @@ func init() {
 	caddy.RegisterModule(FileServer{})
 }
 
-// FileServer implements a static file server responder for Caddy.
+// FileServer implements a handler that serves static files.
+//
+// The path of the file to serve is constructed by joining the site root
+// and the sanitized request path. Any and all files within the root and
+// links with targets outside the site root may therefore be accessed.
+// For example, with a site root of `/www`, requests to `/foo/bar.txt`
+// will serve the file at `/www/foo/bar.txt`.
+//
+// The request path is sanitized using the Go standard library's
+// path.Clean() function (https://pkg.go.dev/path#Clean) before being
+// joined to the root. Request paths must be valid and well-formed.
+//
+// For requests that access directories instead of regular files,
+// Caddy will attempt to serve an index file if present. For example,
+// a request to `/dir/` will attempt to serve `/dir/index.html` if
+// it exists. The index file names to try are configurable. If a
+// requested directory does not have an index file, Caddy writes a
+// 404 response. Alternatively, file browsing can be enabled with
+// the "browse" parameter which shows a list of files when directories
+// are requested if no index file is present.
+//
+// By default, this handler will canonicalize URIs so that requests to
+// directories end with a slash, but requests to regular files do not.
+// This is enforced with HTTP redirects automatically and can be disabled.
+// Canonicalization redirects are not issued, however, if a URI rewrite
+// modified the last component of the path (the filename).
+//
+// This handler sets the Etag and Last-Modified headers for static files.
+// It does not perform MIME sniffing to determine Content-Type based on
+// contents, but does use the extension (if known); see the Go docs for
+// details: https://pkg.go.dev/mime#TypeByExtension
+//
+// The file server properly handles requests with If-Match,
+// If-Unmodified-Since, If-Modified-Since, If-None-Match, Range, and
+// If-Range headers. It includes the file's modification time in the
+// Last-Modified header of the response.
 type FileServer struct {
+	// The file system implementation to use. By default, Caddy uses the local
+	// disk file system.
+	//
+	// File system modules used here must adhere to the following requirements:
+	// - Implement fs.FS interface.
+	// - Support seeking on opened files; i.e.returned fs.File values must
+	//   implement the io.Seeker interface. This is required for determining
+	//   Content-Length and satisfying Range requests.
+	// - fs.File values that represent directories must implement the
+	//   fs.ReadDirFile interface so that directory listings can be procured.
+	FileSystemRaw json.RawMessage `json:"file_system,omitempty" caddy:"namespace=caddy.fs inline_key=backend"`
+	fileSystem    fs.FS
+
 	// The path to the root of the site. Default is `{http.vars.root}` if set,
-	// or current working directory otherwise.
+	// or current working directory otherwise. This should be a trusted value.
+	//
+	// Note that a site root is not a sandbox. Although the file server does
+	// sanitize the request URI to prevent directory traversal, files (including
+	// links) within the site root may be directly accessed based on the request
+	// path. Files and folders within the root should be secure and trustworthy.
 	Root string `json:"root,omitempty"`
 
 	// A list of files or folders to hide; the file server will pretend as if
@@ -63,6 +119,7 @@ type FileServer struct {
 	Hide []string `json:"hide,omitempty"`
 
 	// The names of files to try as index files if a folder is requested.
+	// Default: index.html, index.txt.
 	IndexNames []string `json:"index_names,omitempty"`
 
 	// Enables file listings if a directory was requested and no index
@@ -95,8 +152,7 @@ type FileServer struct {
 	// If no order specified here, the first encoding from the Accept-Encoding header
 	// that both client and server support is used
 	PrecompressedOrder []string `json:"precompressed_order,omitempty"`
-
-	precompressors map[string]encode.Precompressed
+	precompressors     map[string]encode.Precompressed
 
 	logger *zap.Logger
 }
@@ -111,7 +167,19 @@ func (FileServer) CaddyModule() caddy.ModuleInfo {
 
 // Provision sets up the static files responder.
 func (fsrv *FileServer) Provision(ctx caddy.Context) error {
-	fsrv.logger = ctx.Logger(fsrv)
+	fsrv.logger = ctx.Logger()
+
+	// establish which file system (possibly a virtual one) we'll be using
+	if len(fsrv.FileSystemRaw) > 0 {
+		mod, err := ctx.LoadModule(fsrv, "FileSystemRaw")
+		if err != nil {
+			return fmt.Errorf("loading file system module: %v", err)
+		}
+		fsrv.fileSystem = mod.(fs.FS)
+	}
+	if fsrv.fileSystem == nil {
+		fsrv.fileSystem = osFS{}
+	}
 
 	if fsrv.Root == "" {
 		fsrv.Root = "{http.vars.root}"
@@ -131,11 +199,12 @@ func (fsrv *FileServer) Provision(ctx caddy.Context) error {
 		}
 	}
 
+	// support precompressed sidecar files
 	mods, err := ctx.LoadModule(fsrv, "PrecompressedRaw")
 	if err != nil {
 		return fmt.Errorf("loading encoder modules: %v", err)
 	}
-	for modName, modIface := range mods.(map[string]interface{}) {
+	for modName, modIface := range mods.(map[string]any) {
 		p, ok := modIface.(encode.Precompressed)
 		if !ok {
 			return fmt.Errorf("module %s is not precompressor", modName)
@@ -166,16 +235,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 	filesToHide := fsrv.transformHidePaths(repl)
 
 	root := repl.ReplaceAll(fsrv.Root, ".")
-	// PathUnescape returns an error if the escapes aren't well-formed,
-	// meaning the count % matches the RFC. Return early if the escape is
-	// improper.
-	if _, err := url.PathUnescape(r.URL.Path); err != nil {
-		fsrv.logger.Debug("improper path escape",
-			zap.String("site_root", root),
-			zap.String("request_path", r.URL.Path),
-			zap.Error(err))
-		return err
-	}
+
 	filename := caddyhttp.SanitizedPathJoin(root, r.URL.Path)
 
 	fsrv.logger.Debug("sanitized path join",
@@ -184,12 +244,12 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 		zap.String("result", filename))
 
 	// get information about the file
-	info, err := os.Stat(filename)
+	info, err := fs.Stat(fsrv.fileSystem, filename)
 	if err != nil {
-		err = mapDirOpenError(err, filename)
-		if os.IsNotExist(err) {
+		err = fsrv.mapDirOpenError(err, filename)
+		if errors.Is(err, fs.ErrNotExist) {
 			return fsrv.notFound(w, r, next)
-		} else if os.IsPermission(err) {
+		} else if errors.Is(err, fs.ErrPermission) {
 			return caddyhttp.Error(http.StatusForbidden, err)
 		}
 		return caddyhttp.Error(http.StatusInternalServerError, err)
@@ -210,7 +270,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 				continue
 			}
 
-			indexInfo, err := os.Stat(indexPath)
+			indexInfo, err := fs.Stat(fsrv.fileSystem, indexPath)
 			if err != nil {
 				continue
 			}
@@ -280,7 +340,8 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 		}
 	}
 
-	var file *os.File
+	var file fs.File
+	var etag string
 
 	// check for precompressed files
 	for _, ae := range encode.AcceptedEncodings(r, fsrv.PrecompressedOrder) {
@@ -289,7 +350,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 			continue
 		}
 		compressedFilename := filename + precompress.Suffix()
-		compressedInfo, err := os.Stat(compressedFilename)
+		compressedInfo, err := fs.Stat(fsrv.fileSystem, compressedFilename)
 		if err != nil || compressedInfo.IsDir() {
 			fsrv.logger.Debug("precompressed file not accessible", zap.String("filename", compressedFilename), zap.Error(err))
 			continue
@@ -301,12 +362,19 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 			if caddyErr, ok := err.(caddyhttp.HandlerError); ok && caddyErr.StatusCode == http.StatusServiceUnavailable {
 				return err
 			}
+			file = nil
 			continue
 		}
 		defer file.Close()
 		w.Header().Set("Content-Encoding", ae)
 		w.Header().Del("Accept-Ranges")
 		w.Header().Add("Vary", "Accept-Encoding")
+
+		// don't assign info = compressedInfo because sidecars are kind
+		// of transparent; however we do need to set the Etag:
+		// https://caddy.community/t/gzipped-sidecar-file-wrong-same-etag/16793
+		etag = calculateEtag(compressedInfo)
+
 		break
 	}
 
@@ -324,18 +392,18 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 			return err // error is already structured
 		}
 		defer file.Close()
+
+		etag = calculateEtag(info)
 	}
 
-	// set the ETag - note that a conditional If-None-Match request is handled
-	// by http.ServeContent below, which checks against this ETag value
-	w.Header().Set("ETag", calculateEtag(info))
+	// set the Etag - note that a conditional If-None-Match request is handled
+	// by http.ServeContent below, which checks against this Etag value
+	w.Header().Set("Etag", etag)
 
 	if w.Header().Get("Content-Type") == "" {
 		mtyp := mime.TypeByExtension(filepath.Ext(filename))
 		if mtyp == "" {
-			// do not allow Go to sniff the content-type; see
-			// https://www.youtube.com/watch?v=8t8JYpt0egE
-			// TODO: If we want a Content-Type, consider writing a default of application/octet-stream - this is secure but violates spec
+			// do not allow Go to sniff the content-type; see https://www.youtube.com/watch?v=8t8JYpt0egE
 			w.Header()["Content-Type"] = nil
 		} else {
 			w.Header().Set("Content-Type", mtyp)
@@ -375,7 +443,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 	// that errors generated by ServeContent are written immediately
 	// to the response, so we cannot handle them (but errors there
 	// are rare)
-	http.ServeContent(w, r, info.Name(), info.ModTime(), file)
+	http.ServeContent(w, r, info.Name(), info.ModTime(), file.(io.ReadSeeker))
 
 	return nil
 }
@@ -384,10 +452,10 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 // the response is configured to inform the client how to best handle it
 // and a well-described handler error is returned (do not wrap the
 // returned error value).
-func (fsrv *FileServer) openFile(filename string, w http.ResponseWriter) (*os.File, error) {
-	file, err := os.Open(filename)
+func (fsrv *FileServer) openFile(filename string, w http.ResponseWriter) (fs.File, error) {
+	file, err := fsrv.fileSystem.Open(filename)
 	if err != nil {
-		err = mapDirOpenError(err, filename)
+		err = fsrv.mapDirOpenError(err, filename)
 		if os.IsNotExist(err) {
 			fsrv.logger.Debug("file not found", zap.String("filename", filename), zap.Error(err))
 			return nil, caddyhttp.Error(http.StatusNotFound, err)
@@ -412,8 +480,8 @@ func (fsrv *FileServer) openFile(filename string, w http.ResponseWriter) (*os.Fi
 // Adapted from the Go standard library; originally written by Nathaniel Caza.
 // https://go-review.googlesource.com/c/go/+/36635/
 // https://go-review.googlesource.com/c/go/+/36804/
-func mapDirOpenError(originalErr error, name string) error {
-	if os.IsNotExist(originalErr) || os.IsPermission(originalErr) {
+func (fsrv *FileServer) mapDirOpenError(originalErr error, name string) error {
+	if errors.Is(originalErr, fs.ErrNotExist) || errors.Is(originalErr, fs.ErrPermission) {
 		return originalErr
 	}
 
@@ -422,12 +490,12 @@ func mapDirOpenError(originalErr error, name string) error {
 		if parts[i] == "" {
 			continue
 		}
-		fi, err := os.Stat(strings.Join(parts[:i+1], separator))
+		fi, err := fs.Stat(fsrv.fileSystem, strings.Join(parts[:i+1], separator))
 		if err != nil {
 			return originalErr
 		}
 		if !fi.IsDir() {
-			return os.ErrNotExist
+			return fs.ErrNotExist
 		}
 	}
 
@@ -545,6 +613,21 @@ func (wr statusOverrideResponseWriter) WriteHeader(int) {
 	wr.ResponseWriter.WriteHeader(wr.code)
 }
 
+// osFS is a simple fs.FS implementation that uses the local
+// file system. (We do not use os.DirFS because we do our own
+// rooting or path prefixing without being constrained to a single
+// root folder. The standard os.DirFS implementation is problematic
+// since roots can be dynamic in our application.)
+//
+// osFS also implements fs.StatFS, fs.GlobFS, fs.ReadDirFS, and fs.ReadFileFS.
+type osFS struct{}
+
+func (osFS) Open(name string) (fs.File, error)          { return os.Open(name) }
+func (osFS) Stat(name string) (fs.FileInfo, error)      { return os.Stat(name) }
+func (osFS) Glob(pattern string) ([]string, error)      { return filepath.Glob(pattern) }
+func (osFS) ReadDir(name string) ([]fs.DirEntry, error) { return os.ReadDir(name) }
+func (osFS) ReadFile(name string) ([]byte, error)       { return os.ReadFile(name) }
+
 var defaultIndexNames = []string{"index.html", "index.txt"}
 
 const (
@@ -556,4 +639,9 @@ const (
 var (
 	_ caddy.Provisioner           = (*FileServer)(nil)
 	_ caddyhttp.MiddlewareHandler = (*FileServer)(nil)
+
+	_ fs.StatFS     = (*osFS)(nil)
+	_ fs.GlobFS     = (*osFS)(nil)
+	_ fs.ReadDirFS  = (*osFS)(nil)
+	_ fs.ReadFileFS = (*osFS)(nil)
 )

modules/caddyhttp/fileserver/testdata/foodir/bar.txt

@@ -0,0 +1 @@
+foodir/bar.txt

modules/caddyhttp/headers/headers.go

@@ -118,10 +118,16 @@ type HeaderOps struct {
 	// Sets HTTP headers; replaces existing header fields.
 	Set http.Header `json:"set,omitempty"`
 
-	// Names of HTTP header fields to delete.
+	// Names of HTTP header fields to delete. Basic wildcards are supported:
+	//
+	// - Start with `*` for all field names with the given suffix;
+	// - End with `*` for all field names with the given prefix;
+	// - Start and end with `*` for all field names containing a substring.
 	Delete []string `json:"delete,omitempty"`
 
-	// Performs substring replacements of HTTP headers in-situ.
+	// Performs in-situ substring replacements of HTTP headers.
+	// Keys are the field names on which to perform the associated replacements.
+	// If the field name is `*`, the replacements are performed on all header fields.
 	Replace map[string][]Replacement `json:"replace,omitempty"`
 }
 
@@ -188,38 +194,60 @@ type RespHeaderOps struct {
 func (ops HeaderOps) ApplyTo(hdr http.Header, repl *caddy.Replacer) {
 	// add
 	for fieldName, vals := range ops.Add {
-		fieldName = repl.ReplaceAll(fieldName, "")
+		fieldName = repl.ReplaceKnown(fieldName, "")
 		for _, v := range vals {
-			hdr.Add(fieldName, repl.ReplaceAll(v, ""))
+			hdr.Add(fieldName, repl.ReplaceKnown(v, ""))
 		}
 	}
 
 	// set
 	for fieldName, vals := range ops.Set {
-		fieldName = repl.ReplaceAll(fieldName, "")
+		fieldName = repl.ReplaceKnown(fieldName, "")
 		var newVals []string
 		for i := range vals {
 			// append to new slice so we don't overwrite
 			// the original values in ops.Set
-			newVals = append(newVals, repl.ReplaceAll(vals[i], ""))
+			newVals = append(newVals, repl.ReplaceKnown(vals[i], ""))
 		}
 		hdr.Set(fieldName, strings.Join(newVals, ","))
 	}
 
 	// delete
 	for _, fieldName := range ops.Delete {
-		hdr.Del(repl.ReplaceAll(fieldName, ""))
+		fieldName = strings.ToLower(repl.ReplaceKnown(fieldName, ""))
+		switch {
+		case strings.HasPrefix(fieldName, "*") && strings.HasSuffix(fieldName, "*"):
+			for existingField := range hdr {
+				if strings.Contains(strings.ToLower(existingField), fieldName[1:len(fieldName)-1]) {
+					delete(hdr, existingField)
+				}
+			}
+		case strings.HasPrefix(fieldName, "*"):
+			for existingField := range hdr {
+				if strings.HasSuffix(strings.ToLower(existingField), fieldName[1:]) {
+					delete(hdr, existingField)
+				}
+			}
+		case strings.HasSuffix(fieldName, "*"):
+			for existingField := range hdr {
+				if strings.HasPrefix(strings.ToLower(existingField), fieldName[:len(fieldName)-1]) {
+					delete(hdr, existingField)
+				}
+			}
+		default:
+			hdr.Del(fieldName)
+		}
 	}
 
 	// replace
 	for fieldName, replacements := range ops.Replace {
-		fieldName = http.CanonicalHeaderKey(repl.ReplaceAll(fieldName, ""))
+		fieldName = http.CanonicalHeaderKey(repl.ReplaceKnown(fieldName, ""))
 
 		// all fields...
 		if fieldName == "*" {
 			for _, r := range replacements {
-				search := repl.ReplaceAll(r.Search, "")
-				replace := repl.ReplaceAll(r.Replace, "")
+				search := repl.ReplaceKnown(r.Search, "")
+				replace := repl.ReplaceKnown(r.Replace, "")
 				for fieldName, vals := range hdr {
 					for i := range vals {
 						if r.re != nil {
@@ -235,8 +263,8 @@ func (ops HeaderOps) ApplyTo(hdr http.Header, repl *caddy.Replacer) {
 
 		// ...or only with the named field
 		for _, r := range replacements {
-			search := repl.ReplaceAll(r.Search, "")
-			replace := repl.ReplaceAll(r.Replace, "")
+			search := repl.ReplaceKnown(r.Search, "")
+			replace := repl.ReplaceKnown(r.Replace, "")
 			for hdrFieldName, vals := range hdr {
 				// see issue #4330 for why we don't simply use hdr[fieldName]
 				if http.CanonicalHeaderKey(hdrFieldName) != fieldName {

modules/caddyhttp/headers/headers_test.go

@@ -79,6 +79,26 @@ func TestHandler(t *testing.T) {
 				"Keep-Me": []string{"i swear i'm innocent"},
 			},
 		},
+		{
+			handler: Handler{
+				Request: &HeaderOps{
+					Delete: []string{
+						"*-suffix",
+						"prefix-*",
+						"*_*",
+					},
+				},
+			},
+			reqHeader: http.Header{
+				"Header-Suffix": []string{"lalala"},
+				"Prefix-Test":   []string{"asdf"},
+				"Host_Header":   []string{"silly django... sigh"}, // see issue #4830
+				"Keep-Me":       []string{"foofoofoo"},
+			},
+			expectedReqHeader: http.Header{
+				"Keep-Me": []string{"foofoofoo"},
+			},
+		},
 		{
 			handler: Handler{
 				Request: &HeaderOps{

modules/caddyhttp/logging.go

@@ -0,0 +1,144 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddyhttp
+
+import (
+	"errors"
+	"net"
+	"net/http"
+	"strings"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+// ServerLogConfig describes a server's logging configuration. If
+// enabled without customization, all requests to this server are
+// logged to the default logger; logger destinations may be
+// customized per-request-host.
+type ServerLogConfig struct {
+	// The default logger name for all logs emitted by this server for
+	// hostnames that are not in the LoggerNames (logger_names) map.
+	DefaultLoggerName string `json:"default_logger_name,omitempty"`
+
+	// LoggerNames maps request hostnames to a custom logger name.
+	// For example, a mapping of "example.com" to "example" would
+	// cause access logs from requests with a Host of example.com
+	// to be emitted by a logger named "http.log.access.example".
+	LoggerNames map[string]string `json:"logger_names,omitempty"`
+
+	// By default, all requests to this server will be logged if
+	// access logging is enabled. This field lists the request
+	// hosts for which access logging should be disabled.
+	SkipHosts []string `json:"skip_hosts,omitempty"`
+
+	// If true, requests to any host not appearing in the
+	// LoggerNames (logger_names) map will not be logged.
+	SkipUnmappedHosts bool `json:"skip_unmapped_hosts,omitempty"`
+
+	// If true, credentials that are otherwise omitted, will be logged.
+	// The definition of credentials is defined by https://fetch.spec.whatwg.org/#credentials,
+	// and this includes some request and response headers, i.e `Cookie`,
+	// `Set-Cookie`, `Authorization`, and `Proxy-Authorization`.
+	ShouldLogCredentials bool `json:"should_log_credentials,omitempty"`
+}
+
+// wrapLogger wraps logger in a logger named according to user preferences for the given host.
+func (slc ServerLogConfig) wrapLogger(logger *zap.Logger, host string) *zap.Logger {
+	if loggerName := slc.getLoggerName(host); loggerName != "" {
+		return logger.Named(loggerName)
+	}
+	return logger
+}
+
+func (slc ServerLogConfig) getLoggerName(host string) string {
+	tryHost := func(key string) (string, bool) {
+		// first try exact match
+		if loggerName, ok := slc.LoggerNames[key]; ok {
+			return loggerName, ok
+		}
+		// strip port and try again (i.e. Host header of "example.com:1234" should
+		// match "example.com" if there is no "example.com:1234" in the map)
+		hostOnly, _, err := net.SplitHostPort(key)
+		if err != nil {
+			return "", false
+		}
+		loggerName, ok := slc.LoggerNames[hostOnly]
+		return loggerName, ok
+	}
+
+	// try the exact hostname first
+	if loggerName, ok := tryHost(host); ok {
+		return loggerName
+	}
+
+	// try matching wildcard domains if other non-specific loggers exist
+	labels := strings.Split(host, ".")
+	for i := range labels {
+		if labels[i] == "" {
+			continue
+		}
+		labels[i] = "*"
+		wildcardHost := strings.Join(labels, ".")
+		if loggerName, ok := tryHost(wildcardHost); ok {
+			return loggerName
+		}
+	}
+
+	return slc.DefaultLoggerName
+}
+
+func (slc *ServerLogConfig) clone() *ServerLogConfig {
+	clone := &ServerLogConfig{
+		DefaultLoggerName:    slc.DefaultLoggerName,
+		LoggerNames:          make(map[string]string),
+		SkipHosts:            append([]string{}, slc.SkipHosts...),
+		SkipUnmappedHosts:    slc.SkipUnmappedHosts,
+		ShouldLogCredentials: slc.ShouldLogCredentials,
+	}
+	for k, v := range slc.LoggerNames {
+		clone.LoggerNames[k] = v
+	}
+	return clone
+}
+
+// errLogValues inspects err and returns the status code
+// to use, the error log message, and any extra fields.
+// If err is a HandlerError, the returned values will
+// have richer information.
+func errLogValues(err error) (status int, msg string, fields []zapcore.Field) {
+	var handlerErr HandlerError
+	if errors.As(err, &handlerErr) {
+		status = handlerErr.StatusCode
+		if handlerErr.Err == nil {
+			msg = err.Error()
+		} else {
+			msg = handlerErr.Err.Error()
+		}
+		fields = []zapcore.Field{
+			zap.Int("status", handlerErr.StatusCode),
+			zap.String("err_id", handlerErr.ID),
+			zap.String("err_trace", handlerErr.Trace),
+		}
+		return
+	}
+	status = http.StatusInternalServerError
+	msg = err.Error()
+	return
+}
+
+// Variable name used to indicate that this request
+// should be omitted from the access logs
+const SkipLogVar = "skip_log"

modules/caddyhttp/map/caddyfile.go

@@ -27,10 +27,10 @@ func init() {
 
 // parseCaddyfile sets up the map handler from Caddyfile tokens. Syntax:
 //
-//     map [<matcher>] <source> <destinations...> {
-//         [~]<input> <outputs...>
-//         default    <defaults...>
-//     }
+//	map [<matcher>] <source> <destinations...> {
+//	    [~]<input> <outputs...>
+//	    default    <defaults...>
+//	}
 //
 // If the input value is prefixed with a tilde (~), then the input will be parsed as a
 // regular expression.
@@ -76,9 +76,9 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 				continue
 			}
 
-			// every other line maps one input to one or more outputs
+			// every line maps an input value to one or more outputs
 			in := h.Val()
-			var outs []interface{}
+			var outs []any
 			for h.NextArg() {
 				val := h.ScalarVal()
 				if val == "-" {

modules/caddyhttp/map/map.go

@@ -62,6 +62,9 @@ func (Handler) CaddyModule() caddy.ModuleInfo {
 // Provision sets up h.
 func (h *Handler) Provision(_ caddy.Context) error {
 	for j, dest := range h.Destinations {
+		if strings.Count(dest, "{") != 1 || !strings.HasPrefix(dest, "{") {
+			return fmt.Errorf("destination must be a placeholder and only a placeholder")
+		}
 		h.Destinations[j] = strings.Trim(dest, "{}")
 	}
 
@@ -106,6 +109,16 @@ func (h *Handler) Validate() error {
 		}
 		seen[input] = i
 
+		// prevent infinite recursion
+		for _, out := range m.Outputs {
+			for _, dest := range h.Destinations {
+				if strings.Contains(caddy.ToString(out), dest) ||
+					strings.Contains(m.Input, dest) {
+					return fmt.Errorf("mapping %d requires value of {%s} to define value of {%s}: infinite recursion", i, dest, dest)
+				}
+			}
+		}
+
 		// ensure mappings have 1:1 output-to-destination correspondence
 		nOut := len(m.Outputs)
 		if nOut != nDest {
@@ -119,7 +132,7 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhtt
 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
 
 	// defer work until a variable is actually evaluated by using replacer's Map callback
-	repl.Map(func(key string) (interface{}, bool) {
+	repl.Map(func(key string) (any, bool) {
 		// return early if the variable is not even a configured destination
 		destIdx := h.destinationIndex(key)
 		if destIdx < 0 {
@@ -135,21 +148,22 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhtt
 			if output == nil {
 				continue
 			}
+			outputStr := caddy.ToString(output)
+
+			// evaluate regular expression if configured
 			if m.re != nil {
 				var result []byte
 				matches := m.re.FindStringSubmatchIndex(input)
 				if matches == nil {
 					continue
 				}
-				result = m.re.ExpandString(result, output.(string), input, matches)
+				result = m.re.ExpandString(result, outputStr, input, matches)
 				return string(result), true
 			}
+
+			// otherwise simple string comparison
 			if input == m.Input {
-				if outputStr, ok := output.(string); ok {
-					// NOTE: if the output has a placeholder that has the same key as the input, this is infinite recursion
-					return repl.ReplaceAll(outputStr, ""), true
-				}
-				return output, true
+				return repl.ReplaceAll(outputStr, ""), true
 			}
 		}
 
@@ -187,7 +201,7 @@ type Mapping struct {
 	// Upon a match with the input, each output is positionally correlated
 	// with each destination of the parent handler. An output that is null
 	// (nil) will be treated as if it was not mapped at all.
-	Outputs []interface{} `json:"outputs,omitempty"`
+	Outputs []any `json:"outputs,omitempty"`
 
 	re *regexp.Regexp
 }

modules/caddyhttp/map/map_test.go

@@ -15,7 +15,7 @@ func TestHandler(t *testing.T) {
 	for i, tc := range []struct {
 		handler Handler
 		reqURI  string
-		expect  map[string]interface{}
+		expect  map[string]any
 	}{
 		{
 			reqURI: "/foo",
@@ -25,11 +25,11 @@ func TestHandler(t *testing.T) {
 				Mappings: []Mapping{
 					{
 						Input:   "/foo",
-						Outputs: []interface{}{"FOO"},
+						Outputs: []any{"FOO"},
 					},
 				},
 			},
-			expect: map[string]interface{}{
+			expect: map[string]any{
 				"output": "FOO",
 			},
 		},
@@ -41,11 +41,11 @@ func TestHandler(t *testing.T) {
 				Mappings: []Mapping{
 					{
 						InputRegexp: "(/abc)",
-						Outputs:     []interface{}{"ABC"},
+						Outputs:     []any{"ABC"},
 					},
 				},
 			},
-			expect: map[string]interface{}{
+			expect: map[string]any{
 				"output": "ABC",
 			},
 		},
@@ -57,11 +57,11 @@ func TestHandler(t *testing.T) {
 				Mappings: []Mapping{
 					{
 						InputRegexp: "(xyz)",
-						Outputs:     []interface{}{"...${1}..."},
+						Outputs:     []any{"...${1}..."},
 					},
 				},
 			},
-			expect: map[string]interface{}{
+			expect: map[string]any{
 				"output": "...xyz...",
 			},
 		},
@@ -74,11 +74,11 @@ func TestHandler(t *testing.T) {
 				Mappings: []Mapping{
 					{
 						InputRegexp: "(?i)(\\^|`|<|>|%|\\\\|\\{|\\}|\\|)",
-						Outputs:     []interface{}{"3"},
+						Outputs:     []any{"3"},
 					},
 				},
 			},
-			expect: map[string]interface{}{
+			expect: map[string]any{
 				"output": "3",
 			},
 		},
@@ -90,11 +90,11 @@ func TestHandler(t *testing.T) {
 				Mappings: []Mapping{
 					{
 						Input:   "/foo",
-						Outputs: []interface{}{"{testvar}"},
+						Outputs: []any{"{testvar}"},
 					},
 				},
 			},
-			expect: map[string]interface{}{
+			expect: map[string]any{
 				"output": "testing",
 			},
 		},

modules/caddyhttp/matchers_test.go

@@ -158,9 +158,10 @@ func TestHostMatcher(t *testing.T) {
 
 func TestPathMatcher(t *testing.T) {
 	for i, tc := range []struct {
-		match  MatchPath
-		input  string
-		expect bool
+		match        MatchPath // not URI-encoded because not parsing from a URI
+		input        string    // should be valid URI encoding (escaped) since it will become part of a request
+		expect       bool
+		provisionErr bool
 	}{
 		{
 			match:  MatchPath{},
@@ -252,6 +253,11 @@ func TestPathMatcher(t *testing.T) {
 			input:  "/FOOOO",
 			expect: true,
 		},
+		{
+			match:  MatchPath{"*.php"},
+			input:  "/foo/index.php. .",
+			expect: true,
+		},
 		{
 			match:  MatchPath{"/foo/bar.txt"},
 			input:  "/foo/BAR.txt",
@@ -263,10 +269,60 @@ func TestPathMatcher(t *testing.T) {
 			expect: true,
 		},
 		{
-			match:  MatchPath{"/foo*"},
+			match:  MatchPath{"/foo"},
 			input:  "//foo",
 			expect: true,
 		},
+		{
+			match:  MatchPath{"//foo"},
+			input:  "/foo",
+			expect: false,
+		},
+		{
+			match:  MatchPath{"//foo"},
+			input:  "//foo",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"/foo//*"},
+			input:  "/foo//bar",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"/foo//*"},
+			input:  "/foo/%2Fbar",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"/foo/%2F*"},
+			input:  "/foo/%2Fbar",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"/foo/%2F*"},
+			input:  "/foo//bar",
+			expect: false,
+		},
+		{
+			match:  MatchPath{"/foo//bar"},
+			input:  "/foo//bar",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"/foo/*//bar"},
+			input:  "/foo///bar",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"/foo/%*//bar"},
+			input:  "/foo///bar",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"/foo/%*//bar"},
+			input:  "/foo//%2Fbar",
+			expect: true,
+		},
 		{
 			match:  MatchPath{"/foo*"},
 			input:  "/%2F/foo",
@@ -292,8 +348,79 @@ func TestPathMatcher(t *testing.T) {
 			input:  "/foo/bar",
 			expect: true,
 		},
+		// notice these next three test cases are the same normalized path but are written differently
+		{
+			match:  MatchPath{"/%25@.txt"},
+			input:  "/%25@.txt",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"/%25@.txt"},
+			input:  "/%25%40.txt",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"/%25%40.txt"},
+			input:  "/%25%40.txt",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"/bands/*/*"},
+			input:  "/bands/AC%2FDC/T.N.T",
+			expect: false, // because * operates in normalized space
+		},
+		{
+			match:  MatchPath{"/bands/%*/%*"},
+			input:  "/bands/AC%2FDC/T.N.T",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"/bands/%*/%*"},
+			input:  "/bands/AC/DC/T.N.T",
+			expect: false,
+		},
+		{
+			match:  MatchPath{"/bands/%*"},
+			input:  "/bands/AC/DC",
+			expect: false, // not a suffix match
+		},
+		{
+			match:  MatchPath{"/bands/%*"},
+			input:  "/bands/AC%2FDC",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"/foo%2fbar/baz"},
+			input:  "/foo%2Fbar/baz",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"/foo%2fbar/baz"},
+			input:  "/foo/bar/baz",
+			expect: false,
+		},
+		{
+			match:  MatchPath{"/foo/bar/baz"},
+			input:  "/foo%2fbar/baz",
+			expect: true,
+		},
 	} {
-		req := &http.Request{URL: &url.URL{Path: tc.input}}
+		err := tc.match.Provision(caddy.Context{})
+		if err == nil && tc.provisionErr {
+			t.Errorf("Test %d %v: Expected error provisioning, but there was no error", i, tc.match)
+		}
+		if err != nil && !tc.provisionErr {
+			t.Errorf("Test %d %v: Expected no error provisioning, but there was an error: %v", i, tc.match, err)
+		}
+		if tc.provisionErr {
+			continue // if it's not supposed to provision properly, pointless to test it
+		}
+
+		u, err := url.ParseRequestURI(tc.input)
+		if err != nil {
+			t.Fatalf("Test %d (%v): Invalid request URI (should be rejected by Go's HTTP server): %v", i, tc.input, err)
+		}
+		req := &http.Request{URL: u}
 		repl := caddy.NewReplacer()
 		ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
 		req = req.WithContext(ctx)
@@ -387,6 +514,16 @@ func TestPathREMatcher(t *testing.T) {
 			expect:     true,
 			expectRepl: map[string]string{"name.myparam": "bar"},
 		},
+		{
+			match:  MatchPathRE{MatchRegexp{Pattern: "^/%@.txt"}},
+			input:  "/%25@.txt",
+			expect: true,
+		},
+		{
+			match:  MatchPathRE{MatchRegexp{Pattern: "^/%25@.txt"}},
+			input:  "/%25@.txt",
+			expect: false,
+		},
 	} {
 		// compile the regexp and validate its name
 		err := tc.match.Provision(caddy.Context{})
@@ -401,7 +538,11 @@ func TestPathREMatcher(t *testing.T) {
 		}
 
 		// set up the fake request and its Replacer
-		req := &http.Request{URL: &url.URL{Path: tc.input}}
+		u, err := url.ParseRequestURI(tc.input)
+		if err != nil {
+			t.Fatalf("Test %d: Bad input URI: %v", i, err)
+		}
+		req := &http.Request{URL: u}
 		repl := caddy.NewReplacer()
 		ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
 		req = req.WithContext(ctx)
@@ -807,7 +948,7 @@ func TestVarREMatcher(t *testing.T) {
 			req := &http.Request{URL: new(url.URL), Method: http.MethodGet}
 			repl := caddy.NewReplacer()
 			ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
-			ctx = context.WithValue(ctx, VarsCtxKey, make(map[string]interface{}))
+			ctx = context.WithValue(ctx, VarsCtxKey, make(map[string]any))
 			req = req.WithContext(ctx)
 
 			addHTTPVarsToReplacer(repl, req, httptest.NewRecorder())

modules/caddyhttp/metrics.go

@@ -11,6 +11,10 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
+// Metrics configures metrics observations.
+// EXPERIMENTAL and subject to change or removal.
+type Metrics struct{}
+
 var httpMetrics = struct {
 	init             sync.Once
 	requestInFlight  *prometheus.GaugeVec

modules/caddyhttp/push/handler.go

@@ -29,10 +29,24 @@ func init() {
 	caddy.RegisterModule(Handler{})
 }
 
-// Handler is a middleware for manipulating the request body.
+// Handler is a middleware for HTTP/2 server push. Note that
+// HTTP/2 server push has been deprecated by some clients and
+// its use is discouraged unless you can accurately predict
+// which resources actually need to be pushed to the client;
+// it can be difficult to know what the client already has
+// cached. Pushing unnecessary resources results in worse
+// performance. Consider using HTTP 103 Early Hints instead.
+//
+// This handler supports pushing from Link headers; in other
+// words, if the eventual response has Link headers, this
+// handler will push the resources indicated by those headers,
+// even without specifying any resources in its config.
 type Handler struct {
-	Resources []Resource    `json:"resources,omitempty"`
-	Headers   *HeaderConfig `json:"headers,omitempty"`
+	// The resources to push.
+	Resources []Resource `json:"resources,omitempty"`
+
+	// Headers to modify for the push requests.
+	Headers *HeaderConfig `json:"headers,omitempty"`
 
 	logger *zap.Logger
 }
@@ -47,7 +61,7 @@ func (Handler) CaddyModule() caddy.ModuleInfo {
 
 // Provision sets up h.
 func (h *Handler) Provision(ctx caddy.Context) error {
-	h.logger = ctx.Logger(h)
+	h.logger = ctx.Logger()
 	if h.Headers != nil {
 		err := h.Headers.Provision(ctx)
 		if err != nil {

modules/caddyhttp/push/link.go

@@ -52,17 +52,16 @@ func parseLinkHeader(header string) []linkResource {
 		l.uri = strings.TrimSpace(link[li+1 : ri])
 
 		for _, param := range strings.Split(strings.TrimSpace(link[ri+1:]), semicolon) {
-			parts := strings.SplitN(strings.TrimSpace(param), equal, 2)
-			key := strings.TrimSpace(parts[0])
+			before, after, isCut := strings.Cut(strings.TrimSpace(param), equal)
+			key := strings.TrimSpace(before)
 			if key == "" {
 				continue
 			}
-			if len(parts) == 1 {
+			if isCut {
+				l.params[key] = strings.TrimSpace(after)
+			} else {
 				l.params[key] = key
 			}
-			if len(parts) == 2 {
-				l.params[key] = strings.TrimSpace(parts[1])
-			}
 		}
 
 		resources = append(resources, l)

modules/caddyhttp/replacer.go

@@ -57,7 +57,7 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo
 	SetVar(req.Context(), "start_time", time.Now())
 	SetVar(req.Context(), "uuid", new(requestID))
 
-	httpVars := func(key string) (interface{}, bool) {
+	httpVars := func(key string) (any, bool) {
 		if req != nil {
 			// query string parameters
 			if strings.HasPrefix(key, reqURIQueryReplPrefix) {
@@ -143,6 +143,10 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo
 			case "http.request.uri.path.dir":
 				dir, _ := path.Split(req.URL.Path)
 				return dir, true
+			case "http.request.uri.path.file.base":
+				return strings.TrimSuffix(path.Base(req.URL.Path), path.Ext(req.URL.Path)), true
+			case "http.request.uri.path.file.ext":
+				return path.Ext(req.URL.Path), true
 			case "http.request.uri.query":
 				return req.URL.RawQuery, true
 			case "http.request.duration":
@@ -169,7 +173,7 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo
 				req.Body = io.NopCloser(buf)  // replace real body with buffered data
 				return buf.String(), true
 
-				// original request, before any internal changes
+			// original request, before any internal changes
 			case "http.request.orig_method":
 				or, _ := req.Context().Value(OriginalRequestCtxKey).(http.Request)
 				return or.Method, true
@@ -233,7 +237,7 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo
 			// middleware variables
 			if strings.HasPrefix(key, varsReplPrefix) {
 				varName := key[len(varsReplPrefix):]
-				tbl := req.Context().Value(VarsCtxKey).(map[string]interface{})
+				tbl := req.Context().Value(VarsCtxKey).(map[string]any)
 				raw := tbl[varName]
 				// variables can be dynamic, so always return true
 				// even when it may not be set; treat as empty then
@@ -252,13 +256,29 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo
 			}
 		}
 
+		switch {
+		case key == "http.shutting_down":
+			server := req.Context().Value(ServerCtxKey).(*Server)
+			server.shutdownAtMu.RLock()
+			defer server.shutdownAtMu.RUnlock()
+			return !server.shutdownAt.IsZero(), true
+		case key == "http.time_until_shutdown":
+			server := req.Context().Value(ServerCtxKey).(*Server)
+			server.shutdownAtMu.RLock()
+			defer server.shutdownAtMu.RUnlock()
+			if server.shutdownAt.IsZero() {
+				return nil, true
+			}
+			return time.Until(server.shutdownAt), true
+		}
+
 		return nil, false
 	}
 
 	repl.Map(httpVars)
 }
 
-func getReqTLSReplacement(req *http.Request, key string) (interface{}, bool) {
+func getReqTLSReplacement(req *http.Request, key string) (any, bool) {
 	if req == nil || req.TLS == nil {
 		return nil, false
 	}
@@ -279,7 +299,7 @@ func getReqTLSReplacement(req *http.Request, key string) (interface{}, bool) {
 		if strings.HasPrefix(field, "client.san.") {
 			field = field[len("client.san."):]
 			var fieldName string
-			var fieldValue interface{}
+			var fieldValue any
 			switch {
 			case strings.HasPrefix(field, "dns_names"):
 				fieldName = "dns_names"
@@ -383,7 +403,7 @@ func getReqTLSReplacement(req *http.Request, key string) (interface{}, bool) {
 }
 
 // marshalPublicKey returns the byte encoding of pubKey.
-func marshalPublicKey(pubKey interface{}) ([]byte, error) {
+func marshalPublicKey(pubKey any) ([]byte, error) {
 	switch key := pubKey.(type) {
 	case *rsa.PublicKey:
 		return asn1.Marshal(key)

modules/caddyhttp/replacer_test.go

@@ -27,7 +27,7 @@ import (
 )
 
 func TestHTTPVarReplacement(t *testing.T) {
-	req, _ := http.NewRequest("GET", "/", nil)
+	req, _ := http.NewRequest(http.MethodGet, "/foo/bar.tar.gz", nil)
 	repl := caddy.NewReplacer()
 	ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
 	req = req.WithContext(ctx)
@@ -72,114 +72,134 @@ eqp31wM9il1n+guTNyxJd+FzVAH+hCZE5K+tCgVDdVFUlDEHHbS/wqb2PSIoouLV
 	addHTTPVarsToReplacer(repl, req, res)
 
 	for i, tc := range []struct {
-		input  string
+		get    string
 		expect string
 	}{
 		{
-			input:  "{http.request.scheme}",
+			get:    "http.request.scheme",
 			expect: "https",
 		},
 		{
-			input:  "{http.request.host}",
+			get:    "http.request.method",
+			expect: http.MethodGet,
+		},
+		{
+			get:    "http.request.host",
 			expect: "example.com",
 		},
 		{
-			input:  "{http.request.port}",
+			get:    "http.request.port",
 			expect: "80",
 		},
 		{
-			input:  "{http.request.hostport}",
+			get:    "http.request.hostport",
 			expect: "example.com:80",
 		},
 		{
-			input:  "{http.request.remote.host}",
+			get:    "http.request.remote.host",
 			expect: "localhost",
 		},
 		{
-			input:  "{http.request.remote.port}",
+			get:    "http.request.remote.port",
 			expect: "1234",
 		},
 		{
-			input:  "{http.request.host.labels.0}",
+			get:    "http.request.host.labels.0",
 			expect: "com",
 		},
 		{
-			input:  "{http.request.host.labels.1}",
+			get:    "http.request.host.labels.1",
 			expect: "example",
 		},
 		{
-			input:  "{http.request.host.labels.2}",
-			expect: "<empty>",
+			get:    "http.request.host.labels.2",
+			expect: "",
 		},
 		{
-			input:  "{http.request.tls.cipher_suite}",
+			get:    "http.request.uri.path.file",
+			expect: "bar.tar.gz",
+		},
+		{
+			get:    "http.request.uri.path.file.base",
+			expect: "bar.tar",
+		},
+		{
+			// not ideal, but also most correct, given that files can have dots (example: index.<SHA>.html) TODO: maybe this isn't right..
+			get:    "http.request.uri.path.file.ext",
+			expect: ".gz",
+		},
+		{
+			get:    "http.request.tls.cipher_suite",
 			expect: "TLS_AES_256_GCM_SHA384",
 		},
 		{
-			input:  "{http.request.tls.proto}",
+			get:    "http.request.tls.proto",
 			expect: "h2",
 		},
 		{
-			input:  "{http.request.tls.proto_mutual}",
+			get:    "http.request.tls.proto_mutual",
 			expect: "true",
 		},
 		{
-			input:  "{http.request.tls.resumed}",
+			get:    "http.request.tls.resumed",
 			expect: "false",
 		},
 		{
-			input:  "{http.request.tls.server_name}",
+			get:    "http.request.tls.server_name",
 			expect: "foo.com",
 		},
 		{
-			input:  "{http.request.tls.version}",
+			get:    "http.request.tls.version",
 			expect: "tls1.3",
 		},
 		{
-			input:  "{http.request.tls.client.fingerprint}",
+			get:    "http.request.tls.client.fingerprint",
 			expect: "9f57b7b497cceacc5459b76ac1c3afedbc12b300e728071f55f84168ff0f7702",
 		},
 		{
-			input:  "{http.request.tls.client.issuer}",
+			get:    "http.request.tls.client.issuer",
 			expect: "CN=Caddy Test CA",
 		},
 		{
-			input:  "{http.request.tls.client.serial}",
+			get:    "http.request.tls.client.serial",
 			expect: "2",
 		},
 		{
-			input:  "{http.request.tls.client.subject}",
+			get:    "http.request.tls.client.subject",
 			expect: "CN=client.localdomain",
 		},
 		{
-			input:  "{http.request.tls.client.san.dns_names}",
+			get:    "http.request.tls.client.san.dns_names",
 			expect: "[localhost]",
 		},
 		{
-			input:  "{http.request.tls.client.san.dns_names.0}",
+			get:    "http.request.tls.client.san.dns_names.0",
 			expect: "localhost",
 		},
 		{
-			input:  "{http.request.tls.client.san.dns_names.1}",
-			expect: "<empty>",
+			get:    "http.request.tls.client.san.dns_names.1",
+			expect: "",
 		},
 		{
-			input:  "{http.request.tls.client.san.ips}",
+			get:    "http.request.tls.client.san.ips",
 			expect: "[127.0.0.1]",
 		},
 		{
-			input:  "{http.request.tls.client.san.ips.0}",
+			get:    "http.request.tls.client.san.ips.0",
 			expect: "127.0.0.1",
 		},
 		{
-			input:  "{http.request.tls.client.certificate_pem}",
+			get:    "http.request.tls.client.certificate_pem",
 			expect: string(clientCert) + "\n", // returned value comes with a newline appended to it
 		},
 	} {
-		actual := repl.ReplaceAll(tc.input, "<empty>")
+		actual, got := repl.GetString(tc.get)
+		if !got {
+			t.Errorf("Test %d: Expected to recognize the placeholder name, but didn't", i)
+		}
 		if actual != tc.expect {
-			t.Errorf("Test %d: Expected placeholder %s to be '%s' but got '%s'",
-				i, tc.input, tc.expect, actual)
+			t.Errorf("Test %d: Expected %s to be '%s' but got '%s'",
+				i, tc.get, tc.expect, actual)
 		}
 	}
 }

modules/caddyhttp/responsewriter.go

@@ -62,6 +62,16 @@ func (rww *ResponseWriterWrapper) Push(target string, opts *http.PushOptions) er
 	return ErrNotImplemented
 }
 
+// ReadFrom implements io.ReaderFrom. It simply calls the underlying
+// ResponseWriter's ReadFrom method if there is one, otherwise it defaults
+// to io.Copy.
+func (rww *ResponseWriterWrapper) ReadFrom(r io.Reader) (n int64, err error) {
+	if rf, ok := rww.ResponseWriter.(io.ReaderFrom); ok {
+		return rf.ReadFrom(r)
+	}
+	return io.Copy(rww.ResponseWriter, r)
+}
+
 // HTTPInterfaces mix all the interfaces that middleware ResponseWriters need to support.
 type HTTPInterfaces interface {
 	http.ResponseWriter
@@ -111,15 +121,15 @@ type responseRecorder struct {
 //
 // Proper usage of a recorder looks like this:
 //
-//     rec := caddyhttp.NewResponseRecorder(w, buf, shouldBuffer)
-//     err := next.ServeHTTP(rec, req)
-//     if err != nil {
-//         return err
-//     }
-//     if !rec.Buffered() {
-//         return nil
-//     }
-//     // process the buffered response here
+//	rec := caddyhttp.NewResponseRecorder(w, buf, shouldBuffer)
+//	err := next.ServeHTTP(rec, req)
+//	if err != nil {
+//	    return err
+//	}
+//	if !rec.Buffered() {
+//	    return nil
+//	}
+//	// process the buffered response here
 //
 // The header map is not buffered; i.e. the ResponseRecorder's Header()
 // method returns the same header map of the underlying ResponseWriter.
@@ -129,7 +139,7 @@ type responseRecorder struct {
 // Once you are ready to write the response, there are two ways you can
 // do it. The easier way is to have the recorder do it:
 //
-//     rec.WriteResponse()
+//	rec.WriteResponse()
 //
 // This writes the recorded response headers as well as the buffered body.
 // Or, you may wish to do it yourself, especially if you manipulated the
@@ -138,9 +148,12 @@ type responseRecorder struct {
 // recorder's body buffer, but you might have your own body to write
 // instead):
 //
-//     w.WriteHeader(rec.Status())
-//     io.Copy(w, rec.Buffer())
+//	w.WriteHeader(rec.Status())
+//	io.Copy(w, rec.Buffer())
 //
+// As a special case, 1xx responses are not buffered nor recorded
+// because they are not the final response; they are passed through
+// directly to the underlying ResponseWriter.
 func NewResponseRecorder(w http.ResponseWriter, buf *bytes.Buffer, shouldBuffer ShouldBufferFunc) ResponseRecorder {
 	return &responseRecorder{
 		ResponseWriterWrapper: &ResponseWriterWrapper{ResponseWriter: w},
@@ -149,22 +162,29 @@ func NewResponseRecorder(w http.ResponseWriter, buf *bytes.Buffer, shouldBuffer
 	}
 }
 
+// WriteHeader writes the headers with statusCode to the wrapped
+// ResponseWriter unless the response is to be buffered instead.
+// 1xx responses are never buffered.
 func (rr *responseRecorder) WriteHeader(statusCode int) {
 	if rr.wroteHeader {
 		return
 	}
-	rr.statusCode = statusCode
-	rr.wroteHeader = true
 
-	// decide whether we should buffer the response
-	if rr.shouldBuffer == nil {
-		rr.stream = true
-	} else {
-		rr.stream = !rr.shouldBuffer(rr.statusCode, rr.ResponseWriterWrapper.Header())
+	// 1xx responses aren't final; just informational
+	if statusCode < 100 || statusCode > 199 {
+		rr.statusCode = statusCode
+		rr.wroteHeader = true
+
+		// decide whether we should buffer the response
+		if rr.shouldBuffer == nil {
+			rr.stream = true
+		} else {
+			rr.stream = !rr.shouldBuffer(rr.statusCode, rr.ResponseWriterWrapper.Header())
+		}
 	}
 
-	// if not buffered, immediately write header
-	if rr.stream {
+	// if informational or not buffered, immediately write header
+	if rr.stream || (100 <= statusCode && statusCode <= 199) {
 		rr.ResponseWriterWrapper.WriteHeader(rr.statusCode)
 	}
 }
@@ -178,9 +198,26 @@ func (rr *responseRecorder) Write(data []byte) (int, error) {
 	} else {
 		n, err = rr.buf.Write(data)
 	}
-	if err == nil {
-		rr.size += n
+
+	rr.size += n
+	return n, err
+}
+
+func (rr *responseRecorder) ReadFrom(r io.Reader) (int64, error) {
+	rr.WriteHeader(http.StatusOK)
+	var n int64
+	var err error
+	if rr.stream {
+		if rf, ok := rr.ResponseWriter.(io.ReaderFrom); ok {
+			n, err = rf.ReadFrom(r)
+		} else {
+			n, err = io.Copy(rr.ResponseWriter, r)
+		}
+	} else {
+		n, err = rr.buf.ReadFrom(r)
 	}
+
+	rr.size += int(n)
 	return n, err
 }
 
@@ -241,4 +278,10 @@ type ShouldBufferFunc func(status int, header http.Header) bool
 var (
 	_ HTTPInterfaces   = (*ResponseWriterWrapper)(nil)
 	_ ResponseRecorder = (*responseRecorder)(nil)
+
+	// Implementing ReaderFrom can be such a significant
+	// optimization that it should probably be required!
+	// see PR #5022 (25%-50% speedup)
+	_ io.ReaderFrom = (*ResponseWriterWrapper)(nil)
+	_ io.ReaderFrom = (*responseRecorder)(nil)
 )

modules/caddyhttp/responsewriter_test.go

@@ -0,0 +1,165 @@
+package caddyhttp
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+type responseWriterSpy interface {
+	http.ResponseWriter
+	Written() string
+	CalledReadFrom() bool
+}
+
+var (
+	_ responseWriterSpy = (*baseRespWriter)(nil)
+	_ responseWriterSpy = (*readFromRespWriter)(nil)
+)
+
+// a barebones http.ResponseWriter mock
+type baseRespWriter []byte
+
+func (brw *baseRespWriter) Write(d []byte) (int, error) {
+	*brw = append(*brw, d...)
+	return len(d), nil
+}
+func (brw *baseRespWriter) Header() http.Header        { return nil }
+func (brw *baseRespWriter) WriteHeader(statusCode int) {}
+func (brw *baseRespWriter) Written() string            { return string(*brw) }
+func (brw *baseRespWriter) CalledReadFrom() bool       { return false }
+
+// an http.ResponseWriter mock that supports ReadFrom
+type readFromRespWriter struct {
+	baseRespWriter
+	called bool
+}
+
+func (rf *readFromRespWriter) ReadFrom(r io.Reader) (int64, error) {
+	rf.called = true
+	return io.Copy(&rf.baseRespWriter, r)
+}
+
+func (rf *readFromRespWriter) CalledReadFrom() bool { return rf.called }
+
+func TestResponseWriterWrapperReadFrom(t *testing.T) {
+	tests := map[string]struct {
+		responseWriter responseWriterSpy
+		wantReadFrom   bool
+	}{
+		"no ReadFrom": {
+			responseWriter: &baseRespWriter{},
+			wantReadFrom:   false,
+		},
+		"has ReadFrom": {
+			responseWriter: &readFromRespWriter{},
+			wantReadFrom:   true,
+		},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			// what we expect middlewares to do:
+			type myWrapper struct {
+				*ResponseWriterWrapper
+			}
+
+			wrapped := myWrapper{
+				ResponseWriterWrapper: &ResponseWriterWrapper{ResponseWriter: tt.responseWriter},
+			}
+
+			const srcData = "boo!"
+			// hides everything but Read, since strings.Reader implements WriteTo it would
+			// take precedence over our ReadFrom.
+			src := struct{ io.Reader }{strings.NewReader(srcData)}
+
+			fmt.Println(name)
+			if _, err := io.Copy(wrapped, src); err != nil {
+				t.Errorf("Copy() err = %v", err)
+			}
+
+			if got := tt.responseWriter.Written(); got != srcData {
+				t.Errorf("data = %q, want %q", got, srcData)
+			}
+
+			if tt.responseWriter.CalledReadFrom() != tt.wantReadFrom {
+				if tt.wantReadFrom {
+					t.Errorf("ReadFrom() should have been called")
+				} else {
+					t.Errorf("ReadFrom() should not have been called")
+				}
+			}
+		})
+	}
+}
+
+func TestResponseRecorderReadFrom(t *testing.T) {
+	tests := map[string]struct {
+		responseWriter responseWriterSpy
+		shouldBuffer   bool
+		wantReadFrom   bool
+	}{
+		"buffered plain": {
+			responseWriter: &baseRespWriter{},
+			shouldBuffer:   true,
+			wantReadFrom:   false,
+		},
+		"streamed plain": {
+			responseWriter: &baseRespWriter{},
+			shouldBuffer:   false,
+			wantReadFrom:   false,
+		},
+		"buffered ReadFrom": {
+			responseWriter: &readFromRespWriter{},
+			shouldBuffer:   true,
+			wantReadFrom:   false,
+		},
+		"streamed ReadFrom": {
+			responseWriter: &readFromRespWriter{},
+			shouldBuffer:   false,
+			wantReadFrom:   true,
+		},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			var buf bytes.Buffer
+
+			rr := NewResponseRecorder(tt.responseWriter, &buf, func(status int, header http.Header) bool {
+				return tt.shouldBuffer
+			})
+
+			const srcData = "boo!"
+			// hides everything but Read, since strings.Reader implements WriteTo it would
+			// take precedence over our ReadFrom.
+			src := struct{ io.Reader }{strings.NewReader(srcData)}
+
+			if _, err := io.Copy(rr, src); err != nil {
+				t.Errorf("Copy() err = %v", err)
+			}
+
+			wantStreamed := srcData
+			wantBuffered := ""
+			if tt.shouldBuffer {
+				wantStreamed = ""
+				wantBuffered = srcData
+			}
+
+			if got := tt.responseWriter.Written(); got != wantStreamed {
+				t.Errorf("streamed data = %q, want %q", got, wantStreamed)
+			}
+			if got := buf.String(); got != wantBuffered {
+				t.Errorf("buffered data = %q, want %q", got, wantBuffered)
+			}
+
+			if tt.responseWriter.CalledReadFrom() != tt.wantReadFrom {
+				if tt.wantReadFrom {
+					t.Errorf("ReadFrom() should have been called")
+				} else {
+					t.Errorf("ReadFrom() should not have been called")
+				}
+			}
+		})
+	}
+}

modules/caddyhttp/reverseproxy/addresses.go

@@ -80,9 +80,9 @@ func parseUpstreamDialAddress(upstreamAddr string) (string, string, error) {
 		scheme, host, port = toURL.Scheme, toURL.Hostname(), toURL.Port()
 	} else {
 		// extract network manually, since caddy.ParseNetworkAddress() will always add one
-		if idx := strings.Index(upstreamAddr, "/"); idx >= 0 {
-			network = strings.ToLower(strings.TrimSpace(upstreamAddr[:idx]))
-			upstreamAddr = upstreamAddr[idx+1:]
+		if beforeSlash, afterSlash, slashFound := strings.Cut(upstreamAddr, "/"); slashFound {
+			network = strings.ToLower(strings.TrimSpace(beforeSlash))
+			upstreamAddr = afterSlash
 		}
 		var err error
 		host, port, err = net.SplitHostPort(upstreamAddr)
@@ -96,6 +96,12 @@ func parseUpstreamDialAddress(upstreamAddr string) (string, string, error) {
 		}
 	}
 
+	// special case network to support both unix and h2c at the same time
+	if network == "unix+h2c" {
+		network = "unix"
+		scheme = "h2c"
+	}
+
 	// for simplest possible config, we only need to include
 	// the network portion if the user specified one
 	if network != "" {