go.mod: Upgrade various dependencies (#5362)

* chore: Upgrade various dependencies

* Support CEL file matcher with no args

* Document `http.request.orig_uri.path.*`, reorder placeholders in docs

---------

Co-authored-by: Matt Holt <mholt@users.noreply.github.com>

.github/dependabot.yml

@@ -0,0 +1,7 @@
+---
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
-        go: [ '1.18', '1.19' ]
+        go: [ '1.18', '1.20' ]
 
         include:
         # Set the minimum Go patch version for the given Go minor
@@ -27,8 +27,8 @@ jobs:
         - go: '1.18'
           GO_SEMVER: '~1.18.4'
 
-        - go: '1.19'
-          GO_SEMVER: '~1.19.0'
+        - go: '1.20'
+          GO_SEMVER: '~1.20.0'
 
         # Set some variables per OS, usable via ${{ matrix.VAR }}
         # CADDY_BIN_PATH: the path to the compiled Caddy binary, for artifact publishing
@@ -64,7 +64,7 @@ jobs:
     #     go get github.com/axw/gocov/gocov
     #     go get github.com/AlekSi/gocov-xml
     #     go get -u github.com/jstemmer/go-junit-report
-    #     echo "::add-path::$(go env GOPATH)/bin"
+    #     echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
 
     - name: Print Go version and environment
       id: vars
@@ -77,10 +77,10 @@ jobs:
         env
         printf "Git version: $(git version)\n\n"
         # Calculate the short SHA1 hash of the git commit
-        echo "::set-output name=short_sha::$(git rev-parse --short HEAD)"
+        echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
 
     - name: Cache the build cache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         # In order:
         # * Module download cache
@@ -109,7 +109,7 @@ jobs:
         go build -trimpath -ldflags="-w -s" -v
 
     - name: Publish Build Artifact
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v3
       with:
         name: caddy_${{ runner.os }}_go${{ matrix.go }}_${{ steps.vars.outputs.short_sha }}
         path: ${{ matrix.CADDY_BIN_PATH }}
@@ -123,7 +123,7 @@ jobs:
       run: |
         # (go test -v -coverprofile=cover-profile.out -race ./... 2>&1) > test-results/test-result.out
         go test -v -coverprofile="cover-profile.out" -short -race ./...
-        # echo "::set-output name=status::$?"
+        # echo "status=$?" >> $GITHUB_OUTPUT
 
     # Relevant step if we reinvestigate publishing test/coverage reports
     # - name: Prepare coverage reports
@@ -143,7 +143,7 @@ jobs:
   s390x-test:
     name: test (s390x on IBM Z)
     runs-on: ubuntu-latest
-    if: github.event.pull_request.head.repo.full_name == github.repository
+    if: github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]'
     continue-on-error: true  # August 2020: s390x VM is down due to weather and power issues
     steps:
       - name: Checkout code into the Go module directory
@@ -156,17 +156,18 @@ jobs:
           short_sha=$(git rev-parse --short HEAD)
 
           # The environment is fresh, so there's no point in keeping accepting and adding the key.
-          rsync -arz -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" --progress --delete --exclude '.git' . caddy-ci@ci-s390x.caddyserver.com:/var/tmp/"$short_sha"
-          ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -t caddy-ci@ci-s390x.caddyserver.com "cd /var/tmp/$short_sha; go version; go env; printf "\n\n";CGO_ENABLED=0 go test -v ./..."
+          rsync -arz -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" --progress --delete --exclude '.git' . "$CI_USER"@ci-s390x.caddyserver.com:/var/tmp/"$short_sha"
+          ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -t "$CI_USER"@ci-s390x.caddyserver.com "cd /var/tmp/$short_sha; go version; go env; printf "\n\n";CGO_ENABLED=0 go test -v ./..."
           test_result=$?
 
           # There's no need leaving the files around
-          ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null caddy-ci@ci-s390x.caddyserver.com "rm -rf /var/tmp/'$short_sha'"
+          ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$CI_USER"@ci-s390x.caddyserver.com "rm -rf /var/tmp/'$short_sha'"
 
           echo "Test exit code: $test_result"
           exit $test_result
         env:
           SSH_KEY: ${{ secrets.S390X_SSH_KEY }}
+          CI_USER: ${{ secrets.CI_USER }}
 
   goreleaser-check:
     runs-on: ubuntu-latest
@@ -174,7 +175,7 @@ jobs:
       - name: checkout
         uses: actions/checkout@v3
       
-      - uses: goreleaser/goreleaser-action@v2
+      - uses: goreleaser/goreleaser-action@v4
         with:
           version: latest
           args: check

.github/workflows/cross-build.yml

@@ -16,13 +16,13 @@ jobs:
       fail-fast: false
       matrix:
         goos: ['android', 'linux', 'solaris', 'illumos', 'dragonfly', 'freebsd', 'openbsd', 'plan9', 'windows', 'darwin', 'netbsd']
-        go: [ '1.19' ]
+        go: [ '1.20' ]
 
         include:
         # Set the minimum Go patch version for the given Go minor
         # Usable via ${{ matrix.GO_SEMVER }}
-        - go: '1.19'
-          GO_SEMVER: '~1.19.0'
+        - go: '1.20'
+          GO_SEMVER: '~1.20.0'
 
     runs-on: ubuntu-latest
     continue-on-error: true
@@ -44,7 +44,7 @@ jobs:
           env
 
       - name: Cache the build cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           # In order:
           # * Module download cache

.github/workflows/lint.yml

@@ -10,9 +10,15 @@ on:
       - master
       - 2.*
 
+permissions:
+  contents: read
+
 jobs:
   # From https://github.com/golangci/golangci-lint-action
   golangci:
+    permissions:
+      contents: read  # for actions/checkout to fetch code
+      pull-requests: read  # for golangci/golangci-lint-action to fetch pull requests
     name: lint
     strategy:
       matrix:
@@ -28,7 +34,7 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.47
+          version: v1.50
           # Windows times out frequently after about 5m50s if we don't set a longer timeout.
           args: --timeout 10m
           # Optional: show only new issues if it's a pull request. The default value is `false`.

.github/workflows/release.yml

@@ -11,15 +11,22 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        go: [ '1.19' ]
+        go: [ '1.20' ]
 
         include:
         # Set the minimum Go patch version for the given Go minor
         # Usable via ${{ matrix.GO_SEMVER }}
-        - go: '1.19'
-          GO_SEMVER: '~1.19.0'
+        - go: '1.20'
+          GO_SEMVER: '~1.20.0'
 
     runs-on: ${{ matrix.os }}
+    # https://github.com/sigstore/cosign/issues/1258#issuecomment-1002251233
+    # https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect#adding-permissions-settings
+    permissions:
+      id-token: write
+      # https://docs.github.com/en/rest/overview/permissions-required-for-github-apps#permission-on-contents
+      # "Releases" is part of `contents`, so it needs the `write`
+      contents: write
 
     steps:
     - name: Install Go
@@ -54,8 +61,8 @@ jobs:
         go env
         printf "\n\nSystem environment:\n\n"
         env
-        echo "::set-output name=version_tag::${GITHUB_REF/refs\/tags\//}"
-        echo "::set-output name=short_sha::$(git rev-parse --short HEAD)"
+        echo "version_tag=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
+        echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
 
         # Add "pip install" CLI tools to PATH
         echo ~/.local/bin >> $GITHUB_PATH
@@ -67,10 +74,10 @@ jobs:
         TAG_MINOR=`echo ${TAG#v} | sed -e "s#$SEMVER_RE#\2#"`
         TAG_PATCH=`echo ${TAG#v} | sed -e "s#$SEMVER_RE#\3#"`
         TAG_SPECIAL=`echo ${TAG#v} | sed -e "s#$SEMVER_RE#\4#"`
-        echo "::set-output name=tag_major::${TAG_MAJOR}"
-        echo "::set-output name=tag_minor::${TAG_MINOR}"
-        echo "::set-output name=tag_patch::${TAG_PATCH}"
-        echo "::set-output name=tag_special::${TAG_SPECIAL}"
+        echo "tag_major=${TAG_MAJOR}" >> $GITHUB_OUTPUT
+        echo "tag_minor=${TAG_MINOR}" >> $GITHUB_OUTPUT
+        echo "tag_patch=${TAG_PATCH}" >> $GITHUB_OUTPUT
+        echo "tag_special=${TAG_SPECIAL}" >> $GITHUB_OUTPUT
 
     # Cloudsmith CLI tooling for pushing releases
     # See https://help.cloudsmith.io/docs/cli
@@ -88,7 +95,7 @@ jobs:
         git verify-tag "${{ steps.vars.outputs.version_tag }}" || exit 1
 
     - name: Cache the build cache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         # In order:
         # * Module download cache
@@ -109,10 +116,10 @@ jobs:
       run: syft version
     # GoReleaser will take care of publishing those artifacts into the release
     - name: Run GoReleaser
-      uses: goreleaser/goreleaser-action@v2
+      uses: goreleaser/goreleaser-action@v4
       with:
         version: latest
-        args: release --rm-dist
+        args: release --rm-dist --timeout 60m
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         TAG: ${{ steps.vars.outputs.version_tag }}

.github/workflows/release_published.yml

@@ -17,7 +17,7 @@ jobs:
 
     # See https://github.com/peter-evans/repository-dispatch
     - name: Trigger event on caddyserver/dist
-      uses: peter-evans/repository-dispatch@v1
+      uses: peter-evans/repository-dispatch@v2
       with:
         token: ${{ secrets.REPO_DISPATCH_TOKEN }}
         repository: caddyserver/dist
@@ -25,7 +25,7 @@ jobs:
         client-payload: '{"tag": "${{ github.event.release.tag_name }}"}'
 
     - name: Trigger event on caddyserver/caddy-docker
-      uses: peter-evans/repository-dispatch@v1
+      uses: peter-evans/repository-dispatch@v2
       with:
         token: ${{ secrets.REPO_DISPATCH_TOKEN }}
         repository: caddyserver/caddy-docker

.golangci.yml

@@ -96,3 +96,7 @@ issues:
       text: "G404" # G404: Insecure random number source (rand)
       linters:
         - gosec
+    - path: modules/caddyhttp/reverseproxy/streaming.go
+      text: "G404" # G404: Insecure random number source (rand)
+      linters:
+        - gosec

.goreleaser.yml

@@ -4,6 +4,7 @@ before:
     # This is so we can run goreleaser on tag without Git complaining of being dirty. The main.go in cmd/caddy directory 
     # cannot be built within that directory due to changes necessary for the build causing Git to be dirty, which
     # subsequently causes gorleaser to refuse running.
+    - rm -rf caddy-build caddy-dist
     - mkdir -p caddy-build
     - cp cmd/caddy/main.go caddy-build/main.go
     - /bin/sh -c 'cd ./caddy-build && go mod init caddy'
@@ -65,21 +66,41 @@ builds:
   - -mod=readonly
   ldflags:
   - -s -w
+
 signs:
   - cmd: cosign
     signature: "${artifact}.sig"
-    args: ["sign-blob", "--oidc-issuer=https://token.actions.githubusercontent.com", "--output=${signature}", "${artifact}"]
+    certificate: '{{ trimsuffix (trimsuffix .Env.artifact ".zip") ".tar.gz" }}.pem'
+    args: ["sign-blob", "--output-signature=${signature}", "--output-certificate", "${certificate}", "${artifact}"]
     artifacts: all
+
 sboms:
   - artifacts: binary
+    documents:
+      - >-
+        {{ .ProjectName }}_
+        {{- .Version }}_
+        {{- if eq .Os "darwin" }}mac{{ else }}{{ .Os }}{{ end }}_
+        {{- .Arch }}
+        {{- with .Arm }}v{{ . }}{{ end }}
+        {{- with .Mips }}_{{ . }}{{ end }}
+        {{- if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}.sbom
     cmd: syft
-    args: ["$artifact", "--file", "$sbom", "--output", "cyclonedx-json"]
+    args: ["$artifact", "--file", "${document}", "--output", "cyclonedx-json"]
+
 archives:
   - format_overrides:
       - goos: windows
         format: zip
-    replacements:
-      darwin: mac
+    name_template: >-
+      {{ .ProjectName }}_
+      {{- .Version }}_
+      {{- if eq .Os "darwin" }}mac{{ else }}{{ .Os }}{{ end }}_
+      {{- .Arch }}
+      {{- with .Arm }}v{{ . }}{{ end }}
+      {{- with .Mips }}_{{ . }}{{ end }}
+      {{- if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}
+
 checksum:
   algorithm: sha512
 
@@ -124,7 +145,6 @@ nfpms:
       preremove: ./caddy-dist/scripts/preremove.sh
       postremove: ./caddy-dist/scripts/postremove.sh
 
-
 release:
   github:
     owner: caddyserver

README.md

@@ -1,13 +1,19 @@
 <p align="center">
-	<a href="https://caddyserver.com"><img src="https://user-images.githubusercontent.com/1128849/36338535-05fb646a-136f-11e8-987b-e6901e717d5a.png" alt="Caddy" width="450"></a>
+	<a href="https://caddyserver.com">
+		<picture>
+			<source media="(prefers-color-scheme: dark)" srcset="https://user-images.githubusercontent.com/1128849/210187358-e2c39003-9a5e-4dd5-a783-6deb6483ee72.svg">
+			<source media="(prefers-color-scheme: light)" srcset="https://user-images.githubusercontent.com/1128849/210187356-dfb7f1c5-ac2e-43aa-bb23-fc014280ae1f.svg">
+			<img src="https://user-images.githubusercontent.com/1128849/210187356-dfb7f1c5-ac2e-43aa-bb23-fc014280ae1f.svg" alt="Caddy" width="550">
+		</picture>
+	</a>
 	<br>
-	<h3 align="center">a <a href="https://zerossl.com"><img src="https://caddyserver.com/resources/images/zerossl-logo.svg" height="28" valign="middle"></a> project</h3>
+	<h3 align="center">a <a href="https://zerossl.com"><img src="https://user-images.githubusercontent.com/55066419/208327323-2770dc16-ec09-43a0-9035-c5b872c2ad7f.svg" height="28" style="vertical-align: -7.7px" valign="middle"></a> project</h3>
 </p>
 <hr>
 <h3 align="center">Every site on HTTPS</h3>
 <p align="center">Caddy is an extensible server platform that uses TLS by default.</p>
 <p align="center">
-	<a href="https://github.com/caddyserver/caddy/actions?query=workflow%3ACross-Platform"><img src="https://github.com/caddyserver/caddy/workflows/Cross-Platform/badge.svg"></a>
+	<a href="https://github.com/caddyserver/caddy/actions/workflows/ci.yml"><img src="https://github.com/caddyserver/caddy/actions/workflows/ci.yml/badge.svg"></a>
 	<a href="https://pkg.go.dev/github.com/caddyserver/caddy/v2"><img src="https://img.shields.io/badge/godoc-reference-%23007d9c.svg"></a>
 	<br>
 	<a href="https://twitter.com/caddyserver" title="@caddyserver on Twitter"><img src="https://img.shields.io/badge/twitter-@caddyserver-55acee.svg" alt="@caddyserver on Twitter"></a>
@@ -40,7 +46,13 @@
 <p align="center">
 	<b>Powered by</b>
 	<br>
-	<a href="https://github.com/caddyserver/certmagic"><img src="https://user-images.githubusercontent.com/1128849/49704830-49d37200-fbd5-11e8-8385-767e0cd033c3.png" alt="CertMagic" width="250"></a>
+	<a href="https://github.com/caddyserver/certmagic">
+		<picture>
+			<source media="(prefers-color-scheme: dark)" srcset="https://user-images.githubusercontent.com/55066419/206946718-740b6371-3df3-4d72-a822-47e4c48af999.png">
+			<source media="(prefers-color-scheme: light)" srcset="https://user-images.githubusercontent.com/1128849/49704830-49d37200-fbd5-11e8-8385-767e0cd033c3.png">
+			<img src="https://user-images.githubusercontent.com/1128849/49704830-49d37200-fbd5-11e8-8385-767e0cd033c3.png" alt="CertMagic" width="250">
+		</picture>
+	</a>
 </p>
 
 
@@ -78,7 +90,7 @@ Requirements:
 - [Go 1.18 or newer](https://golang.org/dl/)
 
 ### For development
- 
+
 _**Note:** These steps [will not embed proper version information](https://github.com/golang/go/issues/29228). For that, please follow the instructions in the next section._
 
 ```bash
@@ -185,4 +197,4 @@ Matthew Holt began developing Caddy in 2014 while studying computer science at B
 
 Caddy is a project of [ZeroSSL](https://zerossl.com), a Stack Holdings company.
 
-Debian package repository hosting is graciously provided by [Cloudsmith](https://cloudsmith.com). Cloudsmith is the only fully hosted, cloud-native, universal package management solution, that enables your organization to create, store and share packages in any format, to any place, with total confidence.
+Debian package repository hosting is graciously provided by [Cloudsmith](https://cloudsmith.com). Cloudsmith is the only fully hosted, cloud-native, universal package management solution, that enables your organization to create, store and share packages in any format, to any place, with total confidence.

admin.go

@@ -46,6 +46,17 @@ import (
 	"go.uber.org/zap/zapcore"
 )
 
+func init() {
+	// The hard-coded default `DefaultAdminListen` can be overidden
+	// by setting the `CADDY_ADMIN` environment variable.
+	// The environment variable may be used by packagers to change
+	// the default admin address to something more appropriate for
+	// that platform. See #5317 for discussion.
+	if env, exists := os.LookupEnv("CADDY_ADMIN"); exists {
+		DefaultAdminListen = env
+	}
+}
+
 // AdminConfig configures Caddy's API endpoint, which is used
 // to manage Caddy while it is running.
 type AdminConfig struct {
@@ -57,7 +68,9 @@ type AdminConfig struct {
 
 	// The address to which the admin endpoint's listener should
 	// bind itself. Can be any single network address that can be
-	// parsed by Caddy. Default: localhost:2019
+	// parsed by Caddy. Accepts placeholders.
+	// Default: the value of the `CADDY_ADMIN` environment variable,
+	// or `localhost:2019` otherwise.
 	Listen string `json:"listen,omitempty"`
 
 	// If true, CORS headers will be emitted, and requests to the
@@ -156,7 +169,7 @@ type IdentityConfig struct {
 //
 // EXPERIMENTAL: Subject to change.
 type RemoteAdmin struct {
-	// The address on which to start the secure listener.
+	// The address on which to start the secure listener. Accepts placeholders.
 	// Default: :2021
 	Listen string `json:"listen,omitempty"`
 
@@ -382,7 +395,7 @@ func replaceLocalAdminServer(cfg *Config) error {
 
 	handler := cfg.Admin.newAdminHandler(addr, false)
 
-	ln, err := Listen(addr.Network, addr.JoinHostPort(0))
+	ln, err := addr.Listen(context.TODO(), 0, net.ListenConfig{})
 	if err != nil {
 		return err
 	}
@@ -403,7 +416,7 @@ func replaceLocalAdminServer(cfg *Config) error {
 		serverMu.Lock()
 		server := localAdminServer
 		serverMu.Unlock()
-		if err := server.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
+		if err := server.Serve(ln.(net.Listener)); !errors.Is(err, http.ErrServerClosed) {
 			adminLogger.Error("admin server shutdown for unknown reason", zap.Error(err))
 		}
 	}()
@@ -549,10 +562,11 @@ func replaceRemoteAdminServer(ctx Context, cfg *Config) error {
 	serverMu.Unlock()
 
 	// start listener
-	ln, err := Listen(addr.Network, addr.JoinHostPort(0))
+	lnAny, err := addr.Listen(ctx, 0, net.ListenConfig{})
 	if err != nil {
 		return err
 	}
+	ln := lnAny.(net.Listener)
 	ln = tls.NewListener(ln, tlsConfig)
 
 	go func() {
@@ -571,12 +585,13 @@ func replaceRemoteAdminServer(ctx Context, cfg *Config) error {
 }
 
 func (ident *IdentityConfig) certmagicConfig(logger *zap.Logger, makeCache bool) *certmagic.Config {
+	var cmCfg *certmagic.Config
 	if ident == nil {
 		// user might not have configured identity; that's OK, we can still make a
 		// certmagic config, although it'll be mostly useless for remote management
 		ident = new(IdentityConfig)
 	}
-	cmCfg := &certmagic.Config{
+	template := certmagic.Config{
 		Storage: DefaultStorage, // do not act as part of a cluster (this is for the server's local identity)
 		Logger:  logger,
 		Issuers: ident.issuers,
@@ -586,9 +601,11 @@ func (ident *IdentityConfig) certmagicConfig(logger *zap.Logger, makeCache bool)
 			GetConfigForCert: func(certmagic.Certificate) (*certmagic.Config, error) {
 				return cmCfg, nil
 			},
+			Logger: logger.Named("cache"),
 		})
 	}
-	return certmagic.New(identityCertCache, *cmCfg)
+	cmCfg = certmagic.New(identityCertCache, template)
+	return cmCfg
 }
 
 // IdentityCredentials returns this instance's configured, managed identity credentials
@@ -1246,7 +1263,10 @@ func (e APIError) Error() string {
 // parseAdminListenAddr extracts a singular listen address from either addr
 // or defaultAddr, returning the network and the address of the listener.
 func parseAdminListenAddr(addr string, defaultAddr string) (NetworkAddress, error) {
-	input := addr
+	input, err := NewReplacer().ReplaceOrErr(addr, true, true)
+	if err != nil {
+		return NetworkAddress{}, fmt.Errorf("replacing listen address: %v", err)
+	}
 	if input == "" {
 		input = defaultAddr
 	}

admin_test.go

@@ -161,7 +161,7 @@ func (fooModule) Stop() error  { return nil }
 func TestETags(t *testing.T) {
 	RegisterModule(fooModule{})
 
-	if err := Load([]byte(`{"apps": {"foo": {"strField": "abc", "intField": 0}}}`), true); err != nil {
+	if err := Load([]byte(`{"admin": {"listen": "localhost:2999"}, "apps": {"foo": {"strField": "abc", "intField": 0}}}`), true); err != nil {
 		t.Fatalf("loading: %s", err)
 	}
 

caddy.go

@@ -31,6 +31,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/caddyserver/caddy/v2/notify"
@@ -676,6 +677,10 @@ func Validate(cfg *Config) error {
 // Errors are logged along the way, and an appropriate exit
 // code is emitted.
 func exitProcess(ctx context.Context, logger *zap.Logger) {
+	// let the rest of the program know we're quitting
+	atomic.StoreInt32(exiting, 1)
+
+	// give the OS or service/process manager our 2 weeks' notice: we quit
 	if err := notify.Stopping(); err != nil {
 		Log().Error("unable to notify service manager of stopping state", zap.Error(err))
 	}
@@ -739,6 +744,12 @@ func exitProcess(ctx context.Context, logger *zap.Logger) {
 	}()
 }
 
+var exiting = new(int32) // accessed atomically
+
+// Exiting returns true if the process is exiting.
+// EXPERIMENTAL API: subject to change or removal.
+func Exiting() bool { return atomic.LoadInt32(exiting) == 1 }
+
 // Duration can be an integer or a string. An integer is
 // interpreted as nanoseconds. If a string, it is a Go
 // time.Duration value such as `300ms`, `1.5h`, or `2h45m`;
@@ -763,8 +774,12 @@ func (d *Duration) UnmarshalJSON(b []byte) error {
 
 // ParseDuration parses a duration string, adding
 // support for the "d" unit meaning number of days,
-// where a day is assumed to be 24h.
+// where a day is assumed to be 24h. The maximum
+// input string length is 1024.
 func ParseDuration(s string) (time.Duration, error) {
+	if len(s) > 1024 {
+		return 0, fmt.Errorf("parsing duration: input string too long")
+	}
 	var inNumber bool
 	var numStart int
 	for i := 0; i < len(s); i++ {
@@ -809,6 +824,20 @@ func InstanceID() (uuid.UUID, error) {
 	return uuid.ParseBytes(uuidFileBytes)
 }
 
+// CustomVersion is an optional string that overrides Caddy's
+// reported version. It can be helpful when downstream packagers
+// need to manually set Caddy's version. If no other version
+// information is available, the short form version (see
+// Version()) will be set to CustomVersion, and the full version
+// will include CustomVersion at the beginning.
+//
+// Set this variable during `go build` with `-ldflags`:
+//
+//	-ldflags '-X github.com/caddyserver/caddy/v2.CustomVersion=v2.6.2'
+//
+// for example.
+var CustomVersion string
+
 // Version returns the Caddy version in a simple/short form, and
 // a full version string. The short form will not have spaces and
 // is intended for User-Agent strings and similar, but may be
@@ -818,8 +847,10 @@ func InstanceID() (uuid.UUID, error) {
 // build info provided by go.mod dependencies; then it tries to
 // get info from embedded VCS information, which requires having
 // built Caddy from a git repository. If no version is available,
-// this function returns "(devel)" becaise Go uses that, but for
-// the simple form we change it to "unknown".
+// this function returns "(devel)" because Go uses that, but for
+// the simple form we change it to "unknown". If still no version
+// is available (e.g. no VCS repo), then it will use CustomVersion;
+// CustomVersion is always prepended to the full version string.
 //
 // See relevant Go issues: https://github.com/golang/go/issues/29228
 // and https://github.com/golang/go/issues/50603.
@@ -833,13 +864,21 @@ func Version() (simple, full string) {
 	// bi.Main... hopefully.
 	var module *debug.Module
 	bi, ok := debug.ReadBuildInfo()
-	if ok {
-		// find the Caddy module in the dependency list
-		for _, dep := range bi.Deps {
-			if dep.Path == ImportPath {
-				module = dep
-				break
-			}
+	if !ok {
+		if CustomVersion != "" {
+			full = CustomVersion
+			simple = CustomVersion
+			return
+		}
+		full = "unknown"
+		simple = "unknown"
+		return
+	}
+	// find the Caddy module in the dependency list
+	for _, dep := range bi.Deps {
+		if dep.Path == ImportPath {
+			module = dep
+			break
 		}
 	}
 	if module != nil {
@@ -895,8 +934,22 @@ func Version() (simple, full string) {
 		}
 	}
 
+	if full == "" {
+		if CustomVersion != "" {
+			full = CustomVersion
+		} else {
+			full = "unknown"
+		}
+	} else if CustomVersion != "" {
+		full = CustomVersion + " " + full
+	}
+
 	if simple == "" || simple == "(devel)" {
-		simple = "unknown"
+		if CustomVersion != "" {
+			simple = CustomVersion
+		} else {
+			simple = "unknown"
+		}
 	}
 
 	return

caddyconfig/caddyfile/adapter.go

@@ -54,7 +54,7 @@ func (a Adapter) Adapt(body []byte, options map[string]any) ([]byte, []caddyconf
 
 	// lint check: see if input was properly formatted; sometimes messy files files parse
 	// successfully but result in logical errors (the Caddyfile is a bad format, I'm sorry)
-	if warning, different := formattingDifference(filename, body); different {
+	if warning, different := FormattingDifference(filename, body); different {
 		warnings = append(warnings, warning)
 	}
 
@@ -63,10 +63,10 @@ func (a Adapter) Adapt(body []byte, options map[string]any) ([]byte, []caddyconf
 	return result, warnings, err
 }
 
-// formattingDifference returns a warning and true if the formatted version
+// FormattingDifference returns a warning and true if the formatted version
 // is any different from the input; empty warning and false otherwise.
 // TODO: also perform this check on imported files
-func formattingDifference(filename string, body []byte) (caddyconfig.Warning, bool) {
+func FormattingDifference(filename string, body []byte) (caddyconfig.Warning, bool) {
 	// replace windows-style newlines to normalize comparison
 	normalizedBody := bytes.Replace(body, []byte("\r\n"), []byte("\n"), -1)
 

caddyconfig/caddyfile/formatter.go

@@ -153,7 +153,10 @@ func Format(input []byte) []byte {
 			openBraceWritten = true
 			nextLine()
 			newLines = 0
-			nesting++
+			// prevent infinite nesting from ridiculous inputs (issue #4169)
+			if nesting < 10 {
+				nesting++
+			}
 		}
 
 		switch {

caddyconfig/caddyfile/parse.go

@@ -61,20 +61,12 @@ func Parse(filename string, input []byte) ([]ServerBlock, error) {
 // It returns all the tokens from the input, unstructured
 // and in order. It may mutate input as it expands env vars.
 func allTokens(filename string, input []byte) ([]Token, error) {
-	inputCopy, err := replaceEnvVars(input)
-	if err != nil {
-		return nil, err
-	}
-	tokens, err := Tokenize(inputCopy, filename)
-	if err != nil {
-		return nil, err
-	}
-	return tokens, nil
+	return Tokenize(replaceEnvVars(input), filename)
 }
 
 // replaceEnvVars replaces all occurrences of environment variables.
 // It mutates the underlying array and returns the updated slice.
-func replaceEnvVars(input []byte) ([]byte, error) {
+func replaceEnvVars(input []byte) []byte {
 	var offset int
 	for {
 		begin := bytes.Index(input[offset:], spanOpen)
@@ -115,7 +107,7 @@ func replaceEnvVars(input []byte) ([]byte, error) {
 		// continue at the end of the replacement
 		offset = begin + len(envVarBytes)
 	}
-	return input, nil
+	return input
 }
 
 type parser struct {
@@ -397,6 +389,20 @@ func (p *parser) doImport() error {
 			} else {
 				return p.Errf("File to import not found: %s", importPattern)
 			}
+		} else {
+			// See issue #5295 - should skip any files that start with a . when iterating over them.
+			sep := string(filepath.Separator)
+			segGlobPattern := strings.Split(globPattern, sep)
+			if strings.HasPrefix(segGlobPattern[len(segGlobPattern)-1], "*") {
+				var tmpMatches []string
+				for _, m := range matches {
+					seg := strings.Split(m, sep)
+					if !strings.HasPrefix(seg[len(seg)-1], ".") {
+						tmpMatches = append(tmpMatches, m)
+					}
+				}
+				matches = tmpMatches
+			}
 		}
 
 		// collect all the imported tokens
@@ -459,6 +465,12 @@ func (p *parser) doSingleImport(importFile string) ([]Token, error) {
 		return nil, p.Errf("Could not read imported file %s: %v", importFile, err)
 	}
 
+	// only warning in case of empty files
+	if len(input) == 0 || len(strings.TrimSpace(string(input))) == 0 {
+		caddy.Log().Warn("Import file is empty", zap.String("file", importFile))
+		return []Token{}, nil
+	}
+
 	importedTokens, err := allTokens(importFile, input)
 	if err != nil {
 		return nil, p.Errf("Could not read tokens while importing %s: %v", importFile, err)

caddyconfig/caddyfile/parse_test.go

@@ -187,6 +187,23 @@ func TestParseOneAndImport(t *testing.T) {
 
 		{`import testdata/not_found.txt`, true, []string{}, []int{}},
 
+		// empty file should just log a warning, and result in no tokens
+		{`import testdata/empty.txt`, false, []string{}, []int{}},
+
+		{`import testdata/only_white_space.txt`, false, []string{}, []int{}},
+
+		// import path/to/dir/* should skip any files that start with a . when iterating over them.
+		{`localhost
+		  dir1 arg1
+		  import testdata/glob/*`, false, []string{
+			"localhost",
+		}, []int{2, 3, 1}},
+
+		// import path/to/dir/.* should continue to read all dotfiles in a dir.
+		{`import testdata/glob/.*`, false, []string{
+			"host1",
+		}, []int{1, 2}},
+
 		{`""`, false, []string{}, []int{}},
 
 		{``, false, []string{}, []int{}},
@@ -604,10 +621,7 @@ func TestEnvironmentReplacement(t *testing.T) {
 			expect: "}{$",
 		},
 	} {
-		actual, err := replaceEnvVars([]byte(test.input))
-		if err != nil {
-			t.Fatal(err)
-		}
+		actual := replaceEnvVars([]byte(test.input))
 		if !bytes.Equal(actual, []byte(test.expect)) {
 			t.Errorf("Test %d: Expected: '%s' but got '%s'", i, test.expect, actual)
 		}

caddyconfig/caddyfile/testdata/glob/.dotfile.txt

@@ -0,0 +1,4 @@
+host1 {
+	dir1
+	dir2 arg1
+}

caddyconfig/caddyfile/testdata/glob/import_test1.txt

@@ -0,0 +1,2 @@
+dir2 arg1 arg2
+dir3

caddyconfig/caddyfile/testdata/only_white_space.txt

@@ -0,0 +1,7 @@
+
+
+ 
+  
+　
+
+

caddyconfig/httpcaddyfile/addresses.go

@@ -36,12 +36,12 @@ import (
 // server block that share the same address stay grouped together so the config
 // isn't repeated unnecessarily. For example, this Caddyfile:
 //
-// 	example.com {
-// 		bind 127.0.0.1
-// 	}
-// 	www.example.com, example.net/path, localhost:9999 {
-// 		bind 127.0.0.1 1.2.3.4
-// 	}
+//	example.com {
+//		bind 127.0.0.1
+//	}
+//	www.example.com, example.net/path, localhost:9999 {
+//		bind 127.0.0.1 1.2.3.4
+//	}
 //
 // has two server blocks to start with. But expressed in this Caddyfile are
 // actually 4 listener addresses: 127.0.0.1:443, 1.2.3.4:443, 127.0.0.1:9999,
@@ -219,7 +219,7 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 		return nil, fmt.Errorf("[%s] scheme and port violate convention", key)
 	}
 
-	// the bind directive specifies hosts, but is optional
+	// the bind directive specifies hosts (and potentially network), but is optional
 	lnHosts := make([]string, 0, len(sblock.pile["bind"]))
 	for _, cfgVal := range sblock.pile["bind"] {
 		lnHosts = append(lnHosts, cfgVal.Value.([]string)...)
@@ -234,11 +234,23 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 
 	// use a map to prevent duplication
 	listeners := make(map[string]struct{})
-	for _, host := range lnHosts {
-		// host can have network + host (e.g. "tcp6/localhost") but
-		// will/should not have port information because this usually
-		// comes from the bind directive, so we append the port
-		addr, err := caddy.ParseNetworkAddress(host + ":" + lnPort)
+	for _, lnHost := range lnHosts {
+		// normally we would simply append the port,
+		// but if lnHost is IPv6, we need to ensure it
+		// is enclosed in [ ]; net.JoinHostPort does
+		// this for us, but lnHost might also have a
+		// network type in front (e.g. "tcp/") leading
+		// to "[tcp/::1]" which causes parsing failures
+		// later; what we need is "tcp/[::1]", so we have
+		// to split the network and host, then re-combine
+		network, host, ok := strings.Cut(lnHost, "/")
+		if !ok {
+			host = network
+			network = ""
+		}
+		host = strings.Trim(host, "[]") // IPv6
+		networkAddr := caddy.JoinNetworkAddress(network, host, lnPort)
+		addr, err := caddy.ParseNetworkAddress(networkAddr)
 		if err != nil {
 			return nil, fmt.Errorf("parsing network address: %v", err)
 		}

caddyconfig/httpcaddyfile/builtins.go

@@ -24,6 +24,7 @@ import (
 	"reflect"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
@@ -48,12 +49,12 @@ func init() {
 	RegisterHandlerDirective("handle", parseHandle)
 	RegisterDirective("handle_errors", parseHandleErrors)
 	RegisterDirective("log", parseLog)
+	RegisterHandlerDirective("skip_log", parseSkipLog)
 }
 
 // parseBind parses the bind directive. Syntax:
 //
-//     bind <addresses...>
-//
+//	bind <addresses...>
 func parseBind(h Helper) ([]ConfigValue, error) {
 	var lnHosts []string
 	for h.Next() {
@@ -64,28 +65,34 @@ func parseBind(h Helper) ([]ConfigValue, error) {
 
 // parseTLS parses the tls directive. Syntax:
 //
-//     tls [<email>|internal]|[<cert_file> <key_file>] {
-//         protocols <min> [<max>]
-//         ciphers   <cipher_suites...>
-//         curves    <curves...>
-//         client_auth {
-//             mode                   [request|require|verify_if_given|require_and_verify]
-//             trusted_ca_cert        <base64_der>
-//             trusted_ca_cert_file   <filename>
-//             trusted_leaf_cert      <base64_der>
-//             trusted_leaf_cert_file <filename>
-//         }
-//         alpn      <values...>
-//         load      <paths...>
-//         ca        <acme_ca_endpoint>
-//         ca_root   <pem_file>
-//         dns       <provider_name> [...]
-//         on_demand
-//         eab    <key_id> <mac_key>
-//         issuer <module_name> [...]
-//         get_certificate <module_name> [...]
-//     }
-//
+//	tls [<email>|internal]|[<cert_file> <key_file>] {
+//	    protocols <min> [<max>]
+//	    ciphers   <cipher_suites...>
+//	    curves    <curves...>
+//	    client_auth {
+//	        mode                   [request|require|verify_if_given|require_and_verify]
+//	        trusted_ca_cert        <base64_der>
+//	        trusted_ca_cert_file   <filename>
+//	        trusted_leaf_cert      <base64_der>
+//	        trusted_leaf_cert_file <filename>
+//	    }
+//	    alpn                          <values...>
+//	    load                          <paths...>
+//	    ca                            <acme_ca_endpoint>
+//	    ca_root                       <pem_file>
+//	    key_type                      [ed25519|p256|p384|rsa2048|rsa4096]
+//	    dns                           <provider_name> [...]
+//	    propagation_delay             <duration>
+//	    propagation_timeout           <duration>
+//	    resolvers                     <dns_servers...>
+//	    dns_ttl                       <duration>
+//	    dns_challenge_override_domain <domain>
+//	    on_demand
+//	    eab                           <key_id> <mac_key>
+//	    issuer                        <module_name> [...]
+//	    get_certificate               <module_name> [...]
+//	    insecure_secrets_log          <log_file>
+//	}
 func parseTLS(h Helper) ([]ConfigValue, error) {
 	cp := new(caddytls.ConnectionPolicy)
 	var fileLoader caddytls.FileLoader
@@ -363,6 +370,75 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 				}
 				acmeIssuer.Challenges.DNS.Resolvers = args
 
+			case "propagation_delay":
+				arg := h.RemainingArgs()
+				if len(arg) != 1 {
+					return nil, h.ArgErr()
+				}
+				delayStr := arg[0]
+				delay, err := caddy.ParseDuration(delayStr)
+				if err != nil {
+					return nil, h.Errf("invalid propagation_delay duration %s: %v", delayStr, err)
+				}
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
+				if acmeIssuer.Challenges == nil {
+					acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
+				}
+				if acmeIssuer.Challenges.DNS == nil {
+					acmeIssuer.Challenges.DNS = new(caddytls.DNSChallengeConfig)
+				}
+				acmeIssuer.Challenges.DNS.PropagationDelay = caddy.Duration(delay)
+
+			case "propagation_timeout":
+				arg := h.RemainingArgs()
+				if len(arg) != 1 {
+					return nil, h.ArgErr()
+				}
+				timeoutStr := arg[0]
+				var timeout time.Duration
+				if timeoutStr == "-1" {
+					timeout = time.Duration(-1)
+				} else {
+					var err error
+					timeout, err = caddy.ParseDuration(timeoutStr)
+					if err != nil {
+						return nil, h.Errf("invalid propagation_timeout duration %s: %v", timeoutStr, err)
+					}
+				}
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
+				if acmeIssuer.Challenges == nil {
+					acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
+				}
+				if acmeIssuer.Challenges.DNS == nil {
+					acmeIssuer.Challenges.DNS = new(caddytls.DNSChallengeConfig)
+				}
+				acmeIssuer.Challenges.DNS.PropagationTimeout = caddy.Duration(timeout)
+
+			case "dns_ttl":
+				arg := h.RemainingArgs()
+				if len(arg) != 1 {
+					return nil, h.ArgErr()
+				}
+				ttlStr := arg[0]
+				ttl, err := caddy.ParseDuration(ttlStr)
+				if err != nil {
+					return nil, h.Errf("invalid dns_ttl duration %s: %v", ttlStr, err)
+				}
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
+				if acmeIssuer.Challenges == nil {
+					acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
+				}
+				if acmeIssuer.Challenges.DNS == nil {
+					acmeIssuer.Challenges.DNS = new(caddytls.DNSChallengeConfig)
+				}
+				acmeIssuer.Challenges.DNS.TTL = caddy.Duration(ttl)
+
 			case "dns_challenge_override_domain":
 				arg := h.RemainingArgs()
 				if len(arg) != 1 {
@@ -395,6 +471,12 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 				}
 				onDemand = true
 
+			case "insecure_secrets_log":
+				if !h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				cp.InsecureSecretsLog = h.Val()
+
 			default:
 				return nil, h.Errf("unknown subdirective: %s", h.Val())
 			}
@@ -515,8 +597,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 
 // parseRoot parses the root directive. Syntax:
 //
-//     root [<matcher>] <path>
-//
+//	root [<matcher>] <path>
 func parseRoot(h Helper) (caddyhttp.MiddlewareHandler, error) {
 	var root string
 	for h.Next() {
@@ -650,29 +731,20 @@ func parseError(h Helper) (caddyhttp.MiddlewareHandler, error) {
 
 // parseRoute parses the route directive.
 func parseRoute(h Helper) (caddyhttp.MiddlewareHandler, error) {
-	sr := new(caddyhttp.Subroute)
-
 	allResults, err := parseSegmentAsConfig(h)
 	if err != nil {
 		return nil, err
 	}
 
 	for _, result := range allResults {
-		switch handler := result.Value.(type) {
-		case caddyhttp.Route:
-			sr.Routes = append(sr.Routes, handler)
-		case caddyhttp.Subroute:
-			// directives which return a literal subroute instead of a route
-			// means they intend to keep those handlers together without
-			// them being reordered; we're doing that anyway since we're in
-			// the route directive, so just append its handlers
-			sr.Routes = append(sr.Routes, handler.Routes...)
+		switch result.Value.(type) {
+		case caddyhttp.Route, caddyhttp.Subroute:
 		default:
 			return nil, h.Errf("%s directive returned something other than an HTTP route or subroute: %#v (only handler directives can be used in routes)", result.directive, result.Value)
 		}
 	}
 
-	return sr, nil
+	return buildSubroute(allResults, h.groupCounter, false)
 }
 
 func parseHandle(h Helper) (caddyhttp.MiddlewareHandler, error) {
@@ -694,12 +766,11 @@ func parseHandleErrors(h Helper) ([]ConfigValue, error) {
 
 // parseLog parses the log directive. Syntax:
 //
-//     log {
-//         output <writer_module> ...
-//         format <encoder_module> ...
-//         level  <level>
-//     }
-//
+//	log {
+//	    output <writer_module> ...
+//	    format <encoder_module> ...
+//	    level  <level>
+//	}
 func parseLog(h Helper) ([]ConfigValue, error) {
 	return parseLogHelper(h, nil)
 }
@@ -731,7 +802,7 @@ func parseLogHelper(h Helper, globalLogNames map[string]struct{}) ([]ConfigValue
 				// reference the default logger. See the
 				// setupNewDefault function in the logging
 				// package for where this is configured.
-				globalLogName = "default"
+				globalLogName = caddy.DefaultLoggerName
 			}
 
 			// Verify this name is unused.
@@ -858,3 +929,15 @@ func parseLogHelper(h Helper, globalLogNames map[string]struct{}) ([]ConfigValue
 	}
 	return configValues, nil
 }
+
+// parseSkipLog parses the skip_log directive. Syntax:
+//
+//	skip_log [<matcher>]
+func parseSkipLog(h Helper) (caddyhttp.MiddlewareHandler, error) {
+	for h.Next() {
+		if h.NextArg() {
+			return nil, h.ArgErr()
+		}
+	}
+	return caddyhttp.VarsMiddleware{"skip_log": true}, nil
+}

caddyconfig/httpcaddyfile/directives.go

@@ -42,6 +42,7 @@ var directiveOrder = []string{
 	"map",
 	"vars",
 	"root",
+	"skip_log",
 
 	"header",
 	"copy_response_headers", // only in reverse_proxy's handle_response
@@ -288,7 +289,7 @@ func ParseSegmentAsSubroute(h Helper) (caddyhttp.MiddlewareHandler, error) {
 		return nil, err
 	}
 
-	return buildSubroute(allResults, h.groupCounter)
+	return buildSubroute(allResults, h.groupCounter, true)
 }
 
 // parseSegmentAsConfig parses the segment such that its subdirectives
@@ -406,7 +407,7 @@ func sortRoutes(routes []ConfigValue) {
 			return false
 		}
 
-		// decode the path matchers, if there is just one of them
+		// decode the path matchers if there is just one matcher set
 		var iPM, jPM caddyhttp.MatchPath
 		if len(iRoute.MatcherSetsRaw) == 1 {
 			_ = json.Unmarshal(iRoute.MatcherSetsRaw[0]["path"], &iPM)
@@ -415,38 +416,45 @@ func sortRoutes(routes []ConfigValue) {
 			_ = json.Unmarshal(jRoute.MatcherSetsRaw[0]["path"], &jPM)
 		}
 
-		// sort by longer path (more specific) first; missing path
-		// matchers or multi-matchers are treated as zero-length paths
+		// if there is only one path in the path matcher, sort by longer path
+		// (more specific) first; missing path matchers or multi-matchers are
+		// treated as zero-length paths
 		var iPathLen, jPathLen int
-		if len(iPM) > 0 {
+		if len(iPM) == 1 {
 			iPathLen = len(iPM[0])
 		}
-		if len(jPM) > 0 {
+		if len(jPM) == 1 {
 			jPathLen = len(jPM[0])
 		}
 
 		// some directives involve setting values which can overwrite
-		// eachother, so it makes most sense to reverse the order so
+		// each other, so it makes most sense to reverse the order so
 		// that the lease specific matcher is first; everything else
 		// has most-specific matcher first
 		if iDir == "vars" {
-			// if both directives have no path matcher, use whichever one
-			// has no matcher first.
-			if iPathLen == 0 && jPathLen == 0 {
-				return len(iRoute.MatcherSetsRaw) == 0 && len(jRoute.MatcherSetsRaw) > 0
+			// we can only confidently compare path lengths if both
+			// directives have a single path to match (issue #5037)
+			if iPathLen > 0 && jPathLen > 0 {
+				// sort least-specific (shortest) path first
+				return iPathLen < jPathLen
 			}
 
-			// sort with the least-specific (shortest) path first
-			return iPathLen < jPathLen
+			// if both directives don't have a single path to compare,
+			// sort whichever one has no matcher first; if both have
+			// no matcher, sort equally (stable sort preserves order)
+			return len(iRoute.MatcherSetsRaw) == 0 && len(jRoute.MatcherSetsRaw) > 0
 		} else {
-			// if both directives have no path matcher, use whichever one
-			// has any kind of matcher defined first.
-			if iPathLen == 0 && jPathLen == 0 {
-				return len(iRoute.MatcherSetsRaw) > 0 && len(jRoute.MatcherSetsRaw) == 0
+			// we can only confidently compare path lengths if both
+			// directives have a single path to match (issue #5037)
+			if iPathLen > 0 && jPathLen > 0 {
+				// sort most-specific (longest) path first
+				return iPathLen > jPathLen
 			}
 
-			// sort with the most-specific (longest) path first
-			return iPathLen > jPathLen
+			// if both directives don't have a single path to compare,
+			// sort whichever one has a matcher first; if both have
+			// a matcher, sort equally (stable sort preserves order)
+			return len(iRoute.MatcherSetsRaw) > 0 && len(jRoute.MatcherSetsRaw) == 0
 		}
 	})
 }

caddyconfig/httpcaddyfile/httptype.go

@@ -95,11 +95,13 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		{regexp.MustCompile(`{cookie\.([\w-]*)}`), "{http.request.cookie.$1}"},
 		{regexp.MustCompile(`{labels\.([\w-]*)}`), "{http.request.host.labels.$1}"},
 		{regexp.MustCompile(`{path\.([\w-]*)}`), "{http.request.uri.path.$1}"},
+		{regexp.MustCompile(`{file\.([\w-]*)}`), "{http.request.uri.path.file.$1}"},
 		{regexp.MustCompile(`{query\.([\w-]*)}`), "{http.request.uri.query.$1}"},
 		{regexp.MustCompile(`{re\.([\w-]*)\.([\w-]*)}`), "{http.regexp.$1.$2}"},
 		{regexp.MustCompile(`{vars\.([\w-]*)}`), "{http.vars.$1}"},
 		{regexp.MustCompile(`{rp\.([\w-\.]*)}`), "{http.reverse_proxy.$1}"},
 		{regexp.MustCompile(`{err\.([\w-\.]*)}`), "{http.error.$1}"},
+		{regexp.MustCompile(`{file_match\.([\w-]*)}`), "{http.matchers.file.$1}"},
 	}
 
 	for _, sb := range originalServerBlocks {
@@ -217,11 +219,11 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		if ncl.name == "" {
 			return
 		}
-		if ncl.name == "default" {
+		if ncl.name == caddy.DefaultLoggerName {
 			hasDefaultLog = true
 		}
 		if _, ok := options["debug"]; ok && ncl.log.Level == "" {
-			ncl.log.Level = "DEBUG"
+			ncl.log.Level = zap.DebugLevel.CapitalString()
 		}
 		customLogs = append(customLogs, ncl)
 	}
@@ -238,8 +240,8 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		// configure it with any applicable options
 		if _, ok := options["debug"]; ok {
 			customLogs = append(customLogs, namedCustomLog{
-				name: "default",
-				log:  &caddy.CustomLog{Level: "DEBUG"},
+				name: caddy.DefaultLoggerName,
+				log:  &caddy.CustomLog{Level: zap.DebugLevel.CapitalString()},
 			})
 		}
 	}
@@ -284,6 +286,17 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 	if adminConfig, ok := options["admin"].(*caddy.AdminConfig); ok && adminConfig != nil {
 		cfg.Admin = adminConfig
 	}
+
+	if pc, ok := options["persist_config"].(string); ok && pc == "off" {
+		if cfg.Admin == nil {
+			cfg.Admin = new(caddy.AdminConfig)
+		}
+		if cfg.Admin.Config == nil {
+			cfg.Admin.Config = new(caddy.ConfigSettings)
+		}
+		cfg.Admin.Config.Persist = new(bool)
+	}
+
 	if len(customLogs) > 0 {
 		if cfg.Logging == nil {
 			cfg.Logging = &caddy.Logging{
@@ -297,11 +310,11 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 			// most users seem to prefer not writing access logs
 			// to the default log when they are directed to a
 			// file or have any other special customization
-			if ncl.name != "default" && len(ncl.log.Include) > 0 {
-				defaultLog, ok := cfg.Logging.Logs["default"]
+			if ncl.name != caddy.DefaultLoggerName && len(ncl.log.Include) > 0 {
+				defaultLog, ok := cfg.Logging.Logs[caddy.DefaultLoggerName]
 				if !ok {
 					defaultLog = new(caddy.CustomLog)
-					cfg.Logging.Logs["default"] = defaultLog
+					cfg.Logging.Logs[caddy.DefaultLoggerName] = defaultLog
 				}
 				defaultLog.Exclude = append(defaultLog.Exclude, ncl.log.Include...)
 			}
@@ -516,15 +529,6 @@ func (st *ServerType) serversFromPairings(
 		var hasCatchAllTLSConnPolicy, addressQualifiesForTLS bool
 		autoHTTPSWillAddConnPolicy := autoHTTPS != "off"
 
-		// if a catch-all server block (one which accepts all hostnames) exists in this pairing,
-		// we need to know that so that we can configure logs properly (see #3878)
-		var catchAllSblockExists bool
-		for _, sblock := range p.serverBlocks {
-			if len(sblock.hostsFromKeys(false)) == 0 {
-				catchAllSblockExists = true
-			}
-		}
-
 		// if needed, the ServerLogConfig is initialized beforehand so
 		// that all server blocks can populate it with data, even when not
 		// coming with a log directive
@@ -625,7 +629,7 @@ func (st *ServerType) serversFromPairings(
 
 			// set up each handler directive, making sure to honor directive order
 			dirRoutes := sblock.pile["route"]
-			siteSubroute, err := buildSubroute(dirRoutes, groupCounter)
+			siteSubroute, err := buildSubroute(dirRoutes, groupCounter, true)
 			if err != nil {
 				return nil, err
 			}
@@ -656,18 +660,10 @@ func (st *ServerType) serversFromPairings(
 				} else {
 					// map each host to the user's desired logger name
 					for _, h := range sblockLogHosts {
-						// if the custom logger name is non-empty, add it to the map;
-						// otherwise, only map to an empty logger name if this or
-						// another site block on this server has a catch-all host (in
-						// which case only requests with mapped hostnames will be
-						// access-logged, so it'll be necessary to add them to the
-						// map even if they use default logger)
-						if ncl.name != "" || catchAllSblockExists {
-							if srv.Logs.LoggerNames == nil {
-								srv.Logs.LoggerNames = make(map[string]string)
-							}
-							srv.Logs.LoggerNames[h] = ncl.name
+						if srv.Logs.LoggerNames == nil {
+							srv.Logs.LoggerNames = make(map[string]string)
 						}
+						srv.Logs.LoggerNames[h] = ncl.name
 					}
 				}
 			}
@@ -721,7 +717,7 @@ func (st *ServerType) serversFromPairings(
 
 	err := applyServerOptions(servers, options, warnings)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("applying global server options: %v", err)
 	}
 
 	return servers, nil
@@ -922,11 +918,32 @@ func appendSubrouteToRouteList(routeList caddyhttp.RouteList,
 		return routeList
 	}
 
+	// No need to wrap the handlers in a subroute if this is the only server block
+	// and there is no matcher for it (doing so would produce unnecessarily nested
+	// JSON), *unless* there is a host matcher within this site block; if so, then
+	// we still need to wrap in a subroute because otherwise the host matcher from
+	// the inside of the site block would be a top-level host matcher, which is
+	// subject to auto-HTTPS (cert management), and using a host matcher within
+	// a site block is a valid, common pattern for excluding domains from cert
+	// management, leading to unexpected behavior; see issue #5124.
+	wrapInSubroute := true
 	if len(matcherSetsEnc) == 0 && len(p.serverBlocks) == 1 {
-		// no need to wrap the handlers in a subroute if this is
-		// the only server block and there is no matcher for it
-		routeList = append(routeList, subroute.Routes...)
-	} else {
+		var hasHostMatcher bool
+	outer:
+		for _, route := range subroute.Routes {
+			for _, ms := range route.MatcherSetsRaw {
+				for matcherName := range ms {
+					if matcherName == "host" {
+						hasHostMatcher = true
+						break outer
+					}
+				}
+			}
+		}
+		wrapInSubroute = hasHostMatcher
+	}
+
+	if wrapInSubroute {
 		route := caddyhttp.Route{
 			// the semantics of a site block in the Caddyfile dictate
 			// that only the first matching one is evaluated, since
@@ -944,20 +961,25 @@ func appendSubrouteToRouteList(routeList caddyhttp.RouteList,
 		if len(route.MatcherSetsRaw) > 0 || len(route.HandlersRaw) > 0 {
 			routeList = append(routeList, route)
 		}
+	} else {
+		routeList = append(routeList, subroute.Routes...)
 	}
+
 	return routeList
 }
 
 // buildSubroute turns the config values, which are expected to be routes
 // into a clean and orderly subroute that has all the routes within it.
-func buildSubroute(routes []ConfigValue, groupCounter counter) (*caddyhttp.Subroute, error) {
-	for _, val := range routes {
-		if !directiveIsOrdered(val.directive) {
-			return nil, fmt.Errorf("directive '%s' is not ordered, so it cannot be used here", val.directive)
+func buildSubroute(routes []ConfigValue, groupCounter counter, needsSorting bool) (*caddyhttp.Subroute, error) {
+	if needsSorting {
+		for _, val := range routes {
+			if !directiveIsOrdered(val.directive) {
+				return nil, fmt.Errorf("directive '%s' is not an ordered HTTP handler, so it cannot be used here", val.directive)
+			}
 		}
-	}
 
-	sortRoutes(routes)
+		sortRoutes(routes)
+	}
 
 	subroute := new(caddyhttp.Subroute)
 

caddyconfig/httpcaddyfile/options.go

@@ -54,6 +54,7 @@ func init() {
 	RegisterGlobalOption("ocsp_stapling", parseOCSPStaplingOptions)
 	RegisterGlobalOption("log", parseLogOptions)
 	RegisterGlobalOption("preferred_chains", parseOptPreferredChains)
+	RegisterGlobalOption("persist_config", parseOptPersistConfig)
 }
 
 func parseOptTrue(d *caddyfile.Dispenser, _ any) (any, error) { return true, nil }
@@ -386,6 +387,21 @@ func parseOptOnDemand(d *caddyfile.Dispenser, _ any) (any, error) {
 	return ond, nil
 }
 
+func parseOptPersistConfig(d *caddyfile.Dispenser, _ any) (any, error) {
+	d.Next() // consume parameter name
+	if !d.Next() {
+		return "", d.ArgErr()
+	}
+	val := d.Val()
+	if d.Next() {
+		return "", d.ArgErr()
+	}
+	if val != "off" {
+		return "", d.Errf("persist_config must be 'off'")
+	}
+	return val, nil
+}
+
 func parseOptAutoHTTPS(d *caddyfile.Dispenser, _ any) (any, error) {
 	d.Next() // consume parameter name
 	if !d.Next() {
@@ -421,13 +437,13 @@ func parseOCSPStaplingOptions(d *caddyfile.Dispenser, _ any) (any, error) {
 
 // parseLogOptions parses the global log option. Syntax:
 //
-//     log [name] {
-//         output  <writer_module> ...
-//         format  <encoder_module> ...
-//         level   <level>
-//         include <namespaces...>
-//         exclude <namespaces...>
-//     }
+//	log [name] {
+//	    output  <writer_module> ...
+//	    format  <encoder_module> ...
+//	    level   <level>
+//	    include <namespaces...>
+//	    exclude <namespaces...>
+//	}
 //
 // When the name argument is unspecified, this directive modifies the default
 // logger.

caddyconfig/httpcaddyfile/pkiapp.go

@@ -15,6 +15,7 @@
 package httpcaddyfile
 
 import (
+	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddypki"
@@ -26,23 +27,24 @@ func init() {
 
 // parsePKIApp parses the global log option. Syntax:
 //
-//     pki {
-//         ca [<id>] {
-//             name            <name>
-//             root_cn         <name>
-//             intermediate_cn <name>
-//             root {
-//                 cert   <path>
-//                 key    <path>
-//                 format <format>
-//             }
-//             intermediate {
-//                 cert   <path>
-//                 key    <path>
-//                 format <format>
-//             }
-//         }
-//     }
+//	pki {
+//	    ca [<id>] {
+//	        name                  <name>
+//	        root_cn               <name>
+//	        intermediate_cn       <name>
+//	        intermediate_lifetime <duration>
+//	        root {
+//	            cert   <path>
+//	            key    <path>
+//	            format <format>
+//	        }
+//	        intermediate {
+//	            cert   <path>
+//	            key    <path>
+//	            format <format>
+//	        }
+//	    }
+//	}
 //
 // When the CA ID is unspecified, 'local' is assumed.
 func parsePKIApp(d *caddyfile.Dispenser, existingVal any) (any, error) {
@@ -83,6 +85,16 @@ func parsePKIApp(d *caddyfile.Dispenser, existingVal any) (any, error) {
 						}
 						pkiCa.IntermediateCommonName = d.Val()
 
+					case "intermediate_lifetime":
+						if !d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						dur, err := caddy.ParseDuration(d.Val())
+						if err != nil {
+							return nil, err
+						}
+						pkiCa.IntermediateLifetime = caddy.Duration(dur)
+
 					case "root":
 						if pkiCa.Root == nil {
 							pkiCa.Root = new(caddypki.KeyPair)

caddyconfig/httpcaddyfile/serveroptions.go

@@ -33,6 +33,7 @@ type serverOptions struct {
 	ListenerAddress string
 
 	// These will all map 1:1 to the caddyhttp.Server struct
+	Name                 string
 	ListenerWrappersRaw  []json.RawMessage
 	ReadTimeout          caddy.Duration
 	ReadHeaderTimeout    caddy.Duration
@@ -42,7 +43,9 @@ type serverOptions struct {
 	MaxHeaderBytes       int
 	Protocols            []string
 	StrictSNIHost        *bool
+	TrustedProxiesRaw    json.RawMessage
 	ShouldLogCredentials bool
+	Metrics              *caddyhttp.Metrics
 }
 
 func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (any, error) {
@@ -56,6 +59,15 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (any, error) {
 		}
 		for nesting := d.Nesting(); d.NextBlock(nesting); {
 			switch d.Val() {
+			case "name":
+				if serverOpts.ListenerAddress == "" {
+					return nil, d.Errf("cannot set a name for a server without a listener address")
+				}
+				if !d.NextArg() {
+					return nil, d.ArgErr()
+				}
+				serverOpts.Name = d.Val()
+
 			case "listener_wrappers":
 				for nesting := d.Nesting(); d.NextBlock(nesting); {
 					modID := "caddy.listeners." + d.Val()
@@ -161,7 +173,7 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (any, error) {
 					}
 					serverOpts.Protocols = append(serverOpts.Protocols, proto)
 				}
-				if d.NextBlock(0) {
+				if nesting := d.Nesting(); d.NextBlock(nesting) {
 					return nil, d.ArgErr()
 				}
 
@@ -175,6 +187,36 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (any, error) {
 				}
 				serverOpts.StrictSNIHost = &boolVal
 
+			case "trusted_proxies":
+				if !d.NextArg() {
+					return nil, d.Err("trusted_proxies expects an IP range source module name as its first argument")
+				}
+				modID := "http.ip_sources." + d.Val()
+				unm, err := caddyfile.UnmarshalModule(d, modID)
+				if err != nil {
+					return nil, err
+				}
+				source, ok := unm.(caddyhttp.IPRangeSource)
+				if !ok {
+					return nil, fmt.Errorf("module %s (%T) is not an IP range source", modID, unm)
+				}
+				jsonSource := caddyconfig.JSONModuleObject(
+					source,
+					"source",
+					source.(caddy.Module).CaddyModule().ID.Name(),
+					nil,
+				)
+				serverOpts.TrustedProxiesRaw = jsonSource
+
+			case "metrics":
+				if d.NextArg() {
+					return nil, d.ArgErr()
+				}
+				if nesting := d.Nesting(); d.NextBlock(nesting) {
+					return nil, d.ArgErr()
+				}
+				serverOpts.Metrics = new(caddyhttp.Metrics)
+
 			// TODO: DEPRECATED. (August 2022)
 			case "protocol":
 				caddy.Log().Named("caddyfile").Warn("DEPRECATED: protocol sub-option will be removed soon")
@@ -228,7 +270,22 @@ func applyServerOptions(
 		return nil
 	}
 
-	for _, server := range servers {
+	// check for duplicate names, which would clobber the config
+	existingNames := map[string]bool{}
+	for _, opts := range serverOpts {
+		if opts.Name == "" {
+			continue
+		}
+		if existingNames[opts.Name] {
+			return fmt.Errorf("cannot use duplicate server name '%s'", opts.Name)
+		}
+		existingNames[opts.Name] = true
+	}
+
+	// collect the server name overrides
+	nameReplacements := map[string]string{}
+
+	for key, server := range servers {
 		// find the options that apply to this server
 		opts := func() *serverOptions {
 			for _, entry := range serverOpts {
@@ -259,12 +316,24 @@ func applyServerOptions(
 		server.MaxHeaderBytes = opts.MaxHeaderBytes
 		server.Protocols = opts.Protocols
 		server.StrictSNIHost = opts.StrictSNIHost
+		server.TrustedProxiesRaw = opts.TrustedProxiesRaw
+		server.Metrics = opts.Metrics
 		if opts.ShouldLogCredentials {
 			if server.Logs == nil {
 				server.Logs = &caddyhttp.ServerLogConfig{}
 			}
 			server.Logs.ShouldLogCredentials = opts.ShouldLogCredentials
 		}
+
+		if opts.Name != "" {
+			nameReplacements[key] = opts.Name
+		}
+	}
+
+	// rename the servers if marked to do so
+	for old, new := range nameReplacements {
+		servers[new] = servers[old]
+		delete(servers, old)
 	}
 
 	return nil

caddyconfig/httpcaddyfile/tlsapp.go

@@ -44,37 +44,32 @@ func (st ServerType) buildTLSApp(
 	if hp, ok := options["http_port"].(int); ok {
 		httpPort = strconv.Itoa(hp)
 	}
-	httpsPort := strconv.Itoa(caddyhttp.DefaultHTTPSPort)
-	if hsp, ok := options["https_port"].(int); ok {
-		httpsPort = strconv.Itoa(hsp)
+	autoHTTPS := "on"
+	if ah, ok := options["auto_https"].(string); ok {
+		autoHTTPS = ah
 	}
 
-	// count how many server blocks have a TLS-enabled key with
-	// no host, and find all hosts that share a server block with
-	// a hostless key, so that they don't get forgotten/omitted
-	// by auto-HTTPS (since they won't appear in route matchers)
-	var serverBlocksWithTLSHostlessKey int
+	// find all hosts that share a server block with a hostless
+	// key, so that they don't get forgotten/omitted by auto-HTTPS
+	// (since they won't appear in route matchers)
 	httpsHostsSharedWithHostlessKey := make(map[string]struct{})
-	for _, pair := range pairings {
-		for _, sb := range pair.serverBlocks {
-			for _, addr := range sb.keys {
-				if addr.Host == "" {
-					// this address has no hostname, but if it's explicitly set
-					// to HTTPS, then we need to count it as being TLS-enabled
-					if addr.Scheme == "https" || addr.Port == httpsPort {
-						serverBlocksWithTLSHostlessKey++
-					}
-					// this server block has a hostless key, now
-					// go through and add all the hosts to the set
-					for _, otherAddr := range sb.keys {
-						if otherAddr.Original == addr.Original {
-							continue
-						}
-						if otherAddr.Host != "" && otherAddr.Scheme != "http" && otherAddr.Port != httpPort {
-							httpsHostsSharedWithHostlessKey[otherAddr.Host] = struct{}{}
+	if autoHTTPS != "off" {
+		for _, pair := range pairings {
+			for _, sb := range pair.serverBlocks {
+				for _, addr := range sb.keys {
+					if addr.Host == "" {
+						// this server block has a hostless key, now
+						// go through and add all the hosts to the set
+						for _, otherAddr := range sb.keys {
+							if otherAddr.Original == addr.Original {
+								continue
+							}
+							if otherAddr.Host != "" && otherAddr.Scheme != "http" && otherAddr.Port != httpPort {
+								httpsHostsSharedWithHostlessKey[otherAddr.Host] = struct{}{}
+							}
 						}
+						break
 					}
-					break
 				}
 			}
 		}
@@ -134,6 +129,19 @@ func (st ServerType) buildTLSApp(
 					issuers = append(issuers, issuerVal.Value.(certmagic.Issuer))
 				}
 				if ap == catchAllAP && !reflect.DeepEqual(ap.Issuers, issuers) {
+					// this more correctly implements an error check that was removed
+					// below; try it with this config:
+					//
+					// :443 {
+					// 	bind 127.0.0.1
+					// }
+					//
+					// :443 {
+					// 	bind ::1
+					// 	tls {
+					// 		issuer acme
+					// 	}
+					// }
 					return nil, warnings, fmt.Errorf("automation policy from site block is also default/catch-all policy because of key without hostname, and the two are in conflict: %#v != %#v", ap.Issuers, issuers)
 				}
 				ap.Issuers = issuers
@@ -176,29 +184,25 @@ func (st ServerType) buildTLSApp(
 				}
 			}
 
-			// first make sure this block is allowed to create an automation policy;
-			// doing so is forbidden if it has a key with no host (i.e. ":443")
+			// we used to ensure this block is allowed to create an automation policy;
+			// doing so was forbidden if it has a key with no host (i.e. ":443")
 			// and if there is a different server block that also has a key with no
 			// host -- since a key with no host matches any host, we need its
 			// associated automation policy to have an empty Subjects list, i.e. no
 			// host filter, which is indistinguishable between the two server blocks
 			// because automation is not done in the context of a particular server...
 			// this is an example of a poor mapping from Caddyfile to JSON but that's
-			// the least-leaky abstraction I could figure out
-			if len(sblockHosts) == 0 {
-				if serverBlocksWithTLSHostlessKey > 1 {
-					// this server block and at least one other has a key with no host,
-					// making the two indistinguishable; it is misleading to define such
-					// a policy within one server block since it actually will apply to
-					// others as well
-					return nil, warnings, fmt.Errorf("cannot make a TLS automation policy from a server block that has a host-less address when there are other TLS-enabled server block addresses lacking a host")
-				}
-				if catchAllAP == nil {
-					// this server block has a key with no hosts, but there is not yet
-					// a catch-all automation policy (probably because no global options
-					// were set), so this one becomes it
-					catchAllAP = ap
-				}
+			// the least-leaky abstraction I could figure out -- however, this check
+			// was preventing certain listeners, like those provided by plugins, from
+			// being used as desired (see the Tailscale listener plugin), so I removed
+			// the check: and I think since I originally wrote the check I added a new
+			// check above which *properly* detects this ambiguity without breaking the
+			// listener plugin; see the check above with a commented example config
+			if len(sblockHosts) == 0 && catchAllAP == nil {
+				// this server block has a key with no hosts, but there is not yet
+				// a catch-all automation policy (probably because no global options
+				// were set), so this one becomes it
+				catchAllAP = ap
 			}
 
 			// associate our new automation policy with this server block's hosts
@@ -331,10 +335,12 @@ func (st ServerType) buildTLSApp(
 	internalAP := &caddytls.AutomationPolicy{
 		IssuersRaw: []json.RawMessage{json.RawMessage(`{"module":"internal"}`)},
 	}
-	for h := range httpsHostsSharedWithHostlessKey {
-		al = append(al, h)
-		if !certmagic.SubjectQualifiesForPublicCert(h) {
-			internalAP.Subjects = append(internalAP.Subjects, h)
+	if autoHTTPS != "off" {
+		for h := range httpsHostsSharedWithHostlessKey {
+			al = append(al, h)
+			if !certmagic.SubjectQualifiesForPublicCert(h) {
+				internalAP.Subjects = append(internalAP.Subjects, h)
+			}
 		}
 	}
 	if len(al) > 0 {

caddyconfig/httploader.go

@@ -94,7 +94,7 @@ func (hl HTTPLoader) LoadConfig(ctx caddy.Context) ([]byte, error) {
 		}
 	}
 
-	resp, err := client.Do(req)
+	resp, err := doHttpCallWithRetries(ctx, client, req)
 	if err != nil {
 		return nil, err
 	}
@@ -113,12 +113,44 @@ func (hl HTTPLoader) LoadConfig(ctx caddy.Context) ([]byte, error) {
 		return nil, err
 	}
 	for _, warn := range warnings {
-		ctx.Logger(hl).Warn(warn.String())
+		ctx.Logger().Warn(warn.String())
 	}
 
 	return result, nil
 }
 
+func attemptHttpCall(client *http.Client, request *http.Request) (*http.Response, error) {
+	resp, err := client.Do(request)
+	if err != nil {
+		return nil, fmt.Errorf("problem calling http loader url: %v", err)
+	} else if resp.StatusCode < 200 || resp.StatusCode > 499 {
+		resp.Body.Close()
+		return nil, fmt.Errorf("bad response status code from http loader url: %v", resp.StatusCode)
+	}
+	return resp, nil
+}
+
+func doHttpCallWithRetries(ctx caddy.Context, client *http.Client, request *http.Request) (*http.Response, error) {
+	var resp *http.Response
+	var err error
+	const maxAttempts = 10
+
+	for i := 0; i < maxAttempts; i++ {
+		resp, err = attemptHttpCall(client, request)
+		if err != nil && i < maxAttempts-1 {
+			select {
+			case <-time.After(time.Millisecond * 500):
+			case <-ctx.Done():
+				return resp, ctx.Err()
+			}
+		} else {
+			break
+		}
+	}
+
+	return resp, err
+}
+
 func (hl HTTPLoader) makeClient(ctx caddy.Context) (*http.Client, error) {
 	client := &http.Client{
 		Timeout: time.Duration(hl.Timeout),
@@ -129,7 +161,7 @@ func (hl HTTPLoader) makeClient(ctx caddy.Context) (*http.Client, error) {
 
 		// client authentication
 		if hl.TLS.UseServerIdentity {
-			certs, err := ctx.IdentityCredentials(ctx.Logger(hl))
+			certs, err := ctx.IdentityCredentials(ctx.Logger())
 			if err != nil {
 				return nil, fmt.Errorf("getting server identity credentials: %v", err)
 			}

caddytest/caddytest.go

@@ -43,7 +43,7 @@ type Defaults struct {
 
 // Default testing values
 var Default = Defaults{
-	AdminPort:          2019,
+	AdminPort:          2999, // different from what a real server also running on a developer's machine might be
 	Certifcates:        []string{"/caddy.localhost.crt", "/caddy.localhost.key"},
 	TestRequestTimeout: 5 * time.Second,
 	LoadRequestTimeout: 5 * time.Second,
@@ -100,7 +100,7 @@ func (tc *Tester) InitServer(rawConfig string, configType string) {
 		tc.t.Fail()
 	}
 	if err := tc.ensureConfigRunning(rawConfig, configType); err != nil {
-		tc.t.Logf("failed ensurng config is running: %s", err)
+		tc.t.Logf("failed ensuring config is running: %s", err)
 		tc.t.Fail()
 	}
 }
@@ -114,7 +114,7 @@ func (tc *Tester) initServer(rawConfig string, configType string) error {
 		return nil
 	}
 
-	err := validateTestPrerequisites()
+	err := validateTestPrerequisites(tc.t)
 	if err != nil {
 		tc.t.Skipf("skipping tests as failed integration prerequisites. %s", err)
 		return nil
@@ -214,19 +214,24 @@ func (tc *Tester) ensureConfigRunning(rawConfig string, configType string) error
 		return actual
 	}
 
-	for retries := 4; retries > 0; retries-- {
+	for retries := 10; retries > 0; retries-- {
 		if reflect.DeepEqual(expected, fetchConfig(client)) {
 			return nil
 		}
-		time.Sleep(10 * time.Millisecond)
+		time.Sleep(1 * time.Second)
 	}
 	tc.t.Errorf("POSTed configuration isn't active")
 	return errors.New("EnsureConfigRunning: POSTed configuration isn't active")
 }
 
+const initConfig = `{
+	admin localhost:2999
+}
+`
+
 // validateTestPrerequisites ensures the certificates are available in the
 // designated path and Caddy sub-process is running.
-func validateTestPrerequisites() error {
+func validateTestPrerequisites(t *testing.T) error {
 
 	// check certificates are found
 	for _, certName := range Default.Certifcates {
@@ -236,15 +241,27 @@ func validateTestPrerequisites() error {
 	}
 
 	if isCaddyAdminRunning() != nil {
+		// setup the init config file, and set the cleanup afterwards
+		f, err := os.CreateTemp("", "")
+		if err != nil {
+			return err
+		}
+		t.Cleanup(func() {
+			os.Remove(f.Name())
+		})
+		if _, err := f.WriteString(initConfig); err != nil {
+			return err
+		}
+
 		// start inprocess caddy server
-		os.Args = []string{"caddy", "run"}
+		os.Args = []string{"caddy", "run", "--config", f.Name(), "--adapter", "caddyfile"}
 		go func() {
 			caddycmd.Main()
 		}()
 
 		// wait for caddy to start serving the initial config
-		for retries := 4; retries > 0 && isCaddyAdminRunning() != nil; retries-- {
-			time.Sleep(10 * time.Millisecond)
+		for retries := 10; retries > 0 && isCaddyAdminRunning() != nil; retries-- {
+			time.Sleep(1 * time.Second)
 		}
 	}
 

caddytest/integration/autohttps_test.go

@@ -11,6 +11,8 @@ func TestAutoHTTPtoHTTPSRedirectsImplicitPort(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
 	{
+		admin localhost:2999
+		skip_install_trust
 		http_port     9080
 		https_port    9443
 	}
@@ -25,6 +27,8 @@ func TestAutoHTTPtoHTTPSRedirectsExplicitPortSameAsHTTPSPort(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
 	{
+		skip_install_trust
+		admin localhost:2999
 		http_port     9080
 		https_port    9443
 	}
@@ -39,6 +43,8 @@ func TestAutoHTTPtoHTTPSRedirectsExplicitPortDifferentFromHTTPSPort(t *testing.T
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
 	{
+		skip_install_trust
+		admin localhost:2999
 		http_port     9080
 		https_port    9443
 	}
@@ -53,6 +59,9 @@ func TestAutoHTTPRedirectsWithHTTPListenerFirstInAddresses(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
 {
+  "admin": {
+	"listen": "localhost:2999"
+  },
   "apps": {
     "http": {
       "http_port": 9080,
@@ -74,7 +83,14 @@ func TestAutoHTTPRedirectsWithHTTPListenerFirstInAddresses(t *testing.T) {
           ]
         }
       }
-    }
+    },
+	"pki": {
+		"certificate_authorities": {
+			"local": {
+				"install_trust": false
+			}
+		}
+	}
   }
 }
 `, "json")
@@ -85,6 +101,8 @@ func TestAutoHTTPRedirectsInsertedBeforeUserDefinedCatchAll(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
 	{
+		skip_install_trust
+		admin localhost:2999
 		http_port     9080
 		https_port    9443
 		local_certs
@@ -108,6 +126,8 @@ func TestAutoHTTPRedirectsInsertedBeforeUserDefinedCatchAllWithNoExplicitHTTPSit
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
 	{
+		skip_install_trust
+		admin localhost:2999
 		http_port     9080
 		https_port    9443
 		local_certs

caddytest/integration/caddyfile_adapt/acme_server_lifetime.txt

@@ -0,0 +1,108 @@
+{
+	pki {
+		ca internal {
+			name "Internal"
+			root_cn "Internal Root Cert"
+			intermediate_cn "Internal Intermediate Cert"
+		}
+		ca internal-long-lived {
+			name "Long-lived"
+			root_cn "Internal Root Cert 2"
+			intermediate_cn "Internal Intermediate Cert 2"
+		}
+	}
+}
+
+acme-internal.example.com {
+	acme_server {
+		ca internal
+	}
+}
+
+acme-long-lived.example.com {
+	acme_server {
+		ca internal-long-lived
+		lifetime 7d
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"acme-long-lived.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"ca": "internal-long-lived",
+													"handler": "acme_server",
+													"lifetime": 604800000000000
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"acme-internal.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"ca": "internal",
+													"handler": "acme_server"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"pki": {
+			"certificate_authorities": {
+				"internal": {
+					"name": "Internal",
+					"root_common_name": "Internal Root Cert",
+					"intermediate_common_name": "Internal Intermediate Cert"
+				},
+				"internal-long-lived": {
+					"name": "Long-lived",
+					"root_common_name": "Internal Root Cert 2",
+					"intermediate_common_name": "Internal Intermediate Cert 2"
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options_admin_with_persist_config_off.txt

@@ -0,0 +1,36 @@
+{
+	http_port 8080
+	persist_config off
+	admin {
+		origins localhost:2019 [::1]:2019 127.0.0.1:2019 192.168.10.128
+	}
+}
+
+:80
+----------
+{
+	"admin": {
+		"listen": "localhost:2019",
+		"origins": [
+			"localhost:2019",
+			"[::1]:2019",
+			"127.0.0.1:2019",
+			"192.168.10.128"
+		],
+		"config": {
+			"persist": false
+		}
+	},
+	"apps": {
+		"http": {
+			"http_port": 8080,
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options_persist_config.txt

@@ -0,0 +1,25 @@
+{
+	persist_config off
+}
+
+:8881 {
+}
+----------
+{
+	"admin": {
+		"config": {
+			"persist": false
+		}
+	},
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8881"
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options_skip_install_trust.txt

@@ -165,4 +165,4 @@ acme-bar.example.com {
 			}
 		}
 	}
-}
+}

caddytest/integration/caddyfile_adapt/global_server_options_multi.txt

@@ -3,9 +3,7 @@
 		timeouts {
 			idle 90s
 		}
-		protocol {
-			strict_sni_host insecure_off
-		}
+		strict_sni_host insecure_off
 	}
 	servers :80 {
 		timeouts {
@@ -16,9 +14,7 @@
 		timeouts {
 			idle 30s
 		}
-		protocol {
-			strict_sni_host
-		}
+		strict_sni_host
 	}
 }
 

caddytest/integration/caddyfile_adapt/global_server_options_single.txt

@@ -12,8 +12,9 @@
 		}
 		max_header_size 100MB
 		log_credentials
-		strict_sni_host
 		protocols h1 h2 h2c h3
+		strict_sni_host
+		trusted_proxies static private_ranges
 	}
 }
 
@@ -55,6 +56,17 @@ foo.com {
 						}
 					],
 					"strict_sni_host": true,
+					"trusted_proxies": {
+						"ranges": [
+							"192.168.0.0/16",
+							"172.16.0.0/12",
+							"10.0.0.0/8",
+							"127.0.0.1/8",
+							"fd00::/8",
+							"::1"
+						],
+						"source": "static"
+					},
 					"logs": {
 						"should_log_credentials": true
 					},

caddytest/integration/caddyfile_adapt/handle_nested_in_route.txt

@@ -0,0 +1,78 @@
+:8881 {
+	route {
+		handle /foo/* {
+			respond "Foo"
+		}
+		handle {
+			respond "Bar"
+		}
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8881"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"group": "group2",
+											"handle": [
+												{
+													"handler": "subroute",
+													"routes": [
+														{
+															"handle": [
+																{
+																	"body": "Foo",
+																	"handler": "static_response"
+																}
+															]
+														}
+													]
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/foo/*"
+													]
+												}
+											]
+										},
+										{
+											"group": "group2",
+											"handle": [
+												{
+													"handler": "subroute",
+													"routes": [
+														{
+															"handle": [
+																{
+																	"body": "Bar",
+																	"handler": "static_response"
+																}
+															]
+														}
+													]
+												}
+											]
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/log_except_catchall_blocks.txt

@@ -1,5 +1,7 @@
 http://localhost:2020 {
 	log
+	skip_log /first-hidden*
+	skip_log /second-hidden*
 	respond 200
 }
 
@@ -28,6 +30,36 @@ http://localhost:2020 {
 								{
 									"handler": "subroute",
 									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "vars",
+													"skip_log": true
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/second-hidden*"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"handler": "vars",
+													"skip_log": true
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/first-hidden*"
+													]
+												}
+											]
+										},
 										{
 											"handle": [
 												{

caddytest/integration/caddyfile_adapt/log_skip_hosts.txt

@@ -62,6 +62,9 @@ example.com {
 						}
 					],
 					"logs": {
+						"logger_names": {
+							"one.example.com": ""
+						},
 						"skip_hosts": [
 							"three.example.com",
 							"two.example.com",

caddytest/integration/caddyfile_adapt/php_fastcgi_expanded_form.txt

@@ -8,7 +8,7 @@ route {
 		}
 		not path */
 	}
-	redir @canonicalPath {path}/ 308
+	redir @canonicalPath {http.request.orig_uri.path}/ 308
 
 	# If the requested file does not exist, try index files
 	@indexFiles {
@@ -50,7 +50,7 @@ route {
 													"handler": "static_response",
 													"headers": {
 														"Location": [
-															"{http.request.uri.path}/"
+															"{http.request.orig_uri.path}/"
 														]
 													},
 													"status_code": 308
@@ -74,6 +74,7 @@ route {
 											]
 										},
 										{
+											"group": "group0",
 											"handle": [
 												{
 													"handler": "rewrite",
@@ -129,4 +130,4 @@ route {
 			}
 		}
 	}
-}
+}

caddytest/integration/caddyfile_adapt/php_fastcgi_handle_response.txt

@@ -42,7 +42,7 @@
 									"handler": "static_response",
 									"headers": {
 										"Location": [
-											"{http.request.uri.path}/"
+											"{http.request.orig_uri.path}/"
 										]
 									},
 									"status_code": 308

caddytest/integration/caddyfile_adapt/php_fastcgi_matcher.txt

@@ -1,7 +1,12 @@
 :8884
 
-@api host example.com
-php_fastcgi @api localhost:9000
+# the use of a host matcher here should cause this
+# site block to be wrapped in a subroute, even though
+# the site block does not have a hostname; this is
+# to prevent auto-HTTPS from picking up on this host
+# matcher because it is not a key on the site block
+@test host example.com
+php_fastcgi @test localhost:9000
 ----------
 {
 	"apps": {
@@ -13,13 +18,6 @@ php_fastcgi @api localhost:9000
 					],
 					"routes": [
 						{
-							"match": [
-								{
-									"host": [
-										"example.com"
-									]
-								}
-							],
 							"handle": [
 								{
 									"handler": "subroute",
@@ -27,82 +25,99 @@ php_fastcgi @api localhost:9000
 										{
 											"handle": [
 												{
-													"handler": "static_response",
-													"headers": {
-														"Location": [
-															"{http.request.uri.path}/"
-														]
-													},
-													"status_code": 308
-												}
-											],
-											"match": [
-												{
-													"file": {
-														"try_files": [
-															"{http.request.uri.path}/index.php"
-														]
-													},
-													"not": [
+													"handler": "subroute",
+													"routes": [
 														{
-															"path": [
-																"*/"
+															"handle": [
+																{
+																	"handler": "static_response",
+																	"headers": {
+																		"Location": [
+																			"{http.request.orig_uri.path}/"
+																		]
+																	},
+																	"status_code": 308
+																}
+															],
+															"match": [
+																{
+																	"file": {
+																		"try_files": [
+																			"{http.request.uri.path}/index.php"
+																		]
+																	},
+																	"not": [
+																		{
+																			"path": [
+																				"*/"
+																			]
+																		}
+																	]
+																}
+															]
+														},
+														{
+															"handle": [
+																{
+																	"handler": "rewrite",
+																	"uri": "{http.matchers.file.relative}"
+																}
+															],
+															"match": [
+																{
+																	"file": {
+																		"split_path": [
+																			".php"
+																		],
+																		"try_files": [
+																			"{http.request.uri.path}",
+																			"{http.request.uri.path}/index.php",
+																			"index.php"
+																		]
+																	}
+																}
+															]
+														},
+														{
+															"handle": [
+																{
+																	"handler": "reverse_proxy",
+																	"transport": {
+																		"protocol": "fastcgi",
+																		"split_path": [
+																			".php"
+																		]
+																	},
+																	"upstreams": [
+																		{
+																			"dial": "localhost:9000"
+																		}
+																	]
+																}
+															],
+															"match": [
+																{
+																	"path": [
+																		"*.php"
+																	]
+																}
 															]
 														}
 													]
 												}
-											]
-										},
-										{
-											"handle": [
-												{
-													"handler": "rewrite",
-													"uri": "{http.matchers.file.relative}"
-												}
 											],
 											"match": [
 												{
-													"file": {
-														"split_path": [
-															".php"
-														],
-														"try_files": [
-															"{http.request.uri.path}",
-															"{http.request.uri.path}/index.php",
-															"index.php"
-														]
-													}
-												}
-											]
-										},
-										{
-											"handle": [
-												{
-													"handler": "reverse_proxy",
-													"transport": {
-														"protocol": "fastcgi",
-														"split_path": [
-															".php"
-														]
-													},
-													"upstreams": [
-														{
-															"dial": "localhost:9000"
-														}
-													]
-												}
-											],
-											"match": [
-												{
-													"path": [
-														"*.php"
+													"host": [
+														"example.com"
 													]
 												}
 											]
 										}
 									]
 								}
-							]
+							],
+							"terminal": true
 						}
 					]
 				}

caddytest/integration/caddyfile_adapt/php_fastcgi_subdirectives.txt

@@ -43,7 +43,7 @@ php_fastcgi localhost:9000 {
 									"handler": "static_response",
 									"headers": {
 										"Location": [
-											"{http.request.uri.path}/"
+											"{http.request.orig_uri.path}/"
 										]
 									},
 									"status_code": 308

caddytest/integration/caddyfile_adapt/php_fastcgi_try_files_override.txt

@@ -46,7 +46,7 @@ php_fastcgi localhost:9000 {
 									"handler": "static_response",
 									"headers": {
 										"Location": [
-											"{http.request.uri.path}/"
+											"{http.request.orig_uri.path}/"
 										]
 									},
 									"status_code": 308

caddytest/integration/caddyfile_adapt/reverse_proxy_options.txt

@@ -1,4 +1,3 @@
-
 https://example.com {
 	reverse_proxy /path https://localhost:54321 {
 		header_up Host {upstream_hostport}
@@ -24,13 +23,12 @@ https://example.com {
 			max_conns_per_host 5
 			keepalive_idle_conns_per_host 2
 			keepalive_interval 30s
-			
+
 			tls_renegotiation freely
 			tls_except_ports 8181 8182
 		}
 	}
 }
-
 ----------
 {
 	"apps": {

caddytest/integration/caddyfile_adapt/server_names.txt

@@ -0,0 +1,77 @@
+{
+	servers :443 {
+		name https
+	}
+
+	servers :8000 {
+		name app1
+	}
+
+	servers :8001 {
+		name app2
+	}
+
+	servers 123.123.123.123:8002 {
+		name bind-server
+	}
+}
+
+example.com {
+}
+
+:8000 {
+}
+
+:8001, :8002 {
+}
+
+:8002 {
+	bind 123.123.123.123 222.222.222.222
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"app1": {
+					"listen": [
+						":8000"
+					]
+				},
+				"app2": {
+					"listen": [
+						":8001"
+					]
+				},
+				"bind-server": {
+					"listen": [
+						"123.123.123.123:8002",
+						"222.222.222.222:8002"
+					]
+				},
+				"https": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				},
+				"srv4": {
+					"listen": [
+						":8002"
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_automation_policies_10.txt

@@ -0,0 +1,58 @@
+# example from issue #4667
+{
+	auto_https off
+}
+
+https://, example.com {
+	tls test.crt test.key
+	respond "Hello World"
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"body": "Hello World",
+									"handler": "static_response"
+								}
+							]
+						}
+					],
+					"tls_connection_policies": [
+						{
+							"certificate_selection": {
+								"any_tag": [
+									"cert0"
+								]
+							}
+						}
+					],
+					"automatic_https": {
+						"disable": true
+					}
+				}
+			}
+		},
+		"tls": {
+			"certificates": {
+				"load_files": [
+					{
+						"certificate": "test.crt",
+						"key": "test.key",
+						"tags": [
+							"cert0"
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_dns_ttl.txt

@@ -0,0 +1,76 @@
+localhost
+
+respond "hello from localhost"
+tls {
+	dns_ttl 5m10s
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"body": "hello from localhost",
+													"handler": "static_response"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"localhost"
+						],
+						"issuers": [
+							{
+								"challenges": {
+									"dns": {
+										"ttl": 310000000000
+									}
+								},
+								"module": "acme"
+							},
+							{
+								"challenges": {
+									"dns": {
+										"ttl": 310000000000
+									}
+								},
+								"module": "zerossl"
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_explicit_issuer_dns_ttl.txt

@@ -0,0 +1,81 @@
+localhost
+
+respond "hello from localhost"
+tls {
+	issuer acme {
+		dns_ttl 5m10s
+	}
+	issuer zerossl {
+		dns_ttl 10m20s
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"body": "hello from localhost",
+													"handler": "static_response"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"localhost"
+						],
+						"issuers": [
+							{
+								"challenges": {
+									"dns": {
+										"ttl": 310000000000
+									}
+								},
+								"module": "acme"
+							},
+							{
+								"challenges": {
+									"dns": {
+										"ttl": 620000000000
+									}
+								},
+								"module": "zerossl"
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_propagation_options.txt

@@ -0,0 +1,79 @@
+localhost
+
+respond "hello from localhost"
+tls {
+	propagation_delay 5m10s
+	propagation_timeout 10m20s
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"body": "hello from localhost",
+													"handler": "static_response"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"localhost"
+						],
+						"issuers": [
+							{
+								"challenges": {
+									"dns": {
+										"propagation_delay": 310000000000,
+										"propagation_timeout": 620000000000
+									}
+								},
+								"module": "acme"
+							},
+							{
+								"challenges": {
+									"dns": {
+										"propagation_delay": 310000000000,
+										"propagation_timeout": 620000000000
+									}
+								},
+								"module": "zerossl"
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_test.go

@@ -14,8 +14,10 @@ func TestRespond(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(` 
   {
+    admin localhost:2999
     http_port     9080
     https_port    9443
+    grace_period  1ns
   }
   
   localhost:9080 {
@@ -35,8 +37,10 @@ func TestRedirect(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
   {
+    admin localhost:2999
     http_port     9080
     https_port    9443
+    grace_period  1ns
   }
   
   localhost:9080 {
@@ -84,8 +88,11 @@ func TestReadCookie(t *testing.T) {
 	tester.Client.Jar.SetCookies(localhost, []*http.Cookie{&cookie})
 	tester.InitServer(` 
   {
+    skip_install_trust
+    admin localhost:2999
     http_port     9080
     https_port    9443
+    grace_period  1ns
   }
   
   localhost:9080 {
@@ -107,8 +114,11 @@ func TestReplIndex(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
   {
+    skip_install_trust
+    admin localhost:2999
     http_port     9080
     https_port    9443
+    grace_period  1ns
   }
 
   localhost:9080 {

caddytest/integration/handler_test.go

@@ -11,8 +11,11 @@ func TestBrowse(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
 	{
+		skip_install_trust
+		admin localhost:2999
 		http_port     9080
 		https_port    9443
+		grace_period  1ns
 	}
 	http://localhost:9080 {
 		file_server browse

caddytest/integration/map_test.go

@@ -11,8 +11,11 @@ func TestMap(t *testing.T) {
 	// arrange
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`{
+		skip_install_trust
+		admin localhost:2999
 		http_port     9080
 		https_port    9443
+		grace_period  1ns
 	}
 
 	localhost:9080 {
@@ -38,6 +41,8 @@ func TestMapRespondWithDefault(t *testing.T) {
 	// arrange
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`{
+		skip_install_trust
+		admin localhost:2999
 		http_port     9080
 		https_port    9443
 		}
@@ -65,7 +70,17 @@ func TestMapAsJSON(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
 	{
+		"admin": {
+			"listen": "localhost:2999"
+		},
 		"apps": {
+			"pki": {
+				"certificate_authorities" : {
+				  "local" : {
+					"install_trust": false
+				  }
+				}
+			},
 			"http": {
 				"http_port": 9080,
 				"https_port": 9443,

caddytest/integration/pki_test.go

@@ -0,0 +1,101 @@
+package integration
+
+import (
+	"testing"
+
+	"github.com/caddyserver/caddy/v2/caddytest"
+)
+
+func TestLeafCertLifetimeLessThanIntermediate(t *testing.T) {
+	caddytest.AssertLoadError(t, `
+    {
+      "apps": {
+        "http": {
+          "servers": {
+            "srv0": {
+              "listen": [
+                ":443"
+              ],
+              "routes": [
+                {
+                  "handle": [
+                    {
+                      "handler": "subroute",
+                      "routes": [
+                        {
+                          "handle": [
+                            {
+                              "ca": "internal",
+                              "handler": "acme_server",
+                              "lifetime": 604800000000000
+                            }
+                          ]
+                        }
+                      ]
+                    }
+                  ]
+                }
+              ]
+            }
+          }
+        },
+        "pki": {
+          "certificate_authorities": {
+            "internal": {
+              "install_trust": false,
+              "intermediate_lifetime": 604800000000000,
+              "name": "Internal CA"
+            }
+          }
+        }
+      }
+    }
+	`, "json", "certificate lifetime (168h0m0s) should be less than intermediate certificate lifetime (168h0m0s)")
+}
+
+func TestIntermediateLifetimeLessThanRoot(t *testing.T) {
+	caddytest.AssertLoadError(t, `
+    {
+      "apps": {
+        "http": {
+          "servers": {
+            "srv0": {
+              "listen": [
+                ":443"
+              ],
+              "routes": [
+                {
+                  "handle": [
+                    {
+                      "handler": "subroute",
+                      "routes": [
+                        {
+                          "handle": [
+                            {
+                              "ca": "internal",
+                              "handler": "acme_server",
+                              "lifetime": 2592000000000000
+                            }
+                          ]
+                        }
+                      ]
+                    }
+                  ]
+                }
+              ]
+            }
+          }
+        },
+        "pki": {
+          "certificate_authorities": {
+            "internal": {
+              "install_trust": false,
+              "intermediate_lifetime": 311040000000000000,
+              "name": "Internal CA"
+            }
+          }
+        }
+      }
+    }
+	`, "json", "intermediate certificate lifetime must be less than root certificate lifetime (86400h0m0s)")
+}

caddytest/integration/reverseproxy_test.go

@@ -8,6 +8,7 @@ import (
 	"runtime"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/caddyserver/caddy/v2/caddytest"
 )
@@ -16,8 +17,19 @@ func TestSRVReverseProxy(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
 	{
+		"admin": {
+			"listen": "localhost:2999"
+		},
 		"apps": {
+			"pki": {
+				"certificate_authorities" : {
+					"local" : {
+					"install_trust": false
+					}
+				}
+			},
 		  "http": {
+			"grace_period": 1,
 			"servers": {
 			  "srv0": {
 				"listen": [
@@ -49,7 +61,15 @@ func TestSRVWithDial(t *testing.T) {
 	caddytest.AssertLoadError(t, `
 	{
 		"apps": {
+			"pki": {
+				"certificate_authorities" : {
+				  "local" : {
+					"install_trust": false
+				  }
+				}
+			  },
 		  "http": {
+			"grace_period": 1,
 			"servers": {
 			  "srv0": {
 				"listen": [
@@ -113,8 +133,19 @@ func TestDialWithPlaceholderUnix(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
 	{
+		"admin": {
+			"listen": "localhost:2999"
+		},
 		"apps": {
+			"pki": {
+				"certificate_authorities" : {
+				  "local" : {
+					"install_trust": false
+				  }
+				}
+			  },
 		  "http": {
+			"grace_period": 1,
 			"servers": {
 			  "srv0": {
 				"listen": [
@@ -154,8 +185,19 @@ func TestReverseProxyWithPlaceholderDialAddress(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
 	{
+		"admin": {
+			"listen": "localhost:2999"
+		},
 		"apps": {
+			"pki": {
+				"certificate_authorities" : {
+				  "local" : {
+					"install_trust": false
+				  }
+				}
+			  },
 			"http": {
+				"grace_period": 1,
 				"servers": {
 					"srv0": {
 						"listen": [
@@ -237,8 +279,19 @@ func TestReverseProxyWithPlaceholderTCPDialAddress(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
 	{
+		"admin": {
+			"listen": "localhost:2999"
+		},
 		"apps": {
+			"pki": {
+				"certificate_authorities" : {
+				  "local" : {
+					"install_trust": false
+				  }
+				}
+			  },
 			"http": {
+				"grace_period": 1,
 				"servers": {
 					"srv0": {
 						"listen": [
@@ -320,7 +373,15 @@ func TestSRVWithActiveHealthcheck(t *testing.T) {
 	caddytest.AssertLoadError(t, `
 	{
 		"apps": {
+			"pki": {
+				"certificate_authorities" : {
+				  "local" : {
+					"install_trust": false
+				  }
+				}
+			  },
 		  "http": {
+			"grace_period": 1,
 			"servers": {
 			  "srv0": {
 				"listen": [
@@ -357,8 +418,11 @@ func TestReverseProxyHealthCheck(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
 	{
+		skip_install_trust
+		admin localhost:2999
 		http_port     9080
 		https_port    9443
+		grace_period 1ns
 	}
 	http://localhost:2020 {
 		respond "Hello, World!"
@@ -372,12 +436,13 @@ func TestReverseProxyHealthCheck(t *testing.T) {
 	
 			health_uri /health
 			health_port 2021
-			health_interval 2s
-			health_timeout 5s
+			health_interval 10ms
+			health_timeout 100ms
 		}
 	}
   `, "caddyfile")
 
+	time.Sleep(100 * time.Millisecond) // TODO: for some reason this test seems particularly flaky, getting 503 when it should be 200, unless we wait
 	tester.AssertGetResponse("http://localhost:9080/", 200, "Hello, World!")
 }
 
@@ -418,8 +483,11 @@ func TestReverseProxyHealthCheckUnixSocket(t *testing.T) {
 
 	tester.InitServer(fmt.Sprintf(`
 	{
+		skip_install_trust
+		admin localhost:2999
 		http_port     9080
 		https_port    9443
+		grace_period 1ns
 	}
 	http://localhost:9080 {
 		reverse_proxy {
@@ -473,8 +541,11 @@ func TestReverseProxyHealthCheckUnixSocketWithoutPort(t *testing.T) {
 
 	tester.InitServer(fmt.Sprintf(`
 	{
+		skip_install_trust
+		admin localhost:2999
 		http_port     9080
 		https_port    9443
+		grace_period 1ns
 	}
 	http://localhost:9080 {
 		reverse_proxy {

caddytest/integration/sni_test.go

@@ -11,91 +11,95 @@ func TestDefaultSNI(t *testing.T) {
 	// arrange
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`{
-    "apps": {
-      "http": {
-        "http_port": 9080,
-        "https_port": 9443,
-        "servers": {
-          "srv0": {
-            "listen": [
-              ":9443"
-            ],
-            "routes": [
-              {
-                "handle": [
-                  {
-                    "handler": "subroute",
-                    "routes": [
-                      {
-                        "handle": [
-                          {
-                            "body": "hello from a.caddy.localhost",
-                            "handler": "static_response",
-                            "status_code": 200
-                          }
-                        ],
-                        "match": [
-                          {
-                            "path": [
-                              "/version"
-                            ]
-                          }
-                        ]
-                      }
-                    ]
-                  }
-                ],
-                "match": [
-                  {
-                    "host": [
-                      "127.0.0.1"
-                    ]
-                  }
-                ],
-                "terminal": true
-              }
-            ],
-            "tls_connection_policies": [
-              {
-                "certificate_selection": {
-                  "any_tag": ["cert0"]
-                },
-                "match": {
-                  "sni": [
-                    "127.0.0.1"
-                  ]
-                }
-              },
-              {
-                "default_sni": "*.caddy.localhost"
-              }
-            ]
-          }
-        }
-      },
-      "tls": {
-        "certificates": {
-          "load_files": [
-            {
-              "certificate": "/caddy.localhost.crt",
-              "key": "/caddy.localhost.key",
-              "tags": [
-                "cert0"
-              ]
-            }
-          ]
-        }
-      },
-      "pki": {
-        "certificate_authorities" : {
-          "local" : {
-            "install_trust": false
-          }
-        }
-      }
-    }
-  }
-  `, "json")
+		"admin": {
+			"listen": "localhost:2999"
+		},
+		"apps": {
+			"http": {
+				"http_port": 9080,
+				"https_port": 9443,
+				"grace_period": 1,
+				"servers": {
+					"srv0": {
+						"listen": [
+							":9443"
+						],
+						"routes": [
+							{
+								"handle": [
+									{
+										"handler": "subroute",
+										"routes": [
+											{
+												"handle": [
+													{
+														"body": "hello from a.caddy.localhost",
+														"handler": "static_response",
+														"status_code": 200
+													}
+												],
+												"match": [
+													{
+														"path": [
+															"/version"
+														]
+													}
+												]
+											}
+										]
+									}
+								],
+								"match": [
+									{
+										"host": [
+											"127.0.0.1"
+										]
+									}
+								],
+								"terminal": true
+							}
+						],
+						"tls_connection_policies": [
+							{
+								"certificate_selection": {
+									"any_tag": ["cert0"]
+								},
+								"match": {
+									"sni": [
+										"127.0.0.1"
+									]
+								}
+							},
+							{
+								"default_sni": "*.caddy.localhost"
+							}
+						]
+					}
+				}
+			},
+			"tls": {
+				"certificates": {
+					"load_files": [
+						{
+							"certificate": "/caddy.localhost.crt",
+							"key": "/caddy.localhost.key",
+							"tags": [
+								"cert0"
+							]
+						}
+					]
+				}
+			},
+			"pki": {
+				"certificate_authorities" : {
+					"local" : {
+						"install_trust": false
+					}
+				}
+			}
+		}
+	}
+	`, "json")
 
 	// act and assert
 	// makes a request with no sni
@@ -107,96 +111,100 @@ func TestDefaultSNIWithNamedHostAndExplicitIP(t *testing.T) {
 	// arrange
 	tester := caddytest.NewTester(t)
 	tester.InitServer(` 
-  {
-    "apps": {
-      "http": {
-        "http_port": 9080,
-        "https_port": 9443,
-        "servers": {
-          "srv0": {
-            "listen": [
-              ":9443"
-            ],
-            "routes": [
-              {
-                "handle": [
-                  {
-                    "handler": "subroute",
-                    "routes": [
-                      {
-                        "handle": [
-                          {
-                            "body": "hello from a",
-                            "handler": "static_response",
-                            "status_code": 200
-                          }
-                        ],
-                        "match": [
-                          {
-                            "path": [
-                              "/version"
-                            ]
-                          }
-                        ]
-                      }
-                    ]
-                  }
-                ],
-                "match": [
-                  {
-                    "host": [
-                      "a.caddy.localhost",
-                      "127.0.0.1"
-                    ]
-                  }
-                ],
-                "terminal": true
-              }
-            ],
-            "tls_connection_policies": [
-              {
-                "certificate_selection": {
-                  "any_tag": ["cert0"]
-                },
-                "default_sni": "a.caddy.localhost",
-                "match": {
-                  "sni": [
-                    "a.caddy.localhost",
-                    "127.0.0.1",
-                    ""
-                  ]
-                }
-              },
-              {
-                "default_sni": "a.caddy.localhost"
-              }
-            ]
-          }
-        }
-      },
-      "tls": {
-        "certificates": {
-          "load_files": [
-            {
-              "certificate": "/a.caddy.localhost.crt",
-              "key": "/a.caddy.localhost.key",
-              "tags": [
-                "cert0"
-              ]
-            }
-          ]
-        }
-      },
-      "pki": {
-        "certificate_authorities" : {
-          "local" : {
-            "install_trust": false
-          }
-        }
-      }
-    }
-  }
-  `, "json")
+	{
+		"admin": {
+			"listen": "localhost:2999"
+		},
+		"apps": {
+			"http": {
+				"http_port": 9080,
+				"https_port": 9443,
+				"grace_period": 1,
+				"servers": {
+					"srv0": {
+						"listen": [
+							":9443"
+						],
+						"routes": [
+							{
+								"handle": [
+									{
+										"handler": "subroute",
+										"routes": [
+											{
+												"handle": [
+													{
+														"body": "hello from a",
+														"handler": "static_response",
+														"status_code": 200
+													}
+												],
+												"match": [
+													{
+														"path": [
+															"/version"
+														]
+													}
+												]
+											}
+										]
+									}
+								],
+								"match": [
+									{
+										"host": [
+											"a.caddy.localhost",
+											"127.0.0.1"
+										]
+									}
+								],
+								"terminal": true
+							}
+						],
+						"tls_connection_policies": [
+							{
+								"certificate_selection": {
+									"any_tag": ["cert0"]
+								},
+								"default_sni": "a.caddy.localhost",
+								"match": {
+									"sni": [
+										"a.caddy.localhost",
+										"127.0.0.1",
+										""
+									]
+								}
+							},
+							{
+								"default_sni": "a.caddy.localhost"
+							}
+						]
+					}
+				}
+			},
+			"tls": {
+				"certificates": {
+					"load_files": [
+						{
+							"certificate": "/a.caddy.localhost.crt",
+							"key": "/a.caddy.localhost.key",
+							"tags": [
+								"cert0"
+							]
+						}
+					]
+				}
+			},
+			"pki": {
+				"certificate_authorities" : {
+					"local" : {
+						"install_trust": false
+					}
+				}
+			}
+		}
+	}
+	`, "json")
 
 	// act and assert
 	// makes a request with no sni
@@ -207,68 +215,72 @@ func TestDefaultSNIWithPortMappingOnly(t *testing.T) {
 	// arrange
 	tester := caddytest.NewTester(t)
 	tester.InitServer(` 
-  {
-    "apps": {
-      "http": {
-        "http_port": 9080,
-        "https_port": 9443,
-        "servers": {
-          "srv0": {
-            "listen": [
-              ":9443"
-            ],
-            "routes": [
-              {
-                "handle": [
-                  {
-                    "body": "hello from a.caddy.localhost",
-                    "handler": "static_response",
-                    "status_code": 200
-                  }
-                ],
-                "match": [
-                  {
-                    "path": [
-                      "/version"
-                    ]
-                  }
-                ]
-              }
-            ],
-            "tls_connection_policies": [
-              {
-                "certificate_selection": {
-                  "any_tag": ["cert0"]
-                },
-                "default_sni": "a.caddy.localhost"
-              }
-            ]
-          }
-        }
-      },
-      "tls": {
-        "certificates": {
-          "load_files": [
-            {
-              "certificate": "/a.caddy.localhost.crt",
-              "key": "/a.caddy.localhost.key",
-              "tags": [
-                "cert0"
-              ]
-            }
-          ]
-        }
-      },
-      "pki": {
-        "certificate_authorities" : {
-          "local" : {
-            "install_trust": false
-          }
-        }
-      }
-    }
-  }
-  `, "json")
+	{
+		"admin": {
+			"listen": "localhost:2999"
+		},
+		"apps": {
+			"http": {
+				"http_port": 9080,
+				"https_port": 9443,
+				"grace_period": 1,
+				"servers": {
+					"srv0": {
+						"listen": [
+							":9443"
+						],
+						"routes": [
+							{
+								"handle": [
+									{
+										"body": "hello from a.caddy.localhost",
+										"handler": "static_response",
+										"status_code": 200
+									}
+								],
+								"match": [
+									{
+										"path": [
+											"/version"
+										]
+									}
+								]
+							}
+						],
+						"tls_connection_policies": [
+							{
+								"certificate_selection": {
+									"any_tag": ["cert0"]
+								},
+								"default_sni": "a.caddy.localhost"
+							}
+						]
+					}
+				}
+			},
+			"tls": {
+				"certificates": {
+					"load_files": [
+						{
+							"certificate": "/a.caddy.localhost.crt",
+							"key": "/a.caddy.localhost.key",
+							"tags": [
+								"cert0"
+							]
+						}
+					]
+				}
+			},
+			"pki": {
+				"certificate_authorities" : {
+					"local" : {
+						"install_trust": false
+					}
+				}
+			}
+		}
+	}
+	`, "json")
 
 	// act and assert
 	// makes a request with no sni
@@ -278,6 +290,7 @@ func TestDefaultSNIWithPortMappingOnly(t *testing.T) {
 func TestHttpOnlyOnDomainWithSNI(t *testing.T) {
 	caddytest.AssertAdapt(t, `
 	{
+		skip_install_trust
 		default_sni a.caddy.localhost
 	}
 	:80 {
@@ -313,6 +326,13 @@ func TestHttpOnlyOnDomainWithSNI(t *testing.T) {
 					]
 				}
 			}
+		},
+		"pki": {
+			"certificate_authorities": {
+				"local": {
+					"install_trust": false
+				}
+			}
 		}
 	}
 }`)

caddytest/integration/stream_test.go

@@ -23,10 +23,14 @@ func TestH2ToH2CStream(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(` 
   {
+	"admin": {
+		"listen": "localhost:2999"
+	},
     "apps": {
       "http": {
         "http_port": 9080,
         "https_port": 9443,
+		"grace_period": 1,
         "servers": {
           "srv0": {
             "listen": [
@@ -205,6 +209,9 @@ func TestH2ToH1ChunkedResponse(t *testing.T) {
 	tester := caddytest.NewTester(t)
 	tester.InitServer(` 
 {
+	"admin": {
+		"listen": "localhost:2999"
+	},
   "logging": {
     "logs": {
       "default": {
@@ -216,6 +223,7 @@ func TestH2ToH1ChunkedResponse(t *testing.T) {
     "http": {
       "http_port": 9080,
       "https_port": 9443,
+	  "grace_period": 1,
       "servers": {
         "srv0": {
           "listen": [

cmd/caddy/main.go

@@ -19,10 +19,10 @@
 // There is no need to modify the Caddy source code to customize your
 // builds. You can easily build a custom Caddy with these simple steps:
 //
-//   1. Copy this file (main.go) into a new folder
-//   2. Edit the imports below to include the modules you want plugged in
-//   3. Run `go mod init caddy`
-//   4. Run `go install` or `go build` - you now have a custom binary!
+//  1. Copy this file (main.go) into a new folder
+//  2. Edit the imports below to include the modules you want plugged in
+//  3. Run `go mod init caddy`
+//  4. Run `go install` or `go build` - you now have a custom binary!
 //
 // Or you can use xcaddy which does it all for you as a command:
 // https://github.com/caddyserver/xcaddy

cmd/cobra.go

@@ -101,10 +101,10 @@ const fullDocsFooter = `Full documentation is available at:
 https://caddyserver.com/docs/command-line`
 
 func init() {
-	rootCmd.SetHelpTemplate(rootCmd.HelpTemplate() + "\n" + fullDocsFooter)
+	rootCmd.SetHelpTemplate(rootCmd.HelpTemplate() + "\n" + fullDocsFooter + "\n")
 }
 
-func caddyCmdToCoral(caddyCmd Command) *cobra.Command {
+func caddyCmdToCobra(caddyCmd Command) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   caddyCmd.Name,
 		Short: caddyCmd.Short,

cmd/commandfuncs.go

@@ -506,6 +506,15 @@ func cmdAdaptConfig(fl Flags) (int, error) {
 func cmdValidateConfig(fl Flags) (int, error) {
 	validateCmdConfigFlag := fl.String("config")
 	validateCmdAdapterFlag := fl.String("adapter")
+	runCmdLoadEnvfileFlag := fl.String("envfile")
+
+	// load all additional envs as soon as possible
+	if runCmdLoadEnvfileFlag != "" {
+		if err := loadEnvFromFile(runCmdLoadEnvfileFlag); err != nil {
+			return caddy.ExitCodeFailedStartup,
+				fmt.Errorf("loading additional environment variables: %v", err)
+		}
+	}
 
 	input, _, err := LoadConfig(validateCmdConfigFlag, validateCmdAdapterFlag)
 	if err != nil {
@@ -558,7 +567,10 @@ func cmdFmt(fl Flags) (int, error) {
 		if err := os.WriteFile(formatCmdConfigFile, output, 0600); err != nil {
 			return caddy.ExitCodeFailedStartup, fmt.Errorf("overwriting formatted file: %v", err)
 		}
-	} else if fl.Bool("diff") {
+		return caddy.ExitCodeSuccess, nil
+	}
+
+	if fl.Bool("diff") {
 		diff := difflib.Diff(
 			strings.Split(string(input), "\n"),
 			strings.Split(string(output), "\n"))
@@ -576,6 +588,10 @@ func cmdFmt(fl Flags) (int, error) {
 		fmt.Print(string(output))
 	}
 
+	if warning, diff := caddyfile.FormattingDifference(formatCmdConfigFile, input); diff {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("%s:%d: Caddyfile input is not formatted", warning.File, warning.Line)
+	}
+
 	return caddy.ExitCodeSuccess, nil
 }
 
@@ -679,7 +695,7 @@ func DetermineAdminAPIAddress(address string, config []byte, configFile, configA
 				return "", err
 			}
 			if loadedConfigFile == "" {
-				return "", fmt.Errorf("no config file to load")
+				return "", fmt.Errorf("no config file to load; either use --config flag or ensure Caddyfile exists in current directory")
 			}
 		}
 

cmd/commands.go

@@ -200,6 +200,19 @@ config file; otherwise the default is assumed.`,
 		Name:  "version",
 		Func:  cmdVersion,
 		Short: "Prints the version",
+		Long: `
+Prints the version of this Caddy binary.
+
+Version information must be embedded into the binary at compile-time in
+order for Caddy to display anything useful with this command. If Caddy
+is built from within a version control repository, the Go command will
+embed the revision hash if available. However, if Caddy is built in the
+way specified by our online documentation (or by using xcaddy), more
+detailed version information is printed as given by Go modules.
+
+For more details about the full version string, see the Go module
+documentation: https://go.dev/doc/modules/version-numbers
+`,
 	})
 
 	RegisterCommand(Command{
@@ -226,6 +239,24 @@ config file; otherwise the default is assumed.`,
 		Name:  "environ",
 		Func:  cmdEnviron,
 		Short: "Prints the environment",
+		Long: `
+Prints the environment as seen by this Caddy process.
+
+The environment includes variables set in the system. If your Caddy
+configuration uses environment variables (e.g. "{env.VARIABLE}") then
+this command can be useful for verifying that the variables will have
+the values you expect in your config.
+
+Note that environments may be different depending on how you run Caddy.
+Environments for Caddy instances started by service managers such as
+systemd are often different than the environment inherited from your
+shell or terminal.
+
+You can also print the environment the same time you use "caddy run"
+by adding the "--environ" flag.
+
+Environments may contain sensitive data.
+`,
 	})
 
 	RegisterCommand(Command{
@@ -256,16 +287,20 @@ zero exit status will be returned.`,
 	RegisterCommand(Command{
 		Name:  "validate",
 		Func:  cmdValidateConfig,
-		Usage: "--config <path> [--adapter <name>]",
+		Usage: "--config <path> [--adapter <name>] [--envfile <path>]",
 		Short: "Tests whether a configuration file is valid",
 		Long: `
 Loads and provisions the provided config, but does not start running it.
 This reveals any errors with the configuration through the loading and
-provisioning stages.`,
+provisioning stages.
+
+If --envfile is specified, an environment file with environment variables in
+the KEY=VALUE format will be loaded into the Caddy process.`,
 		Flags: func() *flag.FlagSet {
 			fs := flag.NewFlagSet("validate", flag.ExitOnError)
 			fs.String("config", "", "Input configuration file")
 			fs.String("adapter", "", "Name of config adapter")
+			fs.String("envfile", "", "Environment file to load")
 			return fs
 		}(),
 	})
@@ -367,9 +402,11 @@ EXPERIMENTAL: May be changed or removed.
 		Usage: "--directory <path>",
 		Short: "Generates the manual pages for Caddy commands",
 		Long: `
-Generates the manual pages for Caddy commands into the designated directory tagged into section 8 (System Administration).
+Generates the manual pages for Caddy commands into the designated directory
+tagged into section 8 (System Administration).
 
-The manual page files are generated into the directory specified by the argument of --directory. If the directory does not exist, it will be created.
+The manual page files are generated into the directory specified by the
+argument of --directory. If the directory does not exist, it will be created.
 `,
 		Flags: func() *flag.FlagSet {
 			fs := flag.NewFlagSet("manpage", flag.ExitOnError)
@@ -423,7 +460,7 @@ The manual page files are generated into the directory specified by the argument
 	`, rootCmd.Root().Name()),
 		DisableFlagsInUseLine: true,
 		ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
-		Args:                  cobra.ExactValidArgs(1),
+		Args:                  cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			switch args[0] {
 			case "bash":
@@ -471,7 +508,7 @@ func RegisterCommand(cmd Command) {
 	if !commandNameRegex.MatchString(cmd.Name) {
 		panic("invalid command name")
 	}
-	rootCmd.AddCommand(caddyCmdToCoral(cmd))
+	rootCmd.AddCommand(caddyCmdToCobra(cmd))
 }
 
 var commandNameRegex = regexp.MustCompile(`^[a-z0-9]$|^([a-z0-9]+-?[a-z0-9]*)+[a-z0-9]$`)

cmd/main.go

@@ -41,10 +41,15 @@ func init() {
 	// set a fitting User-Agent for ACME requests
 	version, _ := caddy.Version()
 	cleanModVersion := strings.TrimPrefix(version, "v")
-	certmagic.UserAgent = "Caddy/" + cleanModVersion
+	ua := "Caddy/" + cleanModVersion
+	if uaEnv, ok := os.LookupEnv("USERAGENT"); ok {
+		ua = uaEnv + " " + ua
+	}
+	certmagic.UserAgent = ua
 
 	// by using Caddy, user indicates agreement to CA terms
-	// (very important, or ACME account creation will fail!)
+	// (very important, as Caddy is often non-interactive
+	// and thus ACME account creation will fail!)
 	certmagic.DefaultACME.Agreed = true
 }
 

context.go

@@ -442,10 +442,27 @@ func (ctx Context) Storage() certmagic.Storage {
 	return ctx.cfg.storage
 }
 
-// TODO: aw man, can I please change this?
-// Logger returns a logger that can be used by mod.
-func (ctx Context) Logger(mod Module) *zap.Logger {
-	// TODO: if mod is nil, use ctx.Module() instead...
+// Logger returns a logger that is intended for use by the most
+// recent module associated with the context. Callers should not
+// pass in any arguments unless they want to associate with a
+// different module; it panics if more than 1 value is passed in.
+//
+// Originally, this method's signature was `Logger(mod Module)`,
+// requiring that an instance of a Caddy module be passed in.
+// However, that is no longer necessary, as the closest module
+// most recently associated with the context will be automatically
+// assumed. To prevent a sudden breaking change, this method's
+// signature has been changed to be variadic, but we may remove
+// the parameter altogether in the future. Callers should not
+// pass in any argument. If there is valid need to specify a
+// different module, please open an issue to discuss.
+//
+// PARTIALLY DEPRECATED: The Logger(module) form is deprecated and
+// may be removed in the future. Do not pass in any arguments.
+func (ctx Context) Logger(module ...Module) *zap.Logger {
+	if len(module) > 1 {
+		panic("more than 1 module passed in")
+	}
 	if ctx.cfg == nil {
 		// often the case in tests; just use a dev logger
 		l, err := zap.NewDevelopment()
@@ -454,23 +471,13 @@ func (ctx Context) Logger(mod Module) *zap.Logger {
 		}
 		return l
 	}
+	mod := ctx.Module()
+	if len(module) > 0 {
+		mod = module[0]
+	}
 	return ctx.cfg.Logging.Logger(mod)
 }
 
-// TODO: use this
-// // Logger returns a logger that can be used by the current module.
-// func (ctx Context) Log() *zap.Logger {
-// 	if ctx.cfg == nil {
-// 		// often the case in tests; just use a dev logger
-// 		l, err := zap.NewDevelopment()
-// 		if err != nil {
-// 			panic("config missing, unable to create dev logger: " + err.Error())
-// 		}
-// 		return l
-// 	}
-// 	return ctx.cfg.Logging.Logger(ctx.Module())
-// }
-
 // Modules returns the lineage of modules that this context provisioned,
 // with the most recent/current module being last in the list.
 func (ctx Context) Modules() []Module {

go.mod

@@ -3,69 +3,79 @@ module github.com/caddyserver/caddy/v2
 go 1.18
 
 require (
-	github.com/BurntSushi/toml v1.2.0
-	github.com/Masterminds/sprig/v3 v3.2.2
-	github.com/alecthomas/chroma v0.10.0
+	github.com/BurntSushi/toml v1.2.1
+	github.com/Masterminds/sprig/v3 v3.2.3
+	github.com/alecthomas/chroma/v2 v2.5.0
 	github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b
-	github.com/caddyserver/certmagic v0.17.1
-	github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac
+	github.com/caddyserver/certmagic v0.17.2
+	github.com/dustin/go-humanize v1.0.1
 	github.com/go-chi/chi v4.1.2+incompatible
-	github.com/google/cel-go v0.12.4
+	github.com/google/cel-go v0.13.0
 	github.com/google/uuid v1.3.0
-	github.com/klauspost/compress v1.15.9
-	github.com/klauspost/cpuid/v2 v2.1.0
-	github.com/lucas-clemente/quic-go v0.28.2-0.20220813150001-9957668d4301
+	github.com/klauspost/compress v1.15.15
+	github.com/klauspost/cpuid/v2 v2.2.3
 	github.com/mholt/acmez v1.0.4
-	github.com/prometheus/client_golang v1.12.2
-	github.com/smallstep/certificates v0.21.0
-	github.com/smallstep/cli v0.21.0
-	github.com/smallstep/nosql v0.4.0
-	github.com/smallstep/truststore v0.11.0
-	github.com/spf13/cobra v1.1.3
+	github.com/prometheus/client_golang v1.14.0
+	github.com/quic-go/quic-go v0.32.0
+	github.com/smallstep/certificates v0.23.2
+	github.com/smallstep/nosql v0.5.0
+	github.com/smallstep/truststore v0.12.1
+	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
-	github.com/tailscale/tscert v0.0.0-20220316030059-54bbcb9f74e2
-	github.com/yuin/goldmark v1.4.13
-	github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.34.0
-	go.opentelemetry.io/otel v1.9.0
+	github.com/tailscale/tscert v0.0.0-20230124224810-c6dc1f4049b2
+	github.com/yuin/goldmark v1.5.4
+	github.com/yuin/goldmark-highlighting/v2 v2.0.0-20220924101305-151362477c87
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.39.0
+	go.opentelemetry.io/otel v1.13.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.4.0
-	go.opentelemetry.io/otel/sdk v1.4.0
-	go.uber.org/zap v1.21.0
-	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
-	golang.org/x/net v0.0.0-20220812165438-1d4ff48094d1
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
-	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0
+	go.opentelemetry.io/otel/sdk v1.13.0
+	go.uber.org/zap v1.24.0
+	golang.org/x/crypto v0.5.0
+	golang.org/x/net v0.5.0
+	golang.org/x/sync v0.1.0
+	golang.org/x/term v0.5.0
+	google.golang.org/genproto v0.0.0-20230202175211-008b39050e57
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fxamacker/cbor/v2 v2.4.0 // indirect
+	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/mock v1.6.0 // indirect
-	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
+	github.com/onsi/ginkgo/v2 v2.2.0 // indirect
+	github.com/quic-go/qpack v0.4.0 // indirect
+	github.com/quic-go/qtls-go1-18 v0.2.0 // indirect
+	github.com/quic-go/qtls-go1-19 v0.2.0 // indirect
+	github.com/quic-go/qtls-go1-20 v0.1.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
 )
 
 require (
-	filippo.io/edwards25519 v1.0.0-rc.1 // indirect
+	filippo.io/edwards25519 v1.0.0 // indirect
 	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
-	github.com/Masterminds/semver/v3 v3.1.1 // indirect
-	github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed // indirect
+	github.com/Masterminds/semver/v3 v3.2.0 // indirect
+	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/dgraph-io/badger v1.6.2 // indirect
 	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
-	github.com/dgraph-io/ristretto v0.0.4-0.20200906165740-41ebdbffecfd // indirect
+	github.com/dgraph-io/ristretto v0.1.0 // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
-	github.com/dlclark/regexp2 v1.4.0 // indirect
+	github.com/dlclark/regexp2 v1.7.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-kit/kit v0.10.0 // indirect
-	github.com/go-logfmt/logfmt v0.5.0 // indirect
+	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-sql-driver/mysql v1.6.0 // indirect
@@ -73,22 +83,19 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
-	github.com/huandu/xstrings v1.3.2 // indirect
+	github.com/huandu/xstrings v1.3.3 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
-	github.com/jackc/pgconn v1.10.1 // indirect
+	github.com/jackc/pgconn v1.13.0 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.1 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
-	github.com/jackc/pgtype v1.9.0 // indirect
-	github.com/jackc/pgx/v4 v4.14.0 // indirect
+	github.com/jackc/pgtype v1.12.0 // indirect
+	github.com/jackc/pgx/v4 v4.17.2 // indirect
 	github.com/libdns/libdns v0.2.1 // indirect
 	github.com/manifoldco/promptui v0.9.0 // indirect
-	github.com/marten-seemann/qpack v0.2.1 // indirect
-	github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
-	github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.8 // indirect
 	github.com/mattn/go-isatty v0.0.13 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
@@ -98,41 +105,37 @@ require (
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-ps v1.0.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
-	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/onsi/ginkgo v1.16.4 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.32.1 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/rs/xid v1.2.1 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/rs/xid v1.4.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
-	github.com/slackhq/nebula v1.5.2 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/slackhq/nebula v1.6.1 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect
-	github.com/urfave/cli v1.22.5 // indirect
+	github.com/urfave/cli v1.22.12 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
 	go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.0 // indirect
-	go.opentelemetry.io/otel/metric v0.31.0 // indirect
-	go.opentelemetry.io/otel/trace v1.9.0 // indirect
+	go.opentelemetry.io/otel/metric v0.36.0 // indirect
+	go.opentelemetry.io/otel/trace v1.13.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.12.0 // indirect
-	go.step.sm/cli-utils v0.7.3 // indirect
-	go.step.sm/crypto v0.16.2 // indirect
-	go.step.sm/linkedca v0.16.1 // indirect
+	go.step.sm/cli-utils v0.7.5 // indirect
+	go.step.sm/crypto v0.23.2
+	go.step.sm/linkedca v0.19.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10
-	golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect
-	golang.org/x/tools v0.1.10 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/grpc v1.46.0 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	golang.org/x/mod v0.6.0 // indirect
+	golang.org/x/sys v0.5.0
+	golang.org/x/text v0.6.0 // indirect
+	golang.org/x/tools v0.2.0 // indirect
+	google.golang.org/grpc v1.52.3 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	howett.net/plist v1.0.0 // indirect
 )

listen.go

@@ -1,9 +1,26 @@
-//go:build !linux
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// TODO: Go 1.19 introduced the "unix" build tag. We have to support Go 1.18 until Go 1.20 is released.
+// When Go 1.19 is our minimum, change this build tag to simply "!unix".
+// (see similar change needed in listen_unix.go)
+//go:build !(aix || android || darwin || dragonfly || freebsd || hurd || illumos || ios || linux || netbsd || openbsd || solaris)
 
 package caddy
 
 import (
-	"fmt"
+	"context"
 	"net"
 	"sync"
 	"sync/atomic"
@@ -12,21 +29,14 @@ import (
 	"go.uber.org/zap"
 )
 
-func ListenTimeout(network, addr string, keepAlivePeriod time.Duration) (net.Listener, error) {
-	// check to see if plugin provides listener
-	if ln, err := getListenerFromPlugin(network, addr); err != nil || ln != nil {
-		return ln, err
-	}
-
-	lnKey := listenerKey(network, addr)
+func reuseUnixSocket(network, addr string) (any, error) {
+	return nil, nil
+}
 
+func listenTCPOrUnix(ctx context.Context, lnKey string, network, address string, config net.ListenConfig) (net.Listener, error) {
 	sharedLn, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
-		ln, err := net.Listen(network, addr)
+		ln, err := config.Listen(ctx, network, address)
 		if err != nil {
-			// https://github.com/caddyserver/caddy/pull/4534
-			if isUnixNetwork(network) && isListenBindAddressAlreadyInUseError(err) {
-				return nil, fmt.Errorf("%w: this can happen if Caddy was forcefully killed", err)
-			}
 			return nil, err
 		}
 		return &sharedListener{Listener: ln, key: lnKey}, nil
@@ -34,8 +44,7 @@ func ListenTimeout(network, addr string, keepAlivePeriod time.Duration) (net.Lis
 	if err != nil {
 		return nil, err
 	}
-
-	return &fakeCloseListener{sharedListener: sharedLn.(*sharedListener), keepAlivePeriod: keepAlivePeriod}, nil
+	return &fakeCloseListener{sharedListener: sharedLn.(*sharedListener), keepAlivePeriod: config.KeepAlive}, nil
 }
 
 // fakeCloseListener is a private wrapper over a listener that
@@ -139,8 +148,6 @@ func (sl *sharedListener) clearDeadline() error {
 		switch ln := sl.Listener.(type) {
 		case *net.TCPListener:
 			err = ln.SetDeadline(time.Time{})
-		case *net.UnixListener:
-			err = ln.SetDeadline(time.Time{})
 		}
 		sl.deadline = false
 	}
@@ -156,8 +163,6 @@ func (sl *sharedListener) setDeadline() error {
 		switch ln := sl.Listener.(type) {
 		case *net.TCPListener:
 			err = ln.SetDeadline(timeInPast)
-		case *net.UnixListener:
-			err = ln.SetDeadline(timeInPast)
 		}
 		sl.deadline = true
 	}

listen_linux.go

@@ -1,34 +0,0 @@
-package caddy
-
-import (
-	"context"
-	"net"
-	"syscall"
-	"time"
-
-	"go.uber.org/zap"
-	"golang.org/x/sys/unix"
-)
-
-// ListenTimeout is the same as Listen, but with a configurable keep-alive timeout duration.
-func ListenTimeout(network, addr string, keepalivePeriod time.Duration) (net.Listener, error) {
-	// check to see if plugin provides listener
-	if ln, err := getListenerFromPlugin(network, addr); err != nil || ln != nil {
-		return ln, err
-	}
-
-	config := &net.ListenConfig{Control: reusePort, KeepAlive: keepalivePeriod}
-	return config.Listen(context.Background(), network, addr)
-}
-
-func reusePort(network, address string, conn syscall.RawConn) error {
-	return conn.Control(func(descriptor uintptr) {
-		if err := unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
-			Log().Error("setting SO_REUSEPORT",
-				zap.String("network", network),
-				zap.String("address", address),
-				zap.Uintptr("descriptor", descriptor),
-				zap.Error(err))
-		}
-	})
-}

listen_unix.go

@@ -0,0 +1,118 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// TODO: Go 1.19 introduced the "unix" build tag. We have to support Go 1.18 until Go 1.20 is released.
+// When Go 1.19 is our minimum, remove this build tag, since "_unix" in the filename will do this.
+// (see also change needed in listen.go)
+//go:build aix || android || darwin || dragonfly || freebsd || hurd || illumos || ios || linux || netbsd || openbsd || solaris
+
+package caddy
+
+import (
+	"context"
+	"errors"
+	"io/fs"
+	"net"
+	"sync/atomic"
+	"syscall"
+
+	"go.uber.org/zap"
+	"golang.org/x/sys/unix"
+)
+
+// reuseUnixSocket copies and reuses the unix domain socket (UDS) if we already
+// have it open; if not, unlink it so we can have it. No-op if not a unix network.
+func reuseUnixSocket(network, addr string) (any, error) {
+	if !IsUnixNetwork(network) {
+		return nil, nil
+	}
+
+	socketKey := listenerKey(network, addr)
+
+	socket, exists := unixSockets[socketKey]
+	if exists {
+		// make copy of file descriptor
+		socketFile, err := socket.File() // does dup() deep down
+		if err != nil {
+			return nil, err
+		}
+
+		// use copied fd to make new Listener or PacketConn, then replace
+		// it in the map so that future copies always come from the most
+		// recent fd (as the previous ones will be closed, and we'd get
+		// "use of closed network connection" errors) -- note that we
+		// preserve the *pointer* to the counter (not just the value) so
+		// that all socket wrappers will refer to the same value
+		switch unixSocket := socket.(type) {
+		case *unixListener:
+			ln, err := net.FileListener(socketFile)
+			if err != nil {
+				return nil, err
+			}
+			atomic.AddInt32(unixSocket.count, 1)
+			unixSockets[socketKey] = &unixListener{ln.(*net.UnixListener), socketKey, unixSocket.count}
+
+		case *unixConn:
+			pc, err := net.FilePacketConn(socketFile)
+			if err != nil {
+				return nil, err
+			}
+			atomic.AddInt32(unixSocket.count, 1)
+			unixSockets[socketKey] = &unixConn{pc.(*net.UnixConn), addr, socketKey, unixSocket.count}
+		}
+
+		return unixSockets[socketKey], nil
+	}
+
+	// from what I can tell after some quick research, it's quite common for programs to
+	// leave their socket file behind after they close, so the typical pattern is to
+	// unlink it before you bind to it -- this is often crucial if the last program using
+	// it was killed forcefully without a chance to clean up the socket, but there is a
+	// race, as the comment in net.UnixListener.close() explains... oh well, I guess?
+	if err := syscall.Unlink(addr); err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return nil, err
+	}
+
+	return nil, nil
+}
+
+func listenTCPOrUnix(ctx context.Context, lnKey string, network, address string, config net.ListenConfig) (net.Listener, error) {
+	// wrap any Control function set by the user so we can also add our reusePort control without clobbering theirs
+	oldControl := config.Control
+	config.Control = func(network, address string, c syscall.RawConn) error {
+		if oldControl != nil {
+			if err := oldControl(network, address, c); err != nil {
+				return err
+			}
+		}
+		return reusePort(network, address, c)
+	}
+	return config.Listen(ctx, network, address)
+}
+
+// reusePort sets SO_REUSEPORT. Ineffective for unix sockets.
+func reusePort(network, address string, conn syscall.RawConn) error {
+	if IsUnixNetwork(network) {
+		return nil
+	}
+	return conn.Control(func(descriptor uintptr) {
+		if err := unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
+			Log().Error("setting SO_REUSEPORT",
+				zap.String("network", network),
+				zap.String("address", address),
+				zap.Uintptr("descriptor", descriptor),
+				zap.Error(err))
+		}
+	})
+}

listeners.go

@@ -19,231 +19,193 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"net/netip"
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"syscall"
+	"time"
 
-	"github.com/lucas-clemente/quic-go"
-	"github.com/lucas-clemente/quic-go/http3"
+	"github.com/quic-go/quic-go"
+	"github.com/quic-go/quic-go/http3"
 	"go.uber.org/zap"
 )
 
-// Listen is like net.Listen, except Caddy's listeners can overlap
-// each other: multiple listeners may be created on the same socket
-// at the same time. This is useful because during config changes,
-// the new config is started while the old config is still running.
-// When Caddy listeners are closed, the closing logic is virtualized
-// so the underlying socket isn't actually closed until all uses of
-// the socket have been finished. Always be sure to close listeners
-// when you are done with them, just like normal listeners.
-func Listen(network, addr string) (net.Listener, error) {
-	// a 0 timeout means Go uses its default
-	return ListenTimeout(network, addr, 0)
+// NetworkAddress represents one or more network addresses.
+// It contains the individual components for a parsed network
+// address of the form accepted by ParseNetworkAddress().
+type NetworkAddress struct {
+	// Should be a network value accepted by Go's net package or
+	// by a plugin providing a listener for that network type.
+	Network string
+
+	// The "main" part of the network address is the host, which
+	// often takes the form of a hostname, DNS name, IP address,
+	// or socket path.
+	Host string
+
+	// For addresses that contain a port, ranges are given by
+	// [StartPort, EndPort]; i.e. for a single port, StartPort
+	// and EndPort are the same. For no port, they are 0.
+	StartPort uint
+	EndPort   uint
 }
 
-// getListenerFromPlugin returns a listener on the given network and address
-// if a plugin has registered the network name. It may return (nil, nil) if
-// no plugin can provide a listener.
-func getListenerFromPlugin(network, addr string) (net.Listener, error) {
-	network = strings.TrimSpace(strings.ToLower(network))
+// ListenAll calls Listen() for all addresses represented by this struct, i.e. all ports in the range.
+// (If the address doesn't use ports or has 1 port only, then only 1 listener will be created.)
+// It returns an error if any listener failed to bind, and closes any listeners opened up to that point.
+//
+// TODO: Experimental API: subject to change or removal.
+func (na NetworkAddress) ListenAll(ctx context.Context, config net.ListenConfig) ([]any, error) {
+	var listeners []any
+	var err error
 
-	// get listener from plugin if network type is registered
-	if getListener, ok := networkTypes[network]; ok {
-		Log().Debug("getting listener from plugin", zap.String("network", network))
-		return getListener(network, addr)
-	}
-
-	return nil, nil
-}
-
-// ListenPacket returns a net.PacketConn suitable for use in a Caddy module.
-// It is like Listen except for PacketConns.
-// Always be sure to close the PacketConn when you are done.
-func ListenPacket(network, addr string) (net.PacketConn, error) {
-	lnKey := listenerKey(network, addr)
-
-	sharedPc, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
-		pc, err := net.ListenPacket(network, addr)
-		if err != nil {
-			// https://github.com/caddyserver/caddy/pull/4534
-			if isUnixNetwork(network) && isListenBindAddressAlreadyInUseError(err) {
-				return nil, fmt.Errorf("%w: this can happen if Caddy was forcefully killed", err)
+	// if one of the addresses has a failure, we need to close
+	// any that did open a socket to avoid leaking resources
+	defer func() {
+		if err == nil {
+			return
+		}
+		for _, ln := range listeners {
+			if cl, ok := ln.(io.Closer); ok {
+				cl.Close()
 			}
+		}
+	}()
+
+	// an address can contain a port range, which represents multiple addresses;
+	// some addresses don't use ports at all and have a port range size of 1;
+	// whatever the case, iterate each address represented and bind a socket
+	for portOffset := uint(0); portOffset < na.PortRangeSize(); portOffset++ {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+		}
+
+		// create (or reuse) the listener ourselves
+		var ln any
+		ln, err = na.Listen(ctx, portOffset, config)
+		if err != nil {
 			return nil, err
 		}
-		return &sharedPacketConn{PacketConn: pc, key: lnKey}, nil
-	})
-	if err != nil {
-		return nil, err
+		listeners = append(listeners, ln)
 	}
 
-	return &fakeClosePacketConn{sharedPacketConn: sharedPc.(*sharedPacketConn)}, nil
+	return listeners, nil
 }
 
-// ListenQUIC returns a quic.EarlyListener suitable for use in a Caddy module.
-// Note that the context passed to Accept is currently ignored, so using
-// a context other than context.Background is meaningless.
-func ListenQUIC(addr string, tlsConf *tls.Config, activeRequests *int64) (quic.EarlyListener, error) {
-	lnKey := listenerKey("udp", addr)
+// Listen is similar to net.Listen, with a few differences:
+//
+// Listen announces on the network address using the port calculated by adding
+// portOffset to the start port. (For network types that do not use ports, the
+// portOffset is ignored.)
+//
+// The provided ListenConfig is used to create the listener. Its Control function,
+// if set, may be wrapped by an internally-used Control function. The provided
+// context may be used to cancel long operations early. The context is not used
+// to close the listener after it has been created.
+//
+// Caddy's listeners can overlap each other: multiple listeners may be created on
+// the same socket at the same time. This is useful because during config changes,
+// the new config is started while the old config is still running. How this is
+// accomplished varies by platform and network type. For example, on Unix, SO_REUSEPORT
+// is set except on Unix sockets, for which the file descriptor is duplicated and
+// reused; on Windows, the close logic is virtualized using timeouts. Like normal
+// listeners, be sure to Close() them when you are done.
+//
+// This method returns any type, as the implementations of listeners for various
+// network types are not interchangeable. The type of listener returned is switched
+// on the network type. Stream-based networks ("tcp", "unix", "unixpacket", etc.)
+// return a net.Listener; datagram-based networks ("udp", "unixgram", etc.) return
+// a net.PacketConn; and so forth. The actual concrete types are not guaranteed to
+// be standard, exported types (wrapping is necessary to provide graceful reloads).
+//
+// Unix sockets will be unlinked before being created, to ensure we can bind to
+// it even if the previous program using it exited uncleanly; it will also be
+// unlinked upon a graceful exit (or when a new config does not use that socket).
+//
+// TODO: Experimental API: subject to change or removal.
+func (na NetworkAddress) Listen(ctx context.Context, portOffset uint, config net.ListenConfig) (any, error) {
+	if na.IsUnixNetwork() {
+		unixSocketsMu.Lock()
+		defer unixSocketsMu.Unlock()
+	}
 
-	sharedEl, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
-		el, err := quic.ListenAddrEarly(addr, http3.ConfigureTLSConfig(tlsConf), &quic.Config{
-			RequireAddressValidation: func(clientAddr net.Addr) bool {
-				var highLoad bool
-				if activeRequests != nil {
-					highLoad = atomic.LoadInt64(activeRequests) > 1000 // TODO: make tunable?
-				}
-				return highLoad
-			},
+	// check to see if plugin provides listener
+	if ln, err := getListenerFromPlugin(ctx, na.Network, na.JoinHostPort(portOffset), config); ln != nil || err != nil {
+		return ln, err
+	}
+
+	// create (or reuse) the listener ourselves
+	return na.listen(ctx, portOffset, config)
+}
+
+func (na NetworkAddress) listen(ctx context.Context, portOffset uint, config net.ListenConfig) (any, error) {
+	var ln any
+	var err error
+
+	address := na.JoinHostPort(portOffset)
+
+	// if this is a unix socket, see if we already have it open
+	if socket, err := reuseUnixSocket(na.Network, address); socket != nil || err != nil {
+		return socket, err
+	}
+
+	lnKey := listenerKey(na.Network, address)
+
+	switch na.Network {
+	case "tcp", "tcp4", "tcp6", "unix", "unixpacket":
+		ln, err = listenTCPOrUnix(ctx, lnKey, na.Network, address, config)
+	case "unixgram":
+		ln, err = config.ListenPacket(ctx, na.Network, address)
+	case "udp", "udp4", "udp6":
+		sharedPc, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
+			pc, err := config.ListenPacket(ctx, na.Network, address)
+			if err != nil {
+				return nil, err
+			}
+			return &sharedPacketConn{PacketConn: pc, key: lnKey}, nil
 		})
 		if err != nil {
 			return nil, err
 		}
-		return &sharedQuicListener{EarlyListener: el, key: lnKey}, nil
-	})
-
-	ctx, cancel := context.WithCancel(context.Background())
-	return &fakeCloseQuicListener{
-		sharedQuicListener: sharedEl.(*sharedQuicListener),
-		context:            ctx, contextCancel: cancel,
-	}, err
-}
-
-// ListenerUsage returns the current usage count of the given listener address.
-func ListenerUsage(network, addr string) int {
-	count, _ := listenerPool.References(listenerKey(network, addr))
-	return count
-}
-
-func listenerKey(network, addr string) string {
-	return network + "/" + addr
-}
-
-type fakeCloseQuicListener struct {
-	closed              int32 // accessed atomically; belongs to this struct only
-	*sharedQuicListener       // embedded, so we also become a quic.EarlyListener
-	context             context.Context
-	contextCancel       context.CancelFunc
-}
-
-// Currently Accept ignores the passed context, however a situation where
-// someone would need a hotswappable QUIC-only (not http3, since it uses context.Background here)
-// server on which Accept would be called with non-empty contexts
-// (mind that the default net listeners' Accept doesn't take a context argument)
-// sounds way too rare for us to sacrifice efficiency here.
-func (fcql *fakeCloseQuicListener) Accept(_ context.Context) (quic.EarlyConnection, error) {
-	conn, err := fcql.sharedQuicListener.Accept(fcql.context)
-	if err == nil {
-		return conn, nil
+		ln = &fakeClosePacketConn{sharedPacketConn: sharedPc.(*sharedPacketConn)}
+	}
+	if strings.HasPrefix(na.Network, "ip") {
+		ln, err = config.ListenPacket(ctx, na.Network, address)
+	}
+	if err != nil {
+		return nil, err
+	}
+	if ln == nil {
+		return nil, fmt.Errorf("unsupported network type: %s", na.Network)
 	}
 
-	// if the listener is "closed", return a fake closed error instead
-	if atomic.LoadInt32(&fcql.closed) == 1 && errors.Is(err, context.Canceled) {
-		return nil, fakeClosedErr(fcql)
+	// if new listener is a unix socket, make sure we can reuse it later
+	// (we do our own "unlink on close" -- not required, but more tidy)
+	one := int32(1)
+	switch unix := ln.(type) {
+	case *net.UnixListener:
+		unix.SetUnlinkOnClose(false)
+		ln = &unixListener{unix, lnKey, &one}
+		unixSockets[lnKey] = ln.(*unixListener)
+	case *net.UnixConn:
+		ln = &unixConn{unix, address, lnKey, &one}
+		unixSockets[lnKey] = ln.(*unixConn)
 	}
-	return nil, err
-}
 
-func (fcql *fakeCloseQuicListener) Close() error {
-	if atomic.CompareAndSwapInt32(&fcql.closed, 0, 1) {
-		fcql.contextCancel()
-		_, _ = listenerPool.Delete(fcql.sharedQuicListener.key)
-	}
-	return nil
-}
-
-// fakeClosedErr returns an error value that is not temporary
-// nor a timeout, suitable for making the caller think the
-// listener is actually closed
-func fakeClosedErr(l interface{ Addr() net.Addr }) error {
-	return &net.OpError{
-		Op:   "accept",
-		Net:  l.Addr().Network(),
-		Addr: l.Addr(),
-		Err:  errFakeClosed,
-	}
-}
-
-// ErrFakeClosed is the underlying error value returned by
-// fakeCloseListener.Accept() after Close() has been called,
-// indicating that it is pretending to be closed so that the
-// server using it can terminate, while the underlying
-// socket is actually left open.
-var errFakeClosed = fmt.Errorf("listener 'closed' 😉")
-
-// fakeClosePacketConn is like fakeCloseListener, but for PacketConns.
-type fakeClosePacketConn struct {
-	closed            int32 // accessed atomically; belongs to this struct only
-	*sharedPacketConn       // embedded, so we also become a net.PacketConn
-}
-
-func (fcpc *fakeClosePacketConn) Close() error {
-	if atomic.CompareAndSwapInt32(&fcpc.closed, 0, 1) {
-		_, _ = listenerPool.Delete(fcpc.sharedPacketConn.key)
-	}
-	return nil
-}
-
-// Supports QUIC implementation: https://github.com/caddyserver/caddy/issues/3998
-func (fcpc fakeClosePacketConn) SetReadBuffer(bytes int) error {
-	if conn, ok := fcpc.PacketConn.(interface{ SetReadBuffer(int) error }); ok {
-		return conn.SetReadBuffer(bytes)
-	}
-	return fmt.Errorf("SetReadBuffer() not implemented for %T", fcpc.PacketConn)
-}
-
-// Supports QUIC implementation: https://github.com/caddyserver/caddy/issues/3998
-func (fcpc fakeClosePacketConn) SyscallConn() (syscall.RawConn, error) {
-	if conn, ok := fcpc.PacketConn.(interface {
-		SyscallConn() (syscall.RawConn, error)
-	}); ok {
-		return conn.SyscallConn()
-	}
-	return nil, fmt.Errorf("SyscallConn() not implemented for %T", fcpc.PacketConn)
-}
-
-// sharedQuicListener is like sharedListener, but for quic.EarlyListeners.
-type sharedQuicListener struct {
-	quic.EarlyListener
-	key string
-}
-
-// Destruct closes the underlying QUIC listener.
-func (sql *sharedQuicListener) Destruct() error {
-	return sql.EarlyListener.Close()
-}
-
-// sharedPacketConn is like sharedListener, but for net.PacketConns.
-type sharedPacketConn struct {
-	net.PacketConn
-	key string
-}
-
-// Destruct closes the underlying socket.
-func (spc *sharedPacketConn) Destruct() error {
-	return spc.PacketConn.Close()
-}
-
-// NetworkAddress contains the individual components
-// for a parsed network address of the form accepted
-// by ParseNetworkAddress(). Network should be a
-// network value accepted by Go's net package. Port
-// ranges are given by [StartPort, EndPort].
-type NetworkAddress struct {
-	Network   string
-	Host      string
-	StartPort uint
-	EndPort   uint
+	return ln, nil
 }
 
 // IsUnixNetwork returns true if na.Network is
 // unix, unixgram, or unixpacket.
 func (na NetworkAddress) IsUnixNetwork() bool {
-	return isUnixNetwork(na.Network)
+	return IsUnixNetwork(na.Network)
 }
 
 // JoinHostPort is like net.JoinHostPort, but where the port
@@ -255,17 +217,27 @@ func (na NetworkAddress) JoinHostPort(offset uint) string {
 	return net.JoinHostPort(na.Host, strconv.Itoa(int(na.StartPort+offset)))
 }
 
+// Expand returns one NetworkAddress for each port in the port range.
+//
+// This is EXPERIMENTAL and subject to change or removal.
 func (na NetworkAddress) Expand() []NetworkAddress {
 	size := na.PortRangeSize()
 	addrs := make([]NetworkAddress, size)
 	for portOffset := uint(0); portOffset < size; portOffset++ {
-		na2 := na
-		na2.StartPort, na2.EndPort = na.StartPort+portOffset, na.StartPort+portOffset
-		addrs[portOffset] = na2
+		addrs[portOffset] = na.At(portOffset)
 	}
 	return addrs
 }
 
+// At returns a NetworkAddress with a port range of just 1
+// at the given port offset; i.e. a NetworkAddress that
+// represents precisely 1 address only.
+func (na NetworkAddress) At(portOffset uint) NetworkAddress {
+	na2 := na
+	na2.StartPort, na2.EndPort = na.StartPort+portOffset, na.StartPort+portOffset
+	return na2
+}
+
 // PortRangeSize returns how many ports are in
 // pa's port range. Port ranges are inclusive,
 // so the size is the difference of start and
@@ -317,22 +289,9 @@ func (na NetworkAddress) String() string {
 	return JoinNetworkAddress(na.Network, na.Host, na.port())
 }
 
-func isUnixNetwork(netw string) bool {
-	return netw == "unix" || netw == "unixgram" || netw == "unixpacket"
-}
-
-func isListenBindAddressAlreadyInUseError(err error) bool {
-	switch networkOperationError := err.(type) {
-	case *net.OpError:
-		switch syscallError := networkOperationError.Err.(type) {
-		case *os.SyscallError:
-			if syscallError.Syscall == "bind" {
-				return true
-			}
-		}
-	}
-
-	return false
+// IsUnixNetwork returns true if the netw is a unix network.
+func IsUnixNetwork(netw string) bool {
+	return strings.HasPrefix(netw, "unix")
 }
 
 // ParseNetworkAddress parses addr into its individual
@@ -352,7 +311,7 @@ func ParseNetworkAddress(addr string) (NetworkAddress, error) {
 	if network == "" {
 		network = "tcp"
 	}
-	if isUnixNetwork(network) {
+	if IsUnixNetwork(network) {
 		return NetworkAddress{
 			Network: network,
 			Host:    host,
@@ -395,7 +354,7 @@ func SplitNetworkAddress(a string) (network, host, port string, err error) {
 		network = strings.ToLower(strings.TrimSpace(beforeSlash))
 		a = afterSlash
 	}
-	if isUnixNetwork(network) {
+	if IsUnixNetwork(network) {
 		host = a
 		return
 	}
@@ -426,7 +385,7 @@ func JoinNetworkAddress(network, host, port string) string {
 	if network != "" {
 		a = network + "/"
 	}
-	if (host != "" && port == "") || isUnixNetwork(network) {
+	if (host != "" && port == "") || IsUnixNetwork(network) {
 		a += host
 	} else if port != "" {
 		a += net.JoinHostPort(host, port)
@@ -434,6 +393,208 @@ func JoinNetworkAddress(network, host, port string) string {
 	return a
 }
 
+// DEPRECATED: Use NetworkAddress.Listen instead. This function will likely be changed or removed in the future.
+func Listen(network, addr string) (net.Listener, error) {
+	// a 0 timeout means Go uses its default
+	return ListenTimeout(network, addr, 0)
+}
+
+// DEPRECATED: Use NetworkAddress.Listen instead. This function will likely be changed or removed in the future.
+func ListenTimeout(network, addr string, keepalivePeriod time.Duration) (net.Listener, error) {
+	netAddr, err := ParseNetworkAddress(JoinNetworkAddress(network, addr, ""))
+	if err != nil {
+		return nil, err
+	}
+
+	ln, err := netAddr.Listen(context.TODO(), 0, net.ListenConfig{KeepAlive: keepalivePeriod})
+	if err != nil {
+		return nil, err
+	}
+
+	return ln.(net.Listener), nil
+}
+
+// DEPRECATED: Use NetworkAddress.Listen instead. This function will likely be changed or removed in the future.
+func ListenPacket(network, addr string) (net.PacketConn, error) {
+	netAddr, err := ParseNetworkAddress(JoinNetworkAddress(network, addr, ""))
+	if err != nil {
+		return nil, err
+	}
+
+	ln, err := netAddr.Listen(context.TODO(), 0, net.ListenConfig{})
+	if err != nil {
+		return nil, err
+	}
+
+	return ln.(net.PacketConn), nil
+}
+
+// ListenQUIC returns a quic.EarlyListener suitable for use in a Caddy module.
+// The network will be transformed into a QUIC-compatible type (if unix, then
+// unixgram will be used; otherwise, udp will be used).
+//
+// NOTE: This API is EXPERIMENTAL and may be changed or removed.
+//
+// TODO: See if we can find a more elegant solution closer to the new NetworkAddress.Listen API.
+func ListenQUIC(ln net.PacketConn, tlsConf *tls.Config, activeRequests *int64) (quic.EarlyListener, error) {
+	lnKey := listenerKey("quic+"+ln.LocalAddr().Network(), ln.LocalAddr().String())
+
+	sharedEarlyListener, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
+		earlyLn, err := quic.ListenEarly(ln, http3.ConfigureTLSConfig(tlsConf), &quic.Config{
+			RequireAddressValidation: func(clientAddr net.Addr) bool {
+				var highLoad bool
+				if activeRequests != nil {
+					highLoad = atomic.LoadInt64(activeRequests) > 1000 // TODO: make tunable?
+				}
+				return highLoad
+			},
+		})
+		if err != nil {
+			return nil, err
+		}
+		return &sharedQuicListener{EarlyListener: earlyLn, key: lnKey}, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: to serve QUIC over a unix socket, currently we need to hold onto
+	// the underlying net.PacketConn (which we wrap as unixConn to keep count
+	// of closes) because closing the quic.EarlyListener doesn't actually close
+	// the underlying PacketConn, but we need to for unix sockets since we dup
+	// the file descriptor and thus need to close the original; track issue:
+	// https://github.com/quic-go/quic-go/issues/3560#issuecomment-1258959608
+	var unix *unixConn
+	if uc, ok := ln.(*unixConn); ok {
+		unix = uc
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	return &fakeCloseQuicListener{
+		sharedQuicListener: sharedEarlyListener.(*sharedQuicListener),
+		uc:                 unix,
+		context:            ctx,
+		contextCancel:      cancel,
+	}, nil
+}
+
+// ListenerUsage returns the current usage count of the given listener address.
+func ListenerUsage(network, addr string) int {
+	count, _ := listenerPool.References(listenerKey(network, addr))
+	return count
+}
+
+// sharedQuicListener is like sharedListener, but for quic.EarlyListeners.
+type sharedQuicListener struct {
+	quic.EarlyListener
+	key string
+}
+
+// Destruct closes the underlying QUIC listener.
+func (sql *sharedQuicListener) Destruct() error {
+	return sql.EarlyListener.Close()
+}
+
+// sharedPacketConn is like sharedListener, but for net.PacketConns.
+type sharedPacketConn struct {
+	net.PacketConn
+	key string
+}
+
+// Destruct closes the underlying socket.
+func (spc *sharedPacketConn) Destruct() error {
+	return spc.PacketConn.Close()
+}
+
+// fakeClosedErr returns an error value that is not temporary
+// nor a timeout, suitable for making the caller think the
+// listener is actually closed
+func fakeClosedErr(l interface{ Addr() net.Addr }) error {
+	return &net.OpError{
+		Op:   "accept",
+		Net:  l.Addr().Network(),
+		Addr: l.Addr(),
+		Err:  errFakeClosed,
+	}
+}
+
+// errFakeClosed is the underlying error value returned by
+// fakeCloseListener.Accept() after Close() has been called,
+// indicating that it is pretending to be closed so that the
+// server using it can terminate, while the underlying
+// socket is actually left open.
+var errFakeClosed = fmt.Errorf("listener 'closed' 😉")
+
+// fakeClosePacketConn is like fakeCloseListener, but for PacketConns.
+type fakeClosePacketConn struct {
+	closed            int32 // accessed atomically; belongs to this struct only
+	*sharedPacketConn       // embedded, so we also become a net.PacketConn
+}
+
+func (fcpc *fakeClosePacketConn) Close() error {
+	if atomic.CompareAndSwapInt32(&fcpc.closed, 0, 1) {
+		_, _ = listenerPool.Delete(fcpc.sharedPacketConn.key)
+	}
+	return nil
+}
+
+// Supports QUIC implementation: https://github.com/caddyserver/caddy/issues/3998
+func (fcpc fakeClosePacketConn) SetReadBuffer(bytes int) error {
+	if conn, ok := fcpc.PacketConn.(interface{ SetReadBuffer(int) error }); ok {
+		return conn.SetReadBuffer(bytes)
+	}
+	return fmt.Errorf("SetReadBuffer() not implemented for %T", fcpc.PacketConn)
+}
+
+// Supports QUIC implementation: https://github.com/caddyserver/caddy/issues/3998
+func (fcpc fakeClosePacketConn) SyscallConn() (syscall.RawConn, error) {
+	if conn, ok := fcpc.PacketConn.(interface {
+		SyscallConn() (syscall.RawConn, error)
+	}); ok {
+		return conn.SyscallConn()
+	}
+	return nil, fmt.Errorf("SyscallConn() not implemented for %T", fcpc.PacketConn)
+}
+
+type fakeCloseQuicListener struct {
+	closed              int32     // accessed atomically; belongs to this struct only
+	*sharedQuicListener           // embedded, so we also become a quic.EarlyListener
+	uc                  *unixConn // underlying unix socket, if UDS
+	context             context.Context
+	contextCancel       context.CancelFunc
+}
+
+// Currently Accept ignores the passed context, however a situation where
+// someone would need a hotswappable QUIC-only (not http3, since it uses context.Background here)
+// server on which Accept would be called with non-empty contexts
+// (mind that the default net listeners' Accept doesn't take a context argument)
+// sounds way too rare for us to sacrifice efficiency here.
+func (fcql *fakeCloseQuicListener) Accept(_ context.Context) (quic.EarlyConnection, error) {
+	conn, err := fcql.sharedQuicListener.Accept(fcql.context)
+	if err == nil {
+		return conn, nil
+	}
+
+	// if the listener is "closed", return a fake closed error instead
+	if atomic.LoadInt32(&fcql.closed) == 1 && errors.Is(err, context.Canceled) {
+		return nil, fakeClosedErr(fcql)
+	}
+	return nil, err
+}
+
+func (fcql *fakeCloseQuicListener) Close() error {
+	if atomic.CompareAndSwapInt32(&fcql.closed, 0, 1) {
+		fcql.contextCancel()
+		_, _ = listenerPool.Delete(fcql.sharedQuicListener.key)
+		if fcql.uc != nil {
+			// unix sockets need to be closed ourselves because we dup() the file
+			// descriptor when we reuse them, so this avoids a resource leak
+			fcql.uc.Close()
+		}
+	}
+	return nil
+}
+
 // RegisterNetwork registers a network type with Caddy so that if a listener is
 // created for that network type, getListener will be invoked to get the listener.
 // This should be called during init() and will panic if the network type is standard
@@ -455,11 +616,77 @@ func RegisterNetwork(network string, getListener ListenerFunc) {
 	networkTypes[network] = getListener
 }
 
+type unixListener struct {
+	*net.UnixListener
+	mapKey string
+	count  *int32 // accessed atomically
+}
+
+func (uln *unixListener) Close() error {
+	newCount := atomic.AddInt32(uln.count, -1)
+	if newCount == 0 {
+		defer func() {
+			addr := uln.Addr().String()
+			unixSocketsMu.Lock()
+			delete(unixSockets, uln.mapKey)
+			unixSocketsMu.Unlock()
+			_ = syscall.Unlink(addr)
+		}()
+	}
+	return uln.UnixListener.Close()
+}
+
+type unixConn struct {
+	*net.UnixConn
+	filename string
+	mapKey   string
+	count    *int32 // accessed atomically
+}
+
+func (uc *unixConn) Close() error {
+	newCount := atomic.AddInt32(uc.count, -1)
+	if newCount == 0 {
+		defer func() {
+			unixSocketsMu.Lock()
+			delete(unixSockets, uc.mapKey)
+			unixSocketsMu.Unlock()
+			_ = syscall.Unlink(uc.filename)
+		}()
+	}
+	return uc.UnixConn.Close()
+}
+
+// unixSockets keeps track of the currently-active unix sockets
+// so we can transfer their FDs gracefully during reloads.
+var (
+	unixSockets = make(map[string]interface {
+		File() (*os.File, error)
+	})
+	unixSocketsMu sync.Mutex
+)
+
+// getListenerFromPlugin returns a listener on the given network and address
+// if a plugin has registered the network name. It may return (nil, nil) if
+// no plugin can provide a listener.
+func getListenerFromPlugin(ctx context.Context, network, addr string, config net.ListenConfig) (any, error) {
+	// get listener from plugin if network type is registered
+	if getListener, ok := networkTypes[network]; ok {
+		Log().Debug("getting listener from plugin", zap.String("network", network))
+		return getListener(ctx, network, addr, config)
+	}
+
+	return nil, nil
+}
+
+func listenerKey(network, addr string) string {
+	return network + "/" + addr
+}
+
 // ListenerFunc is a function that can return a listener given a network and address.
 // The listeners must be capable of overlapping: with Caddy, new configs are loaded
 // before old ones are unloaded, so listeners may overlap briefly if the configs
 // both need the same listener. EXPERIMENTAL and subject to change.
-type ListenerFunc func(network, addr string) (net.Listener, error)
+type ListenerFunc func(ctx context.Context, network, addr string, cfg net.ListenConfig) (any, error)
 
 var networkTypes = map[string]ListenerFunc{}
 

logging.go

@@ -105,7 +105,7 @@ func (logging *Logging) openLogs(ctx Context) error {
 	// then set up any other custom logs
 	for name, l := range logging.Logs {
 		// the default log is already set up
-		if name == "default" {
+		if name == DefaultLoggerName {
 			continue
 		}
 
@@ -138,7 +138,7 @@ func (logging *Logging) setupNewDefault(ctx Context) error {
 
 	// extract the user-defined default log, if any
 	newDefault := new(defaultCustomLog)
-	if userDefault, ok := logging.Logs["default"]; ok {
+	if userDefault, ok := logging.Logs[DefaultLoggerName]; ok {
 		newDefault.CustomLog = userDefault
 	} else {
 		// if none, make one with our own default settings
@@ -147,7 +147,7 @@ func (logging *Logging) setupNewDefault(ctx Context) error {
 		if err != nil {
 			return fmt.Errorf("setting up default Caddy log: %v", err)
 		}
-		logging.Logs["default"] = newDefault.CustomLog
+		logging.Logs[DefaultLoggerName] = newDefault.CustomLog
 	}
 
 	// set up this new log
@@ -702,6 +702,8 @@ var (
 
 var writers = NewUsagePool()
 
+const DefaultLoggerName = "default"
+
 // Interface guards
 var (
 	_ io.WriteCloser = (*notClosable)(nil)

modules/caddyevents/app.go

@@ -119,7 +119,7 @@ func (App) CaddyModule() caddy.ModuleInfo {
 
 // Provision sets up the app.
 func (app *App) Provision(ctx caddy.Context) error {
-	app.logger = ctx.Logger(app)
+	app.logger = ctx.Logger()
 	app.subscriptions = make(map[string]map[caddy.ModuleID][]Handler)
 
 	for _, sub := range app.Subscriptions {
@@ -201,6 +201,9 @@ func (app *App) On(eventName string, handler Handler) error {
 // Emit creates and dispatches an event named eventName to all relevant handlers with
 // the metadata data. Events are emitted and propagated synchronously. The returned Event
 // value will have any additional information from the invoked handlers.
+//
+// Note that the data map is not copied, for efficiency. After Emit() is called, the
+// data passed in should not be changed in other goroutines.
 func (app *App) Emit(ctx caddy.Context, eventName string, data map[string]any) Event {
 	logger := app.logger.With(zap.String("name", eventName))
 
@@ -212,11 +215,11 @@ func (app *App) Emit(ctx caddy.Context, eventName string, data map[string]any) E
 	eventName = strings.ToLower(eventName)
 
 	e := Event{
+		Data:   data,
 		id:     id,
 		ts:     time.Now(),
 		name:   eventName,
 		origin: ctx.Module(),
-		data:   data,
 	}
 
 	logger = logger.With(
@@ -244,12 +247,12 @@ func (app *App) Emit(ctx caddy.Context, eventName string, data map[string]any) E
 		case "event.module":
 			return e.origin.CaddyModule().ID, true
 		case "event.data":
-			return e.data, true
+			return e.Data, true
 		}
 
 		if strings.HasPrefix(key, "event.data.") {
 			key = strings.TrimPrefix(key, "event.data.")
-			if val, ok := data[key]; ok {
+			if val, ok := e.Data[key]; ok {
 				return val, true
 			}
 		}
@@ -257,7 +260,7 @@ func (app *App) Emit(ctx caddy.Context, eventName string, data map[string]any) E
 		return nil, false
 	})
 
-	logger.Debug("event", zap.Any("data", e.data))
+	logger.Debug("event", zap.Any("data", e.Data))
 
 	// invoke handlers bound to the event by name and also all events; this for loop
 	// iterates twice at most: once for the event name, once for "" (all events)
@@ -314,26 +317,40 @@ func (app *App) Emit(ctx caddy.Context, eventName string, data map[string]any) E
 }
 
 // Event represents something that has happened or is happening.
+// An Event value is not synchronized, so it should be copied if
+// being used in goroutines.
+//
+// EXPERIMENTAL: As with the rest of this package, events are
+// subject to change.
 type Event struct {
-	id     uuid.UUID
-	ts     time.Time
-	name   string
-	origin caddy.Module
-	data   map[string]any
-
 	// If non-nil, the event has been aborted, meaning
 	// propagation has stopped to other handlers and
 	// the code should stop what it was doing. Emitters
 	// may choose to use this as a signal to adjust their
 	// code path appropriately.
 	Aborted error
+
+	// The data associated with the event. Usually the
+	// original emitter will be the only one to set or
+	// change these values, but the field is exported
+	// so handlers can have full access if needed.
+	// However, this map is not synchronized, so
+	// handlers must not use this map directly in new
+	// goroutines; instead, copy the map to use it in a
+	// goroutine.
+	Data map[string]any
+
+	id     uuid.UUID
+	ts     time.Time
+	name   string
+	origin caddy.Module
 }
 
 // CloudEvent exports event e as a structure that, when
 // serialized as JSON, is compatible with the
 // CloudEvents spec.
 func (e Event) CloudEvent() CloudEvent {
-	dataJSON, _ := json.Marshal(e.data)
+	dataJSON, _ := json.Marshal(e.Data)
 	return CloudEvent{
 		ID:              e.id.String(),
 		Source:          e.origin.CaddyModule().String(),

modules/caddyhttp/app.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"net"
 	"net/http"
 	"strconv"
 	"sync"
@@ -53,19 +54,20 @@ func init() {
 // `{http.request.duration_ms}` | Same as 'duration', but in milliseconds.
 // `{http.request.uuid}` | The request unique identifier
 // `{http.request.header.*}` | Specific request header field
-// `{http.request.host.labels.*}` | Request host labels (0-based from right); e.g. for foo.example.com: 0=com, 1=example, 2=foo
 // `{http.request.host}` | The host part of the request's Host header
+// `{http.request.host.labels.*}` | Request host labels (0-based from right); e.g. for foo.example.com: 0=com, 1=example, 2=foo
 // `{http.request.hostport}` | The host and port from the request's Host header
 // `{http.request.method}` | The request method
 // `{http.request.orig_method}` | The request's original method
+// `{http.request.orig_uri}` | The request's original URI
+// `{http.request.orig_uri.path}` | The request's original path
+// `{http.request.orig_uri.path.*}` | Parts of the original path, split by `/` (0-based from left)
 // `{http.request.orig_uri.path.dir}` | The request's original directory
 // `{http.request.orig_uri.path.file}` | The request's original filename
-// `{http.request.orig_uri.path}` | The request's original path
 // `{http.request.orig_uri.query}` | The request's original query string (without `?`)
-// `{http.request.orig_uri}` | The request's original URI
 // `{http.request.port}` | The port part of the request's Host header
 // `{http.request.proto}` | The protocol of the request
-// `{http.request.remote.host}` | The host part of the remote client's address
+// `{http.request.remote.host}` | The host (IP) part of the remote client's address
 // `{http.request.remote.port}` | The port part of the remote client's address
 // `{http.request.remote}` | The address of the remote client
 // `{http.request.scheme}` | The request scheme
@@ -87,13 +89,13 @@ func init() {
 // `{http.request.tls.client.san.emails.*}` | SAN email addresses (index optional)
 // `{http.request.tls.client.san.ips.*}` | SAN IP addresses (index optional)
 // `{http.request.tls.client.san.uris.*}` | SAN URIs (index optional)
+// `{http.request.uri}` | The full request URI
+// `{http.request.uri.path}` | The path component of the request URI
 // `{http.request.uri.path.*}` | Parts of the path, split by `/` (0-based from left)
 // `{http.request.uri.path.dir}` | The directory, excluding leaf filename
 // `{http.request.uri.path.file}` | The filename of the path, excluding directory
-// `{http.request.uri.path}` | The path component of the request URI
-// `{http.request.uri.query.*}` | Individual query string value
 // `{http.request.uri.query}` | The query string (without `?`)
-// `{http.request.uri}` | The full request URI
+// `{http.request.uri.query.*}` | Individual query string value
 // `{http.response.header.*}` | Specific response header field
 // `{http.vars.*}` | Custom variables in the HTTP handler chain
 // `{http.shutting_down}` | True if the HTTP app is shutting down
@@ -160,7 +162,7 @@ func (app *App) Provision(ctx caddy.Context) error {
 	}
 	app.tlsApp = tlsAppIface.(*caddytls.TLS)
 	app.ctx = ctx
-	app.logger = ctx.Logger(app)
+	app.logger = ctx.Logger()
 
 	eventsAppIface, err := ctx.App("events")
 	if err != nil {
@@ -178,7 +180,9 @@ func (app *App) Provision(ctx caddy.Context) error {
 	}
 
 	// prepare each server
+	oldContext := ctx.Context
 	for srvName, srv := range app.Servers {
+		ctx.Context = context.WithValue(oldContext, ServerCtxKey, srv)
 		srv.name = srvName
 		srv.tlsApp = app.tlsApp
 		srv.events = eventsAppIface.(*caddyevents.App)
@@ -219,6 +223,15 @@ func (app *App) Provision(ctx caddy.Context) error {
 			srv.StrictSNIHost = &trueBool
 		}
 
+		// set up the trusted proxies source
+		for srv.TrustedProxiesRaw != nil {
+			val, err := ctx.LoadModule(srv, "TrustedProxiesRaw")
+			if err != nil {
+				return fmt.Errorf("loading trusted proxies modules: %v", err)
+			}
+			srv.trustedProxies = val.(IPRangeSource)
+		}
+
 		// process each listener address
 		for i := range srv.Listen {
 			lnOut, err := repl.ReplaceOrErr(srv.Listen[i], true, true)
@@ -263,7 +276,7 @@ func (app *App) Provision(ctx caddy.Context) error {
 		// route handler so that important security checks are done, etc.
 		primaryRoute := emptyHandler
 		if srv.Routes != nil {
-			err := srv.Routes.ProvisionHandlers(ctx)
+			err := srv.Routes.ProvisionHandlers(ctx, srv.Metrics)
 			if err != nil {
 				return fmt.Errorf("server %s: setting up route handlers: %v", srvName, err)
 			}
@@ -293,7 +306,7 @@ func (app *App) Provision(ctx caddy.Context) error {
 			srv.IdleTimeout = defaultIdleTimeout
 		}
 	}
-
+	ctx.Context = oldContext
 	return nil
 }
 
@@ -365,6 +378,7 @@ func (app *App) Start() error {
 		// this TLS config is used by the std lib to choose the actual TLS config for connections
 		// by looking through the connection policies to find the first one that matches
 		tlsCfg := srv.TLSConnPolicies.TLSConfig(app.ctx)
+		srv.configureServer(srv.server)
 
 		// enable H2C if configured
 		if srv.protocol("h2c") {
@@ -384,10 +398,11 @@ func (app *App) Start() error {
 			for portOffset := uint(0); portOffset < listenAddr.PortRangeSize(); portOffset++ {
 				// create the listener for this socket
 				hostport := listenAddr.JoinHostPort(portOffset)
-				ln, err := caddy.ListenTimeout(listenAddr.Network, hostport, time.Duration(srv.KeepAliveInterval))
+				lnAny, err := listenAddr.Listen(app.ctx, portOffset, net.ListenConfig{KeepAlive: time.Duration(srv.KeepAliveInterval)})
 				if err != nil {
-					return fmt.Errorf("%s: listening on %s: %v", listenAddr.Network, hostport, err)
+					return fmt.Errorf("listening on %s: %v", listenAddr.At(portOffset), err)
 				}
+				ln := lnAny.(net.Listener)
 
 				// wrap listener before TLS (up to the TLS placeholder wrapper)
 				var lnWrapperIdx int
@@ -407,9 +422,26 @@ func (app *App) Start() error {
 
 					// enable HTTP/3 if configured
 					if srv.protocol("h3") {
-						app.logger.Info("enabling HTTP/3 listener", zap.String("addr", hostport))
-						if err := srv.serveHTTP3(hostport, tlsCfg); err != nil {
-							return err
+						// Can't serve HTTP/3 on the same socket as HTTP/1 and 2 because it uses
+						// a different transport mechanism... which is fine, but the OS doesn't
+						// differentiate between a SOCK_STREAM file and a SOCK_DGRAM file; they
+						// are still one file on the system. So even though "unixpacket" and
+						// "unixgram" are different network types just as "tcp" and "udp" are,
+						// the OS will not let us use the same file as both STREAM and DGRAM.
+						if len(srv.Protocols) > 1 && listenAddr.IsUnixNetwork() {
+							app.logger.Warn("HTTP/3 disabled because Unix can't multiplex STREAM and DGRAM on same socket",
+								zap.String("file", hostport))
+							for i := range srv.Protocols {
+								if srv.Protocols[i] == "h3" {
+									srv.Protocols = append(srv.Protocols[:i], srv.Protocols[i+1:]...)
+									break
+								}
+							}
+						} else {
+							app.logger.Info("enabling HTTP/3 listener", zap.String("addr", hostport))
+							if err := srv.serveHTTP3(listenAddr.At(portOffset), tlsCfg); err != nil {
+								return err
+							}
 						}
 					}
 				}
@@ -421,11 +453,10 @@ func (app *App) Start() error {
 
 				// if binding to port 0, the OS chooses a port for us;
 				// but the user won't know the port unless we print it
-				if listenAddr.StartPort == 0 && listenAddr.EndPort == 0 {
+				if !listenAddr.IsUnixNetwork() && listenAddr.StartPort == 0 && listenAddr.EndPort == 0 {
 					app.logger.Info("port 0 listener",
 						zap.String("input_address", lnAddr),
-						zap.String("actual_address", ln.Addr().String()),
-					)
+						zap.String("actual_address", ln.Addr().String()))
 				}
 
 				app.logger.Debug("starting server loop",
@@ -502,22 +533,74 @@ func (app *App) Stop() error {
 		app.logger.Debug("servers shutting down with eternal grace period")
 	}
 
-	// shut down servers
-	for _, server := range app.Servers {
+	// goroutines aren't guaranteed to be scheduled right away,
+	// so we'll use one WaitGroup to wait for all the goroutines
+	// to start their server shutdowns, and another to wait for
+	// them to finish; we'll always block for them to start so
+	// that when we return the caller can be confident* that the
+	// old servers are no longer accepting new connections
+	// (* the scheduler might still pause them right before
+	// calling Shutdown(), but it's unlikely)
+	var startedShutdown, finishedShutdown sync.WaitGroup
+
+	// these will run in goroutines
+	stopServer := func(server *Server) {
+		defer finishedShutdown.Done()
+		startedShutdown.Done()
+
 		if err := server.server.Shutdown(ctx); err != nil {
 			app.logger.Error("server shutdown",
 				zap.Error(err),
 				zap.Strings("addresses", server.Listen))
 		}
+	}
+	stopH3Server := func(server *Server) {
+		defer finishedShutdown.Done()
+		startedShutdown.Done()
 
-		if server.h3server != nil {
-			// TODO: CloseGracefully, once implemented upstream (see https://github.com/lucas-clemente/quic-go/issues/2103)
-			if err := server.h3server.Close(); err != nil {
-				app.logger.Error("HTTP/3 server shutdown",
+		if server.h3server == nil {
+			return
+		}
+
+		// TODO: we have to manually close our listeners because quic-go won't
+		// close listeners it didn't create along with the server itself...
+		// see https://github.com/quic-go/quic-go/issues/3560
+		for _, el := range server.h3listeners {
+			if err := el.Close(); err != nil {
+				app.logger.Error("HTTP/3 listener close",
 					zap.Error(err),
-					zap.Strings("addresses", server.Listen))
+					zap.String("address", el.LocalAddr().String()))
 			}
 		}
+
+		// TODO: CloseGracefully, once implemented upstream (see https://github.com/quic-go/quic-go/issues/2103)
+		if err := server.h3server.Close(); err != nil {
+			app.logger.Error("HTTP/3 server shutdown",
+				zap.Error(err),
+				zap.Strings("addresses", server.Listen))
+		}
+	}
+
+	for _, server := range app.Servers {
+		startedShutdown.Add(2)
+		finishedShutdown.Add(2)
+		go stopServer(server)
+		go stopH3Server(server)
+	}
+
+	// block until all the goroutines have been run by the scheduler;
+	// this means that they have likely called Shutdown() by now
+	startedShutdown.Wait()
+
+	// if the process is exiting, we need to block here and wait
+	// for the grace periods to complete, otherwise the process will
+	// terminate before the servers are finished shutting down; but
+	// we don't really need to wait for the grace period to finish
+	// if the process isn't exiting (but note that frequent config
+	// reloads with long grace periods for a sustained length of time
+	// may deplete resources)
+	if caddy.Exiting() {
+		finishedShutdown.Wait()
 	}
 
 	return nil

modules/caddyhttp/autohttps.go

@@ -378,19 +378,29 @@ redirServersLoop:
 		// we'll create a new server for all the listener addresses
 		// that are unused and serve the remaining redirects from it
 		for _, srv := range app.Servers {
-			if srv.hasListenerAddress(redirServerAddr) {
-				// find the index of the route after the last route with a host
-				// matcher, then insert the redirects there, but before any
-				// user-defined catch-all routes
-				// see https://github.com/caddyserver/caddy/issues/3212
-				insertIndex := srv.findLastRouteWithHostMatcher()
-				srv.Routes = append(srv.Routes[:insertIndex], append(routes, srv.Routes[insertIndex:]...)...)
-
-				// append our catch-all route in case the user didn't define their own
-				srv.Routes = appendCatchAll(srv.Routes)
-
-				continue redirServersLoop
+			// only look at servers which listen on an address which
+			// we want to add redirects to
+			if !srv.hasListenerAddress(redirServerAddr) {
+				continue
 			}
+
+			// find the index of the route after the last route with a host
+			// matcher, then insert the redirects there, but before any
+			// user-defined catch-all routes
+			// see https://github.com/caddyserver/caddy/issues/3212
+			insertIndex := srv.findLastRouteWithHostMatcher()
+
+			// add the redirects at the insert index, except for when
+			// we have a catch-all for HTTPS, in which case the user's
+			// defined catch-all should take precedence. See #4829
+			if len(uniqueDomainsForCerts) != 0 {
+				srv.Routes = append(srv.Routes[:insertIndex], append(routes, srv.Routes[insertIndex:]...)...)
+			}
+
+			// append our catch-all route in case the user didn't define their own
+			srv.Routes = appendCatchAll(srv.Routes)
+
+			continue redirServersLoop
 		}
 
 		// no server with this listener address exists;

modules/caddyhttp/caddyauth/basicauth.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/caddyserver/caddy/v2"
+	"golang.org/x/sync/singleflight"
 )
 
 func init() {
@@ -142,6 +143,7 @@ func (hba *HTTPBasicAuth) Provision(ctx caddy.Context) error {
 	if hba.HashCache != nil {
 		hba.HashCache.cache = make(map[string]bool)
 		hba.HashCache.mu = new(sync.RWMutex)
+		hba.HashCache.g = new(singleflight.Group)
 	}
 
 	return nil
@@ -190,12 +192,18 @@ func (hba HTTPBasicAuth) correctPassword(account Account, plaintextPassword []by
 	if ok {
 		return same, nil
 	}
-
 	// slow track: do the expensive op, then add it to the cache
-	same, err := compare()
+	// but perform it in a singleflight group so that multiple
+	// parallel requests using the same password don't cause a
+	// thundering herd problem by all performing the same hashing
+	// operation before the first one finishes and caches it.
+	v, err, _ := hba.HashCache.g.Do(cacheKey, func() (any, error) {
+		return compare()
+	})
 	if err != nil {
 		return false, err
 	}
+	same = v.(bool)
 	hba.HashCache.mu.Lock()
 	if len(hba.HashCache.cache) >= 1000 {
 		hba.HashCache.makeRoom() // keep cache size under control
@@ -223,6 +231,7 @@ func (hba HTTPBasicAuth) promptForCredentials(w http.ResponseWriter, err error)
 // compute on every HTTP request.
 type Cache struct {
 	mu *sync.RWMutex
+	g  *singleflight.Group
 
 	// map of concatenated hashed password + plaintext password + salt, to result
 	cache map[string]bool

modules/caddyhttp/caddyauth/caddyauth.go

@@ -56,7 +56,7 @@ func (Authentication) CaddyModule() caddy.ModuleInfo {
 
 // Provision sets up a.
 func (a *Authentication) Provision(ctx caddy.Context) error {
-	a.logger = ctx.Logger(a)
+	a.logger = ctx.Logger()
 	a.Providers = make(map[string]Authenticator)
 	mods, err := ctx.LoadModule(a, "ProvidersRaw")
 	if err != nil {

modules/caddyhttp/caddyauth/caddyfile.go

@@ -27,10 +27,10 @@ func init() {
 
 // parseCaddyfile sets up the handler from Caddyfile tokens. Syntax:
 //
-//     basicauth [<matcher>] [<hash_algorithm> [<realm>]] {
-//         <username> <hashed_password_base64> [<salt_base64>]
-//         ...
-//     }
+//	basicauth [<matcher>] [<hash_algorithm> [<realm>]] {
+//	    <username> <hashed_password_base64> [<salt_base64>]
+//	    ...
+//	}
 //
 // If no hash algorithm is supplied, bcrypt will be assumed.
 func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {

modules/caddyhttp/caddyauth/hashes.go

@@ -92,7 +92,7 @@ func (ScryptHash) CaddyModule() caddy.ModuleInfo {
 // Provision sets up s.
 func (s *ScryptHash) Provision(ctx caddy.Context) error {
 	s.SetDefaults()
-	ctx.Logger(s).Warn("use of 'scrypt' is deprecated, please use 'bcrypt' instead")
+	ctx.Logger().Warn("use of 'scrypt' is deprecated, please use 'bcrypt' instead")
 	return nil
 }
 

modules/caddyhttp/caddyhttp.go

@@ -232,7 +232,7 @@ func SanitizedPathJoin(root, reqPath string) string {
 		root = "."
 	}
 
-	path := filepath.Join(root, filepath.Clean("/"+reqPath))
+	path := filepath.Join(root, path.Clean("/"+reqPath))
 
 	// filepath.Join also cleans the path, and cleaning strips
 	// the trailing slash, so we need to re-add it afterwards.

modules/caddyhttp/caddyhttp_test.go

@@ -87,7 +87,7 @@ func TestSanitizedPathJoin(t *testing.T) {
 		}
 		actual := SanitizedPathJoin(tc.inputRoot, u.Path)
 		if actual != tc.expect {
-			t.Errorf("Test %d: SanitizedPathJoin('%s', '%s') =>  %s (expected '%s')",
+			t.Errorf("Test %d: SanitizedPathJoin('%s', '%s') =>  '%s' (expected '%s')",
 				i, tc.inputRoot, tc.inputPath, actual, tc.expect)
 		}
 	}

modules/caddyhttp/celmatcher.go

@@ -90,7 +90,7 @@ func (m *MatchExpression) UnmarshalJSON(data []byte) error {
 
 // Provision sets ups m.
 func (m *MatchExpression) Provision(ctx caddy.Context) error {
-	m.log = ctx.Logger(m)
+	m.log = ctx.Logger()
 
 	// replace placeholders with a function call - this is just some
 	// light (and possibly naïve) syntactic sugar
@@ -124,7 +124,7 @@ func (m *MatchExpression) Provision(ctx caddy.Context) error {
 
 	// create the CEL environment
 	env, err := cel.NewEnv(
-		cel.Function(placeholderFuncName, cel.SingletonBinaryImpl(m.caddyPlaceholderFunc), cel.Overload(
+		cel.Function(placeholderFuncName, cel.SingletonBinaryBinding(m.caddyPlaceholderFunc), cel.Overload(
 			placeholderFuncName+"_httpRequest_string",
 			[]*cel.Type{httpRequestObjectType, cel.StringType},
 			cel.AnyType,
@@ -294,13 +294,13 @@ type CELLibraryProducer interface {
 // CELMatcherImpl creates a new cel.Library based on the following pieces of
 // data:
 //
-// - macroName: the function name to be used within CEL. This will be a macro
-//   and not a function proper.
-// - funcName: the function overload name generated by the CEL macro used to
-//   represent the matcher.
-// - matcherDataTypes: the argument types to the macro.
-// - fac: a matcherFactory implementation which converts from CEL constant
-//   values to a Matcher instance.
+//   - macroName: the function name to be used within CEL. This will be a macro
+//     and not a function proper.
+//   - funcName: the function overload name generated by the CEL macro used to
+//     represent the matcher.
+//   - matcherDataTypes: the argument types to the macro.
+//   - fac: a matcherFactory implementation which converts from CEL constant
+//     values to a Matcher instance.
 //
 // Note, macro names and function names must not collide with other macros or
 // functions exposed within CEL expressions, or an error will be produced
@@ -345,7 +345,7 @@ func CELMatcherImpl(macroName, funcName string, matcherDataTypes []*cel.Type, fa
 		cel.Macros(macro),
 		cel.Function(funcName,
 			cel.Overload(funcName, append([]*cel.Type{requestType}, matcherDataTypes...), cel.BoolType),
-			cel.SingletonBinaryImpl(CELMatcherRuntimeFunction(funcName, fac))),
+			cel.SingletonBinaryBinding(CELMatcherRuntimeFunction(funcName, fac))),
 	}
 	programOptions := []cel.ProgramOption{
 		cel.CustomDecorator(CELMatcherDecorator(funcName, fac)),

modules/caddyhttp/encode/caddyfile.go

@@ -39,18 +39,18 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 
 // UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax:
 //
-//     encode [<matcher>] <formats...> {
-//         gzip           [<level>]
-//         zstd
-//         minimum_length <length>
-//         # response matcher block
-//         match {
-//             status <code...>
-//             header <field> [<value>]
-//         }
-//         # or response matcher single line syntax
-//         match [header <field> [<value>]] | [status <code...>]
-//     }
+//	encode [<matcher>] <formats...> {
+//	    gzip           [<level>]
+//	    zstd
+//	    minimum_length <length>
+//	    # response matcher block
+//	    match {
+//	        status <code...>
+//	        header <field> [<value>]
+//	    }
+//	    # or response matcher single line syntax
+//	    match [header <field> [<value>]] | [status <code...>]
+//	}
 //
 // Specifying the formats on the first line will use those formats' defaults.
 func (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {

modules/caddyhttp/encode/encode.go

@@ -20,7 +20,6 @@
 package encode
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"math"
@@ -118,14 +117,20 @@ func (enc *Encode) Validate() error {
 	return nil
 }
 
+func isEncodeAllowed(h http.Header) bool {
+	return !strings.Contains(h.Get("Cache-Control"), "no-transform")
+}
+
 func (enc *Encode) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
-	for _, encName := range AcceptedEncodings(r, enc.Prefer) {
-		if _, ok := enc.writerPools[encName]; !ok {
-			continue // encoding not offered
+	if isEncodeAllowed(r.Header) {
+		for _, encName := range AcceptedEncodings(r, enc.Prefer) {
+			if _, ok := enc.writerPools[encName]; !ok {
+				continue // encoding not offered
+			}
+			w = enc.openResponseWriter(encName, w)
+			defer w.(*responseWriter).Close()
+			break
 		}
-		w = enc.openResponseWriter(encName, w)
-		defer w.(*responseWriter).Close()
-		break
 	}
 	return next.ServeHTTP(w, r)
 }
@@ -160,13 +165,12 @@ func (enc *Encode) openResponseWriter(encodingName string, w http.ResponseWriter
 // initResponseWriter initializes the responseWriter instance
 // allocated in openResponseWriter, enabling mid-stack inlining.
 func (enc *Encode) initResponseWriter(rw *responseWriter, encodingName string, wrappedRW http.ResponseWriter) *responseWriter {
-	buf := bufPool.Get().(*bytes.Buffer)
-	buf.Reset()
-
-	// The allocation of ResponseWriterWrapper might be optimized as well.
-	rw.ResponseWriterWrapper = &caddyhttp.ResponseWriterWrapper{ResponseWriter: wrappedRW}
+	if httpInterfaces, ok := wrappedRW.(caddyhttp.HTTPInterfaces); ok {
+		rw.HTTPInterfaces = httpInterfaces
+	} else {
+		rw.HTTPInterfaces = &caddyhttp.ResponseWriterWrapper{ResponseWriter: wrappedRW}
+	}
 	rw.encodingName = encodingName
-	rw.buf = buf
 	rw.config = enc
 
 	return rw
@@ -176,10 +180,9 @@ func (enc *Encode) initResponseWriter(rw *responseWriter, encodingName string, w
 // using the encoding represented by encodingName and
 // configured by config.
 type responseWriter struct {
-	*caddyhttp.ResponseWriterWrapper
+	caddyhttp.HTTPInterfaces
 	encodingName string
 	w            Encoder
-	buf          *bytes.Buffer
 	config       *Encode
 	statusCode   int
 	wroteHeader  bool
@@ -206,28 +209,33 @@ func (rw *responseWriter) Flush() {
 		// to rw.Write (see bug in #4314)
 		return
 	}
-	rw.ResponseWriterWrapper.Flush()
+	rw.HTTPInterfaces.Flush()
 }
 
 // Write writes to the response. If the response qualifies,
 // it is encoded using the encoder, which is initialized
 // if not done so already.
 func (rw *responseWriter) Write(p []byte) (int, error) {
-	var n, written int
-	var err error
+	// ignore zero data writes, probably head request
+	if len(p) == 0 {
+		return 0, nil
+	}
 
-	if rw.buf != nil && rw.config.MinLength > 0 {
-		written = rw.buf.Len()
-		_, err := rw.buf.Write(p)
-		if err != nil {
-			return 0, err
+	// sniff content-type and determine content-length
+	if !rw.wroteHeader && rw.config.MinLength > 0 {
+		var gtMinLength bool
+		if len(p) > rw.config.MinLength {
+			gtMinLength = true
+		} else if cl, err := strconv.Atoi(rw.Header().Get("Content-Length")); err == nil && cl > rw.config.MinLength {
+			gtMinLength = true
+		}
+
+		if gtMinLength {
+			if rw.Header().Get("Content-Type") == "" {
+				rw.Header().Set("Content-Type", http.DetectContentType(p))
+			}
+			rw.init()
 		}
-		rw.init()
-		p = rw.buf.Bytes()
-		defer func() {
-			bufPool.Put(rw.buf)
-			rw.buf = nil
-		}()
 	}
 
 	// before we write to the response, we need to make
@@ -236,63 +244,41 @@ func (rw *responseWriter) Write(p []byte) (int, error) {
 	// and if so, that means we haven't written the
 	// header OR the default status code will be written
 	// by the standard library
-	if rw.statusCode > 0 {
-		rw.ResponseWriter.WriteHeader(rw.statusCode)
-		rw.statusCode = 0
+	if !rw.wroteHeader {
+		if rw.statusCode != 0 {
+			rw.HTTPInterfaces.WriteHeader(rw.statusCode)
+		}
 		rw.wroteHeader = true
 	}
 
-	switch {
-	case rw.w != nil:
-		n, err = rw.w.Write(p)
-	default:
-		n, err = rw.ResponseWriter.Write(p)
+	if rw.w != nil {
+		return rw.w.Write(p)
+	} else {
+		return rw.HTTPInterfaces.Write(p)
 	}
-	n -= written
-	if n < 0 {
-		n = 0
-	}
-	return n, err
 }
 
 // Close writes any remaining buffered response and
 // deallocates any active resources.
 func (rw *responseWriter) Close() error {
-	var err error
-	// only attempt to write the remaining buffered response
-	// if there are any bytes left to write; otherwise, if
-	// the handler above us returned an error without writing
-	// anything, we'd write to the response when we instead
-	// should simply let the error propagate back down; this
-	// is why the check for rw.buf.Len() > 0 is crucial
-	if rw.buf != nil && rw.buf.Len() > 0 {
-		rw.init()
-		p := rw.buf.Bytes()
-		defer func() {
-			bufPool.Put(rw.buf)
-			rw.buf = nil
-		}()
-		switch {
-		case rw.w != nil:
-			_, err = rw.w.Write(p)
-		default:
-			_, err = rw.ResponseWriter.Write(p)
+	// didn't write, probably head request
+	if !rw.wroteHeader {
+		cl, err := strconv.Atoi(rw.Header().Get("Content-Length"))
+		if err == nil && cl > rw.config.MinLength {
+			rw.init()
+		}
+
+		// issue #5059, don't write status code if not set explicitly.
+		if rw.statusCode != 0 {
+			rw.HTTPInterfaces.WriteHeader(rw.statusCode)
 		}
-	} else if rw.statusCode != 0 {
-		// it is possible that a body was not written, and
-		// a header was not even written yet, even though
-		// we are closing; ensure the proper status code is
-		// written exactly once, or we risk breaking requests
-		// that rely on If-None-Match, for example
-		rw.ResponseWriter.WriteHeader(rw.statusCode)
-		rw.statusCode = 0
 		rw.wroteHeader = true
 	}
+
+	var err error
 	if rw.w != nil {
-		err2 := rw.w.Close()
-		if err2 != nil && err == nil {
-			err = err2
-		}
+		err = rw.w.Close()
+		rw.w.Reset(nil)
 		rw.config.writerPools[rw.encodingName].Put(rw.w)
 		rw.w = nil
 	}
@@ -301,17 +287,16 @@ func (rw *responseWriter) Close() error {
 
 // init should be called before we write a response, if rw.buf has contents.
 func (rw *responseWriter) init() {
-	if rw.Header().Get("Content-Encoding") == "" &&
-		rw.buf.Len() >= rw.config.MinLength &&
+	if rw.Header().Get("Content-Encoding") == "" && isEncodeAllowed(rw.Header()) &&
 		rw.config.Match(rw) {
 
 		rw.w = rw.config.writerPools[rw.encodingName].Get().(Encoder)
-		rw.w.Reset(rw.ResponseWriter)
+		rw.w.Reset(rw.HTTPInterfaces)
 		rw.Header().Del("Content-Length") // https://github.com/golang/go/issues/14975
 		rw.Header().Set("Content-Encoding", rw.encodingName)
 		rw.Header().Add("Vary", "Accept-Encoding")
+		rw.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-encoded content
 	}
-	rw.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-encoded content
 }
 
 // AcceptedEncodings returns the list of encodings that the
@@ -417,12 +402,6 @@ type Precompressed interface {
 	Suffix() string
 }
 
-var bufPool = sync.Pool{
-	New: func() any {
-		return new(bytes.Buffer)
-	},
-}
-
 // defaultMinLength is the minimum length at which to compress content.
 const defaultMinLength = 512
 

modules/caddyhttp/encode/encode_test.go

@@ -261,3 +261,50 @@ func TestValidate(t *testing.T) {
 
 	}
 }
+
+func TestIsEncodeAllowed(t *testing.T) {
+	testCases := []struct {
+		name     string
+		headers  http.Header
+		expected bool
+	}{
+		{
+			name:     "Without any headers",
+			headers:  http.Header{},
+			expected: true,
+		},
+		{
+			name: "Without Cache-Control HTTP header",
+			headers: http.Header{
+				"Accept-Encoding": {"gzip"},
+			},
+			expected: true,
+		},
+		{
+			name: "Cache-Control HTTP header ending with no-transform directive",
+			headers: http.Header{
+				"Accept-Encoding": {"gzip"},
+				"Cache-Control":   {"no-cache; no-transform"},
+			},
+			expected: false,
+		},
+		{
+			name: "With Cache-Control HTTP header no-transform as Cache-Extension value",
+			headers: http.Header{
+				"Accept-Encoding": {"gzip"},
+				"Cache-Control":   {`no-store; no-cache; community="no-transform"`},
+			},
+			expected: false,
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			if result := isEncodeAllowed(test.headers); result != test.expected {
+				t.Errorf("The headers given to the isEncodeAllowed should return %t, %t given.",
+					result,
+					test.expected)
+			}
+		})
+	}
+}

modules/caddyhttp/fileserver/browse.go

@@ -16,9 +16,11 @@ package fileserver
 
 import (
 	"bytes"
+	"context"
 	_ "embed"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/fs"
 	"net/http"
 	"os"
@@ -81,7 +83,7 @@ func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter,
 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
 
 	// calling path.Clean here prevents weird breadcrumbs when URL paths are sketchy like /%2e%2e%2f
-	listing, err := fsrv.loadDirectoryContents(dir.(fs.ReadDirFile), root, path.Clean(r.URL.Path), repl)
+	listing, err := fsrv.loadDirectoryContents(r.Context(), dir.(fs.ReadDirFile), root, path.Clean(r.URL.Path), repl)
 	switch {
 	case os.IsPermission(err):
 		return caddyhttp.Error(http.StatusForbidden, err)
@@ -135,16 +137,16 @@ func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter,
 	return nil
 }
 
-func (fsrv *FileServer) loadDirectoryContents(dir fs.ReadDirFile, root, urlPath string, repl *caddy.Replacer) (browseTemplateContext, error) {
+func (fsrv *FileServer) loadDirectoryContents(ctx context.Context, dir fs.ReadDirFile, root, urlPath string, repl *caddy.Replacer) (browseTemplateContext, error) {
 	files, err := dir.ReadDir(10000) // TODO: this limit should probably be configurable
-	if err != nil {
+	if err != nil && err != io.EOF {
 		return browseTemplateContext{}, err
 	}
 
 	// user can presumably browse "up" to parent folder if path is longer than "/"
 	canGoUp := len(urlPath) > 1
 
-	return fsrv.directoryListing(files, canGoUp, root, urlPath, repl), nil
+	return fsrv.directoryListing(ctx, files, canGoUp, root, urlPath, repl), nil
 }
 
 // browseApplyQueryParams applies query parameters to the listing.
@@ -210,7 +212,7 @@ func (fsrv *FileServer) isSymlinkTargetDir(f fs.FileInfo, root, urlPath string)
 		return false
 	}
 	target := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, f.Name()))
-	targetInfo, err := fsrv.fileSystem.Stat(target)
+	targetInfo, err := fs.Stat(fsrv.fileSystem, target)
 	if err != nil {
 		return false
 	}

modules/caddyhttp/fileserver/browse.html

@@ -27,6 +27,10 @@ a:visited {
 	color: #800080;
 }
 
+a:visited:hover {
+	color: #b900b9;
+}
+
 header,
 #summary {
 	padding-left: 5%;
@@ -244,6 +248,14 @@ footer {
 		color: #62b2fd;
 	}
 
+	a:visited {
+		color: #c269c2;
+	}
+
+	a:visited:hover {
+		color: #d03cd0;
+	}
+
 	tr {
 		border-bottom: 1px dashed rgba(255, 255, 255, 0.12);
 	}

modules/caddyhttp/fileserver/browsetplcontext.go

@@ -15,6 +15,7 @@
 package fileserver
 
 import (
+	"context"
 	"io/fs"
 	"net/url"
 	"os"
@@ -30,13 +31,17 @@ import (
 	"go.uber.org/zap"
 )
 
-func (fsrv *FileServer) directoryListing(entries []fs.DirEntry, canGoUp bool, root, urlPath string, repl *caddy.Replacer) browseTemplateContext {
+func (fsrv *FileServer) directoryListing(ctx context.Context, entries []fs.DirEntry, canGoUp bool, root, urlPath string, repl *caddy.Replacer) browseTemplateContext {
 	filesToHide := fsrv.transformHidePaths(repl)
 
 	var dirCount, fileCount int
 	fileInfos := []fileInfo{}
 
 	for _, entry := range entries {
+		if err := ctx.Err(); err != nil {
+			break
+		}
+
 		name := entry.Name()
 
 		if fileHidden(name, filesToHide) {
@@ -65,7 +70,7 @@ func (fsrv *FileServer) directoryListing(entries []fs.DirEntry, canGoUp bool, ro
 		fileIsSymlink := isSymlink(info)
 		if fileIsSymlink {
 			path := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, info.Name()))
-			fileInfo, err := fsrv.fileSystem.Stat(path)
+			fileInfo, err := fs.Stat(fsrv.fileSystem, path)
 			if err == nil {
 				size = fileInfo.Size()
 			}

modules/caddyhttp/fileserver/caddyfile.go

@@ -77,11 +77,11 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 				if err != nil {
 					return nil, err
 				}
-				statFS, ok := unm.(fs.StatFS)
+				fsys, ok := unm.(fs.FS)
 				if !ok {
-					return nil, h.Errf("module %s (%T) is not a supported file system implementation (requires fs.StatFS)", modID, unm)
+					return nil, h.Errf("module %s (%T) is not a supported file system implementation (requires fs.FS)", modID, unm)
 				}
-				fsrv.FileSystemRaw = caddyconfig.JSONModuleObject(statFS, "backend", name, nil)
+				fsrv.FileSystemRaw = caddyconfig.JSONModuleObject(fsys, "backend", name, nil)
 
 			case "hide":
 				fsrv.Hide = h.RemainingArgs()

modules/caddyhttp/fileserver/command.go

@@ -27,6 +27,7 @@ import (
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	caddytpl "github.com/caddyserver/caddy/v2/modules/caddyhttp/templates"
 	"github.com/caddyserver/certmagic"
+	"go.uber.org/zap"
 )
 
 func init() {
@@ -56,6 +57,7 @@ respond with a file listing.`,
 			fs.Bool("browse", false, "Enable directory browsing")
 			fs.Bool("templates", false, "Enable template rendering")
 			fs.Bool("access-log", false, "Enable the access log")
+			fs.Bool("debug", false, "Enable verbose debug logs")
 			return fs
 		}(),
 	})
@@ -70,6 +72,7 @@ func cmdFileServer(fs caddycmd.Flags) (int, error) {
 	browse := fs.Bool("browse")
 	templates := fs.Bool("templates")
 	accessLog := fs.Bool("access-log")
+	debug := fs.Bool("debug")
 
 	var handlers []json.RawMessage
 
@@ -130,6 +133,14 @@ func cmdFileServer(fs caddycmd.Flags) (int, error) {
 		},
 	}
 
+	if debug {
+		cfg.Logging = &caddy.Logging{
+			Logs: map[string]*caddy.CustomLog{
+				"default": {Level: zap.DebugLevel.CapitalString()},
+			},
+		}
+	}
+
 	err := caddy.Run(cfg)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, err

modules/caddyhttp/fileserver/matcher.go

@@ -64,7 +64,7 @@ type MatchFile struct {
 	// The file system implementation to use. By default, the
 	// local disk file system will be used.
 	FileSystemRaw json.RawMessage `json:"file_system,omitempty" caddy:"namespace=caddy.fs inline_key=backend"`
-	fileSystem    fs.StatFS
+	fileSystem    fs.FS
 
 	// The root directory, used for creating absolute
 	// file paths, and required when working with
@@ -163,6 +163,8 @@ func (m *MatchFile) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 //
 // Example:
 //
+//	expression file()
+//	expression file({http.request.uri.path}, '/index.php')
 //	expression file({'root': '/srv', 'try_files': [{http.request.uri.path}, '/index.php'], 'try_policy': 'first_exist', 'split_path': ['.php']})
 func (MatchFile) CELLibrary(ctx caddy.Context) (cel.Library, error) {
 	requestType := cel.ObjectType("http.Request")
@@ -199,7 +201,7 @@ func (MatchFile) CELLibrary(ctx caddy.Context) (cel.Library, error) {
 		cel.Function("file", cel.Overload("file_request_map", []*cel.Type{requestType, caddyhttp.CELTypeJSON}, cel.BoolType)),
 		cel.Function("file_request_map",
 			cel.Overload("file_request_map", []*cel.Type{requestType, caddyhttp.CELTypeJSON}, cel.BoolType),
-			cel.SingletonBinaryImpl(caddyhttp.CELMatcherRuntimeFunction("file_request_map", matcherFactory))),
+			cel.SingletonBinaryBinding(caddyhttp.CELMatcherRuntimeFunction("file_request_map", matcherFactory))),
 	}
 
 	programOptions := []cel.ProgramOption{
@@ -212,18 +214,21 @@ func (MatchFile) CELLibrary(ctx caddy.Context) (cel.Library, error) {
 func celFileMatcherMacroExpander() parser.MacroExpander {
 	return func(eh parser.ExprHelper, target *exprpb.Expr, args []*exprpb.Expr) (*exprpb.Expr, *common.Error) {
 		if len(args) == 0 {
-			return nil, &common.Error{
-				Message: "matcher requires at least one argument",
-			}
+			return eh.GlobalCall("file",
+				eh.Ident("request"),
+				eh.NewMap(),
+			), nil
 		}
 		if len(args) == 1 {
 			arg := args[0]
 			if isCELStringLiteral(arg) || isCELCaddyPlaceholderCall(arg) {
 				return eh.GlobalCall("file",
 					eh.Ident("request"),
-					eh.NewMap(
-						eh.NewMapEntry(eh.LiteralString("try_files"), eh.NewList(arg)),
-					),
+					eh.NewMap(eh.NewMapEntry(
+						eh.LiteralString("try_files"),
+						eh.NewList(arg),
+						false,
+					)),
 				), nil
 			}
 			if isCELTryFilesLiteral(arg) {
@@ -245,18 +250,18 @@ func celFileMatcherMacroExpander() parser.MacroExpander {
 		}
 		return eh.GlobalCall("file",
 			eh.Ident("request"),
-			eh.NewMap(
-				eh.NewMapEntry(
-					eh.LiteralString("try_files"), eh.NewList(args...),
-				),
-			),
+			eh.NewMap(eh.NewMapEntry(
+				eh.LiteralString("try_files"),
+				eh.NewList(args...),
+				false,
+			)),
 		), nil
 	}
 }
 
 // Provision sets up m's defaults.
 func (m *MatchFile) Provision(ctx caddy.Context) error {
-	m.logger = ctx.Logger(m)
+	m.logger = ctx.Logger()
 
 	// establish the file system to use
 	if len(m.FileSystemRaw) > 0 {
@@ -264,7 +269,7 @@ func (m *MatchFile) Provision(ctx caddy.Context) error {
 		if err != nil {
 			return fmt.Errorf("loading file system module: %v", err)
 		}
-		m.fileSystem = mod.(fs.StatFS)
+		m.fileSystem = mod.(fs.FS)
 	}
 	if m.fileSystem == nil {
 		m.fileSystem = osFS{}
@@ -418,7 +423,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
 		for _, pattern := range m.TryFiles {
 			candidates := makeCandidates(pattern)
 			for _, c := range candidates {
-				info, err := m.fileSystem.Stat(c.fullpath)
+				info, err := fs.Stat(m.fileSystem, c.fullpath)
 				if err == nil && info.Size() > largestSize {
 					largestSize = info.Size()
 					largest = c
@@ -439,7 +444,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
 		for _, pattern := range m.TryFiles {
 			candidates := makeCandidates(pattern)
 			for _, c := range candidates {
-				info, err := m.fileSystem.Stat(c.fullpath)
+				info, err := fs.Stat(m.fileSystem, c.fullpath)
 				if err == nil && (smallestSize == 0 || info.Size() < smallestSize) {
 					smallestSize = info.Size()
 					smallest = c
@@ -459,7 +464,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
 		for _, pattern := range m.TryFiles {
 			candidates := makeCandidates(pattern)
 			for _, c := range candidates {
-				info, err := m.fileSystem.Stat(c.fullpath)
+				info, err := fs.Stat(m.fileSystem, c.fullpath)
 				if err == nil &&
 					(recentInfo == nil || info.ModTime().After(recentInfo.ModTime())) {
 					recent = c
@@ -498,7 +503,7 @@ func parseErrorCode(input string) error {
 // NOT end in a forward slash, the file must NOT
 // be a directory.
 func (m MatchFile) strictFileExists(file string) (os.FileInfo, bool) {
-	info, err := m.fileSystem.Stat(file)
+	info, err := fs.Stat(m.fileSystem, file)
 	if err != nil {
 		// in reality, this can be any error
 		// such as permission or even obscure

modules/caddyhttp/fileserver/matcher_test.go

@@ -62,6 +62,12 @@ func TestFileMatcher(t *testing.T) {
 			expectedType: "file",
 			matched:      true,
 		},
+		{
+			path:         "/foo.txt?a=b",
+			expectedPath: "/foo.txt",
+			expectedType: "file",
+			matched:      true,
+		},
 		{
 			path:         "/foodir",
 			expectedPath: "/foodir/",
@@ -211,6 +217,12 @@ func TestPHPFileMatcher(t *testing.T) {
 			expectedType: "file",
 			matched:      false,
 		},
+		{
+			path:         "/index.php?path={path}&{query}",
+			expectedPath: "/index.php",
+			expectedType: "file",
+			matched:      true,
+		},
 	} {
 		m := &MatchFile{
 			fileSystem: osFS{},
@@ -280,8 +292,8 @@ var (
 			expression: &caddyhttp.MatchExpression{
 				Expr: `file()`,
 			},
-			urlTarget: "https://example.com/foo",
-			wantErr:   true,
+			urlTarget:  "https://example.com/foo.txt",
+			wantResult: true,
 		},
 		{
 			name: "file error bad try files (MatchFile)",

modules/caddyhttp/fileserver/staticfiles.go

@@ -26,6 +26,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -83,14 +84,14 @@ type FileServer struct {
 	// disk file system.
 	//
 	// File system modules used here must adhere to the following requirements:
-	// - Implement fs.StatFS interface.
+	// - Implement fs.FS interface.
 	// - Support seeking on opened files; i.e.returned fs.File values must
 	//   implement the io.Seeker interface. This is required for determining
 	//   Content-Length and satisfying Range requests.
 	// - fs.File values that represent directories must implement the
 	//   fs.ReadDirFile interface so that directory listings can be procured.
 	FileSystemRaw json.RawMessage `json:"file_system,omitempty" caddy:"namespace=caddy.fs inline_key=backend"`
-	fileSystem    fs.StatFS
+	fileSystem    fs.FS
 
 	// The path to the root of the site. Default is `{http.vars.root}` if set,
 	// or current working directory otherwise. This should be a trusted value.
@@ -167,7 +168,7 @@ func (FileServer) CaddyModule() caddy.ModuleInfo {
 
 // Provision sets up the static files responder.
 func (fsrv *FileServer) Provision(ctx caddy.Context) error {
-	fsrv.logger = ctx.Logger(fsrv)
+	fsrv.logger = ctx.Logger()
 
 	// establish which file system (possibly a virtual one) we'll be using
 	if len(fsrv.FileSystemRaw) > 0 {
@@ -175,7 +176,7 @@ func (fsrv *FileServer) Provision(ctx caddy.Context) error {
 		if err != nil {
 			return fmt.Errorf("loading file system module: %v", err)
 		}
-		fsrv.fileSystem = mod.(fs.StatFS)
+		fsrv.fileSystem = mod.(fs.FS)
 	}
 	if fsrv.fileSystem == nil {
 		fsrv.fileSystem = osFS{}
@@ -232,6 +233,19 @@ func (fsrv *FileServer) Provision(ctx caddy.Context) error {
 func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
 
+	if runtime.GOOS == "windows" {
+		// reject paths with Alternate Data Streams (ADS)
+		if strings.Contains(r.URL.Path, ":") {
+			return caddyhttp.Error(http.StatusBadRequest, fmt.Errorf("illegal ADS path"))
+		}
+		// reject paths with "8.3" short names
+		trimmedPath := strings.TrimRight(r.URL.Path, ". ") // Windows ignores trailing dots and spaces, sigh
+		if len(path.Base(trimmedPath)) <= 12 && strings.Contains(trimmedPath, "~") {
+			return caddyhttp.Error(http.StatusBadRequest, fmt.Errorf("illegal short name"))
+		}
+		// both of those could bypass file hiding or possibly leak information even if the file is not hidden
+	}
+
 	filesToHide := fsrv.transformHidePaths(repl)
 
 	root := repl.ReplaceAll(fsrv.Root, ".")
@@ -244,10 +258,10 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 		zap.String("result", filename))
 
 	// get information about the file
-	info, err := fsrv.fileSystem.Stat(filename)
+	info, err := fs.Stat(fsrv.fileSystem, filename)
 	if err != nil {
 		err = fsrv.mapDirOpenError(err, filename)
-		if errors.Is(err, fs.ErrNotExist) {
+		if errors.Is(err, fs.ErrNotExist) || errors.Is(err, fs.ErrInvalid) {
 			return fsrv.notFound(w, r, next)
 		} else if errors.Is(err, fs.ErrPermission) {
 			return caddyhttp.Error(http.StatusForbidden, err)
@@ -270,7 +284,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 				continue
 			}
 
-			indexInfo, err := fsrv.fileSystem.Stat(indexPath)
+			indexInfo, err := fs.Stat(fsrv.fileSystem, indexPath)
 			if err != nil {
 				continue
 			}
@@ -350,7 +364,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 			continue
 		}
 		compressedFilename := filename + precompress.Suffix()
-		compressedInfo, err := fsrv.fileSystem.Stat(compressedFilename)
+		compressedInfo, err := fs.Stat(fsrv.fileSystem, compressedFilename)
 		if err != nil || compressedInfo.IsDir() {
 			fsrv.logger.Debug("precompressed file not accessible", zap.String("filename", compressedFilename), zap.Error(err))
 			continue
@@ -396,6 +410,14 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 		etag = calculateEtag(info)
 	}
 
+	// at this point, we're serving a file; Go std lib supports only
+	// GET and HEAD, which is sensible for a static file server - reject
+	// any other methods (see issue #5166)
+	if r.Method != http.MethodGet && r.Method != http.MethodHead {
+		w.Header().Add("Allow", "GET, HEAD")
+		return caddyhttp.Error(http.StatusMethodNotAllowed, nil)
+	}
+
 	// set the Etag - note that a conditional If-None-Match request is handled
 	// by http.ServeContent below, which checks against this Etag value
 	w.Header().Set("Etag", etag)
@@ -490,7 +512,7 @@ func (fsrv *FileServer) mapDirOpenError(originalErr error, name string) error {
 		if parts[i] == "" {
 			continue
 		}
-		fi, err := fsrv.fileSystem.Stat(strings.Join(parts[:i+1], separator))
+		fi, err := fs.Stat(fsrv.fileSystem, strings.Join(parts[:i+1], separator))
 		if err != nil {
 			return originalErr
 		}
@@ -613,13 +635,13 @@ func (wr statusOverrideResponseWriter) WriteHeader(int) {
 	wr.ResponseWriter.WriteHeader(wr.code)
 }
 
-// osFS is a simple fs.StatFS implementation that uses the local
+// osFS is a simple fs.FS implementation that uses the local
 // file system. (We do not use os.DirFS because we do our own
 // rooting or path prefixing without being constrained to a single
 // root folder. The standard os.DirFS implementation is problematic
 // since roots can be dynamic in our application.)
 //
-// osFS also implements fs.GlobFS, fs.ReadDirFS, and fs.ReadFileFS.
+// osFS also implements fs.StatFS, fs.GlobFS, fs.ReadDirFS, and fs.ReadFileFS.
 type osFS struct{}
 
 func (osFS) Open(name string) (fs.File, error)          { return os.Open(name) }
@@ -640,6 +662,7 @@ var (
 	_ caddy.Provisioner           = (*FileServer)(nil)
 	_ caddyhttp.MiddlewareHandler = (*FileServer)(nil)
 
+	_ fs.StatFS     = (*osFS)(nil)
 	_ fs.GlobFS     = (*osFS)(nil)
 	_ fs.ReadDirFS  = (*osFS)(nil)
 	_ fs.ReadFileFS = (*osFS)(nil)

modules/caddyhttp/headers/caddyfile.go

@@ -32,12 +32,12 @@ func init() {
 // parseCaddyfile sets up the handler for response headers from
 // Caddyfile tokens. Syntax:
 //
-//     header [<matcher>] [[+|-|?]<field> [<value|regexp>] [<replacement>]] {
-//         [+]<field> [<value|regexp> [<replacement>]]
-//         ?<field> <default_value>
-//         -<field>
-//         [defer]
-//     }
+//	header [<matcher>] [[+|-|?]<field> [<value|regexp>] [<replacement>]] {
+//	    [+]<field> [<value|regexp> [<replacement>]]
+//	    ?<field> <default_value>
+//	    -<field>
+//	    [defer]
+//	}
 //
 // Either a block can be opened or a single header field can be configured
 // in the first line, but not both in the same directive. Header operations
@@ -148,8 +148,7 @@ func parseCaddyfile(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error)
 // parseReqHdrCaddyfile sets up the handler for request headers
 // from Caddyfile tokens. Syntax:
 //
-//     request_header [<matcher>] [[+|-]<field> [<value|regexp>] [<replacement>]]
-//
+//	request_header [<matcher>] [[+|-]<field> [<value|regexp>] [<replacement>]]
 func parseReqHdrCaddyfile(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error) {
 	if !h.Next() {
 		return nil, h.ArgErr()

modules/caddyhttp/headers/headers.go

@@ -332,7 +332,10 @@ func (rww *responseWriterWrapper) WriteHeader(status int) {
 	if rww.wroteHeader {
 		return
 	}
-	rww.wroteHeader = true
+	// 1xx responses aren't final; just informational
+	if status < 100 || status > 199 {
+		rww.wroteHeader = true
+	}
 	if rww.require == nil || rww.require.Match(status, rww.ResponseWriterWrapper.Header()) {
 		if rww.headerOps != nil {
 			rww.headerOps.ApplyTo(rww.ResponseWriterWrapper.Header(), rww.replacer)

modules/caddyhttp/ip_range.go

@@ -0,0 +1,142 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddyhttp
+
+import (
+	"fmt"
+	"net/http"
+	"net/netip"
+	"strings"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+)
+
+func init() {
+	caddy.RegisterModule(StaticIPRange{})
+}
+
+// IPRangeSource gets a list of IP ranges.
+//
+// The request is passed as an argument to allow plugin implementations
+// to have more flexibility. But, a plugin MUST NOT modify the request.
+// The caller will have read the `r.RemoteAddr` before getting IP ranges.
+//
+// This should be a very fast function -- instant if possible.
+// The list of IP ranges should be sourced as soon as possible if loaded
+// from an external source (i.e. initially loaded during Provisioning),
+// so that it's ready to be used when requests start getting handled.
+// A read lock should probably be used to get the cached value if the
+// ranges can change at runtime (e.g. periodically refreshed).
+// Using a `caddy.UsagePool` may be a good idea to avoid having refetch
+// the values when a config reload occurs, which would waste time.
+//
+// If the list of IP ranges cannot be sourced, then provisioning SHOULD
+// fail. Getting the IP ranges at runtime MUST NOT fail, because it would
+// cancel incoming requests. If refreshing the list fails, then the
+// previous list of IP ranges should continue to be returned so that the
+// server can continue to operate normally.
+type IPRangeSource interface {
+	GetIPRanges(*http.Request) []netip.Prefix
+}
+
+// StaticIPRange provides a static range of IP address prefixes (CIDRs).
+type StaticIPRange struct {
+	// A static list of IP ranges (supports CIDR notation).
+	Ranges []string `json:"ranges,omitempty"`
+
+	// Holds the parsed CIDR ranges from Ranges.
+	ranges []netip.Prefix
+}
+
+// CaddyModule returns the Caddy module information.
+func (StaticIPRange) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "http.ip_sources.static",
+		New: func() caddy.Module { return new(StaticIPRange) },
+	}
+}
+
+func (s *StaticIPRange) Provision(ctx caddy.Context) error {
+	for _, str := range s.Ranges {
+		prefix, err := CIDRExpressionToPrefix(str)
+		if err != nil {
+			return err
+		}
+		s.ranges = append(s.ranges, prefix)
+	}
+
+	return nil
+}
+
+func (s *StaticIPRange) GetIPRanges(_ *http.Request) []netip.Prefix {
+	return s.ranges
+}
+
+// UnmarshalCaddyfile implements caddyfile.Unmarshaler.
+func (m *StaticIPRange) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	if !d.Next() {
+		return nil
+	}
+	for d.NextArg() {
+		if d.Val() == "private_ranges" {
+			m.Ranges = append(m.Ranges, PrivateRangesCIDR()...)
+			continue
+		}
+		m.Ranges = append(m.Ranges, d.Val())
+	}
+	return nil
+}
+
+// CIDRExpressionToPrefix takes a string which could be either a
+// CIDR expression or a single IP address, and returns a netip.Prefix.
+func CIDRExpressionToPrefix(expr string) (netip.Prefix, error) {
+	// Having a slash means it should be a CIDR expression
+	if strings.Contains(expr, "/") {
+		prefix, err := netip.ParsePrefix(expr)
+		if err != nil {
+			return netip.Prefix{}, fmt.Errorf("parsing CIDR expression: '%s': %v", expr, err)
+		}
+		return prefix, nil
+	}
+
+	// Otherwise it's likely a single IP address
+	parsed, err := netip.ParseAddr(expr)
+	if err != nil {
+		return netip.Prefix{}, fmt.Errorf("invalid IP address: '%s': %v", expr, err)
+	}
+	prefix := netip.PrefixFrom(parsed, parsed.BitLen())
+	return prefix, nil
+}
+
+// PrivateRangesCIDR returns a list of private CIDR range
+// strings, which can be used as a configuration shortcut.
+func PrivateRangesCIDR() []string {
+	return []string{
+		"192.168.0.0/16",
+		"172.16.0.0/12",
+		"10.0.0.0/8",
+		"127.0.0.1/8",
+		"fd00::/8",
+		"::1",
+	}
+}
+
+// Interface guards
+var (
+	_ caddy.Provisioner     = (*StaticIPRange)(nil)
+	_ caddyfile.Unmarshaler = (*StaticIPRange)(nil)
+	_ IPRangeSource         = (*StaticIPRange)(nil)
+)

modules/caddyhttp/logging.go

@@ -0,0 +1,144 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddyhttp
+
+import (
+	"errors"
+	"net"
+	"net/http"
+	"strings"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+// ServerLogConfig describes a server's logging configuration. If
+// enabled without customization, all requests to this server are
+// logged to the default logger; logger destinations may be
+// customized per-request-host.
+type ServerLogConfig struct {
+	// The default logger name for all logs emitted by this server for
+	// hostnames that are not in the LoggerNames (logger_names) map.
+	DefaultLoggerName string `json:"default_logger_name,omitempty"`
+
+	// LoggerNames maps request hostnames to a custom logger name.
+	// For example, a mapping of "example.com" to "example" would
+	// cause access logs from requests with a Host of example.com
+	// to be emitted by a logger named "http.log.access.example".
+	LoggerNames map[string]string `json:"logger_names,omitempty"`
+
+	// By default, all requests to this server will be logged if
+	// access logging is enabled. This field lists the request
+	// hosts for which access logging should be disabled.
+	SkipHosts []string `json:"skip_hosts,omitempty"`
+
+	// If true, requests to any host not appearing in the
+	// LoggerNames (logger_names) map will not be logged.
+	SkipUnmappedHosts bool `json:"skip_unmapped_hosts,omitempty"`
+
+	// If true, credentials that are otherwise omitted, will be logged.
+	// The definition of credentials is defined by https://fetch.spec.whatwg.org/#credentials,
+	// and this includes some request and response headers, i.e `Cookie`,
+	// `Set-Cookie`, `Authorization`, and `Proxy-Authorization`.
+	ShouldLogCredentials bool `json:"should_log_credentials,omitempty"`
+}
+
+// wrapLogger wraps logger in a logger named according to user preferences for the given host.
+func (slc ServerLogConfig) wrapLogger(logger *zap.Logger, host string) *zap.Logger {
+	if loggerName := slc.getLoggerName(host); loggerName != "" {
+		return logger.Named(loggerName)
+	}
+	return logger
+}
+
+func (slc ServerLogConfig) getLoggerName(host string) string {
+	tryHost := func(key string) (string, bool) {
+		// first try exact match
+		if loggerName, ok := slc.LoggerNames[key]; ok {
+			return loggerName, ok
+		}
+		// strip port and try again (i.e. Host header of "example.com:1234" should
+		// match "example.com" if there is no "example.com:1234" in the map)
+		hostOnly, _, err := net.SplitHostPort(key)
+		if err != nil {
+			return "", false
+		}
+		loggerName, ok := slc.LoggerNames[hostOnly]
+		return loggerName, ok
+	}
+
+	// try the exact hostname first
+	if loggerName, ok := tryHost(host); ok {
+		return loggerName
+	}
+
+	// try matching wildcard domains if other non-specific loggers exist
+	labels := strings.Split(host, ".")
+	for i := range labels {
+		if labels[i] == "" {
+			continue
+		}
+		labels[i] = "*"
+		wildcardHost := strings.Join(labels, ".")
+		if loggerName, ok := tryHost(wildcardHost); ok {
+			return loggerName
+		}
+	}
+
+	return slc.DefaultLoggerName
+}
+
+func (slc *ServerLogConfig) clone() *ServerLogConfig {
+	clone := &ServerLogConfig{
+		DefaultLoggerName:    slc.DefaultLoggerName,
+		LoggerNames:          make(map[string]string),
+		SkipHosts:            append([]string{}, slc.SkipHosts...),
+		SkipUnmappedHosts:    slc.SkipUnmappedHosts,
+		ShouldLogCredentials: slc.ShouldLogCredentials,
+	}
+	for k, v := range slc.LoggerNames {
+		clone.LoggerNames[k] = v
+	}
+	return clone
+}
+
+// errLogValues inspects err and returns the status code
+// to use, the error log message, and any extra fields.
+// If err is a HandlerError, the returned values will
+// have richer information.
+func errLogValues(err error) (status int, msg string, fields []zapcore.Field) {
+	var handlerErr HandlerError
+	if errors.As(err, &handlerErr) {
+		status = handlerErr.StatusCode
+		if handlerErr.Err == nil {
+			msg = err.Error()
+		} else {
+			msg = handlerErr.Err.Error()
+		}
+		fields = []zapcore.Field{
+			zap.Int("status", handlerErr.StatusCode),
+			zap.String("err_id", handlerErr.ID),
+			zap.String("err_trace", handlerErr.Trace),
+		}
+		return
+	}
+	status = http.StatusInternalServerError
+	msg = err.Error()
+	return
+}
+
+// Variable name used to indicate that this request
+// should be omitted from the access logs
+const SkipLogVar = "skip_log"

modules/caddyhttp/map/map.go

@@ -40,6 +40,7 @@ type Handler struct {
 	Source string `json:"source,omitempty"`
 
 	// Destinations are the names of placeholders in which to store the outputs.
+	// Destination values should be wrapped in braces, for example, {my_placeholder}.
 	Destinations []string `json:"destinations,omitempty"`
 
 	// Mappings from source values (inputs) to destination values (outputs).
@@ -109,22 +110,13 @@ func (h *Handler) Validate() error {
 		}
 		seen[input] = i
 
-		// prevent infinite recursion
-		for _, out := range m.Outputs {
-			for _, dest := range h.Destinations {
-				if strings.Contains(caddy.ToString(out), dest) ||
-					strings.Contains(m.Input, dest) {
-					return fmt.Errorf("mapping %d requires value of {%s} to define value of {%s}: infinite recursion", i, dest, dest)
-				}
-			}
-		}
-
 		// ensure mappings have 1:1 output-to-destination correspondence
 		nOut := len(m.Outputs)
 		if nOut != nDest {
 			return fmt.Errorf("mapping %d has %d outputs but there are %d destinations defined", i, nOut, nDest)
 		}
 	}
+
 	return nil
 }
 
@@ -169,7 +161,7 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhtt
 
 		// fall back to default if no match or if matched nil value
 		if len(h.Defaults) > destIdx {
-			return h.Defaults[destIdx], true
+			return repl.ReplaceAll(h.Defaults[destIdx], ""), true
 		}
 
 		return nil, true

modules/caddyhttp/map/map_test.go

@@ -98,6 +98,28 @@ func TestHandler(t *testing.T) {
 				"output": "testing",
 			},
 		},
+		{
+			reqURI: "/foo",
+			handler: Handler{
+				Source:       "{http.request.uri.path}",
+				Destinations: []string{"{output}"},
+				Defaults:     []string{"default"},
+			},
+			expect: map[string]any{
+				"output": "default",
+			},
+		},
+		{
+			reqURI: "/foo",
+			handler: Handler{
+				Source:       "{http.request.uri.path}",
+				Destinations: []string{"{output}"},
+				Defaults:     []string{"{testvar}"},
+			},
+			expect: map[string]any{
+				"output": "testing",
+			},
+		},
 	} {
 		if err := tc.handler.Provision(caddy.Context{}); err != nil {
 			t.Fatalf("Test %d: Provisioning handler: %v", i, err)

modules/caddyhttp/matchers.go

@@ -123,6 +123,7 @@ type (
 	// keyed by the query keys, with an array of string values to match for that key.
 	// Query key matches are exact, but wildcards may be used for value matches. Both
 	// keys and values may be placeholders.
+	//
 	// An example of the structure to match `?key=value&topic=api&query=something` is:
 	//
 	// ```json
@@ -135,6 +136,13 @@ type (
 	//
 	// Invalid query strings, including those with bad escapings or illegal characters
 	// like semicolons, will fail to parse and thus fail to match.
+	//
+	// **NOTE:** Notice that query string values are arrays, not singular values. This is
+	// because repeated keys are valid in query strings, and each one may have a
+	// different value. This matcher will match for a key if any one of its configured
+	// values is assigned in the query string. Backend applications relying on query
+	// strings MUST take into consideration that query string values are arrays and can
+	// have multiple values.
 	MatchQuery url.Values
 
 	// MatchHeader matches requests by header fields. The key is the field
@@ -144,6 +152,13 @@ type (
 	// surrounding the value with the wildcard `*` character, respectively.
 	// If a list is null, the header must not exist. If the list is empty,
 	// the field must simply exist, regardless of its value.
+	//
+	// **NOTE:** Notice that header values are arrays, not singular values. This is
+	// because repeated fields are valid in headers, and each one may have a
+	// different value. This matcher will match for a field if any one of its configured
+	// values matches in the header. Backend applications relying on headers MUST take
+	// into consideration that header field values are arrays and can have multiple
+	// values.
 	MatchHeader http.Header
 
 	// MatchHeaderRE matches requests by a regular expression on header fields.
@@ -156,7 +171,9 @@ type (
 	MatchHeaderRE map[string]*MatchRegexp
 
 	// MatchProtocol matches requests by protocol. Recognized values are
-	// "http", "https", and "grpc".
+	// "http", "https", and "grpc" for broad protocol matches, or specific
+	// HTTP versions can be specified like so: "http/1", "http/1.1",
+	// "http/2", "http/3", or minimum versions: "http/2+", etc.
 	MatchProtocol string
 
 	// MatchRemoteIP matches requests by client IP (or CIDR range).
@@ -374,7 +391,11 @@ func (MatchPath) CaddyModule() caddy.ModuleInfo {
 // Provision lower-cases the paths in m to ensure case-insensitive matching.
 func (m MatchPath) Provision(_ caddy.Context) error {
 	for i := range m {
-		// TODO: if m[i] == "*", put it first and delete all others (will always match)
+		if m[i] == "*" && i > 0 {
+			// will always match, so just put it first
+			m[0] = m[i]
+			break
+		}
 		m[i] = strings.ToLower(m[i])
 	}
 	return nil
@@ -1001,6 +1022,12 @@ func (m *MatchHeaderRE) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 			val = second
 		}
 
+		// If there's already a pattern for this field
+		// then we would end up overwriting the old one
+		if (*m)[field] != nil {
+			return d.Errf("header_regexp matcher can only be used once per named matcher, per header field: %s", field)
+		}
+
 		(*m)[field] = &MatchRegexp{Pattern: val, Name: name}
 
 		if d.NextBlock(0) {
@@ -1254,14 +1281,7 @@ func (m *MatchRemoteIP) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 				continue
 			}
 			if d.Val() == "private_ranges" {
-				m.Ranges = append(m.Ranges, []string{
-					"192.168.0.0/16",
-					"172.16.0.0/12",
-					"10.0.0.0/8",
-					"127.0.0.1/8",
-					"fd00::/8",
-					"::1",
-				}...)
+				m.Ranges = append(m.Ranges, PrivateRangesCIDR()...)
 				continue
 			}
 			m.Ranges = append(m.Ranges, d.Val())
@@ -1316,7 +1336,7 @@ func (MatchRemoteIP) CELLibrary(ctx caddy.Context) (cel.Library, error) {
 
 // Provision parses m's IP ranges, either from IP or CIDR expressions.
 func (m *MatchRemoteIP) Provision(ctx caddy.Context) error {
-	m.logger = ctx.Logger(m)
+	m.logger = ctx.Logger()
 	for _, str := range m.Ranges {
 		// Exclude the zone_id from the IP
 		if strings.Contains(str, "%") {
@@ -1469,6 +1489,13 @@ func (mre *MatchRegexp) Match(input string, repl *caddy.Replacer) bool {
 // UnmarshalCaddyfile implements caddyfile.Unmarshaler.
 func (mre *MatchRegexp) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	for d.Next() {
+		// If this is the second iteration of the loop
+		// then there's more than one path_regexp matcher
+		// and we would end up overwriting the old one
+		if mre.Pattern != "" {
+			return d.Err("regular expression can only be used once per named matcher")
+		}
+
 		args := d.RemainingArgs()
 		switch len(args) {
 		case 1: