noot

* reverseproxy: add Max-Age option to sticky cookie

* Update selectionpolicies.go

Co-authored-by: Francis Lavoie <lavofr@gmail.com>

* Update selectionpolicies.go

Co-authored-by: Francis Lavoie <lavofr@gmail.com>

---------

Co-authored-by: Francis Lavoie <lavofr@gmail.com>

.editorconfig

@@ -1,5 +1,5 @@
 [*]
 end_of_line = lf
 
-[caddytest/integration/caddyfile_adapt/*.txt]
+[caddytest/integration/caddyfile_adapt/*.caddyfiletest]
 indent_style = tab

.github/CONTRIBUTING.md

@@ -25,7 +25,7 @@ Other menu items:
 
 You can have a huge impact on the project by helping with its code. To contribute code to Caddy, first submit or comment in an issue to discuss your contribution, then open a [pull request](https://github.com/caddyserver/caddy/pulls) (PR). If you're new to our community, that's okay: **we gladly welcome pull requests from anyone, regardless of your native language or coding experience.** You can get familiar with Caddy's code base by using [code search at Sourcegraph](https://sourcegraph.com/github.com/caddyserver/caddy).
 
-We hold contributions to a high standard for quality :bowtie:, so don't be surprised if we ask for revisions—even if it seems small or insignificant. Please don't take it personally. :blue_heart: If your change is on the right track, we can guide you to make it mergable.
+We hold contributions to a high standard for quality :bowtie:, so don't be surprised if we ask for revisions—even if it seems small or insignificant. Please don't take it personally. :blue_heart: If your change is on the right track, we can guide you to make it mergeable.
 
 Here are some of the expectations we have of contributors:
 

.github/SECURITY.md

@@ -7,7 +7,7 @@ The Caddy project would like to make sure that it stays on top of all practicall
 
 | Version | Supported          |
 | ------- | ------------------ |
-| 2.x     | :white_check_mark: |
+| 2.x     | ✔️ |
 | 1.x     | :x:                |
 | < 1.x   | :x:                |
 
@@ -24,7 +24,7 @@ We do not accept reports if the steps imply or require a compromised system or t
 
 Client-side exploits are out of scope. In other words, it is not a bug in Caddy if the web browser does something unsafe, even if the downloaded content was served by Caddy. (Those kinds of exploits can generally be mitigated by proper configuration of HTTP headers.) As a general rule, the content served by Caddy is not considered in scope because content is configurable by the site owner or the associated web application.
 
-Security bugs in code dependencies are out of scope. Instead, if a dependency has patched a relevant security bug, please feel free to open a public issue or pull request to update that dependency in our code.
+Security bugs in code dependencies (including Go's standard library) are out of scope. Instead, if a dependency has patched a relevant security bug, please feel free to open a public issue or pull request to update that dependency in our code.
 
 
 ## Reporting a Vulnerability
@@ -42,7 +42,7 @@ We'll need enough information to verify the bug and make a patch. To speed thing
 - Specific minimal steps to reproduce the issue from scratch
 - A working patch
 
-Please DO NOT use containers, VMs, cloud instances or services, or any other complex infrastructure in your steps. Always prefer `curl` instead of web browsers.
+Please DO NOT use containers, VMs, cloud instances or services, or any other complex infrastructure in your steps. Always prefer `curl -v` instead of web browsers.
 
 We consider publicly-registered domain names to be public information. This necessary in order to maintain the integrity of certificate transparency, public DNS, and other public trust systems. Do not redact domain names from your reports. The actual content of your domain name affects Caddy's behavior, so we need the exact domain name(s) to reproduce with, or your report will be ignored.
 

.github/workflows/ci.yml

@@ -18,41 +18,50 @@ jobs:
       # Default is true, cancels jobs for other platforms in the matrix if one fails
       fail-fast: false
       matrix:
-        os: [ ubuntu-latest, macos-latest, windows-latest ]
+        os: 
-        go: [ '1.19', '1.20' ]
+          - linux
+          - mac
+          - windows
+        go: 
+          - '1.21'
+          - '1.22'
 
         include:
         # Set the minimum Go patch version for the given Go minor
         # Usable via ${{ matrix.GO_SEMVER }}
-        - go: '1.19'
+        - go: '1.21'
-          GO_SEMVER: '~1.19.6'
+          GO_SEMVER: '~1.21.0'
 
-        - go: '1.20'
+        - go: '1.22'
-          GO_SEMVER: '~1.20.1'
+          GO_SEMVER: '~1.22.3'
 
         # Set some variables per OS, usable via ${{ matrix.VAR }}
+        # OS_LABEL: the VM label from GitHub Actions (see https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories)
         # CADDY_BIN_PATH: the path to the compiled Caddy binary, for artifact publishing
         # SUCCESS: the typical value for $? per OS (Windows/pwsh returns 'True')
-        - os: ubuntu-latest
+        - os: linux
+          OS_LABEL: ubuntu-latest
           CADDY_BIN_PATH: ./cmd/caddy/caddy
           SUCCESS: 0
 
-        - os: macos-latest
+        - os: mac
+          OS_LABEL: macos-14
           CADDY_BIN_PATH: ./cmd/caddy/caddy
           SUCCESS: 0
 
-        - os: windows-latest
+        - os: windows
+          OS_LABEL: windows-latest
           CADDY_BIN_PATH: ./cmd/caddy/caddy.exe
           SUCCESS: 'True'
 
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.OS_LABEL }}
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Install Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: ${{ matrix.GO_SEMVER }}
         check-latest: true
@@ -68,6 +77,7 @@ jobs:
 
     - name: Print Go version and environment
       id: vars
+      shell: bash
       run: |
         printf "Using go at: $(which go)\n"
         printf "Go version: $(go version)\n"
@@ -89,13 +99,20 @@ jobs:
       env:
         CGO_ENABLED: 0
       run: |
-        go build -trimpath -ldflags="-w -s" -v
+        go build -tags nobdger -trimpath -ldflags="-w -s" -v
+
+    - name: Smoke test Caddy
+      working-directory: ./cmd/caddy
+      run: |
+        ./caddy start
+        ./caddy stop
 
     - name: Publish Build Artifact
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: caddy_${{ runner.os }}_go${{ matrix.go }}_${{ steps.vars.outputs.short_sha }}
         path: ${{ matrix.CADDY_BIN_PATH }}
+        compression-level: 0
 
     # Commented bits below were useful to allow the job to continue
     # even if the tests fail, so we can publish the report separately
@@ -105,7 +122,7 @@ jobs:
       # continue-on-error: true
       run: |
         # (go test -v -coverprofile=cover-profile.out -race ./... 2>&1) > test-results/test-result.out
-        go test -v -coverprofile="cover-profile.out" -short -race ./...
+        go test -tags nobadger -v -coverprofile="cover-profile.out" -short -race ./...
         # echo "status=$?" >> $GITHUB_OUTPUT
 
     # Relevant step if we reinvestigate publishing test/coverage reports
@@ -118,7 +135,7 @@ jobs:
 
     # To return the correct result even though we set 'continue-on-error: true'
     # - name: Coerce correct build result
-    #   if: matrix.os != 'windows-latest' && steps.step_test.outputs.status != ${{ matrix.SUCCESS }}
+    #   if: matrix.os != 'windows' && steps.step_test.outputs.status != ${{ matrix.SUCCESS }}
     #   run: |
     #     echo "step_test ${{ steps.step_test.outputs.status }}\n"
     #     exit 1
@@ -130,7 +147,7 @@ jobs:
     continue-on-error: true  # August 2020: s390x VM is down due to weather and power issues
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Run Tests
         run: |
           mkdir -p ~/.ssh && echo -e "${SSH_KEY//_/\\n}" > ~/.ssh/id_ecdsa && chmod og-rwx ~/.ssh/id_ecdsa
@@ -140,7 +157,7 @@ jobs:
 
           # The environment is fresh, so there's no point in keeping accepting and adding the key.
           rsync -arz -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" --progress --delete --exclude '.git' . "$CI_USER"@ci-s390x.caddyserver.com:/var/tmp/"$short_sha"
-          ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -t "$CI_USER"@ci-s390x.caddyserver.com "cd /var/tmp/$short_sha; go version; go env; printf "\n\n";CGO_ENABLED=0 go test -v ./..."
+          ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -t "$CI_USER"@ci-s390x.caddyserver.com "cd /var/tmp/$short_sha; go version; go env; printf "\n\n";CGO_ENABLED=0 go test -tags nobadger -v ./..."
           test_result=$?
 
           # There's no need leaving the files around
@@ -156,11 +173,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       
-      - uses: goreleaser/goreleaser-action@v4
+      - uses: goreleaser/goreleaser-action@v6
         with:
           version: latest
           args: check
-        env:
-          TAG: ${{ steps.vars.outputs.version_tag }}

.github/workflows/cross-build.yml

@@ -11,27 +11,38 @@ on:
       - 2.*
 
 jobs:
-  cross-build-test:
+  build:
     strategy:
       fail-fast: false
       matrix:
-        goos: ['android', 'linux', 'solaris', 'illumos', 'dragonfly', 'freebsd', 'openbsd', 'plan9', 'windows', 'darwin', 'netbsd']
+        goos: 
-        go: [ '1.20' ]
+          - 'aix'
+          - 'linux'
+          - 'solaris'
+          - 'illumos'
+          - 'dragonfly'
+          - 'freebsd'
+          - 'openbsd'
+          - 'windows'
+          - 'darwin'
+          - 'netbsd'
+        go: 
+          - '1.22'
 
         include:
         # Set the minimum Go patch version for the given Go minor
         # Usable via ${{ matrix.GO_SEMVER }}
-        - go: '1.20'
+        - go: '1.22'
-          GO_SEMVER: '~1.20.1'
+          GO_SEMVER: '~1.22.3'
 
     runs-on: ubuntu-latest
     continue-on-error: true
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.GO_SEMVER }}
           check-latest: true
@@ -50,12 +61,9 @@ jobs:
         env:
           CGO_ENABLED: 0
           GOOS: ${{ matrix.goos }}
+          GOARCH: ${{ matrix.goos == 'aix' && 'ppc64' || 'amd64' }}
         shell: bash
         continue-on-error: true
         working-directory: ./cmd/caddy
         run: |
-          GOOS=$GOOS go build -trimpath -o caddy-"$GOOS"-amd64 2> /dev/null
+          GOOS=$GOOS GOARCH=$GOARCH go build -tags nobadger -trimpath -o caddy-"$GOOS"-$GOARCH 2> /dev/null
-          if [ $? -ne 0 ]; then
-            echo "::warning ::$GOOS Build Failed"
-            exit 0
-          fi

.github/workflows/lint.yml

@@ -17,25 +17,51 @@ jobs:
   # From https://github.com/golangci/golangci-lint-action
   golangci:
     permissions:
-      contents: read  # for actions/checkout to fetch code
+      contents: read # for actions/checkout to fetch code
-      pull-requests: read  # for golangci/golangci-lint-action to fetch pull requests
+      pull-requests: read # for golangci/golangci-lint-action to fetch pull requests
     name: lint
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os:
-    runs-on: ${{ matrix.os }}
+          - linux
+          - mac
+          - windows
+
+        include:
+        - os: linux
+          OS_LABEL: ubuntu-latest
+
+        - os: mac
+          OS_LABEL: macos-14
+
+        - os: windows
+          OS_LABEL: windows-latest
+
+    runs-on: ${{ matrix.OS_LABEL }}
+
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '~1.19.6'
+          go-version: '~1.22.3'
           check-latest: true
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.50
+          version: v1.55
+
           # Windows times out frequently after about 5m50s if we don't set a longer timeout.
           args: --timeout 10m
+
           # Optional: show only new issues if it's a pull request. The default value is `false`.
           # only-new-issues: true
+
+  govulncheck:
+    runs-on: ubuntu-latest
+    steps:
+      - name: govulncheck
+        uses: golang/govulncheck-action@v1
+        with:
+          go-version-input: '~1.22.3'
+          check-latest: true

.github/workflows/release.yml

@@ -10,14 +10,16 @@ jobs:
     name: Release
     strategy:
       matrix:
-        os: [ ubuntu-latest ]
+        os: 
-        go: [ '1.20' ]
+          - ubuntu-latest
+        go: 
+          - '1.22'
 
         include:
         # Set the minimum Go patch version for the given Go minor
         # Usable via ${{ matrix.GO_SEMVER }}
-        - go: '1.20'
+        - go: '1.22'
-          GO_SEMVER: '~1.20.1'
+          GO_SEMVER: '~1.22.3'
 
     runs-on: ${{ matrix.os }}
     # https://github.com/sigstore/cosign/issues/1258#issuecomment-1002251233
@@ -30,18 +32,18 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
     - name: Install Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: ${{ matrix.GO_SEMVER }}
         check-latest: true
 
     # Force fetch upstream tags -- because 65 minutes
-    # tl;dr: actions/checkout@v3 runs this line:
+    # tl;dr: actions/checkout@v4 runs this line:
     #   git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +ebc278ec98bb24f2852b61fde2a9bf2e3d83818b:refs/tags/
     # which makes its own local lightweight tag, losing all the annotations in the process. Our earlier script ran:
     #   git fetch --prune --unshallow
@@ -104,10 +106,10 @@ jobs:
       run: syft version
     # GoReleaser will take care of publishing those artifacts into the release
     - name: Run GoReleaser
-      uses: goreleaser/goreleaser-action@v4
+      uses: goreleaser/goreleaser-action@v6
       with:
         version: latest
-        args: release --rm-dist --timeout 60m
+        args: release --clean --timeout 60m
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         TAG: ${{ steps.vars.outputs.version_tag }}

.github/workflows/release_published.yml

@@ -10,14 +10,15 @@ jobs:
     name: Release Published
     strategy:
       matrix:
-        os: [ ubuntu-latest ]
+        os: 
+          - ubuntu-latest
     runs-on: ${{ matrix.os }}
 
     steps:
 
     # See https://github.com/peter-evans/repository-dispatch
     - name: Trigger event on caddyserver/dist
-      uses: peter-evans/repository-dispatch@v2
+      uses: peter-evans/repository-dispatch@v3
       with:
         token: ${{ secrets.REPO_DISPATCH_TOKEN }}
         repository: caddyserver/dist
@@ -25,7 +26,7 @@ jobs:
         client-payload: '{"tag": "${{ github.event.release.tag_name }}"}'
 
     - name: Trigger event on caddyserver/caddy-docker
-      uses: peter-evans/repository-dispatch@v2
+      uses: peter-evans/repository-dispatch@v3
       with:
         token: ${{ secrets.REPO_DISPATCH_TOKEN }}
         repository: caddyserver/caddy-docker

.gitignore

@@ -3,6 +3,7 @@ _gitignore/
 Caddyfile
 Caddyfile.*
 !caddyfile/
+!caddyfile.go
 
 # artifacts from pprof tooling
 *.prof
@@ -12,6 +13,7 @@ Caddyfile.*
 cmd/caddy/caddy
 cmd/caddy/caddy.exe
 cmd/caddy/tmp/*.exe
+cmd/caddy/.env
 
 # mac specific
 .DS_Store

.golangci.yml

@@ -2,35 +2,81 @@ linters-settings:
   errcheck:
     ignore: fmt:.*,go.uber.org/zap/zapcore:^Add.*
     ignoretests: true
+  gci:
+    sections:
+      - standard # Standard section: captures all standard packages.
+      - default # Default section: contains all imports that could not be matched to another section type.
+      - prefix(github.com/caddyserver/caddy/v2/cmd) # ensure that this is always at the top and always has a line break.
+      - prefix(github.com/caddyserver/caddy) # Custom section: groups all imports with the specified Prefix.
+    # Skip generated files.
+    # Default: true
+    skip-generated: true
+    # Enable custom order of sections.
+    # If `true`, make the section order the same as the order of `sections`.
+    # Default: false
+    custom-order: true
+  exhaustive:
+    ignore-enum-types: reflect.Kind|svc.Cmd
 
 linters:
   disable-all: true
   enable:
+    - asasalint
+    - asciicheck
+    - bidichk
     - bodyclose
+    - decorder
+    - dogsled
+    - dupl
+    - dupword
+    - durationcheck
     - errcheck
+    - errname
+    - exhaustive
+    - exportloopref
+    - gci
     - gofmt
     - goimports
+    - gofumpt
     - gosec
     - gosimple
     - govet
     - ineffassign
+    - importas
     - misspell
     - prealloc
+    - promlinter
+    - sloglint
+    - sqlclosecheck
     - staticcheck
+    - tenv
+    - testableexamples
+    - testifylint
+    - tparallel
     - typecheck
     - unconvert
     - unused
+    - wastedassign
+    - whitespace
+    - zerologlint
   # these are implicitly disabled:
-  # - asciicheck
+  # - containedctx
+  # - contextcheck
+  # - cyclop
   # - depguard
-  # - dogsled
+  # - errchkjson
-  # - dupl
+  # - errorlint
-  # - exhaustive
+  # - exhaustruct
-  # - exportloopref
+  # - execinquery
+  # - exhaustruct
+  # - forbidigo
+  # - forcetypeassert
   # - funlen
-  # - gci
+  # - ginkgolinter
+  # - gocheckcompilerdirectives
   # - gochecknoglobals
   # - gochecknoinits
+  # - gochecksumtype
   # - gocognit
   # - goconst
   # - gocritic
@@ -38,27 +84,47 @@ linters:
   # - godot
   # - godox
   # - goerr113
-  # - gofumpt
   # - goheader
-  # - golint
   # - gomnd
+  # - gomoddirectives
   # - gomodguard
   # - goprintffuncname
-  # - interfacer
+  # - gosmopolitan
+  # - grouper
+  # - inamedparam
+  # - interfacebloat
+  # - ireturn
   # - lll
-  # - maligned
+  # - loggercheck
+  # - maintidx
+  # - makezero
+  # - mirror
+  # - musttag
   # - nakedret
   # - nestif
+  # - nilerr
+  # - nilnil
   # - nlreturn
   # - noctx
   # - nolintlint
+  # - nonamedreturns
+  # - nosprintfhostport
+  # - paralleltest
+  # - perfsprint
+  # - predeclared
+  # - protogetter
+  # - reassign
+  # - revive
   # - rowserrcheck
-  # - scopelint
-  # - sqlclosecheck
   # - stylecheck
+  # - tagalign
+  # - tagliatelle
   # - testpackage
+  # - thelper
   # - unparam
-  # - whitespace
+  # - usestdlibvars
+  # - varnamelen
+  # - wrapcheck
   # - wsl
 
 run:
@@ -77,23 +143,26 @@ output:
 issues:
   exclude-rules:
     # we aren't calling unknown URL
-    - text: "G107" # G107: Url provided to HTTP request as taint input
+    - text: 'G107' # G107: Url provided to HTTP request as taint input
       linters:
         - gosec
     # as a web server that's expected to handle any template, this is totally in the hands of the user.
-    - text: "G203" # G203: Use of unescaped data in HTML templates
+    - text: 'G203' # G203: Use of unescaped data in HTML templates
       linters:
         - gosec
     # we're shelling out to known commands, not relying on user-defined input.
-    - text: "G204" # G204: Audit use of command execution
+    - text: 'G204' # G204: Audit use of command execution
       linters:
         - gosec
     # the choice of weakrand is deliberate, hence the named import "weakrand"
     - path: modules/caddyhttp/reverseproxy/selectionpolicies.go
-      text: "G404" # G404: Insecure random number source (rand)
+      text: 'G404' # G404: Insecure random number source (rand)
       linters:
         - gosec
     - path: modules/caddyhttp/reverseproxy/streaming.go
-      text: "G404" # G404: Insecure random number source (rand)
+      text: 'G404' # G404: Insecure random number source (rand)
       linters:
         - gosec
+    - path: modules/logging/filters.go
+      linters:
+        - dupl

.goreleaser.yml

@@ -1,3 +1,5 @@
+version: 2
+
 before:
   hooks:
     # The build is done in this particular way to build Caddy in a designated directory named in .gitignore.
@@ -43,6 +45,7 @@ builds:
   - arm64
   - s390x
   - ppc64le
+  - riscv64
   goarm:
   - "5"
   - "6"
@@ -54,14 +57,20 @@ builds:
       goarch: ppc64le
     - goos: darwin
       goarch: s390x
+    - goos: darwin
+      goarch: riscv64
     - goos: windows
       goarch: ppc64le
     - goos: windows
       goarch: s390x
+    - goos: windows
+      goarch: riscv64
     - goos: freebsd
       goarch: ppc64le
     - goos: freebsd
       goarch: s390x
+    - goos: freebsd
+      goarch: riscv64
     - goos: freebsd
       goarch: arm
       goarm: "5"
@@ -70,6 +79,8 @@ builds:
   - -mod=readonly
   ldflags:
   - -s -w
+  tags:
+  - nobadger
 
 signs:
   - cmd: cosign
@@ -106,11 +117,10 @@ archives:
       {{- with .Mips }}_{{ . }}{{ end }}
       {{- if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}
   
-  # packge the 'caddy-build' directory into a tarball,
+  # package the 'caddy-build' directory into a tarball,
   # allowing users to build the exact same set of files as ours.
   - id: source
     meta: true
-    rlcp: true
     name_template: "{{ .ProjectName }}_{{ .Version }}_buildable-artifact"
     files:
       - src: LICENSE
@@ -127,14 +137,6 @@ source:
   name_template: '{{ .ProjectName }}_{{ .Version }}_src'
   format: 'tar.gz'
 
-  # This will make the destination paths be relative to the longest common
-  # path prefix between all the files matched and the source glob.
-  # Enabling this essentially mimic the behavior of nfpm's contents section.
-  # It will be the default by June 2023.
-  #
-  # Default: false
-  rlcp: true
-
   # Additional files/template/globs you want to add to the source archive.
   #
   # Default: empty.

README.md

@@ -56,7 +56,7 @@
 </p>
 
 
-## [Features](https://caddyserver.com/v2)
+## [Features](https://caddyserver.com/features)
 
 - **Easy configuration** with the [Caddyfile](https://caddyserver.com/docs/caddyfile)
 - **Powerful configuration** with its [native JSON config](https://caddyserver.com/docs/json/)
@@ -75,7 +75,7 @@
 - **Runs anywhere** with **no external dependencies** (not even libc)
 - Written in Go, a language with higher **memory safety guarantees** than other servers
 - Actually **fun to use**
-- So much more to [discover](https://caddyserver.com/v2)
+- So much more to [discover](https://caddyserver.com/features)
 
 ## Install
 
@@ -87,7 +87,7 @@ See [our online documentation](https://caddyserver.com/docs/install) for other i
 
 Requirements:
 
-- [Go 1.19 or newer](https://golang.org/dl/)
+- [Go 1.21 or newer](https://golang.org/dl/)
 
 ### For development
 

admin.go

@@ -26,7 +26,6 @@ import (
 	"expvar"
 	"fmt"
 	"hash"
-	"hash/fnv"
 	"io"
 	"net"
 	"net/http"
@@ -41,13 +40,14 @@ import (
 	"time"
 
 	"github.com/caddyserver/certmagic"
+	"github.com/cespare/xxhash/v2"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 )
 
 func init() {
-	// The hard-coded default `DefaultAdminListen` can be overidden
+	// The hard-coded default `DefaultAdminListen` can be overridden
 	// by setting the `CADDY_ADMIN` environment variable.
 	// The environment variable may be used by packagers to change
 	// the default admin address to something more appropriate for
@@ -318,7 +318,32 @@ func (admin AdminConfig) allowedOrigins(addr NetworkAddress) []*url.URL {
 				// messages. If the requested URI does not include an Internet host
 				// name for the service being requested, then the Host header field MUST
 				// be given with an empty value."
+				//
+				// UPDATE July 2023: Go broke this by patching a minor security bug in 1.20.6.
+				// Understandable, but frustrating. See:
+				// https://github.com/golang/go/issues/60374
+				// See also the discussion here:
+				// https://github.com/golang/go/issues/61431
+				//
+				// We can no longer conform to RFC 2616 Section 14.26 from either Go or curl
+				// in purity. (Curl allowed no host between 7.40 and 7.50, but now requires a
+				// bogus host; see https://superuser.com/a/925610.) If we disable Host/Origin
+				// security checks, the infosec community assures me that it is secure to do
+				// so, because:
+				// 1) Browsers do not allow access to unix sockets
+				// 2) DNS is irrelevant to unix sockets
+				//
+				// I am not quite ready to trust either of those external factors, so instead
+				// of disabling Host/Origin checks, we now allow specific Host values when
+				// accessing the admin endpoint over unix sockets. I definitely don't trust
+				// DNS (e.g. I don't trust 'localhost' to always resolve to the local host),
+				// and IP shouldn't even be used, but if it is for some reason, I think we can
+				// at least be reasonably assured that 127.0.0.1 and ::1 route to the local
+				// machine, meaning that a hypothetical browser origin would have to be on the
+				// local machine as well.
 				uniqueOrigins[""] = struct{}{}
+				uniqueOrigins["127.0.0.1"] = struct{}{}
+				uniqueOrigins["::1"] = struct{}{}
 			} else {
 				uniqueOrigins[net.JoinHostPort("localhost", addr.port())] = struct{}{}
 				uniqueOrigins[net.JoinHostPort("::1", addr.port())] = struct{}{}
@@ -449,7 +474,6 @@ func manageIdentity(ctx Context, cfg *Config) error {
 	// import the caddytls package -- but it works
 	if cfg.Admin.Identity.IssuersRaw == nil {
 		cfg.Admin.Identity.IssuersRaw = []json.RawMessage{
-			json.RawMessage(`{"module": "zerossl"}`),
 			json.RawMessage(`{"module": "acme"}`),
 		}
 	}
@@ -921,7 +945,7 @@ func (h adminHandler) originAllowed(origin *url.URL) bool {
 
 // etagHasher returns a the hasher we used on the config to both
 // produce and verify ETags.
-func etagHasher() hash.Hash32 { return fnv.New32a() }
+func etagHasher() hash.Hash { return xxhash.New() }
 
 // makeEtag returns an Etag header value (including quotes) for
 // the given config path and hash of contents at that path.
@@ -929,17 +953,28 @@ func makeEtag(path string, hash hash.Hash) string {
 	return fmt.Sprintf(`"%s %x"`, path, hash.Sum(nil))
 }
 
+// This buffer pool is used to keep buffers for
+// reading the config file during eTag header generation
+var bufferPool = sync.Pool{
+	New: func() any {
+		return new(bytes.Buffer)
+	},
+}
+
 func handleConfig(w http.ResponseWriter, r *http.Request) error {
 	switch r.Method {
 	case http.MethodGet:
 		w.Header().Set("Content-Type", "application/json")
-		// Set the ETag as a trailer header.
-		// The alternative is to write the config to a buffer, and
-		// then hash that.
-		w.Header().Set("Trailer", "ETag")
-
 		hash := etagHasher()
-		configWriter := io.MultiWriter(w, hash)
+
+		// Read the config into a buffer instead of writing directly to
+		// the response writer, as we want to set the ETag as the header,
+		// not the trailer.
+		buf := bufferPool.Get().(*bytes.Buffer)
+		buf.Reset()
+		defer bufferPool.Put(buf)
+
+		configWriter := io.MultiWriter(buf, hash)
 		err := readConfig(r.URL.Path, configWriter)
 		if err != nil {
 			return APIError{HTTPStatus: http.StatusBadRequest, Err: err}
@@ -948,6 +983,10 @@ func handleConfig(w http.ResponseWriter, r *http.Request) error {
 		// we could consider setting up a sync.Pool for the summed
 		// hashes to reduce GC pressure.
 		w.Header().Set("Etag", makeEtag(r.URL.Path, hash))
+		_, err = w.Write(buf.Bytes())
+		if err != nil {
+			return APIError{HTTPStatus: http.StatusInternalServerError, Err: err}
+		}
 
 		return nil
 
@@ -1016,9 +1055,9 @@ func handleConfigID(w http.ResponseWriter, r *http.Request) error {
 	id := parts[2]
 
 	// map the ID to the expanded path
-	currentCtxMu.RLock()
+	rawCfgMu.RLock()
 	expanded, ok := rawCfgIndex[id]
-	defer currentCtxMu.RUnlock()
+	rawCfgMu.RUnlock()
 	if !ok {
 		return APIError{
 			HTTPStatus: http.StatusNotFound,
@@ -1171,15 +1210,27 @@ traverseLoop:
 					}
 				case http.MethodPut:
 					if _, ok := v[part]; ok {
-						return fmt.Errorf("[%s] key already exists: %s", path, part)
+						return APIError{
+							HTTPStatus: http.StatusConflict,
+							Err:        fmt.Errorf("[%s] key already exists: %s", path, part),
+						}
 					}
 					v[part] = val
 				case http.MethodPatch:
 					if _, ok := v[part]; !ok {
-						return fmt.Errorf("[%s] key does not exist: %s", path, part)
+						return APIError{
+							HTTPStatus: http.StatusNotFound,
+							Err:        fmt.Errorf("[%s] key does not exist: %s", path, part),
+						}
 					}
 					v[part] = val
 				case http.MethodDelete:
+					if _, ok := v[part]; !ok {
+						return APIError{
+							HTTPStatus: http.StatusNotFound,
+							Err:        fmt.Errorf("[%s] key does not exist: %s", path, part),
+						}
+					}
 					delete(v, part)
 				default:
 					return fmt.Errorf("unrecognized method %s", method)
@@ -1321,7 +1372,7 @@ var (
 // will get deleted before the process gracefully exits.
 func PIDFile(filename string) error {
 	pid := []byte(strconv.Itoa(os.Getpid()) + "\n")
-	err := os.WriteFile(filename, pid, 0600)
+	err := os.WriteFile(filename, pid, 0o600)
 	if err != nil {
 		return err
 	}

admin_test.go

@@ -75,6 +75,12 @@ func TestUnsyncedConfigAccess(t *testing.T) {
 			path:   "/bar/qq",
 			expect: `{"foo": "jet", "bar": {"aa": "bb"}, "list": ["a", "b", "c"]}`,
 		},
+		{
+			method:    "DELETE",
+			path:      "/bar/qq",
+			expect:    `{"foo": "jet", "bar": {"aa": "bb"}, "list": ["a", "b", "c"]}`,
+			shouldErr: true,
+		},
 		{
 			method:  "POST",
 			path:    "/list",

caddy.go

@@ -20,8 +20,10 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"errors"
+	"flag"
 	"fmt"
 	"io"
+	"io/fs"
 	"log"
 	"net/http"
 	"os"
@@ -34,10 +36,12 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/caddyserver/caddy/v2/notify"
 	"github.com/caddyserver/certmagic"
 	"github.com/google/uuid"
 	"go.uber.org/zap"
+
+	"github.com/caddyserver/caddy/v2/internal/filesystems"
+	"github.com/caddyserver/caddy/v2/notify"
 )
 
 // Config is the top (or beginning) of the Caddy configuration structure.
@@ -82,6 +86,9 @@ type Config struct {
 	storage certmagic.Storage
 
 	cancelFunc context.CancelFunc
+
+	// filesystems is a dict of filesystems that will later be loaded from and added to.
+	filesystems FileSystems
 }
 
 // App is a thing that Caddy runs.
@@ -156,8 +163,8 @@ func changeConfig(method, path string, input []byte, ifMatchHeader string, force
 		return fmt.Errorf("method not allowed")
 	}
 
-	currentCtxMu.Lock()
+	rawCfgMu.Lock()
-	defer currentCtxMu.Unlock()
+	defer rawCfgMu.Unlock()
 
 	if ifMatchHeader != "" {
 		// expect the first and last character to be quotes
@@ -257,8 +264,8 @@ func changeConfig(method, path string, input []byte, ifMatchHeader string, force
 // readConfig traverses the current config to path
 // and writes its JSON encoding to out.
 func readConfig(path string, out io.Writer) error {
-	currentCtxMu.RLock()
+	rawCfgMu.RLock()
-	defer currentCtxMu.RUnlock()
+	defer rawCfgMu.RUnlock()
 	return unsyncedConfigAccess(http.MethodGet, path, nil, out)
 }
 
@@ -305,7 +312,7 @@ func indexConfigObjects(ptr any, configPath string, index map[string]string) err
 // it as the new config, replacing any other current config.
 // It does NOT update the raw config state, as this is a
 // lower-level function; most callers will want to use Load
-// instead. A write lock on currentCtxMu is required! If
+// instead. A write lock on rawCfgMu is required! If
 // allowPersist is false, it will not be persisted to disk,
 // even if it is configured to.
 func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
@@ -340,8 +347,10 @@ func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
 	}
 
 	// swap old context (including its config) with the new one
+	currentCtxMu.Lock()
 	oldCtx := currentCtx
 	currentCtx = ctx
+	currentCtxMu.Unlock()
 
 	// Stop, Cleanup each old app
 	unsyncedStop(oldCtx)
@@ -354,13 +363,13 @@ func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
 			newCfg.Admin.Config.Persist == nil ||
 			*newCfg.Admin.Config.Persist) {
 		dir := filepath.Dir(ConfigAutosavePath)
-		err := os.MkdirAll(dir, 0700)
+		err := os.MkdirAll(dir, 0o700)
 		if err != nil {
 			Log().Error("unable to create folder for config autosave",
 				zap.String("dir", dir),
 				zap.Error(err))
 		} else {
-			err := os.WriteFile(ConfigAutosavePath, cfgJSON, 0600)
+			err := os.WriteFile(ConfigAutosavePath, cfgJSON, 0o600)
 			if err == nil {
 				Log().Info("autosaved config (load with --resume flag)", zap.String("file", ConfigAutosavePath))
 			} else {
@@ -389,6 +398,58 @@ func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
 // will want to use Run instead, which also
 // updates the config's raw state.
 func run(newCfg *Config, start bool) (Context, error) {
+	ctx, err := provisionContext(newCfg, start)
+	if err != nil {
+		return ctx, err
+	}
+
+	if !start {
+		return ctx, nil
+	}
+
+	// Provision any admin routers which may need to access
+	// some of the other apps at runtime
+	err = ctx.cfg.Admin.provisionAdminRouters(ctx)
+	if err != nil {
+		return ctx, err
+	}
+
+	// Start
+	err = func() error {
+		started := make([]string, 0, len(ctx.cfg.apps))
+		for name, a := range ctx.cfg.apps {
+			err := a.Start()
+			if err != nil {
+				// an app failed to start, so we need to stop
+				// all other apps that were already started
+				for _, otherAppName := range started {
+					err2 := ctx.cfg.apps[otherAppName].Stop()
+					if err2 != nil {
+						err = fmt.Errorf("%v; additionally, aborting app %s: %v",
+							err, otherAppName, err2)
+					}
+				}
+				return fmt.Errorf("%s app module: start: %v", name, err)
+			}
+			started = append(started, name)
+		}
+		return nil
+	}()
+	if err != nil {
+		return ctx, err
+	}
+
+	// now that the user's config is running, finish setting up anything else,
+	// such as remote admin endpoint, config loader, etc.
+	return ctx, finishSettingUp(ctx, ctx.cfg)
+}
+
+// provisionContext creates a new context from the given configuration and provisions
+// storage and apps.
+// If `newCfg` is nil a new empty configuration will be created.
+// If `replaceAdminServer` is true any currently active admin server will be replaced
+// with a new admin server based on the provided configuration.
+func provisionContext(newCfg *Config, replaceAdminServer bool) (Context, error) {
 	// because we will need to roll back any state
 	// modifications if this function errors, we
 	// keep a single error value and scope all
@@ -436,13 +497,16 @@ func run(newCfg *Config, start bool) (Context, error) {
 	}
 
 	// start the admin endpoint (and stop any prior one)
-	if start {
+	if replaceAdminServer {
 		err = replaceLocalAdminServer(newCfg)
 		if err != nil {
 			return ctx, fmt.Errorf("starting caddy administration endpoint: %v", err)
 		}
 	}
 
+	// create the new filesystem map
+	newCfg.filesystems = &filesystems.FilesystemMap{}
+
 	// prepare the new config for use
 	newCfg.apps = make(map[string]App)
 
@@ -480,49 +544,16 @@ func run(newCfg *Config, start bool) (Context, error) {
 		}
 		return nil
 	}()
-	if err != nil {
+	return ctx, err
-		return ctx, err
+}
-	}
 
-	if !start {
+// ProvisionContext creates a new context from the configuration and provisions storage
-		return ctx, nil
+// and app modules.
-	}
+// The function is intended for testing and advanced use cases only, typically `Run` should be
-
+// use to ensure a fully functional caddy instance.
-	// Provision any admin routers which may need to access
+// EXPERIMENTAL: While this is public the interface and implementation details of this function may change.
-	// some of the other apps at runtime
+func ProvisionContext(newCfg *Config) (Context, error) {
-	err = newCfg.Admin.provisionAdminRouters(ctx)
+	return provisionContext(newCfg, false)
-	if err != nil {
-		return ctx, err
-	}
-
-	// Start
-	err = func() error {
-		started := make([]string, 0, len(newCfg.apps))
-		for name, a := range newCfg.apps {
-			err := a.Start()
-			if err != nil {
-				// an app failed to start, so we need to stop
-				// all other apps that were already started
-				for _, otherAppName := range started {
-					err2 := newCfg.apps[otherAppName].Stop()
-					if err2 != nil {
-						err = fmt.Errorf("%v; additionally, aborting app %s: %v",
-							err, otherAppName, err2)
-					}
-				}
-				return fmt.Errorf("%s app module: start: %v", name, err)
-			}
-			started = append(started, name)
-		}
-		return nil
-	}()
-	if err != nil {
-		return ctx, err
-	}
-
-	// now that the user's config is running, finish setting up anything else,
-	// such as remote admin endpoint, config loader, etc.
-	return ctx, finishSettingUp(ctx, newCfg)
 }
 
 // finishSettingUp should be run after all apps have successfully started.
@@ -627,22 +658,35 @@ type ConfigLoader interface {
 // stop the others. Stop should only be called
 // if not replacing with a new config.
 func Stop() error {
+	currentCtxMu.RLock()
+	ctx := currentCtx
+	currentCtxMu.RUnlock()
+
+	rawCfgMu.Lock()
+	unsyncedStop(ctx)
+
 	currentCtxMu.Lock()
-	defer currentCtxMu.Unlock()
-	unsyncedStop(currentCtx)
 	currentCtx = Context{}
+	currentCtxMu.Unlock()
+
 	rawCfgJSON = nil
 	rawCfgIndex = nil
 	rawCfg[rawConfigKey] = nil
+	rawCfgMu.Unlock()
+
 	return nil
 }
 
-// unsyncedStop stops cfg from running, but has
+// unsyncedStop stops ctx from running, but has
-// no locking around cfg. It is a no-op if cfg is
+// no locking around ctx. It is a no-op if ctx has a
-// nil. If any app returns an error when stopping,
+// nil cfg. If any app returns an error when stopping,
 // it is logged and the function continues stopping
 // the next app. This function assumes all apps in
-// cfg were successfully started first.
+// ctx were successfully started first.
+//
+// A lock on rawCfgMu is required, even though this
+// function does not access rawCfg, that lock
+// synchronizes the stop/start of apps.
 func unsyncedStop(ctx Context) {
 	if ctx.cfg == nil {
 		return
@@ -691,6 +735,7 @@ func exitProcess(ctx context.Context, logger *zap.Logger) {
 	logger.Warn("exiting; byeee!! 👋")
 
 	exitCode := ExitCodeSuccess
+	lastContext := ActiveContext()
 
 	// stop all apps
 	if err := Stop(); err != nil {
@@ -712,6 +757,16 @@ func exitProcess(ctx context.Context, logger *zap.Logger) {
 		}
 	}
 
+	// execute any process-exit callbacks
+	for _, exitFunc := range lastContext.exitFuncs {
+		exitFunc(ctx)
+	}
+	exitFuncsMu.Lock()
+	for _, exitFunc := range exitFuncs {
+		exitFunc(ctx)
+	}
+	exitFuncsMu.Unlock()
+
 	// shut down admin endpoint(s) in goroutines so that
 	// if this function was called from an admin handler,
 	// it has a chance to return gracefully
@@ -724,7 +779,10 @@ func exitProcess(ctx context.Context, logger *zap.Logger) {
 			} else {
 				logger.Error("unclean shutdown")
 			}
-			os.Exit(exitCode)
+			// check if we are in test environment, and dont call exit if we are
+			if flag.Lookup("test.v") == nil && !strings.Contains(os.Args[0], ".test") {
+				os.Exit(exitCode)
+			}
 		}()
 
 		if remoteAdminServer != nil {
@@ -750,6 +808,23 @@ var exiting = new(int32) // accessed atomically
 // EXPERIMENTAL API: subject to change or removal.
 func Exiting() bool { return atomic.LoadInt32(exiting) == 1 }
 
+// OnExit registers a callback to invoke during process exit.
+// This registration is PROCESS-GLOBAL, meaning that each
+// function should only be registered once forever, NOT once
+// per config load (etc).
+//
+// EXPERIMENTAL API: subject to change or removal.
+func OnExit(f func(context.Context)) {
+	exitFuncsMu.Lock()
+	exitFuncs = append(exitFuncs, f)
+	exitFuncsMu.Unlock()
+}
+
+var (
+	exitFuncs   []func(context.Context)
+	exitFuncsMu sync.Mutex
+)
+
 // Duration can be an integer or a string. An integer is
 // interpreted as nanoseconds. If a string, it is a Go
 // time.Duration value such as `300ms`, `1.5h`, or `2h45m`;
@@ -809,14 +884,19 @@ func ParseDuration(s string) (time.Duration, error) {
 // regardless of storage configuration, since each instance is intended to
 // have its own unique ID.
 func InstanceID() (uuid.UUID, error) {
-	uuidFilePath := filepath.Join(AppDataDir(), "instance.uuid")
+	appDataDir := AppDataDir()
+	uuidFilePath := filepath.Join(appDataDir, "instance.uuid")
 	uuidFileBytes, err := os.ReadFile(uuidFilePath)
-	if os.IsNotExist(err) {
+	if errors.Is(err, fs.ErrNotExist) {
 		uuid, err := uuid.NewRandom()
 		if err != nil {
 			return uuid, err
 		}
-		err = os.WriteFile(uuidFilePath, []byte(uuid.String()), 0600)
+		err = os.MkdirAll(appDataDir, 0o700)
+		if err != nil {
+			return uuid, err
+		}
+		err = os.WriteFile(uuidFilePath, []byte(uuid.String()), 0o600)
 		return uuid, err
 	} else if err != nil {
 		return [16]byte{}, err
@@ -969,14 +1049,12 @@ type CtxKey string
 
 // This group of variables pertains to the current configuration.
 var (
-	// currentCtxMu protects everything in this var block.
-	currentCtxMu sync.RWMutex
-
 	// currentCtx is the root context for the currently-running
 	// configuration, which can be accessed through this value.
 	// If the Config contained in this value is not nil, then
 	// a config is currently active/running.
-	currentCtx Context
+	currentCtx   Context
+	currentCtxMu sync.RWMutex
 
 	// rawCfg is the current, generic-decoded configuration;
 	// we initialize it as a map with one field ("config")
@@ -994,6 +1072,10 @@ var (
 	// rawCfgIndex is the map of user-assigned ID to expanded
 	// path, for converting /id/ paths to /config/ paths.
 	rawCfgIndex map[string]string
+
+	// rawCfgMu protects all the rawCfg fields and also
+	// essentially synchronizes config changes/reloads.
+	rawCfgMu sync.RWMutex
 )
 
 // errSameConfig is returned if the new config is the same

caddyconfig/caddyfile/adapter.go

@@ -52,7 +52,7 @@ func (a Adapter) Adapt(body []byte, options map[string]any) ([]byte, []caddyconf
 		return nil, warnings, err
 	}
 
-	// lint check: see if input was properly formatted; sometimes messy files files parse
+	// lint check: see if input was properly formatted; sometimes messy files parse
 	// successfully but result in logical errors (the Caddyfile is a bad format, I'm sorry)
 	if warning, different := FormattingDifference(filename, body); different {
 		warnings = append(warnings, warning)
@@ -92,30 +92,26 @@ func FormattingDifference(filename string, body []byte) (caddyconfig.Warning, bo
 	}, true
 }
 
-// Unmarshaler is a type that can unmarshal
+// Unmarshaler is a type that can unmarshal Caddyfile tokens to
-// Caddyfile tokens to set itself up for a
+// set itself up for a JSON encoding. The goal of an unmarshaler
-// JSON encoding. The goal of an unmarshaler
+// is not to set itself up for actual use, but to set itself up for
-// is not to set itself up for actual use,
+// being marshaled into JSON. Caddyfile-unmarshaled values will not
-// but to set itself up for being marshaled
+// be used directly; they will be encoded as JSON and then used from
-// into JSON. Caddyfile-unmarshaled values
+// that. Implementations _may_ be able to support multiple segments
-// will not be used directly; they will be
+// (instances of their directive or batch of tokens); typically this
-// encoded as JSON and then used from that.
+// means wrapping parsing logic in a loop: `for d.Next() { ... }`.
-// Implementations must be able to support
+// More commonly, only a single segment is supported, so a simple
-// multiple segments (instances of their
+// `d.Next()` at the start should be used to consume the module
-// directive or batch of tokens); typically
+// identifier token (directive name, etc).
-// this means wrapping all token logic in
-// a loop: `for d.Next() { ... }`.
 type Unmarshaler interface {
 	UnmarshalCaddyfile(d *Dispenser) error
 }
 
 // ServerType is a type that can evaluate a Caddyfile and set up a caddy config.
 type ServerType interface {
-	// Setup takes the server blocks which
+	// Setup takes the server blocks which contain tokens,
-	// contain tokens, as well as options
+	// as well as options (e.g. CLI flags) and creates a
-	// (e.g. CLI flags) and creates a Caddy
+	// Caddy config, along with any warnings or an error.
-	// config, along with any warnings or
-	// an error.
 	Setup([]ServerBlock, map[string]any) (*caddy.Config, []caddyconfig.Warning, error)
 }
 

caddyconfig/caddyfile/dispenser.go

@@ -30,6 +30,10 @@ type Dispenser struct {
 	tokens  []Token
 	cursor  int
 	nesting int
+
+	// A map of arbitrary context data that can be used
+	// to pass through some information to unmarshalers.
+	context map[string]any
 }
 
 // NewDispenser returns a Dispenser filled with the given tokens.
@@ -106,7 +110,7 @@ func (d *Dispenser) nextOnSameLine() bool {
 	}
 	curr := d.tokens[d.cursor]
 	next := d.tokens[d.cursor+1]
-	if curr.File == next.File && curr.Line+curr.NumLineBreaks() == next.Line {
+	if !isNextOnNewLine(curr, next) {
 		d.cursor++
 		return true
 	}
@@ -127,7 +131,7 @@ func (d *Dispenser) NextLine() bool {
 	}
 	curr := d.tokens[d.cursor]
 	next := d.tokens[d.cursor+1]
-	if curr.File != next.File || curr.Line+curr.NumLineBreaks() < next.Line {
+	if isNextOnNewLine(curr, next) {
 		d.cursor++
 		return true
 	}
@@ -391,22 +395,22 @@ func (d *Dispenser) Reset() {
 // an argument.
 func (d *Dispenser) ArgErr() error {
 	if d.Val() == "{" {
-		return d.Err("Unexpected token '{', expecting argument")
+		return d.Err("unexpected token '{', expecting argument")
 	}
-	return d.Errf("Wrong argument count or unexpected line ending after '%s'", d.Val())
+	return d.Errf("wrong argument count or unexpected line ending after '%s'", d.Val())
 }
 
 // SyntaxErr creates a generic syntax error which explains what was
 // found and what was expected.
 func (d *Dispenser) SyntaxErr(expected string) error {
-	msg := fmt.Sprintf("%s:%d - Syntax error: Unexpected token '%s', expecting '%s', import chain: ['%s']", d.File(), d.Line(), d.Val(), expected, strings.Join(d.Token().imports, "','"))
+	msg := fmt.Sprintf("syntax error: unexpected token '%s', expecting '%s', at %s:%d import chain: ['%s']", d.Val(), expected, d.File(), d.Line(), strings.Join(d.Token().imports, "','"))
 	return errors.New(msg)
 }
 
 // EOFErr returns an error indicating that the dispenser reached
 // the end of the input when searching for the next token.
 func (d *Dispenser) EOFErr() error {
-	return d.Errf("Unexpected EOF")
+	return d.Errf("unexpected EOF")
 }
 
 // Err generates a custom parse-time error with a message of msg.
@@ -421,7 +425,10 @@ func (d *Dispenser) Errf(format string, args ...any) error {
 
 // WrapErr takes an existing error and adds the Caddyfile file and line number.
 func (d *Dispenser) WrapErr(err error) error {
-	return fmt.Errorf("%s:%d - Error during parsing: %w, import chain: ['%s']", d.File(), d.Line(), err, strings.Join(d.Token().imports, "','"))
+	if len(d.Token().imports) > 0 {
+		return fmt.Errorf("%w, at %s:%d import chain ['%s']", err, d.File(), d.Line(), strings.Join(d.Token().imports, "','"))
+	}
+	return fmt.Errorf("%w, at %s:%d", err, d.File(), d.Line())
 }
 
 // Delete deletes the current token and returns the updated slice
@@ -451,6 +458,34 @@ func (d *Dispenser) DeleteN(amount int) []Token {
 	return d.tokens
 }
 
+// SetContext sets a key-value pair in the context map.
+func (d *Dispenser) SetContext(key string, value any) {
+	if d.context == nil {
+		d.context = make(map[string]any)
+	}
+	d.context[key] = value
+}
+
+// GetContext gets the value of a key in the context map.
+func (d *Dispenser) GetContext(key string) any {
+	if d.context == nil {
+		return nil
+	}
+	return d.context[key]
+}
+
+// GetContextString gets the value of a key in the context map
+// as a string, or an empty string if the key does not exist.
+func (d *Dispenser) GetContextString(key string) string {
+	if d.context == nil {
+		return ""
+	}
+	if val, ok := d.context[key].(string); ok {
+		return val
+	}
+	return ""
+}
+
 // isNewLine determines whether the current token is on a different
 // line (higher line number) than the previous token. It handles imported
 // tokens correctly. If there isn't a previous token, it returns true.
@@ -464,17 +499,7 @@ func (d *Dispenser) isNewLine() bool {
 
 	prev := d.tokens[d.cursor-1]
 	curr := d.tokens[d.cursor]
-
+	return isNextOnNewLine(prev, curr)
-	// If the previous token is from a different file,
-	// we can assume it's from a different line
-	if prev.File != curr.File {
-		return true
-	}
-
-	// If the previous token (incl line breaks) ends
-	// on a line earlier than the current token,
-	// then the current token is on a new line
-	return prev.Line+prev.NumLineBreaks() < curr.Line
 }
 
 // isNextOnNewLine determines whether the current token is on a different
@@ -490,15 +515,7 @@ func (d *Dispenser) isNextOnNewLine() bool {
 
 	curr := d.tokens[d.cursor]
 	next := d.tokens[d.cursor+1]
-
+	return isNextOnNewLine(curr, next)
-	// If the next token is from a different file,
-	// we can assume it's from a different line
-	if curr.File != next.File {
-		return true
-	}
-
-	// If the current token (incl line breaks) ends
-	// on a line earlier than the next token,
-	// then the next token is on a new line
-	return curr.Line+curr.NumLineBreaks() < next.Line
 }
+
+const MatcherNameCtxKey = "matcher_name"

caddyconfig/caddyfile/dispenser_test.go

@@ -305,7 +305,7 @@ func TestDispenser_ArgErr_Err(t *testing.T) {
 		t.Errorf("Expected error message with custom message in it ('foobar'); got '%v'", err)
 	}
 
-	var ErrBarIsFull = errors.New("bar is full")
+	ErrBarIsFull := errors.New("bar is full")
 	bookingError := d.Errf("unable to reserve: %w", ErrBarIsFull)
 	if !errors.Is(bookingError, ErrBarIsFull) {
 		t.Errorf("Errf(): should be able to unwrap the error chain")

caddyconfig/caddyfile/formatter.go

@@ -17,6 +17,7 @@ package caddyfile
 import (
 	"bytes"
 	"io"
+	"slices"
 	"unicode"
 )
 
@@ -31,6 +32,14 @@ func Format(input []byte) []byte {
 	out := new(bytes.Buffer)
 	rdr := bytes.NewReader(input)
 
+	type heredocState int
+
+	const (
+		heredocClosed  heredocState = 0
+		heredocOpening heredocState = 1
+		heredocOpened  heredocState = 2
+	)
+
 	var (
 		last rune // the last character that was written to the result
 
@@ -47,6 +56,11 @@ func Format(input []byte) []byte {
 		quoted  bool // whether we're in a quoted segment
 		escaped bool // whether current char is escaped
 
+		heredoc              heredocState // whether we're in a heredoc
+		heredocEscaped       bool         // whether heredoc is escaped
+		heredocMarker        []rune
+		heredocClosingMarker []rune
+
 		nesting int // indentation level
 	)
 
@@ -75,6 +89,62 @@ func Format(input []byte) []byte {
 			panic(err)
 		}
 
+		// detect whether we have the start of a heredoc
+		if !quoted && !(heredoc != heredocClosed || heredocEscaped) &&
+			space && last == '<' && ch == '<' {
+			write(ch)
+			heredoc = heredocOpening
+			space = false
+			continue
+		}
+
+		if heredoc == heredocOpening {
+			if ch == '\n' {
+				if len(heredocMarker) > 0 && heredocMarkerRegexp.MatchString(string(heredocMarker)) {
+					heredoc = heredocOpened
+				} else {
+					heredocMarker = nil
+					heredoc = heredocClosed
+					nextLine()
+					continue
+				}
+				write(ch)
+				continue
+			}
+			if unicode.IsSpace(ch) {
+				// a space means it's just a regular token and not a heredoc
+				heredocMarker = nil
+				heredoc = heredocClosed
+			} else {
+				heredocMarker = append(heredocMarker, ch)
+				write(ch)
+				continue
+			}
+		}
+		// if we're in a heredoc, all characters are read&write as-is
+		if heredoc == heredocOpened {
+			heredocClosingMarker = append(heredocClosingMarker, ch)
+			if len(heredocClosingMarker) > len(heredocMarker)+1 { // We assert that the heredocClosingMarker is followed by a unicode.Space
+				heredocClosingMarker = heredocClosingMarker[1:]
+			}
+			// check if we're done
+			if unicode.IsSpace(ch) && slices.Equal(heredocClosingMarker[:len(heredocClosingMarker)-1], heredocMarker) {
+				heredocMarker = nil
+				heredocClosingMarker = nil
+				heredoc = heredocClosed
+			} else {
+				write(ch)
+				if ch == '\n' {
+					heredocClosingMarker = heredocClosingMarker[:0]
+				}
+				continue
+			}
+		}
+
+		if last == '<' && space {
+			space = false
+		}
+
 		if comment {
 			if ch == '\n' {
 				comment = false
@@ -98,6 +168,9 @@ func Format(input []byte) []byte {
 		}
 
 		if escaped {
+			if ch == '<' {
+				heredocEscaped = true
+			}
 			write(ch)
 			escaped = false
 			continue
@@ -117,6 +190,7 @@ func Format(input []byte) []byte {
 
 		if unicode.IsSpace(ch) {
 			space = true
+			heredocEscaped = false
 			if ch == '\n' {
 				newLines++
 			}
@@ -205,6 +279,11 @@ func Format(input []byte) []byte {
 			write('{')
 			openBraceWritten = true
 		}
+
+		if spacePrior && ch == '<' {
+			space = true
+		}
+
 		write(ch)
 
 		beginningOfLine = false

caddyconfig/caddyfile/formatter_test.go

@@ -362,6 +362,76 @@ block {
 
 block {
 }
+`,
+		},
+		{
+			description: "keep heredoc as-is",
+			input: `block {
+	heredoc <<HEREDOC
+	Here's more than one space       Here's more than one space
+	HEREDOC
+}
+`,
+			expect: `block {
+	heredoc <<HEREDOC
+	Here's more than one space       Here's more than one space
+	HEREDOC
+}
+`,
+		},
+		{
+			description: "Mixing heredoc with regular part",
+			input: `block {
+	heredoc <<HEREDOC
+	Here's more than one space       Here's more than one space
+	HEREDOC
+	respond "More than one space will be eaten"     200
+}
+
+block2 {
+	heredoc <<HEREDOC
+	Here's more than one space       Here's more than one space
+	HEREDOC
+	respond "More than one space will be eaten" 200
+}
+`,
+			expect: `block {
+	heredoc <<HEREDOC
+	Here's more than one space       Here's more than one space
+	HEREDOC
+	respond "More than one space will be eaten" 200
+}
+
+block2 {
+	heredoc <<HEREDOC
+	Here's more than one space       Here's more than one space
+	HEREDOC
+	respond "More than one space will be eaten" 200
+}
+`,
+		},
+		{
+			description: "Heredoc as regular token",
+			input: `block {
+	heredoc <<HEREDOC                                 "More than one space will be eaten"
+}
+`,
+			expect: `block {
+	heredoc <<HEREDOC "More than one space will be eaten"
+}
+`,
+		},
+		{
+			description: "Escape heredoc",
+			input: `block {
+	heredoc \<<HEREDOC
+	respond "More than one space will be eaten"                           200
+}
+`,
+			expect: `block {
+	heredoc \<<HEREDOC
+	respond "More than one space will be eaten" 200
+}
 `,
 		},
 	} {

caddyconfig/caddyfile/importargs.go

@@ -19,8 +19,9 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/caddyserver/caddy/v2"
 	"go.uber.org/zap"
+
+	"github.com/caddyserver/caddy/v2"
 )
 
 // parseVariadic determines if the token is a variadic placeholder,
@@ -51,6 +52,13 @@ func parseVariadic(token Token, argCount int) (bool, int, int) {
 		return false, 0, 0
 	}
 
+	// A valid token may contain several placeholders, and
+	// they may be separated by ":". It's not variadic.
+	// https://github.com/caddyserver/caddy/issues/5716
+	if strings.Contains(start, "}") || strings.Contains(end, "{") {
+		return false, 0, 0
+	}
+
 	var (
 		startIndex = 0
 		endIndex   = argCount
@@ -93,6 +101,11 @@ func makeArgsReplacer(args []string) *caddy.Replacer {
 		// TODO: Remove the deprecated {args.*} placeholder
 		// support at some point in the future
 		if matches := argsRegexpIndexDeprecated.FindStringSubmatch(key); len(matches) > 0 {
+			// What's matched may be a substring of the key
+			if matches[0] != key {
+				return nil, false
+			}
+
 			value, err := strconv.Atoi(matches[1])
 			if err != nil {
 				caddy.Log().Named("caddyfile").Warn(
@@ -111,6 +124,11 @@ func makeArgsReplacer(args []string) *caddy.Replacer {
 
 		// Handle args[*] form
 		if matches := argsRegexpIndex.FindStringSubmatch(key); len(matches) > 0 {
+			// What's matched may be a substring of the key
+			if matches[0] != key {
+				return nil, false
+			}
+
 			if strings.Contains(matches[1], ":") {
 				caddy.Log().Named("caddyfile").Warn(
 					"Variadic placeholder {args[" + matches[1] + "]} must be a token on its own")

caddyconfig/caddyfile/importgraph.go

@@ -21,19 +21,20 @@ import (
 type adjacency map[string][]string
 
 type importGraph struct {
-	nodes map[string]bool
+	nodes map[string]struct{}
 	edges adjacency
 }
 
 func (i *importGraph) addNode(name string) {
 	if i.nodes == nil {
-		i.nodes = make(map[string]bool)
+		i.nodes = make(map[string]struct{})
 	}
 	if _, exists := i.nodes[name]; exists {
 		return
 	}
-	i.nodes[name] = true
+	i.nodes[name] = struct{}{}
 }
+
 func (i *importGraph) addNodes(names []string) {
 	for _, name := range names {
 		i.addNode(name)
@@ -43,6 +44,7 @@ func (i *importGraph) addNodes(names []string) {
 func (i *importGraph) removeNode(name string) {
 	delete(i.nodes, name)
 }
+
 func (i *importGraph) removeNodes(names []string) {
 	for _, name := range names {
 		i.removeNode(name)
@@ -64,7 +66,7 @@ func (i *importGraph) addEdge(from, to string) error {
 	}
 
 	if i.nodes == nil {
-		i.nodes = make(map[string]bool)
+		i.nodes = make(map[string]struct{})
 	}
 	if i.edges == nil {
 		i.edges = make(adjacency)
@@ -73,6 +75,7 @@ func (i *importGraph) addEdge(from, to string) error {
 	i.edges[from] = append(i.edges[from], to)
 	return nil
 }
+
 func (i *importGraph) addEdges(from string, tos []string) error {
 	for _, to := range tos {
 		err := i.addEdge(from, to)

caddyconfig/caddyfile/lexer.go

@@ -137,18 +137,32 @@ func (l *lexer) next() (bool, error) {
 		}
 
 		// detect whether we have the start of a heredoc
-		if !inHeredoc && !heredocEscaped && len(val) > 1 && string(val[:2]) == "<<" {
+		if !(quoted || btQuoted) && !(inHeredoc || heredocEscaped) &&
-			if ch == '<' {
+			len(val) > 1 && string(val[:2]) == "<<" {
-				return false, fmt.Errorf("too many '<' for heredoc on line #%d; only use two, for example <<END", l.line)
+			// a space means it's just a regular token and not a heredoc
+			if ch == ' ' {
+				return makeToken(0), nil
 			}
+
+			// skip CR, we only care about LF
 			if ch == '\r' {
 				continue
 			}
+
 			// after hitting a newline, we know that the heredoc marker
 			// is the characters after the two << and the newline.
 			// we reset the val because the heredoc is syntax we don't
 			// want to keep.
 			if ch == '\n' {
+				if len(val) == 2 {
+					return false, fmt.Errorf("missing opening heredoc marker on line #%d; must contain only alpha-numeric characters, dashes and underscores; got empty string", l.line)
+				}
+
+				// check if there's too many <
+				if string(val[:3]) == "<<<" {
+					return false, fmt.Errorf("too many '<' for heredoc on line #%d; only use two, for example <<END", l.line)
+				}
+
 				heredocMarker = string(val[2:])
 				if !heredocMarkerRegexp.Match([]byte(heredocMarker)) {
 					return false, fmt.Errorf("heredoc marker on line #%d must contain only alpha-numeric characters, dashes and underscores; got '%s'", l.line, heredocMarker)
@@ -172,7 +186,7 @@ func (l *lexer) next() (bool, error) {
 			}
 
 			// check if we're done, i.e. that the last few characters are the marker
-			if len(val) > len(heredocMarker) && heredocMarker == string(val[len(val)-len(heredocMarker):]) {
+			if len(val) >= len(heredocMarker) && heredocMarker == string(val[len(val)-len(heredocMarker):]) {
 				// set the final value
 				val, err = l.finalizeHeredoc(val, heredocMarker)
 				if err != nil {
@@ -299,6 +313,11 @@ func (l *lexer) finalizeHeredoc(val []rune, marker string) ([]rune, error) {
 	// iterate over each line and strip the whitespace from the front
 	var out string
 	for lineNum, lineText := range lines[:len(lines)-1] {
+		if lineText == "" || lineText == "\r" {
+			out += "\n"
+			continue
+		}
+
 		// find an exact match for the padding
 		index := strings.Index(lineText, paddingToStrip)
 
@@ -321,6 +340,8 @@ func (l *lexer) finalizeHeredoc(val []rune, marker string) ([]rune, error) {
 	return []rune(out), nil
 }
 
+// Quoted returns true if the token was enclosed in quotes
+// (i.e. double quotes, backticks, or heredoc).
 func (t Token) Quoted() bool {
 	return t.wasQuoted > 0
 }
@@ -337,4 +358,42 @@ func (t Token) NumLineBreaks() int {
 	return lineBreaks
 }
 
+// Clone returns a deep copy of the token.
+func (t Token) Clone() Token {
+	return Token{
+		File:          t.File,
+		imports:       append([]string{}, t.imports...),
+		Line:          t.Line,
+		Text:          t.Text,
+		wasQuoted:     t.wasQuoted,
+		heredocMarker: t.heredocMarker,
+		snippetName:   t.snippetName,
+	}
+}
+
 var heredocMarkerRegexp = regexp.MustCompile("^[A-Za-z0-9_-]+$")
+
+// isNextOnNewLine tests whether t2 is on a different line from t1
+func isNextOnNewLine(t1, t2 Token) bool {
+	// If the second token is from a different file,
+	// we can assume it's from a different line
+	if t1.File != t2.File {
+		return true
+	}
+
+	// If the second token is from a different import chain,
+	// we can assume it's from a different line
+	if len(t1.imports) != len(t2.imports) {
+		return true
+	}
+	for i, im := range t1.imports {
+		if im != t2.imports[i] {
+			return true
+		}
+	}
+
+	// If the first token (incl line breaks) ends
+	// on a line earlier than the next token,
+	// then the second token is on a new line
+	return t1.Line+t1.NumLineBreaks() < t2.Line
+}

caddyconfig/caddyfile/lexer_test.go

@@ -285,6 +285,18 @@ EOF same-line-arg
 		},
 		{
 			input: []byte(`heredoc <<EOF
+EOF
+	HERE same-line-arg
+	`),
+			expected: []Token{
+				{Line: 1, Text: `heredoc`},
+				{Line: 1, Text: ``},
+				{Line: 3, Text: `HERE`},
+				{Line: 3, Text: `same-line-arg`},
+			},
+		},
+		{
+			input: []byte(`heredoc <<EOF
 		EOF same-line-arg
 	`),
 			expected: []Token{
@@ -322,15 +334,59 @@ EOF same-line-arg
 			},
 		},
 		{
-			input: []byte(`heredoc <EOF
+			input: []byte(`escaped-heredoc \<< >>`),
+			expected: []Token{
+				{Line: 1, Text: `escaped-heredoc`},
+				{Line: 1, Text: `<<`},
+				{Line: 1, Text: `>>`},
+			},
+		},
+		{
+			input: []byte(`not-a-heredoc <EOF
 	content
-	EOF same-line-arg
 	`),
 			expected: []Token{
-				{Line: 1, Text: `heredoc`},
+				{Line: 1, Text: `not-a-heredoc`},
 				{Line: 1, Text: `<EOF`},
 				{Line: 2, Text: `content`},
-				{Line: 3, Text: `EOF`},
+			},
+		},
+		{
+			input: []byte(`not-a-heredoc <<<EOF content`),
+			expected: []Token{
+				{Line: 1, Text: `not-a-heredoc`},
+				{Line: 1, Text: `<<<EOF`},
+				{Line: 1, Text: `content`},
+			},
+		},
+		{
+			input: []byte(`not-a-heredoc "<<" ">>"`),
+			expected: []Token{
+				{Line: 1, Text: `not-a-heredoc`},
+				{Line: 1, Text: `<<`},
+				{Line: 1, Text: `>>`},
+			},
+		},
+		{
+			input: []byte(`not-a-heredoc << >>`),
+			expected: []Token{
+				{Line: 1, Text: `not-a-heredoc`},
+				{Line: 1, Text: `<<`},
+				{Line: 1, Text: `>>`},
+			},
+		},
+		{
+			input: []byte(`not-a-heredoc <<HERE SAME LINE
+	content
+	HERE same-line-arg
+	`),
+			expected: []Token{
+				{Line: 1, Text: `not-a-heredoc`},
+				{Line: 1, Text: `<<HERE`},
+				{Line: 1, Text: `SAME`},
+				{Line: 1, Text: `LINE`},
+				{Line: 2, Text: `content`},
+				{Line: 3, Text: `HERE`},
 				{Line: 3, Text: `same-line-arg`},
 			},
 		},
@@ -366,12 +422,9 @@ EOF same-line-arg
 			},
 		},
 		{
-			input: []byte(`heredoc <<HERE SAME LINE
+			input:        []byte("not-a-heredoc <<\n"),
-	content
-	HERE same-line-arg
-	`),
 			expectErr:    true,
-			errorMessage: "heredoc marker on line #1 must contain only alpha-numeric characters, dashes and underscores; got 'HERE SAME LINE'",
+			errorMessage: "missing opening heredoc marker on line #1; must contain only alpha-numeric characters, dashes and underscores; got empty string",
 		},
 		{
 			input: []byte(`heredoc <<<EOF
@@ -404,6 +457,48 @@ EOF same-line-arg
 			expectErr:    true,
 			errorMessage: "mismatched leading whitespace in heredoc <<EOF on line #2 [        content], expected whitespace [\t\t] to match the closing marker",
 		},
+		{
+			input: []byte(`heredoc <<EOF
+The next line is a blank line
+
+The previous line is a blank line
+EOF`),
+			expected: []Token{
+				{Line: 1, Text: "heredoc"},
+				{Line: 1, Text: "The next line is a blank line\n\nThe previous line is a blank line"},
+			},
+		},
+		{
+			input: []byte(`heredoc <<EOF
+	One tab indented heredoc with blank next line
+
+	One tab indented heredoc with blank previous line
+	EOF`),
+			expected: []Token{
+				{Line: 1, Text: "heredoc"},
+				{Line: 1, Text: "One tab indented heredoc with blank next line\n\nOne tab indented heredoc with blank previous line"},
+			},
+		},
+		{
+			input: []byte(`heredoc <<EOF
+The next line is a blank line with one tab
+	
+The previous line is a blank line with one tab
+EOF`),
+			expected: []Token{
+				{Line: 1, Text: "heredoc"},
+				{Line: 1, Text: "The next line is a blank line with one tab\n\t\nThe previous line is a blank line with one tab"},
+			},
+		},
+		{
+			input: []byte(`heredoc <<EOF
+		The next line is a blank line with one tab less than the correct indentation
+	
+		The previous line is a blank line with one tab less than the correct indentation
+		EOF`),
+			expectErr:    true,
+			errorMessage: "mismatched leading whitespace in heredoc <<EOF on line #3 [\t], expected whitespace [\t\t] to match the closing marker",
+		},
 	}
 
 	for i, testCase := range testCases {

caddyconfig/caddyfile/parse.go

@@ -22,8 +22,9 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/caddyserver/caddy/v2"
 	"go.uber.org/zap"
+
+	"github.com/caddyserver/caddy/v2"
 )
 
 // Parse parses the input just enough to group tokens, in
@@ -49,7 +50,7 @@ func Parse(filename string, input []byte) ([]ServerBlock, error) {
 	p := parser{
 		Dispenser: NewDispenser(tokens),
 		importGraph: importGraph{
-			nodes: make(map[string]bool),
+			nodes: make(map[string]struct{}),
 			edges: make(adjacency),
 		},
 	}
@@ -159,14 +160,14 @@ func (p *parser) begin() error {
 	}
 
 	if ok, name := p.isNamedRoute(); ok {
-		// named routes only have one key, the route name
-		p.block.Keys = []string{name}
-		p.block.IsNamedRoute = true
-
 		// we just need a dummy leading token to ease parsing later
 		nameToken := p.Token()
 		nameToken.Text = name
 
+		// named routes only have one key, the route name
+		p.block.Keys = []Token{nameToken}
+		p.block.IsNamedRoute = true
+
 		// get all the tokens from the block, including the braces
 		tokens, err := p.blockTokens(true)
 		if err != nil {
@@ -210,10 +211,16 @@ func (p *parser) addresses() error {
 	var expectingAnother bool
 
 	for {
-		tkn := p.Val()
+		value := p.Val()
+		token := p.Token()
 
-		// special case: import directive replaces tokens during parse-time
+		// Reject request matchers if trying to define them globally
-		if tkn == "import" && p.isNewLine() {
+		if strings.HasPrefix(value, "@") {
+			return p.Errf("request matchers may not be defined globally, they must be in a site block; found %s", value)
+		}
+
+		// Special case: import directive replaces tokens during parse-time
+		if value == "import" && p.isNewLine() {
 			err := p.doImport(0)
 			if err != nil {
 				return err
@@ -222,9 +229,9 @@ func (p *parser) addresses() error {
 		}
 
 		// Open brace definitely indicates end of addresses
-		if tkn == "{" {
+		if value == "{" {
 			if expectingAnother {
-				return p.Errf("Expected another address but had '%s' - check for extra comma", tkn)
+				return p.Errf("Expected another address but had '%s' - check for extra comma", value)
 			}
 			// Mark this server block as being defined with braces.
 			// This is used to provide a better error message when
@@ -236,15 +243,15 @@ func (p *parser) addresses() error {
 		}
 
 		// Users commonly forget to place a space between the address and the '{'
-		if strings.HasSuffix(tkn, "{") {
+		if strings.HasSuffix(value, "{") {
-			return p.Errf("Site addresses cannot end with a curly brace: '%s' - put a space between the token and the brace", tkn)
+			return p.Errf("Site addresses cannot end with a curly brace: '%s' - put a space between the token and the brace", value)
 		}
 
-		if tkn != "" { // empty token possible if user typed ""
+		if value != "" { // empty token possible if user typed ""
 			// Trailing comma indicates another address will follow, which
 			// may possibly be on the next line
-			if tkn[len(tkn)-1] == ',' {
+			if value[len(value)-1] == ',' {
-				tkn = tkn[:len(tkn)-1]
+				value = value[:len(value)-1]
 				expectingAnother = true
 			} else {
 				expectingAnother = false // but we may still see another one on this line
@@ -253,11 +260,12 @@ func (p *parser) addresses() error {
 			// If there's a comma here, it's probably because they didn't use a space
 			// between their two domains, e.g. "foo.com,bar.com", which would not be
 			// parsed as two separate site addresses.
-			if strings.Contains(tkn, ",") {
+			if strings.Contains(value, ",") {
-				return p.Errf("Site addresses cannot contain a comma ',': '%s' - put a space after the comma to separate site addresses", tkn)
+				return p.Errf("Site addresses cannot contain a comma ',': '%s' - put a space after the comma to separate site addresses", value)
 			}
 
-			p.block.Keys = append(p.block.Keys, tkn)
+			token.Text = value
+			p.block.Keys = append(p.block.Keys, token)
 		}
 
 		// Advance token and possibly break out of loop or return error
@@ -356,9 +364,45 @@ func (p *parser) doImport(nesting int) error {
 	// set up a replacer for non-variadic args replacement
 	repl := makeArgsReplacer(args)
 
+	// grab all the tokens (if it exists) from within a block that follows the import
+	var blockTokens []Token
+	for currentNesting := p.Nesting(); p.NextBlock(currentNesting); {
+		blockTokens = append(blockTokens, p.Token())
+	}
+	// initialize with size 1
+	blockMapping := make(map[string][]Token, 1)
+	if len(blockTokens) > 0 {
+		// use such tokens to create a new dispenser, and then use it to parse each block
+		bd := NewDispenser(blockTokens)
+		for bd.Next() {
+			// see if we can grab a key
+			var currentMappingKey string
+			if bd.Val() == "{" {
+				return p.Err("anonymous blocks are not supported")
+			}
+			currentMappingKey = bd.Val()
+			currentMappingTokens := []Token{}
+			// read all args until end of line / {
+			if bd.NextArg() {
+				currentMappingTokens = append(currentMappingTokens, bd.Token())
+				for bd.NextArg() {
+					currentMappingTokens = append(currentMappingTokens, bd.Token())
+				}
+				// TODO(elee1766): we don't enter another mapping here because it's annoying to extract the { and } properly.
+				// maybe someone can do that in the future
+			} else {
+				// attempt to enter a block and add tokens to the currentMappingTokens
+				for mappingNesting := bd.Nesting(); bd.NextBlock(mappingNesting); {
+					currentMappingTokens = append(currentMappingTokens, bd.Token())
+				}
+			}
+			blockMapping[currentMappingKey] = currentMappingTokens
+		}
+	}
+
 	// splice out the import directive and its arguments
 	// (2 tokens, plus the length of args)
-	tokensBefore := p.tokens[:p.cursor-1-len(args)]
+	tokensBefore := p.tokens[:p.cursor-1-len(args)-len(blockTokens)]
 	tokensAfter := p.tokens[p.cursor+1:]
 	var importedTokens []Token
 	var nodes []string
@@ -392,7 +436,6 @@ func (p *parser) doImport(nesting int) error {
 			return p.Errf("Glob pattern may only contain one wildcard (*), but has others: %s", globPattern)
 		}
 		matches, err = filepath.Glob(globPattern)
-
 		if err != nil {
 			return p.Errf("Failed to use import pattern %s: %v", importPattern, err)
 		}
@@ -464,7 +507,7 @@ func (p *parser) doImport(nesting int) error {
 		// format, won't check for nesting correctness or any other error, that's what parser does.
 		if !maybeSnippet && nesting == 0 {
 			// first of the line
-			if i == 0 || importedTokens[i-1].File != token.File || importedTokens[i-1].Line+importedTokens[i-1].NumLineBreaks() < token.Line {
+			if i == 0 || isNextOnNewLine(tokensCopy[i-1], token) {
 				index = 0
 			} else {
 				index++
@@ -488,6 +531,33 @@ func (p *parser) doImport(nesting int) error {
 				maybeSnippet = false
 			}
 		}
+		// if it is {block}, we substitute with all tokens in the block
+		// if it is {blocks.*}, we substitute with the tokens in the mapping for the *
+		var skip bool
+		var tokensToAdd []Token
+		switch {
+		case token.Text == "{block}":
+			tokensToAdd = blockTokens
+		case strings.HasPrefix(token.Text, "{blocks.") && strings.HasSuffix(token.Text, "}"):
+			// {blocks.foo.bar} will be extracted to key `foo.bar`
+			blockKey := strings.TrimPrefix(strings.TrimSuffix(token.Text, "}"), "{blocks.")
+			val, ok := blockMapping[blockKey]
+			if ok {
+				tokensToAdd = val
+			}
+		default:
+			skip = true
+		}
+		if !skip {
+			if len(tokensToAdd) == 0 {
+				// if there is no content in the snippet block, don't do any replacement
+				// this allows snippets which contained {block}/{block.*} before this change to continue functioning as normal
+				tokensCopy = append(tokensCopy, token)
+			} else {
+				tokensCopy = append(tokensCopy, tokensToAdd...)
+			}
+			continue
+		}
 
 		if maybeSnippet {
 			tokensCopy = append(tokensCopy, token)
@@ -509,7 +579,7 @@ func (p *parser) doImport(nesting int) error {
 	// splice the imported tokens in the place of the import statement
 	// and rewind cursor so Next() will land on first imported token
 	p.tokens = append(tokensBefore, append(tokensCopy, tokensAfter...)...)
-	p.cursor -= len(args) + 1
+	p.cursor -= len(args) + len(blockTokens) + 1
 
 	return nil
 }
@@ -565,7 +635,6 @@ func (p *parser) doSingleImport(importFile string) ([]Token, error) {
 // are loaded into the current server block for later use
 // by directive setup functions.
 func (p *parser) directive() error {
-
 	// a segment is a list of tokens associated with this directive
 	var segment Segment
 
@@ -637,8 +706,8 @@ func (p *parser) closeCurlyBrace() error {
 func (p *parser) isNamedRoute() (bool, string) {
 	keys := p.block.Keys
 	// A named route block is a single key with parens, prefixed with &.
-	if len(keys) == 1 && strings.HasPrefix(keys[0], "&(") && strings.HasSuffix(keys[0], ")") {
+	if len(keys) == 1 && strings.HasPrefix(keys[0].Text, "&(") && strings.HasSuffix(keys[0].Text, ")") {
-		return true, strings.TrimSuffix(keys[0][2:], ")")
+		return true, strings.TrimSuffix(keys[0].Text[2:], ")")
 	}
 	return false, ""
 }
@@ -646,8 +715,8 @@ func (p *parser) isNamedRoute() (bool, string) {
 func (p *parser) isSnippet() (bool, string) {
 	keys := p.block.Keys
 	// A snippet block is a single key with parens. Nothing else qualifies.
-	if len(keys) == 1 && strings.HasPrefix(keys[0], "(") && strings.HasSuffix(keys[0], ")") {
+	if len(keys) == 1 && strings.HasPrefix(keys[0].Text, "(") && strings.HasSuffix(keys[0].Text, ")") {
-		return true, strings.TrimSuffix(keys[0][1:], ")")
+		return true, strings.TrimSuffix(keys[0].Text[1:], ")")
 	}
 	return false, ""
 }
@@ -691,11 +760,19 @@ func (p *parser) blockTokens(retainCurlies bool) ([]Token, error) {
 // grouped by segments.
 type ServerBlock struct {
 	HasBraces    bool
-	Keys         []string
+	Keys         []Token
 	Segments     []Segment
 	IsNamedRoute bool
 }
 
+func (sb ServerBlock) GetKeysText() []string {
+	res := []string{}
+	for _, k := range sb.Keys {
+		res = append(res, k.Text)
+	}
+	return res
+}
+
 // DispenseDirective returns a dispenser that contains
 // all the tokens in the server block.
 func (sb ServerBlock) DispenseDirective(dir string) *Dispenser {

caddyconfig/caddyfile/parse_test.go

@@ -22,7 +22,7 @@ import (
 )
 
 func TestParseVariadic(t *testing.T) {
-	var args = make([]string, 10)
+	args := make([]string, 10)
 	for i, tc := range []struct {
 		input  string
 		result bool
@@ -91,6 +91,10 @@ func TestParseVariadic(t *testing.T) {
 			input:  "{args[0:10]}",
 			result: true,
 		},
+		{
+			input:  "{args[0]}:{args[1]}:{args[2]}",
+			result: false,
+		},
 	} {
 		token := Token{
 			File: "test",
@@ -107,7 +111,6 @@ func TestAllTokens(t *testing.T) {
 	input := []byte("a b c\nd e")
 	expected := []string{"a", "b", "c", "d", "e"}
 	tokens, err := allTokens("TestAllTokens", input)
-
 	if err != nil {
 		t.Fatalf("Expected no error, got %v", err)
 	}
@@ -145,10 +148,11 @@ func TestParseOneAndImport(t *testing.T) {
 			"localhost",
 		}, []int{1}},
 
-		{`localhost:1234
+		{
+			`localhost:1234
 		  dir1 foo bar`, false, []string{
-			"localhost:1234",
+				"localhost:1234",
-		}, []int{3},
+			}, []int{3},
 		},
 
 		{`localhost {
@@ -343,7 +347,7 @@ func TestParseOneAndImport(t *testing.T) {
 				i, len(test.keys), len(result.Keys))
 			continue
 		}
-		for j, addr := range result.Keys {
+		for j, addr := range result.GetKeysText() {
 			if addr != test.keys[j] {
 				t.Errorf("Test %d, key %d: Expected '%s', but was '%s'",
 					i, j, test.keys[j], addr)
@@ -375,8 +379,9 @@ func TestRecursiveImport(t *testing.T) {
 	}
 
 	isExpected := func(got ServerBlock) bool {
-		if len(got.Keys) != 1 || got.Keys[0] != "localhost" {
+		textKeys := got.GetKeysText()
-			t.Errorf("got keys unexpected: expect localhost, got %v", got.Keys)
+		if len(textKeys) != 1 || textKeys[0] != "localhost" {
+			t.Errorf("got keys unexpected: expect localhost, got %v", textKeys)
 			return false
 		}
 		if len(got.Segments) != 2 {
@@ -403,13 +408,13 @@ func TestRecursiveImport(t *testing.T) {
 	err = os.WriteFile(recursiveFile1, []byte(
 		`localhost
 		dir1
-		import recursive_import_test2`), 0644)
+		import recursive_import_test2`), 0o644)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer os.Remove(recursiveFile1)
 
-	err = os.WriteFile(recursiveFile2, []byte("dir2 1"), 0644)
+	err = os.WriteFile(recursiveFile2, []byte("dir2 1"), 0o644)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -437,7 +442,7 @@ func TestRecursiveImport(t *testing.T) {
 	err = os.WriteFile(recursiveFile1, []byte(
 		`localhost
 		dir1
-		import `+recursiveFile2), 0644)
+		import `+recursiveFile2), 0o644)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -470,8 +475,9 @@ func TestDirectiveImport(t *testing.T) {
 	}
 
 	isExpected := func(got ServerBlock) bool {
-		if len(got.Keys) != 1 || got.Keys[0] != "localhost" {
+		textKeys := got.GetKeysText()
-			t.Errorf("got keys unexpected: expect localhost, got %v", got.Keys)
+		if len(textKeys) != 1 || textKeys[0] != "localhost" {
+			t.Errorf("got keys unexpected: expect localhost, got %v", textKeys)
 			return false
 		}
 		if len(got.Segments) != 2 {
@@ -491,7 +497,7 @@ func TestDirectiveImport(t *testing.T) {
 	}
 
 	err = os.WriteFile(directiveFile, []byte(`prop1 1
-	prop2 2`), 0644)
+	prop2 2`), 0o644)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -612,7 +618,7 @@ func TestParseAll(t *testing.T) {
 					i, len(test.keys[j]), j, len(block.Keys))
 				continue
 			}
-			for k, addr := range block.Keys {
+			for k, addr := range block.GetKeysText() {
 				if addr != test.keys[j][k] {
 					t.Errorf("Test %d, block %d, key %d: Expected '%s', but got '%s'",
 						i, j, k, test.keys[j][k], addr)
@@ -718,6 +724,36 @@ func TestEnvironmentReplacement(t *testing.T) {
 	}
 }
 
+func TestImportReplacementInJSONWithBrace(t *testing.T) {
+	for i, test := range []struct {
+		args   []string
+		input  string
+		expect string
+	}{
+		{
+			args:   []string{"123"},
+			input:  "{args[0]}",
+			expect: "123",
+		},
+		{
+			args:   []string{"123"},
+			input:  `{"key":"{args[0]}"}`,
+			expect: `{"key":"123"}`,
+		},
+		{
+			args:   []string{"123", "123"},
+			input:  `{"key":[{args[0]},{args[1]}]}`,
+			expect: `{"key":[123,123]}`,
+		},
+	} {
+		repl := makeArgsReplacer(test.args)
+		actual := repl.ReplaceKnown(test.input, "")
+		if actual != test.expect {
+			t.Errorf("Test %d: Expected: '%s' but got '%s'", i, test.expect, actual)
+		}
+	}
+}
+
 func TestSnippets(t *testing.T) {
 	p := testParser(`
 		(common) {
@@ -735,7 +771,7 @@ func TestSnippets(t *testing.T) {
 	if len(blocks) != 1 {
 		t.Fatalf("Expect exactly one server block. Got %d.", len(blocks))
 	}
-	if actual, expected := blocks[0].Keys[0], "http://example.com"; expected != actual {
+	if actual, expected := blocks[0].GetKeysText()[0], "http://example.com"; expected != actual {
 		t.Errorf("Expected server name to be '%s' but was '%s'", expected, actual)
 	}
 	if len(blocks[0].Segments) != 2 {
@@ -767,7 +803,7 @@ func TestImportedFilesIgnoreNonDirectiveImportTokens(t *testing.T) {
 	fileName := writeStringToTempFileOrDie(t, `
 		http://example.com {
 			# This isn't an import directive, it's just an arg with value 'import'
-			basicauth / import password
+			basic_auth / import password
 		}
 	`)
 	// Parse the root file that imports the other one.
@@ -778,12 +814,12 @@ func TestImportedFilesIgnoreNonDirectiveImportTokens(t *testing.T) {
 	}
 	auth := blocks[0].Segments[0]
 	line := auth[0].Text + " " + auth[1].Text + " " + auth[2].Text + " " + auth[3].Text
-	if line != "basicauth / import password" {
+	if line != "basic_auth / import password" {
 		// Previously, it would be changed to:
-		//   basicauth / import /path/to/test/dir/password
+		//   basic_auth / import /path/to/test/dir/password
 		// referencing a file that (probably) doesn't exist and changing the
 		// password!
-		t.Errorf("Expected basicauth tokens to be 'basicauth / import password' but got %#q", line)
+		t.Errorf("Expected basic_auth tokens to be 'basic_auth / import password' but got %#q", line)
 	}
 }
 
@@ -810,7 +846,7 @@ func TestSnippetAcrossMultipleFiles(t *testing.T) {
 	if len(blocks) != 1 {
 		t.Fatalf("Expect exactly one server block. Got %d.", len(blocks))
 	}
-	if actual, expected := blocks[0].Keys[0], "http://example.com"; expected != actual {
+	if actual, expected := blocks[0].GetKeysText()[0], "http://example.com"; expected != actual {
 		t.Errorf("Expected server name to be '%s' but was '%s'", expected, actual)
 	}
 	if len(blocks[0].Segments) != 1 {
@@ -821,6 +857,29 @@ func TestSnippetAcrossMultipleFiles(t *testing.T) {
 	}
 }
 
+func TestRejectsGlobalMatcher(t *testing.T) {
+	p := testParser(`
+		@rejected path /foo
+
+		(common) {
+			gzip foo
+			errors stderr
+		}
+
+		http://example.com {
+			import common
+		}
+	`)
+	_, err := p.parseAll()
+	if err == nil {
+		t.Fatal("Expected an error, but got nil")
+	}
+	expected := "request matchers may not be defined globally, they must be in a site block; found @rejected, at Testfile:2"
+	if err.Error() != expected {
+		t.Errorf("Expected error to be '%s' but got '%v'", expected, err)
+	}
+}
+
 func testParser(input string) parser {
 	return parser{Dispenser: NewTestDispenser(input)}
 }

caddyconfig/configadapters.go

@@ -135,11 +135,4 @@ func (am adapterModule) CaddyModule() caddy.ModuleInfo {
 	}
 }
 
-// UnwrapAdapter return the original Adapter, this method must be exported
-// to be type-assertable 🤷
-// hack, https://github.com/caddyserver/caddy/issues/5621
-func (am adapterModule) UnwrapAdapter() any {
-	return am.Adapter
-}
-
 var configAdapters = make(map[string]Adapter)

caddyconfig/httpcaddyfile/addresses.go

@@ -24,10 +24,11 @@ import (
 	"strings"
 	"unicode"
 
+	"github.com/caddyserver/certmagic"
+
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
-	"github.com/caddyserver/certmagic"
 )
 
 // mapAddressToServerBlocks returns a map of listener address to list of server
@@ -77,7 +78,8 @@ import (
 // multiple addresses to the same lists of server blocks (a many:many mapping).
 // (Doing this is essentially a map-reduce technique.)
 func (st *ServerType) mapAddressToServerBlocks(originalServerBlocks []serverBlock,
-	options map[string]any) (map[string][]serverBlock, error) {
+	options map[string]any,
+) (map[string][]serverBlock, error) {
 	sbmap := make(map[string][]serverBlock)
 
 	for i, sblock := range originalServerBlocks {
@@ -86,15 +88,15 @@ func (st *ServerType) mapAddressToServerBlocks(originalServerBlocks []serverBloc
 		// will be served by them; this has the effect of treating each
 		// key of a server block as its own, but without having to repeat its
 		// contents in cases where multiple keys really can be served together
-		addrToKeys := make(map[string][]string)
+		addrToKeys := make(map[string][]caddyfile.Token)
 		for j, key := range sblock.block.Keys {
 			// a key can have multiple listener addresses if there are multiple
 			// arguments to the 'bind' directive (although they will all have
 			// the same port, since the port is defined by the key or is implicit
 			// through automatic HTTPS)
-			addrs, err := st.listenerAddrsForServerBlockKey(sblock, key, options)
+			addrs, err := st.listenerAddrsForServerBlockKey(sblock, key.Text, options)
 			if err != nil {
-				return nil, fmt.Errorf("server block %d, key %d (%s): determining listener address: %v", i, j, key, err)
+				return nil, fmt.Errorf("server block %d, key %d (%s): determining listener address: %v", i, j, key.Text, err)
 			}
 
 			// associate this key with each listener address it is served on
@@ -120,9 +122,9 @@ func (st *ServerType) mapAddressToServerBlocks(originalServerBlocks []serverBloc
 			// parse keys so that we only have to do it once
 			parsedKeys := make([]Address, 0, len(keys))
 			for _, key := range keys {
-				addr, err := ParseAddress(key)
+				addr, err := ParseAddress(key.Text)
 				if err != nil {
-					return nil, fmt.Errorf("parsing key '%s': %v", key, err)
+					return nil, fmt.Errorf("parsing key '%s': %v", key.Text, err)
 				}
 				parsedKeys = append(parsedKeys, addr.Normalize())
 			}
@@ -187,13 +189,25 @@ func (st *ServerType) consolidateAddrMappings(addrToServerBlocks map[string][]se
 // listenerAddrsForServerBlockKey essentially converts the Caddyfile
 // site addresses to Caddy listener addresses for each server block.
 func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key string,
-	options map[string]any) ([]string, error) {
+	options map[string]any,
+) ([]string, error) {
 	addr, err := ParseAddress(key)
 	if err != nil {
 		return nil, fmt.Errorf("parsing key: %v", err)
 	}
 	addr = addr.Normalize()
 
+	switch addr.Scheme {
+	case "wss":
+		return nil, fmt.Errorf("the scheme wss:// is only supported in browsers; use https:// instead")
+	case "ws":
+		return nil, fmt.Errorf("the scheme ws:// is only supported in browsers; use http:// instead")
+	case "https", "http", "":
+		// Do nothing or handle the valid schemes
+	default:
+		return nil, fmt.Errorf("unsupported URL scheme %s://", addr.Scheme)
+	}
+
 	// figure out the HTTP and HTTPS ports; either
 	// use defaults, or override with user config
 	httpPort, httpsPort := strconv.Itoa(caddyhttp.DefaultHTTPPort), strconv.Itoa(caddyhttp.DefaultHTTPSPort)

caddyconfig/httpcaddyfile/builtins_test.go

@@ -25,11 +25,12 @@ func TestLogDirectiveSyntax(t *testing.T) {
 		{
 			input: `:8080 {
 				log {
+					core mock
 					output file foo.log
 				}
 			}
 			`,
-			output:      `{"logging":{"logs":{"default":{"exclude":["http.log.access.log0"]},"log0":{"writer":{"filename":"foo.log","output":"file"},"include":["http.log.access.log0"]}}},"apps":{"http":{"servers":{"srv0":{"listen":[":8080"],"logs":{"default_logger_name":"log0"}}}}}}`,
+			output:      `{"logging":{"logs":{"default":{"exclude":["http.log.access.log0"]},"log0":{"writer":{"filename":"foo.log","output":"file"},"core":{"module":"mock"},"include":["http.log.access.log0"]}}},"apps":{"http":{"servers":{"srv0":{"listen":[":8080"],"logs":{"default_logger_name":"log0"}}}}}}`,
 			expectError: false,
 		},
 		{
@@ -52,12 +53,14 @@ func TestLogDirectiveSyntax(t *testing.T) {
 		},
 		{
 			input: `:8080 {
-				log invalid {
+				log name-override {
+					core mock
 					output file foo.log
 				}
 			}
 			`,
-			expectError: true,
+			output:      `{"logging":{"logs":{"default":{"exclude":["http.log.access.name-override"]},"name-override":{"writer":{"filename":"foo.log","output":"file"},"core":{"module":"mock"},"include":["http.log.access.name-override"]}}},"apps":{"http":{"servers":{"srv0":{"listen":[":8080"],"logs":{"default_logger_name":"name-override"}}}}}}`,
+			expectError: false,
 		},
 	} {
 
@@ -277,3 +280,76 @@ func TestImportErrorLine(t *testing.T) {
 		}
 	}
 }
+
+func TestNestedImport(t *testing.T) {
+	for i, tc := range []struct {
+		input     string
+		errorFunc func(err error) bool
+	}{
+		{
+			input: `(t1) {
+						respond {args[0]} {args[1]}
+					}
+					
+					(t2) {
+						import t1 {args[0]} 202
+					}
+					
+					:8080 {
+						handle {
+							import t2 "foobar"
+						}
+					}`,
+			errorFunc: func(err error) bool {
+				return err == nil
+			},
+		},
+		{
+			input: `(t1) {
+						respond {args[:]}
+					}
+					
+					(t2) {
+						import t1 {args[0]} {args[1]}
+					}
+					
+					:8080 {
+						handle {
+							import t2 "foobar" 202
+						}
+					}`,
+			errorFunc: func(err error) bool {
+				return err == nil
+			},
+		},
+		{
+			input: `(t1) {
+						respond {args[0]} {args[1]}
+					}
+					
+					(t2) {
+						import t1 {args[:]}
+					}
+					
+					:8080 {
+						handle {
+							import t2 "foobar" 202
+						}
+					}`,
+			errorFunc: func(err error) bool {
+				return err == nil
+			},
+		},
+	} {
+		adapter := caddyfile.Adapter{
+			ServerType: ServerType{},
+		}
+
+		_, _, err := adapter.Adapt([]byte(tc.input), nil)
+
+		if !tc.errorFunc(err) {
+			t.Errorf("Test %d error expectation failed, got %s", i, err)
+			continue
+		}
+	}
+}

caddyconfig/httpcaddyfile/directives.go

@@ -27,22 +27,33 @@ import (
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 )
 
-// directiveOrder specifies the order
+// defaultDirectiveOrder specifies the default order
-// to apply directives in HTTP routes.
+// to apply directives in HTTP routes. This must only
+// consist of directives that are included in Caddy's
+// standard distribution.
 //
-// The root directive goes first in case rewrites or
+// e.g. The 'root' directive goes near the start in
-// redirects depend on existence of files, i.e. the
+// case rewrites or redirects depend on existence of
-// file matcher, which must know the root first.
+// files, i.e. the file matcher, which must know the
+// root first.
 //
-// The header directive goes second so that headers
+// e.g. The 'header' directive goes before 'redir' so
-// can be manipulated before doing redirects.
+// that headers can be manipulated before doing redirects.
-var directiveOrder = []string{
+//
+// e.g. The 'respond' directive is near the end because it
+// writes a response and terminates the middleware chain.
+var defaultDirectiveOrder = []string{
 	"tracing",
 
+	// set variables that may be used by other directives
 	"map",
 	"vars",
+	"fs",
 	"root",
-	"skip_log",
+	"log_append",
+	"skip_log", // TODO: deprecated, renamed to log_skip
+	"log_skip",
+	"log_name",
 
 	"header",
 	"copy_response_headers", // only in reverse_proxy's handle_response
@@ -57,11 +68,13 @@ var directiveOrder = []string{
 	"try_files",
 
 	// middleware handlers; some wrap responses
-	"basicauth",
+	"basicauth", // TODO: deprecated, renamed to basic_auth
+	"basic_auth",
 	"forward_auth",
 	"request_header",
 	"encode",
 	"push",
+	"intercept",
 	"templates",
 
 	// special routing & dispatching directives
@@ -82,6 +95,11 @@ var directiveOrder = []string{
 	"acme_server",
 }
 
+// directiveOrder specifies the order to apply directives
+// in HTTP routes, after being modified by either the
+// plugins or by the user via the "order" global option.
+var directiveOrder = defaultDirectiveOrder
+
 // directiveIsOrdered returns true if dir is
 // a known, ordered (sorted) directive.
 func directiveIsOrdered(dir string) bool {
@@ -128,6 +146,58 @@ func RegisterHandlerDirective(dir string, setupFunc UnmarshalHandlerFunc) {
 	})
 }
 
+// RegisterDirectiveOrder registers the default order for a
+// directive from a plugin.
+//
+// This is useful when a plugin has a well-understood place
+// it should run in the middleware pipeline, and it allows
+// users to avoid having to define the order themselves.
+//
+// The directive dir may be placed in the position relative
+// to ('before' or 'after') a directive included in Caddy's
+// standard distribution. It cannot be relative to another
+// plugin's directive.
+//
+// EXPERIMENTAL: This API may change or be removed.
+func RegisterDirectiveOrder(dir string, position Positional, standardDir string) {
+	// check if directive was already ordered
+	if directiveIsOrdered(dir) {
+		panic("directive '" + dir + "' already ordered")
+	}
+
+	if position != Before && position != After {
+		panic("the 2nd argument must be either 'before' or 'after', got '" + position + "'")
+	}
+
+	// check if directive exists in standard distribution, since
+	// we can't allow plugins to depend on one another; we can't
+	// guarantee the order that plugins are loaded in.
+	foundStandardDir := false
+	for _, d := range defaultDirectiveOrder {
+		if d == standardDir {
+			foundStandardDir = true
+		}
+	}
+	if !foundStandardDir {
+		panic("the 3rd argument '" + standardDir + "' must be a directive that exists in the standard distribution of Caddy")
+	}
+
+	// insert directive into proper position
+	newOrder := directiveOrder
+	for i, d := range newOrder {
+		if d != standardDir {
+			continue
+		}
+		if position == Before {
+			newOrder = append(newOrder[:i], append([]string{dir}, newOrder[i:]...)...)
+		} else if position == After {
+			newOrder = append(newOrder[:i+1], append([]string{dir}, newOrder[i+1:]...)...)
+		}
+		break
+	}
+	directiveOrder = newOrder
+}
+
 // RegisterGlobalOption registers a unique global option opt with
 // an associated unmarshaling (setup) function. When the global
 // option opt is encountered in a Caddyfile, setupFunc will be
@@ -217,7 +287,8 @@ func (h Helper) ExtractMatcherSet() (caddy.ModuleMap, error) {
 
 // NewRoute returns config values relevant to creating a new HTTP route.
 func (h Helper) NewRoute(matcherSet caddy.ModuleMap,
-	handler caddyhttp.MiddlewareHandler) []ConfigValue {
+	handler caddyhttp.MiddlewareHandler,
+) []ConfigValue {
 	mod, err := caddy.GetModule(caddy.GetModuleID(handler))
 	if err != nil {
 		*h.warnings = append(*h.warnings, caddyconfig.Warning{
@@ -269,12 +340,6 @@ func (h Helper) GroupRoutes(vals []ConfigValue) {
 	}
 }
 
-// NewBindAddresses returns config values relevant to adding
-// listener bind addresses to the config.
-func (h Helper) NewBindAddresses(addrs []string) []ConfigValue {
-	return []ConfigValue{{Class: "bind", Value: addrs}}
-}
-
 // WithDispenser returns a new instance based on d. All others Helper
 // fields are copied, so typically maps are shared with this new instance.
 func (h Helper) WithDispenser(d *caddyfile.Dispenser) Helper {
@@ -557,6 +622,16 @@ func (sb serverBlock) isAllHTTP() bool {
 	return true
 }
 
+// Positional are the supported modes for ordering directives.
+type Positional string
+
+const (
+	Before Positional = "before"
+	After  Positional = "after"
+	First  Positional = "first"
+	Last   Positional = "last"
+)
+
 type (
 	// UnmarshalFunc is a function which can unmarshal Caddyfile
 	// tokens into zero or more config values using a Helper type.

caddyconfig/httpcaddyfile/directives_test.go

@@ -31,20 +31,23 @@ func TestHostsFromKeys(t *testing.T) {
 			[]Address{
 				{Original: ":2015", Port: "2015"},
 			},
-			[]string{}, []string{},
+			[]string{},
+			[]string{},
 		},
 		{
 			[]Address{
 				{Original: ":443", Port: "443"},
 			},
-			[]string{}, []string{},
+			[]string{},
+			[]string{},
 		},
 		{
 			[]Address{
 				{Original: "foo", Host: "foo"},
 				{Original: ":2015", Port: "2015"},
 			},
-			[]string{}, []string{"foo"},
+			[]string{},
+			[]string{"foo"},
 		},
 		{
 			[]Address{

caddyconfig/httpcaddyfile/httptype.go

@@ -17,19 +17,21 @@ package httpcaddyfile
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 	"reflect"
-	"regexp"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
 
+	"go.uber.org/zap"
+
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddypki"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
-	"go.uber.org/zap"
 )
 
 func init() {
@@ -48,8 +50,7 @@ type App struct {
 }
 
 // ServerType can set up a config from an HTTP Caddyfile.
-type ServerType struct {
+type ServerType struct{}
-}
 
 // Setup makes a config from the tokens.
 func (st ServerType) Setup(
@@ -64,8 +65,11 @@ func (st ServerType) Setup(
 	originalServerBlocks := make([]serverBlock, 0, len(inputServerBlocks))
 	for _, sblock := range inputServerBlocks {
 		for j, k := range sblock.Keys {
-			if j == 0 && strings.HasPrefix(k, "@") {
+			if j == 0 && strings.HasPrefix(k.Text, "@") {
-				return nil, warnings, fmt.Errorf("cannot define a matcher outside of a site block: '%s'", k)
+				return nil, warnings, fmt.Errorf("%s:%d: cannot define a matcher outside of a site block: '%s'", k.File, k.Line, k.Text)
+			}
+			if _, ok := registeredDirectives[k.Text]; ok {
+				return nil, warnings, fmt.Errorf("%s:%d: parsed '%s' as a site address, but it is a known directive; directives must appear in a site block", k.File, k.Line, k.Text)
 			}
 		}
 		originalServerBlocks = append(originalServerBlocks, serverBlock{
@@ -81,46 +85,18 @@ func (st ServerType) Setup(
 		return nil, warnings, err
 	}
 
-	originalServerBlocks, err = st.extractNamedRoutes(originalServerBlocks, options, &warnings)
+	// this will replace both static and user-defined placeholder shorthands
+	// with actual identifiers used by Caddy
+	replacer := NewShorthandReplacer()
+
+	originalServerBlocks, err = st.extractNamedRoutes(originalServerBlocks, options, &warnings, replacer)
 	if err != nil {
 		return nil, warnings, err
 	}
 
-	// replace shorthand placeholders (which are convenient
-	// when writing a Caddyfile) with their actual placeholder
-	// identifiers or variable names
-	replacer := strings.NewReplacer(placeholderShorthands()...)
-
-	// these are placeholders that allow a user-defined final
-	// parameters, but we still want to provide a shorthand
-	// for those, so we use a regexp to replace
-	regexpReplacements := []struct {
-		search  *regexp.Regexp
-		replace string
-	}{
-		{regexp.MustCompile(`{header\.([\w-]*)}`), "{http.request.header.$1}"},
-		{regexp.MustCompile(`{cookie\.([\w-]*)}`), "{http.request.cookie.$1}"},
-		{regexp.MustCompile(`{labels\.([\w-]*)}`), "{http.request.host.labels.$1}"},
-		{regexp.MustCompile(`{path\.([\w-]*)}`), "{http.request.uri.path.$1}"},
-		{regexp.MustCompile(`{file\.([\w-]*)}`), "{http.request.uri.path.file.$1}"},
-		{regexp.MustCompile(`{query\.([\w-]*)}`), "{http.request.uri.query.$1}"},
-		{regexp.MustCompile(`{re\.([\w-]*)\.([\w-]*)}`), "{http.regexp.$1.$2}"},
-		{regexp.MustCompile(`{vars\.([\w-]*)}`), "{http.vars.$1}"},
-		{regexp.MustCompile(`{rp\.([\w-\.]*)}`), "{http.reverse_proxy.$1}"},
-		{regexp.MustCompile(`{err\.([\w-\.]*)}`), "{http.error.$1}"},
-		{regexp.MustCompile(`{file_match\.([\w-]*)}`), "{http.matchers.file.$1}"},
-	}
-
 	for _, sb := range originalServerBlocks {
-		for _, segment := range sb.block.Segments {
+		for i := range sb.block.Segments {
-			for i := 0; i < len(segment); i++ {
+			replacer.ApplyToSegment(&sb.block.Segments[i])
-				// simple string replacements
-				segment[i].Text = replacer.Replace(segment[i].Text)
-				// complex regexp replacements
-				for _, r := range regexpReplacements {
-					segment[i].Text = r.search.ReplaceAllString(segment[i].Text, r.replace)
-				}
-			}
 		}
 
 		if len(sb.block.Keys) == 0 {
@@ -241,7 +217,7 @@ func (st ServerType) Setup(
 		if ncl.name == caddy.DefaultLoggerName {
 			hasDefaultLog = true
 		}
-		if _, ok := options["debug"]; ok && ncl.log.Level == "" {
+		if _, ok := options["debug"]; ok && ncl.log != nil && ncl.log.Level == "" {
 			ncl.log.Level = zap.DebugLevel.CapitalString()
 		}
 		customLogs = append(customLogs, ncl)
@@ -298,6 +274,12 @@ func (st ServerType) Setup(
 	if !reflect.DeepEqual(pkiApp, &caddypki.PKI{CAs: make(map[string]*caddypki.CA)}) {
 		cfg.AppsRaw["pki"] = caddyconfig.JSON(pkiApp, &warnings)
 	}
+	if filesystems, ok := options["filesystem"].(caddy.Module); ok {
+		cfg.AppsRaw["caddy.filesystems"] = caddyconfig.JSON(
+			filesystems,
+			&warnings)
+	}
+
 	if storageCvtr, ok := options["storage"].(caddy.StorageConverter); ok {
 		cfg.StorageRaw = caddyconfig.JSONModuleObject(storageCvtr,
 			"module",
@@ -307,7 +289,6 @@ func (st ServerType) Setup(
 	if adminConfig, ok := options["admin"].(*caddy.AdminConfig); ok && adminConfig != nil {
 		cfg.Admin = adminConfig
 	}
-
 	if pc, ok := options["persist_config"].(string); ok && pc == "off" {
 		if cfg.Admin == nil {
 			cfg.Admin = new(caddy.AdminConfig)
@@ -324,7 +305,21 @@ func (st ServerType) Setup(
 				Logs: make(map[string]*caddy.CustomLog),
 			}
 		}
+
+		// Add the default log first if defined, so that it doesn't
+		// accidentally get re-created below due to the Exclude logic
 		for _, ncl := range customLogs {
+			if ncl.name == caddy.DefaultLoggerName && ncl.log != nil {
+				cfg.Logging.Logs[caddy.DefaultLoggerName] = ncl.log
+				break
+			}
+		}
+
+		// Add the rest of the custom logs
+		for _, ncl := range customLogs {
+			if ncl.log == nil || ncl.name == caddy.DefaultLoggerName {
+				continue
+			}
 			if ncl.name != "" {
 				cfg.Logging.Logs[ncl.name] = ncl.log
 			}
@@ -338,8 +333,16 @@ func (st ServerType) Setup(
 					cfg.Logging.Logs[caddy.DefaultLoggerName] = defaultLog
 				}
 				defaultLog.Exclude = append(defaultLog.Exclude, ncl.log.Include...)
+
+				// avoid duplicates by sorting + compacting
+				sort.Strings(defaultLog.Exclude)
+				defaultLog.Exclude = slices.Compact[[]string, string](defaultLog.Exclude)
 			}
 		}
+		// we may have not actually added anything, so remove if empty
+		if len(cfg.Logging.Logs) == 0 {
+			cfg.Logging = nil
+		}
 	}
 
 	return cfg, warnings, nil
@@ -429,6 +432,7 @@ func (ServerType) extractNamedRoutes(
 	serverBlocks []serverBlock,
 	options map[string]any,
 	warnings *[]caddyconfig.Warning,
+	replacer ShorthandReplacer,
 ) ([]serverBlock, error) {
 	namedRoutes := map[string]*caddyhttp.Route{}
 
@@ -454,11 +458,14 @@ func (ServerType) extractNamedRoutes(
 			continue
 		}
 
-		// zip up all the segments since ParseSegmentAsSubroute
-		// was designed to take a directive+
 		wholeSegment := caddyfile.Segment{}
-		for _, segment := range sb.block.Segments {
+		for i := range sb.block.Segments {
-			wholeSegment = append(wholeSegment, segment...)
+			// replace user-defined placeholder shorthands in extracted named routes
+			replacer.ApplyToSegment(&sb.block.Segments[i])
+
+			// zip up all the segments since ParseSegmentAsSubroute
+			// was designed to take a directive+
+			wholeSegment = append(wholeSegment, sb.block.Segments[i]...)
 		}
 
 		h := Helper{
@@ -486,7 +493,7 @@ func (ServerType) extractNamedRoutes(
 			route.HandlersRaw = []json.RawMessage{caddyconfig.JSONModuleObject(handler, "handler", subroute.CaddyModule().ID.Name(), h.warnings)}
 		}
 
-		namedRoutes[sb.block.Keys[0]] = &route
+		namedRoutes[sb.block.GetKeysText()[0]] = &route
 	}
 	options["named_routes"] = namedRoutes
 
@@ -524,12 +531,12 @@ func (st *ServerType) serversFromPairings(
 		// address), otherwise their routes will improperly be added
 		// to the same server (see issue #4635)
 		for j, sblock1 := range p.serverBlocks {
-			for _, key := range sblock1.block.Keys {
+			for _, key := range sblock1.block.GetKeysText() {
 				for k, sblock2 := range p.serverBlocks {
 					if k == j {
 						continue
 					}
-					if sliceContains(sblock2.block.Keys, key) {
+					if sliceContains(sblock2.block.GetKeysText(), key) {
 						return nil, fmt.Errorf("ambiguous site definition: %s", key)
 					}
 				}
@@ -687,6 +694,7 @@ func (st *ServerType) serversFromPairings(
 					}
 
 					if len(hosts) > 0 {
+						slices.Sort(hosts) // for deterministic JSON output
 						cp.MatchersRaw = caddy.ModuleMap{
 							"sni": caddyconfig.JSON(hosts, warnings), // make sure to match all hosts, not just auto-HTTPS-qualified ones
 						}
@@ -718,10 +726,20 @@ func (st *ServerType) serversFromPairings(
 					}
 				}
 
+				// If TLS is specified as directive, it will also result in 1 or more connection policy being created
+				// Thus, catch-all address with non-standard port, e.g. :8443, can have TLS enabled without
+				// specifying prefix "https://"
+				// Second part of the condition is to allow creating TLS conn policy even though `auto_https` has been disabled
+				// ensuring compatibility with behavior described in below link
+				// https://caddy.community/t/making-sense-of-auto-https-and-why-disabling-it-still-serves-https-instead-of-http/9761
+				createdTLSConnPolicies, ok := sblock.pile["tls.connection_policy"]
+				hasTLSEnabled := (ok && len(createdTLSConnPolicies) > 0) ||
+					(addr.Host != "" && srv.AutoHTTPS != nil && !sliceContains(srv.AutoHTTPS.Skip, addr.Host))
+
 				// we'll need to remember if the address qualifies for auto-HTTPS, so we
 				// can add a TLS conn policy if necessary
 				if addr.Scheme == "https" ||
-					(addr.Scheme != "http" && addr.Host != "" && addr.Port != httpPort) {
+					(addr.Scheme != "http" && addr.Port != httpPort && hasTLSEnabled) {
 					addressQualifiesForTLS = true
 				}
 				// predict whether auto-HTTPS will add the conn policy for us; if so, we
@@ -759,10 +777,19 @@ func (st *ServerType) serversFromPairings(
 				if srv.Errors == nil {
 					srv.Errors = new(caddyhttp.HTTPErrorConfig)
 				}
+				sort.SliceStable(errorSubrouteVals, func(i, j int) bool {
+					sri, srj := errorSubrouteVals[i].Value.(*caddyhttp.Subroute), errorSubrouteVals[j].Value.(*caddyhttp.Subroute)
+					if len(sri.Routes[0].MatcherSetsRaw) == 0 && len(srj.Routes[0].MatcherSetsRaw) != 0 {
+						return false
+					}
+					return true
+				})
+				errorsSubroute := &caddyhttp.Subroute{}
 				for _, val := range errorSubrouteVals {
 					sr := val.Value.(*caddyhttp.Subroute)
-					srv.Errors.Routes = appendSubrouteToRouteList(srv.Errors.Routes, sr, matcherSetsEnc, p, warnings)
+					errorsSubroute.Routes = append(errorsSubroute.Routes, sr.Routes...)
 				}
+				srv.Errors.Routes = appendSubrouteToRouteList(srv.Errors.Routes, errorsSubroute, matcherSetsEnc, p, warnings)
 			}
 
 			// add log associations
@@ -770,17 +797,39 @@ func (st *ServerType) serversFromPairings(
 			sblockLogHosts := sblock.hostsFromKeys(true)
 			for _, cval := range sblock.pile["custom_log"] {
 				ncl := cval.Value.(namedCustomLog)
-				if sblock.hasHostCatchAllKey() {
+
+				// if `no_hostname` is set, then this logger will not
+				// be associated with any of the site block's hostnames,
+				// and only be usable via the `log_name` directive
+				// or the `access_logger_names` variable
+				if ncl.noHostname {
+					continue
+				}
+
+				if sblock.hasHostCatchAllKey() && len(ncl.hostnames) == 0 {
 					// all requests for hosts not able to be listed should use
 					// this log because it's a catch-all-hosts server block
 					srv.Logs.DefaultLoggerName = ncl.name
-				} else {
+				} else if len(ncl.hostnames) > 0 {
-					// map each host to the user's desired logger name
+					// if the logger overrides the hostnames, map that to the logger name
-					for _, h := range sblockLogHosts {
+					for _, h := range ncl.hostnames {
 						if srv.Logs.LoggerNames == nil {
-							srv.Logs.LoggerNames = make(map[string]string)
+							srv.Logs.LoggerNames = make(map[string]caddyhttp.StringArray)
 						}
-						srv.Logs.LoggerNames[h] = ncl.name
+						srv.Logs.LoggerNames[h] = append(srv.Logs.LoggerNames[h], ncl.name)
+					}
+				} else {
+					// otherwise, map each host to the logger name
+					for _, h := range sblockLogHosts {
+						// strip the port from the host, if any
+						host, _, err := net.SplitHostPort(h)
+						if err != nil {
+							host = h
+						}
+						if srv.Logs.LoggerNames == nil {
+							srv.Logs.LoggerNames = make(map[string]caddyhttp.StringArray)
+						}
+						srv.Logs.LoggerNames[host] = append(srv.Logs.LoggerNames[host], ncl.name)
 					}
 				}
 			}
@@ -797,6 +846,11 @@ func (st *ServerType) serversFromPairings(
 			}
 		}
 
+		// sort for deterministic JSON output
+		if srv.Logs != nil {
+			slices.Sort(srv.Logs.SkipHosts)
+		}
+
 		// a server cannot (natively) serve both HTTP and HTTPS at the
 		// same time, so make sure the configuration isn't in conflict
 		err := detectConflictingSchemes(srv, p.serverBlocks, options)
@@ -1028,8 +1082,8 @@ func appendSubrouteToRouteList(routeList caddyhttp.RouteList,
 	subroute *caddyhttp.Subroute,
 	matcherSetsEnc []caddy.ModuleMap,
 	p sbAddrAssociation,
-	warnings *[]caddyconfig.Warning) caddyhttp.RouteList {
+	warnings *[]caddyconfig.Warning,
-
+) caddyhttp.RouteList {
 	// nothing to do if... there's nothing to do
 	if len(matcherSetsEnc) == 0 && len(subroute.Routes) == 0 && subroute.Errors == nil {
 		return routeList
@@ -1237,19 +1291,24 @@ func matcherSetFromMatcherToken(
 	if tkn.Text == "*" {
 		// match all requests == no matchers, so nothing to do
 		return nil, true, nil
-	} else if strings.HasPrefix(tkn.Text, "/") {
+	}
-		// convenient way to specify a single path match
+
+	// convenient way to specify a single path match
+	if strings.HasPrefix(tkn.Text, "/") {
 		return caddy.ModuleMap{
 			"path": caddyconfig.JSON(caddyhttp.MatchPath{tkn.Text}, warnings),
 		}, true, nil
-	} else if strings.HasPrefix(tkn.Text, matcherPrefix) {
+	}
-		// pre-defined matcher
+
+	// pre-defined matcher
+	if strings.HasPrefix(tkn.Text, matcherPrefix) {
 		m, ok := matcherDefs[tkn.Text]
 		if !ok {
 			return nil, false, fmt.Errorf("unrecognized matcher name: %+v", tkn.Text)
 		}
 		return m, true, nil
 	}
+
 	return nil, false, nil
 }
 
@@ -1339,68 +1398,83 @@ func (st *ServerType) compileEncodedMatcherSets(sblock serverBlock) ([]caddy.Mod
 }
 
 func parseMatcherDefinitions(d *caddyfile.Dispenser, matchers map[string]caddy.ModuleMap) error {
-	for d.Next() {
+	d.Next() // advance to the first token
-		// this is the "name" for "named matchers"
-		definitionName := d.Val()
 
-		if _, ok := matchers[definitionName]; ok {
+	// this is the "name" for "named matchers"
-			return fmt.Errorf("matcher is defined more than once: %s", definitionName)
+	definitionName := d.Val()
+
+	if _, ok := matchers[definitionName]; ok {
+		return fmt.Errorf("matcher is defined more than once: %s", definitionName)
+	}
+	matchers[definitionName] = make(caddy.ModuleMap)
+
+	// given a matcher name and the tokens following it, parse
+	// the tokens as a matcher module and record it
+	makeMatcher := func(matcherName string, tokens []caddyfile.Token) error {
+		// create a new dispenser from the tokens
+		dispenser := caddyfile.NewDispenser(tokens)
+
+		// set the matcher name (without @) in the dispenser context so
+		// that matcher modules can access it to use it as their name
+		// (e.g. regexp matchers which use the name for capture groups)
+		dispenser.SetContext(caddyfile.MatcherNameCtxKey, definitionName[1:])
+
+		mod, err := caddy.GetModule("http.matchers." + matcherName)
+		if err != nil {
+			return fmt.Errorf("getting matcher module '%s': %v", matcherName, err)
 		}
-		matchers[definitionName] = make(caddy.ModuleMap)
+		unm, ok := mod.New().(caddyfile.Unmarshaler)
+		if !ok {
+			return fmt.Errorf("matcher module '%s' is not a Caddyfile unmarshaler", matcherName)
+		}
+		err = unm.UnmarshalCaddyfile(dispenser)
+		if err != nil {
+			return err
+		}
+		rm, ok := unm.(caddyhttp.RequestMatcher)
+		if !ok {
+			return fmt.Errorf("matcher module '%s' is not a request matcher", matcherName)
+		}
+		matchers[definitionName][matcherName] = caddyconfig.JSON(rm, nil)
+		return nil
+	}
 
-		// given a matcher name and the tokens following it, parse
+	// if the next token is quoted, we can assume it's not a matcher name
-		// the tokens as a matcher module and record it
+	// and that it's probably an 'expression' matcher
-		makeMatcher := func(matcherName string, tokens []caddyfile.Token) error {
+	if d.NextArg() {
-			mod, err := caddy.GetModule("http.matchers." + matcherName)
+		if d.Token().Quoted() {
-			if err != nil {
+			// since it was missing the matcher name, we insert a token
-				return fmt.Errorf("getting matcher module '%s': %v", matcherName, err)
+			// in front of the expression token itself; we use Clone() to
-			}
+			// make the new token to keep the same the import location as
-			unm, ok := mod.New().(caddyfile.Unmarshaler)
+			// the next token, if this is within a snippet or imported file.
-			if !ok {
+			// see https://github.com/caddyserver/caddy/issues/6287
-				return fmt.Errorf("matcher module '%s' is not a Caddyfile unmarshaler", matcherName)
+			expressionToken := d.Token().Clone()
-			}
+			expressionToken.Text = "expression"
-			err = unm.UnmarshalCaddyfile(caddyfile.NewDispenser(tokens))
+			err := makeMatcher("expression", []caddyfile.Token{expressionToken, d.Token()})
 			if err != nil {
 				return err
 			}
-			rm, ok := unm.(caddyhttp.RequestMatcher)
-			if !ok {
-				return fmt.Errorf("matcher module '%s' is not a request matcher", matcherName)
-			}
-			matchers[definitionName][matcherName] = caddyconfig.JSON(rm, nil)
 			return nil
 		}
 
-		// if the next token is quoted, we can assume it's not a matcher name
+		// if it wasn't quoted, then we need to rewind after calling
-		// and that it's probably an 'expression' matcher
+		// d.NextArg() so the below properly grabs the matcher name
-		if d.NextArg() {
+		d.Prev()
-			if d.Token().Quoted() {
+	}
-				err := makeMatcher("expression", []caddyfile.Token{d.Token()})
-				if err != nil {
-					return err
-				}
-				continue
-			}
 
-			// if it wasn't quoted, then we need to rewind after calling
+	// in case there are multiple instances of the same matcher, concatenate
-			// d.NextArg() so the below properly grabs the matcher name
+	// their tokens (we expect that UnmarshalCaddyfile should be able to
-			d.Prev()
+	// handle more than one segment); otherwise, we'd overwrite other
-		}
+	// instances of the matcher in this set
-
+	tokensByMatcherName := make(map[string][]caddyfile.Token)
-		// in case there are multiple instances of the same matcher, concatenate
+	for nesting := d.Nesting(); d.NextArg() || d.NextBlock(nesting); {
-		// their tokens (we expect that UnmarshalCaddyfile should be able to
+		matcherName := d.Val()
-		// handle more than one segment); otherwise, we'd overwrite other
+		tokensByMatcherName[matcherName] = append(tokensByMatcherName[matcherName], d.NextSegment()...)
-		// instances of the matcher in this set
+	}
-		tokensByMatcherName := make(map[string][]caddyfile.Token)
+	for matcherName, tokens := range tokensByMatcherName {
-		for nesting := d.Nesting(); d.NextArg() || d.NextBlock(nesting); {
+		err := makeMatcher(matcherName, tokens)
-			matcherName := d.Val()
+		if err != nil {
-			tokensByMatcherName[matcherName] = append(tokensByMatcherName[matcherName], d.NextSegment()...)
+			return err
-		}
-		for matcherName, tokens := range tokensByMatcherName {
-			err := makeMatcher(matcherName, tokens)
-			if err != nil {
-				return err
-			}
 		}
 	}
 	return nil
@@ -1418,37 +1492,6 @@ func encodeMatcherSet(matchers map[string]caddyhttp.RequestMatcher) (caddy.Modul
 	return msEncoded, nil
 }
 
-// placeholderShorthands returns a slice of old-new string pairs,
-// where the left of the pair is a placeholder shorthand that may
-// be used in the Caddyfile, and the right is the replacement.
-func placeholderShorthands() []string {
-	return []string{
-		"{dir}", "{http.request.uri.path.dir}",
-		"{file}", "{http.request.uri.path.file}",
-		"{host}", "{http.request.host}",
-		"{hostport}", "{http.request.hostport}",
-		"{port}", "{http.request.port}",
-		"{method}", "{http.request.method}",
-		"{path}", "{http.request.uri.path}",
-		"{query}", "{http.request.uri.query}",
-		"{remote}", "{http.request.remote}",
-		"{remote_host}", "{http.request.remote.host}",
-		"{remote_port}", "{http.request.remote.port}",
-		"{scheme}", "{http.request.scheme}",
-		"{uri}", "{http.request.uri}",
-		"{tls_cipher}", "{http.request.tls.cipher_suite}",
-		"{tls_version}", "{http.request.tls.version}",
-		"{tls_client_fingerprint}", "{http.request.tls.client.fingerprint}",
-		"{tls_client_issuer}", "{http.request.tls.client.issuer}",
-		"{tls_client_serial}", "{http.request.tls.client.serial}",
-		"{tls_client_subject}", "{http.request.tls.client.subject}",
-		"{tls_client_certificate_pem}", "{http.request.tls.client.certificate_pem}",
-		"{tls_client_certificate_der_base64}", "{http.request.tls.client.certificate_der_base64}",
-		"{upstream_hostport}", "{http.reverse_proxy.upstream.hostport}",
-		"{client_ip}", "{http.vars.client_ip}",
-	}
-}
-
 // WasReplacedPlaceholderShorthand checks if a token string was
 // likely a replaced shorthand of the known Caddyfile placeholder
 // replacement outputs. Useful to prevent some user-defined map
@@ -1564,8 +1607,10 @@ func (c counter) nextGroup() string {
 }
 
 type namedCustomLog struct {
-	name string
+	name       string
-	log  *caddy.CustomLog
+	hostnames  []string
+	log        *caddy.CustomLog
+	noHostname bool
 }
 
 // sbAddrAssociation is a mapping from a list of
@@ -1576,8 +1621,10 @@ type sbAddrAssociation struct {
 	serverBlocks []serverBlock
 }
 
-const matcherPrefix = "@"
+const (
-const namedRouteKey = "named_route"
+	matcherPrefix = "@"
+	namedRouteKey = "named_route"
+)
 
 // Interface guard
 var _ caddyfile.ServerType = (*ServerType)(nil)

caddyconfig/httpcaddyfile/options.go

@@ -17,12 +17,13 @@ package httpcaddyfile
 import (
 	"strconv"
 
+	"github.com/caddyserver/certmagic"
+	"github.com/mholt/acmez/v2/acme"
+
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
-	"github.com/caddyserver/certmagic"
-	"github.com/mholt/acmez/acme"
 )
 
 func init() {
@@ -53,6 +54,7 @@ func init() {
 	RegisterGlobalOption("auto_https", parseOptAutoHTTPS)
 	RegisterGlobalOption("servers", parseServerOptions)
 	RegisterGlobalOption("ocsp_stapling", parseOCSPStaplingOptions)
+	RegisterGlobalOption("cert_lifetime", parseOptDuration)
 	RegisterGlobalOption("log", parseLogOptions)
 	RegisterGlobalOption("preferred_chains", parseOptPreferredChains)
 	RegisterGlobalOption("persist_config", parseOptPersistConfig)
@@ -61,105 +63,103 @@ func init() {
 func parseOptTrue(d *caddyfile.Dispenser, _ any) (any, error) { return true, nil }
 
 func parseOptHTTPPort(d *caddyfile.Dispenser, _ any) (any, error) {
+	d.Next() // consume option name
 	var httpPort int
-	for d.Next() {
+	var httpPortStr string
-		var httpPortStr string
+	if !d.AllArgs(&httpPortStr) {
-		if !d.AllArgs(&httpPortStr) {
+		return 0, d.ArgErr()
-			return 0, d.ArgErr()
+	}
-		}
+	var err error
-		var err error
+	httpPort, err = strconv.Atoi(httpPortStr)
-		httpPort, err = strconv.Atoi(httpPortStr)
+	if err != nil {
-		if err != nil {
+		return 0, d.Errf("converting port '%s' to integer value: %v", httpPortStr, err)
-			return 0, d.Errf("converting port '%s' to integer value: %v", httpPortStr, err)
-		}
 	}
 	return httpPort, nil
 }
 
 func parseOptHTTPSPort(d *caddyfile.Dispenser, _ any) (any, error) {
+	d.Next() // consume option name
 	var httpsPort int
-	for d.Next() {
+	var httpsPortStr string
-		var httpsPortStr string
+	if !d.AllArgs(&httpsPortStr) {
-		if !d.AllArgs(&httpsPortStr) {
+		return 0, d.ArgErr()
-			return 0, d.ArgErr()
+	}
-		}
+	var err error
-		var err error
+	httpsPort, err = strconv.Atoi(httpsPortStr)
-		httpsPort, err = strconv.Atoi(httpsPortStr)
+	if err != nil {
-		if err != nil {
+		return 0, d.Errf("converting port '%s' to integer value: %v", httpsPortStr, err)
-			return 0, d.Errf("converting port '%s' to integer value: %v", httpsPortStr, err)
-		}
 	}
 	return httpsPort, nil
 }
 
 func parseOptOrder(d *caddyfile.Dispenser, _ any) (any, error) {
+	d.Next() // consume option name
+
+	// get directive name
+	if !d.Next() {
+		return nil, d.ArgErr()
+	}
+	dirName := d.Val()
+	if _, ok := registeredDirectives[dirName]; !ok {
+		return nil, d.Errf("%s is not a registered directive", dirName)
+	}
+
+	// get positional token
+	if !d.Next() {
+		return nil, d.ArgErr()
+	}
+	pos := Positional(d.Val())
+
 	newOrder := directiveOrder
 
-	for d.Next() {
+	// if directive exists, first remove it
-		// get directive name
+	for i, d := range newOrder {
-		if !d.Next() {
+		if d == dirName {
-			return nil, d.ArgErr()
+			newOrder = append(newOrder[:i], newOrder[i+1:]...)
-		}
+			break
-		dirName := d.Val()
-		if _, ok := registeredDirectives[dirName]; !ok {
-			return nil, d.Errf("%s is not a registered directive", dirName)
 		}
+	}
 
-		// get positional token
+	// act on the positional
-		if !d.Next() {
+	switch pos {
-			return nil, d.ArgErr()
+	case First:
-		}
+		newOrder = append([]string{dirName}, newOrder...)
-		pos := d.Val()
-
-		// if directive exists, first remove it
-		for i, d := range newOrder {
-			if d == dirName {
-				newOrder = append(newOrder[:i], newOrder[i+1:]...)
-				break
-			}
-		}
-
-		// act on the positional
-		switch pos {
-		case "first":
-			newOrder = append([]string{dirName}, newOrder...)
-			if d.NextArg() {
-				return nil, d.ArgErr()
-			}
-			directiveOrder = newOrder
-			return newOrder, nil
-		case "last":
-			newOrder = append(newOrder, dirName)
-			if d.NextArg() {
-				return nil, d.ArgErr()
-			}
-			directiveOrder = newOrder
-			return newOrder, nil
-		case "before":
-		case "after":
-		default:
-			return nil, d.Errf("unknown positional '%s'", pos)
-		}
-
-		// get name of other directive
-		if !d.NextArg() {
-			return nil, d.ArgErr()
-		}
-		otherDir := d.Val()
 		if d.NextArg() {
 			return nil, d.ArgErr()
 		}
+		directiveOrder = newOrder
+		return newOrder, nil
+	case Last:
+		newOrder = append(newOrder, dirName)
+		if d.NextArg() {
+			return nil, d.ArgErr()
+		}
+		directiveOrder = newOrder
+		return newOrder, nil
+	case Before:
+	case After:
+	default:
+		return nil, d.Errf("unknown positional '%s'", pos)
+	}
 
-		// insert directive into proper position
+	// get name of other directive
-		for i, d := range newOrder {
+	if !d.NextArg() {
-			if d == otherDir {
+		return nil, d.ArgErr()
-				if pos == "before" {
+	}
-					newOrder = append(newOrder[:i], append([]string{dirName}, newOrder[i:]...)...)
+	otherDir := d.Val()
-				} else if pos == "after" {
+	if d.NextArg() {
-					newOrder = append(newOrder[:i+1], append([]string{dirName}, newOrder[i+1:]...)...)
+		return nil, d.ArgErr()
-				}
+	}
-				break
+
+	// insert directive into proper position
+	for i, d := range newOrder {
+		if d == otherDir {
+			if pos == Before {
+				newOrder = append(newOrder[:i], append([]string{dirName}, newOrder[i:]...)...)
+			} else if pos == After {
+				newOrder = append(newOrder[:i+1], append([]string{dirName}, newOrder[i+1:]...)...)
 			}
+			break
 		}
 	}
 
@@ -213,66 +213,67 @@ func parseOptACMEDNS(d *caddyfile.Dispenser, _ any) (any, error) {
 	if err != nil {
 		return nil, err
 	}
-	prov, ok := unm.(certmagic.ACMEDNSProvider)
+	prov, ok := unm.(certmagic.DNSProvider)
 	if !ok {
-		return nil, d.Errf("module %s (%T) is not a certmagic.ACMEDNSProvider", modID, unm)
+		return nil, d.Errf("module %s (%T) is not a certmagic.DNSProvider", modID, unm)
 	}
 	return prov, nil
 }
 
 func parseOptACMEEAB(d *caddyfile.Dispenser, _ any) (any, error) {
 	eab := new(acme.EAB)
-	for d.Next() {
+	d.Next() // consume option name
-		if d.NextArg() {
+	if d.NextArg() {
-			return nil, d.ArgErr()
+		return nil, d.ArgErr()
-		}
+	}
-		for nesting := d.Nesting(); d.NextBlock(nesting); {
+	for d.NextBlock(0) {
-			switch d.Val() {
+		switch d.Val() {
-			case "key_id":
+		case "key_id":
-				if !d.NextArg() {
+			if !d.NextArg() {
-					return nil, d.ArgErr()
+				return nil, d.ArgErr()
-				}
-				eab.KeyID = d.Val()
-
-			case "mac_key":
-				if !d.NextArg() {
-					return nil, d.ArgErr()
-				}
-				eab.MACKey = d.Val()
-
-			default:
-				return nil, d.Errf("unrecognized parameter '%s'", d.Val())
 			}
+			eab.KeyID = d.Val()
+
+		case "mac_key":
+			if !d.NextArg() {
+				return nil, d.ArgErr()
+			}
+			eab.MACKey = d.Val()
+
+		default:
+			return nil, d.Errf("unrecognized parameter '%s'", d.Val())
 		}
 	}
 	return eab, nil
 }
 
 func parseOptCertIssuer(d *caddyfile.Dispenser, existing any) (any, error) {
+	d.Next() // consume option name
+
 	var issuers []certmagic.Issuer
 	if existing != nil {
 		issuers = existing.([]certmagic.Issuer)
 	}
-	for d.Next() { // consume option name
+
-		if !d.Next() { // get issuer module name
+	// get issuer module name
-			return nil, d.ArgErr()
+	if !d.Next() {
-		}
+		return nil, d.ArgErr()
-		modID := "tls.issuance." + d.Val()
-		unm, err := caddyfile.UnmarshalModule(d, modID)
-		if err != nil {
-			return nil, err
-		}
-		iss, ok := unm.(certmagic.Issuer)
-		if !ok {
-			return nil, d.Errf("module %s (%T) is not a certmagic.Issuer", modID, unm)
-		}
-		issuers = append(issuers, iss)
 	}
+	modID := "tls.issuance." + d.Val()
+	unm, err := caddyfile.UnmarshalModule(d, modID)
+	if err != nil {
+		return nil, err
+	}
+	iss, ok := unm.(certmagic.Issuer)
+	if !ok {
+		return nil, d.Errf("module %s (%T) is not a certmagic.Issuer", modID, unm)
+	}
+	issuers = append(issuers, iss)
 	return issuers, nil
 }
 
 func parseOptSingleString(d *caddyfile.Dispenser, _ any) (any, error) {
-	d.Next() // consume parameter name
+	d.Next() // consume option name
 	if !d.Next() {
 		return "", d.ArgErr()
 	}
@@ -284,7 +285,7 @@ func parseOptSingleString(d *caddyfile.Dispenser, _ any) (any, error) {
 }
 
 func parseOptStringList(d *caddyfile.Dispenser, _ any) (any, error) {
-	d.Next() // consume parameter name
+	d.Next() // consume option name
 	val := d.RemainingArgs()
 	if len(val) == 0 {
 		return "", d.ArgErr()
@@ -293,33 +294,33 @@ func parseOptStringList(d *caddyfile.Dispenser, _ any) (any, error) {
 }
 
 func parseOptAdmin(d *caddyfile.Dispenser, _ any) (any, error) {
+	d.Next() // consume option name
+
 	adminCfg := new(caddy.AdminConfig)
-	for d.Next() {
+	if d.NextArg() {
-		if d.NextArg() {
+		listenAddress := d.Val()
-			listenAddress := d.Val()
+		if listenAddress == "off" {
-			if listenAddress == "off" {
+			adminCfg.Disabled = true
-				adminCfg.Disabled = true
+			if d.Next() { // Do not accept any remaining options including block
-				if d.Next() { // Do not accept any remaining options including block
+				return nil, d.Err("No more option is allowed after turning off admin config")
-					return nil, d.Err("No more option is allowed after turning off admin config")
+			}
-				}
+		} else {
-			} else {
+			adminCfg.Listen = listenAddress
-				adminCfg.Listen = listenAddress
+			if d.NextArg() { // At most 1 arg is allowed
-				if d.NextArg() { // At most 1 arg is allowed
+				return nil, d.ArgErr()
-					return nil, d.ArgErr()
-				}
 			}
 		}
-		for nesting := d.Nesting(); d.NextBlock(nesting); {
+	}
-			switch d.Val() {
+	for d.NextBlock(0) {
-			case "enforce_origin":
+		switch d.Val() {
-				adminCfg.EnforceOrigin = true
+		case "enforce_origin":
+			adminCfg.EnforceOrigin = true
 
-			case "origins":
+		case "origins":
-				adminCfg.Origins = d.RemainingArgs()
+			adminCfg.Origins = d.RemainingArgs()
 
-			default:
+		default:
-				return nil, d.Errf("unrecognized parameter '%s'", d.Val())
+			return nil, d.Errf("unrecognized parameter '%s'", d.Val())
-			}
 		}
 	}
 	if adminCfg.Listen == "" && !adminCfg.Disabled {
@@ -329,57 +330,84 @@ func parseOptAdmin(d *caddyfile.Dispenser, _ any) (any, error) {
 }
 
 func parseOptOnDemand(d *caddyfile.Dispenser, _ any) (any, error) {
+	d.Next() // consume option name
+	if d.NextArg() {
+		return nil, d.ArgErr()
+	}
+
 	var ond *caddytls.OnDemandConfig
-	for d.Next() {
-		if d.NextArg() {
-			return nil, d.ArgErr()
-		}
-		for nesting := d.Nesting(); d.NextBlock(nesting); {
-			switch d.Val() {
-			case "ask":
-				if !d.NextArg() {
-					return nil, d.ArgErr()
-				}
-				if ond == nil {
-					ond = new(caddytls.OnDemandConfig)
-				}
-				ond.Ask = d.Val()
 
-			case "interval":
+	for nesting := d.Nesting(); d.NextBlock(nesting); {
-				if !d.NextArg() {
+		switch d.Val() {
-					return nil, d.ArgErr()
+		case "ask":
-				}
+			if !d.NextArg() {
-				dur, err := caddy.ParseDuration(d.Val())
+				return nil, d.ArgErr()
-				if err != nil {
-					return nil, err
-				}
-				if ond == nil {
-					ond = new(caddytls.OnDemandConfig)
-				}
-				if ond.RateLimit == nil {
-					ond.RateLimit = new(caddytls.RateLimit)
-				}
-				ond.RateLimit.Interval = caddy.Duration(dur)
-
-			case "burst":
-				if !d.NextArg() {
-					return nil, d.ArgErr()
-				}
-				burst, err := strconv.Atoi(d.Val())
-				if err != nil {
-					return nil, err
-				}
-				if ond == nil {
-					ond = new(caddytls.OnDemandConfig)
-				}
-				if ond.RateLimit == nil {
-					ond.RateLimit = new(caddytls.RateLimit)
-				}
-				ond.RateLimit.Burst = burst
-
-			default:
-				return nil, d.Errf("unrecognized parameter '%s'", d.Val())
 			}
+			if ond == nil {
+				ond = new(caddytls.OnDemandConfig)
+			}
+			if ond.PermissionRaw != nil {
+				return nil, d.Err("on-demand TLS permission module (or 'ask') already specified")
+			}
+			perm := caddytls.PermissionByHTTP{Endpoint: d.Val()}
+			ond.PermissionRaw = caddyconfig.JSONModuleObject(perm, "module", "http", nil)
+
+		case "permission":
+			if !d.NextArg() {
+				return nil, d.ArgErr()
+			}
+			if ond == nil {
+				ond = new(caddytls.OnDemandConfig)
+			}
+			if ond.PermissionRaw != nil {
+				return nil, d.Err("on-demand TLS permission module (or 'ask') already specified")
+			}
+			modName := d.Val()
+			modID := "tls.permission." + modName
+			unm, err := caddyfile.UnmarshalModule(d, modID)
+			if err != nil {
+				return nil, err
+			}
+			perm, ok := unm.(caddytls.OnDemandPermission)
+			if !ok {
+				return nil, d.Errf("module %s (%T) is not an on-demand TLS permission module", modID, unm)
+			}
+			ond.PermissionRaw = caddyconfig.JSONModuleObject(perm, "module", modName, nil)
+
+		case "interval":
+			if !d.NextArg() {
+				return nil, d.ArgErr()
+			}
+			dur, err := caddy.ParseDuration(d.Val())
+			if err != nil {
+				return nil, err
+			}
+			if ond == nil {
+				ond = new(caddytls.OnDemandConfig)
+			}
+			if ond.RateLimit == nil {
+				ond.RateLimit = new(caddytls.RateLimit)
+			}
+			ond.RateLimit.Interval = caddy.Duration(dur)
+
+		case "burst":
+			if !d.NextArg() {
+				return nil, d.ArgErr()
+			}
+			burst, err := strconv.Atoi(d.Val())
+			if err != nil {
+				return nil, err
+			}
+			if ond == nil {
+				ond = new(caddytls.OnDemandConfig)
+			}
+			if ond.RateLimit == nil {
+				ond.RateLimit = new(caddytls.RateLimit)
+			}
+			ond.RateLimit.Burst = burst
+
+		default:
+			return nil, d.Errf("unrecognized parameter '%s'", d.Val())
 		}
 	}
 	if ond == nil {
@@ -389,7 +417,7 @@ func parseOptOnDemand(d *caddyfile.Dispenser, _ any) (any, error) {
 }
 
 func parseOptPersistConfig(d *caddyfile.Dispenser, _ any) (any, error) {
-	d.Next() // consume parameter name
+	d.Next() // consume option name
 	if !d.Next() {
 		return "", d.ArgErr()
 	}
@@ -404,7 +432,7 @@ func parseOptPersistConfig(d *caddyfile.Dispenser, _ any) (any, error) {
 }
 
 func parseOptAutoHTTPS(d *caddyfile.Dispenser, _ any) (any, error) {
-	d.Next() // consume parameter name
+	d.Next() // consume option name
 	if !d.Next() {
 		return "", d.ArgErr()
 	}

caddyconfig/httpcaddyfile/pkiapp.go

@@ -48,124 +48,124 @@ func init() {
 //
 // When the CA ID is unspecified, 'local' is assumed.
 func parsePKIApp(d *caddyfile.Dispenser, existingVal any) (any, error) {
-	pki := &caddypki.PKI{CAs: make(map[string]*caddypki.CA)}
+	d.Next() // consume app name
 
-	for d.Next() {
+	pki := &caddypki.PKI{
-		for nesting := d.Nesting(); d.NextBlock(nesting); {
+		CAs: make(map[string]*caddypki.CA),
-			switch d.Val() {
+	}
-			case "ca":
+	for d.NextBlock(0) {
-				pkiCa := new(caddypki.CA)
+		switch d.Val() {
+		case "ca":
+			pkiCa := new(caddypki.CA)
+			if d.NextArg() {
+				pkiCa.ID = d.Val()
 				if d.NextArg() {
-					pkiCa.ID = d.Val()
+					return nil, d.ArgErr()
-					if d.NextArg() {
+				}
+			}
+			if pkiCa.ID == "" {
+				pkiCa.ID = caddypki.DefaultCAID
+			}
+
+			for nesting := d.Nesting(); d.NextBlock(nesting); {
+				switch d.Val() {
+				case "name":
+					if !d.NextArg() {
 						return nil, d.ArgErr()
 					}
-				}
+					pkiCa.Name = d.Val()
-				if pkiCa.ID == "" {
-					pkiCa.ID = caddypki.DefaultCAID
-				}
 
-				for nesting := d.Nesting(); d.NextBlock(nesting); {
+				case "root_cn":
-					switch d.Val() {
+					if !d.NextArg() {
-					case "name":
+						return nil, d.ArgErr()
-						if !d.NextArg() {
-							return nil, d.ArgErr()
-						}
-						pkiCa.Name = d.Val()
-
-					case "root_cn":
-						if !d.NextArg() {
-							return nil, d.ArgErr()
-						}
-						pkiCa.RootCommonName = d.Val()
-
-					case "intermediate_cn":
-						if !d.NextArg() {
-							return nil, d.ArgErr()
-						}
-						pkiCa.IntermediateCommonName = d.Val()
-
-					case "intermediate_lifetime":
-						if !d.NextArg() {
-							return nil, d.ArgErr()
-						}
-						dur, err := caddy.ParseDuration(d.Val())
-						if err != nil {
-							return nil, err
-						}
-						pkiCa.IntermediateLifetime = caddy.Duration(dur)
-
-					case "root":
-						if pkiCa.Root == nil {
-							pkiCa.Root = new(caddypki.KeyPair)
-						}
-						for nesting := d.Nesting(); d.NextBlock(nesting); {
-							switch d.Val() {
-							case "cert":
-								if !d.NextArg() {
-									return nil, d.ArgErr()
-								}
-								pkiCa.Root.Certificate = d.Val()
-
-							case "key":
-								if !d.NextArg() {
-									return nil, d.ArgErr()
-								}
-								pkiCa.Root.PrivateKey = d.Val()
-
-							case "format":
-								if !d.NextArg() {
-									return nil, d.ArgErr()
-								}
-								pkiCa.Root.Format = d.Val()
-
-							default:
-								return nil, d.Errf("unrecognized pki ca root option '%s'", d.Val())
-							}
-						}
-
-					case "intermediate":
-						if pkiCa.Intermediate == nil {
-							pkiCa.Intermediate = new(caddypki.KeyPair)
-						}
-						for nesting := d.Nesting(); d.NextBlock(nesting); {
-							switch d.Val() {
-							case "cert":
-								if !d.NextArg() {
-									return nil, d.ArgErr()
-								}
-								pkiCa.Intermediate.Certificate = d.Val()
-
-							case "key":
-								if !d.NextArg() {
-									return nil, d.ArgErr()
-								}
-								pkiCa.Intermediate.PrivateKey = d.Val()
-
-							case "format":
-								if !d.NextArg() {
-									return nil, d.ArgErr()
-								}
-								pkiCa.Intermediate.Format = d.Val()
-
-							default:
-								return nil, d.Errf("unrecognized pki ca intermediate option '%s'", d.Val())
-							}
-						}
-
-					default:
-						return nil, d.Errf("unrecognized pki ca option '%s'", d.Val())
 					}
+					pkiCa.RootCommonName = d.Val()
+
+				case "intermediate_cn":
+					if !d.NextArg() {
+						return nil, d.ArgErr()
+					}
+					pkiCa.IntermediateCommonName = d.Val()
+
+				case "intermediate_lifetime":
+					if !d.NextArg() {
+						return nil, d.ArgErr()
+					}
+					dur, err := caddy.ParseDuration(d.Val())
+					if err != nil {
+						return nil, err
+					}
+					pkiCa.IntermediateLifetime = caddy.Duration(dur)
+
+				case "root":
+					if pkiCa.Root == nil {
+						pkiCa.Root = new(caddypki.KeyPair)
+					}
+					for nesting := d.Nesting(); d.NextBlock(nesting); {
+						switch d.Val() {
+						case "cert":
+							if !d.NextArg() {
+								return nil, d.ArgErr()
+							}
+							pkiCa.Root.Certificate = d.Val()
+
+						case "key":
+							if !d.NextArg() {
+								return nil, d.ArgErr()
+							}
+							pkiCa.Root.PrivateKey = d.Val()
+
+						case "format":
+							if !d.NextArg() {
+								return nil, d.ArgErr()
+							}
+							pkiCa.Root.Format = d.Val()
+
+						default:
+							return nil, d.Errf("unrecognized pki ca root option '%s'", d.Val())
+						}
+					}
+
+				case "intermediate":
+					if pkiCa.Intermediate == nil {
+						pkiCa.Intermediate = new(caddypki.KeyPair)
+					}
+					for nesting := d.Nesting(); d.NextBlock(nesting); {
+						switch d.Val() {
+						case "cert":
+							if !d.NextArg() {
+								return nil, d.ArgErr()
+							}
+							pkiCa.Intermediate.Certificate = d.Val()
+
+						case "key":
+							if !d.NextArg() {
+								return nil, d.ArgErr()
+							}
+							pkiCa.Intermediate.PrivateKey = d.Val()
+
+						case "format":
+							if !d.NextArg() {
+								return nil, d.ArgErr()
+							}
+							pkiCa.Intermediate.Format = d.Val()
+
+						default:
+							return nil, d.Errf("unrecognized pki ca intermediate option '%s'", d.Val())
+						}
+					}
+
+				default:
+					return nil, d.Errf("unrecognized pki ca option '%s'", d.Val())
 				}
-
-				pki.CAs[pkiCa.ID] = pkiCa
-
-			default:
-				return nil, d.Errf("unrecognized pki option '%s'", d.Val())
 			}
+
+			pki.CAs[pkiCa.ID] = pkiCa
+
+		default:
+			return nil, d.Errf("unrecognized pki option '%s'", d.Val())
 		}
 	}
-
 	return pki, nil
 }
 
@@ -174,7 +174,6 @@ func (st ServerType) buildPKIApp(
 	options map[string]any,
 	warnings []caddyconfig.Warning,
 ) (*caddypki.PKI, []caddyconfig.Warning, error) {
-
 	skipInstallTrust := false
 	if _, ok := options["skip_install_trust"]; ok {
 		skipInstallTrust = true

caddyconfig/httpcaddyfile/serveroptions.go

@@ -18,11 +18,12 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/dustin/go-humanize"
+
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
-	"github.com/dustin/go-humanize"
 )
 
 // serverOptions collects server config overrides parsed from Caddyfile global options
@@ -41,232 +42,219 @@ type serverOptions struct {
 	IdleTimeout          caddy.Duration
 	KeepAliveInterval    caddy.Duration
 	MaxHeaderBytes       int
+	EnableFullDuplex     bool
 	Protocols            []string
 	StrictSNIHost        *bool
 	TrustedProxiesRaw    json.RawMessage
+	TrustedProxiesStrict int
 	ClientIPHeaders      []string
 	ShouldLogCredentials bool
 	Metrics              *caddyhttp.Metrics
+	Trace                bool // TODO: EXPERIMENTAL
 }
 
 func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (any, error) {
+	d.Next() // consume option name
+
 	serverOpts := serverOptions{}
-	for d.Next() {
+	if d.NextArg() {
+		serverOpts.ListenerAddress = d.Val()
 		if d.NextArg() {
-			serverOpts.ListenerAddress = d.Val()
+			return nil, d.ArgErr()
-			if d.NextArg() {
+		}
+	}
+	for d.NextBlock(0) {
+		switch d.Val() {
+		case "name":
+			if serverOpts.ListenerAddress == "" {
+				return nil, d.Errf("cannot set a name for a server without a listener address")
+			}
+			if !d.NextArg() {
 				return nil, d.ArgErr()
 			}
-		}
+			serverOpts.Name = d.Val()
-		for nesting := d.Nesting(); d.NextBlock(nesting); {
-			switch d.Val() {
-			case "name":
-				if serverOpts.ListenerAddress == "" {
-					return nil, d.Errf("cannot set a name for a server without a listener address")
-				}
-				if !d.NextArg() {
-					return nil, d.ArgErr()
-				}
-				serverOpts.Name = d.Val()
 
-			case "listener_wrappers":
+		case "listener_wrappers":
-				for nesting := d.Nesting(); d.NextBlock(nesting); {
+			for nesting := d.Nesting(); d.NextBlock(nesting); {
-					modID := "caddy.listeners." + d.Val()
+				modID := "caddy.listeners." + d.Val()
-					unm, err := caddyfile.UnmarshalModule(d, modID)
-					if err != nil {
-						return nil, err
-					}
-					listenerWrapper, ok := unm.(caddy.ListenerWrapper)
-					if !ok {
-						return nil, fmt.Errorf("module %s (%T) is not a listener wrapper", modID, unm)
-					}
-					jsonListenerWrapper := caddyconfig.JSONModuleObject(
-						listenerWrapper,
-						"wrapper",
-						listenerWrapper.(caddy.Module).CaddyModule().ID.Name(),
-						nil,
-					)
-					serverOpts.ListenerWrappersRaw = append(serverOpts.ListenerWrappersRaw, jsonListenerWrapper)
-				}
-
-			case "timeouts":
-				for nesting := d.Nesting(); d.NextBlock(nesting); {
-					switch d.Val() {
-					case "read_body":
-						if !d.NextArg() {
-							return nil, d.ArgErr()
-						}
-						dur, err := caddy.ParseDuration(d.Val())
-						if err != nil {
-							return nil, d.Errf("parsing read_body timeout duration: %v", err)
-						}
-						serverOpts.ReadTimeout = caddy.Duration(dur)
-
-					case "read_header":
-						if !d.NextArg() {
-							return nil, d.ArgErr()
-						}
-						dur, err := caddy.ParseDuration(d.Val())
-						if err != nil {
-							return nil, d.Errf("parsing read_header timeout duration: %v", err)
-						}
-						serverOpts.ReadHeaderTimeout = caddy.Duration(dur)
-
-					case "write":
-						if !d.NextArg() {
-							return nil, d.ArgErr()
-						}
-						dur, err := caddy.ParseDuration(d.Val())
-						if err != nil {
-							return nil, d.Errf("parsing write timeout duration: %v", err)
-						}
-						serverOpts.WriteTimeout = caddy.Duration(dur)
-
-					case "idle":
-						if !d.NextArg() {
-							return nil, d.ArgErr()
-						}
-						dur, err := caddy.ParseDuration(d.Val())
-						if err != nil {
-							return nil, d.Errf("parsing idle timeout duration: %v", err)
-						}
-						serverOpts.IdleTimeout = caddy.Duration(dur)
-
-					default:
-						return nil, d.Errf("unrecognized timeouts option '%s'", d.Val())
-					}
-				}
-			case "keepalive_interval":
-				if !d.NextArg() {
-					return nil, d.ArgErr()
-				}
-				dur, err := caddy.ParseDuration(d.Val())
-				if err != nil {
-					return nil, d.Errf("parsing keepalive interval duration: %v", err)
-				}
-				serverOpts.KeepAliveInterval = caddy.Duration(dur)
-
-			case "max_header_size":
-				var sizeStr string
-				if !d.AllArgs(&sizeStr) {
-					return nil, d.ArgErr()
-				}
-				size, err := humanize.ParseBytes(sizeStr)
-				if err != nil {
-					return nil, d.Errf("parsing max_header_size: %v", err)
-				}
-				serverOpts.MaxHeaderBytes = int(size)
-
-			case "log_credentials":
-				if d.NextArg() {
-					return nil, d.ArgErr()
-				}
-				serverOpts.ShouldLogCredentials = true
-
-			case "protocols":
-				protos := d.RemainingArgs()
-				for _, proto := range protos {
-					if proto != "h1" && proto != "h2" && proto != "h2c" && proto != "h3" {
-						return nil, d.Errf("unknown protocol '%s': expected h1, h2, h2c, or h3", proto)
-					}
-					if sliceContains(serverOpts.Protocols, proto) {
-						return nil, d.Errf("protocol %s specified more than once", proto)
-					}
-					serverOpts.Protocols = append(serverOpts.Protocols, proto)
-				}
-				if nesting := d.Nesting(); d.NextBlock(nesting) {
-					return nil, d.ArgErr()
-				}
-
-			case "strict_sni_host":
-				if d.NextArg() && d.Val() != "insecure_off" && d.Val() != "on" {
-					return nil, d.Errf("strict_sni_host only supports 'on' or 'insecure_off', got '%s'", d.Val())
-				}
-				boolVal := true
-				if d.Val() == "insecure_off" {
-					boolVal = false
-				}
-				serverOpts.StrictSNIHost = &boolVal
-
-			case "trusted_proxies":
-				if !d.NextArg() {
-					return nil, d.Err("trusted_proxies expects an IP range source module name as its first argument")
-				}
-				modID := "http.ip_sources." + d.Val()
 				unm, err := caddyfile.UnmarshalModule(d, modID)
 				if err != nil {
 					return nil, err
 				}
-				source, ok := unm.(caddyhttp.IPRangeSource)
+				listenerWrapper, ok := unm.(caddy.ListenerWrapper)
 				if !ok {
-					return nil, fmt.Errorf("module %s (%T) is not an IP range source", modID, unm)
+					return nil, fmt.Errorf("module %s (%T) is not a listener wrapper", modID, unm)
 				}
-				jsonSource := caddyconfig.JSONModuleObject(
+				jsonListenerWrapper := caddyconfig.JSONModuleObject(
-					source,
+					listenerWrapper,
-					"source",
+					"wrapper",
-					source.(caddy.Module).CaddyModule().ID.Name(),
+					listenerWrapper.(caddy.Module).CaddyModule().ID.Name(),
 					nil,
 				)
-				serverOpts.TrustedProxiesRaw = jsonSource
+				serverOpts.ListenerWrappersRaw = append(serverOpts.ListenerWrappersRaw, jsonListenerWrapper)
-
-			case "client_ip_headers":
-				headers := d.RemainingArgs()
-				for _, header := range headers {
-					if sliceContains(serverOpts.ClientIPHeaders, header) {
-						return nil, d.Errf("client IP header %s specified more than once", header)
-					}
-					serverOpts.ClientIPHeaders = append(serverOpts.ClientIPHeaders, header)
-				}
-				if nesting := d.Nesting(); d.NextBlock(nesting) {
-					return nil, d.ArgErr()
-				}
-
-			case "metrics":
-				if d.NextArg() {
-					return nil, d.ArgErr()
-				}
-				if nesting := d.Nesting(); d.NextBlock(nesting) {
-					return nil, d.ArgErr()
-				}
-				serverOpts.Metrics = new(caddyhttp.Metrics)
-
-			// TODO: DEPRECATED. (August 2022)
-			case "protocol":
-				caddy.Log().Named("caddyfile").Warn("DEPRECATED: protocol sub-option will be removed soon")
-
-				for nesting := d.Nesting(); d.NextBlock(nesting); {
-					switch d.Val() {
-					case "allow_h2c":
-						caddy.Log().Named("caddyfile").Warn("DEPRECATED: allow_h2c will be removed soon; use protocols option instead")
-
-						if d.NextArg() {
-							return nil, d.ArgErr()
-						}
-						if sliceContains(serverOpts.Protocols, "h2c") {
-							return nil, d.Errf("protocol h2c already specified")
-						}
-						serverOpts.Protocols = append(serverOpts.Protocols, "h2c")
-
-					case "strict_sni_host":
-						caddy.Log().Named("caddyfile").Warn("DEPRECATED: protocol > strict_sni_host in this position will be removed soon; move up to the servers block instead")
-
-						if d.NextArg() && d.Val() != "insecure_off" && d.Val() != "on" {
-							return nil, d.Errf("strict_sni_host only supports 'on' or 'insecure_off', got '%s'", d.Val())
-						}
-						boolVal := true
-						if d.Val() == "insecure_off" {
-							boolVal = false
-						}
-						serverOpts.StrictSNIHost = &boolVal
-
-					default:
-						return nil, d.Errf("unrecognized protocol option '%s'", d.Val())
-					}
-				}
-
-			default:
-				return nil, d.Errf("unrecognized servers option '%s'", d.Val())
 			}
+
+		case "timeouts":
+			for nesting := d.Nesting(); d.NextBlock(nesting); {
+				switch d.Val() {
+				case "read_body":
+					if !d.NextArg() {
+						return nil, d.ArgErr()
+					}
+					dur, err := caddy.ParseDuration(d.Val())
+					if err != nil {
+						return nil, d.Errf("parsing read_body timeout duration: %v", err)
+					}
+					serverOpts.ReadTimeout = caddy.Duration(dur)
+
+				case "read_header":
+					if !d.NextArg() {
+						return nil, d.ArgErr()
+					}
+					dur, err := caddy.ParseDuration(d.Val())
+					if err != nil {
+						return nil, d.Errf("parsing read_header timeout duration: %v", err)
+					}
+					serverOpts.ReadHeaderTimeout = caddy.Duration(dur)
+
+				case "write":
+					if !d.NextArg() {
+						return nil, d.ArgErr()
+					}
+					dur, err := caddy.ParseDuration(d.Val())
+					if err != nil {
+						return nil, d.Errf("parsing write timeout duration: %v", err)
+					}
+					serverOpts.WriteTimeout = caddy.Duration(dur)
+
+				case "idle":
+					if !d.NextArg() {
+						return nil, d.ArgErr()
+					}
+					dur, err := caddy.ParseDuration(d.Val())
+					if err != nil {
+						return nil, d.Errf("parsing idle timeout duration: %v", err)
+					}
+					serverOpts.IdleTimeout = caddy.Duration(dur)
+
+				default:
+					return nil, d.Errf("unrecognized timeouts option '%s'", d.Val())
+				}
+			}
+		case "keepalive_interval":
+			if !d.NextArg() {
+				return nil, d.ArgErr()
+			}
+			dur, err := caddy.ParseDuration(d.Val())
+			if err != nil {
+				return nil, d.Errf("parsing keepalive interval duration: %v", err)
+			}
+			serverOpts.KeepAliveInterval = caddy.Duration(dur)
+
+		case "max_header_size":
+			var sizeStr string
+			if !d.AllArgs(&sizeStr) {
+				return nil, d.ArgErr()
+			}
+			size, err := humanize.ParseBytes(sizeStr)
+			if err != nil {
+				return nil, d.Errf("parsing max_header_size: %v", err)
+			}
+			serverOpts.MaxHeaderBytes = int(size)
+
+		case "enable_full_duplex":
+			if d.NextArg() {
+				return nil, d.ArgErr()
+			}
+			serverOpts.EnableFullDuplex = true
+
+		case "log_credentials":
+			if d.NextArg() {
+				return nil, d.ArgErr()
+			}
+			serverOpts.ShouldLogCredentials = true
+
+		case "protocols":
+			protos := d.RemainingArgs()
+			for _, proto := range protos {
+				if proto != "h1" && proto != "h2" && proto != "h2c" && proto != "h3" {
+					return nil, d.Errf("unknown protocol '%s': expected h1, h2, h2c, or h3", proto)
+				}
+				if sliceContains(serverOpts.Protocols, proto) {
+					return nil, d.Errf("protocol %s specified more than once", proto)
+				}
+				serverOpts.Protocols = append(serverOpts.Protocols, proto)
+			}
+			if nesting := d.Nesting(); d.NextBlock(nesting) {
+				return nil, d.ArgErr()
+			}
+
+		case "strict_sni_host":
+			if d.NextArg() && d.Val() != "insecure_off" && d.Val() != "on" {
+				return nil, d.Errf("strict_sni_host only supports 'on' or 'insecure_off', got '%s'", d.Val())
+			}
+			boolVal := true
+			if d.Val() == "insecure_off" {
+				boolVal = false
+			}
+			serverOpts.StrictSNIHost = &boolVal
+
+		case "trusted_proxies":
+			if !d.NextArg() {
+				return nil, d.Err("trusted_proxies expects an IP range source module name as its first argument")
+			}
+			modID := "http.ip_sources." + d.Val()
+			unm, err := caddyfile.UnmarshalModule(d, modID)
+			if err != nil {
+				return nil, err
+			}
+			source, ok := unm.(caddyhttp.IPRangeSource)
+			if !ok {
+				return nil, fmt.Errorf("module %s (%T) is not an IP range source", modID, unm)
+			}
+			jsonSource := caddyconfig.JSONModuleObject(
+				source,
+				"source",
+				source.(caddy.Module).CaddyModule().ID.Name(),
+				nil,
+			)
+			serverOpts.TrustedProxiesRaw = jsonSource
+
+		case "trusted_proxies_strict":
+			if d.NextArg() {
+				return nil, d.ArgErr()
+			}
+			serverOpts.TrustedProxiesStrict = 1
+
+		case "client_ip_headers":
+			headers := d.RemainingArgs()
+			for _, header := range headers {
+				if sliceContains(serverOpts.ClientIPHeaders, header) {
+					return nil, d.Errf("client IP header %s specified more than once", header)
+				}
+				serverOpts.ClientIPHeaders = append(serverOpts.ClientIPHeaders, header)
+			}
+			if nesting := d.Nesting(); d.NextBlock(nesting) {
+				return nil, d.ArgErr()
+			}
+
+		case "metrics":
+			if d.NextArg() {
+				return nil, d.ArgErr()
+			}
+			if nesting := d.Nesting(); d.NextBlock(nesting) {
+				return nil, d.ArgErr()
+			}
+			serverOpts.Metrics = new(caddyhttp.Metrics)
+
+		case "trace":
+			if d.NextArg() {
+				return nil, d.ArgErr()
+			}
+			serverOpts.Trace = true
+
+		default:
+			return nil, d.Errf("unrecognized servers option '%s'", d.Val())
 		}
 	}
 	return serverOpts, nil
@@ -276,7 +264,7 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (any, error) {
 func applyServerOptions(
 	servers map[string]*caddyhttp.Server,
 	options map[string]any,
-	warnings *[]caddyconfig.Warning,
+	_ *[]caddyconfig.Warning,
 ) error {
 	serverOpts, ok := options["servers"].([]serverOptions)
 	if !ok {
@@ -327,17 +315,26 @@ func applyServerOptions(
 		server.IdleTimeout = opts.IdleTimeout
 		server.KeepAliveInterval = opts.KeepAliveInterval
 		server.MaxHeaderBytes = opts.MaxHeaderBytes
+		server.EnableFullDuplex = opts.EnableFullDuplex
 		server.Protocols = opts.Protocols
 		server.StrictSNIHost = opts.StrictSNIHost
 		server.TrustedProxiesRaw = opts.TrustedProxiesRaw
 		server.ClientIPHeaders = opts.ClientIPHeaders
+		server.TrustedProxiesStrict = opts.TrustedProxiesStrict
 		server.Metrics = opts.Metrics
 		if opts.ShouldLogCredentials {
 			if server.Logs == nil {
-				server.Logs = &caddyhttp.ServerLogConfig{}
+				server.Logs = new(caddyhttp.ServerLogConfig)
 			}
 			server.Logs.ShouldLogCredentials = opts.ShouldLogCredentials
 		}
+		if opts.Trace {
+			// TODO: THIS IS EXPERIMENTAL (MAY 2024)
+			if server.Logs == nil {
+				server.Logs = new(caddyhttp.ServerLogConfig)
+			}
+			server.Logs.Trace = opts.Trace
+		}
 
 		if opts.Name != "" {
 			nameReplacements[key] = opts.Name

caddyconfig/httpcaddyfile/shorthands.go

@@ -0,0 +1,94 @@
+package httpcaddyfile
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+)
+
+type ComplexShorthandReplacer struct {
+	search  *regexp.Regexp
+	replace string
+}
+
+type ShorthandReplacer struct {
+	complex []ComplexShorthandReplacer
+	simple  *strings.Replacer
+}
+
+func NewShorthandReplacer() ShorthandReplacer {
+	// replace shorthand placeholders (which are convenient
+	// when writing a Caddyfile) with their actual placeholder
+	// identifiers or variable names
+	replacer := strings.NewReplacer(placeholderShorthands()...)
+
+	// these are placeholders that allow a user-defined final
+	// parameters, but we still want to provide a shorthand
+	// for those, so we use a regexp to replace
+	regexpReplacements := []ComplexShorthandReplacer{
+		{regexp.MustCompile(`{header\.([\w-]*)}`), "{http.request.header.$1}"},
+		{regexp.MustCompile(`{cookie\.([\w-]*)}`), "{http.request.cookie.$1}"},
+		{regexp.MustCompile(`{labels\.([\w-]*)}`), "{http.request.host.labels.$1}"},
+		{regexp.MustCompile(`{path\.([\w-]*)}`), "{http.request.uri.path.$1}"},
+		{regexp.MustCompile(`{file\.([\w-]*)}`), "{http.request.uri.path.file.$1}"},
+		{regexp.MustCompile(`{query\.([\w-]*)}`), "{http.request.uri.query.$1}"},
+		{regexp.MustCompile(`{re\.([\w-\.]*)}`), "{http.regexp.$1}"},
+		{regexp.MustCompile(`{vars\.([\w-]*)}`), "{http.vars.$1}"},
+		{regexp.MustCompile(`{rp\.([\w-\.]*)}`), "{http.reverse_proxy.$1}"},
+		{regexp.MustCompile(`{resp\.([\w-\.]*)}`), "{http.intercept.$1}"},
+		{regexp.MustCompile(`{err\.([\w-\.]*)}`), "{http.error.$1}"},
+		{regexp.MustCompile(`{file_match\.([\w-]*)}`), "{http.matchers.file.$1}"},
+	}
+
+	return ShorthandReplacer{
+		complex: regexpReplacements,
+		simple:  replacer,
+	}
+}
+
+// placeholderShorthands returns a slice of old-new string pairs,
+// where the left of the pair is a placeholder shorthand that may
+// be used in the Caddyfile, and the right is the replacement.
+func placeholderShorthands() []string {
+	return []string{
+		"{dir}", "{http.request.uri.path.dir}",
+		"{file}", "{http.request.uri.path.file}",
+		"{host}", "{http.request.host}",
+		"{hostport}", "{http.request.hostport}",
+		"{port}", "{http.request.port}",
+		"{method}", "{http.request.method}",
+		"{path}", "{http.request.uri.path}",
+		"{query}", "{http.request.uri.query}",
+		"{remote}", "{http.request.remote}",
+		"{remote_host}", "{http.request.remote.host}",
+		"{remote_port}", "{http.request.remote.port}",
+		"{scheme}", "{http.request.scheme}",
+		"{uri}", "{http.request.uri}",
+		"{uuid}", "{http.request.uuid}",
+		"{tls_cipher}", "{http.request.tls.cipher_suite}",
+		"{tls_version}", "{http.request.tls.version}",
+		"{tls_client_fingerprint}", "{http.request.tls.client.fingerprint}",
+		"{tls_client_issuer}", "{http.request.tls.client.issuer}",
+		"{tls_client_serial}", "{http.request.tls.client.serial}",
+		"{tls_client_subject}", "{http.request.tls.client.subject}",
+		"{tls_client_certificate_pem}", "{http.request.tls.client.certificate_pem}",
+		"{tls_client_certificate_der_base64}", "{http.request.tls.client.certificate_der_base64}",
+		"{upstream_hostport}", "{http.reverse_proxy.upstream.hostport}",
+		"{client_ip}", "{http.vars.client_ip}",
+	}
+}
+
+// ApplyToSegment replaces shorthand placeholder to its full placeholder, understandable by Caddy.
+func (s ShorthandReplacer) ApplyToSegment(segment *caddyfile.Segment) {
+	if segment != nil {
+		for i := 0; i < len(*segment); i++ {
+			// simple string replacements
+			(*segment)[i].Text = s.simple.Replace((*segment)[i].Text)
+			// complex regexp replacements
+			for _, r := range s.complex {
+				(*segment)[i].Text = r.search.ReplaceAllString((*segment)[i].Text, r.replace)
+			}
+		}
+	}
+}

caddyconfig/httpcaddyfile/tlsapp.go

@@ -23,12 +23,13 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/caddyserver/certmagic"
+	"github.com/mholt/acmez/v2/acme"
+
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
-	"github.com/caddyserver/certmagic"
-	"github.com/mholt/acmez/acme"
 )
 
 func (st ServerType) buildTLSApp(
@@ -36,7 +37,6 @@ func (st ServerType) buildTLSApp(
 	options map[string]any,
 	warnings []caddyconfig.Warning,
 ) (*caddytls.TLS, []caddyconfig.Warning, error) {
-
 	tlsApp := &caddytls.TLS{CertificatesRaw: make(caddy.ModuleMap)}
 	var certLoaders []caddytls.CertificateLoader
 
@@ -118,6 +118,11 @@ func (st ServerType) buildTLSApp(
 				ap.OnDemand = true
 			}
 
+			// reuse private keys tls
+			if _, ok := sblock.pile["tls.reuse_private_keys"]; ok {
+				ap.ReusePrivateKeys = true
+			}
+
 			if keyTypeVals, ok := sblock.pile["tls.key_type"]; ok {
 				ap.KeyType = keyTypeVals[0].Value.(string)
 			}
@@ -219,7 +224,7 @@ func (st ServerType) buildTLSApp(
 				var internal, external []string
 				for _, s := range ap.SubjectsRaw {
 					// do not create Issuers for Tailscale domains; they will be given a Manager instead
-					if strings.HasSuffix(strings.ToLower(s), ".ts.net") {
+					if isTailscaleDomain(s) {
 						continue
 					}
 					if !certmagic.SubjectQualifiesForCert(s) {
@@ -339,7 +344,7 @@ func (st ServerType) buildTLSApp(
 	internalAP := &caddytls.AutomationPolicy{
 		IssuersRaw: []json.RawMessage{json.RawMessage(`{"module":"internal"}`)},
 	}
-	if autoHTTPS != "off" {
+	if autoHTTPS != "off" && autoHTTPS != "disable_certs" {
 		for h := range httpsHostsSharedWithHostlessKey {
 			al = append(al, h)
 			if !certmagic.SubjectQualifiesForPublicCert(h) {
@@ -373,15 +378,12 @@ func (st ServerType) buildTLSApp(
 				if len(ap.Issuers) == 0 && automationPolicyHasAllPublicNames(ap) {
 					// for public names, create default issuers which will later be filled in with configured global defaults
 					// (internal names will implicitly use the internal issuer at auto-https time)
-					ap.Issuers = caddytls.DefaultIssuers()
+					emailStr, _ := globalEmail.(string)
+					ap.Issuers = caddytls.DefaultIssuers(emailStr)
 
 					// if a specific endpoint is configured, can't use multiple default issuers
 					if globalACMECA != nil {
-						if strings.Contains(globalACMECA.(string), "zerossl") {
+						ap.Issuers = []certmagic.Issuer{new(caddytls.ACMEIssuer)}
-							ap.Issuers = []certmagic.Issuer{&caddytls.ZeroSSLIssuer{ACMEIssuer: new(caddytls.ACMEIssuer)}}
-						} else {
-							ap.Issuers = []certmagic.Issuer{new(caddytls.ACMEIssuer)}
-						}
 					}
 				}
 			}
@@ -454,6 +456,8 @@ func fillInGlobalACMEDefaults(issuer certmagic.Issuer, options map[string]any) e
 	globalACMEDNS := options["acme_dns"]
 	globalACMEEAB := options["acme_eab"]
 	globalPreferredChains := options["preferred_chains"]
+	globalCertLifetime := options["cert_lifetime"]
+	globalHTTPPort, globalHTTPSPort := options["http_port"], options["https_port"]
 
 	if globalEmail != nil && acmeIssuer.Email == "" {
 		acmeIssuer.Email = globalEmail.(string)
@@ -477,6 +481,27 @@ func fillInGlobalACMEDefaults(issuer certmagic.Issuer, options map[string]any) e
 	if globalPreferredChains != nil && acmeIssuer.PreferredChains == nil {
 		acmeIssuer.PreferredChains = globalPreferredChains.(*caddytls.ChainPreference)
 	}
+	if globalHTTPPort != nil && (acmeIssuer.Challenges == nil || acmeIssuer.Challenges.HTTP == nil || acmeIssuer.Challenges.HTTP.AlternatePort == 0) {
+		if acmeIssuer.Challenges == nil {
+			acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
+		}
+		if acmeIssuer.Challenges.HTTP == nil {
+			acmeIssuer.Challenges.HTTP = new(caddytls.HTTPChallengeConfig)
+		}
+		acmeIssuer.Challenges.HTTP.AlternatePort = globalHTTPPort.(int)
+	}
+	if globalHTTPSPort != nil && (acmeIssuer.Challenges == nil || acmeIssuer.Challenges.TLSALPN == nil || acmeIssuer.Challenges.TLSALPN.AlternatePort == 0) {
+		if acmeIssuer.Challenges == nil {
+			acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
+		}
+		if acmeIssuer.Challenges.TLSALPN == nil {
+			acmeIssuer.Challenges.TLSALPN = new(caddytls.TLSALPNChallengeConfig)
+		}
+		acmeIssuer.Challenges.TLSALPN.AlternatePort = globalHTTPSPort.(int)
+	}
+	if globalCertLifetime != nil && acmeIssuer.CertificateLifetime == 0 {
+		acmeIssuer.CertificateLifetime = globalCertLifetime.(caddy.Duration)
+	}
 	return nil
 }
 
@@ -485,7 +510,11 @@ func fillInGlobalACMEDefaults(issuer certmagic.Issuer, options map[string]any) e
 // for any other automation policies. A nil policy (and no error) will be
 // returned if there are no default/global options. However, if always is
 // true, a non-nil value will always be returned (unless there is an error).
-func newBaseAutomationPolicy(options map[string]any, warnings []caddyconfig.Warning, always bool) (*caddytls.AutomationPolicy, error) {
+func newBaseAutomationPolicy(
+	options map[string]any,
+	_ []caddyconfig.Warning,
+	always bool,
+) (*caddytls.AutomationPolicy, error) {
 	issuers, hasIssuers := options["cert_issuer"]
 	_, hasLocalCerts := options["local_certs"]
 	keyType, hasKeyType := options["key_type"]
@@ -582,10 +611,12 @@ outer:
 			// eaten up by the one with subjects; and if both have subjects, we
 			// need to combine their lists
 			if reflect.DeepEqual(aps[i].IssuersRaw, aps[j].IssuersRaw) &&
+				reflect.DeepEqual(aps[i].ManagersRaw, aps[j].ManagersRaw) &&
 				bytes.Equal(aps[i].StorageRaw, aps[j].StorageRaw) &&
 				aps[i].MustStaple == aps[j].MustStaple &&
 				aps[i].KeyType == aps[j].KeyType &&
 				aps[i].OnDemand == aps[j].OnDemand &&
+				aps[i].ReusePrivateKeys == aps[j].ReusePrivateKeys &&
 				aps[i].RenewalWindowRatio == aps[j].RenewalWindowRatio {
 				if len(aps[i].SubjectsRaw) > 0 && len(aps[j].SubjectsRaw) == 0 {
 					// later policy (at j) has no subjects ("catch-all"), so we can
@@ -659,17 +690,33 @@ func automationPolicyShadows(i int, aps []*caddytls.AutomationPolicy) int {
 // subjectQualifiesForPublicCert is like certmagic.SubjectQualifiesForPublicCert() except
 // that this allows domains with multiple wildcard levels like '*.*.example.com' to qualify
 // if the automation policy has OnDemand enabled (i.e. this function is more lenient).
+//
+// IP subjects are considered as non-qualifying for public certs. Technically, there are
+// now public ACME CAs as well as non-ACME CAs that issue IP certificates. But this function
+// is used solely for implicit automation (defaults), where it gets really complicated to
+// keep track of which issuers support IP certificates in which circumstances. Currently,
+// issuers that support IP certificates are very few, and all require some sort of config
+// from the user anyway (such as an account credential). Since we cannot implicitly and
+// automatically get public IP certs without configuration from the user, we treat IPs as
+// not qualifying for public certificates. Users should expressly configure an issuer
+// that supports IP certs for that purpose.
 func subjectQualifiesForPublicCert(ap *caddytls.AutomationPolicy, subj string) bool {
 	return !certmagic.SubjectIsIP(subj) &&
 		!certmagic.SubjectIsInternal(subj) &&
 		(strings.Count(subj, "*.") < 2 || ap.OnDemand)
 }
 
+// automationPolicyHasAllPublicNames returns true if all the names on the policy
+// do NOT qualify for public certs OR are tailscale domains.
 func automationPolicyHasAllPublicNames(ap *caddytls.AutomationPolicy) bool {
 	for _, subj := range ap.SubjectsRaw {
-		if !subjectQualifiesForPublicCert(ap, subj) {
+		if !subjectQualifiesForPublicCert(ap, subj) || isTailscaleDomain(subj) {
 			return false
 		}
 	}
 	return true
 }
+
+func isTailscaleDomain(name string) bool {
+	return strings.HasSuffix(strings.ToLower(name), ".ts.net")
+}

caddyconfig/httploader.go

@@ -30,8 +30,14 @@ func init() {
 	caddy.RegisterModule(HTTPLoader{})
 }
 
-// HTTPLoader can load Caddy configs over HTTP(S). It can adapt the config
+// HTTPLoader can load Caddy configs over HTTP(S).
-// based on the Content-Type header of the HTTP response.
+//
+// If the response is not a JSON config, a config adapter must be specified
+// either in the loader config (`adapter`), or in the Content-Type HTTP header
+// returned in the HTTP response from the server. The Content-Type header is
+// read just like the admin API's `/load` endpoint. Uf you don't have control
+// over the HTTP server (but can still trust its response), you can override
+// the Content-Type header by setting the `adapter` property in this config.
 type HTTPLoader struct {
 	// The method for the request. Default: GET
 	Method string `json:"method,omitempty"`
@@ -45,6 +51,11 @@ type HTTPLoader struct {
 	// Maximum time allowed for a complete connection and request.
 	Timeout caddy.Duration `json:"timeout,omitempty"`
 
+	// The name of the config adapter to use, if any. Only needed
+	// if the HTTP response is not a JSON config and if the server's
+	// Content-Type header is missing or incorrect.
+	Adapter string `json:"adapter,omitempty"`
+
 	TLS *struct {
 		// Present this instance's managed remote identity credentials to the server.
 		UseServerIdentity bool `json:"use_server_identity,omitempty"`
@@ -108,7 +119,12 @@ func (hl HTTPLoader) LoadConfig(ctx caddy.Context) ([]byte, error) {
 		return nil, err
 	}
 
-	result, warnings, err := adaptByContentType(resp.Header.Get("Content-Type"), body)
+	// adapt the config based on either manually-configured adapter or server's response header
+	ct := resp.Header.Get("Content-Type")
+	if hl.Adapter != "" {
+		ct = "text/" + hl.Adapter
+	}
+	result, warnings, err := adaptByContentType(ct, body)
 	if err != nil {
 		return nil, err
 	}
@@ -165,19 +181,16 @@ func (hl HTTPLoader) makeClient(ctx caddy.Context) (*http.Client, error) {
 			if err != nil {
 				return nil, fmt.Errorf("getting server identity credentials: %v", err)
 			}
-			if tlsConfig == nil {
+			// See https://github.com/securego/gosec/issues/1054#issuecomment-2072235199
-				tlsConfig = new(tls.Config)
+			//nolint:gosec
-			}
+			tlsConfig = &tls.Config{Certificates: certs}
-			tlsConfig.Certificates = certs
 		} else if hl.TLS.ClientCertificateFile != "" && hl.TLS.ClientCertificateKeyFile != "" {
 			cert, err := tls.LoadX509KeyPair(hl.TLS.ClientCertificateFile, hl.TLS.ClientCertificateKeyFile)
 			if err != nil {
 				return nil, err
 			}
-			if tlsConfig == nil {
+			//nolint:gosec
-				tlsConfig = new(tls.Config)
+			tlsConfig = &tls.Config{Certificates: []tls.Certificate{cert}}
-			}
-			tlsConfig.Certificates = []tls.Certificate{cert}
 		}
 
 		// trusted server certs

caddytest/caddytest.go

@@ -1,11 +1,9 @@
 package caddytest
 
 import (
-	"bytes"
 	"context"
 	"crypto/tls"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -13,16 +11,11 @@ import (
 	"net/http"
 	"net/http/cookiejar"
 	"os"
-	"path"
+	"strconv"
-	"reflect"
-	"regexp"
-	"runtime"
 	"strings"
-	"testing"
+	"sync/atomic"
 	"time"
 
-	"github.com/aryann/difflib"
-	"github.com/caddyserver/caddy/v2/caddyconfig"
 	caddycmd "github.com/caddyserver/caddy/v2/cmd"
 
 	// plug in Caddy modules here
@@ -31,10 +24,8 @@ import (
 
 // Defaults store any configuration required to make the tests run
 type Defaults struct {
-	// Port we expect caddy to listening on
-	AdminPort int
 	// Certificates we expect to be loaded before attempting to run the tests
-	Certifcates []string
+	Certificates []string
 	// TestRequestTimeout is the time to wait for a http request to
 	TestRequestTimeout time.Duration
 	// LoadRequestTimeout is the time to wait for the config to be loaded against the caddy server
@@ -43,30 +34,31 @@ type Defaults struct {
 
 // Default testing values
 var Default = Defaults{
-	AdminPort:          2999, // different from what a real server also running on a developer's machine might be
+	Certificates:       []string{"/caddy.localhost.crt", "/caddy.localhost.key"},
-	Certifcates:        []string{"/caddy.localhost.crt", "/caddy.localhost.key"},
 	TestRequestTimeout: 5 * time.Second,
 	LoadRequestTimeout: 5 * time.Second,
 }
 
-var (
-	matchKey  = regexp.MustCompile(`(/[\w\d\.]+\.key)`)
-	matchCert = regexp.MustCompile(`(/[\w\d\.]+\.crt)`)
-)
-
 // Tester represents an instance of a test client.
 type Tester struct {
-	Client       *http.Client
+	Client *http.Client
-	configLoaded bool
+
-	t            *testing.T
+	adminPort int
+
+	portOne int
+	portTwo int
+
+	started        atomic.Bool
+	configLoaded   bool
+	configFileName string
+	envFileName    string
 }
 
 // NewTester will create a new testing client with an attached cookie jar
-func NewTester(t *testing.T) *Tester {
+func NewTester() (*Tester, error) {
-
 	jar, err := cookiejar.New(nil)
 	if err != nil {
-		t.Fatalf("failed to create cookiejar: %s", err)
+		return nil, fmt.Errorf("failed to create cookiejar: %w", err)
 	}
 
 	return &Tester{
@@ -76,8 +68,7 @@ func NewTester(t *testing.T) *Tester {
 			Timeout:   Default.TestRequestTimeout,
 		},
 		configLoaded: false,
-		t:            t,
+	}, nil
-	}
 }
 
 type configLoadError struct {
@@ -91,61 +82,90 @@ func timeElapsed(start time.Time, name string) {
 	log.Printf("%s took %s", name, elapsed)
 }
 
-// InitServer this will configure the server with a configurion of a specific
+// launch caddy will start the server
-// type. The configType must be either "json" or the adapter type.
+func (tc *Tester) LaunchCaddy() error {
-func (tc *Tester) InitServer(rawConfig string, configType string) {
+	if !tc.started.CompareAndSwap(false, true) {
-
+		return fmt.Errorf("already launched caddy with this tester")
-	if err := tc.initServer(rawConfig, configType); err != nil {
-		tc.t.Logf("failed to load config: %s", err)
-		tc.t.Fail()
 	}
-	if err := tc.ensureConfigRunning(rawConfig, configType); err != nil {
+	if err := tc.startServer(); err != nil {
-		tc.t.Logf("failed ensuring config is running: %s", err)
+		return fmt.Errorf("failed to start server: %w", err)
-		tc.t.Fail()
 	}
+	return nil
 }
 
-// InitServer this will configure the server with a configurion of a specific
+func (tc *Tester) CleanupCaddy() error {
-// type. The configType must be either "json" or the adapter type.
+	// now shutdown the server, since the test is done.
-func (tc *Tester) initServer(rawConfig string, configType string) error {
+	defer func() {
-
+		// try to remove  pthe tmp config file we created
-	if testing.Short() {
+		if tc.configFileName != "" {
-		tc.t.SkipNow()
+			os.Remove(tc.configFileName)
-		return nil
-	}
-
-	err := validateTestPrerequisites(tc.t)
-	if err != nil {
-		tc.t.Skipf("skipping tests as failed integration prerequisites. %s", err)
-		return nil
-	}
-
-	tc.t.Cleanup(func() {
-		if tc.t.Failed() && tc.configLoaded {
-
-			res, err := http.Get(fmt.Sprintf("http://localhost:%d/config/", Default.AdminPort))
-			if err != nil {
-				tc.t.Log("unable to read the current config")
-				return
-			}
-			defer res.Body.Close()
-			body, _ := io.ReadAll(res.Body)
-
-			var out bytes.Buffer
-			_ = json.Indent(&out, body, "", "  ")
-			tc.t.Logf("----------- failed with config -----------\n%s", out.String())
 		}
-	})
+		if tc.envFileName != "" {
+			os.Remove(tc.envFileName)
+		}
+	}()
+	resp, err := http.Post(fmt.Sprintf("http://localhost:%d/stop", tc.adminPort), "", nil)
+	if err != nil {
+		return fmt.Errorf("couldn't stop caddytest server: %w", err)
+	}
+	resp.Body.Close()
+	for retries := 0; retries < 10; retries++ {
+		if tc.isCaddyAdminRunning() != nil {
+			return nil
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
 
-	rawConfig = prependCaddyFilePath(rawConfig)
+	return fmt.Errorf("timed out waiting for caddytest server to stop")
+}
+
+func (tc *Tester) AdminPort() int {
+	return tc.adminPort
+}
+
+func (tc *Tester) PortOne() int {
+	return tc.portOne
+}
+
+func (tc *Tester) PortTwo() int {
+	return tc.portTwo
+}
+
+func (tc *Tester) ReplaceTestingPlaceholders(x string) string {
+	x = strings.ReplaceAll(x, "{$TESTING_CADDY_ADMIN_BIND}", fmt.Sprintf("localhost:%d", tc.adminPort))
+	x = strings.ReplaceAll(x, "{$TESTING_CADDY_ADMIN_PORT}", fmt.Sprintf("%d", tc.adminPort))
+	x = strings.ReplaceAll(x, "{$TESTING_CADDY_PORT_ONE}", fmt.Sprintf("%d", tc.portOne))
+	x = strings.ReplaceAll(x, "{$TESTING_CADDY_PORT_TWO}", fmt.Sprintf("%d", tc.portTwo))
+	return x
+}
+
+// LoadConfig loads the config to the tester server and also ensures that the config was loaded
+// it should not be run
+func (tc *Tester) LoadConfig(rawConfig string, configType string) error {
+	if tc.adminPort == 0 {
+		return fmt.Errorf("load config called where startServer didnt succeed")
+	}
+	rawConfig = tc.ReplaceTestingPlaceholders(rawConfig)
+	// replace special testing placeholders so we can have our admin api be on a random port
+	// normalize JSON config
+	if configType == "json" {
+		var conf any
+		if err := json.Unmarshal([]byte(rawConfig), &conf); err != nil {
+			return err
+		}
+		c, err := json.Marshal(conf)
+		if err != nil {
+			return err
+		}
+		rawConfig = string(c)
+	}
 	client := &http.Client{
 		Timeout: Default.LoadRequestTimeout,
 	}
 	start := time.Now()
-	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/load", Default.AdminPort), strings.NewReader(rawConfig))
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/load", tc.adminPort), strings.NewReader(rawConfig))
 	if err != nil {
-		tc.t.Errorf("failed to create request. %s", err)
+		return fmt.Errorf("failed to create request. %w", err)
-		return err
 	}
 
 	if configType == "json" {
@@ -156,16 +176,14 @@ func (tc *Tester) initServer(rawConfig string, configType string) error {
 
 	res, err := client.Do(req)
 	if err != nil {
-		tc.t.Errorf("unable to contact caddy server. %s", err)
+		return fmt.Errorf("unable to contact caddy server. %w", err)
-		return err
 	}
 	timeElapsed(start, "caddytest: config load time")
 
 	defer res.Body.Close()
 	body, err := io.ReadAll(res.Body)
 	if err != nil {
-		tc.t.Errorf("unable to read response. %s", err)
+		return fmt.Errorf("unable to read response. %w", err)
-		return err
 	}
 
 	if res.StatusCode != 200 {
@@ -173,138 +191,117 @@ func (tc *Tester) initServer(rawConfig string, configType string) error {
 	}
 
 	tc.configLoaded = true
+
+	// if the config is not loaded at this point, it is a bug in caddy's config.Load
+	// the contract for config.Load states that the config must be loaded before it returns, and that it will
+	// error if the config fails to apply
 	return nil
 }
 
-func (tc *Tester) ensureConfigRunning(rawConfig string, configType string) error {
+func (tc *Tester) GetCurrentConfig(receiver any) error {
-	expectedBytes := []byte(prependCaddyFilePath(rawConfig))
-	if configType != "json" {
-		adapter := caddyconfig.GetAdapter(configType)
-		if adapter == nil {
-			return fmt.Errorf("adapter of config type is missing: %s", configType)
-		}
-		expectedBytes, _, _ = adapter.Adapt([]byte(rawConfig), nil)
-	}
-
-	var expected any
-	err := json.Unmarshal(expectedBytes, &expected)
-	if err != nil {
-		return err
-	}
-
 	client := &http.Client{
 		Timeout: Default.LoadRequestTimeout,
 	}
 
-	fetchConfig := func(client *http.Client) any {
+	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/config/", tc.adminPort))
-		resp, err := client.Get(fmt.Sprintf("http://localhost:%d/config/", Default.AdminPort))
+	if err != nil {
-		if err != nil {
+		return err
-			return nil
-		}
-		defer resp.Body.Close()
-		actualBytes, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return nil
-		}
-		var actual any
-		err = json.Unmarshal(actualBytes, &actual)
-		if err != nil {
-			return nil
-		}
-		return actual
 	}
-
+	defer resp.Body.Close()
-	for retries := 10; retries > 0; retries-- {
+	actualBytes, err := io.ReadAll(resp.Body)
-		if reflect.DeepEqual(expected, fetchConfig(client)) {
+	if err != nil {
-			return nil
+		return err
-		}
-		time.Sleep(1 * time.Second)
 	}
-	tc.t.Errorf("POSTed configuration isn't active")
+	err = json.Unmarshal(actualBytes, receiver)
-	return errors.New("EnsureConfigRunning: POSTed configuration isn't active")
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
-const initConfig = `{
+func getFreePort() (int, error) {
-	admin localhost:2999
+	lr, err := net.Listen("tcp", "localhost:0")
-}
+	if err != nil {
-`
+		return 0, err
-
-// validateTestPrerequisites ensures the certificates are available in the
-// designated path and Caddy sub-process is running.
-func validateTestPrerequisites(t *testing.T) error {
-
-	// check certificates are found
-	for _, certName := range Default.Certifcates {
-		if _, err := os.Stat(getIntegrationDir() + certName); os.IsNotExist(err) {
-			return fmt.Errorf("caddy integration test certificates (%s) not found", certName)
-		}
 	}
+	port := strings.Split(lr.Addr().String(), ":")
+	if len(port) < 2 {
+		return 0, fmt.Errorf("no port available")
+	}
+	i, err := strconv.Atoi(port[1])
+	if err != nil {
+		return 0, err
+	}
+	err = lr.Close()
+	if err != nil {
+		return 0, fmt.Errorf("failed to close listener: %w", err)
+	}
+	return i, nil
+}
 
-	if isCaddyAdminRunning() != nil {
+// launches caddy, and then ensures the Caddy sub-process is running.
-		// setup the init config file, and set the cleanup afterwards
+func (tc *Tester) startServer() error {
+	if tc.isCaddyAdminRunning() == nil {
+		return fmt.Errorf("caddy test admin port still in use")
+	}
+	a, err := getFreePort()
+	if err != nil {
+		return fmt.Errorf("could not find a open port to listen on: %w", err)
+	}
+	tc.adminPort = a
+	tc.portOne, err = getFreePort()
+	if err != nil {
+		return fmt.Errorf("could not find a open portOne: %w", err)
+	}
+	tc.portTwo, err = getFreePort()
+	if err != nil {
+		return fmt.Errorf("could not find a open portOne: %w", err)
+	}
+	// setup the init config file, and set the cleanup afterwards
+	{
 		f, err := os.CreateTemp("", "")
 		if err != nil {
 			return err
 		}
-		t.Cleanup(func() {
+		tc.configFileName = f.Name()
-			os.Remove(f.Name())
+
-		})
+		initConfig := fmt.Sprintf(`{
+	admin localhost:%d
+}`, a)
 		if _, err := f.WriteString(initConfig); err != nil {
 			return err
 		}
+	}
 
-		// start inprocess caddy server
+	// start inprocess caddy server
-		os.Args = []string{"caddy", "run", "--config", f.Name(), "--adapter", "caddyfile"}
+	go func() {
-		go func() {
+		_ = caddycmd.MainForTesting("run", "--config", tc.configFileName, "--adapter", "caddyfile")
-			caddycmd.Main()
+	}()
-		}()
+	// wait for caddy admin api to start. it should happen quickly.
-
+	for retries := 10; retries > 0 && tc.isCaddyAdminRunning() != nil; retries-- {
-		// wait for caddy to start serving the initial config
+		time.Sleep(100 * time.Millisecond)
-		for retries := 10; retries > 0 && isCaddyAdminRunning() != nil; retries-- {
-			time.Sleep(1 * time.Second)
-		}
 	}
 
 	// one more time to return the error
-	return isCaddyAdminRunning()
+	return tc.isCaddyAdminRunning()
 }
 
-func isCaddyAdminRunning() error {
+func (tc *Tester) isCaddyAdminRunning() error {
 	// assert that caddy is running
 	client := &http.Client{
 		Timeout: Default.LoadRequestTimeout,
 	}
-	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/config/", Default.AdminPort))
+	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/config/", tc.adminPort))
 	if err != nil {
-		return fmt.Errorf("caddy integration test caddy server not running. Expected to be listening on localhost:%d", Default.AdminPort)
+		return fmt.Errorf("caddy integration test caddy server not running. Expected to be listening on localhost:%d", tc.adminPort)
 	}
 	resp.Body.Close()
 
 	return nil
 }
 
-func getIntegrationDir() string {
-
-	_, filename, _, ok := runtime.Caller(1)
-	if !ok {
-		panic("unable to determine the current file path")
-	}
-
-	return path.Dir(filename)
-}
-
-// use the convention to replace /[certificatename].[crt|key] with the full path
-// this helps reduce the noise in test configurations and also allow this
-// to run in any path
-func prependCaddyFilePath(rawConfig string) string {
-	r := matchKey.ReplaceAllString(rawConfig, getIntegrationDir()+"$1")
-	r = matchCert.ReplaceAllString(r, getIntegrationDir()+"$1")
-	return r
-}
-
 // CreateTestingTransport creates a testing transport that forces call dialing connections to happen locally
 func CreateTestingTransport() *http.Transport {
-
 	dialer := net.Dialer{
 		Timeout:   5 * time.Second,
 		KeepAlive: 5 * time.Second,
@@ -329,241 +326,3 @@ func CreateTestingTransport() *http.Transport {
 		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true}, //nolint:gosec
 	}
 }
-
-// AssertLoadError will load a config and expect an error
-func AssertLoadError(t *testing.T, rawConfig string, configType string, expectedError string) {
-
-	tc := NewTester(t)
-
-	err := tc.initServer(rawConfig, configType)
-	if !strings.Contains(err.Error(), expectedError) {
-		t.Errorf("expected error \"%s\" but got \"%s\"", expectedError, err.Error())
-	}
-}
-
-// AssertRedirect makes a request and asserts the redirection happens
-func (tc *Tester) AssertRedirect(requestURI string, expectedToLocation string, expectedStatusCode int) *http.Response {
-
-	redirectPolicyFunc := func(req *http.Request, via []*http.Request) error {
-		return http.ErrUseLastResponse
-	}
-
-	// using the existing client, we override the check redirect policy for this test
-	old := tc.Client.CheckRedirect
-	tc.Client.CheckRedirect = redirectPolicyFunc
-	defer func() { tc.Client.CheckRedirect = old }()
-
-	resp, err := tc.Client.Get(requestURI)
-	if err != nil {
-		tc.t.Errorf("failed to call server %s", err)
-		return nil
-	}
-
-	if expectedStatusCode != resp.StatusCode {
-		tc.t.Errorf("requesting \"%s\" expected status code: %d but got %d", requestURI, expectedStatusCode, resp.StatusCode)
-	}
-
-	loc, err := resp.Location()
-	if err != nil {
-		tc.t.Errorf("requesting \"%s\" expected location: \"%s\" but got error: %s", requestURI, expectedToLocation, err)
-	}
-	if loc == nil && expectedToLocation != "" {
-		tc.t.Errorf("requesting \"%s\" expected a Location header, but didn't get one", requestURI)
-	}
-	if loc != nil {
-		if expectedToLocation != loc.String() {
-			tc.t.Errorf("requesting \"%s\" expected location: \"%s\" but got \"%s\"", requestURI, expectedToLocation, loc.String())
-		}
-	}
-
-	return resp
-}
-
-// CompareAdapt adapts a config and then compares it against an expected result
-func CompareAdapt(t *testing.T, filename, rawConfig string, adapterName string, expectedResponse string) bool {
-
-	cfgAdapter := caddyconfig.GetAdapter(adapterName)
-	if cfgAdapter == nil {
-		t.Logf("unrecognized config adapter '%s'", adapterName)
-		return false
-	}
-
-	options := make(map[string]any)
-
-	result, warnings, err := cfgAdapter.Adapt([]byte(rawConfig), options)
-	if err != nil {
-		t.Logf("adapting config using %s adapter: %v", adapterName, err)
-		return false
-	}
-
-	// prettify results to keep tests human-manageable
-	var prettyBuf bytes.Buffer
-	err = json.Indent(&prettyBuf, result, "", "\t")
-	if err != nil {
-		return false
-	}
-	result = prettyBuf.Bytes()
-
-	if len(warnings) > 0 {
-		for _, w := range warnings {
-			t.Logf("warning: %s:%d: %s: %s", filename, w.Line, w.Directive, w.Message)
-		}
-	}
-
-	diff := difflib.Diff(
-		strings.Split(expectedResponse, "\n"),
-		strings.Split(string(result), "\n"))
-
-	// scan for failure
-	failed := false
-	for _, d := range diff {
-		if d.Delta != difflib.Common {
-			failed = true
-			break
-		}
-	}
-
-	if failed {
-		for _, d := range diff {
-			switch d.Delta {
-			case difflib.Common:
-				fmt.Printf("  %s\n", d.Payload)
-			case difflib.LeftOnly:
-				fmt.Printf(" - %s\n", d.Payload)
-			case difflib.RightOnly:
-				fmt.Printf(" + %s\n", d.Payload)
-			}
-		}
-		return false
-	}
-	return true
-}
-
-// AssertAdapt adapts a config and then tests it against an expected result
-func AssertAdapt(t *testing.T, rawConfig string, adapterName string, expectedResponse string) {
-	ok := CompareAdapt(t, "Caddyfile", rawConfig, adapterName, expectedResponse)
-	if !ok {
-		t.Fail()
-	}
-}
-
-// Generic request functions
-
-func applyHeaders(t *testing.T, req *http.Request, requestHeaders []string) {
-	requestContentType := ""
-	for _, requestHeader := range requestHeaders {
-		arr := strings.SplitAfterN(requestHeader, ":", 2)
-		k := strings.TrimRight(arr[0], ":")
-		v := strings.TrimSpace(arr[1])
-		if k == "Content-Type" {
-			requestContentType = v
-		}
-		t.Logf("Request header: %s => %s", k, v)
-		req.Header.Set(k, v)
-	}
-
-	if requestContentType == "" {
-		t.Logf("Content-Type header not provided")
-	}
-}
-
-// AssertResponseCode will execute the request and verify the status code, returns a response for additional assertions
-func (tc *Tester) AssertResponseCode(req *http.Request, expectedStatusCode int) *http.Response {
-
-	resp, err := tc.Client.Do(req)
-	if err != nil {
-		tc.t.Fatalf("failed to call server %s", err)
-	}
-
-	if expectedStatusCode != resp.StatusCode {
-		tc.t.Errorf("requesting \"%s\" expected status code: %d but got %d", req.RequestURI, expectedStatusCode, resp.StatusCode)
-	}
-
-	return resp
-}
-
-// AssertResponse request a URI and assert the status code and the body contains a string
-func (tc *Tester) AssertResponse(req *http.Request, expectedStatusCode int, expectedBody string) (*http.Response, string) {
-
-	resp := tc.AssertResponseCode(req, expectedStatusCode)
-
-	defer resp.Body.Close()
-	bytes, err := io.ReadAll(resp.Body)
-	if err != nil {
-		tc.t.Fatalf("unable to read the response body %s", err)
-	}
-
-	body := string(bytes)
-
-	if body != expectedBody {
-		tc.t.Errorf("requesting \"%s\" expected response body \"%s\" but got \"%s\"", req.RequestURI, expectedBody, body)
-	}
-
-	return resp, body
-}
-
-// Verb specific test functions
-
-// AssertGetResponse GET a URI and expect a statusCode and body text
-func (tc *Tester) AssertGetResponse(requestURI string, expectedStatusCode int, expectedBody string) (*http.Response, string) {
-
-	req, err := http.NewRequest("GET", requestURI, nil)
-	if err != nil {
-		tc.t.Fatalf("unable to create request %s", err)
-	}
-
-	return tc.AssertResponse(req, expectedStatusCode, expectedBody)
-}
-
-// AssertDeleteResponse request a URI and expect a statusCode and body text
-func (tc *Tester) AssertDeleteResponse(requestURI string, expectedStatusCode int, expectedBody string) (*http.Response, string) {
-
-	req, err := http.NewRequest("DELETE", requestURI, nil)
-	if err != nil {
-		tc.t.Fatalf("unable to create request %s", err)
-	}
-
-	return tc.AssertResponse(req, expectedStatusCode, expectedBody)
-}
-
-// AssertPostResponseBody POST to a URI and assert the response code and body
-func (tc *Tester) AssertPostResponseBody(requestURI string, requestHeaders []string, requestBody *bytes.Buffer, expectedStatusCode int, expectedBody string) (*http.Response, string) {
-
-	req, err := http.NewRequest("POST", requestURI, requestBody)
-	if err != nil {
-		tc.t.Errorf("failed to create request %s", err)
-		return nil, ""
-	}
-
-	applyHeaders(tc.t, req, requestHeaders)
-
-	return tc.AssertResponse(req, expectedStatusCode, expectedBody)
-}
-
-// AssertPutResponseBody PUT to a URI and assert the response code and body
-func (tc *Tester) AssertPutResponseBody(requestURI string, requestHeaders []string, requestBody *bytes.Buffer, expectedStatusCode int, expectedBody string) (*http.Response, string) {
-
-	req, err := http.NewRequest("PUT", requestURI, requestBody)
-	if err != nil {
-		tc.t.Errorf("failed to create request %s", err)
-		return nil, ""
-	}
-
-	applyHeaders(tc.t, req, requestHeaders)
-
-	return tc.AssertResponse(req, expectedStatusCode, expectedBody)
-}
-
-// AssertPatchResponseBody PATCH to a URI and assert the response code and body
-func (tc *Tester) AssertPatchResponseBody(requestURI string, requestHeaders []string, requestBody *bytes.Buffer, expectedStatusCode int, expectedBody string) (*http.Response, string) {
-
-	req, err := http.NewRequest("PATCH", requestURI, requestBody)
-	if err != nil {
-		tc.t.Errorf("failed to create request %s", err)
-		return nil, ""
-	}
-
-	applyHeaders(tc.t, req, requestHeaders)
-
-	return tc.AssertResponse(req, expectedStatusCode, expectedBody)
-}

caddytest/caddytest_assert.go

@@ -0,0 +1,116 @@
+package caddytest
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/aryann/difflib"
+	"github.com/stretchr/testify/require"
+
+	"github.com/caddyserver/caddy/v2/caddyconfig"
+)
+
+// AssertLoadError will load a config and expect an error
+func AssertLoadError(t *testing.T, rawConfig string, configType string, expectedError string) {
+	tc, err := NewTester()
+	require.NoError(t, err)
+	err = tc.LaunchCaddy()
+	require.NoError(t, err)
+
+	err = tc.LoadConfig(rawConfig, configType)
+	if !strings.Contains(err.Error(), expectedError) {
+		t.Errorf("expected error \"%s\" but got \"%s\"", expectedError, err.Error())
+	}
+	_ = tc.CleanupCaddy()
+}
+
+// CompareAdapt adapts a config and then compares it against an expected result
+func CompareAdapt(t testing.TB, filename, rawConfig string, adapterName string, expectedResponse string) bool {
+	cfgAdapter := caddyconfig.GetAdapter(adapterName)
+	if cfgAdapter == nil {
+		t.Logf("unrecognized config adapter '%s'", adapterName)
+		return false
+	}
+
+	options := make(map[string]any)
+
+	result, warnings, err := cfgAdapter.Adapt([]byte(rawConfig), options)
+	if err != nil {
+		t.Logf("adapting config using %s adapter: %v", adapterName, err)
+		return false
+	}
+
+	// prettify results to keep tests human-manageable
+	var prettyBuf bytes.Buffer
+	err = json.Indent(&prettyBuf, result, "", "\t")
+	if err != nil {
+		return false
+	}
+	result = prettyBuf.Bytes()
+
+	if len(warnings) > 0 {
+		for _, w := range warnings {
+			t.Logf("warning: %s:%d: %s: %s", filename, w.Line, w.Directive, w.Message)
+		}
+	}
+
+	diff := difflib.Diff(
+		strings.Split(expectedResponse, "\n"),
+		strings.Split(string(result), "\n"))
+
+	// scan for failure
+	failed := false
+	for _, d := range diff {
+		if d.Delta != difflib.Common {
+			failed = true
+			break
+		}
+	}
+
+	if failed {
+		for _, d := range diff {
+			switch d.Delta {
+			case difflib.Common:
+				fmt.Printf("  %s\n", d.Payload)
+			case difflib.LeftOnly:
+				fmt.Printf(" - %s\n", d.Payload)
+			case difflib.RightOnly:
+				fmt.Printf(" + %s\n", d.Payload)
+			}
+		}
+		return false
+	}
+	return true
+}
+
+// AssertAdapt adapts a config and then tests it against an expected result
+func AssertAdapt(t testing.TB, rawConfig string, adapterName string, expectedResponse string) {
+	ok := CompareAdapt(t, "Caddyfile", rawConfig, adapterName, expectedResponse)
+	if !ok {
+		t.Fail()
+	}
+}
+
+// Generic request functions
+
+func applyHeaders(t testing.TB, req *http.Request, requestHeaders []string) {
+	requestContentType := ""
+	for _, requestHeader := range requestHeaders {
+		arr := strings.SplitAfterN(requestHeader, ":", 2)
+		k := strings.TrimRight(arr[0], ":")
+		v := strings.TrimSpace(arr[1])
+		if k == "Content-Type" {
+			requestContentType = v
+		}
+		t.Logf("Request header: %s => %s", k, v)
+		req.Header.Set(k, v)
+	}
+
+	if requestContentType == "" {
+		t.Logf("Content-Type header not provided")
+	}
+}

caddytest/caddytest_test.go

@@ -1,20 +1,22 @@
 package caddytest
 
 import (
+	"fmt"
+	"net/http"
 	"strings"
 	"testing"
 )
 
 func TestReplaceCertificatePaths(t *testing.T) {
-	rawConfig := `a.caddy.localhost:9443 {
+	rawConfig := `a.caddy.localhost:9443{
 		tls /caddy.localhost.crt /caddy.localhost.key {
 		}
 
 		redir / https://b.caddy.localhost:9443/version 301
-    
+
 		respond /version 200 {
 		  body "hello from a.caddy.localhost"
-		}	
+		}
 	  }`
 
 	r := prependCaddyFilePath(rawConfig)
@@ -31,3 +33,98 @@ func TestReplaceCertificatePaths(t *testing.T) {
 		t.Error("expected redirect uri to be unchanged")
 	}
 }
+
+func TestLoadUnorderedJSON(t *testing.T) {
+	harness := StartHarness(t)
+	harness.LoadConfig(`
+	{
+		"logging": {
+			"logs": {
+				"default": {
+					"level": "DEBUG",
+					"writer": {
+						"output": "stdout"
+					}
+				},
+				"sStdOutLogs": {
+					"level": "DEBUG",
+					"writer": {
+						"output": "stdout"
+					},
+					"include": [
+						"http.*",
+						"admin.*"
+					]
+				},
+				"sFileLogs": {
+					"level": "DEBUG",
+					"writer": {
+						"output": "stdout"
+					},
+					"include": [
+						"http.*",
+						"admin.*"
+					]
+				}
+			}
+		},
+		"admin": {
+			"listen": "{$TESTING_CADDY_ADMIN_BIND}"
+		},
+		"apps": {
+			"pki": {
+				"certificate_authorities" : {
+				  "local" : {
+					"install_trust": false
+				  }
+				}
+			},
+			"http": {
+				"http_port": {$TESTING_CADDY_PORT_ONE},
+				"https_port": {$TESTING_CADDY_PORT_TWO},
+				"servers": {
+					"s_server": {
+						"listen": [
+							":{$TESTING_CADDY_PORT_ONE}",
+							":{$TESTING_CADDY_PORT_TWO}"
+						],
+						"routes": [
+							{
+								"handle": [
+									{
+										"handler": "static_response",
+										"body": "Hello"
+									}
+								]
+							},
+							{
+								"match": [
+									{
+										"host": [
+											"localhost",
+											"127.0.0.1"
+										]
+									}
+								]
+							}
+						],
+						"logs": {
+							"default_logger_name": "sStdOutLogs",
+							"logger_names": {
+								"localhost": "sStdOutLogs",
+								"127.0.0.1": "sFileLogs"
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+  `, "json")
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:%d/", harness.Tester().PortOne()), nil)
+	if err != nil {
+		t.Fail()
+		return
+	}
+	harness.AssertResponseCode(req, 200)
+}

caddytest/integration/acme_test.go

@@ -0,0 +1,201 @@
+package integration
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddytest"
+	"github.com/mholt/acmez/v2"
+	"github.com/mholt/acmez/v2/acme"
+	smallstepacme "github.com/smallstep/certificates/acme"
+	"go.uber.org/zap"
+)
+
+const acmeChallengePort = 9081
+
+// Test the basic functionality of Caddy's ACME server
+func TestACMEServerWithDefaults(t *testing.T) {
+	ctx := context.Background()
+	harness := caddytest.StartHarness(t)
+	harness.LoadConfig(`
+	{
+		skip_install_trust
+		admin {$TESTING_CADDY_ADMIN_BIND}
+		http_port     {$TESTING_CADDY_PORT_ONE}
+		https_port    {$TESTING_CADDY_PORT_TWO}
+		local_certs
+	}
+	acme.localhost {
+		acme_server
+	}
+  `, "caddyfile")
+
+	logger := caddy.Log().Named("acmeserver")
+	client := acmez.Client{
+		Client: &acme.Client{
+			Directory:  fmt.Sprintf("https://acme.localhost:%d/acme/local/directory", harness.Tester().PortTwo()),
+			HTTPClient: harness.Client(),
+			Logger:     logger,
+		},
+		ChallengeSolvers: map[string]acmez.Solver{
+			acme.ChallengeTypeHTTP01: &naiveHTTPSolver{logger: logger},
+		},
+	}
+
+	accountPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Errorf("generating account key: %v", err)
+	}
+	account := acme.Account{
+		Contact:              []string{"mailto:you@example.com"},
+		TermsOfServiceAgreed: true,
+		PrivateKey:           accountPrivateKey,
+	}
+	account, err = client.NewAccount(ctx, account)
+	if err != nil {
+		t.Errorf("new account: %v", err)
+		return
+	}
+
+	// Every certificate needs a key.
+	certPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Errorf("generating certificate key: %v", err)
+		return
+	}
+
+	certs, err := client.ObtainCertificateForSANs(ctx, account, certPrivateKey, []string{"localhost"})
+	if err != nil {
+		t.Errorf("obtaining certificate: %v", err)
+		return
+	}
+
+	// ACME servers should usually give you the entire certificate chain
+	// in PEM format, and sometimes even alternate chains! It's up to you
+	// which one(s) to store and use, but whatever you do, be sure to
+	// store the certificate and key somewhere safe and secure, i.e. don't
+	// lose them!
+	for _, cert := range certs {
+		t.Logf("Certificate %q:\n%s\n\n", cert.URL, cert.ChainPEM)
+	}
+}
+
+func TestACMEServerWithMismatchedChallenges(t *testing.T) {
+	ctx := context.Background()
+	logger := caddy.Log().Named("acmez")
+
+	harness := caddytest.StartHarness(t)
+	harness.LoadConfig(`
+	{
+		skip_install_trust
+		admin {$TESTING_CADDY_ADMIN_BIND}
+		http_port     {$TESTING_CADDY_PORT_ONE}
+		https_port    {$TESTING_CADDY_PORT_TWO}
+		local_certs
+	}
+	acme.localhost {
+		acme_server {
+			challenges tls-alpn-01
+		}
+	}
+  `, "caddyfile")
+
+	client := acmez.Client{
+		Client: &acme.Client{
+			Directory:  fmt.Sprintf("https://acme.localhost:%d/acme/local/directory", harness.Tester().PortTwo()),
+			HTTPClient: harness.Client(),
+			Logger:     logger,
+		},
+		ChallengeSolvers: map[string]acmez.Solver{
+			acme.ChallengeTypeHTTP01: &naiveHTTPSolver{logger: logger},
+		},
+	}
+
+	accountPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Errorf("generating account key: %v", err)
+	}
+	account := acme.Account{
+		Contact:              []string{"mailto:you@example.com"},
+		TermsOfServiceAgreed: true,
+		PrivateKey:           accountPrivateKey,
+	}
+	account, err = client.NewAccount(ctx, account)
+	if err != nil {
+		t.Errorf("new account: %v", err)
+		return
+	}
+
+	// Every certificate needs a key.
+	certPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Errorf("generating certificate key: %v", err)
+		return
+	}
+
+	certs, err := client.ObtainCertificateForSANs(ctx, account, certPrivateKey, []string{"localhost"})
+	if len(certs) > 0 {
+		t.Errorf("expected '0' certificates, but received '%d'", len(certs))
+	}
+	if err == nil {
+		t.Error("expected errors, but received none")
+	}
+	const expectedErrMsg = "no solvers available for remaining challenges (configured=[http-01] offered=[tls-alpn-01] remaining=[tls-alpn-01])"
+	if !strings.Contains(err.Error(), expectedErrMsg) {
+		t.Errorf(`received error message does not match expectation: expected="%s" received="%s"`, expectedErrMsg, err.Error())
+	}
+}
+
+// naiveHTTPSolver is a no-op acmez.Solver for example purposes only.
+type naiveHTTPSolver struct {
+	srv    *http.Server
+	logger *zap.Logger
+}
+
+func (s *naiveHTTPSolver) Present(ctx context.Context, challenge acme.Challenge) error {
+	smallstepacme.InsecurePortHTTP01 = acmeChallengePort
+	s.srv = &http.Server{
+		Addr: fmt.Sprintf(":%d", acmeChallengePort),
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			host, _, err := net.SplitHostPort(r.Host)
+			if err != nil {
+				host = r.Host
+			}
+			s.logger.Info("received request on challenge server", zap.String("path", r.URL.Path))
+			if r.Method == "GET" && r.URL.Path == challenge.HTTP01ResourcePath() && strings.EqualFold(host, challenge.Identifier.Value) {
+				w.Header().Add("Content-Type", "text/plain")
+				w.Write([]byte(challenge.KeyAuthorization))
+				r.Close = true
+				s.logger.Info("served key authentication",
+					zap.String("identifier", challenge.Identifier.Value),
+					zap.String("challenge", "http-01"),
+					zap.String("remote", r.RemoteAddr),
+				)
+			}
+		}),
+	}
+	l, err := net.Listen("tcp", fmt.Sprintf(":%d", acmeChallengePort))
+	if err != nil {
+		return err
+	}
+	s.logger.Info("present challenge", zap.Any("challenge", challenge))
+	go s.srv.Serve(l)
+	return nil
+}
+
+func (s naiveHTTPSolver) CleanUp(ctx context.Context, challenge acme.Challenge) error {
+	smallstepacme.InsecurePortHTTP01 = 0
+	s.logger.Info("cleanup", zap.Any("challenge", challenge))
+	if s.srv != nil {
+		s.srv.Close()
+	}
+	return nil
+}

caddytest/integration/acmeserver_test.go

@@ -0,0 +1,197 @@
+package integration
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddytest"
+	"github.com/mholt/acmez/v2"
+	"github.com/mholt/acmez/v2/acme"
+)
+
+func TestACMEServerDirectory(t *testing.T) {
+	harness := caddytest.StartHarness(t)
+	harness.LoadConfig(`
+	{
+		skip_install_trust
+		local_certs
+		admin {$TESTING_CADDY_ADMIN_BIND}
+		http_port     {$TESTING_CADDY_PORT_ONE}
+		https_port    {$TESTING_CADDY_PORT_TWO}
+		pki {
+			ca local {
+				name "Caddy Local Authority"
+			}
+		}
+	}
+	acme.localhost:{$TESTING_CADDY_PORT_TWO} {
+		acme_server
+	}
+  `, "caddyfile")
+	harness.AssertGetResponse(
+		fmt.Sprintf("https://acme.localhost:%d/acme/local/directory", harness.Tester().PortTwo()),
+		200,
+		fmt.Sprintf(`{"newNonce":"https://acme.localhost:%[1]d/acme/local/new-nonce","newAccount":"https://acme.localhost:%[1]d/acme/local/new-account","newOrder":"https://acme.localhost:%[1]d/acme/local/new-order","revokeCert":"https://acme.localhost:%[1]d/acme/local/revoke-cert","keyChange":"https://acme.localhost:%[1]d/acme/local/key-change"}
+`, harness.Tester().PortTwo()))
+}
+
+func TestACMEServerAllowPolicy(t *testing.T) {
+	harness := caddytest.StartHarness(t)
+	harness.LoadConfig(`
+	{
+		skip_install_trust
+		local_certs
+		admin {$TESTING_CADDY_ADMIN_BIND}
+		http_port     {$TESTING_CADDY_PORT_ONE}
+		https_port    {$TESTING_CADDY_PORT_TWO}
+		pki {
+			ca local {
+				name "Caddy Local Authority"
+			}
+		}
+	}
+	acme.localhost {
+		acme_server {
+			challenges http-01
+			allow {
+				domains localhost
+			}
+		}
+	}
+  `, "caddyfile")
+
+	ctx := context.Background()
+	logger := caddy.Log().Named("acmez")
+
+	client := acmez.Client{
+		Client: &acme.Client{
+			Directory:  fmt.Sprintf("https://acme.localhost:%d/acme/local/directory", harness.Tester().PortTwo()),
+			HTTPClient: harness.Client(),
+			Logger:     logger,
+		},
+		ChallengeSolvers: map[string]acmez.Solver{
+			acme.ChallengeTypeHTTP01: &naiveHTTPSolver{logger: logger},
+		},
+	}
+
+	accountPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Errorf("generating account key: %v", err)
+	}
+	account := acme.Account{
+		Contact:              []string{"mailto:you@example.com"},
+		TermsOfServiceAgreed: true,
+		PrivateKey:           accountPrivateKey,
+	}
+	account, err = client.NewAccount(ctx, account)
+	if err != nil {
+		t.Errorf("new account: %v", err)
+		return
+	}
+
+	// Every certificate needs a key.
+	certPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Errorf("generating certificate key: %v", err)
+		return
+	}
+	{
+		certs, err := client.ObtainCertificateForSANs(ctx, account, certPrivateKey, []string{"localhost"})
+		if err != nil {
+			t.Errorf("obtaining certificate for allowed domain: %v", err)
+			return
+		}
+
+		// ACME servers should usually give you the entire certificate chain
+		// in PEM format, and sometimes even alternate chains! It's up to you
+		// which one(s) to store and use, but whatever you do, be sure to
+		// store the certificate and key somewhere safe and secure, i.e. don't
+		// lose them!
+		for _, cert := range certs {
+			t.Logf("Certificate %q:\n%s\n\n", cert.URL, cert.ChainPEM)
+		}
+	}
+	{
+		_, err := client.ObtainCertificateForSANs(ctx, account, certPrivateKey, []string{"not-matching.localhost"})
+		if err == nil {
+			t.Errorf("obtaining certificate for 'not-matching.localhost' domain")
+		} else if err != nil && !strings.Contains(err.Error(), "urn:ietf:params:acme:error:rejectedIdentifier") {
+			t.Logf("unexpected error: %v", err)
+		}
+	}
+}
+
+func TestACMEServerDenyPolicy(t *testing.T) {
+	harness := caddytest.StartHarness(t)
+	harness.LoadConfig(`
+	{
+		skip_install_trust
+		local_certs
+		admin {$TESTING_CADDY_ADMIN_BIND}
+		http_port     {$TESTING_CADDY_PORT_ONE}
+		https_port    {$TESTING_CADDY_PORT_TWO}
+		pki {
+			ca local {
+				name "Caddy Local Authority"
+			}
+		}
+	}
+	acme.localhost {
+		acme_server {
+			deny {
+				domains deny.localhost
+			}
+		}
+	}
+  `, "caddyfile")
+
+	ctx := context.Background()
+	logger := caddy.Log().Named("acmez")
+
+	client := acmez.Client{
+		Client: &acme.Client{
+			Directory:  fmt.Sprintf("https://acme.localhost:%d/acme/local/directory", harness.Tester().PortTwo()),
+			HTTPClient: harness.Client(),
+			Logger:     logger,
+		},
+		ChallengeSolvers: map[string]acmez.Solver{
+			acme.ChallengeTypeHTTP01: &naiveHTTPSolver{logger: logger},
+		},
+	}
+
+	accountPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Errorf("generating account key: %v", err)
+	}
+	account := acme.Account{
+		Contact:              []string{"mailto:you@example.com"},
+		TermsOfServiceAgreed: true,
+		PrivateKey:           accountPrivateKey,
+	}
+	account, err = client.NewAccount(ctx, account)
+	if err != nil {
+		t.Errorf("new account: %v", err)
+		return
+	}
+
+	// Every certificate needs a key.
+	certPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Errorf("generating certificate key: %v", err)
+		return
+	}
+	{
+		_, err := client.ObtainCertificateForSANs(ctx, account, certPrivateKey, []string{"deny.localhost"})
+		if err == nil {
+			t.Errorf("obtaining certificate for 'deny.localhost' domain")
+		} else if !strings.Contains(err.Error(), "urn:ietf:params:acme:error:rejectedIdentifier") {
+			t.Logf("unexpected error: %v", err)
+		}
+	}
+}

caddytest/integration/autohttps_test.go

@@ -1,6 +1,7 @@
 package integration
 
 import (
+	"fmt"
 	"net/http"
 	"testing"
 
@@ -8,69 +9,69 @@ import (
 )
 
 func TestAutoHTTPtoHTTPSRedirectsImplicitPort(t *testing.T) {
-	tester := caddytest.NewTester(t)
+	harness := caddytest.StartHarness(t)
-	tester.InitServer(`
+	harness.LoadConfig(`
 	{
-		admin localhost:2999
+		admin {$TESTING_CADDY_ADMIN_BIND}
 		skip_install_trust
-		http_port     9080
+		http_port     {$TESTING_CADDY_PORT_ONE}
-		https_port    9443
+		https_port    {$TESTING_CADDY_PORT_TWO}
 	}
 	localhost
 	respond "Yahaha! You found me!"
   `, "caddyfile")
 
-	tester.AssertRedirect("http://localhost:9080/", "https://localhost/", http.StatusPermanentRedirect)
+	harness.AssertRedirect(fmt.Sprintf("http://localhost:%d/", harness.Tester().PortOne()), "https://localhost/", http.StatusPermanentRedirect)
 }
 
 func TestAutoHTTPtoHTTPSRedirectsExplicitPortSameAsHTTPSPort(t *testing.T) {
-	tester := caddytest.NewTester(t)
+	harness := caddytest.StartHarness(t)
-	tester.InitServer(`
+	harness.LoadConfig(`
 	{
 		skip_install_trust
-		admin localhost:2999
+		admin {$TESTING_CADDY_ADMIN_BIND}
-		http_port     9080
+		http_port     {$TESTING_CADDY_PORT_ONE}
-		https_port    9443
+		https_port    {$TESTING_CADDY_PORT_TWO}
 	}
-	localhost:9443
+	localhost:{$TESTING_CADDY_PORT_TWO}
 	respond "Yahaha! You found me!"
   `, "caddyfile")
 
-	tester.AssertRedirect("http://localhost:9080/", "https://localhost/", http.StatusPermanentRedirect)
+	harness.AssertRedirect(fmt.Sprintf("http://localhost:%d/", harness.Tester().PortOne()), "https://localhost/", http.StatusPermanentRedirect)
 }
 
 func TestAutoHTTPtoHTTPSRedirectsExplicitPortDifferentFromHTTPSPort(t *testing.T) {
-	tester := caddytest.NewTester(t)
+	harness := caddytest.StartHarness(t)
-	tester.InitServer(`
+	harness.LoadConfig(`
 	{
 		skip_install_trust
-		admin localhost:2999
+		admin {$TESTING_CADDY_ADMIN_BIND}
-		http_port     9080
+		http_port     {$TESTING_CADDY_PORT_ONE}
-		https_port    9443
+		https_port    {$TESTING_CADDY_PORT_TWO}
 	}
 	localhost:1234
 	respond "Yahaha! You found me!"
   `, "caddyfile")
 
-	tester.AssertRedirect("http://localhost:9080/", "https://localhost:1234/", http.StatusPermanentRedirect)
+	harness.AssertRedirect(fmt.Sprintf("http://localhost:%d/", harness.Tester().PortOne()), "https://localhost:1234/", http.StatusPermanentRedirect)
 }
 
 func TestAutoHTTPRedirectsWithHTTPListenerFirstInAddresses(t *testing.T) {
-	tester := caddytest.NewTester(t)
+	harness := caddytest.StartHarness(t)
-	tester.InitServer(`
+	harness.LoadConfig(`
 {
   "admin": {
-	"listen": "localhost:2999"
+	"listen": "{$TESTING_CADDY_ADMIN_BIND}"
   },
   "apps": {
     "http": {
-      "http_port": 9080,
+      "http_port": {$TESTING_CADDY_PORT_ONE},
-      "https_port": 9443,
+      "https_port": {$TESTING_CADDY_PORT_TWO},
       "servers": {
         "ingress_server": {
           "listen": [
-            ":9080",
+            ":{$TESTING_CADDY_PORT_ONE}",
-            ":9443"
+            ":{$TESTING_CADDY_PORT_TWO}"
           ],
           "routes": [
             {
@@ -94,52 +95,52 @@ func TestAutoHTTPRedirectsWithHTTPListenerFirstInAddresses(t *testing.T) {
   }
 }
 `, "json")
-	tester.AssertRedirect("http://localhost:9080/", "https://localhost/", http.StatusPermanentRedirect)
+	harness.AssertRedirect(fmt.Sprintf("http://localhost:%d/", harness.Tester().PortOne()), "https://localhost/", http.StatusPermanentRedirect)
 }
 
 func TestAutoHTTPRedirectsInsertedBeforeUserDefinedCatchAll(t *testing.T) {
-	tester := caddytest.NewTester(t)
+	harness := caddytest.StartHarness(t)
-	tester.InitServer(`
+	harness.LoadConfig(`
 	{
 		skip_install_trust
-		admin localhost:2999
+		admin {$TESTING_CADDY_ADMIN_BIND}
-		http_port     9080
+		http_port     {$TESTING_CADDY_PORT_ONE}
-		https_port    9443
+		https_port    {$TESTING_CADDY_PORT_TWO}
 		local_certs
 	}
-	http://:9080 {
+	http://:{$TESTING_CADDY_PORT_ONE} {
 		respond "Foo"
 	}
-	http://baz.localhost:9080 {
+	http://baz.localhost:{$TESTING_CADDY_PORT_ONE} {
 		respond "Baz"
 	}
 	bar.localhost {
 		respond "Bar"
 	}
   `, "caddyfile")
-	tester.AssertRedirect("http://bar.localhost:9080/", "https://bar.localhost/", http.StatusPermanentRedirect)
+	harness.AssertRedirect(fmt.Sprintf("http://bar.localhost:%d/", harness.Tester().PortOne()), "https://bar.localhost/", http.StatusPermanentRedirect)
-	tester.AssertGetResponse("http://foo.localhost:9080/", 200, "Foo")
+	harness.AssertGetResponse(fmt.Sprintf("http://foo.localhost:%d/", harness.Tester().PortOne()), 200, "Foo")
-	tester.AssertGetResponse("http://baz.localhost:9080/", 200, "Baz")
+	harness.AssertGetResponse(fmt.Sprintf("http://baz.localhost:%d/", harness.Tester().PortOne()), 200, "Baz")
 }
 
 func TestAutoHTTPRedirectsInsertedBeforeUserDefinedCatchAllWithNoExplicitHTTPSite(t *testing.T) {
-	tester := caddytest.NewTester(t)
+	harness := caddytest.StartHarness(t)
-	tester.InitServer(`
+	harness.LoadConfig(`
 	{
 		skip_install_trust
-		admin localhost:2999
+		admin {$TESTING_CADDY_ADMIN_BIND}
-		http_port     9080
+		http_port     {$TESTING_CADDY_PORT_ONE}
-		https_port    9443
+		https_port    {$TESTING_CADDY_PORT_TWO}
 		local_certs
 	}
-	http://:9080 {
+	http://:{$TESTING_CADDY_PORT_ONE} {
 		respond "Foo"
 	}
 	bar.localhost {
 		respond "Bar"
 	}
   `, "caddyfile")
-	tester.AssertRedirect("http://bar.localhost:9080/", "https://bar.localhost/", http.StatusPermanentRedirect)
+	harness.AssertRedirect(fmt.Sprintf("http://bar.localhost:%d/", harness.Tester().PortOne()), "https://bar.localhost/", http.StatusPermanentRedirect)
-	tester.AssertGetResponse("http://foo.localhost:9080/", 200, "Foo")
+	harness.AssertGetResponse(fmt.Sprintf("http://foo.localhost:%d/", harness.Tester().PortOne()), 200, "Foo")
-	tester.AssertGetResponse("http://baz.localhost:9080/", 200, "Foo")
+	harness.AssertGetResponse(fmt.Sprintf("http://baz.localhost:%d/", harness.Tester().PortOne()), 200, "Foo")
 }

caddytest/integration/caddyfile_adapt/acme_server_custom_challenges.caddyfiletest

@@ -0,0 +1,65 @@
+{
+	pki {
+		ca custom-ca {
+			name "Custom CA"
+		}
+	}
+}
+
+acme.example.com {
+	acme_server {
+		ca custom-ca
+		challenges dns-01
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"acme.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"ca": "custom-ca",
+													"challenges": [
+														"dns-01"
+													],
+													"handler": "acme_server"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"pki": {
+			"certificate_authorities": {
+				"custom-ca": {
+					"name": "Custom CA"
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/acme_server_default_challenges.caddyfiletest

@@ -0,0 +1,62 @@
+{
+	pki {
+		ca custom-ca {
+			name "Custom CA"
+		}
+	}
+}
+
+acme.example.com {
+	acme_server {
+		ca custom-ca
+		challenges
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"acme.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"ca": "custom-ca",
+													"handler": "acme_server"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"pki": {
+			"certificate_authorities": {
+				"custom-ca": {
+					"name": "Custom CA"
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/acme_server_multi_custom_challenges.caddyfiletest

@@ -0,0 +1,66 @@
+{
+	pki {
+		ca custom-ca {
+			name "Custom CA"
+		}
+	}
+}
+
+acme.example.com {
+	acme_server {
+		ca custom-ca
+		challenges dns-01 http-01
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"acme.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"ca": "custom-ca",
+													"challenges": [
+														"dns-01",
+														"http-01"
+													],
+													"handler": "acme_server"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"pki": {
+			"certificate_authorities": {
+				"custom-ca": {
+					"name": "Custom CA"
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/acme_server_sign_with_root.caddyfiletest

@@ -0,0 +1,67 @@
+{
+	pki {
+		ca internal {
+			name "Internal"
+			root_cn "Internal Root Cert"
+			intermediate_cn "Internal Intermediate Cert"
+		}
+    }
+}
+
+acme.example.com {
+	acme_server {
+		ca internal
+		sign_with_root
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"acme.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"ca": "internal",
+													"handler": "acme_server",
+													"sign_with_root": true
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"pki": {
+			"certificate_authorities": {
+				"internal": {
+					"name": "Internal",
+					"root_common_name": "Internal Root Cert",
+					"intermediate_common_name": "Internal Intermediate Cert"
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/enable_tls_for_catch_all_site.caddyfiletest

@@ -0,0 +1,37 @@
+:8443 {
+	tls internal {
+		on_demand
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8443"
+					],
+					"tls_connection_policies": [
+						{}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"issuers": [
+							{
+								"module": "internal"
+							}
+						],
+						"on_demand": true
+					}
+				]
+			}
+		}
+	}
+}
+

caddytest/integration/caddyfile_adapt/encode_options.caddyfiletest

@@ -11,6 +11,7 @@ encode gzip zstd {
 		header Content-Type application/xhtml+xml*
 		header Content-Type application/atom+xml*
 		header Content-Type application/rss+xml*
+		header Content-Type application/wasm*
 		header Content-Type image/svg+xml*
 	}
 }
@@ -47,6 +48,7 @@ encode {
 												"application/xhtml+xml*",
 												"application/atom+xml*",
 												"application/rss+xml*",
+												"application/wasm*",
 												"image/svg+xml*"
 											]
 										},

caddytest/integration/caddyfile_adapt/error_multi_site_blocks.caddyfiletest

@@ -0,0 +1,245 @@
+foo.localhost {
+	root * /srv
+	error /private* "Unauthorized" 410
+	error /fivehundred* "Internal Server Error" 500
+
+	handle_errors 5xx {
+		respond "Error In range [500 .. 599]"
+	}
+	handle_errors 410 {
+		respond "404 or 410 error"
+	}
+}
+
+bar.localhost {
+	root * /srv
+	error /private* "Unauthorized" 410
+	error /fivehundred* "Internal Server Error" 500
+
+	handle_errors 5xx {
+		respond "Error In range [500 .. 599] from second site"
+	}
+	handle_errors 410 {
+		respond "404 or 410 error from second site"
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"foo.localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "vars",
+													"root": "/srv"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Internal Server Error",
+													"handler": "error",
+													"status_code": 500
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/fivehundred*"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Unauthorized",
+													"handler": "error",
+													"status_code": 410
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/private*"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"bar.localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "vars",
+													"root": "/srv"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Internal Server Error",
+													"handler": "error",
+													"status_code": 500
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/fivehundred*"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Unauthorized",
+													"handler": "error",
+													"status_code": 410
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/private*"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"errors": {
+						"routes": [
+							{
+								"match": [
+									{
+										"host": [
+											"foo.localhost"
+										]
+									}
+								],
+								"handle": [
+									{
+										"handler": "subroute",
+										"routes": [
+											{
+												"handle": [
+													{
+														"body": "404 or 410 error",
+														"handler": "static_response"
+													}
+												],
+												"match": [
+													{
+														"expression": "{http.error.status_code} in [410]"
+													}
+												]
+											},
+											{
+												"handle": [
+													{
+														"body": "Error In range [500 .. 599]",
+														"handler": "static_response"
+													}
+												],
+												"match": [
+													{
+														"expression": "{http.error.status_code} \u003e= 500 \u0026\u0026 {http.error.status_code} \u003c= 599"
+													}
+												]
+											}
+										]
+									}
+								],
+								"terminal": true
+							},
+							{
+								"match": [
+									{
+										"host": [
+											"bar.localhost"
+										]
+									}
+								],
+								"handle": [
+									{
+										"handler": "subroute",
+										"routes": [
+											{
+												"handle": [
+													{
+														"body": "404 or 410 error from second site",
+														"handler": "static_response"
+													}
+												],
+												"match": [
+													{
+														"expression": "{http.error.status_code} in [410]"
+													}
+												]
+											},
+											{
+												"handle": [
+													{
+														"body": "Error In range [500 .. 599] from second site",
+														"handler": "static_response"
+													}
+												],
+												"match": [
+													{
+														"expression": "{http.error.status_code} \u003e= 500 \u0026\u0026 {http.error.status_code} \u003c= 599"
+													}
+												]
+											}
+										]
+									}
+								],
+								"terminal": true
+							}
+						]
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/error_range_codes.caddyfiletest

@@ -0,0 +1,120 @@
+{
+	http_port 3010
+}
+localhost:3010 {
+	root * /srv
+	error /private* "Unauthorized" 410
+	error /hidden* "Not found" 404
+
+	handle_errors 4xx {
+		respond "Error in the [400 .. 499] range"
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"http_port": 3010,
+			"servers": {
+				"srv0": {
+					"listen": [
+						":3010"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "vars",
+													"root": "/srv"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Unauthorized",
+													"handler": "error",
+													"status_code": 410
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/private*"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Not found",
+													"handler": "error",
+													"status_code": 404
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/hidden*"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"errors": {
+						"routes": [
+							{
+								"match": [
+									{
+										"host": [
+											"localhost"
+										]
+									}
+								],
+								"handle": [
+									{
+										"handler": "subroute",
+										"routes": [
+											{
+												"handle": [
+													{
+														"body": "Error in the [400 .. 499] range",
+														"handler": "static_response"
+													}
+												],
+												"match": [
+													{
+														"expression": "{http.error.status_code} \u003e= 400 \u0026\u0026 {http.error.status_code} \u003c= 499"
+													}
+												]
+											}
+										]
+									}
+								],
+								"terminal": true
+							}
+						]
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/error_range_simple_codes.caddyfiletest

@@ -0,0 +1,153 @@
+{
+	http_port 2099
+}
+localhost:2099 {
+	root * /srv
+	error /private* "Unauthorized" 410
+	error /threehundred* "Moved Permanently" 301
+	error /internalerr* "Internal Server Error" 500
+
+	handle_errors 500 3xx {
+		respond "Error code is equal to 500 or in the [300..399] range"
+	}
+	handle_errors 4xx {
+		respond "Error in the [400 .. 499] range"
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"http_port": 2099,
+			"servers": {
+				"srv0": {
+					"listen": [
+						":2099"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "vars",
+													"root": "/srv"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Moved Permanently",
+													"handler": "error",
+													"status_code": 301
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/threehundred*"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Internal Server Error",
+													"handler": "error",
+													"status_code": 500
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/internalerr*"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Unauthorized",
+													"handler": "error",
+													"status_code": 410
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/private*"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"errors": {
+						"routes": [
+							{
+								"match": [
+									{
+										"host": [
+											"localhost"
+										]
+									}
+								],
+								"handle": [
+									{
+										"handler": "subroute",
+										"routes": [
+											{
+												"handle": [
+													{
+														"body": "Error in the [400 .. 499] range",
+														"handler": "static_response"
+													}
+												],
+												"match": [
+													{
+														"expression": "{http.error.status_code} \u003e= 400 \u0026\u0026 {http.error.status_code} \u003c= 499"
+													}
+												]
+											},
+											{
+												"handle": [
+													{
+														"body": "Error code is equal to 500 or in the [300..399] range",
+														"handler": "static_response"
+													}
+												],
+												"match": [
+													{
+														"expression": "{http.error.status_code} \u003e= 300 \u0026\u0026 {http.error.status_code} \u003c= 399 || {http.error.status_code} in [500]"
+													}
+												]
+											}
+										]
+									}
+								],
+								"terminal": true
+							}
+						]
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/error_simple_codes.caddyfiletest

@@ -0,0 +1,120 @@
+{
+	http_port 3010
+}
+localhost:3010 {
+	root * /srv
+	error /private* "Unauthorized" 410
+	error /hidden* "Not found" 404
+
+	handle_errors 404 410 {
+		respond "404 or 410 error"
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"http_port": 3010,
+			"servers": {
+				"srv0": {
+					"listen": [
+						":3010"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "vars",
+													"root": "/srv"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Unauthorized",
+													"handler": "error",
+													"status_code": 410
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/private*"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Not found",
+													"handler": "error",
+													"status_code": 404
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/hidden*"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"errors": {
+						"routes": [
+							{
+								"match": [
+									{
+										"host": [
+											"localhost"
+										]
+									}
+								],
+								"handle": [
+									{
+										"handler": "subroute",
+										"routes": [
+											{
+												"handle": [
+													{
+														"body": "404 or 410 error",
+														"handler": "static_response"
+													}
+												],
+												"match": [
+													{
+														"expression": "{http.error.status_code} in [404, 410]"
+													}
+												]
+											}
+										]
+									}
+								],
+								"terminal": true
+							}
+						]
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/error_sort.caddyfiletest

@@ -0,0 +1,148 @@
+{
+	http_port 2099
+}
+localhost:2099 {
+	root * /srv
+	error /private* "Unauthorized" 410
+	error /hidden* "Not found" 404
+	error /internalerr* "Internal Server Error" 500
+
+	handle_errors {
+		respond "Fallback route: code outside the [400..499] range"
+	}
+	handle_errors 4xx {
+		respond "Error in the [400 .. 499] range"
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"http_port": 2099,
+			"servers": {
+				"srv0": {
+					"listen": [
+						":2099"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "vars",
+													"root": "/srv"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Internal Server Error",
+													"handler": "error",
+													"status_code": 500
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/internalerr*"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Unauthorized",
+													"handler": "error",
+													"status_code": 410
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/private*"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Not found",
+													"handler": "error",
+													"status_code": 404
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/hidden*"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"errors": {
+						"routes": [
+							{
+								"match": [
+									{
+										"host": [
+											"localhost"
+										]
+									}
+								],
+								"handle": [
+									{
+										"handler": "subroute",
+										"routes": [
+											{
+												"handle": [
+													{
+														"body": "Error in the [400 .. 499] range",
+														"handler": "static_response"
+													}
+												],
+												"match": [
+													{
+														"expression": "{http.error.status_code} \u003e= 400 \u0026\u0026 {http.error.status_code} \u003c= 499"
+													}
+												]
+											},
+											{
+												"handle": [
+													{
+														"body": "Fallback route: code outside the [400..499] range",
+														"handler": "static_response"
+													}
+												]
+											}
+										]
+									}
+								],
+								"terminal": true
+							}
+						]
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/expression_quotes.caddyfiletest

@@ -1,3 +1,7 @@
+(snippet) {
+	@g `{http.error.status_code} == 404`
+}
+
 example.com
 
 @a expression {http.error.status_code} == 400
@@ -14,6 +18,12 @@ abort @d
 
 @e expression `{http.error.status_code} == 404`
 abort @e
+
+@f `{http.error.status_code} == 404`
+abort @f
+
+import snippet
+abort @g
 ----------
 {
 	"apps": {
@@ -84,7 +94,10 @@ abort @e
 											],
 											"match": [
 												{
-													"expression": "{http.error.status_code} == 403"
+													"expression": {
+														"expr": "{http.error.status_code} == 403",
+														"name": "d"
+													}
 												}
 											]
 										},
@@ -97,7 +110,42 @@ abort @e
 											],
 											"match": [
 												{
-													"expression": "{http.error.status_code} == 404"
+													"expression": {
+														"expr": "{http.error.status_code} == 404",
+														"name": "e"
+													}
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": {
+														"expr": "{http.error.status_code} == 404",
+														"name": "f"
+													}
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": {
+														"expr": "{http.error.status_code} == 404",
+														"name": "g"
+													}
 												}
 											]
 										}

caddytest/integration/caddyfile_adapt/file_server_etag_file_extensions.caddyfiletest

@@ -0,0 +1,40 @@
+:8080 {
+	root * ./
+	file_server {
+		etag_file_extensions .b3sum .sha256
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8080"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "vars",
+									"root": "./"
+								},
+								{
+									"etag_file_extensions": [
+										".b3sum",
+										".sha256"
+									],
+									"handler": "file_server",
+									"hide": [
+										"./Caddyfile"
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options.caddyfiletest

@@ -69,11 +69,14 @@
 					}
 				],
 				"on_demand": {
+					"permission": {
+						"endpoint": "https://example.com",
+						"module": "http"
+					},
 					"rate_limit": {
 						"interval": 30000000000,
 						"burst": 20
-					},
+					}
-					"ask": "https://example.com"
 				}
 			},
 			"disable_ocsp_stapling": true

caddytest/integration/caddyfile_adapt/global_options_acme.caddyfiletest

@@ -63,6 +63,14 @@
 						"issuers": [
 							{
 								"ca": "https://example.com",
+								"challenges": {
+									"http": {
+										"alternate_port": 8080
+									},
+									"tls-alpn": {
+										"alternate_port": 8443
+									}
+								},
 								"email": "test@example.com",
 								"external_account": {
 									"key_id": "4K2scIVbBpNd-78scadB2g",
@@ -78,11 +86,14 @@
 					}
 				],
 				"on_demand": {
+					"permission": {
+						"endpoint": "https://example.com",
+						"module": "http"
+					},
 					"rate_limit": {
 						"interval": 30000000000,
 						"burst": 20
-					},
+					}
-					"ask": "https://example.com"
 				},
 				"ocsp_interval": 172800000000000,
 				"renew_interval": 86400000000000,

caddytest/integration/caddyfile_adapt/global_options_admin.caddyfiletest

@@ -71,11 +71,14 @@
 					}
 				],
 				"on_demand": {
+					"permission": {
+						"endpoint": "https://example.com",
+						"module": "http"
+					},
 					"rate_limit": {
 						"interval": 30000000000,
 						"burst": 20
-					},
+					}
-					"ask": "https://example.com"
 				}
 			}
 		}

caddytest/integration/caddyfile_adapt/global_options_preferred_chains.caddyfiletest

@@ -40,12 +40,6 @@ example.com
 								"preferred_chains": {
 									"smallest": true
 								}
-							},
-							{
-								"module": "zerossl",
-								"preferred_chains": {
-									"smallest": true
-								}
 							}
 						]
 					}

caddytest/integration/caddyfile_adapt/global_server_options_single.caddyfiletest

@@ -11,6 +11,7 @@
 			idle 30s
 		}
 		max_header_size 100MB
+		enable_full_duplex
 		log_credentials
 		protocols h1 h2 h2c h3
 		strict_sni_host
@@ -45,6 +46,7 @@ foo.com {
 					"write_timeout": 30000000000,
 					"idle_timeout": 30000000000,
 					"max_header_bytes": 100000000,
+					"enable_full_duplex": true,
 					"routes": [
 						{
 							"match": [

caddytest/integration/caddyfile_adapt/http_valid_directive_like_site_address.caddyfiletest

@@ -0,0 +1,46 @@
+http://handle {
+	file_server
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"handle"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "file_server",
+													"hide": [
+														"./Caddyfile"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/import_args_snippet.caddyfiletest

@@ -72,8 +72,12 @@ b.example.com {
 					],
 					"logs": {
 						"logger_names": {
-							"a.example.com": "log0",
+							"a.example.com": [
-							"b.example.com": "log1"
+								"log0"
+							],
+							"b.example.com": [
+								"log1"
+							]
 						}
 					}
 				}