Merge branch 'master' into proxy-res-templates

Fixes #5037

.editorconfig

@@ -0,0 +1,5 @@
+[*]
+end_of_line = lf
+
+[caddytest/integration/caddyfile_adapt/*.txt]
+indent_style = tab

.gitattributes

@@ -0,0 +1 @@
+*.go text eol=lf

.github/SECURITY.md

@@ -2,9 +2,6 @@
 
 The Caddy project would like to make sure that it stays on top of all practically-exploitable vulnerabilities.
 
-Some security problems are more the result of interplay between different components of the Web, rather than a vulnerability in the web server itself. Please report only vulnerabilities in the web server itself, as we cannot coerce the rest of the Web to be fixed (for example, we do not consider IP spoofing or BGP hijacks a vulnerability in the Caddy web server).
-
-Please note that we consider publicly-registered domain names to be public information. This necessary in order to maintain the integrity of certificate transparency, public DNS, and other public trust systems.
 
 ## Supported Versions
 
@@ -14,11 +11,46 @@ Please note that we consider publicly-registered domain names to be public infor
 | 1.x     | :x:                |
 | < 1.x   | :x:                |
 
+
+## Acceptable Scope
+
+A security report must demonstrate a security bug in the source code from this repository.
+
+Some security problems are the result of interplay between different components of the Web, rather than a vulnerability in the web server itself. Please only report vulnerabilities in the web server itself, as we cannot coerce the rest of the Web to be fixed (for example, we do not consider IP spoofing, BGP hijacks, or missing/misconfigured HTTP headers a vulnerability in the Caddy web server).
+
+Vulnerabilities caused by misconfigurations are out of scope. Yes, it is entirely possible to craft and use a configuration that is unsafe, just like with every other web server; we recommend against doing that.
+
+We do not accept reports if the steps imply or require a compromised system or third-party software, as we cannot control those. We expect that users secure their own systems and keep all their software patched. For example, if untrusted users are able to upload/write/host arbitrary files in the web root directory, it is NOT a security bug in Caddy if those files get served to clients; however, it _would_ be a valid report if a bug in Caddy's source code unintentionally gave unauthorized users the ability to upload unsafe files or delete files without relying on an unpatched system or piece of software.
+
+Client-side exploits are out of scope. In other words, it is not a bug in Caddy if the web browser does something unsafe, even if the downloaded content was served by Caddy. (Those kinds of exploits can generally be mitigated by proper configuration of HTTP headers.) As a general rule, the content served by Caddy is not considered in scope because content is configurable by the site owner or the associated web application.
+
+Security bugs in code dependencies are out of scope. Instead, if a dependency has patched a relevant security bug, please feel free to open a public issue or pull request to update that dependency in our code.
+
+
 ## Reporting a Vulnerability
 
-Please email Matt Holt (the author) directly: matt [at] lightcodelabs [dot com].
+We get a lot of difficult reports that turn out to be invalid. Clear, obvious reports tend to be the most credible (but are also rare).
 
-We'll need enough information to verify the bug and make a patch. It will speed things up if you suggest a working patch, such as a code diff, and explain why and how it works. Reports that are not actionable, do not contain enough information, are too pushy/demanding, or are not able to convince us that it is a viable and practical attack on the web server itself may be deferred to a later time or possibly ignored, resources permitting. Priority will be given to credible, responsible reports that are constructive, specific, and actionable. Thank you for understanding.
+First please ensure your report falls within the accepted scope of security bugs (above).
+
+We'll need enough information to verify the bug and make a patch. To speed things up, please include:
+
+- Most minimal possible config (without redactions!)
+- Command(s)
+- Precise HTTP requests (`curl -v` and its output please)
+- Full log output (please enable debug mode)
+- Specific minimal steps to reproduce the issue from scratch
+- A working patch
+
+Please DO NOT use containers, VMs, cloud instances or services, or any other complex infrastructure in your steps. Always prefer `curl` instead of web browsers.
+
+We consider publicly-registered domain names to be public information. This necessary in order to maintain the integrity of certificate transparency, public DNS, and other public trust systems. Do not redact domain names from your reports. The actual content of your domain name affects Caddy's behavior, so we need the exact domain name(s) to reproduce with, or your report will be ignored.
+
+It will speed things up if you suggest a working patch, such as a code diff, and explain why and how it works. Reports that are not actionable, do not contain enough information, are too pushy/demanding, or are not able to convince us that it is a viable and practical attack on the web server itself may be deferred to a later time or possibly ignored, depending on available resources. Priority will be given to credible, responsible reports that are constructive, specific, and actionable. (We get a lot of invalid reports.) Thank you for understanding.
+
+When you are ready, please email Matt Holt (the author) directly: matt at dyanim dot com.
+
+Please don't encrypt the email body. It only makes the process more complicated.
 
 Please also understand that due to our nature as an open source project, we do not have a budget to award security bounties. We can only thank you.
 

.github/workflows/ci.yml

@@ -19,12 +19,20 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
-        go: [ '1.15', '1.16' ]
+        go: [ '1.18', '1.19' ]
+
+        include:
+        # Set the minimum Go patch version for the given Go minor
+        # Usable via ${{ matrix.GO_SEMVER }}
+        - go: '1.18'
+          GO_SEMVER: '~1.18.4'
+
+        - go: '1.19'
+          GO_SEMVER: '~1.19.0'
 
         # Set some variables per OS, usable via ${{ matrix.VAR }}
         # CADDY_BIN_PATH: the path to the compiled Caddy binary, for artifact publishing
         # SUCCESS: the typical value for $? per OS (Windows/pwsh returns 'True')
-        include:
         - os: ubuntu-latest
           CADDY_BIN_PATH: ./cmd/caddy/caddy
           SUCCESS: 0
@@ -41,12 +49,13 @@ jobs:
 
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
-        go-version: ${{ matrix.go }}
+        go-version: ${{ matrix.GO_SEMVER }}
+        check-latest: true
 
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # These tools would be useful if we later decide to reinvestigate
     # publishing test/coverage reports to some tool for easier consumption
@@ -69,12 +78,20 @@ jobs:
         printf "Git version: $(git version)\n\n"
         # Calculate the short SHA1 hash of the git commit
         echo "::set-output name=short_sha::$(git rev-parse --short HEAD)"
-        echo "::set-output name=go_cache::$(go env GOCACHE)"
 
     - name: Cache the build cache
       uses: actions/cache@v2
       with:
-        path: ${{ steps.vars.outputs.go_cache }}
+        # In order:
+        # * Module download cache
+        # * Build cache (Linux)
+        # * Build cache (Mac)
+        # * Build cache (Windows)
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
+          ~/Library/Caches/go-build
+          ~\AppData\Local\go-build
         key: ${{ runner.os }}-${{ matrix.go }}-go-ci-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-${{ matrix.go }}-go-ci
@@ -130,7 +147,7 @@ jobs:
     continue-on-error: true  # August 2020: s390x VM is down due to weather and power issues
     steps:
       - name: Checkout code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Run Tests
         run: |
           mkdir -p ~/.ssh && echo -e "${SSH_KEY//_/\\n}" > ~/.ssh/id_ecdsa && chmod og-rwx ~/.ssh/id_ecdsa
@@ -155,7 +172,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+      
       - uses: goreleaser/goreleaser-action@v2
         with:
           version: latest

.github/workflows/cross-build.yml

@@ -16,14 +16,22 @@ jobs:
       fail-fast: false
       matrix:
         goos: ['android', 'linux', 'solaris', 'illumos', 'dragonfly', 'freebsd', 'openbsd', 'plan9', 'windows', 'darwin', 'netbsd']
-        go: [ '1.15', '1.16' ]
+        go: [ '1.19' ]
+
+        include:
+        # Set the minimum Go patch version for the given Go minor
+        # Usable via ${{ matrix.GO_SEMVER }}
+        - go: '1.19'
+          GO_SEMVER: '~1.19.0'
+
     runs-on: ubuntu-latest
     continue-on-error: true
     steps:
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
-          go-version: ${{ matrix.go }}
+          go-version: ${{ matrix.GO_SEMVER }}
+          check-latest: true
 
       - name: Print Go version and environment
         id: vars
@@ -34,18 +42,22 @@ jobs:
           go env
           printf "\n\nSystem environment:\n\n"
           env
-          echo "::set-output name=go_cache::$(go env GOCACHE)"
 
       - name: Cache the build cache
         uses: actions/cache@v2
         with:
-          path: ${{ steps.vars.outputs.go_cache }}
+          # In order:
+          # * Module download cache
+          # * Build cache (Linux)
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
           key: cross-build-go${{ matrix.go }}-${{ matrix.goos }}-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             cross-build-go${{ matrix.go }}-${{ matrix.goos }}
 
       - name: Checkout code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Run Build
         env:

.github/workflows/lint.yml

@@ -14,12 +14,22 @@ jobs:
   # From https://github.com/golangci/golangci-lint-action
   golangci:
     name: lint
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v2
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
         with:
-          version: v1.31
+          go-version: '~1.18.4'
+          check-latest: true
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.47
+          # Windows times out frequently after about 5m50s if we don't set a longer timeout.
+          args: --timeout 10m
           # Optional: show only new issues if it's a pull request. The default value is `false`.
           # only-new-issues: true

.github/workflows/release.yml

@@ -11,22 +11,37 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        go: [ '1.16' ]
+        go: [ '1.19' ]
+
+        include:
+        # Set the minimum Go patch version for the given Go minor
+        # Usable via ${{ matrix.GO_SEMVER }}
+        - go: '1.19'
+          GO_SEMVER: '~1.19.0'
+
     runs-on: ${{ matrix.os }}
+    # https://github.com/sigstore/cosign/issues/1258#issuecomment-1002251233
+    # https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect#adding-permissions-settings
+    permissions:
+      id-token: write
+      # https://docs.github.com/en/rest/overview/permissions-required-for-github-apps#permission-on-contents
+      # "Releases" is part of `contents`, so it needs the `write`
+      contents: write
 
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
-        go-version: ${{ matrix.go }}
+        go-version: ${{ matrix.GO_SEMVER }}
+        check-latest: true
 
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
 
     # Force fetch upstream tags -- because 65 minutes
-    # tl;dr: actions/checkout@v2 runs this line:
+    # tl;dr: actions/checkout@v3 runs this line:
     #   git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +ebc278ec98bb24f2852b61fde2a9bf2e3d83818b:refs/tags/
     # which makes its own local lightweight tag, losing all the annotations in the process. Our earlier script ran:
     #   git fetch --prune --unshallow
@@ -48,7 +63,6 @@ jobs:
         env
         echo "::set-output name=version_tag::${GITHUB_REF/refs\/tags\//}"
         echo "::set-output name=short_sha::$(git rev-parse --short HEAD)"
-        echo "::set-output name=go_cache::$(go env GOCACHE)"
 
         # Add "pip install" CLI tools to PATH
         echo ~/.local/bin >> $GITHUB_PATH
@@ -83,11 +97,23 @@ jobs:
     - name: Cache the build cache
       uses: actions/cache@v2
       with:
-        path: ${{ steps.vars.outputs.go_cache }}
+        # In order:
+        # * Module download cache
+        # * Build cache (Linux)
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
         key: ${{ runner.os }}-go${{ matrix.go }}-release-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go${{ matrix.go }}-release
-
+    - name: Install Cosign
+      uses: sigstore/cosign-installer@main
+    - name: Cosign version
+      run: cosign version
+    - name: Install Syft
+      uses: anchore/sbom-action/download-syft@main
+    - name: Syft version
+      run: syft version
     # GoReleaser will take care of publishing those artifacts into the release
     - name: Run GoReleaser
       uses: goreleaser/goreleaser-action@v2
@@ -97,9 +123,10 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         TAG: ${{ steps.vars.outputs.version_tag }}
+        COSIGN_EXPERIMENTAL: 1
 
     # Only publish on non-special tags (e.g. non-beta)
-    # We will continue to push to Gemfury for the forseeable future, although
+    # We will continue to push to Gemfury for the foreseeable future, although
     # Cloudsmith is probably better, to not break things for existing users of Gemfury.
     # See https://gemfury.com/caddy/deb:caddy
     - name: Publish .deb to Gemfury

.gitignore

@@ -1,6 +1,7 @@
 _gitignore/
 *.log
 Caddyfile
+Caddyfile.*
 !caddyfile/
 
 # artifacts from pprof tooling

.golangci.yml

@@ -1,6 +1,6 @@
 linters-settings:
   errcheck:
-    ignore: fmt:.*,io/ioutil:^Read.*,go.uber.org/zap/zapcore:^Add.*
+    ignore: fmt:.*,go.uber.org/zap/zapcore:^Add.*
     ignoretests: true
 
 linters:

.goreleaser.yml

@@ -6,8 +6,7 @@ before:
     # subsequently causes gorleaser to refuse running.
     - mkdir -p caddy-build
     - cp cmd/caddy/main.go caddy-build/main.go
-    - cp ./go.mod caddy-build/go.mod
-    - sed -i.bkp 's|github.com/caddyserver/caddy/v2|caddy|g' ./caddy-build/go.mod
+    - /bin/sh -c 'cd ./caddy-build && go mod init caddy'
     # GoReleaser doesn't seem to offer {{.Tag}} at this stage, so we have to embed it into the env
     # so we run: TAG=$(git describe --abbrev=0) goreleaser release --rm-dist --skip-publish --skip-validate
     - go mod edit -require=github.com/caddyserver/caddy/v2@{{.Env.TAG}} ./caddy-build/go.mod
@@ -15,7 +14,11 @@ before:
     # run `go mod tidy`. The `/bin/sh -c '...'` is because goreleaser can't find cd in PATH without shell invocation.
     - /bin/sh -c 'cd ./caddy-build && go mod tidy'
     - git clone --depth 1 https://github.com/caddyserver/dist caddy-dist
+    - mkdir -p caddy-dist/man
     - go mod download
+    - go run cmd/caddy/main.go manpage --directory ./caddy-dist/man
+    - gzip -r ./caddy-dist/man/
+    - /bin/sh -c 'go run cmd/caddy/main.go completion bash > ./caddy-dist/scripts/bash-completion'
 
 builds:
 - env:
@@ -36,9 +39,9 @@ builds:
   - s390x
   - ppc64le
   goarm:
-  - 5
-  - 6
-  - 7
+  - "5"
+  - "6"
+  - "7"
   ignore:
     - goos: darwin
       goarch: arm
@@ -56,12 +59,25 @@ builds:
       goarch: s390x
     - goos: freebsd
       goarch: arm
-      goarm: 5
+      goarm: "5"
   flags:
   - -trimpath
+  - -mod=readonly
   ldflags:
   - -s -w
-
+signs:
+  - cmd: cosign
+    signature: "${artifact}.sig"
+    certificate: '{{ trimsuffix .Env.artifact ".tar.gz" }}.pem'
+    args: ["sign-blob", "--output-signature=${signature}", "--output-certificate", "${certificate}", "${artifact}"]
+    artifacts: all
+sboms:
+  - artifacts: binary
+    # defaults to
+    # documents:
+    # - "{{ .Binary }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}.sbom"
+    cmd: syft
+    args: ["$artifact", "--file", "${document}", "--output", "cyclonedx-json"]
 archives:
   - format_overrides:
       - goos: windows
@@ -75,11 +91,11 @@ nfpms:
   - id: default
     package_name: caddy
 
-    vendor: Light Code Labs
+    vendor: Dyanim
     homepage: https://caddyserver.com
     maintainer: Matthew Holt <mholt@users.noreply.github.com>
     description: |
-      Powerful, enterprise-ready, open source web server with automatic HTTPS written in Go
+      Caddy - Powerful, enterprise-ready, open source web server with automatic HTTPS written in Go
     license: Apache 2.0
 
     formats:
@@ -97,13 +113,16 @@ nfpms:
       - src: ./caddy-dist/welcome/index.html
         dst: /usr/share/caddy/index.html
       
-      - src: ./caddy-dist/scripts/completions/bash-completion
+      - src: ./caddy-dist/scripts/bash-completion
         dst: /etc/bash_completion.d/caddy
     
       - src: ./caddy-dist/config/Caddyfile
         dst: /etc/caddy/Caddyfile
         type: config
 
+      - src: ./caddy-dist/man/*
+        dst: /usr/share/man/man8/
+
     scripts:
       postinstall: ./caddy-dist/scripts/postinstall.sh
       preremove: ./caddy-dist/scripts/preremove.sh

README.md

@@ -57,25 +57,25 @@
 	- Multi-issuer fallback
 - **Stays up when other servers go down** due to TLS/OCSP/certificate-related issues
 - **Production-ready** after serving trillions of requests and managing millions of TLS certificates
-- **Scales to tens of thousands of sites** ... and probably more
-- **HTTP/1.1, HTTP/2, and experimental HTTP/3** support
+- **Scales to hundreds of thousands of sites** as proven in production
+- **HTTP/1.1, HTTP/2, and HTTP/3** supported all by default
 - **Highly extensible** [modular architecture](https://caddyserver.com/docs/architecture) lets Caddy do anything without bloat
 - **Runs anywhere** with **no external dependencies** (not even libc)
 - Written in Go, a language with higher **memory safety guarantees** than other servers
 - Actually **fun to use**
-- So, so much more to [discover](https://caddyserver.com/v2)
+- So much more to [discover](https://caddyserver.com/v2)
 
 ## Install
 
-The simplest, cross-platform way is to download from [GitHub Releases](https://github.com/caddyserver/caddy/releases) and place the executable file in your PATH.
+The simplest, cross-platform way to get started is to download Caddy from [GitHub Releases](https://github.com/caddyserver/caddy/releases) and place the executable file in your PATH.
 
-For other install options, see https://caddyserver.com/docs/install.
+See [our online documentation](https://caddyserver.com/docs/install) for other install instructions.
 
 ## Build from source
 
 Requirements:
 
-- [Go 1.15 or newer](https://golang.org/dl/)
+- [Go 1.18 or newer](https://golang.org/dl/)
 
 ### For development
  
@@ -164,9 +164,9 @@ The docs are also open source. You can contribute to them here: https://github.c
 
 ## Getting help
 
-- We **strongly recommend** that all professionals or companies using Caddy get a support contract through [Ardan Labs](https://www.ardanlabs.com/my/contact-us?dd=caddy) before help is needed.
+- We advise companies using Caddy to secure a support contract through [Ardan Labs](https://www.ardanlabs.com/my/contact-us?dd=caddy) before help is needed.
 
-- A [sponsorship](https://github.com/sponsors/mholt) goes a long way! If Caddy is benefitting your company, please consider a sponsorship! This not only helps fund full-time work to ensure the longevity of the project, it's also a great look for your company to your customers and potential customers!
+- A [sponsorship](https://github.com/sponsors/mholt) goes a long way! We can offer private help to sponsors. If Caddy is benefitting your company, please consider a sponsorship. This not only helps fund full-time work to ensure the longevity of the project, it provides your company the resources, support, and discounts you need; along with being a great look for your company to your customers and potential customers!
 
 - Individuals can exchange help for free on our community forum at https://caddy.community. Remember that people give help out of their spare time and good will. The best way to get help is to give it first!
 
@@ -176,6 +176,8 @@ Please use our [issue tracker](https://github.com/caddyserver/caddy/issues) only
 
 ## About
 
+Matthew Holt began developing Caddy in 2014 while studying computer science at Brigham Young University. (The name "Caddy" was chosen because this software helps with the tedious, mundane tasks of serving the Web, and is also a single place for multiple things to be organized together.) It soon became the first web server to use HTTPS automatically and by default, and now has hundreds of contributors and has served trillions of HTTPS requests.
+
 **The name "Caddy" is trademarked.** The name of the software is "Caddy", not "Caddy Server" or "CaddyServer". Please call it "Caddy" or, if you wish to clarify, "the Caddy web server". Caddy is a registered trademark of Stack Holdings GmbH.
 
 - _Project on Twitter: [@caddyserver](https://twitter.com/caddyserver)_

admin.go

@@ -25,8 +25,9 @@ import (
 	"errors"
 	"expvar"
 	"fmt"
+	"hash"
+	"hash/fnv"
 	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/pprof"
@@ -39,10 +40,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/caddyserver/caddy/v2/notify"
 	"github.com/caddyserver/certmagic"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 )
 
 // AdminConfig configures Caddy's API endpoint, which is used
@@ -92,6 +93,10 @@ type AdminConfig struct {
 	//
 	// EXPERIMENTAL: This feature is subject to change.
 	Remote *RemoteAdmin `json:"remote,omitempty"`
+
+	// Holds onto the routers so that we can later provision them
+	// if they require provisioning.
+	routers []AdminRouter
 }
 
 // ConfigSettings configures the management of configuration.
@@ -101,14 +106,26 @@ type ConfigSettings struct {
 	// are not persisted; only configs that are pushed to Caddy get persisted.
 	Persist *bool `json:"persist,omitempty"`
 
-	// Loads a configuration to use. This is helpful if your configs are
-	// managed elsewhere, and you want Caddy to pull its config dynamically
+	// Loads a new configuration. This is helpful if your configs are
+	// managed elsewhere and you want Caddy to pull its config dynamically
 	// when it starts. The pulled config completely replaces the current
 	// one, just like any other config load. It is an error if a pulled
-	// config is configured to pull another config.
+	// config is configured to pull another config without a load_delay,
+	// as this creates a tight loop.
 	//
 	// EXPERIMENTAL: Subject to change.
 	LoadRaw json.RawMessage `json:"load,omitempty" caddy:"namespace=caddy.config_loaders inline_key=module"`
+
+	// The duration after which to load config. If set, config will be pulled
+	// from the config loader after this duration. A delay is required if a
+	// dynamically-loaded config is configured to load yet another config. To
+	// load configs on a regular interval, ensure this value is set the same
+	// on all loaded configs; it can also be variable if needed, and to stop
+	// the loop, simply remove dynamic config loading from the next-loaded
+	// config.
+	//
+	// EXPERIMENTAL: Subject to change.
+	LoadDelay Duration `json:"load_delay,omitempty"`
 }
 
 // IdentityConfig configures management of this server's identity. An identity
@@ -178,7 +195,7 @@ type AdminPermissions struct {
 
 // newAdminHandler reads admin's config and returns an http.Handler suitable
 // for use in an admin endpoint server, which will be listening on listenAddr.
-func (admin AdminConfig) newAdminHandler(addr NetworkAddress, remote bool) adminHandler {
+func (admin *AdminConfig) newAdminHandler(addr NetworkAddress, remote bool) adminHandler {
 	muxWrap := adminHandler{mux: http.NewServeMux()}
 
 	// secure the local or remote endpoint respectively
@@ -187,6 +204,7 @@ func (admin AdminConfig) newAdminHandler(addr NetworkAddress, remote bool) admin
 	} else {
 		muxWrap.enforceHost = !addr.isWildcardInterface()
 		muxWrap.allowedOrigins = admin.allowedOrigins(addr)
+		muxWrap.enforceOrigin = admin.EnforceOrigin
 	}
 
 	addRouteWithMetrics := func(pattern string, handlerLabel string, h http.Handler) {
@@ -237,17 +255,39 @@ func (admin AdminConfig) newAdminHandler(addr NetworkAddress, remote bool) admin
 		for _, route := range router.Routes() {
 			addRoute(route.Pattern, handlerLabel, route.Handler)
 		}
+		admin.routers = append(admin.routers, router)
 	}
 
 	return muxWrap
 }
 
+// provisionAdminRouters provisions all the router modules
+// in the admin.api namespace that need provisioning.
+func (admin *AdminConfig) provisionAdminRouters(ctx Context) error {
+	for _, router := range admin.routers {
+		provisioner, ok := router.(Provisioner)
+		if !ok {
+			continue
+		}
+
+		err := provisioner.Provision(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	// We no longer need the routers once provisioned, allow for GC
+	admin.routers = nil
+
+	return nil
+}
+
 // allowedOrigins returns a list of origins that are allowed.
 // If admin.Origins is nil (null), the provided listen address
 // will be used as the default origin. If admin.Origins is
 // empty, no origins will be allowed, effectively bricking the
 // endpoint for non-unix-socket endpoints, but whatever.
-func (admin AdminConfig) allowedOrigins(addr NetworkAddress) []string {
+func (admin AdminConfig) allowedOrigins(addr NetworkAddress) []*url.URL {
 	uniqueOrigins := make(map[string]struct{})
 	for _, o := range admin.Origins {
 		uniqueOrigins[o] = struct{}{}
@@ -271,8 +311,23 @@ func (admin AdminConfig) allowedOrigins(addr NetworkAddress) []string {
 			uniqueOrigins[addr.JoinHostPort(0)] = struct{}{}
 		}
 	}
-	allowed := make([]string, 0, len(uniqueOrigins))
-	for origin := range uniqueOrigins {
+	allowed := make([]*url.URL, 0, len(uniqueOrigins))
+	for originStr := range uniqueOrigins {
+		var origin *url.URL
+		if strings.Contains(originStr, "://") {
+			var err error
+			origin, err = url.Parse(originStr)
+			if err != nil {
+				continue
+			}
+			origin.Path = ""
+			origin.RawPath = ""
+			origin.Fragment = ""
+			origin.RawFragment = ""
+			origin.RawQuery = ""
+		} else {
+			origin = &url.URL{Host: originStr}
+		}
 		allowed = append(allowed, origin)
 	}
 	return allowed
@@ -284,17 +339,19 @@ func (admin AdminConfig) allowedOrigins(addr NetworkAddress) []string {
 // that there is always an admin server (unless it is explicitly
 // configured to be disabled).
 func replaceLocalAdminServer(cfg *Config) error {
-	// always be sure to close down the old admin endpoint
+	// always* be sure to close down the old admin endpoint
 	// as gracefully as possible, even if the new one is
 	// disabled -- careful to use reference to the current
 	// (old) admin endpoint since it will be different
 	// when the function returns
+	// (* except if the new one fails to start)
 	oldAdminServer := localAdminServer
+	var err error
 	defer func() {
 		// do the shutdown asynchronously so that any
 		// current API request gets a response; this
 		// goroutine may last a few seconds
-		if oldAdminServer != nil {
+		if oldAdminServer != nil && err == nil {
 			go func(oldAdminServer *http.Server) {
 				err := stopAdminServer(oldAdminServer)
 				if err != nil {
@@ -304,31 +361,33 @@ func replaceLocalAdminServer(cfg *Config) error {
 		}
 	}()
 
-	// always get a valid admin config
-	adminConfig := DefaultAdminConfig
-	if cfg != nil && cfg.Admin != nil {
-		adminConfig = cfg.Admin
+	// set a default if admin wasn't otherwise configured
+	if cfg.Admin == nil {
+		cfg.Admin = &AdminConfig{
+			Listen: DefaultAdminListen,
+		}
 	}
 
 	// if new admin endpoint is to be disabled, we're done
-	if adminConfig.Disabled {
+	if cfg.Admin.Disabled {
 		Log().Named("admin").Warn("admin endpoint disabled")
 		return nil
 	}
 
 	// extract a singular listener address
-	addr, err := parseAdminListenAddr(adminConfig.Listen, DefaultAdminListen)
+	addr, err := parseAdminListenAddr(cfg.Admin.Listen, DefaultAdminListen)
 	if err != nil {
 		return err
 	}
 
-	handler := adminConfig.newAdminHandler(addr, false)
+	handler := cfg.Admin.newAdminHandler(addr, false)
 
 	ln, err := Listen(addr.Network, addr.JoinHostPort(0))
 	if err != nil {
 		return err
 	}
 
+	serverMu.Lock()
 	localAdminServer = &http.Server{
 		Addr:              addr.String(), // for logging purposes only
 		Handler:           handler,
@@ -337,18 +396,22 @@ func replaceLocalAdminServer(cfg *Config) error {
 		IdleTimeout:       60 * time.Second,
 		MaxHeaderBytes:    1024 * 64,
 	}
+	serverMu.Unlock()
 
 	adminLogger := Log().Named("admin")
 	go func() {
-		if err := localAdminServer.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
+		serverMu.Lock()
+		server := localAdminServer
+		serverMu.Unlock()
+		if err := server.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
 			adminLogger.Error("admin server shutdown for unknown reason", zap.Error(err))
 		}
 	}()
 
 	adminLogger.Info("admin endpoint started",
 		zap.String("address", addr.String()),
-		zap.Bool("enforce_origin", adminConfig.EnforceOrigin),
-		zap.Strings("origins", handler.allowedOrigins))
+		zap.Bool("enforce_origin", cfg.Admin.EnforceOrigin),
+		zap.Array("origins", loggableURLArray(handler.allowedOrigins)))
 
 	if !handler.enforceHost {
 		adminLogger.Warn("admin endpoint on open interface; host checking disabled",
@@ -364,11 +427,6 @@ func manageIdentity(ctx Context, cfg *Config) error {
 		return nil
 	}
 
-	oldIdentityCertCache := identityCertCache
-	if oldIdentityCertCache != nil {
-		defer oldIdentityCertCache.Stop()
-	}
-
 	// set default issuers; this is pretty hacky because we can't
 	// import the caddytls package -- but it works
 	if cfg.Admin.Identity.IssuersRaw == nil {
@@ -384,13 +442,18 @@ func manageIdentity(ctx Context, cfg *Config) error {
 		if err != nil {
 			return fmt.Errorf("loading identity issuer modules: %s", err)
 		}
-		for _, issVal := range val.([]interface{}) {
+		for _, issVal := range val.([]any) {
 			cfg.Admin.Identity.issuers = append(cfg.Admin.Identity.issuers, issVal.(certmagic.Issuer))
 		}
 	}
 
+	// we'll make a new cache when we make the CertMagic config, so stop any previous cache
+	if identityCertCache != nil {
+		identityCertCache.Stop()
+	}
+
 	logger := Log().Named("admin.identity")
-	cmCfg := cfg.Admin.Identity.certmagicConfig(logger)
+	cmCfg := cfg.Admin.Identity.certmagicConfig(logger, true)
 
 	// issuers have circular dependencies with the configs because,
 	// as explained in the caddytls package, they need access to the
@@ -456,7 +519,10 @@ func replaceRemoteAdminServer(ctx Context, cfg *Config) error {
 	}
 
 	// create TLS config that will enforce mutual authentication
-	cmCfg := cfg.Admin.Identity.certmagicConfig(remoteLogger)
+	if identityCertCache == nil {
+		return fmt.Errorf("cannot enable remote admin without a certificate cache; configure identity management to initialize a certificate cache")
+	}
+	cmCfg := cfg.Admin.Identity.certmagicConfig(remoteLogger, false)
 	tlsConfig := cmCfg.TLSConfig()
 	tlsConfig.NextProtos = nil // this server does not solve ACME challenges
 	tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
@@ -468,6 +534,7 @@ func replaceRemoteAdminServer(ctx Context, cfg *Config) error {
 		return err
 	}
 
+	serverMu.Lock()
 	// create secure HTTP server
 	remoteAdminServer = &http.Server{
 		Addr:              addr.String(), // for logging purposes only
@@ -479,6 +546,7 @@ func replaceRemoteAdminServer(ctx Context, cfg *Config) error {
 		MaxHeaderBytes:    1024 * 64,
 		ErrorLog:          serverLogger,
 	}
+	serverMu.Unlock()
 
 	// start listener
 	ln, err := Listen(addr.Network, addr.JoinHostPort(0))
@@ -488,7 +556,10 @@ func replaceRemoteAdminServer(ctx Context, cfg *Config) error {
 	ln = tls.NewListener(ln, tlsConfig)
 
 	go func() {
-		if err := remoteAdminServer.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
+		serverMu.Lock()
+		server := remoteAdminServer
+		serverMu.Unlock()
+		if err := server.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
 			remoteLogger.Error("admin remote server shutdown for unknown reason", zap.Error(err))
 		}
 	}()
@@ -499,7 +570,7 @@ func replaceRemoteAdminServer(ctx Context, cfg *Config) error {
 	return nil
 }
 
-func (ident *IdentityConfig) certmagicConfig(logger *zap.Logger) *certmagic.Config {
+func (ident *IdentityConfig) certmagicConfig(logger *zap.Logger, makeCache bool) *certmagic.Config {
 	if ident == nil {
 		// user might not have configured identity; that's OK, we can still make a
 		// certmagic config, although it'll be mostly useless for remote management
@@ -510,7 +581,7 @@ func (ident *IdentityConfig) certmagicConfig(logger *zap.Logger) *certmagic.Conf
 		Logger:  logger,
 		Issuers: ident.issuers,
 	}
-	if identityCertCache == nil {
+	if makeCache {
 		identityCertCache = certmagic.NewCache(certmagic.CacheOptions{
 			GetConfigForCert: func(certmagic.Certificate) (*certmagic.Config, error) {
 				return cmCfg, nil
@@ -533,7 +604,7 @@ func (ctx Context) IdentityCredentials(logger *zap.Logger) ([]tls.Certificate, e
 	if logger == nil {
 		logger = Log()
 	}
-	magic := ident.certmagicConfig(logger)
+	magic := ident.certmagicConfig(logger, false)
 	return magic.ClientCredentials(ctx, ident.Identifiers)
 }
 
@@ -632,10 +703,10 @@ type AdminRoute struct {
 type adminHandler struct {
 	mux *http.ServeMux
 
-	// security for local/plaintext) endpoint, on by default
+	// security for local/plaintext endpoint
 	enforceOrigin  bool
 	enforceHost    bool
-	allowedOrigins []string
+	allowedOrigins []*url.URL
 
 	// security for remote/encrypted endpoint
 	remoteControl *RemoteAdmin
@@ -644,11 +715,17 @@ type adminHandler struct {
 // ServeHTTP is the external entry point for API requests.
 // It will only be called once per request.
 func (h adminHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ip, port, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		ip = r.RemoteAddr
+		port = ""
+	}
 	log := Log().Named("admin.api").With(
 		zap.String("method", r.Method),
 		zap.String("host", r.Host),
 		zap.String("uri", r.RequestURI),
-		zap.String("remote_addr", r.RemoteAddr),
+		zap.String("remote_ip", ip),
+		zap.String("remote_port", port),
 		zap.Reflect("headers", r.Header),
 	)
 	if r.TLS != nil {
@@ -755,8 +832,8 @@ func (h adminHandler) handleError(w http.ResponseWriter, r *http.Request, err er
 // rebinding attacks.
 func (h adminHandler) checkHost(r *http.Request) error {
 	var allowed bool
-	for _, allowedHost := range h.allowedOrigins {
-		if r.Host == allowedHost {
+	for _, allowedOrigin := range h.allowedOrigins {
+		if r.Host == allowedOrigin.Host {
 			allowed = true
 			break
 		}
@@ -775,59 +852,81 @@ func (h adminHandler) checkHost(r *http.Request) error {
 // sites from issuing requests to our listener. It
 // returns the origin that was obtained from r.
 func (h adminHandler) checkOrigin(r *http.Request) (string, error) {
-	origin := h.getOriginHost(r)
-	if origin == "" {
-		return origin, APIError{
+	originStr, origin := h.getOrigin(r)
+	if origin == nil {
+		return "", APIError{
 			HTTPStatus: http.StatusForbidden,
-			Err:        fmt.Errorf("missing required Origin header"),
+			Err:        fmt.Errorf("required Origin header is missing or invalid"),
 		}
 	}
 	if !h.originAllowed(origin) {
-		return origin, APIError{
+		return "", APIError{
 			HTTPStatus: http.StatusForbidden,
-			Err:        fmt.Errorf("client is not allowed to access from origin %s", origin),
+			Err:        fmt.Errorf("client is not allowed to access from origin '%s'", originStr),
 		}
 	}
-	return origin, nil
+	return origin.String(), nil
 }
 
-func (h adminHandler) getOriginHost(r *http.Request) string {
+func (h adminHandler) getOrigin(r *http.Request) (string, *url.URL) {
 	origin := r.Header.Get("Origin")
 	if origin == "" {
 		origin = r.Header.Get("Referer")
 	}
 	originURL, err := url.Parse(origin)
-	if err == nil && originURL.Host != "" {
-		origin = originURL.Host
+	if err != nil {
+		return origin, nil
 	}
-	return origin
+	originURL.Path = ""
+	originURL.RawPath = ""
+	originURL.Fragment = ""
+	originURL.RawFragment = ""
+	originURL.RawQuery = ""
+	return origin, originURL
 }
 
-func (h adminHandler) originAllowed(origin string) bool {
+func (h adminHandler) originAllowed(origin *url.URL) bool {
 	for _, allowedOrigin := range h.allowedOrigins {
-		originCopy := origin
-		if !strings.Contains(allowedOrigin, "://") {
-			// no scheme specified, so allow both
-			originCopy = strings.TrimPrefix(originCopy, "http://")
-			originCopy = strings.TrimPrefix(originCopy, "https://")
+		if allowedOrigin.Scheme != "" && origin.Scheme != allowedOrigin.Scheme {
+			continue
 		}
-		if originCopy == allowedOrigin {
+		if origin.Host == allowedOrigin.Host {
 			return true
 		}
 	}
 	return false
 }
 
+// etagHasher returns a the hasher we used on the config to both
+// produce and verify ETags.
+func etagHasher() hash.Hash32 { return fnv.New32a() }
+
+// makeEtag returns an Etag header value (including quotes) for
+// the given config path and hash of contents at that path.
+func makeEtag(path string, hash hash.Hash) string {
+	return fmt.Sprintf(`"%s %x"`, path, hash.Sum(nil))
+}
+
 func handleConfig(w http.ResponseWriter, r *http.Request) error {
 	switch r.Method {
 	case http.MethodGet:
 		w.Header().Set("Content-Type", "application/json")
+		// Set the ETag as a trailer header.
+		// The alternative is to write the config to a buffer, and
+		// then hash that.
+		w.Header().Set("Trailer", "ETag")
 
-		err := readConfig(r.URL.Path, w)
+		hash := etagHasher()
+		configWriter := io.MultiWriter(w, hash)
+		err := readConfig(r.URL.Path, configWriter)
 		if err != nil {
 			return APIError{HTTPStatus: http.StatusBadRequest, Err: err}
 		}
 
+		// we could consider setting up a sync.Pool for the summed
+		// hashes to reduce GC pressure.
+		w.Header().Set("Etag", makeEtag(r.URL.Path, hash))
+
 		return nil
 
 	case http.MethodPost,
@@ -861,8 +960,8 @@ func handleConfig(w http.ResponseWriter, r *http.Request) error {
 
 		forceReload := r.Header.Get("Cache-Control") == "must-revalidate"
 
-		err := changeConfig(r.Method, r.URL.Path, body, forceReload)
-		if err != nil {
+		err := changeConfig(r.Method, r.URL.Path, body, r.Header.Get("If-Match"), forceReload)
+		if err != nil && !errors.Is(err, errSameConfig) {
 			return err
 		}
 
@@ -881,19 +980,28 @@ func handleConfigID(w http.ResponseWriter, r *http.Request) error {
 
 	parts := strings.Split(idPath, "/")
 	if len(parts) < 3 || parts[2] == "" {
-		return fmt.Errorf("request path is missing object ID")
+		return APIError{
+			HTTPStatus: http.StatusBadRequest,
+			Err:        fmt.Errorf("request path is missing object ID"),
+		}
 	}
 	if parts[0] != "" || parts[1] != "id" {
-		return fmt.Errorf("malformed object path")
+		return APIError{
+			HTTPStatus: http.StatusBadRequest,
+			Err:        fmt.Errorf("malformed object path"),
+		}
 	}
 	id := parts[2]
 
 	// map the ID to the expanded path
-	currentCfgMu.RLock()
+	currentCtxMu.RLock()
 	expanded, ok := rawCfgIndex[id]
-	defer currentCfgMu.RUnlock()
+	defer currentCtxMu.RUnlock()
 	if !ok {
-		return fmt.Errorf("unknown object ID '%s'", id)
+		return APIError{
+			HTTPStatus: http.StatusNotFound,
+			Err:        fmt.Errorf("unknown object ID '%s'", id),
+		}
 	}
 
 	// piece the full URL path back together
@@ -911,11 +1019,7 @@ func handleStop(w http.ResponseWriter, r *http.Request) error {
 		}
 	}
 
-	if err := notify.NotifyStopping(); err != nil {
-		Log().Error("unable to notify stopping to service manager", zap.Error(err))
-	}
-
-	exitProcess(Log().Named("admin.api"))
+	exitProcess(context.Background(), Log().Named("admin.api"))
 	return nil
 }
 
@@ -923,11 +1027,11 @@ func handleStop(w http.ResponseWriter, r *http.Request) error {
 // the operation at path according to method, using body and out as
 // needed. This is a low-level, unsynchronized function; most callers
 // will want to use changeConfig or readConfig instead. This requires a
-// read or write lock on currentCfgMu, depending on method (GET needs
+// read or write lock on currentCtxMu, depending on method (GET needs
 // only a read lock; all others need a write lock).
 func unsyncedConfigAccess(method, path string, body []byte, out io.Writer) error {
 	var err error
-	var val interface{}
+	var val any
 
 	// if there is a request body, decode it into the
 	// variable that will be set in the config according
@@ -964,16 +1068,16 @@ func unsyncedConfigAccess(method, path string, body []byte, out io.Writer) error
 		parts = parts[:len(parts)-1]
 	}
 
-	var ptr interface{} = rawCfg
+	var ptr any = rawCfg
 
 traverseLoop:
 	for i, part := range parts {
 		switch v := ptr.(type) {
-		case map[string]interface{}:
+		case map[string]any:
 			// if the next part enters a slice, and the slice is our destination,
 			// handle it specially (because appending to the slice copies the slice
 			// header, which does not replace the original one like we want)
-			if arr, ok := v[part].([]interface{}); ok && i == len(parts)-2 {
+			if arr, ok := v[part].([]any); ok && i == len(parts)-2 {
 				var idx int
 				if method != http.MethodPost {
 					idxStr := parts[len(parts)-1]
@@ -995,7 +1099,7 @@ traverseLoop:
 					}
 				case http.MethodPost:
 					if ellipses {
-						valArray, ok := val.([]interface{})
+						valArray, ok := val.([]any)
 						if !ok {
 							return fmt.Errorf("final element is not an array")
 						}
@@ -1030,9 +1134,9 @@ traverseLoop:
 				case http.MethodPost:
 					// if the part is an existing list, POST appends to
 					// it, otherwise it just sets or creates the value
-					if arr, ok := v[part].([]interface{}); ok {
+					if arr, ok := v[part].([]any); ok {
 						if ellipses {
-							valArray, ok := val.([]interface{})
+							valArray, ok := val.([]any)
 							if !ok {
 								return fmt.Errorf("final element is not an array")
 							}
@@ -1063,12 +1167,12 @@ traverseLoop:
 				// might not exist yet; that's OK but we need to make them as
 				// we go, while we still have a pointer from the level above
 				if v[part] == nil && method == http.MethodPut {
-					v[part] = make(map[string]interface{})
+					v[part] = make(map[string]any)
 				}
 				ptr = v[part]
 			}
 
-		case []interface{}:
+		case []any:
 			partInt, err := strconv.Atoi(part)
 			if err != nil {
 				return fmt.Errorf("[/%s] invalid array index '%s': %v",
@@ -1090,7 +1194,7 @@ traverseLoop:
 
 // RemoveMetaFields removes meta fields like "@id" from a JSON message
 // by using a simple regular expression. (An alternate way to do this
-// would be to delete them from the raw, map[string]interface{}
+// would be to delete them from the raw, map[string]any
 // representation as they are indexed, then iterate the index we made
 // and add them back after encoding as JSON, but this is simpler.)
 func RemoveMetaFields(rawJSON []byte) []byte {
@@ -1165,6 +1269,18 @@ func decodeBase64DERCert(certStr string) (*x509.Certificate, error) {
 	return x509.ParseCertificate(derBytes)
 }
 
+type loggableURLArray []*url.URL
+
+func (ua loggableURLArray) MarshalLogArray(enc zapcore.ArrayEncoder) error {
+	if ua == nil {
+		return nil
+	}
+	for _, u := range ua {
+		enc.AppendString(u.String())
+	}
+	return nil
+}
+
 var (
 	// DefaultAdminListen is the address for the local admin
 	// listener, if none is specified at startup.
@@ -1174,19 +1290,13 @@ var (
 	// (TLS-authenticated) admin listener, if enabled and not
 	// specified otherwise.
 	DefaultRemoteAdminListen = ":2021"
-
-	// DefaultAdminConfig is the default configuration
-	// for the local administration endpoint.
-	DefaultAdminConfig = &AdminConfig{
-		Listen: DefaultAdminListen,
-	}
 )
 
 // PIDFile writes a pidfile to the file at filename. It
 // will get deleted before the process gracefully exits.
 func PIDFile(filename string) error {
 	pid := []byte(strconv.Itoa(os.Getpid()) + "\n")
-	err := ioutil.WriteFile(filename, pid, 0600)
+	err := os.WriteFile(filename, pid, 0600)
 	if err != nil {
 		return err
 	}
@@ -1216,13 +1326,14 @@ const (
 )
 
 var bufPool = sync.Pool{
-	New: func() interface{} {
+	New: func() any {
 		return new(bytes.Buffer)
 	},
 }
 
 // keep a reference to admin endpoint singletons while they're active
 var (
+	serverMu                            sync.Mutex
 	localAdminServer, remoteAdminServer *http.Server
 	identityCertCache                   *certmagic.Cache
 )

admin_test.go

@@ -16,10 +16,31 @@ package caddy
 
 import (
 	"encoding/json"
+	"fmt"
+	"net/http"
 	"reflect"
+	"sync"
 	"testing"
 )
 
+var testCfg = []byte(`{
+			"apps": {
+				"http": {
+					"servers": {
+						"myserver": {
+							"listen": ["tcp/localhost:8080-8084"],
+							"read_timeout": "30s"
+						},
+						"yourserver": {
+							"listen": ["127.0.0.1:5000"],
+							"read_header_timeout": "15s"
+						}
+					}
+				}
+			}
+		}
+		`)
+
 func TestUnsyncedConfigAccess(t *testing.T) {
 	// each test is performed in sequence, so
 	// each change builds on the previous ones;
@@ -94,7 +115,7 @@ func TestUnsyncedConfigAccess(t *testing.T) {
 		}
 
 		// decode the expected config so we can do a convenient DeepEqual
-		var expectedDecoded interface{}
+		var expectedDecoded any
 		err = json.Unmarshal([]byte(tc.expect), &expectedDecoded)
 		if err != nil {
 			t.Fatalf("Test %d: Unmarshaling expected config: %v", i, err)
@@ -108,25 +129,71 @@ func TestUnsyncedConfigAccess(t *testing.T) {
 	}
 }
 
-func BenchmarkLoad(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		cfg := []byte(`{
-			"apps": {
-				"http": {
-					"servers": {
-						"myserver": {
-							"listen": ["tcp/localhost:8080-8084"],
-							"read_timeout": "30s"
-						},
-						"yourserver": {
-							"listen": ["127.0.0.1:5000"],
-							"read_header_timeout": "15s"
-						}
-					}
-				}
-			}
-		}
-		`)
-		Load(cfg, true)
+// TestLoadConcurrent exercises Load under concurrent conditions
+// and is most useful under test with `-race` enabled.
+func TestLoadConcurrent(t *testing.T) {
+	var wg sync.WaitGroup
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		go func() {
+			_ = Load(testCfg, true)
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+}
+
+type fooModule struct {
+	IntField int
+	StrField string
+}
+
+func (fooModule) CaddyModule() ModuleInfo {
+	return ModuleInfo{
+		ID:  "foo",
+		New: func() Module { return new(fooModule) },
+	}
+}
+func (fooModule) Start() error { return nil }
+func (fooModule) Stop() error  { return nil }
+
+func TestETags(t *testing.T) {
+	RegisterModule(fooModule{})
+
+	if err := Load([]byte(`{"apps": {"foo": {"strField": "abc", "intField": 0}}}`), true); err != nil {
+		t.Fatalf("loading: %s", err)
+	}
+
+	const key = "/" + rawConfigKey + "/apps/foo"
+
+	// try update the config with the wrong etag
+	err := changeConfig(http.MethodPost, key, []byte(`{"strField": "abc", "intField": 1}}`), fmt.Sprintf(`"/%s not_an_etag"`, rawConfigKey), false)
+	if apiErr, ok := err.(APIError); !ok || apiErr.HTTPStatus != http.StatusPreconditionFailed {
+		t.Fatalf("expected precondition failed; got %v", err)
+	}
+
+	// get the etag
+	hash := etagHasher()
+	if err := readConfig(key, hash); err != nil {
+		t.Fatalf("reading: %s", err)
+	}
+
+	// do the same update with the correct key
+	err = changeConfig(http.MethodPost, key, []byte(`{"strField": "abc", "intField": 1}`), makeEtag(key, hash), false)
+	if err != nil {
+		t.Fatalf("expected update to work; got %v", err)
+	}
+
+	// now try another update. The hash should no longer match and we should get precondition failed
+	err = changeConfig(http.MethodPost, key, []byte(`{"strField": "abc", "intField": 2}`), makeEtag(key, hash), false)
+	if apiErr, ok := err.(APIError); !ok || apiErr.HTTPStatus != http.StatusPreconditionFailed {
+		t.Fatalf("expected precondition failed; got %v", err)
+	}
+}
+
+func BenchmarkLoad(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Load(testCfg, true)
 	}
 }

caddy.go

@@ -17,10 +17,11 @@ package caddy
 import (
 	"bytes"
 	"context"
+	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -101,26 +102,50 @@ func Run(cfg *Config) error {
 // if it is different from the current config or
 // forceReload is true.
 func Load(cfgJSON []byte, forceReload bool) error {
-	if err := notify.NotifyReloading(); err != nil {
-		Log().Error("unable to notify reloading to service manager", zap.Error(err))
+	if err := notify.Reloading(); err != nil {
+		Log().Error("unable to notify service manager of reloading state", zap.Error(err))
 	}
 
+	// after reload, notify system of success or, if
+	// failure, update with status (error message)
+	var err error
 	defer func() {
-		if err := notify.NotifyReadiness(); err != nil {
-			Log().Error("unable to notify readiness to service manager", zap.Error(err))
+		if err != nil {
+			if notifyErr := notify.Error(err, 0); notifyErr != nil {
+				Log().Error("unable to notify to service manager of reload error",
+					zap.Error(notifyErr),
+					zap.String("reload_err", err.Error()))
+			}
+			return
+		}
+		if err := notify.Ready(); err != nil {
+			Log().Error("unable to notify to service manager of ready state", zap.Error(err))
 		}
 	}()
 
-	return changeConfig(http.MethodPost, "/"+rawConfigKey, cfgJSON, forceReload)
+	err = changeConfig(http.MethodPost, "/"+rawConfigKey, cfgJSON, "", forceReload)
+	if errors.Is(err, errSameConfig) {
+		err = nil // not really an error
+	}
+
+	return err
 }
 
 // changeConfig changes the current config (rawCfg) according to the
 // method, traversed via the given path, and uses the given input as
 // the new value (if applicable; i.e. "DELETE" doesn't have an input).
 // If the resulting config is the same as the previous, no reload will
-// occur unless forceReload is true. This function is safe for
+// occur unless forceReload is true. If the config is unchanged and not
+// forcefully reloaded, then errConfigUnchanged This function is safe for
 // concurrent use.
-func changeConfig(method, path string, input []byte, forceReload bool) error {
+// The ifMatchHeader can optionally be given a string of the format:
+//
+//	"<path> <hash>"
+//
+// where <path> is the absolute path in the config and <hash> is the expected hash of
+// the config at that path. If the hash in the ifMatchHeader doesn't match
+// the hash of the config, then an APIError with status 412 will be returned.
+func changeConfig(method, path string, input []byte, ifMatchHeader string, forceReload bool) error {
 	switch method {
 	case http.MethodGet,
 		http.MethodHead,
@@ -130,8 +155,42 @@ func changeConfig(method, path string, input []byte, forceReload bool) error {
 		return fmt.Errorf("method not allowed")
 	}
 
-	currentCfgMu.Lock()
-	defer currentCfgMu.Unlock()
+	currentCtxMu.Lock()
+	defer currentCtxMu.Unlock()
+
+	if ifMatchHeader != "" {
+		// expect the first and last character to be quotes
+		if len(ifMatchHeader) < 2 || ifMatchHeader[0] != '"' || ifMatchHeader[len(ifMatchHeader)-1] != '"' {
+			return APIError{
+				HTTPStatus: http.StatusBadRequest,
+				Err:        fmt.Errorf("malformed If-Match header; expect quoted string"),
+			}
+		}
+
+		// read out the parts
+		parts := strings.Fields(ifMatchHeader[1 : len(ifMatchHeader)-1])
+		if len(parts) != 2 {
+			return APIError{
+				HTTPStatus: http.StatusBadRequest,
+				Err:        fmt.Errorf("malformed If-Match header; expect format \"<path> <hash>\""),
+			}
+		}
+
+		// get the current hash of the config
+		// at the given path
+		hash := etagHasher()
+		err := unsyncedConfigAccess(http.MethodGet, parts[0], nil, hash)
+		if err != nil {
+			return err
+		}
+
+		if hex.EncodeToString(hash.Sum(nil)) != parts[1] {
+			return APIError{
+				HTTPStatus: http.StatusPreconditionFailed,
+				Err:        fmt.Errorf("If-Match header did not match current config hash"),
+			}
+		}
+	}
 
 	err := unsyncedConfigAccess(method, path, input, nil)
 	if err != nil {
@@ -149,8 +208,8 @@ func changeConfig(method, path string, input []byte, forceReload bool) error {
 
 	// if nothing changed, no need to do a whole reload unless the client forces it
 	if !forceReload && bytes.Equal(rawCfgJSON, newCfg) {
-		Log().Named("admin.api").Info("config is unchanged")
-		return nil
+		Log().Info("config is unchanged")
+		return errSameConfig
 	}
 
 	// find any IDs in this config and index them
@@ -172,7 +231,7 @@ func changeConfig(method, path string, input []byte, forceReload bool) error {
 			// with what caddy is still running; we need to
 			// unmarshal it again because it's likely that
 			// pointers deep in our rawCfg map were modified
-			var oldCfg interface{}
+			var oldCfg any
 			err2 := json.Unmarshal(rawCfgJSON, &oldCfg)
 			if err2 != nil {
 				err = fmt.Errorf("%v; additionally, restoring old config: %v", err, err2)
@@ -197,18 +256,18 @@ func changeConfig(method, path string, input []byte, forceReload bool) error {
 // readConfig traverses the current config to path
 // and writes its JSON encoding to out.
 func readConfig(path string, out io.Writer) error {
-	currentCfgMu.RLock()
-	defer currentCfgMu.RUnlock()
+	currentCtxMu.RLock()
+	defer currentCtxMu.RUnlock()
 	return unsyncedConfigAccess(http.MethodGet, path, nil, out)
 }
 
 // indexConfigObjects recursively searches ptr for object fields named
 // "@id" and maps that ID value to the full configPath in the index.
 // This function is NOT safe for concurrent access; obtain a write lock
-// on currentCfgMu.
-func indexConfigObjects(ptr interface{}, configPath string, index map[string]string) error {
+// on currentCtxMu.
+func indexConfigObjects(ptr any, configPath string, index map[string]string) error {
 	switch val := ptr.(type) {
-	case map[string]interface{}:
+	case map[string]any:
 		for k, v := range val {
 			if k == idKey {
 				switch idVal := v.(type) {
@@ -227,7 +286,7 @@ func indexConfigObjects(ptr interface{}, configPath string, index map[string]str
 				return err
 			}
 		}
-	case []interface{}:
+	case []any:
 		// traverse each element of the array recursively
 		for i := range val {
 			err := indexConfigObjects(val[i], path.Join(configPath, strconv.Itoa(i)), index)
@@ -245,7 +304,7 @@ func indexConfigObjects(ptr interface{}, configPath string, index map[string]str
 // it as the new config, replacing any other current config.
 // It does NOT update the raw config state, as this is a
 // lower-level function; most callers will want to use Load
-// instead. A write lock on currentCfgMu is required! If
+// instead. A write lock on currentCtxMu is required! If
 // allowPersist is false, it will not be persisted to disk,
 // even if it is configured to.
 func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
@@ -268,22 +327,23 @@ func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
 		newCfg != nil &&
 		newCfg.Admin != nil &&
 		newCfg.Admin.Config != nil &&
-		newCfg.Admin.Config.LoadRaw != nil {
-		return fmt.Errorf("recursive config loading detected: pulled configs cannot pull other configs")
+		newCfg.Admin.Config.LoadRaw != nil &&
+		newCfg.Admin.Config.LoadDelay <= 0 {
+		return fmt.Errorf("recursive config loading detected: pulled configs cannot pull other configs without positive load_delay")
 	}
 
 	// run the new config and start all its apps
-	err = run(newCfg, true)
+	ctx, err := run(newCfg, true)
 	if err != nil {
 		return err
 	}
 
-	// swap old config with the new one
-	oldCfg := currentCfg
-	currentCfg = newCfg
+	// swap old context (including its config) with the new one
+	oldCtx := currentCtx
+	currentCtx = ctx
 
 	// Stop, Cleanup each old app
-	unsyncedStop(oldCfg)
+	unsyncedStop(oldCtx)
 
 	// autosave a non-nil config, if not disabled
 	if allowPersist &&
@@ -299,7 +359,7 @@ func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
 				zap.String("dir", dir),
 				zap.Error(err))
 		} else {
-			err := ioutil.WriteFile(ConfigAutosavePath, cfgJSON, 0600)
+			err := os.WriteFile(ConfigAutosavePath, cfgJSON, 0600)
 			if err == nil {
 				Log().Info("autosaved config (load with --resume flag)", zap.String("file", ConfigAutosavePath))
 			} else {
@@ -327,7 +387,7 @@ func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
 // This is a low-level function; most callers
 // will want to use Run instead, which also
 // updates the config's raw state.
-func run(newCfg *Config, start bool) error {
+func run(newCfg *Config, start bool) (Context, error) {
 	// because we will need to roll back any state
 	// modifications if this function errors, we
 	// keep a single error value and scope all
@@ -358,8 +418,8 @@ func run(newCfg *Config, start bool) error {
 			cancel()
 
 			// also undo any other state changes we made
-			if currentCfg != nil {
-				certmagic.Default.Storage = currentCfg.storage
+			if currentCtx.cfg != nil {
+				certmagic.Default.Storage = currentCtx.cfg.storage
 			}
 		}
 	}()
@@ -371,14 +431,14 @@ func run(newCfg *Config, start bool) error {
 	}
 	err = newCfg.Logging.openLogs(ctx)
 	if err != nil {
-		return err
+		return ctx, err
 	}
 
 	// start the admin endpoint (and stop any prior one)
 	if start {
 		err = replaceLocalAdminServer(newCfg)
 		if err != nil {
-			return fmt.Errorf("starting caddy administration endpoint: %v", err)
+			return ctx, fmt.Errorf("starting caddy administration endpoint: %v", err)
 		}
 	}
 
@@ -407,7 +467,7 @@ func run(newCfg *Config, start bool) error {
 		return nil
 	}()
 	if err != nil {
-		return err
+		return ctx, err
 	}
 
 	// Load and Provision each app and their submodules
@@ -420,16 +480,23 @@ func run(newCfg *Config, start bool) error {
 		return nil
 	}()
 	if err != nil {
-		return err
+		return ctx, err
 	}
 
 	if !start {
-		return nil
+		return ctx, nil
+	}
+
+	// Provision any admin routers which may need to access
+	// some of the other apps at runtime
+	err = newCfg.Admin.provisionAdminRouters(ctx)
+	if err != nil {
+		return ctx, err
 	}
 
 	// Start
 	err = func() error {
-		var started []string
+		started := make([]string, 0, len(newCfg.apps))
 		for name, a := range newCfg.apps {
 			err := a.Start()
 			if err != nil {
@@ -449,12 +516,12 @@ func run(newCfg *Config, start bool) error {
 		return nil
 	}()
 	if err != nil {
-		return err
+		return ctx, err
 	}
 
 	// now that the user's config is running, finish setting up anything else,
 	// such as remote admin endpoint, config loader, etc.
-	return finishSettingUp(ctx, newCfg)
+	return ctx, finishSettingUp(ctx, newCfg)
 }
 
 // finishSettingUp should be run after all apps have successfully started.
@@ -480,30 +547,74 @@ func finishSettingUp(ctx Context, cfg *Config) error {
 		if err != nil {
 			return fmt.Errorf("loading config loader module: %s", err)
 		}
-		loadedConfig, err := val.(ConfigLoader).LoadConfig(ctx)
-		if err != nil {
-			return fmt.Errorf("loading dynamic config from %T: %v", val, err)
+
+		logger := Log().Named("config_loader").With(
+			zap.String("module", val.(Module).CaddyModule().ID.Name()),
+			zap.Int("load_delay", int(cfg.Admin.Config.LoadDelay)))
+
+		runLoadedConfig := func(config []byte) error {
+			logger.Info("applying dynamically-loaded config")
+			err := changeConfig(http.MethodPost, "/"+rawConfigKey, config, "", false)
+			if errors.Is(err, errSameConfig) {
+				return err
+			}
+			if err != nil {
+				logger.Error("failed to run dynamically-loaded config", zap.Error(err))
+				return err
+			}
+			logger.Info("successfully applied dynamically-loaded config")
+			return nil
 		}
 
-		// do this in a goroutine so current config can finish being loaded; otherwise deadlock
-		go func() {
-			Log().Info("applying dynamically-loaded config", zap.String("loader_module", val.(Module).CaddyModule().ID.Name()))
-			currentCfgMu.Lock()
-			err := unsyncedDecodeAndRun(loadedConfig, false)
-			currentCfgMu.Unlock()
-			if err == nil {
-				Log().Info("dynamically-loaded config applied successfully")
-			} else {
-				Log().Error("running dynamically-loaded config failed", zap.Error(err))
+		if cfg.Admin.Config.LoadDelay > 0 {
+			go func() {
+				// the loop is here to iterate ONLY if there is an error, a no-op config load,
+				// or an unchanged config; in which case we simply wait the delay and try again
+				for {
+					timer := time.NewTimer(time.Duration(cfg.Admin.Config.LoadDelay))
+					select {
+					case <-timer.C:
+						loadedConfig, err := val.(ConfigLoader).LoadConfig(ctx)
+						if err != nil {
+							logger.Error("failed loading dynamic config; will retry", zap.Error(err))
+							continue
+						}
+						if loadedConfig == nil {
+							logger.Info("dynamically-loaded config was nil; will retry")
+							continue
+						}
+						err = runLoadedConfig(loadedConfig)
+						if errors.Is(err, errSameConfig) {
+							logger.Info("dynamically-loaded config was unchanged; will retry")
+							continue
+						}
+					case <-ctx.Done():
+						if !timer.Stop() {
+							<-timer.C
+						}
+						logger.Info("stopping dynamic config loading")
+					}
+					break
+				}
+			}()
+		} else {
+			// if no LoadDelay is provided, will load config synchronously
+			loadedConfig, err := val.(ConfigLoader).LoadConfig(ctx)
+			if err != nil {
+				return fmt.Errorf("loading dynamic config from %T: %v", val, err)
 			}
-		}()
+			// do this in a goroutine so current config can finish being loaded; otherwise deadlock
+			go func() { _ = runLoadedConfig(loadedConfig) }()
+		}
 	}
 
 	return nil
 }
 
-// ConfigLoader is a type that can load a Caddy config. The
-// returned config must be valid Caddy JSON.
+// ConfigLoader is a type that can load a Caddy config. If
+// the return value is non-nil, it must be valid Caddy JSON;
+// if nil or with non-nil error, it is considered to be a
+// no-op load and may be retried later.
 type ConfigLoader interface {
 	LoadConfig(Context) ([]byte, error)
 }
@@ -515,10 +626,10 @@ type ConfigLoader interface {
 // stop the others. Stop should only be called
 // if not replacing with a new config.
 func Stop() error {
-	currentCfgMu.Lock()
-	defer currentCfgMu.Unlock()
-	unsyncedStop(currentCfg)
-	currentCfg = nil
+	currentCtxMu.Lock()
+	defer currentCtxMu.Unlock()
+	unsyncedStop(currentCtx)
+	currentCtx = Context{}
 	rawCfgJSON = nil
 	rawCfgIndex = nil
 	rawCfg[rawConfigKey] = nil
@@ -531,13 +642,13 @@ func Stop() error {
 // it is logged and the function continues stopping
 // the next app. This function assumes all apps in
 // cfg were successfully started first.
-func unsyncedStop(cfg *Config) {
-	if cfg == nil {
+func unsyncedStop(ctx Context) {
+	if ctx.cfg == nil {
 		return
 	}
 
 	// stop each app
-	for name, a := range cfg.apps {
+	for name, a := range ctx.cfg.apps {
 		err := a.Stop()
 		if err != nil {
 			log.Printf("[ERROR] stop %s: %v", name, err)
@@ -545,13 +656,13 @@ func unsyncedStop(cfg *Config) {
 	}
 
 	// clean up all modules
-	cfg.cancelFunc()
+	ctx.cfg.cancelFunc()
 }
 
 // Validate loads, provisions, and validates
 // cfg, but does not start running it.
 func Validate(cfg *Config) error {
-	err := run(cfg, false)
+	_, err := run(cfg, false)
 	if err == nil {
 		cfg.cancelFunc() // call Cleanup on all modules
 	}
@@ -564,7 +675,11 @@ func Validate(cfg *Config) error {
 // PID file, and shuts down admin endpoint(s) in a goroutine.
 // Errors are logged along the way, and an appropriate exit
 // code is emitted.
-func exitProcess(logger *zap.Logger) {
+func exitProcess(ctx context.Context, logger *zap.Logger) {
+	if err := notify.Stopping(); err != nil {
+		Log().Error("unable to notify service manager of stopping state", zap.Error(err))
+	}
+
 	if logger == nil {
 		logger = Log()
 	}
@@ -579,7 +694,7 @@ func exitProcess(logger *zap.Logger) {
 	}
 
 	// clean up certmagic locks
-	certmagic.CleanUpOwnLocks(logger)
+	certmagic.CleanUpOwnLocks(ctx, logger)
 
 	// remove pidfile
 	if pidfile != "" {
@@ -680,13 +795,13 @@ func ParseDuration(s string) (time.Duration, error) {
 // have its own unique ID.
 func InstanceID() (uuid.UUID, error) {
 	uuidFilePath := filepath.Join(AppDataDir(), "instance.uuid")
-	uuidFileBytes, err := ioutil.ReadFile(uuidFilePath)
+	uuidFileBytes, err := os.ReadFile(uuidFilePath)
 	if os.IsNotExist(err) {
 		uuid, err := uuid.NewRandom()
 		if err != nil {
 			return uuid, err
 		}
-		err = ioutil.WriteFile(uuidFilePath, []byte(uuid.String()), 0600)
+		err = os.WriteFile(uuidFilePath, []byte(uuid.String()), 0600)
 		return uuid, err
 	} else if err != nil {
 		return [16]byte{}, err
@@ -694,36 +809,106 @@ func InstanceID() (uuid.UUID, error) {
 	return uuid.ParseBytes(uuidFileBytes)
 }
 
-// GoModule returns the build info of this Caddy
-// build from debug.BuildInfo (requires Go modules).
-// If no version information is available, a non-nil
-// value will still be returned, but with an
-// unknown version.
-func GoModule() *debug.Module {
-	var mod debug.Module
-	return goModule(&mod)
-}
-
-// goModule holds the actual implementation of GoModule.
-// Allocating debug.Module in GoModule() and passing a
-// reference to goModule enables mid-stack inlining.
-func goModule(mod *debug.Module) *debug.Module {
-	mod.Version = "unknown"
+// Version returns the Caddy version in a simple/short form, and
+// a full version string. The short form will not have spaces and
+// is intended for User-Agent strings and similar, but may be
+// omitting valuable information. Note that Caddy must be compiled
+// in a special way to properly embed complete version information.
+// First this function tries to get the version from the embedded
+// build info provided by go.mod dependencies; then it tries to
+// get info from embedded VCS information, which requires having
+// built Caddy from a git repository. If no version is available,
+// this function returns "(devel)" becaise Go uses that, but for
+// the simple form we change it to "unknown".
+//
+// See relevant Go issues: https://github.com/golang/go/issues/29228
+// and https://github.com/golang/go/issues/50603.
+//
+// This function is experimental and subject to change or removal.
+func Version() (simple, full string) {
+	// the currently-recommended way to build Caddy involves
+	// building it as a dependency so we can extract version
+	// information from go.mod tooling; once the upstream
+	// Go issues are fixed, we should just be able to use
+	// bi.Main... hopefully.
+	var module *debug.Module
 	bi, ok := debug.ReadBuildInfo()
 	if ok {
-		mod.Path = bi.Main.Path
-		// The recommended way to build Caddy involves
-		// creating a separate main module, which
-		// TODO: track related Go issue: https://github.com/golang/go/issues/29228
-		// once that issue is fixed, we should just be able to use bi.Main... hopefully.
+		// find the Caddy module in the dependency list
 		for _, dep := range bi.Deps {
 			if dep.Path == ImportPath {
-				return dep
+				module = dep
+				break
 			}
 		}
-		return &bi.Main
 	}
-	return mod
+	if module != nil {
+		simple, full = module.Version, module.Version
+		if module.Sum != "" {
+			full += " " + module.Sum
+		}
+		if module.Replace != nil {
+			full += " => " + module.Replace.Path
+			if module.Replace.Version != "" {
+				simple = module.Replace.Version + "_custom"
+				full += "@" + module.Replace.Version
+			}
+			if module.Replace.Sum != "" {
+				full += " " + module.Replace.Sum
+			}
+		}
+	}
+
+	if full == "" {
+		var vcsRevision string
+		var vcsTime time.Time
+		var vcsModified bool
+		for _, setting := range bi.Settings {
+			switch setting.Key {
+			case "vcs.revision":
+				vcsRevision = setting.Value
+			case "vcs.time":
+				vcsTime, _ = time.Parse(time.RFC3339, setting.Value)
+			case "vcs.modified":
+				vcsModified, _ = strconv.ParseBool(setting.Value)
+			}
+		}
+
+		if vcsRevision != "" {
+			var modified string
+			if vcsModified {
+				modified = "+modified"
+			}
+			full = fmt.Sprintf("%s%s (%s)", vcsRevision, modified, vcsTime.Format(time.RFC822))
+			simple = vcsRevision
+
+			// use short checksum for simple, if hex-only
+			if _, err := hex.DecodeString(simple); err == nil {
+				simple = simple[:8]
+			}
+
+			// append date to simple since it can be convenient
+			// to know the commit date as part of the version
+			if !vcsTime.IsZero() {
+				simple += "-" + vcsTime.Format("20060102")
+			}
+		}
+	}
+
+	if simple == "" || simple == "(devel)" {
+		simple = "unknown"
+	}
+
+	return
+}
+
+// ActiveContext returns the currently-active context.
+// This function is experimental and might be changed
+// or removed in the future.
+func ActiveContext() Context {
+	currentCtxMu.RLock()
+	defer currentCtxMu.RUnlock()
+	return currentCtx
 }
 
 // CtxKey is a value type for use with context.WithValue.
@@ -731,18 +916,21 @@ type CtxKey string
 
 // This group of variables pertains to the current configuration.
 var (
-	// currentCfgMu protects everything in this var block.
-	currentCfgMu sync.RWMutex
+	// currentCtxMu protects everything in this var block.
+	currentCtxMu sync.RWMutex
 
-	// currentCfg is the currently-running configuration.
-	currentCfg *Config
+	// currentCtx is the root context for the currently-running
+	// configuration, which can be accessed through this value.
+	// If the Config contained in this value is not nil, then
+	// a config is currently active/running.
+	currentCtx Context
 
 	// rawCfg is the current, generic-decoded configuration;
 	// we initialize it as a map with one field ("config")
 	// to maintain parity with the API endpoint and to avoid
 	// the special case of having to access/mutate the variable
 	// directly without traversing into it.
-	rawCfg = map[string]interface{}{
+	rawCfg = map[string]any{
 		rawConfigKey: nil,
 	}
 
@@ -755,5 +943,11 @@ var (
 	rawCfgIndex map[string]string
 )
 
+// errSameConfig is returned if the new config is the same
+// as the old one. This isn't usually an actual, actionable
+// error; it's mostly a sentinel value.
+var errSameConfig = errors.New("config is unchanged")
+
 // ImportPath is the package import path for Caddy core.
+// This identifier may be removed in the future.
 const ImportPath = "github.com/caddyserver/caddy/v2"

caddyconfig/caddyfile/adapter.go

@@ -29,12 +29,12 @@ type Adapter struct {
 }
 
 // Adapt converts the Caddyfile config in body to Caddy JSON.
-func (a Adapter) Adapt(body []byte, options map[string]interface{}) ([]byte, []caddyconfig.Warning, error) {
+func (a Adapter) Adapt(body []byte, options map[string]any) ([]byte, []caddyconfig.Warning, error) {
 	if a.ServerType == nil {
 		return nil, nil, fmt.Errorf("no server type")
 	}
 	if options == nil {
-		options = make(map[string]interface{})
+		options = make(map[string]any)
 	}
 
 	filename, _ := options["filename"].(string)
@@ -88,7 +88,7 @@ func formattingDifference(filename string, body []byte) (caddyconfig.Warning, bo
 	return caddyconfig.Warning{
 		File:    filename,
 		Line:    line,
-		Message: "input is not formatted with 'caddy fmt'",
+		Message: "Caddyfile input is not formatted; run the 'caddy fmt' command to fix inconsistencies",
 	}, true
 }
 
@@ -116,7 +116,7 @@ type ServerType interface {
 	// (e.g. CLI flags) and creates a Caddy
 	// config, along with any warnings or
 	// an error.
-	Setup([]ServerBlock, map[string]interface{}) (*caddy.Config, []caddyconfig.Warning, error)
+	Setup([]ServerBlock, map[string]any) (*caddy.Config, []caddyconfig.Warning, error)
 }
 
 // UnmarshalModule instantiates a module with the given ID and invokes

caddyconfig/caddyfile/dispenser.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"strconv"
 	"strings"
 )
 
@@ -145,15 +146,15 @@ func (d *Dispenser) NextLine() bool {
 //
 // Proper use of this method looks like this:
 //
-//     for nesting := d.Nesting(); d.NextBlock(nesting); {
-//     }
+//	for nesting := d.Nesting(); d.NextBlock(nesting); {
+//	}
 //
 // However, in simple cases where it is known that the
 // Dispenser is new and has not already traversed state
 // by a loop over NextBlock(), this will do:
 //
-//     for d.NextBlock(0) {
-//     }
+//	for d.NextBlock(0) {
+//	}
 //
 // As with other token parsing logic, a loop over
 // NextBlock() should be contained within a loop over
@@ -201,6 +202,43 @@ func (d *Dispenser) Val() string {
 	return d.tokens[d.cursor].Text
 }
 
+// ValRaw gets the raw text of the current token (including quotes).
+// If there is no token loaded, it returns empty string.
+func (d *Dispenser) ValRaw() string {
+	if d.cursor < 0 || d.cursor >= len(d.tokens) {
+		return ""
+	}
+	quote := d.tokens[d.cursor].wasQuoted
+	if quote > 0 {
+		return string(quote) + d.tokens[d.cursor].Text + string(quote) // string literal
+	}
+	return d.tokens[d.cursor].Text
+}
+
+// ScalarVal gets value of the current token, converted to the closest
+// scalar type. If there is no token loaded, it returns nil.
+func (d *Dispenser) ScalarVal() any {
+	if d.cursor < 0 || d.cursor >= len(d.tokens) {
+		return nil
+	}
+	quote := d.tokens[d.cursor].wasQuoted
+	text := d.tokens[d.cursor].Text
+
+	if quote > 0 {
+		return text // string literal
+	}
+	if num, err := strconv.Atoi(text); err == nil {
+		return num
+	}
+	if num, err := strconv.ParseFloat(text, 64); err == nil {
+		return num
+	}
+	if bool, err := strconv.ParseBool(text); err == nil {
+		return bool
+	}
+	return text
+}
+
 // Line gets the line number of the current token.
 // If there is no token loaded, it returns 0.
 func (d *Dispenser) Line() int {
@@ -249,6 +287,19 @@ func (d *Dispenser) AllArgs(targets ...*string) bool {
 	return true
 }
 
+// CountRemainingArgs counts the amount of remaining arguments
+// (tokens on the same line) without consuming the tokens.
+func (d *Dispenser) CountRemainingArgs() int {
+	count := 0
+	for d.NextArg() {
+		count++
+	}
+	for i := 0; i < count; i++ {
+		d.Prev()
+	}
+	return count
+}
+
 // RemainingArgs loads any more arguments (tokens on the same line)
 // into a slice and returns them. Open curly brace tokens also indicate
 // the end of arguments, and the curly brace is not included in
@@ -261,6 +312,18 @@ func (d *Dispenser) RemainingArgs() []string {
 	return args
 }
 
+// RemainingArgsRaw loads any more arguments (tokens on the same line,
+// retaining quotes) into a slice and returns them. Open curly brace
+// tokens also indicate the end of arguments, and the curly brace is
+// not included in the return value nor is it loaded.
+func (d *Dispenser) RemainingArgsRaw() []string {
+	var args []string
+	for d.NextArg() {
+		args = append(args, d.ValRaw())
+	}
+	return args
+}
+
 // NewFromNextSegment returns a new dispenser with a copy of
 // the tokens from the current token until the end of the
 // "directive" whether that be to the end of the line or
@@ -345,13 +408,17 @@ func (d *Dispenser) EOFErr() error {
 
 // Err generates a custom parse-time error with a message of msg.
 func (d *Dispenser) Err(msg string) error {
-	msg = fmt.Sprintf("%s:%d - Error during parsing: %s", d.File(), d.Line(), msg)
-	return errors.New(msg)
+	return d.Errf(msg)
 }
 
 // Errf is like Err, but for formatted error messages
-func (d *Dispenser) Errf(format string, args ...interface{}) error {
-	return d.Err(fmt.Sprintf(format, args...))
+func (d *Dispenser) Errf(format string, args ...any) error {
+	return d.WrapErr(fmt.Errorf(format, args...))
+}
+
+// WrapErr takes an existing error and adds the Caddyfile file and line number.
+func (d *Dispenser) WrapErr(err error) error {
+	return fmt.Errorf("%s:%d - Error during parsing: %w", d.File(), d.Line(), err)
 }
 
 // Delete deletes the current token and returns the updated slice
@@ -391,6 +458,60 @@ func (d *Dispenser) isNewLine() bool {
 	if d.cursor > len(d.tokens)-1 {
 		return false
 	}
-	return d.tokens[d.cursor-1].File != d.tokens[d.cursor].File ||
-		d.tokens[d.cursor-1].Line+d.numLineBreaks(d.cursor-1) < d.tokens[d.cursor].Line
+
+	prev := d.tokens[d.cursor-1]
+	curr := d.tokens[d.cursor]
+
+	// If the previous token is from a different file,
+	// we can assume it's from a different line
+	if prev.File != curr.File {
+		return true
+	}
+
+	// The previous token may contain line breaks if
+	// it was quoted and spanned multiple lines. e.g:
+	//
+	// dir "foo
+	//   bar
+	//   baz"
+	prevLineBreaks := d.numLineBreaks(d.cursor - 1)
+
+	// If the previous token (incl line breaks) ends
+	// on a line earlier than the current token,
+	// then the current token is on a new line
+	return prev.Line+prevLineBreaks < curr.Line
+}
+
+// isNextOnNewLine determines whether the current token is on a different
+// line (higher line number) than the next token. It handles imported
+// tokens correctly. If there isn't a next token, it returns true.
+func (d *Dispenser) isNextOnNewLine() bool {
+	if d.cursor < 0 {
+		return false
+	}
+	if d.cursor >= len(d.tokens)-1 {
+		return true
+	}
+
+	curr := d.tokens[d.cursor]
+	next := d.tokens[d.cursor+1]
+
+	// If the next token is from a different file,
+	// we can assume it's from a different line
+	if curr.File != next.File {
+		return true
+	}
+
+	// The current token may contain line breaks if
+	// it was quoted and spanned multiple lines. e.g:
+	//
+	// dir "foo
+	//   bar
+	//   baz"
+	currLineBreaks := d.numLineBreaks(d.cursor)
+
+	// If the current token (incl line breaks) ends
+	// on a line earlier than the next token,
+	// then the next token is on a new line
+	return curr.Line+currLineBreaks < next.Line
 }

caddyconfig/caddyfile/dispenser_test.go

@@ -15,6 +15,7 @@
 package caddyfile
 
 import (
+	"errors"
 	"reflect"
 	"strings"
 	"testing"
@@ -303,4 +304,10 @@ func TestDispenser_ArgErr_Err(t *testing.T) {
 	if !strings.Contains(err.Error(), "foobar") {
 		t.Errorf("Expected error message with custom message in it ('foobar'); got '%v'", err)
 	}
+
+	var ErrBarIsFull = errors.New("bar is full")
+	bookingError := d.Errf("unable to reserve: %w", ErrBarIsFull)
+	if !errors.Is(bookingError, ErrBarIsFull) {
+		t.Errorf("Errf(): should be able to unwrap the error chain")
+	}
 }

caddyconfig/caddyfile/formatter_fuzz.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build gofuzz
+//go:build gofuzz
 
 package caddyfile
 

caddyconfig/caddyfile/formatter_test.go

@@ -179,6 +179,11 @@ d {
 	{$F}
 }`,
 		},
+		{
+			description: "env var placeholders with port",
+			input:       `:{$PORT}`,
+			expect:      `:{$PORT}`,
+		},
 		{
 			description: "comments",
 			input: `#a "\n"

caddyconfig/caddyfile/lexer.go

@@ -1,4 +1,4 @@
-// Copyright 2015 Light Code Labs, LLC
+// Copyright 2015 Matthew Holt and The Caddy Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -38,6 +38,7 @@ type (
 		File        string
 		Line        int
 		Text        string
+		wasQuoted   rune // enclosing quote character, if any
 		inSnippet   bool
 		snippetName string
 	}
@@ -78,8 +79,9 @@ func (l *lexer) next() bool {
 	var val []rune
 	var comment, quoted, btQuoted, escaped bool
 
-	makeToken := func() bool {
+	makeToken := func(quoted rune) bool {
 		l.token.Text = string(val)
+		l.token.wasQuoted = quoted
 		return true
 	}
 
@@ -87,7 +89,7 @@ func (l *lexer) next() bool {
 		ch, _, err := l.reader.ReadRune()
 		if err != nil {
 			if len(val) > 0 {
-				return makeToken()
+				return makeToken(0)
 			}
 			if err == io.EOF {
 				return false
@@ -110,10 +112,10 @@ func (l *lexer) next() bool {
 				escaped = false
 			} else {
 				if quoted && ch == '"' {
-					return makeToken()
+					return makeToken('"')
 				}
 				if btQuoted && ch == '`' {
-					return makeToken()
+					return makeToken('`')
 				}
 			}
 			if ch == '\n' {
@@ -139,7 +141,7 @@ func (l *lexer) next() bool {
 				comment = false
 			}
 			if len(val) > 0 {
-				return makeToken()
+				return makeToken(0)
 			}
 			continue
 		}
@@ -189,3 +191,7 @@ func Tokenize(input []byte, filename string) ([]Token, error) {
 	}
 	return tokens, nil
 }
+
+func (t Token) Quoted() bool {
+	return t.wasQuoted > 0
+}

caddyconfig/caddyfile/lexer_fuzz.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build gofuzz
+//go:build gofuzz
 
 package caddyfile
 

caddyconfig/caddyfile/lexer_test.go

@@ -1,4 +1,4 @@
-// Copyright 2015 Light Code Labs, LLC
+// Copyright 2015 Matthew Holt and The Caddy Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

caddyconfig/caddyfile/parse.go

@@ -1,4 +1,4 @@
-// Copyright 2015 Light Code Labs, LLC
+// Copyright 2015 Matthew Holt and The Caddy Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,14 +17,14 @@ package caddyfile
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
-	"log"
+	"io"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/caddyserver/caddy/v2"
+	"go.uber.org/zap"
 )
 
 // Parse parses the input just enough to group tokens, in
@@ -37,7 +37,13 @@ import (
 // Environment variables in {$ENVIRONMENT_VARIABLE} notation
 // will be replaced before parsing begins.
 func Parse(filename string, input []byte) ([]ServerBlock, error) {
-	tokens, err := allTokens(filename, input)
+	// unfortunately, we must copy the input because parsing must
+	// remain a read-only operation, but we have to expand environment
+	// variables before we parse, which changes the underlying array (#4422)
+	inputCopy := make([]byte, len(input))
+	copy(inputCopy, input)
+
+	tokens, err := allTokens(filename, inputCopy)
 	if err != nil {
 		return nil, err
 	}
@@ -51,7 +57,23 @@ func Parse(filename string, input []byte) ([]ServerBlock, error) {
 	return p.parseAll()
 }
 
+// allTokens lexes the entire input, but does not parse it.
+// It returns all the tokens from the input, unstructured
+// and in order. It may mutate input as it expands env vars.
+func allTokens(filename string, input []byte) ([]Token, error) {
+	inputCopy, err := replaceEnvVars(input)
+	if err != nil {
+		return nil, err
+	}
+	tokens, err := Tokenize(inputCopy, filename)
+	if err != nil {
+		return nil, err
+	}
+	return tokens, nil
+}
+
 // replaceEnvVars replaces all occurrences of environment variables.
+// It mutates the underlying array and returns the updated slice.
 func replaceEnvVars(input []byte) ([]byte, error) {
 	var offset int
 	for {
@@ -96,21 +118,6 @@ func replaceEnvVars(input []byte) ([]byte, error) {
 	return input, nil
 }
 
-// allTokens lexes the entire input, but does not parse it.
-// It returns all the tokens from the input, unstructured
-// and in order.
-func allTokens(filename string, input []byte) ([]Token, error) {
-	input, err := replaceEnvVars(input)
-	if err != nil {
-		return nil, err
-	}
-	tokens, err := Tokenize(input, filename)
-	if err != nil {
-		return nil, err
-	}
-	return tokens, nil
-}
-
 type parser struct {
 	*Dispenser
 	block           ServerBlock // current server block being parsed
@@ -211,6 +218,12 @@ func (p *parser) addresses() error {
 			if expectingAnother {
 				return p.Errf("Expected another address but had '%s' - check for extra comma", tkn)
 			}
+			// Mark this server block as being defined with braces.
+			// This is used to provide a better error message when
+			// the user may have tried to define two server blocks
+			// without having used braces, which are required in
+			// that case.
+			p.block.HasBraces = true
 			break
 		}
 
@@ -229,6 +242,13 @@ func (p *parser) addresses() error {
 				expectingAnother = false // but we may still see another one on this line
 			}
 
+			// If there's a comma here, it's probably because they didn't use a space
+			// between their two domains, e.g. "foo.com,bar.com", which would not be
+			// parsed as two separate site addresses.
+			if strings.Contains(tkn, ",") {
+				return p.Errf("Site addresses cannot contain a comma ',': '%s' - put a space after the comma to separate site addresses", tkn)
+			}
+
 			p.block.Keys = append(p.block.Keys, tkn)
 		}
 
@@ -373,7 +393,7 @@ func (p *parser) doImport() error {
 		}
 		if len(matches) == 0 {
 			if strings.ContainsAny(globPattern, "*?[]") {
-				log.Printf("[WARNING] No files matching import glob pattern: %s", importPattern)
+				caddy.Log().Warn("No files matching import glob pattern", zap.String("pattern", importPattern))
 			} else {
 				return p.Errf("File to import not found: %s", importPattern)
 			}
@@ -434,7 +454,7 @@ func (p *parser) doSingleImport(importFile string) ([]Token, error) {
 		return nil, p.Errf("Could not import %s: is a directory", importFile)
 	}
 
-	input, err := ioutil.ReadAll(file)
+	input, err := io.ReadAll(file)
 	if err != nil {
 		return nil, p.Errf("Could not read imported file %s: %v", importFile, err)
 	}
@@ -474,6 +494,13 @@ func (p *parser) directive() error {
 	for p.Next() {
 		if p.Val() == "{" {
 			p.nesting++
+			if !p.isNextOnNewLine() && p.Token().wasQuoted == 0 {
+				return p.Err("Unexpected next token after '{' on same line")
+			}
+		} else if p.Val() == "{}" {
+			if p.isNextOnNewLine() && p.Token().wasQuoted == 0 {
+				return p.Err("Unexpected '{}' at end of line")
+			}
 		} else if p.isNewLine() && p.nesting == 0 {
 			p.cursor-- // read too far
 			break
@@ -564,8 +591,9 @@ func (p *parser) snippetTokens() ([]Token, error) {
 // head of the server block with tokens, which are
 // grouped by segments.
 type ServerBlock struct {
-	Keys     []string
-	Segments []Segment
+	HasBraces bool
+	Keys      []string
+	Segments  []Segment
 }
 
 // DispenseDirective returns a dispenser that contains

caddyconfig/caddyfile/parse_test.go

@@ -1,4 +1,4 @@
-// Copyright 2015 Light Code Labs, LLC
+// Copyright 2015 Matthew Holt and The Caddy Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,7 +16,6 @@ package caddyfile
 
 import (
 	"bytes"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
@@ -192,6 +191,20 @@ func TestParseOneAndImport(t *testing.T) {
 
 		{``, false, []string{}, []int{}},
 
+		// Unexpected next token after '{' on same line
+		{`localhost
+		  dir1 { a b }`, true, []string{"localhost"}, []int{}},
+		// Workaround with quotes
+		{`localhost
+		  dir1 "{" a b "}"`, false, []string{"localhost"}, []int{5}},
+
+		// Unexpected '{}' at end of line
+		{`localhost
+		  dir1 {}`, true, []string{"localhost"}, []int{}},
+		// Workaround with quotes
+		{`localhost
+		  dir1 "{}"`, false, []string{"localhost"}, []int{2}},
+
 		// import with args
 		{`import testdata/import_args0.txt a`, false, []string{"a"}, []int{}},
 		{`import testdata/import_args1.txt a b`, false, []string{"a", "b"}, []int{}},
@@ -280,7 +293,7 @@ func TestRecursiveImport(t *testing.T) {
 	}
 
 	// test relative recursive import
-	err = ioutil.WriteFile(recursiveFile1, []byte(
+	err = os.WriteFile(recursiveFile1, []byte(
 		`localhost
 		dir1
 		import recursive_import_test2`), 0644)
@@ -289,7 +302,7 @@ func TestRecursiveImport(t *testing.T) {
 	}
 	defer os.Remove(recursiveFile1)
 
-	err = ioutil.WriteFile(recursiveFile2, []byte("dir2 1"), 0644)
+	err = os.WriteFile(recursiveFile2, []byte("dir2 1"), 0644)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -314,7 +327,7 @@ func TestRecursiveImport(t *testing.T) {
 	}
 
 	// test absolute recursive import
-	err = ioutil.WriteFile(recursiveFile1, []byte(
+	err = os.WriteFile(recursiveFile1, []byte(
 		`localhost
 		dir1
 		import `+recursiveFile2), 0644)
@@ -370,7 +383,7 @@ func TestDirectiveImport(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = ioutil.WriteFile(directiveFile, []byte(`prop1 1
+	err = os.WriteFile(directiveFile, []byte(`prop1 1
 	prop2 2`), 0644)
 	if err != nil {
 		t.Fatal(err)
@@ -633,7 +646,7 @@ func TestSnippets(t *testing.T) {
 }
 
 func writeStringToTempFileOrDie(t *testing.T, str string) (pathToFile string) {
-	file, err := ioutil.TempFile("", t.Name())
+	file, err := os.CreateTemp("", t.Name())
 	if err != nil {
 		panic(err) // get a stack trace so we know where this was called from.
 	}

caddyconfig/configadapters.go

@@ -24,7 +24,7 @@ import (
 // Adapter is a type which can adapt a configuration to Caddy JSON.
 // It returns the results and any warnings, or an error.
 type Adapter interface {
-	Adapt(body []byte, options map[string]interface{}) ([]byte, []Warning, error)
+	Adapt(body []byte, options map[string]any) ([]byte, []Warning, error)
 }
 
 // Warning represents a warning or notice related to conversion.
@@ -48,7 +48,7 @@ func (w Warning) String() string {
 // are converted to warnings. This is convenient when filling config
 // structs that require a json.RawMessage, without having to worry
 // about errors.
-func JSON(val interface{}, warnings *[]Warning) json.RawMessage {
+func JSON(val any, warnings *[]Warning) json.RawMessage {
 	b, err := json.Marshal(val)
 	if err != nil {
 		if warnings != nil {
@@ -64,9 +64,9 @@ func JSON(val interface{}, warnings *[]Warning) json.RawMessage {
 // for encoding module values where the module name has to be described within
 // the object by a certain key; for example, `"handler": "file_server"` for a
 // file server HTTP handler (fieldName="handler" and fieldVal="file_server").
-// The val parameter must encode into a map[string]interface{} (i.e. it must be
+// The val parameter must encode into a map[string]any (i.e. it must be
 // a struct or map). Any errors are converted into warnings.
-func JSONModuleObject(val interface{}, fieldName, fieldVal string, warnings *[]Warning) json.RawMessage {
+func JSONModuleObject(val any, fieldName, fieldVal string, warnings *[]Warning) json.RawMessage {
 	// encode to a JSON object first
 	enc, err := json.Marshal(val)
 	if err != nil {
@@ -77,7 +77,7 @@ func JSONModuleObject(val interface{}, fieldName, fieldVal string, warnings *[]W
 	}
 
 	// then decode the object
-	var tmp map[string]interface{}
+	var tmp map[string]any
 	err = json.Unmarshal(enc, &tmp)
 	if err != nil {
 		if warnings != nil {

caddyconfig/httpcaddyfile/addresses.go

@@ -17,6 +17,7 @@ package httpcaddyfile
 import (
 	"fmt"
 	"net"
+	"net/netip"
 	"reflect"
 	"sort"
 	"strconv"
@@ -76,7 +77,7 @@ import (
 // multiple addresses to the same lists of server blocks (a many:many mapping).
 // (Doing this is essentially a map-reduce technique.)
 func (st *ServerType) mapAddressToServerBlocks(originalServerBlocks []serverBlock,
-	options map[string]interface{}) (map[string][]serverBlock, error) {
+	options map[string]any) (map[string][]serverBlock, error) {
 	sbmap := make(map[string][]serverBlock)
 
 	for i, sblock := range originalServerBlocks {
@@ -102,12 +103,20 @@ func (st *ServerType) mapAddressToServerBlocks(originalServerBlocks []serverBloc
 			}
 		}
 
+		// make a slice of the map keys so we can iterate in sorted order
+		addrs := make([]string, 0, len(addrToKeys))
+		for k := range addrToKeys {
+			addrs = append(addrs, k)
+		}
+		sort.Strings(addrs)
+
 		// now that we know which addresses serve which keys of this
 		// server block, we iterate that mapping and create a list of
 		// new server blocks for each address where the keys of the
 		// server block are only the ones which use the address; but
 		// the contents (tokens) are of course the same
-		for addr, keys := range addrToKeys {
+		for _, addr := range addrs {
+			keys := addrToKeys[addr]
 			// parse keys so that we only have to do it once
 			parsedKeys := make([]Address, 0, len(keys))
 			for _, key := range keys {
@@ -161,6 +170,7 @@ func (st *ServerType) consolidateAddrMappings(addrToServerBlocks map[string][]se
 				delete(addrToServerBlocks, otherAddr)
 			}
 		}
+		sort.Strings(a.addresses)
 
 		sbaddrs = append(sbaddrs, a)
 	}
@@ -174,8 +184,10 @@ func (st *ServerType) consolidateAddrMappings(addrToServerBlocks map[string][]se
 	return sbaddrs
 }
 
+// listenerAddrsForServerBlockKey essentially converts the Caddyfile
+// site addresses to Caddy listener addresses for each server block.
 func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key string,
-	options map[string]interface{}) ([]string, error) {
+	options map[string]any) ([]string, error) {
 	addr, err := ParseAddress(key)
 	if err != nil {
 		return nil, fmt.Errorf("parsing key: %v", err)
@@ -208,23 +220,29 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 	}
 
 	// the bind directive specifies hosts, but is optional
-	lnHosts := make([]string, 0, len(sblock.pile))
+	lnHosts := make([]string, 0, len(sblock.pile["bind"]))
 	for _, cfgVal := range sblock.pile["bind"] {
 		lnHosts = append(lnHosts, cfgVal.Value.([]string)...)
 	}
 	if len(lnHosts) == 0 {
-		lnHosts = []string{""}
+		if defaultBind, ok := options["default_bind"].([]string); ok {
+			lnHosts = defaultBind
+		} else {
+			lnHosts = []string{""}
+		}
 	}
 
 	// use a map to prevent duplication
 	listeners := make(map[string]struct{})
 	for _, host := range lnHosts {
-		addr, err := caddy.ParseNetworkAddress(host)
-		if err == nil && addr.IsUnixNetwork() {
-			listeners[host] = struct{}{}
-		} else {
-			listeners[net.JoinHostPort(host, lnPort)] = struct{}{}
+		// host can have network + host (e.g. "tcp6/localhost") but
+		// will/should not have port information because this usually
+		// comes from the bind directive, so we append the port
+		addr, err := caddy.ParseNetworkAddress(host + ":" + lnPort)
+		if err != nil {
+			return nil, fmt.Errorf("parsing network address: %v", err)
 		}
+		listeners[addr.String()] = struct{}{}
 	}
 
 	// now turn map into list
@@ -232,6 +250,7 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 	for lnStr := range listeners {
 		listenersList = append(listenersList, lnStr)
 	}
+	sort.Strings(listenersList)
 
 	return listenersList, nil
 }
@@ -336,8 +355,10 @@ func (a Address) Normalize() Address {
 
 	// ensure host is normalized if it's an IP address
 	host := strings.TrimSpace(a.Host)
-	if ip := net.ParseIP(host); ip != nil {
-		host = ip.String()
+	if ip, err := netip.ParseAddr(host); err == nil {
+		if ip.Is6() && !ip.Is4() && !ip.Is4In6() {
+			host = ip.String()
+		}
 	}
 
 	return Address{
@@ -349,28 +370,6 @@ func (a Address) Normalize() Address {
 	}
 }
 
-// Key returns a string form of a, much like String() does, but this
-// method doesn't add anything default that wasn't in the original.
-func (a Address) Key() string {
-	res := ""
-	if a.Scheme != "" {
-		res += a.Scheme + "://"
-	}
-	if a.Host != "" {
-		res += a.Host
-	}
-	// insert port only if the original has its own explicit port
-	if a.Port != "" &&
-		len(a.Original) >= len(res) &&
-		strings.HasPrefix(a.Original[len(res):], ":"+a.Port) {
-		res += ":" + a.Port
-	}
-	if a.Path != "" {
-		res += a.Path
-	}
-	return res
-}
-
 // lowerExceptPlaceholders lowercases s except within
 // placeholders (substrings in non-escaped '{ }' spans).
 // See https://github.com/caddyserver/caddy/issues/3264

caddyconfig/httpcaddyfile/addresses_fuzz.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build gofuzz
+//go:build gofuzz
 
 package httpcaddyfile
 

caddyconfig/httpcaddyfile/addresses_test.go

@@ -106,67 +106,128 @@ func TestAddressString(t *testing.T) {
 func TestKeyNormalization(t *testing.T) {
 	testCases := []struct {
 		input  string
-		expect string
+		expect Address
 	}{
 		{
-			input:  "example.com",
-			expect: "example.com",
+			input: "example.com",
+			expect: Address{
+				Host: "example.com",
+			},
 		},
 		{
-			input:  "http://host:1234/path",
-			expect: "http://host:1234/path",
+			input: "http://host:1234/path",
+			expect: Address{
+				Scheme: "http",
+				Host:   "host",
+				Port:   "1234",
+				Path:   "/path",
+			},
 		},
 		{
-			input:  "HTTP://A/ABCDEF",
-			expect: "http://a/ABCDEF",
+			input: "HTTP://A/ABCDEF",
+			expect: Address{
+				Scheme: "http",
+				Host:   "a",
+				Path:   "/ABCDEF",
+			},
 		},
 		{
-			input:  "A/ABCDEF",
-			expect: "a/ABCDEF",
+			input: "A/ABCDEF",
+			expect: Address{
+				Host: "a",
+				Path: "/ABCDEF",
+			},
 		},
 		{
-			input:  "A:2015/Path",
-			expect: "a:2015/Path",
+			input: "A:2015/Path",
+			expect: Address{
+				Host: "a",
+				Port: "2015",
+				Path: "/Path",
+			},
 		},
 		{
-			input:  "sub.{env.MY_DOMAIN}",
-			expect: "sub.{env.MY_DOMAIN}",
+			input: "sub.{env.MY_DOMAIN}",
+			expect: Address{
+				Host: "sub.{env.MY_DOMAIN}",
+			},
 		},
 		{
-			input:  "sub.ExAmPle",
-			expect: "sub.example",
+			input: "sub.ExAmPle",
+			expect: Address{
+				Host: "sub.example",
+			},
 		},
 		{
-			input:  "sub.\\{env.MY_DOMAIN\\}",
-			expect: "sub.\\{env.my_domain\\}",
+			input: "sub.\\{env.MY_DOMAIN\\}",
+			expect: Address{
+				Host: "sub.\\{env.my_domain\\}",
+			},
 		},
 		{
-			input:  "sub.{env.MY_DOMAIN}.com",
-			expect: "sub.{env.MY_DOMAIN}.com",
+			input: "sub.{env.MY_DOMAIN}.com",
+			expect: Address{
+				Host: "sub.{env.MY_DOMAIN}.com",
+			},
 		},
 		{
-			input:  ":80",
-			expect: ":80",
+			input: ":80",
+			expect: Address{
+				Port: "80",
+			},
 		},
 		{
-			input:  ":443",
-			expect: ":443",
+			input: ":443",
+			expect: Address{
+				Port: "443",
+			},
 		},
 		{
-			input:  ":1234",
-			expect: ":1234",
+			input: ":1234",
+			expect: Address{
+				Port: "1234",
+			},
 		},
 		{
 			input:  "",
-			expect: "",
+			expect: Address{},
 		},
 		{
 			input:  ":",
-			expect: "",
+			expect: Address{},
 		},
 		{
-			input:  "[::]",
-			expect: "::",
+			input: "[::]",
+			expect: Address{
+				Host: "::",
+			},
+		},
+		{
+			input: "127.0.0.1",
+			expect: Address{
+				Host: "127.0.0.1",
+			},
+		},
+		{
+			input: "[2001:db8:85a3:8d3:1319:8a2e:370:7348]:1234",
+			expect: Address{
+				Host: "2001:db8:85a3:8d3:1319:8a2e:370:7348",
+				Port: "1234",
+			},
+		},
+		{
+			// IPv4 address in IPv6 form (#4381)
+			input: "[::ffff:cff4:e77d]:1234",
+			expect: Address{
+				Host: "::ffff:cff4:e77d",
+				Port: "1234",
+			},
+		},
+		{
+			input: "::ffff:cff4:e77d",
+			expect: Address{
+				Host: "::ffff:cff4:e77d",
+			},
 		},
 	}
 	for i, tc := range testCases {
@@ -175,9 +236,18 @@ func TestKeyNormalization(t *testing.T) {
 			t.Errorf("Test %d: Parsing address '%s': %v", i, tc.input, err)
 			continue
 		}
-		if actual := addr.Normalize().Key(); actual != tc.expect {
-			t.Errorf("Test %d: Input '%s': Expected '%s' but got '%s'", i, tc.input, tc.expect, actual)
+		actual := addr.Normalize()
+		if actual.Scheme != tc.expect.Scheme {
+			t.Errorf("Test %d: Input '%s': Expected Scheme='%s' but got Scheme='%s'", i, tc.input, tc.expect.Scheme, actual.Scheme)
+		}
+		if actual.Host != tc.expect.Host {
+			t.Errorf("Test %d: Input '%s': Expected Host='%s' but got Host='%s'", i, tc.input, tc.expect.Host, actual.Host)
+		}
+		if actual.Port != tc.expect.Port {
+			t.Errorf("Test %d: Input '%s': Expected Port='%s' but got Port='%s'", i, tc.input, tc.expect.Port, actual.Port)
+		}
+		if actual.Path != tc.expect.Path {
+			t.Errorf("Test %d: Input '%s': Expected Path='%s' but got Path='%s'", i, tc.input, tc.expect.Path, actual.Path)
 		}
-
 	}
 }

caddyconfig/httpcaddyfile/builtins.go

@@ -19,8 +19,8 @@ import (
 	"encoding/pem"
 	"fmt"
 	"html"
-	"io/ioutil"
 	"net/http"
+	"os"
 	"reflect"
 	"strconv"
 	"strings"
@@ -39,6 +39,7 @@ func init() {
 	RegisterDirective("bind", parseBind)
 	RegisterDirective("tls", parseTLS)
 	RegisterHandlerDirective("root", parseRoot)
+	RegisterHandlerDirective("vars", parseVars)
 	RegisterHandlerDirective("redir", parseRedir)
 	RegisterHandlerDirective("respond", parseRespond)
 	RegisterHandlerDirective("abort", parseAbort)
@@ -82,6 +83,7 @@ func parseBind(h Helper) ([]ConfigValue, error) {
 //         on_demand
 //         eab    <key_id> <mac_key>
 //         issuer <module_name> [...]
+//         get_certificate <module_name> [...]
 //     }
 //
 func parseTLS(h Helper) ([]ConfigValue, error) {
@@ -93,6 +95,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 	var keyType string
 	var internalIssuer *caddytls.InternalIssuer
 	var issuers []certmagic.Issuer
+	var certManagers []certmagic.Manager
 	var onDemand bool
 
 	for h.Next() {
@@ -230,7 +233,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 							return nil, h.ArgErr()
 						}
 						filename := h.Val()
-						certDataPEM, err := ioutil.ReadFile(filename)
+						certDataPEM, err := os.ReadFile(filename)
 						if err != nil {
 							return nil, err
 						}
@@ -307,6 +310,22 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 				}
 				issuers = append(issuers, issuer)
 
+			case "get_certificate":
+				if !h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				modName := h.Val()
+				modID := "tls.get_certificate." + modName
+				unm, err := caddyfile.UnmarshalModule(h.Dispenser, modID)
+				if err != nil {
+					return nil, err
+				}
+				certManager, ok := unm.(certmagic.Manager)
+				if !ok {
+					return nil, h.Errf("module %s (%T) is not a certmagic.CertificateManager", modID, unm)
+				}
+				certManagers = append(certManagers, certManager)
+
 			case "dns":
 				if !h.NextArg() {
 					return nil, h.ArgErr()
@@ -344,6 +363,22 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 				}
 				acmeIssuer.Challenges.DNS.Resolvers = args
 
+			case "dns_challenge_override_domain":
+				arg := h.RemainingArgs()
+				if len(arg) != 1 {
+					return nil, h.ArgErr()
+				}
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
+				if acmeIssuer.Challenges == nil {
+					acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
+				}
+				if acmeIssuer.Challenges.DNS == nil {
+					acmeIssuer.Challenges.DNS = new(caddytls.DNSChallengeConfig)
+				}
+				acmeIssuer.Challenges.DNS.OverrideDomain = arg[0]
+
 			case "ca_root":
 				arg := h.RemainingArgs()
 				if len(arg) != 1 {
@@ -453,6 +488,12 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 			Value: true,
 		})
 	}
+	for _, certManager := range certManagers {
+		configVals = append(configVals, ConfigValue{
+			Class: "tls.cert_manager",
+			Value: certManager,
+		})
+	}
 
 	// custom certificate selection
 	if len(certSelector.AnyTag) > 0 {
@@ -490,10 +531,22 @@ func parseRoot(h Helper) (caddyhttp.MiddlewareHandler, error) {
 	return caddyhttp.VarsMiddleware{"root": root}, nil
 }
 
+// parseVars parses the vars directive. See its UnmarshalCaddyfile method for syntax.
+func parseVars(h Helper) (caddyhttp.MiddlewareHandler, error) {
+	v := new(caddyhttp.VarsMiddleware)
+	err := v.UnmarshalCaddyfile(h.Dispenser)
+	return v, err
+}
+
 // parseRedir parses the redir directive. Syntax:
 //
-//     redir [<matcher>] <to> [<code>]
+//	redir [<matcher>] <to> [<code>]
 //
+// <code> can be "permanent" for 301, "temporary" for 302 (default),
+// a placeholder, or any number in the 3xx range or 401. The special
+// code "html" can be used to redirect only browser clients (will
+// respond with HTTP 200 and no Location header; redirect is performed
+// with JS and a meta tag).
 func parseRedir(h Helper) (caddyhttp.MiddlewareHandler, error) {
 	if !h.Next() {
 		return nil, h.ArgErr()
@@ -510,6 +563,7 @@ func parseRedir(h Helper) (caddyhttp.MiddlewareHandler, error) {
 	}
 
 	var body string
+	var hdr http.Header
 	switch code {
 	case "permanent":
 		code = "301"
@@ -530,20 +584,37 @@ func parseRedir(h Helper) (caddyhttp.MiddlewareHandler, error) {
 `
 		safeTo := html.EscapeString(to)
 		body = fmt.Sprintf(metaRedir, safeTo, safeTo, safeTo, safeTo)
-		code = "302"
+		code = "200" // don't redirect non-browser clients
 	default:
+		// Allow placeholders for the code
+		if strings.HasPrefix(code, "{") {
+			break
+		}
+		// Try to validate as an integer otherwise
 		codeInt, err := strconv.Atoi(code)
 		if err != nil {
 			return nil, h.Errf("Not a supported redir code type or not valid integer: '%s'", code)
 		}
-		if codeInt < 300 || codeInt > 399 {
-			return nil, h.Errf("Redir code not in the 3xx range: '%v'", codeInt)
+		// Sometimes, a 401 with Location header is desirable because
+		// requests made with XHR will "eat" the 3xx redirect; so if
+		// the intent was to redirect to an auth page, a 3xx won't
+		// work. Responding with 401 allows JS code to read the
+		// Location header and do a window.location redirect manually.
+		// see https://stackoverflow.com/a/2573589/846934
+		// see https://github.com/oauth2-proxy/oauth2-proxy/issues/1522
+		if codeInt < 300 || (codeInt > 399 && codeInt != 401) {
+			return nil, h.Errf("Redir code not in the 3xx range or 401: '%v'", codeInt)
 		}
 	}
 
+	// don't redirect non-browser clients
+	if code != "200" {
+		hdr = http.Header{"Location": []string{to}}
+	}
+
 	return caddyhttp.StaticResponse{
 		StatusCode: caddyhttp.WeakString(code),
-		Headers:    http.Header{"Location": []string{to}},
+		Headers:    hdr,
 		Body:       body,
 	}, nil
 }

caddyconfig/httpcaddyfile/builtins_test.go

@@ -37,8 +37,7 @@ func TestLogDirectiveSyntax(t *testing.T) {
 					format filter {
 						wrap console
 						fields {
-							common_log delete
-							request>remote_addr ip_mask {
+							request>remote_ip ip_mask {
 								ipv4 24
 								ipv6 32
 							}
@@ -47,7 +46,7 @@ func TestLogDirectiveSyntax(t *testing.T) {
 				}
 			}
 			`,
-			output:      `{"logging":{"logs":{"default":{"exclude":["http.log.access.log0"]},"log0":{"encoder":{"fields":{"common_log":{"filter":"delete"},"request\u003eremote_addr":{"filter":"ip_mask","ipv4_cidr":24,"ipv6_cidr":32}},"format":"filter","wrap":{"format":"console"}},"include":["http.log.access.log0"]}}},"apps":{"http":{"servers":{"srv0":{"listen":[":8080"],"logs":{"default_logger_name":"log0"}}}}}}`,
+			output:      `{"logging":{"logs":{"default":{"exclude":["http.log.access.log0"]},"log0":{"encoder":{"fields":{"request\u003eremote_ip":{"filter":"ip_mask","ipv4_cidr":24,"ipv6_cidr":32}},"format":"filter","wrap":{"format":"console"}},"include":["http.log.access.log0"]}}},"apps":{"http":{"servers":{"srv0":{"listen":[":8080"],"logs":{"default_logger_name":"log0"}}}}}}`,
 			expectError: false,
 		},
 		{
@@ -149,6 +148,27 @@ func TestRedirDirectiveSyntax(t *testing.T) {
 			}`,
 			expectError: false,
 		},
+		{
+			// this is now allowed so a Location header
+			// can be written and consumed by JS
+			// in the case of XHR requests
+			input: `:8080 {
+				redir * :8081 401
+			}`,
+			expectError: false,
+		},
+		{
+			input: `:8080 {
+				redir * :8081 402
+			}`,
+			expectError: true,
+		},
+		{
+			input: `:8080 {
+				redir * :8081 {http.reverse_proxy.status_code}
+			}`,
+			expectError: false,
+		},
 		{
 			input: `:8080 {
 				redir /old.html /new.html htlm
@@ -161,12 +181,6 @@ func TestRedirDirectiveSyntax(t *testing.T) {
 			}`,
 			expectError: true,
 		},
-		{
-			input: `:8080 {
-				redir * :8081 400
-			}`,
-			expectError: true,
-		},
 		{
 			input: `:8080 {
 				redir * :8081 temp

caddyconfig/httpcaddyfile/directives.go

@@ -37,40 +37,47 @@ import (
 // The header directive goes second so that headers
 // can be manipulated before doing redirects.
 var directiveOrder = []string{
+	"tracing",
+
 	"map",
+	"vars",
 	"root",
 
 	"header",
+	"copy_response_headers", // only in reverse_proxy's handle_response
 	"request_body",
 
 	"redir",
-	"rewrite",
 
-	// URI manipulation
+	// incoming request manipulation
+	"method",
+	"rewrite",
 	"uri",
 	"try_files",
 
 	// middleware handlers; some wrap responses
 	"basicauth",
+	"forward_auth",
 	"request_header",
 	"encode",
+	"push",
 	"templates",
 
 	// special routing & dispatching directives
 	"handle",
 	"handle_path",
 	"route",
-	"push",
 
 	// handlers that typically respond to requests
+	"abort",
+	"error",
+	"copy_response", // only in reverse_proxy's handle_response
 	"respond",
 	"metrics",
 	"reverse_proxy",
 	"php_fastcgi",
 	"file_server",
 	"acme_server",
-	"abort",
-	"error",
 }
 
 // directiveIsOrdered returns true if dir is
@@ -135,8 +142,8 @@ func RegisterGlobalOption(opt string, setupFunc UnmarshalGlobalFunc) {
 type Helper struct {
 	*caddyfile.Dispenser
 	// State stores intermediate variables during caddyfile adaptation.
-	State        map[string]interface{}
-	options      map[string]interface{}
+	State        map[string]any
+	options      map[string]any
 	warnings     *[]caddyconfig.Warning
 	matcherDefs  map[string]caddy.ModuleMap
 	parentBlock  caddyfile.ServerBlock
@@ -144,7 +151,7 @@ type Helper struct {
 }
 
 // Option gets the option keyed by name.
-func (h Helper) Option(name string) interface{} {
+func (h Helper) Option(name string) any {
 	return h.options[name]
 }
 
@@ -168,7 +175,7 @@ func (h Helper) Caddyfiles() []string {
 }
 
 // JSON converts val into JSON. Any errors are added to warnings.
-func (h Helper) JSON(val interface{}) json.RawMessage {
+func (h Helper) JSON(val any) json.RawMessage {
 	return caddyconfig.JSON(val, h.warnings)
 }
 
@@ -329,7 +336,7 @@ func parseSegmentAsConfig(h Helper) ([]ConfigValue, error) {
 			dir := seg.Directive()
 			dirFunc, ok := registeredDirectives[dir]
 			if !ok {
-				return nil, h.Errf("unrecognized directive: %s", dir)
+				return nil, h.Errf("unrecognized directive: %s - are you sure your Caddyfile structure (nesting and braces) is correct?", dir)
 			}
 
 			subHelper := h
@@ -340,6 +347,9 @@ func parseSegmentAsConfig(h Helper) ([]ConfigValue, error) {
 			if err != nil {
 				return nil, h.Errf("parsing caddyfile tokens for '%s': %v", dir, err)
 			}
+
+			dir = normalizeDirectiveName(dir)
+
 			for _, result := range results {
 				result.directive = dir
 				allResults = append(allResults, result)
@@ -365,7 +375,7 @@ type ConfigValue struct {
 	// The value to be used when building the config.
 	// Generally its type is associated with the
 	// name of the Class.
-	Value interface{}
+	Value any
 
 	directive string
 }
@@ -396,7 +406,7 @@ func sortRoutes(routes []ConfigValue) {
 			return false
 		}
 
-		// decode the path matchers, if there is just one of them
+		// decode the path matchers if there is just one matcher set
 		var iPM, jPM caddyhttp.MatchPath
 		if len(iRoute.MatcherSetsRaw) == 1 {
 			_ = json.Unmarshal(iRoute.MatcherSetsRaw[0]["path"], &iPM)
@@ -405,24 +415,46 @@ func sortRoutes(routes []ConfigValue) {
 			_ = json.Unmarshal(jRoute.MatcherSetsRaw[0]["path"], &jPM)
 		}
 
-		// sort by longer path (more specific) first; missing path
-		// matchers or multi-matchers are treated as zero-length paths
+		// if there is only one path in the path matcher, sort by longer path
+		// (more specific) first; missing path matchers or multi-matchers are
+		// treated as zero-length paths
 		var iPathLen, jPathLen int
-		if len(iPM) > 0 {
+		if len(iPM) == 1 {
 			iPathLen = len(iPM[0])
 		}
-		if len(jPM) > 0 {
+		if len(jPM) == 1 {
 			jPathLen = len(jPM[0])
 		}
 
-		// if both directives have no path matcher, use whichever one
-		// has any kind of matcher defined first.
-		if iPathLen == 0 && jPathLen == 0 {
+		// some directives involve setting values which can overwrite
+		// each other, so it makes most sense to reverse the order so
+		// that the lease specific matcher is first; everything else
+		// has most-specific matcher first
+		if iDir == "vars" {
+			// we can only confidently compare path lengths if both
+			// directives have a single path to match (issue #5037)
+			if iPathLen > 0 && jPathLen > 0 {
+				// sort least-specific (shortest) path first
+				return iPathLen < jPathLen
+			}
+
+			// if both directives don't have a single path to compare,
+			// sort whichever one has no matcher first; if both have
+			// no matcher, sort equally (stable sort preserves order)
+			return len(iRoute.MatcherSetsRaw) == 0 && len(jRoute.MatcherSetsRaw) > 0
+		} else {
+			// we can only confidently compare path lengths if both
+			// directives have a single path to match (issue #5037)
+			if iPathLen > 0 && jPathLen > 0 {
+				// sort most-specific (longest) path first
+				return iPathLen > jPathLen
+			}
+
+			// if both directives don't have a single path to compare,
+			// sort whichever one has a matcher first; if both have
+			// a matcher, sort equally (stable sort preserves order)
 			return len(iRoute.MatcherSetsRaw) > 0 && len(jRoute.MatcherSetsRaw) == 0
 		}
-
-		// sort with the most-specific (longest) path first
-		return iPathLen > jPathLen
 	})
 }
 
@@ -478,6 +510,27 @@ func (sb serverBlock) hostsFromKeys(loggerMode bool) []string {
 	return sblockHosts
 }
 
+func (sb serverBlock) hostsFromKeysNotHTTP(httpPort string) []string {
+	// ensure each entry in our list is unique
+	hostMap := make(map[string]struct{})
+	for _, addr := range sb.keys {
+		if addr.Host == "" {
+			continue
+		}
+		if addr.Scheme != "http" && addr.Port != httpPort {
+			hostMap[addr.Host] = struct{}{}
+		}
+	}
+
+	// convert map to slice
+	sblockHosts := make([]string, 0, len(hostMap))
+	for host := range hostMap {
+		sblockHosts = append(sblockHosts, host)
+	}
+
+	return sblockHosts
+}
+
 // hasHostCatchAllKey returns true if sb has a key that
 // omits a host portion, i.e. it "catches all" hosts.
 func (sb serverBlock) hasHostCatchAllKey() bool {
@@ -489,6 +542,17 @@ func (sb serverBlock) hasHostCatchAllKey() bool {
 	return false
 }
 
+// isAllHTTP returns true if all sb keys explicitly specify
+// the http:// scheme
+func (sb serverBlock) isAllHTTP() bool {
+	for _, addr := range sb.keys {
+		if addr.Scheme != "http" {
+			return false
+		}
+	}
+	return true
+}
+
 type (
 	// UnmarshalFunc is a function which can unmarshal Caddyfile
 	// tokens into zero or more config values using a Helper type.
@@ -510,7 +574,7 @@ type (
 	// tokens from a global option. It is passed the tokens to parse and
 	// existing value from the previous instance of this global option
 	// (if any). It returns the value to associate with this global option.
-	UnmarshalGlobalFunc func(d *caddyfile.Dispenser, existingVal interface{}) (interface{}, error)
+	UnmarshalGlobalFunc func(d *caddyfile.Dispenser, existingVal any) (any, error)
 )
 
 var registeredDirectives = make(map[string]UnmarshalFunc)

caddyconfig/httpcaddyfile/httptype.go

@@ -17,7 +17,6 @@ package httpcaddyfile
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"reflect"
 	"regexp"
 	"sort"
@@ -30,6 +29,7 @@ import (
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddypki"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
+	"go.uber.org/zap"
 )
 
 func init() {
@@ -53,27 +53,18 @@ type ServerType struct {
 
 // Setup makes a config from the tokens.
 func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
-	options map[string]interface{}) (*caddy.Config, []caddyconfig.Warning, error) {
+	options map[string]any) (*caddy.Config, []caddyconfig.Warning, error) {
 	var warnings []caddyconfig.Warning
 	gc := counter{new(int)}
-	state := make(map[string]interface{})
+	state := make(map[string]any)
 
-	// load all the server blocks and associate them with a "pile"
-	// of config values; also prohibit duplicate keys because they
-	// can make a config confusing if more than one server block is
-	// chosen to handle a request - we actually will make each
-	// server block's route terminal so that only one will run
-	sbKeys := make(map[string]struct{})
+	// load all the server blocks and associate them with a "pile" of config values
 	originalServerBlocks := make([]serverBlock, 0, len(inputServerBlocks))
-	for i, sblock := range inputServerBlocks {
+	for _, sblock := range inputServerBlocks {
 		for j, k := range sblock.Keys {
 			if j == 0 && strings.HasPrefix(k, "@") {
 				return nil, warnings, fmt.Errorf("cannot define a matcher outside of a site block: '%s'", k)
 			}
-			if _, ok := sbKeys[k]; ok {
-				return nil, warnings, fmt.Errorf("duplicate site address not allowed: '%s' in %v (site block %d, key %d)", k, sblock.Keys, i, j)
-			}
-			sbKeys[k] = struct{}{}
 		}
 		originalServerBlocks = append(originalServerBlocks, serverBlock{
 			block: sblock,
@@ -88,32 +79,10 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		return nil, warnings, err
 	}
 
-	// replace shorthand placeholders (which are
-	// convenient when writing a Caddyfile) with
-	// their actual placeholder identifiers or
-	// variable names
-	replacer := strings.NewReplacer(
-		"{dir}", "{http.request.uri.path.dir}",
-		"{file}", "{http.request.uri.path.file}",
-		"{host}", "{http.request.host}",
-		"{hostport}", "{http.request.hostport}",
-		"{port}", "{http.request.port}",
-		"{method}", "{http.request.method}",
-		"{path}", "{http.request.uri.path}",
-		"{query}", "{http.request.uri.query}",
-		"{remote}", "{http.request.remote}",
-		"{remote_host}", "{http.request.remote.host}",
-		"{remote_port}", "{http.request.remote.port}",
-		"{scheme}", "{http.request.scheme}",
-		"{uri}", "{http.request.uri}",
-		"{tls_cipher}", "{http.request.tls.cipher_suite}",
-		"{tls_version}", "{http.request.tls.version}",
-		"{tls_client_fingerprint}", "{http.request.tls.client.fingerprint}",
-		"{tls_client_issuer}", "{http.request.tls.client.issuer}",
-		"{tls_client_serial}", "{http.request.tls.client.serial}",
-		"{tls_client_subject}", "{http.request.tls.client.subject}",
-		"{tls_client_certificate_pem}", "{http.request.tls.client.certificate_pem}",
-	)
+	// replace shorthand placeholders (which are convenient
+	// when writing a Caddyfile) with their actual placeholder
+	// identifiers or variable names
+	replacer := strings.NewReplacer(placeholderShorthands()...)
 
 	// these are placeholders that allow a user-defined final
 	// parameters, but we still want to provide a shorthand
@@ -122,11 +91,17 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		search  *regexp.Regexp
 		replace string
 	}{
-		{regexp.MustCompile(`{query\.([\w-]*)}`), "{http.request.uri.query.$1}"},
-		{regexp.MustCompile(`{labels\.([\w-]*)}`), "{http.request.host.labels.$1}"},
 		{regexp.MustCompile(`{header\.([\w-]*)}`), "{http.request.header.$1}"},
+		{regexp.MustCompile(`{cookie\.([\w-]*)}`), "{http.request.cookie.$1}"},
+		{regexp.MustCompile(`{labels\.([\w-]*)}`), "{http.request.host.labels.$1}"},
 		{regexp.MustCompile(`{path\.([\w-]*)}`), "{http.request.uri.path.$1}"},
+		{regexp.MustCompile(`{file\.([\w-]*)}`), "{http.request.uri.path.file.$1}"},
+		{regexp.MustCompile(`{query\.([\w-]*)}`), "{http.request.uri.query.$1}"},
 		{regexp.MustCompile(`{re\.([\w-]*)\.([\w-]*)}`), "{http.regexp.$1.$2}"},
+		{regexp.MustCompile(`{vars\.([\w-]*)}`), "{http.vars.$1}"},
+		{regexp.MustCompile(`{rp\.([\w-\.]*)}`), "{http.reverse_proxy.$1}"},
+		{regexp.MustCompile(`{err\.([\w-\.]*)}`), "{http.error.$1}"},
+		{regexp.MustCompile(`{file_match\.([\w-]*)}`), "{http.matchers.file.$1}"},
 	}
 
 	for _, sb := range originalServerBlocks {
@@ -169,7 +144,11 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 			dirFunc, ok := registeredDirectives[dir]
 			if !ok {
 				tkn := segment[0]
-				return nil, warnings, fmt.Errorf("%s:%d: unrecognized directive: %s", tkn.File, tkn.Line, dir)
+				message := "%s:%d: unrecognized directive: %s"
+				if !sb.block.HasBraces {
+					message += "\nDid you mean to define a second site? If so, you must use curly braces around each site to separate their configurations."
+				}
+				return nil, warnings, fmt.Errorf(message, tkn.File, tkn.Line, dir)
 			}
 
 			h := Helper{
@@ -187,13 +166,7 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 				return nil, warnings, fmt.Errorf("parsing caddyfile tokens for '%s': %v", dir, err)
 			}
 
-			// As a special case, we want "handle_path" to be sorted
-			// at the same level as "handle", so we force them to use
-			// the same directive name after their parsing is complete.
-			// See https://github.com/caddyserver/caddy/issues/3675#issuecomment-678042377
-			if dir == "handle_path" {
-				dir = "handle"
-			}
+			dir = normalizeDirectiveName(dir)
 
 			for _, result := range results {
 				result.directive = dir
@@ -220,10 +193,11 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 
 	// now that each server is configured, make the HTTP app
 	httpApp := caddyhttp.App{
-		HTTPPort:    tryInt(options["http_port"], &warnings),
-		HTTPSPort:   tryInt(options["https_port"], &warnings),
-		GracePeriod: tryDuration(options["grace_period"], &warnings),
-		Servers:     servers,
+		HTTPPort:      tryInt(options["http_port"], &warnings),
+		HTTPSPort:     tryInt(options["https_port"], &warnings),
+		GracePeriod:   tryDuration(options["grace_period"], &warnings),
+		ShutdownDelay: tryDuration(options["shutdown_delay"], &warnings),
+		Servers:       servers,
 	}
 
 	// then make the TLS app
@@ -253,20 +227,13 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		}
 		customLogs = append(customLogs, ncl)
 	}
+
 	// Apply global log options, when set
 	if options["log"] != nil {
 		for _, logValue := range options["log"].([]ConfigValue) {
 			addCustomLog(logValue.Value.(namedCustomLog))
 		}
 	}
-	// Apply server-specific log options
-	for _, p := range pairings {
-		for _, sb := range p.serverBlocks {
-			for _, clVal := range sb.pile["custom_log"] {
-				addCustomLog(clVal.Value.(namedCustomLog))
-			}
-		}
-	}
 
 	if !hasDefaultLog {
 		// if the default log was not customized, ensure we
@@ -279,6 +246,15 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		}
 	}
 
+	// Apply server-specific log options
+	for _, p := range pairings {
+		for _, sb := range p.serverBlocks {
+			for _, clVal := range sb.pile["custom_log"] {
+				addCustomLog(clVal.Value.(namedCustomLog))
+			}
+		}
+	}
+
 	// annnd the top-level config, then we're done!
 	cfg := &caddy.Config{AppsRaw: make(caddy.ModuleMap)}
 
@@ -341,14 +317,14 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 // which is expected to be the first server block if it has zero
 // keys. It returns the updated list of server blocks with the
 // global options block removed, and updates options accordingly.
-func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options map[string]interface{}) ([]serverBlock, error) {
+func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options map[string]any) ([]serverBlock, error) {
 	if len(serverBlocks) == 0 || len(serverBlocks[0].block.Keys) > 0 {
 		return serverBlocks, nil
 	}
 
 	for _, segment := range serverBlocks[0].block.Segments {
 		opt := segment.Directive()
-		var val interface{}
+		var val any
 		var err error
 		disp := caddyfile.NewDispenser(segment)
 
@@ -418,7 +394,7 @@ func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options
 // to server blocks. Each pairing is essentially a server definition.
 func (st *ServerType) serversFromPairings(
 	pairings []sbAddrAssociation,
-	options map[string]interface{},
+	options map[string]any,
 	warnings *[]caddyconfig.Warning,
 	groupCounter counter,
 ) (map[string]*caddyhttp.Server, error) {
@@ -439,6 +415,23 @@ func (st *ServerType) serversFromPairings(
 	}
 
 	for i, p := range pairings {
+		// detect ambiguous site definitions: server blocks which
+		// have the same host bound to the same interface (listener
+		// address), otherwise their routes will improperly be added
+		// to the same server (see issue #4635)
+		for j, sblock1 := range p.serverBlocks {
+			for _, key := range sblock1.block.Keys {
+				for k, sblock2 := range p.serverBlocks {
+					if k == j {
+						continue
+					}
+					if sliceContains(sblock2.block.Keys, key) {
+						return nil, fmt.Errorf("ambiguous site definition: %s", key)
+					}
+				}
+			}
+		}
+
 		srv := &caddyhttp.Server{
 			Listen: p.addresses,
 		}
@@ -446,17 +439,29 @@ func (st *ServerType) serversFromPairings(
 		// handle the auto_https global option
 		if autoHTTPS != "on" {
 			srv.AutoHTTPS = new(caddyhttp.AutoHTTPSConfig)
-			if autoHTTPS == "off" {
+			switch autoHTTPS {
+			case "off":
 				srv.AutoHTTPS.Disabled = true
-			}
-			if autoHTTPS == "disable_redirects" {
+			case "disable_redirects":
 				srv.AutoHTTPS.DisableRedir = true
-			}
-			if autoHTTPS == "ignore_loaded_certs" {
+			case "disable_certs":
+				srv.AutoHTTPS.DisableCerts = true
+			case "ignore_loaded_certs":
 				srv.AutoHTTPS.IgnoreLoadedCerts = true
 			}
 		}
 
+		// Using paths in site addresses is deprecated
+		// See ParseAddress() where parsing should later reject paths
+		// See https://github.com/caddyserver/caddy/pull/4728 for a full explanation
+		for _, sblock := range p.serverBlocks {
+			for _, addr := range sblock.keys {
+				if addr.Path != "" {
+					caddy.Log().Named("caddyfile").Warn("Using a path in a site address is deprecated; please use the 'handle' directive instead", zap.String("address", addr.String()))
+				}
+			}
+		}
+
 		// sort server blocks by their keys; this is important because
 		// only the first matching site should be evaluated, and we should
 		// attempt to match most specific site first (host and path), in
@@ -522,6 +527,16 @@ func (st *ServerType) serversFromPairings(
 			}
 		}
 
+		// if needed, the ServerLogConfig is initialized beforehand so
+		// that all server blocks can populate it with data, even when not
+		// coming with a log directive
+		for _, sblock := range p.serverBlocks {
+			if len(sblock.pile["custom_log"]) != 0 {
+				srv.Logs = new(caddyhttp.ServerLogConfig)
+				break
+			}
+		}
+
 		// create a subroute for each site in the server block
 		for _, sblock := range p.serverBlocks {
 			matcherSetsEnc, err := st.compileEncodedMatcherSets(sblock)
@@ -534,7 +549,7 @@ func (st *ServerType) serversFromPairings(
 			// emit warnings if user put unspecified IP addresses; they probably want the bind directive
 			for _, h := range hosts {
 				if h == "0.0.0.0" || h == "::" {
-					log.Printf("[WARNING] Site block has unspecified IP address %s which only matches requests having that Host header; you probably want the 'bind' directive to configure the socket", h)
+					caddy.Log().Named("caddyfile").Warn("Site block has an unspecified IP address which only matches requests having that Host header; you probably want the 'bind' directive to configure the socket", zap.String("address", h))
 				}
 			}
 
@@ -570,7 +585,7 @@ func (st *ServerType) serversFromPairings(
 			}
 
 			for _, addr := range sblock.keys {
-				// if server only uses HTTPS port, auto-HTTPS will not apply
+				// if server only uses HTTP port, auto-HTTPS will not apply
 				if listenersUseAnyPortOtherThan(srv.Listen, httpPort) {
 					// exclude any hosts that were defined explicitly with "http://"
 					// in the key from automated cert management (issue #2998)
@@ -636,9 +651,6 @@ func (st *ServerType) serversFromPairings(
 			sblockLogHosts := sblock.hostsFromKeys(true)
 			for _, cval := range sblock.pile["custom_log"] {
 				ncl := cval.Value.(namedCustomLog)
-				if srv.Logs == nil {
-					srv.Logs = new(caddyhttp.ServerLogConfig)
-				}
 				if sblock.hasHostCatchAllKey() {
 					// all requests for hosts not able to be listed should use
 					// this log because it's a catch-all-hosts server block
@@ -717,7 +729,7 @@ func (st *ServerType) serversFromPairings(
 	return servers, nil
 }
 
-func detectConflictingSchemes(srv *caddyhttp.Server, serverBlocks []serverBlock, options map[string]interface{}) error {
+func detectConflictingSchemes(srv *caddyhttp.Server, serverBlocks []serverBlock, options map[string]any) error {
 	httpPort := strconv.Itoa(caddyhttp.DefaultHTTPPort)
 	if hp, ok := options["http_port"].(int); ok {
 		httpPort = strconv.Itoa(hp)
@@ -943,7 +955,7 @@ func appendSubrouteToRouteList(routeList caddyhttp.RouteList,
 func buildSubroute(routes []ConfigValue, groupCounter counter) (*caddyhttp.Subroute, error) {
 	for _, val := range routes {
 		if !directiveIsOrdered(val.directive) {
-			return nil, fmt.Errorf("directive '%s' is not ordered, so it cannot be used here", val.directive)
+			return nil, fmt.Errorf("directive '%s' is not an ordered HTTP handler, so it cannot be used here", val.directive)
 		}
 	}
 
@@ -1048,6 +1060,19 @@ func buildSubroute(routes []ConfigValue, groupCounter counter) (*caddyhttp.Subro
 	return subroute, nil
 }
 
+// normalizeDirectiveName ensures directives that should be sorted
+// at the same level are named the same before sorting happens.
+func normalizeDirectiveName(directive string) string {
+	// As a special case, we want "handle_path" to be sorted
+	// at the same level as "handle", so we force them to use
+	// the same directive name after their parsing is complete.
+	// See https://github.com/caddyserver/caddy/issues/3675#issuecomment-678042377
+	if directive == "handle_path" {
+		directive = "handle"
+	}
+	return directive
+}
+
 // consolidateRoutes combines routes with the same properties
 // (same matchers, same Terminal and Group settings) for a
 // cleaner overall output.
@@ -1178,6 +1203,7 @@ func (st *ServerType) compileEncodedMatcherSets(sblock serverBlock) ([]caddy.Mod
 
 func parseMatcherDefinitions(d *caddyfile.Dispenser, matchers map[string]caddy.ModuleMap) error {
 	for d.Next() {
+		// this is the "name" for "named matchers"
 		definitionName := d.Val()
 
 		if _, ok := matchers[definitionName]; ok {
@@ -1185,16 +1211,9 @@ func parseMatcherDefinitions(d *caddyfile.Dispenser, matchers map[string]caddy.M
 		}
 		matchers[definitionName] = make(caddy.ModuleMap)
 
-		// in case there are multiple instances of the same matcher, concatenate
-		// their tokens (we expect that UnmarshalCaddyfile should be able to
-		// handle more than one segment); otherwise, we'd overwrite other
-		// instances of the matcher in this set
-		tokensByMatcherName := make(map[string][]caddyfile.Token)
-		for nesting := d.Nesting(); d.NextArg() || d.NextBlock(nesting); {
-			matcherName := d.Val()
-			tokensByMatcherName[matcherName] = append(tokensByMatcherName[matcherName], d.NextSegment()...)
-		}
-		for matcherName, tokens := range tokensByMatcherName {
+		// given a matcher name and the tokens following it, parse
+		// the tokens as a matcher module and record it
+		makeMatcher := func(matcherName string, tokens []caddyfile.Token) error {
 			mod, err := caddy.GetModule("http.matchers." + matcherName)
 			if err != nil {
 				return fmt.Errorf("getting matcher module '%s': %v", matcherName, err)
@@ -1212,6 +1231,39 @@ func parseMatcherDefinitions(d *caddyfile.Dispenser, matchers map[string]caddy.M
 				return fmt.Errorf("matcher module '%s' is not a request matcher", matcherName)
 			}
 			matchers[definitionName][matcherName] = caddyconfig.JSON(rm, nil)
+			return nil
+		}
+
+		// if the next token is quoted, we can assume it's not a matcher name
+		// and that it's probably an 'expression' matcher
+		if d.NextArg() {
+			if d.Token().Quoted() {
+				err := makeMatcher("expression", []caddyfile.Token{d.Token()})
+				if err != nil {
+					return err
+				}
+				continue
+			}
+
+			// if it wasn't quoted, then we need to rewind after calling
+			// d.NextArg() so the below properly grabs the matcher name
+			d.Prev()
+		}
+
+		// in case there are multiple instances of the same matcher, concatenate
+		// their tokens (we expect that UnmarshalCaddyfile should be able to
+		// handle more than one segment); otherwise, we'd overwrite other
+		// instances of the matcher in this set
+		tokensByMatcherName := make(map[string][]caddyfile.Token)
+		for nesting := d.Nesting(); d.NextArg() || d.NextBlock(nesting); {
+			matcherName := d.Val()
+			tokensByMatcherName[matcherName] = append(tokensByMatcherName[matcherName], d.NextSegment()...)
+		}
+		for matcherName, tokens := range tokensByMatcherName {
+			err := makeMatcher(matcherName, tokens)
+			if err != nil {
+				return err
+			}
 		}
 	}
 	return nil
@@ -1229,9 +1281,61 @@ func encodeMatcherSet(matchers map[string]caddyhttp.RequestMatcher) (caddy.Modul
 	return msEncoded, nil
 }
 
+// placeholderShorthands returns a slice of old-new string pairs,
+// where the left of the pair is a placeholder shorthand that may
+// be used in the Caddyfile, and the right is the replacement.
+func placeholderShorthands() []string {
+	return []string{
+		"{dir}", "{http.request.uri.path.dir}",
+		"{file}", "{http.request.uri.path.file}",
+		"{host}", "{http.request.host}",
+		"{hostport}", "{http.request.hostport}",
+		"{port}", "{http.request.port}",
+		"{method}", "{http.request.method}",
+		"{path}", "{http.request.uri.path}",
+		"{query}", "{http.request.uri.query}",
+		"{remote}", "{http.request.remote}",
+		"{remote_host}", "{http.request.remote.host}",
+		"{remote_port}", "{http.request.remote.port}",
+		"{scheme}", "{http.request.scheme}",
+		"{uri}", "{http.request.uri}",
+		"{tls_cipher}", "{http.request.tls.cipher_suite}",
+		"{tls_version}", "{http.request.tls.version}",
+		"{tls_client_fingerprint}", "{http.request.tls.client.fingerprint}",
+		"{tls_client_issuer}", "{http.request.tls.client.issuer}",
+		"{tls_client_serial}", "{http.request.tls.client.serial}",
+		"{tls_client_subject}", "{http.request.tls.client.subject}",
+		"{tls_client_certificate_pem}", "{http.request.tls.client.certificate_pem}",
+		"{tls_client_certificate_der_base64}", "{http.request.tls.client.certificate_der_base64}",
+		"{upstream_hostport}", "{http.reverse_proxy.upstream.hostport}",
+	}
+}
+
+// WasReplacedPlaceholderShorthand checks if a token string was
+// likely a replaced shorthand of the known Caddyfile placeholder
+// replacement outputs. Useful to prevent some user-defined map
+// output destinations from overlapping with one of the
+// predefined shorthands.
+func WasReplacedPlaceholderShorthand(token string) string {
+	prev := ""
+	for i, item := range placeholderShorthands() {
+		// only look at every 2nd item, which is the replacement
+		if i%2 == 0 {
+			prev = item
+			continue
+		}
+		if strings.Trim(token, "{}") == strings.Trim(item, "{}") {
+			// we return the original shorthand so it
+			// can be used for an error message
+			return prev
+		}
+	}
+	return ""
+}
+
 // tryInt tries to convert val to an integer. If it fails,
 // it downgrades the error to a warning and returns 0.
-func tryInt(val interface{}, warnings *[]caddyconfig.Warning) int {
+func tryInt(val any, warnings *[]caddyconfig.Warning) int {
 	intVal, ok := val.(int)
 	if val != nil && !ok && warnings != nil {
 		*warnings = append(*warnings, caddyconfig.Warning{Message: "not an integer type"})
@@ -1239,7 +1343,7 @@ func tryInt(val interface{}, warnings *[]caddyconfig.Warning) int {
 	return intVal
 }
 
-func tryString(val interface{}, warnings *[]caddyconfig.Warning) string {
+func tryString(val any, warnings *[]caddyconfig.Warning) string {
 	stringVal, ok := val.(string)
 	if val != nil && !ok && warnings != nil {
 		*warnings = append(*warnings, caddyconfig.Warning{Message: "not a string type"})
@@ -1247,7 +1351,7 @@ func tryString(val interface{}, warnings *[]caddyconfig.Warning) string {
 	return stringVal
 }
 
-func tryDuration(val interface{}, warnings *[]caddyconfig.Warning) caddy.Duration {
+func tryDuration(val any, warnings *[]caddyconfig.Warning) caddy.Duration {
 	durationVal, ok := val.(caddy.Duration)
 	if val != nil && !ok && warnings != nil {
 		*warnings = append(*warnings, caddyconfig.Warning{Message: "not a duration type"})

caddyconfig/httpcaddyfile/options.go

@@ -29,16 +29,21 @@ func init() {
 	RegisterGlobalOption("debug", parseOptTrue)
 	RegisterGlobalOption("http_port", parseOptHTTPPort)
 	RegisterGlobalOption("https_port", parseOptHTTPSPort)
+	RegisterGlobalOption("default_bind", parseOptStringList)
 	RegisterGlobalOption("grace_period", parseOptDuration)
+	RegisterGlobalOption("shutdown_delay", parseOptDuration)
 	RegisterGlobalOption("default_sni", parseOptSingleString)
 	RegisterGlobalOption("order", parseOptOrder)
 	RegisterGlobalOption("storage", parseOptStorage)
 	RegisterGlobalOption("storage_clean_interval", parseOptDuration)
+	RegisterGlobalOption("renew_interval", parseOptDuration)
+	RegisterGlobalOption("ocsp_interval", parseOptDuration)
 	RegisterGlobalOption("acme_ca", parseOptSingleString)
 	RegisterGlobalOption("acme_ca_root", parseOptSingleString)
 	RegisterGlobalOption("acme_dns", parseOptACMEDNS)
 	RegisterGlobalOption("acme_eab", parseOptACMEEAB)
 	RegisterGlobalOption("cert_issuer", parseOptCertIssuer)
+	RegisterGlobalOption("skip_install_trust", parseOptTrue)
 	RegisterGlobalOption("email", parseOptSingleString)
 	RegisterGlobalOption("admin", parseOptAdmin)
 	RegisterGlobalOption("on_demand_tls", parseOptOnDemand)
@@ -48,11 +53,12 @@ func init() {
 	RegisterGlobalOption("servers", parseServerOptions)
 	RegisterGlobalOption("ocsp_stapling", parseOCSPStaplingOptions)
 	RegisterGlobalOption("log", parseLogOptions)
+	RegisterGlobalOption("preferred_chains", parseOptPreferredChains)
 }
 
-func parseOptTrue(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) { return true, nil }
+func parseOptTrue(d *caddyfile.Dispenser, _ any) (any, error) { return true, nil }
 
-func parseOptHTTPPort(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptHTTPPort(d *caddyfile.Dispenser, _ any) (any, error) {
 	var httpPort int
 	for d.Next() {
 		var httpPortStr string
@@ -68,7 +74,7 @@ func parseOptHTTPPort(d *caddyfile.Dispenser, _ interface{}) (interface{}, error
 	return httpPort, nil
 }
 
-func parseOptHTTPSPort(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptHTTPSPort(d *caddyfile.Dispenser, _ any) (any, error) {
 	var httpsPort int
 	for d.Next() {
 		var httpsPortStr string
@@ -84,7 +90,7 @@ func parseOptHTTPSPort(d *caddyfile.Dispenser, _ interface{}) (interface{}, erro
 	return httpsPort, nil
 }
 
-func parseOptOrder(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptOrder(d *caddyfile.Dispenser, _ any) (any, error) {
 	newOrder := directiveOrder
 
 	for d.Next() {
@@ -160,7 +166,7 @@ func parseOptOrder(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
 	return newOrder, nil
 }
 
-func parseOptStorage(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptStorage(d *caddyfile.Dispenser, _ any) (any, error) {
 	if !d.Next() { // consume option name
 		return nil, d.ArgErr()
 	}
@@ -179,7 +185,7 @@ func parseOptStorage(d *caddyfile.Dispenser, _ interface{}) (interface{}, error)
 	return storage, nil
 }
 
-func parseOptDuration(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptDuration(d *caddyfile.Dispenser, _ any) (any, error) {
 	if !d.Next() { // consume option name
 		return nil, d.ArgErr()
 	}
@@ -193,7 +199,7 @@ func parseOptDuration(d *caddyfile.Dispenser, _ interface{}) (interface{}, error
 	return caddy.Duration(dur), nil
 }
 
-func parseOptACMEDNS(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptACMEDNS(d *caddyfile.Dispenser, _ any) (any, error) {
 	if !d.Next() { // consume option name
 		return nil, d.ArgErr()
 	}
@@ -212,7 +218,7 @@ func parseOptACMEDNS(d *caddyfile.Dispenser, _ interface{}) (interface{}, error)
 	return prov, nil
 }
 
-func parseOptACMEEAB(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptACMEEAB(d *caddyfile.Dispenser, _ any) (any, error) {
 	eab := new(acme.EAB)
 	for d.Next() {
 		if d.NextArg() {
@@ -240,7 +246,7 @@ func parseOptACMEEAB(d *caddyfile.Dispenser, _ interface{}) (interface{}, error)
 	return eab, nil
 }
 
-func parseOptCertIssuer(d *caddyfile.Dispenser, existing interface{}) (interface{}, error) {
+func parseOptCertIssuer(d *caddyfile.Dispenser, existing any) (any, error) {
 	var issuers []certmagic.Issuer
 	if existing != nil {
 		issuers = existing.([]certmagic.Issuer)
@@ -263,7 +269,7 @@ func parseOptCertIssuer(d *caddyfile.Dispenser, existing interface{}) (interface
 	return issuers, nil
 }
 
-func parseOptSingleString(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptSingleString(d *caddyfile.Dispenser, _ any) (any, error) {
 	d.Next() // consume parameter name
 	if !d.Next() {
 		return "", d.ArgErr()
@@ -275,7 +281,16 @@ func parseOptSingleString(d *caddyfile.Dispenser, _ interface{}) (interface{}, e
 	return val, nil
 }
 
-func parseOptAdmin(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptStringList(d *caddyfile.Dispenser, _ any) (any, error) {
+	d.Next() // consume parameter name
+	val := d.RemainingArgs()
+	if len(val) == 0 {
+		return "", d.ArgErr()
+	}
+	return val, nil
+}
+
+func parseOptAdmin(d *caddyfile.Dispenser, _ any) (any, error) {
 	adminCfg := new(caddy.AdminConfig)
 	for d.Next() {
 		if d.NextArg() {
@@ -311,7 +326,7 @@ func parseOptAdmin(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
 	return adminCfg, nil
 }
 
-func parseOptOnDemand(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptOnDemand(d *caddyfile.Dispenser, _ any) (any, error) {
 	var ond *caddytls.OnDemandConfig
 	for d.Next() {
 		if d.NextArg() {
@@ -371,7 +386,7 @@ func parseOptOnDemand(d *caddyfile.Dispenser, _ interface{}) (interface{}, error
 	return ond, nil
 }
 
-func parseOptAutoHTTPS(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOptAutoHTTPS(d *caddyfile.Dispenser, _ any) (any, error) {
 	d.Next() // consume parameter name
 	if !d.Next() {
 		return "", d.ArgErr()
@@ -380,17 +395,17 @@ func parseOptAutoHTTPS(d *caddyfile.Dispenser, _ interface{}) (interface{}, erro
 	if d.Next() {
 		return "", d.ArgErr()
 	}
-	if val != "off" && val != "disable_redirects" && val != "ignore_loaded_certs" {
-		return "", d.Errf("auto_https must be one of 'off', 'disable_redirects' or 'ignore_loaded_certs'")
+	if val != "off" && val != "disable_redirects" && val != "disable_certs" && val != "ignore_loaded_certs" {
+		return "", d.Errf("auto_https must be one of 'off', 'disable_redirects', 'disable_certs', or 'ignore_loaded_certs'")
 	}
 	return val, nil
 }
 
-func parseServerOptions(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseServerOptions(d *caddyfile.Dispenser, _ any) (any, error) {
 	return unmarshalCaddyfileServerOptions(d)
 }
 
-func parseOCSPStaplingOptions(d *caddyfile.Dispenser, _ interface{}) (interface{}, error) {
+func parseOCSPStaplingOptions(d *caddyfile.Dispenser, _ any) (any, error) {
 	d.Next() // consume option name
 	var val string
 	if !d.AllArgs(&val) {
@@ -416,8 +431,7 @@ func parseOCSPStaplingOptions(d *caddyfile.Dispenser, _ interface{}) (interface{
 //
 // When the name argument is unspecified, this directive modifies the default
 // logger.
-//
-func parseLogOptions(d *caddyfile.Dispenser, existingVal interface{}) (interface{}, error) {
+func parseLogOptions(d *caddyfile.Dispenser, existingVal any) (any, error) {
 	currentNames := make(map[string]struct{})
 	if existingVal != nil {
 		innerVals, ok := existingVal.([]ConfigValue)
@@ -451,3 +465,8 @@ func parseLogOptions(d *caddyfile.Dispenser, existingVal interface{}) (interface
 
 	return configValues, nil
 }
+
+func parseOptPreferredChains(d *caddyfile.Dispenser, _ any) (any, error) {
+	d.Next()
+	return caddytls.ParseCaddyfilePreferredChainsOptions(d)
+}

caddyconfig/httpcaddyfile/pkiapp.go

@@ -16,26 +16,203 @@ package httpcaddyfile
 
 import (
 	"github.com/caddyserver/caddy/v2/caddyconfig"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddypki"
 )
 
-func (st ServerType) buildPKIApp(
-	pairings []sbAddrAssociation,
-	options map[string]interface{},
-	warnings []caddyconfig.Warning,
-) (*caddypki.PKI, []caddyconfig.Warning, error) {
+func init() {
+	RegisterGlobalOption("pki", parsePKIApp)
+}
 
-	pkiApp := &caddypki.PKI{CAs: make(map[string]*caddypki.CA)}
+// parsePKIApp parses the global log option. Syntax:
+//
+//     pki {
+//         ca [<id>] {
+//             name            <name>
+//             root_cn         <name>
+//             intermediate_cn <name>
+//             root {
+//                 cert   <path>
+//                 key    <path>
+//                 format <format>
+//             }
+//             intermediate {
+//                 cert   <path>
+//                 key    <path>
+//                 format <format>
+//             }
+//         }
+//     }
+//
+// When the CA ID is unspecified, 'local' is assumed.
+func parsePKIApp(d *caddyfile.Dispenser, existingVal any) (any, error) {
+	pki := &caddypki.PKI{CAs: make(map[string]*caddypki.CA)}
 
-	for _, p := range pairings {
-		for _, sblock := range p.serverBlocks {
-			// find all the CAs that were defined and add them to the app config
-			for _, caCfgValue := range sblock.pile["pki.ca"] {
-				ca := caCfgValue.Value.(*caddypki.CA)
-				pkiApp.CAs[ca.ID] = ca
+	for d.Next() {
+		for nesting := d.Nesting(); d.NextBlock(nesting); {
+			switch d.Val() {
+			case "ca":
+				pkiCa := new(caddypki.CA)
+				if d.NextArg() {
+					pkiCa.ID = d.Val()
+					if d.NextArg() {
+						return nil, d.ArgErr()
+					}
+				}
+				if pkiCa.ID == "" {
+					pkiCa.ID = caddypki.DefaultCAID
+				}
+
+				for nesting := d.Nesting(); d.NextBlock(nesting); {
+					switch d.Val() {
+					case "name":
+						if !d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						pkiCa.Name = d.Val()
+
+					case "root_cn":
+						if !d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						pkiCa.RootCommonName = d.Val()
+
+					case "intermediate_cn":
+						if !d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						pkiCa.IntermediateCommonName = d.Val()
+
+					case "root":
+						if pkiCa.Root == nil {
+							pkiCa.Root = new(caddypki.KeyPair)
+						}
+						for nesting := d.Nesting(); d.NextBlock(nesting); {
+							switch d.Val() {
+							case "cert":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Root.Certificate = d.Val()
+
+							case "key":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Root.PrivateKey = d.Val()
+
+							case "format":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Root.Format = d.Val()
+
+							default:
+								return nil, d.Errf("unrecognized pki ca root option '%s'", d.Val())
+							}
+						}
+
+					case "intermediate":
+						if pkiCa.Intermediate == nil {
+							pkiCa.Intermediate = new(caddypki.KeyPair)
+						}
+						for nesting := d.Nesting(); d.NextBlock(nesting); {
+							switch d.Val() {
+							case "cert":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Intermediate.Certificate = d.Val()
+
+							case "key":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Intermediate.PrivateKey = d.Val()
+
+							case "format":
+								if !d.NextArg() {
+									return nil, d.ArgErr()
+								}
+								pkiCa.Intermediate.Format = d.Val()
+
+							default:
+								return nil, d.Errf("unrecognized pki ca intermediate option '%s'", d.Val())
+							}
+						}
+
+					default:
+						return nil, d.Errf("unrecognized pki ca option '%s'", d.Val())
+					}
+				}
+
+				pki.CAs[pkiCa.ID] = pkiCa
+
+			default:
+				return nil, d.Errf("unrecognized pki option '%s'", d.Val())
 			}
 		}
 	}
 
+	return pki, nil
+}
+
+func (st ServerType) buildPKIApp(
+	pairings []sbAddrAssociation,
+	options map[string]any,
+	warnings []caddyconfig.Warning,
+) (*caddypki.PKI, []caddyconfig.Warning, error) {
+
+	skipInstallTrust := false
+	if _, ok := options["skip_install_trust"]; ok {
+		skipInstallTrust = true
+	}
+	falseBool := false
+
+	// Load the PKI app configured via global options
+	var pkiApp *caddypki.PKI
+	unwrappedPki, ok := options["pki"].(*caddypki.PKI)
+	if ok {
+		pkiApp = unwrappedPki
+	} else {
+		pkiApp = &caddypki.PKI{CAs: make(map[string]*caddypki.CA)}
+	}
+	for _, ca := range pkiApp.CAs {
+		if skipInstallTrust {
+			ca.InstallTrust = &falseBool
+		}
+		pkiApp.CAs[ca.ID] = ca
+	}
+
+	// Add in the CAs configured via directives
+	for _, p := range pairings {
+		for _, sblock := range p.serverBlocks {
+			// find all the CAs that were defined and add them to the app config
+			// i.e. from any "acme_server" directives
+			for _, caCfgValue := range sblock.pile["pki.ca"] {
+				ca := caCfgValue.Value.(*caddypki.CA)
+				if skipInstallTrust {
+					ca.InstallTrust = &falseBool
+				}
+
+				// the CA might already exist from global options, so
+				// don't overwrite it in that case
+				if _, ok := pkiApp.CAs[ca.ID]; !ok {
+					pkiApp.CAs[ca.ID] = ca
+				}
+			}
+		}
+	}
+
+	// if there was no CAs defined in any of the servers,
+	// and we were requested to not install trust, then
+	// add one for the default/local CA to do so
+	if len(pkiApp.CAs) == 0 && skipInstallTrust {
+		ca := new(caddypki.CA)
+		ca.ID = caddypki.DefaultCAID
+		ca.InstallTrust = &falseBool
+		pkiApp.CAs[ca.ID] = ca
+	}
+
 	return pkiApp, warnings, nil
 }

caddyconfig/httpcaddyfile/serveroptions.go

@@ -33,18 +33,19 @@ type serverOptions struct {
 	ListenerAddress string
 
 	// These will all map 1:1 to the caddyhttp.Server struct
-	ListenerWrappersRaw []json.RawMessage
-	ReadTimeout         caddy.Duration
-	ReadHeaderTimeout   caddy.Duration
-	WriteTimeout        caddy.Duration
-	IdleTimeout         caddy.Duration
-	MaxHeaderBytes      int
-	AllowH2C            bool
-	ExperimentalHTTP3   bool
-	StrictSNIHost       *bool
+	ListenerWrappersRaw  []json.RawMessage
+	ReadTimeout          caddy.Duration
+	ReadHeaderTimeout    caddy.Duration
+	WriteTimeout         caddy.Duration
+	IdleTimeout          caddy.Duration
+	KeepAliveInterval    caddy.Duration
+	MaxHeaderBytes       int
+	Protocols            []string
+	StrictSNIHost        *bool
+	ShouldLogCredentials bool
 }
 
-func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error) {
+func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (any, error) {
 	serverOpts := serverOptions{}
 	for d.Next() {
 		if d.NextArg() {
@@ -122,6 +123,15 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error
 						return nil, d.Errf("unrecognized timeouts option '%s'", d.Val())
 					}
 				}
+			case "keepalive_interval":
+				if !d.NextArg() {
+					return nil, d.ArgErr()
+				}
+				dur, err := caddy.ParseDuration(d.Val())
+				if err != nil {
+					return nil, d.Errf("parsing keepalive interval duration: %v", err)
+				}
+				serverOpts.KeepAliveInterval = caddy.Duration(dur)
 
 			case "max_header_size":
 				var sizeStr string
@@ -134,27 +144,65 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error
 				}
 				serverOpts.MaxHeaderBytes = int(size)
 
+			case "log_credentials":
+				if d.NextArg() {
+					return nil, d.ArgErr()
+				}
+				serverOpts.ShouldLogCredentials = true
+
+			case "protocols":
+				protos := d.RemainingArgs()
+				for _, proto := range protos {
+					if proto != "h1" && proto != "h2" && proto != "h2c" && proto != "h3" {
+						return nil, d.Errf("unknown protocol '%s': expected h1, h2, h2c, or h3", proto)
+					}
+					if sliceContains(serverOpts.Protocols, proto) {
+						return nil, d.Errf("protocol %s specified more than once", proto)
+					}
+					serverOpts.Protocols = append(serverOpts.Protocols, proto)
+				}
+				if d.NextBlock(0) {
+					return nil, d.ArgErr()
+				}
+
+			case "strict_sni_host":
+				if d.NextArg() && d.Val() != "insecure_off" && d.Val() != "on" {
+					return nil, d.Errf("strict_sni_host only supports 'on' or 'insecure_off', got '%s'", d.Val())
+				}
+				boolVal := true
+				if d.Val() == "insecure_off" {
+					boolVal = false
+				}
+				serverOpts.StrictSNIHost = &boolVal
+
+			// TODO: DEPRECATED. (August 2022)
 			case "protocol":
+				caddy.Log().Named("caddyfile").Warn("DEPRECATED: protocol sub-option will be removed soon")
+
 				for nesting := d.Nesting(); d.NextBlock(nesting); {
 					switch d.Val() {
 					case "allow_h2c":
-						if d.NextArg() {
-							return nil, d.ArgErr()
-						}
-						serverOpts.AllowH2C = true
+						caddy.Log().Named("caddyfile").Warn("DEPRECATED: allow_h2c will be removed soon; use protocols option instead")
 
-					case "experimental_http3":
 						if d.NextArg() {
 							return nil, d.ArgErr()
 						}
-						serverOpts.ExperimentalHTTP3 = true
+						if sliceContains(serverOpts.Protocols, "h2c") {
+							return nil, d.Errf("protocol h2c already specified")
+						}
+						serverOpts.Protocols = append(serverOpts.Protocols, "h2c")
 
 					case "strict_sni_host":
-						if d.NextArg() {
-							return nil, d.ArgErr()
+						caddy.Log().Named("caddyfile").Warn("DEPRECATED: protocol > strict_sni_host in this position will be removed soon; move up to the servers block instead")
+
+						if d.NextArg() && d.Val() != "insecure_off" && d.Val() != "on" {
+							return nil, d.Errf("strict_sni_host only supports 'on' or 'insecure_off', got '%s'", d.Val())
 						}
-						trueBool := true
-						serverOpts.StrictSNIHost = &trueBool
+						boolVal := true
+						if d.Val() == "insecure_off" {
+							boolVal = false
+						}
+						serverOpts.StrictSNIHost = &boolVal
 
 					default:
 						return nil, d.Errf("unrecognized protocol option '%s'", d.Val())
@@ -172,20 +220,9 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error
 // applyServerOptions sets the server options on the appropriate servers
 func applyServerOptions(
 	servers map[string]*caddyhttp.Server,
-	options map[string]interface{},
+	options map[string]any,
 	warnings *[]caddyconfig.Warning,
 ) error {
-	// If experimental HTTP/3 is enabled, enable it on each server.
-	// We already know there won't be a conflict with serverOptions because
-	// we validated earlier that "experimental_http3" cannot be set at the same
-	// time as "servers"
-	if enableH3, ok := options["experimental_http3"].(bool); ok && enableH3 {
-		*warnings = append(*warnings, caddyconfig.Warning{Message: "the 'experimental_http3' global option is deprecated, please use the 'servers > protocol > experimental_http3' option instead"})
-		for _, srv := range servers {
-			srv.ExperimentalHTTP3 = true
-		}
-	}
-
 	serverOpts, ok := options["servers"].([]serverOptions)
 	if !ok {
 		return nil
@@ -218,10 +255,16 @@ func applyServerOptions(
 		server.ReadHeaderTimeout = opts.ReadHeaderTimeout
 		server.WriteTimeout = opts.WriteTimeout
 		server.IdleTimeout = opts.IdleTimeout
+		server.KeepAliveInterval = opts.KeepAliveInterval
 		server.MaxHeaderBytes = opts.MaxHeaderBytes
-		server.AllowH2C = opts.AllowH2C
-		server.ExperimentalHTTP3 = opts.ExperimentalHTTP3
+		server.Protocols = opts.Protocols
 		server.StrictSNIHost = opts.StrictSNIHost
+		if opts.ShouldLogCredentials {
+			if server.Logs == nil {
+				server.Logs = &caddyhttp.ServerLogConfig{}
+			}
+			server.Logs.ShouldLogCredentials = opts.ShouldLogCredentials
+		}
 	}
 
 	return nil

caddyconfig/httpcaddyfile/tlsapp.go

@@ -33,7 +33,7 @@ import (
 
 func (st ServerType) buildTLSApp(
 	pairings []sbAddrAssociation,
-	options map[string]interface{},
+	options map[string]any,
 	warnings []caddyconfig.Warning,
 ) (*caddytls.TLS, []caddyconfig.Warning, error) {
 
@@ -101,6 +101,12 @@ func (st ServerType) buildTLSApp(
 		}
 
 		for _, sblock := range p.serverBlocks {
+			// check the scheme of all the site addresses,
+			// skip building AP if they all had http://
+			if sblock.isAllHTTP() {
+				continue
+			}
+
 			// get values that populate an automation policy for this block
 			ap, err := newBaseAutomationPolicy(options, warnings, true)
 			if err != nil {
@@ -133,6 +139,13 @@ func (st ServerType) buildTLSApp(
 				ap.Issuers = issuers
 			}
 
+			// certificate managers
+			if certManagerVals, ok := sblock.pile["tls.cert_manager"]; ok {
+				for _, certManager := range certManagerVals {
+					certGetterName := certManager.Value.(caddy.Module).CaddyModule().ID.Name()
+					ap.ManagersRaw = append(ap.ManagersRaw, caddyconfig.JSONModuleObject(certManager.Value, "via", certGetterName, &warnings))
+				}
+			}
 			// custom bind host
 			for _, cfgVal := range sblock.pile["bind"] {
 				for _, iss := range ap.Issuers {
@@ -189,7 +202,7 @@ func (st ServerType) buildTLSApp(
 			}
 
 			// associate our new automation policy with this server block's hosts
-			ap.Subjects = sblockHosts
+			ap.Subjects = sblock.hostsFromKeysNotHTTP(httpPort)
 			sort.Strings(ap.Subjects) // solely for deterministic test results
 
 			// if a combination of public and internal names were given
@@ -211,7 +224,7 @@ func (st ServerType) buildTLSApp(
 					// it that we would need to check here) since the hostname is known at handshake;
 					// and it is unexpected to switch to internal issuer when the user wants to get
 					// regular certificates on-demand for a class of certs like *.*.tld.
-					if !certmagic.SubjectIsIP(s) && !certmagic.SubjectIsInternal(s) && (strings.Count(s, "*.") < 2 || ap.OnDemand) {
+					if subjectQualifiesForPublicCert(ap, s) {
 						external = append(external, s)
 					} else {
 						internal = append(internal, s)
@@ -286,6 +299,27 @@ func (st ServerType) buildTLSApp(
 		tlsApp.Automation.StorageCleanInterval = storageCleanInterval
 	}
 
+	// set the expired certificates renew interval if configured
+	if renewCheckInterval, ok := options["renew_interval"].(caddy.Duration); ok {
+		if tlsApp.Automation == nil {
+			tlsApp.Automation = new(caddytls.AutomationConfig)
+		}
+		tlsApp.Automation.RenewCheckInterval = renewCheckInterval
+	}
+
+	// set the OCSP check interval if configured
+	if ocspCheckInterval, ok := options["ocsp_interval"].(caddy.Duration); ok {
+		if tlsApp.Automation == nil {
+			tlsApp.Automation = new(caddytls.AutomationConfig)
+		}
+		tlsApp.Automation.OCSPCheckInterval = ocspCheckInterval
+	}
+
+	// set whether OCSP stapling should be disabled for manually-managed certificates
+	if ocspConfig, ok := options["ocsp_stapling"].(certmagic.OCSPConfig); ok {
+		tlsApp.DisableOCSPStapling = ocspConfig.DisableStapling
+	}
+
 	// if any hostnames appear on the same server block as a key with
 	// no host, they will not be used with route matchers because the
 	// hostless key matches all hosts, therefore, it wouldn't be
@@ -321,10 +355,14 @@ func (st ServerType) buildTLSApp(
 		globalACMECARoot := options["acme_ca_root"]
 		globalACMEDNS := options["acme_dns"]
 		globalACMEEAB := options["acme_eab"]
-		hasGlobalACMEDefaults := globalEmail != nil || globalACMECA != nil || globalACMECARoot != nil || globalACMEDNS != nil || globalACMEEAB != nil
+		globalPreferredChains := options["preferred_chains"]
+		hasGlobalACMEDefaults := globalEmail != nil || globalACMECA != nil || globalACMECARoot != nil || globalACMEDNS != nil || globalACMEEAB != nil || globalPreferredChains != nil
 		if hasGlobalACMEDefaults {
-			for _, ap := range tlsApp.Automation.Policies {
-				if len(ap.Issuers) == 0 {
+			for i := 0; i < len(tlsApp.Automation.Policies); i++ {
+				ap := tlsApp.Automation.Policies[i]
+				if len(ap.Issuers) == 0 && automationPolicyHasAllPublicNames(ap) {
+					// for public names, create default issuers which will later be filled in with configured global defaults
+					// (internal names will implicitly use the internal issuer at auto-https time)
 					ap.Issuers = caddytls.DefaultIssuers()
 
 					// if a specific endpoint is configured, can't use multiple default issuers
@@ -390,7 +428,7 @@ func (st ServerType) buildTLSApp(
 
 type acmeCapable interface{ GetACMEIssuer() *caddytls.ACMEIssuer }
 
-func fillInGlobalACMEDefaults(issuer certmagic.Issuer, options map[string]interface{}) error {
+func fillInGlobalACMEDefaults(issuer certmagic.Issuer, options map[string]any) error {
 	acmeWrapper, ok := issuer.(acmeCapable)
 	if !ok {
 		return nil
@@ -405,6 +443,7 @@ func fillInGlobalACMEDefaults(issuer certmagic.Issuer, options map[string]interf
 	globalACMECARoot := options["acme_ca_root"]
 	globalACMEDNS := options["acme_dns"]
 	globalACMEEAB := options["acme_eab"]
+	globalPreferredChains := options["preferred_chains"]
 
 	if globalEmail != nil && acmeIssuer.Email == "" {
 		acmeIssuer.Email = globalEmail.(string)
@@ -425,6 +464,9 @@ func fillInGlobalACMEDefaults(issuer certmagic.Issuer, options map[string]interf
 	if globalACMEEAB != nil && acmeIssuer.ExternalAccount == nil {
 		acmeIssuer.ExternalAccount = globalACMEEAB.(*acme.EAB)
 	}
+	if globalPreferredChains != nil && acmeIssuer.PreferredChains == nil {
+		acmeIssuer.PreferredChains = globalPreferredChains.(*caddytls.ChainPreference)
+	}
 	return nil
 }
 
@@ -433,7 +475,7 @@ func fillInGlobalACMEDefaults(issuer certmagic.Issuer, options map[string]interf
 // for any other automation policies. A nil policy (and no error) will be
 // returned if there are no default/global options. However, if always is
 // true, a non-nil value will always be returned (unless there is an error).
-func newBaseAutomationPolicy(options map[string]interface{}, warnings []caddyconfig.Warning, always bool) (*caddytls.AutomationPolicy, error) {
+func newBaseAutomationPolicy(options map[string]any, warnings []caddyconfig.Warning, always bool) (*caddytls.AutomationPolicy, error) {
 	issuers, hasIssuers := options["cert_issuer"]
 	_, hasLocalCerts := options["local_certs"]
 	keyType, hasKeyType := options["key_type"]
@@ -489,16 +531,23 @@ func consolidateAutomationPolicies(aps []*caddytls.AutomationPolicy) []*caddytls
 	})
 
 	emptyAPCount := 0
+	origLenAPs := len(aps)
 	// compute the number of empty policies (disregarding subjects) - see #4128
 	emptyAP := new(caddytls.AutomationPolicy)
 	for i := 0; i < len(aps); i++ {
 		emptyAP.Subjects = aps[i].Subjects
 		if reflect.DeepEqual(aps[i], emptyAP) {
 			emptyAPCount++
+			if !automationPolicyHasAllPublicNames(aps[i]) {
+				// if this automation policy has internal names, we might as well remove it
+				// so auto-https can implicitly use the internal issuer
+				aps = append(aps[:i], aps[i+1:]...)
+				i--
+			}
 		}
 	}
 	// If all policies are empty, we can return nil, as there is no need to set any policy
-	if emptyAPCount == len(aps) {
+	if emptyAPCount == origLenAPs {
 		return nil
 	}
 
@@ -596,3 +645,21 @@ func automationPolicyShadows(i int, aps []*caddytls.AutomationPolicy) int {
 	}
 	return -1
 }
+
+// subjectQualifiesForPublicCert is like certmagic.SubjectQualifiesForPublicCert() except
+// that this allows domains with multiple wildcard levels like '*.*.example.com' to qualify
+// if the automation policy has OnDemand enabled (i.e. this function is more lenient).
+func subjectQualifiesForPublicCert(ap *caddytls.AutomationPolicy, subj string) bool {
+	return !certmagic.SubjectIsIP(subj) &&
+		!certmagic.SubjectIsInternal(subj) &&
+		(strings.Count(subj, "*.") < 2 || ap.OnDemand)
+}
+
+func automationPolicyHasAllPublicNames(ap *caddytls.AutomationPolicy) bool {
+	for _, subj := range ap.Subjects {
+		if !subjectQualifiesForPublicCert(ap, subj) {
+			return false
+		}
+	}
+	return true
+}

caddyconfig/httploader.go

@@ -1,11 +1,26 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package caddyconfig
 
 import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
+	"os"
 	"time"
 
 	"github.com/caddyserver/caddy/v2"
@@ -56,21 +71,28 @@ func (HTTPLoader) CaddyModule() caddy.ModuleInfo {
 
 // LoadConfig loads a Caddy config.
 func (hl HTTPLoader) LoadConfig(ctx caddy.Context) ([]byte, error) {
+	repl := caddy.NewReplacer()
+
 	client, err := hl.makeClient(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	method := hl.Method
+	method := repl.ReplaceAll(hl.Method, "")
 	if method == "" {
 		method = http.MethodGet
 	}
 
-	req, err := http.NewRequest(method, hl.URL, nil)
+	url := repl.ReplaceAll(hl.URL, "")
+	req, err := http.NewRequest(method, url, nil)
 	if err != nil {
 		return nil, err
 	}
-	req.Header = hl.Headers
+	for key, vals := range hl.Headers {
+		for _, val := range vals {
+			req.Header.Add(repl.ReplaceAll(key, ""), repl.ReplaceKnown(val, ""))
+		}
+	}
 
 	resp, err := client.Do(req)
 	if err != nil {
@@ -81,7 +103,7 @@ func (hl HTTPLoader) LoadConfig(ctx caddy.Context) ([]byte, error) {
 		return nil, fmt.Errorf("server responded with HTTP %d", resp.StatusCode)
 	}
 
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, err
 	}
@@ -130,7 +152,7 @@ func (hl HTTPLoader) makeClient(ctx caddy.Context) (*http.Client, error) {
 		if len(hl.TLS.RootCAPEMFiles) > 0 {
 			rootPool := x509.NewCertPool()
 			for _, pemFile := range hl.TLS.RootCAPEMFiles {
-				pemData, err := ioutil.ReadFile(pemFile)
+				pemData, err := os.ReadFile(pemFile)
 				if err != nil {
 					return nil, fmt.Errorf("failed reading ca cert: %v", err)
 				}

caddyconfig/load.go

@@ -58,6 +58,10 @@ func (al adminLoad) Routes() []caddy.AdminRoute {
 			Pattern: "/load",
 			Handler: caddy.AdminHandlerFunc(al.handleLoad),
 		},
+		{
+			Pattern: "/adapt",
+			Handler: caddy.AdminHandlerFunc(al.handleAdapt),
+		},
 	}
 }
 
@@ -122,7 +126,48 @@ func (adminLoad) handleLoad(w http.ResponseWriter, r *http.Request) error {
 	return nil
 }
 
-// adaptByContentType adapts body to Caddy JSON using the adapter specified by contenType.
+// handleAdapt adapts the given Caddy config to JSON and responds with the result.
+func (adminLoad) handleAdapt(w http.ResponseWriter, r *http.Request) error {
+	if r.Method != http.MethodPost {
+		return caddy.APIError{
+			HTTPStatus: http.StatusMethodNotAllowed,
+			Err:        fmt.Errorf("method not allowed"),
+		}
+	}
+
+	buf := bufPool.Get().(*bytes.Buffer)
+	buf.Reset()
+	defer bufPool.Put(buf)
+
+	_, err := io.Copy(buf, r.Body)
+	if err != nil {
+		return caddy.APIError{
+			HTTPStatus: http.StatusBadRequest,
+			Err:        fmt.Errorf("reading request body: %v", err),
+		}
+	}
+
+	result, warnings, err := adaptByContentType(r.Header.Get("Content-Type"), buf.Bytes())
+	if err != nil {
+		return caddy.APIError{
+			HTTPStatus: http.StatusBadRequest,
+			Err:        err,
+		}
+	}
+
+	out := struct {
+		Warnings []Warning       `json:"warnings,omitempty"`
+		Result   json.RawMessage `json:"result"`
+	}{
+		Warnings: warnings,
+		Result:   result,
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	return json.NewEncoder(w).Encode(out)
+}
+
+// adaptByContentType adapts body to Caddy JSON using the adapter specified by contentType.
 // If contentType is empty or ends with "/json", the input will be returned, as a no-op.
 func adaptByContentType(contentType string, body []byte) ([]byte, []Warning, error) {
 	// assume JSON as the default
@@ -144,12 +189,11 @@ func adaptByContentType(contentType string, body []byte) ([]byte, []Warning, err
 	}
 
 	// adapter name should be suffix of MIME type
-	slashIdx := strings.Index(ct, "/")
-	if slashIdx < 0 {
+	_, adapterName, slashFound := strings.Cut(ct, "/")
+	if !slashFound {
 		return nil, nil, fmt.Errorf("malformed Content-Type")
 	}
 
-	adapterName := ct[slashIdx+1:]
 	cfgAdapter := GetAdapter(adapterName)
 	if cfgAdapter == nil {
 		return nil, nil, fmt.Errorf("unrecognized config adapter '%s'", adapterName)
@@ -164,7 +208,7 @@ func adaptByContentType(contentType string, body []byte) ([]byte, []Warning, err
 }
 
 var bufPool = sync.Pool{
-	New: func() interface{} {
+	New: func() any {
 		return new(bytes.Buffer)
 	},
 }

caddytest/caddytest.go

@@ -7,7 +7,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"net"
 	"net/http"
@@ -100,7 +100,7 @@ func (tc *Tester) InitServer(rawConfig string, configType string) {
 		tc.t.Fail()
 	}
 	if err := tc.ensureConfigRunning(rawConfig, configType); err != nil {
-		tc.t.Logf("failed ensurng config is running: %s", err)
+		tc.t.Logf("failed ensuring config is running: %s", err)
 		tc.t.Fail()
 	}
 }
@@ -129,7 +129,7 @@ func (tc *Tester) initServer(rawConfig string, configType string) error {
 				return
 			}
 			defer res.Body.Close()
-			body, _ := ioutil.ReadAll(res.Body)
+			body, _ := io.ReadAll(res.Body)
 
 			var out bytes.Buffer
 			_ = json.Indent(&out, body, "", "  ")
@@ -162,7 +162,7 @@ func (tc *Tester) initServer(rawConfig string, configType string) error {
 	timeElapsed(start, "caddytest: config load time")
 
 	defer res.Body.Close()
-	body, err := ioutil.ReadAll(res.Body)
+	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		tc.t.Errorf("unable to read response. %s", err)
 		return err
@@ -186,7 +186,7 @@ func (tc *Tester) ensureConfigRunning(rawConfig string, configType string) error
 		expectedBytes, _, _ = adapter.Adapt([]byte(rawConfig), nil)
 	}
 
-	var expected interface{}
+	var expected any
 	err := json.Unmarshal(expectedBytes, &expected)
 	if err != nil {
 		return err
@@ -196,17 +196,17 @@ func (tc *Tester) ensureConfigRunning(rawConfig string, configType string) error
 		Timeout: Default.LoadRequestTimeout,
 	}
 
-	fetchConfig := func(client *http.Client) interface{} {
+	fetchConfig := func(client *http.Client) any {
 		resp, err := client.Get(fmt.Sprintf("http://localhost:%d/config/", Default.AdminPort))
 		if err != nil {
 			return nil
 		}
 		defer resp.Body.Close()
-		actualBytes, err := ioutil.ReadAll(resp.Body)
+		actualBytes, err := io.ReadAll(resp.Body)
 		if err != nil {
 			return nil
 		}
-		var actual interface{}
+		var actual any
 		err = json.Unmarshal(actualBytes, &actual)
 		if err != nil {
 			return nil
@@ -371,7 +371,7 @@ func CompareAdapt(t *testing.T, filename, rawConfig string, adapterName string,
 		return false
 	}
 
-	options := make(map[string]interface{})
+	options := make(map[string]any)
 
 	result, warnings, err := cfgAdapter.Adapt([]byte(rawConfig), options)
 	if err != nil {
@@ -471,7 +471,7 @@ func (tc *Tester) AssertResponse(req *http.Request, expectedStatusCode int, expe
 	resp := tc.AssertResponseCode(req, expectedStatusCode)
 
 	defer resp.Body.Close()
-	bytes, err := ioutil.ReadAll(resp.Body)
+	bytes, err := io.ReadAll(resp.Body)
 	if err != nil {
 		tc.t.Fatalf("unable to read the response body %s", err)
 	}

caddytest/integration/autohttps_test.go

@@ -103,3 +103,23 @@ func TestAutoHTTPRedirectsInsertedBeforeUserDefinedCatchAll(t *testing.T) {
 	tester.AssertGetResponse("http://foo.localhost:9080/", 200, "Foo")
 	tester.AssertGetResponse("http://baz.localhost:9080/", 200, "Baz")
 }
+
+func TestAutoHTTPRedirectsInsertedBeforeUserDefinedCatchAllWithNoExplicitHTTPSite(t *testing.T) {
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+	{
+		http_port     9080
+		https_port    9443
+		local_certs
+	}
+	http://:9080 {
+		respond "Foo"
+	}
+	bar.localhost {
+		respond "Bar"
+	}
+  `, "caddyfile")
+	tester.AssertRedirect("http://bar.localhost:9080/", "https://bar.localhost/", http.StatusPermanentRedirect)
+	tester.AssertGetResponse("http://foo.localhost:9080/", 200, "Foo")
+	tester.AssertGetResponse("http://baz.localhost:9080/", 200, "Foo")
+}

caddytest/integration/caddyfile_adapt/bind_ipv6.txt

@@ -0,0 +1,29 @@
+example.com {
+	bind tcp6/[::]
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						"tcp6/[::]:443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/error_example.txt

@@ -0,0 +1,138 @@
+example.com {
+	root * /srv
+
+	# Trigger errors for certain paths
+	error /private* "Unauthorized" 403
+	error /hidden* "Not found" 404
+
+	# Handle the error by serving an HTML page 
+	handle_errors {
+		rewrite * /{http.error.status_code}.html
+		file_server
+	}
+
+	file_server
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "vars",
+													"root": "/srv"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Unauthorized",
+													"handler": "error",
+													"status_code": 403
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/private*"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"error": "Not found",
+													"handler": "error",
+													"status_code": 404
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/hidden*"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"handler": "file_server",
+													"hide": [
+														"./Caddyfile"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"errors": {
+						"routes": [
+							{
+								"match": [
+									{
+										"host": [
+											"example.com"
+										]
+									}
+								],
+								"handle": [
+									{
+										"handler": "subroute",
+										"routes": [
+											{
+												"group": "group0",
+												"handle": [
+													{
+														"handler": "rewrite",
+														"uri": "/{http.error.status_code}.html"
+													}
+												]
+											},
+											{
+												"handle": [
+													{
+														"handler": "file_server",
+														"hide": [
+															"./Caddyfile"
+														]
+													}
+												]
+											}
+										]
+									}
+								],
+								"terminal": true
+							}
+						]
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/expression_quotes.txt

@@ -0,0 +1,114 @@
+example.com
+
+@a expression {http.error.status_code} == 400
+abort @a
+
+@b expression {http.error.status_code} == "401"
+abort @b
+
+@c expression {http.error.status_code} == `402`
+abort @c
+
+@d expression "{http.error.status_code} == 403"
+abort @d
+
+@e expression `{http.error.status_code} == 404`
+abort @e
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == 400"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == \"401\""
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == `402`"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == 403"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"abort": true,
+													"handler": "static_response"
+												}
+											],
+											"match": [
+												{
+													"expression": "{http.error.status_code} == 404"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/file_server_disable_canonical_uris.txt

@@ -0,0 +1,32 @@
+:80
+
+file_server {
+	disable_canonical_uris
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"canonical_uris": false,
+									"handler": "file_server",
+									"hide": [
+										"./Caddyfile"
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/file_server_pass_thru.txt

@@ -0,0 +1,32 @@
+:80
+
+file_server {
+	pass_thru
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "file_server",
+									"hide": [
+										"./Caddyfile"
+									],
+									"pass_thru": true
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/forward_auth_authelia.txt

@@ -0,0 +1,111 @@
+app.example.com {
+	forward_auth authelia:9091 {
+		uri /api/verify?rd=https://authelia.example.com
+		copy_headers Remote-User Remote-Groups Remote-Name Remote-Email
+	}
+
+	reverse_proxy backend:8080
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"app.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handle_response": [
+														{
+															"match": {
+																"status_code": [
+																	2
+																]
+															},
+															"routes": [
+																{
+																	"handle": [
+																		{
+																			"handler": "headers",
+																			"request": {
+																				"set": {
+																					"Remote-Email": [
+																						"{http.reverse_proxy.header.Remote-Email}"
+																					],
+																					"Remote-Groups": [
+																						"{http.reverse_proxy.header.Remote-Groups}"
+																					],
+																					"Remote-Name": [
+																						"{http.reverse_proxy.header.Remote-Name}"
+																					],
+																					"Remote-User": [
+																						"{http.reverse_proxy.header.Remote-User}"
+																					]
+																				}
+																			}
+																		}
+																	]
+																}
+															]
+														}
+													],
+													"handler": "reverse_proxy",
+													"headers": {
+														"request": {
+															"set": {
+																"X-Forwarded-Method": [
+																	"{http.request.method}"
+																],
+																"X-Forwarded-Uri": [
+																	"{http.request.uri}"
+																]
+															}
+														}
+													},
+													"rewrite": {
+														"method": "GET",
+														"uri": "/api/verify?rd=https://authelia.example.com"
+													},
+													"upstreams": [
+														{
+															"dial": "authelia:9091"
+														}
+													]
+												},
+												{
+													"handler": "reverse_proxy",
+													"upstreams": [
+														{
+															"dial": "backend:8080"
+														}
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/forward_auth_rename_headers.txt

@@ -0,0 +1,90 @@
+:8881
+
+forward_auth localhost:9000 {
+	uri /auth
+	copy_headers A>1 B C>3 {
+		D
+		E>5
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8881"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handle_response": [
+										{
+											"match": {
+												"status_code": [
+													2
+												]
+											},
+											"routes": [
+												{
+													"handle": [
+														{
+															"handler": "headers",
+															"request": {
+																"set": {
+																	"1": [
+																		"{http.reverse_proxy.header.A}"
+																	],
+																	"3": [
+																		"{http.reverse_proxy.header.C}"
+																	],
+																	"5": [
+																		"{http.reverse_proxy.header.E}"
+																	],
+																	"B": [
+																		"{http.reverse_proxy.header.B}"
+																	],
+																	"D": [
+																		"{http.reverse_proxy.header.D}"
+																	]
+																}
+															}
+														}
+													]
+												}
+											]
+										}
+									],
+									"handler": "reverse_proxy",
+									"headers": {
+										"request": {
+											"set": {
+												"X-Forwarded-Method": [
+													"{http.request.method}"
+												],
+												"X-Forwarded-Uri": [
+													"{http.request.uri}"
+												]
+											}
+										}
+									},
+									"rewrite": {
+										"method": "GET",
+										"uri": "/auth"
+									},
+									"upstreams": [
+										{
+											"dial": "localhost:9000"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options.txt

@@ -3,6 +3,7 @@
 	http_port 8080
 	https_port 8443
 	grace_period 5s
+	shutdown_delay 10s
 	default_sni localhost
 	order root first
 	storage file_system {
@@ -10,6 +11,7 @@
 	}
 	acme_ca https://example.com
 	acme_ca_root /path/to/ca.crt
+	ocsp_stapling off
 
 	email test@example.com
 	admin off
@@ -44,6 +46,7 @@
 			"http_port": 8080,
 			"https_port": 8443,
 			"grace_period": 5000000000,
+			"shutdown_delay": 10000000000,
 			"servers": {
 				"srv0": {
 					"listen": [
@@ -61,7 +64,8 @@
 								"module": "internal"
 							}
 						],
-						"key_type": "ed25519"
+						"key_type": "ed25519",
+						"disable_ocsp_stapling": true
 					}
 				],
 				"on_demand": {
@@ -71,7 +75,8 @@
 					},
 					"ask": "https://example.com"
 				}
-			}
+			},
+			"disable_ocsp_stapling": true
 		}
 	}
 }

caddytest/integration/caddyfile_adapt/global_options_acme.txt

@@ -21,6 +21,8 @@
 		burst 20
 	}
 	storage_clean_interval 7d
+	renew_interval 1d
+	ocsp_interval 2d
 
 	key_type ed25519
 }
@@ -82,6 +84,8 @@
 					},
 					"ask": "https://example.com"
 				},
+				"ocsp_interval": 172800000000000,
+				"renew_interval": 86400000000000,
 				"storage_clean_interval": 604800000000000
 			}
 		}

caddytest/integration/caddyfile_adapt/global_options_debug_with_access_log.txt

@@ -0,0 +1,45 @@
+{
+	debug
+}
+
+:8881 {
+	log {
+		format console
+	}
+}
+----------
+{
+	"logging": {
+		"logs": {
+			"default": {
+				"level": "DEBUG",
+				"exclude": [
+					"http.log.access.log0"
+				]
+			},
+			"log0": {
+				"encoder": {
+					"format": "console"
+				},
+				"level": "DEBUG",
+				"include": [
+					"http.log.access.log0"
+				]
+			}
+		}
+	},
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8881"
+					],
+					"logs": {
+						"default_logger_name": "log0"
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options_default_bind.txt

@@ -0,0 +1,54 @@
+{
+	default_bind tcp4/0.0.0.0 tcp6/[::]
+}
+
+example.com {
+}
+
+example.org:12345 {
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						"tcp4/0.0.0.0:12345",
+						"tcp6/[::]:12345"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.org"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				},
+				"srv1": {
+					"listen": [
+						"tcp4/0.0.0.0:443",
+						"tcp6/[::]:443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options_log_custom.txt

@@ -3,8 +3,7 @@
 		format filter {
 			wrap console
 			fields {
-				common_log delete
-				request>remote_addr ip_mask {
+				request>remote_ip ip_mask {
 					ipv4 24
 					ipv6 32
 				}
@@ -19,10 +18,7 @@
 			"custom-logger": {
 				"encoder": {
 					"fields": {
-						"common_log": {
-							"filter": "delete"
-						},
-						"request\u003eremote_addr": {
+						"request\u003eremote_ip": {
 							"filter": "ip_mask",
 							"ipv4_cidr": 24,
 							"ipv6_cidr": 32

caddytest/integration/caddyfile_adapt/global_options_preferred_chains.txt

@@ -0,0 +1,56 @@
+{
+	preferred_chains smallest
+}
+
+example.com
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"example.com"
+						],
+						"issuers": [
+							{
+								"module": "acme",
+								"preferred_chains": {
+									"smallest": true
+								}
+							},
+							{
+								"module": "zerossl",
+								"preferred_chains": {
+									"smallest": true
+								}
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_options_skip_install_trust.txt

@@ -0,0 +1,168 @@
+{
+	skip_install_trust
+	pki {
+		ca {
+			name "Local"
+			root_cn "Custom Local Root Name"
+			intermediate_cn "Custom Local Intermediate Name"
+			root {
+				cert /path/to/cert.pem
+				key /path/to/key.pem
+				format pem_file
+			}
+			intermediate {
+				cert /path/to/cert.pem
+				key /path/to/key.pem
+				format pem_file
+			}
+		}
+		ca foo {
+			name "Foo"
+			root_cn "Custom Foo Root Name"
+			intermediate_cn "Custom Foo Intermediate Name"
+		}
+	}
+}
+
+a.example.com {
+	tls internal
+}
+
+acme.example.com {
+	acme_server {
+		ca foo
+	}
+}
+
+acme-bar.example.com {
+	acme_server {
+		ca bar
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"acme-bar.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"ca": "bar",
+													"handler": "acme_server"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"acme.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"ca": "foo",
+													"handler": "acme_server"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"a.example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"pki": {
+			"certificate_authorities": {
+				"bar": {
+					"install_trust": false
+				},
+				"foo": {
+					"name": "Foo",
+					"root_common_name": "Custom Foo Root Name",
+					"intermediate_common_name": "Custom Foo Intermediate Name",
+					"install_trust": false
+				},
+				"local": {
+					"name": "Local",
+					"root_common_name": "Custom Local Root Name",
+					"intermediate_common_name": "Custom Local Intermediate Name",
+					"install_trust": false,
+					"root": {
+						"certificate": "/path/to/cert.pem",
+						"private_key": "/path/to/key.pem",
+						"format": "pem_file"
+					},
+					"intermediate": {
+						"certificate": "/path/to/cert.pem",
+						"private_key": "/path/to/key.pem",
+						"format": "pem_file"
+					}
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"acme-bar.example.com",
+							"acme.example.com"
+						]
+					},
+					{
+						"subjects": [
+							"a.example.com"
+						],
+						"issuers": [
+							{
+								"module": "internal"
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_server_options_multi.txt

@@ -3,6 +3,9 @@
 		timeouts {
 			idle 90s
 		}
+		protocol {
+			strict_sni_host insecure_off
+		}
 	}
 	servers :80 {
 		timeouts {
@@ -13,6 +16,9 @@
 		timeouts {
 			idle 30s
 		}
+		protocol {
+			strict_sni_host
+		}
 	}
 }
 
@@ -46,7 +52,8 @@ http://bar.com {
 							],
 							"terminal": true
 						}
-					]
+					],
+					"strict_sni_host": true
 				},
 				"srv1": {
 					"listen": [
@@ -70,7 +77,8 @@ http://bar.com {
 					"listen": [
 						":8080"
 					],
-					"idle_timeout": 90000000000
+					"idle_timeout": 90000000000,
+					"strict_sni_host": false
 				}
 			}
 		}

caddytest/integration/caddyfile_adapt/global_server_options_single.txt

@@ -1,6 +1,7 @@
 {
 	servers {
 		listener_wrappers {
+			http_redirect
 			tls
 		}
 		timeouts {
@@ -10,11 +11,9 @@
 			idle 30s
 		}
 		max_header_size 100MB
-		protocol {
-			allow_h2c
-			experimental_http3
-			strict_sni_host
-		}
+		log_credentials
+		strict_sni_host
+		protocols h1 h2 h2c h3
 	}
 }
 
@@ -31,6 +30,9 @@ foo.com {
 						":443"
 					],
 					"listener_wrappers": [
+						{
+							"wrapper": "http_redirect"
+						},
 						{
 							"wrapper": "tls"
 						}
@@ -53,8 +55,15 @@ foo.com {
 						}
 					],
 					"strict_sni_host": true,
-					"experimental_http3": true,
-					"allow_h2c": true
+					"logs": {
+						"should_log_credentials": true
+					},
+					"protocols": [
+						"h1",
+						"h2",
+						"h2c",
+						"h3"
+					]
 				}
 			}
 		}

caddytest/integration/caddyfile_adapt/header.txt

@@ -13,6 +13,10 @@
 	header @images {
 		Cache-Control "public, max-age=3600, stale-while-revalidate=86400"
 	}
+	header {
+		+Link "Foo"
+		+Link "Bar"
+	}
 }
 ----------
 {
@@ -121,6 +125,17 @@
 											]
 										}
 									}
+								},
+								{
+									"handler": "headers",
+									"response": {
+										"add": {
+											"Link": [
+												"Foo",
+												"Bar"
+											]
+										}
+									}
 								}
 							]
 						}

caddytest/integration/caddyfile_adapt/log_filters.txt

@@ -5,12 +5,24 @@ log {
 	format filter {
 		wrap console
 		fields {
+			uri query {
+				replace foo REDACTED
+				delete bar
+				hash baz
+			}
 			request>headers>Authorization replace REDACTED
 			request>headers>Server delete
-			request>remote_addr ip_mask {
+			request>headers>Cookie cookie {
+				replace foo REDACTED
+				delete bar
+				hash baz
+			}
+			request>remote_ip ip_mask {
 				ipv4 24
 				ipv6 32
 			}
+			request>headers>Regexp regexp secret REDACTED
+			request>headers>Hash hash
 		}
 	}
 }
@@ -33,13 +45,57 @@ log {
 							"filter": "replace",
 							"value": "REDACTED"
 						},
+						"request\u003eheaders\u003eCookie": {
+							"actions": [
+								{
+									"name": "foo",
+									"type": "replace",
+									"value": "REDACTED"
+								},
+								{
+									"name": "bar",
+									"type": "delete"
+								},
+								{
+									"name": "baz",
+									"type": "hash"
+								}
+							],
+							"filter": "cookie"
+						},
+						"request\u003eheaders\u003eHash": {
+							"filter": "hash"
+						},
+						"request\u003eheaders\u003eRegexp": {
+							"filter": "regexp",
+							"regexp": "secret",
+							"value": "REDACTED"
+						},
 						"request\u003eheaders\u003eServer": {
 							"filter": "delete"
 						},
-						"request\u003eremote_addr": {
+						"request\u003eremote_ip": {
 							"filter": "ip_mask",
 							"ipv4_cidr": 24,
 							"ipv6_cidr": 32
+						},
+						"uri": {
+							"actions": [
+								{
+									"parameter": "foo",
+									"type": "replace",
+									"value": "REDACTED"
+								},
+								{
+									"parameter": "bar",
+									"type": "delete"
+								},
+								{
+									"parameter": "baz",
+									"type": "hash"
+								}
+							],
+							"filter": "query"
 						}
 					},
 					"format": "filter",

caddytest/integration/caddyfile_adapt/log_roll_days.txt

@@ -3,6 +3,8 @@
 log {
 	output file /var/log/access.log {
 		roll_size 1gb
+		roll_uncompressed
+		roll_local_time
 		roll_keep 5
 		roll_keep_for 90d
 	}
@@ -20,8 +22,10 @@ log {
 				"writer": {
 					"filename": "/var/log/access.log",
 					"output": "file",
+					"roll_gzip": false,
 					"roll_keep": 5,
 					"roll_keep_days": 90,
+					"roll_local_time": true,
 					"roll_size_mb": 954
 				},
 				"include": [

caddytest/integration/caddyfile_adapt/log_skip_hosts.txt

@@ -0,0 +1,75 @@
+one.example.com {
+	log
+}
+
+two.example.com {
+}
+
+three.example.com {
+}
+
+example.com {
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"three.example.com"
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"one.example.com"
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"two.example.com"
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"logs": {
+						"skip_hosts": [
+							"three.example.com",
+							"two.example.com",
+							"example.com"
+						]
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/map_and_vars_with_raw_types.txt

@@ -0,0 +1,126 @@
+example.com
+
+map {host}             {my_placeholder}  {magic_number} {
+	# Should output boolean "true" and an integer
+	example.com        true              3
+
+	# Should output a string and null
+	foo.example.com    "string value"
+
+	# Should output two strings (quoted int)
+	(.*)\.example.com  "${1} subdomain"  "5"
+
+	# Should output null and a string (quoted int)
+	~.*\.net$          -                 `7`
+
+	# Should output a float and the string "false"
+	~.*\.xyz$          123.456           "false"
+
+	# Should output two strings, second being escaped quote
+	default            "unknown domain"  \"""
+}
+
+vars foo bar
+vars {
+	abc true
+	def 1
+	ghi 2.3
+	jkl "mn op"
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"defaults": [
+														"unknown domain",
+														"\""
+													],
+													"destinations": [
+														"{my_placeholder}",
+														"{magic_number}"
+													],
+													"handler": "map",
+													"mappings": [
+														{
+															"input": "example.com",
+															"outputs": [
+																true,
+																3
+															]
+														},
+														{
+															"input": "foo.example.com",
+															"outputs": [
+																"string value",
+																null
+															]
+														},
+														{
+															"input": "(.*)\\.example.com",
+															"outputs": [
+																"${1} subdomain",
+																"5"
+															]
+														},
+														{
+															"input_regexp": ".*\\.net$",
+															"outputs": [
+																null,
+																"7"
+															]
+														},
+														{
+															"input_regexp": ".*\\.xyz$",
+															"outputs": [
+																123.456,
+																"false"
+															]
+														}
+													],
+													"source": "{http.request.host}"
+												},
+												{
+													"foo": "bar",
+													"handler": "vars"
+												},
+												{
+													"abc": true,
+													"def": 1,
+													"ghi": 2.3,
+													"handler": "vars",
+													"jkl": "mn op"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/matcher_syntax.txt

@@ -19,24 +19,30 @@
 	@matcher6 vars_regexp "{http.request.uri}" `\.([a-f0-9]{6})\.(css|js)$`
 	respond @matcher6 "from vars_regexp matcher without name"
 
-	@matcher7 {
+	@matcher7 `path('/foo*') && method('GET')`
+	respond @matcher7 "inline expression matcher shortcut"
+
+	@matcher8 {
 		header Foo bar
 		header Foo foobar
 		header Bar foo
 	}
-	respond @matcher7 "header matcher merging values of the same field"
+	respond @matcher8 "header matcher merging values of the same field"
 
-	@matcher8 {
+	@matcher9 {
 		query foo=bar foo=baz bar=foo
 		query bar=baz
 	}
-	respond @matcher8 "query matcher merging pairs with the same keys"
+	respond @matcher9 "query matcher merging pairs with the same keys"
 
-	@matcher9 {
+	@matcher10 {
 		header !Foo
 		header Bar foo
 	}
-	respond @matcher9 "header matcher with null field matcher"
+	respond @matcher10 "header matcher with null field matcher"
+
+	@matcher11 remote_ip private_ranges
+	respond @matcher11 "remote_ip matcher with private ranges"
 }
 ----------
 {
@@ -101,7 +107,9 @@
 							"match": [
 								{
 									"vars": {
-										"{http.request.uri}": "/vars-matcher"
+										"{http.request.uri}": [
+											"/vars-matcher"
+										]
 									}
 								}
 							],
@@ -147,6 +155,19 @@
 								}
 							]
 						},
+						{
+							"match": [
+								{
+									"expression": "path('/foo*') \u0026\u0026 method('GET')"
+								}
+							],
+							"handle": [
+								{
+									"body": "inline expression matcher shortcut",
+									"handler": "static_response"
+								}
+							]
+						},
 						{
 							"match": [
 								{
@@ -207,6 +228,28 @@
 									"handler": "static_response"
 								}
 							]
+						},
+						{
+							"match": [
+								{
+									"remote_ip": {
+										"ranges": [
+											"192.168.0.0/16",
+											"172.16.0.0/12",
+											"10.0.0.0/8",
+											"127.0.0.1/8",
+											"fd00::/8",
+											"::1"
+										]
+									}
+								}
+							],
+							"handle": [
+								{
+									"body": "remote_ip matcher with private ranges",
+									"handler": "static_response"
+								}
+							]
 						}
 					]
 				}

caddytest/integration/caddyfile_adapt/method_directive.txt

@@ -0,0 +1,27 @@
+:8080 {
+	method FOO
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8080"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "rewrite",
+									"method": "FOO"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/php_fastcgi_handle_response.txt

@@ -0,0 +1,145 @@
+:8881 {
+	php_fastcgi app:9000 {
+		env FOO bar
+
+		@error status 4xx
+		handle_response @error {
+			root * /errors
+			rewrite * /{http.reverse_proxy.status_code}.html
+			file_server
+		}
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8881"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"file": {
+										"try_files": [
+											"{http.request.uri.path}/index.php"
+										]
+									},
+									"not": [
+										{
+											"path": [
+												"*/"
+											]
+										}
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "static_response",
+									"headers": {
+										"Location": [
+											"{http.request.uri.path}/"
+										]
+									},
+									"status_code": 308
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"file": {
+										"try_files": [
+											"{http.request.uri.path}",
+											"{http.request.uri.path}/index.php",
+											"index.php"
+										],
+										"split_path": [
+											".php"
+										]
+									}
+								}
+							],
+							"handle": [
+								{
+									"handler": "rewrite",
+									"uri": "{http.matchers.file.relative}"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"path": [
+										"*.php"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handle_response": [
+										{
+											"match": {
+												"status_code": [
+													4
+												]
+											},
+											"routes": [
+												{
+													"handle": [
+														{
+															"handler": "vars",
+															"root": "/errors"
+														}
+													]
+												},
+												{
+													"group": "group0",
+													"handle": [
+														{
+															"handler": "rewrite",
+															"uri": "/{http.reverse_proxy.status_code}.html"
+														}
+													]
+												},
+												{
+													"handle": [
+														{
+															"handler": "file_server",
+															"hide": [
+																"./Caddyfile"
+															]
+														}
+													]
+												}
+											]
+										}
+									],
+									"handler": "reverse_proxy",
+									"transport": {
+										"env": {
+											"FOO": "bar"
+										},
+										"protocol": "fastcgi",
+										"split_path": [
+											".php"
+										]
+									},
+									"upstreams": [
+										{
+											"dial": "app:9000"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/php_fastcgi_try_files_override.txt

@@ -0,0 +1,124 @@
+:8884
+
+php_fastcgi localhost:9000 {
+	# some php_fastcgi-specific subdirectives
+	split .php .php5
+	env VAR1 value1
+	env VAR2 value2
+	root /var/www
+	try_files {path} {path}/index.php =404
+	dial_timeout 3s
+	read_timeout 10s
+	write_timeout 20s
+
+	# passed through to reverse_proxy (directive order doesn't matter!)
+	lb_policy random
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"file": {
+										"try_files": [
+											"{http.request.uri.path}/index.php"
+										]
+									},
+									"not": [
+										{
+											"path": [
+												"*/"
+											]
+										}
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "static_response",
+									"headers": {
+										"Location": [
+											"{http.request.uri.path}/"
+										]
+									},
+									"status_code": 308
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"file": {
+										"try_files": [
+											"{http.request.uri.path}",
+											"{http.request.uri.path}/index.php",
+											"=404"
+										],
+										"split_path": [
+											".php",
+											".php5"
+										]
+									}
+								}
+							],
+							"handle": [
+								{
+									"handler": "rewrite",
+									"uri": "{http.matchers.file.relative}"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"path": [
+										"*.php",
+										"*.php5"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "reverse_proxy",
+									"load_balancing": {
+										"selection_policy": {
+											"policy": "random"
+										}
+									},
+									"transport": {
+										"dial_timeout": 3000000000,
+										"env": {
+											"VAR1": "value1",
+											"VAR2": "value2"
+										},
+										"protocol": "fastcgi",
+										"read_timeout": 10000000000,
+										"root": "/var/www",
+										"split_path": [
+											".php",
+											".php5"
+										],
+										"write_timeout": 20000000000
+									},
+									"upstreams": [
+										{
+											"dial": "localhost:9000"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/reverse_proxy_dynamic_upstreams.txt

@@ -0,0 +1,116 @@
+:8884 {
+	reverse_proxy {
+		dynamic a foo 9000
+	}
+
+	reverse_proxy {
+		dynamic a {
+			name foo
+			port 9000
+			refresh 5m
+			resolvers 8.8.8.8 8.8.4.4
+			dial_timeout 2s
+			dial_fallback_delay 300ms
+		}
+	}
+}
+
+:8885 {
+	reverse_proxy {
+		dynamic srv _api._tcp.example.com
+	}
+
+	reverse_proxy {
+		dynamic srv {
+			service api
+			proto tcp
+			name example.com
+			refresh 5m
+			resolvers 8.8.8.8 8.8.4.4
+			dial_timeout 1s
+			dial_fallback_delay -1s
+		}
+	}
+}
+
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"dynamic_upstreams": {
+										"name": "foo",
+										"port": "9000",
+										"source": "a"
+									},
+									"handler": "reverse_proxy"
+								},
+								{
+									"dynamic_upstreams": {
+										"dial_fallback_delay": 300000000,
+										"dial_timeout": 2000000000,
+										"name": "foo",
+										"port": "9000",
+										"refresh": 300000000000,
+										"resolver": {
+											"addresses": [
+												"8.8.8.8",
+												"8.8.4.4"
+											]
+										},
+										"source": "a"
+									},
+									"handler": "reverse_proxy"
+								}
+							]
+						}
+					]
+				},
+				"srv1": {
+					"listen": [
+						":8885"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"dynamic_upstreams": {
+										"name": "_api._tcp.example.com",
+										"source": "srv"
+									},
+									"handler": "reverse_proxy"
+								},
+								{
+									"dynamic_upstreams": {
+										"dial_fallback_delay": -1000000000,
+										"dial_timeout": 1000000000,
+										"name": "example.com",
+										"proto": "tcp",
+										"refresh": 300000000000,
+										"resolver": {
+											"addresses": [
+												"8.8.8.8",
+												"8.8.4.4"
+											]
+										},
+										"service": "api",
+										"source": "srv"
+									},
+									"handler": "reverse_proxy"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/reverse_proxy_h2c_shorthand.txt

@@ -1,6 +1,8 @@
 :8884
 
 reverse_proxy h2c://localhost:8080
+
+reverse_proxy unix+h2c//run/app.sock
 ----------
 {
 	"apps": {
@@ -27,6 +29,21 @@ reverse_proxy h2c://localhost:8080
 											"dial": "localhost:8080"
 										}
 									]
+								},
+								{
+									"handler": "reverse_proxy",
+									"transport": {
+										"protocol": "http",
+										"versions": [
+											"h2c",
+											"2"
+										]
+									},
+									"upstreams": [
+										{
+											"dial": "unix//run/app.sock"
+										}
+									]
 								}
 							]
 						}

caddytest/integration/caddyfile_adapt/reverse_proxy_handle_response.txt

@@ -1,6 +1,14 @@
 :8884
 
 reverse_proxy 127.0.0.1:65535 {
+	@500 status 500
+	replace_status @500 400
+
+	@all status 2xx 3xx 4xx 5xx
+	replace_status @all {http.error.status_code}
+
+	replace_status {http.error.status_code}
+
 	@accel header X-Accel-Redirect *
 	handle_response @accel {
 		respond "Header X-Accel-Redirect!"
@@ -39,8 +47,19 @@ reverse_proxy 127.0.0.1:65535 {
 		respond "Headers Foo, Bar AND statuses 401, 403 and 404!"
 	}
 
-	@changeStatus status 500
-	handle_response @changeStatus 400
+	@200 status 200
+	handle_response @200 {
+		copy_response_headers {
+			include Foo Bar
+		}
+		respond "Copied headers from the response"
+	}
+
+	@201 status 201
+	handle_response @201 {
+		header Foo "Copying the response"
+		copy_response 404
+	}
 }
 ----------
 {
@@ -56,6 +75,25 @@ reverse_proxy 127.0.0.1:65535 {
 							"handle": [
 								{
 									"handle_response": [
+										{
+											"match": {
+												"status_code": [
+													500
+												]
+											},
+											"status_code": 400
+										},
+										{
+											"match": {
+												"status_code": [
+													2,
+													3,
+													4,
+													5
+												]
+											},
+											"status_code": "{http.error.status_code}"
+										},
 										{
 											"match": {
 												"headers": {
@@ -158,10 +196,56 @@ reverse_proxy 127.0.0.1:65535 {
 										{
 											"match": {
 												"status_code": [
-													500
+													200
 												]
 											},
-											"status_code": 400
+											"routes": [
+												{
+													"handle": [
+														{
+															"handler": "copy_response_headers",
+															"include": [
+																"Foo",
+																"Bar"
+															]
+														},
+														{
+															"body": "Copied headers from the response",
+															"handler": "static_response"
+														}
+													]
+												}
+											]
+										},
+										{
+											"match": {
+												"status_code": [
+													201
+												]
+											},
+											"routes": [
+												{
+													"handle": [
+														{
+															"handler": "headers",
+															"response": {
+																"set": {
+																	"Foo": [
+																		"Copying the response"
+																	]
+																}
+															}
+														},
+														{
+															"handler": "copy_response",
+															"status_code": 404
+														}
+													]
+												}
+											]
+										},
+										{
+											"status_code": "{http.error.status_code}"
 										},
 										{
 											"routes": [

caddytest/integration/caddyfile_adapt/reverse_proxy_health_headers.txt

@@ -7,6 +7,7 @@ reverse_proxy 127.0.0.1:65535 {
 		X-Header-Keys VbG4NZwWnipo 335Q9/MhqcNU3s2TO
 		X-Empty-Value
 	}
+	health_uri /health
 }
 ----------
 {
@@ -38,7 +39,8 @@ reverse_proxy 127.0.0.1:65535 {
 													"VbG4NZwWnipo",
 													"335Q9/MhqcNU3s2TO"
 												]
-											}
+											},
+											"uri": "/health"
 										}
 									},
 									"upstreams": [

caddytest/integration/caddyfile_adapt/reverse_proxy_load_balance.txt

@@ -0,0 +1,64 @@
+:8884
+
+reverse_proxy 127.0.0.1:65535 {
+	lb_policy first
+	lb_retries 5
+	lb_try_duration 10s
+	lb_try_interval 500ms
+	lb_retry_match {
+		path /foo*
+		method POST
+	}
+	lb_retry_match path /bar*
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "reverse_proxy",
+									"load_balancing": {
+										"retries": 5,
+										"retry_match": [
+											{
+												"method": [
+													"POST"
+												],
+												"path": [
+													"/foo*"
+												]
+											},
+											{
+												"path": [
+													"/bar*"
+												]
+											}
+										],
+										"selection_policy": {
+											"policy": "first"
+										},
+										"try_duration": 10000000000,
+										"try_interval": 500000000
+									},
+									"upstreams": [
+										{
+											"dial": "127.0.0.1:65535"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/reverse_proxy_options.txt

@@ -1,11 +1,11 @@
 
 https://example.com {
-	reverse_proxy /path http://localhost:54321 {
-		header_up Host {host}
-		header_up X-Real-IP {remote}
-		header_up X-Forwarded-For {remote}
-		header_up X-Forwarded-Port {server_port}
-		header_up X-Forwarded-Proto "http"
+	reverse_proxy /path https://localhost:54321 {
+		header_up Host {upstream_hostport}
+		header_up Foo bar
+
+		method GET
+		rewrite /rewritten?uri={uri}
 
 		buffer_requests
 
@@ -17,11 +17,16 @@ https://example.com {
 			dial_fallback_delay 5s
 			response_header_timeout 8s
 			expect_continue_timeout 9s
+			resolvers 8.8.8.8 8.8.4.4
 
 			versions h2c 2
 			compression off
 			max_conns_per_host 5
-			max_idle_conns_per_host 2
+			keepalive_idle_conns_per_host 2
+			keepalive_interval 30s
+			
+			tls_renegotiation freely
+			tls_except_ports 8181 8182
 		}
 	}
 }
@@ -56,35 +61,46 @@ https://example.com {
 													"headers": {
 														"request": {
 															"set": {
+																"Foo": [
+																	"bar"
+																],
 																"Host": [
-																	"{http.request.host}"
-																],
-																"X-Forwarded-For": [
-																	"{http.request.remote}"
-																],
-																"X-Forwarded-Port": [
-																	"{server_port}"
-																],
-																"X-Forwarded-Proto": [
-																	"http"
-																],
-																"X-Real-Ip": [
-																	"{http.request.remote}"
+																	"{http.reverse_proxy.upstream.hostport}"
 																]
 															}
 														}
 													},
+													"rewrite": {
+														"method": "GET",
+														"uri": "/rewritten?uri={http.request.uri}"
+													},
 													"transport": {
 														"compression": false,
 														"dial_fallback_delay": 5000000000,
 														"dial_timeout": 3000000000,
 														"expect_continue_timeout": 9000000000,
+														"keep_alive": {
+															"max_idle_conns_per_host": 2,
+															"probe_interval": 30000000000
+														},
 														"max_conns_per_host": 5,
-														"max_idle_conns_per_host": 2,
 														"max_response_header_size": 30000000,
 														"protocol": "http",
 														"read_buffer_size": 10000000,
+														"resolver": {
+															"addresses": [
+																"8.8.8.8",
+																"8.8.4.4"
+															]
+														},
 														"response_header_timeout": 8000000000,
+														"tls": {
+															"except_ports": [
+																"8181",
+																"8182"
+															],
+															"renegotiation": "freely"
+														},
 														"versions": [
 															"h2c",
 															"2"

caddytest/integration/caddyfile_adapt/reverse_proxy_trusted_proxies.txt

@@ -0,0 +1,56 @@
+:8884
+
+reverse_proxy 127.0.0.1:65535 {
+	trusted_proxies 127.0.0.1
+}
+
+reverse_proxy 127.0.0.1:65535 {
+	trusted_proxies private_ranges
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "reverse_proxy",
+									"trusted_proxies": [
+										"127.0.0.1"
+									],
+									"upstreams": [
+										{
+											"dial": "127.0.0.1:65535"
+										}
+									]
+								},
+								{
+									"handler": "reverse_proxy",
+									"trusted_proxies": [
+										"192.168.0.0/16",
+										"172.16.0.0/12",
+										"10.0.0.0/8",
+										"127.0.0.1/8",
+										"fd00::/8",
+										"::1"
+									],
+									"upstreams": [
+										{
+											"dial": "127.0.0.1:65535"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/sort_directives_within_handle.txt

@@ -0,0 +1,133 @@
+*.example.com {
+	@foo host foo.example.com
+	handle @foo {
+		handle_path /strip* {
+			respond "this should be first"
+		}
+		handle {
+			respond "this should be second"
+		}
+	}
+	handle {
+		respond "this should be last"
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"*.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"group": "group5",
+											"handle": [
+												{
+													"handler": "subroute",
+													"routes": [
+														{
+															"group": "group2",
+															"handle": [
+																{
+																	"handler": "subroute",
+																	"routes": [
+																		{
+																			"handle": [
+																				{
+																					"handler": "rewrite",
+																					"strip_path_prefix": "/strip"
+																				}
+																			]
+																		},
+																		{
+																			"handle": [
+																				{
+																					"body": "this should be first",
+																					"handler": "static_response"
+																				}
+																			]
+																		}
+																	]
+																}
+															],
+															"match": [
+																{
+																	"path": [
+																		"/strip*"
+																	]
+																}
+															]
+														},
+														{
+															"group": "group2",
+															"handle": [
+																{
+																	"handler": "subroute",
+																	"routes": [
+																		{
+																			"handle": [
+																				{
+																					"body": "this should be second",
+																					"handler": "static_response"
+																				}
+																			]
+																		}
+																	]
+																}
+															]
+														}
+													]
+												}
+											],
+											"match": [
+												{
+													"host": [
+														"foo.example.com"
+													]
+												}
+											]
+										},
+										{
+											"group": "group5",
+											"handle": [
+												{
+													"handler": "subroute",
+													"routes": [
+														{
+															"handle": [
+																{
+																	"body": "this should be last",
+																	"handler": "static_response"
+																}
+															]
+														}
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/sort_vars_in_reverse.txt

@@ -0,0 +1,59 @@
+:80
+
+vars /foobar foo last
+vars /foo foo middle
+vars * foo first
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"foo": "first",
+									"handler": "vars"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"path": [
+										"/foo"
+									]
+								}
+							],
+							"handle": [
+								{
+									"foo": "middle",
+									"handler": "vars"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"path": [
+										"/foobar"
+									]
+								}
+							],
+							"handle": [
+								{
+									"foo": "last",
+									"handler": "vars"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_acme_preferred_chains.txt

@@ -0,0 +1,57 @@
+localhost
+
+tls {
+	issuer acme {
+		preferred_chains {
+			any_common_name "Generic CA 1" "Generic CA 2"
+		}
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"localhost"
+						],
+						"issuers": [
+							{
+								"module": "acme",
+								"preferred_chains": {
+									"any_common_name": [
+										"Generic CA 1",
+										"Generic CA 2"
+									]
+								}
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_automation_policies_7.txt

@@ -0,0 +1,68 @@
+# (this Caddyfile is contrived, but based on issues #4176 and #4198)
+
+http://example.com {
+}
+
+https://example.com {
+	tls internal
+}
+
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				},
+				"srv1": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"example.com"
+						],
+						"issuers": [
+							{
+								"module": "internal"
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_automation_policies_8.txt

@@ -0,0 +1,98 @@
+# (this Caddyfile is contrived, but based on issues #4176 and #4198)
+
+http://example.com {
+}
+
+https://example.com {
+	tls abc@example.com
+}
+
+http://localhost:8081 {
+}
+
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				},
+				"srv1": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				},
+				"srv2": {
+					"listen": [
+						":8081"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"automatic_https": {
+						"skip": [
+							"localhost"
+						]
+					}
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"example.com"
+						],
+						"issuers": [
+							{
+								"email": "abc@example.com",
+								"module": "acme"
+							},
+							{
+								"email": "abc@example.com",
+								"module": "zerossl"
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_automation_policies_9.txt

@@ -0,0 +1,56 @@
+# example from issue #4640
+http://foo:8447, http://127.0.0.1:8447 {
+	reverse_proxy 127.0.0.1:8080
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8447"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"foo",
+										"127.0.0.1"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "reverse_proxy",
+													"upstreams": [
+														{
+															"dial": "127.0.0.1:8080"
+														}
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"automatic_https": {
+						"skip": [
+							"foo",
+							"127.0.0.1"
+						]
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_automation_policies_global_email_localhost.txt

@@ -0,0 +1,66 @@
+{
+	email foo@bar
+}
+
+localhost {
+}
+
+example.com {
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"example.com"
+						],
+						"issuers": [
+							{
+								"email": "foo@bar",
+								"module": "acme"
+							},
+							{
+								"email": "foo@bar",
+								"module": "zerossl"
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_internal_options.txt

@@ -0,0 +1,54 @@
+a.example.com {
+	tls {
+		issuer internal {
+			ca foo
+			lifetime 24h
+			sign_with_root
+		}
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"a.example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"a.example.com"
+						],
+						"issuers": [
+							{
+								"ca": "foo",
+								"lifetime": 86400000000000,
+								"module": "internal",
+								"sign_with_root": true
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_propagation_timeout.txt

@@ -0,0 +1,85 @@
+localhost
+
+respond "hello from localhost"
+tls {
+	issuer acme {
+		propagation_delay 5m10s
+		propagation_timeout 10m20s
+	}
+	issuer zerossl {
+		propagation_delay 5m30s
+		propagation_timeout -1
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"body": "hello from localhost",
+													"handler": "static_response"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"localhost"
+						],
+						"issuers": [
+							{
+								"challenges": {
+									"dns": {
+										"propagation_delay": 310000000000,
+										"propagation_timeout": 620000000000
+									}
+								},
+								"module": "acme"
+							},
+							{
+								"challenges": {
+									"dns": {
+										"propagation_delay": 330000000000,
+										"propagation_timeout": -1
+									}
+								},
+								"module": "zerossl"
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tracing.txt

@@ -0,0 +1,36 @@
+:80 {
+	tracing /myhandler {
+		span my-span
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"path": [
+										"/myhandler"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "tracing",
+									"span": "my-span"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt_test.go

@@ -3,7 +3,7 @@ package integration
 import (
 	jsonMod "encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -14,7 +14,7 @@ import (
 
 func TestCaddyfileAdaptToJSON(t *testing.T) {
 	// load the list of test files from the dir
-	files, err := ioutil.ReadDir("./caddyfile_adapt")
+	files, err := os.ReadDir("./caddyfile_adapt")
 	if err != nil {
 		t.Errorf("failed to read caddyfile_adapt dir: %s", err)
 	}
@@ -29,7 +29,7 @@ func TestCaddyfileAdaptToJSON(t *testing.T) {
 
 		// read the test file
 		filename := f.Name()
-		data, err := ioutil.ReadFile("./caddyfile_adapt/" + filename)
+		data, err := os.ReadFile("./caddyfile_adapt/" + filename)
 		if err != nil {
 			t.Errorf("failed to read %s dir: %s", filename, err)
 		}

caddytest/integration/caddyfile_test.go

@@ -68,7 +68,7 @@ func TestDuplicateHosts(t *testing.T) {
     }
     `,
 		"caddyfile",
-		"duplicate site address not allowed")
+		"ambiguous site definition")
 }
 
 func TestReadCookie(t *testing.T) {
@@ -101,3 +101,27 @@ func TestReadCookie(t *testing.T) {
 	// act and assert
 	tester.AssertGetResponse("http://localhost:9080/cookie.html", 200, "<h2>Cookie.ClientName caddytest</h2>")
 }
+
+func TestReplIndex(t *testing.T) {
+
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+  {
+    http_port     9080
+    https_port    9443
+  }
+
+  localhost:9080 {
+    templates {
+      root testdata
+    }
+    file_server {
+      root testdata
+      index "index.{host}.html"
+    }
+  }
+  `, "caddyfile")
+
+	// act and assert
+	tester.AssertGetResponse("http://localhost:9080/", 200, "")
+}

caddytest/integration/map_test.go

@@ -60,7 +60,7 @@ func TestMapRespondWithDefault(t *testing.T) {
 	tester.AssertPostResponseBody("http://localhost:9080/version", []string{}, bytes.NewBuffer([]byte{}), 200, "hello from localhost unknown")
 }
 
-func TestMapAsJson(t *testing.T) {
+func TestMapAsJSON(t *testing.T) {
 	// arrange
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`
@@ -85,7 +85,7 @@ func TestMapAsJson(t *testing.T) {
 													{
 														"handler": "map",
 														"source": "{http.request.method}",
-														"destinations": ["dest-name"],
+														"destinations": ["{dest-name}"],
 														"defaults": ["unknown"],
 														"mappings": [
 															{

caddytest/integration/reverseproxy_test.go

@@ -2,7 +2,6 @@ package integration
 
 import (
 	"fmt"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
@@ -85,7 +84,7 @@ func TestDialWithPlaceholderUnix(t *testing.T) {
 		t.SkipNow()
 	}
 
-	f, err := ioutil.TempFile("", "*.sock")
+	f, err := os.CreateTemp("", "*.sock")
 	if err != nil {
 		t.Errorf("failed to create TempFile: %s", err)
 		return
@@ -371,7 +370,7 @@ func TestReverseProxyHealthCheck(t *testing.T) {
 		reverse_proxy {
 			to localhost:2020
 	
-			health_path /health
+			health_uri /health
 			health_port 2021
 			health_interval 2s
 			health_timeout 5s
@@ -387,7 +386,7 @@ func TestReverseProxyHealthCheckUnixSocket(t *testing.T) {
 		t.SkipNow()
 	}
 	tester := caddytest.NewTester(t)
-	f, err := ioutil.TempFile("", "*.sock")
+	f, err := os.CreateTemp("", "*.sock")
 	if err != nil {
 		t.Errorf("failed to create TempFile: %s", err)
 		return
@@ -426,7 +425,7 @@ func TestReverseProxyHealthCheckUnixSocket(t *testing.T) {
 		reverse_proxy {
 			to unix/%s
 	
-			health_path /health
+			health_uri /health
 			health_port 2021
 			health_interval 2s
 			health_timeout 5s
@@ -436,3 +435,57 @@ func TestReverseProxyHealthCheckUnixSocket(t *testing.T) {
 
 	tester.AssertGetResponse("http://localhost:9080/", 200, "Hello, World!")
 }
+
+func TestReverseProxyHealthCheckUnixSocketWithoutPort(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.SkipNow()
+	}
+	tester := caddytest.NewTester(t)
+	f, err := os.CreateTemp("", "*.sock")
+	if err != nil {
+		t.Errorf("failed to create TempFile: %s", err)
+		return
+	}
+	// a hack to get a file name within a valid path to use as socket
+	socketName := f.Name()
+	os.Remove(f.Name())
+
+	server := http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			if strings.HasPrefix(req.URL.Path, "/health") {
+				w.Write([]byte("ok"))
+				return
+			}
+			w.Write([]byte("Hello, World!"))
+		}),
+	}
+
+	unixListener, err := net.Listen("unix", socketName)
+	if err != nil {
+		t.Errorf("failed to listen on the socket: %s", err)
+		return
+	}
+	go server.Serve(unixListener)
+	t.Cleanup(func() {
+		server.Close()
+	})
+	runtime.Gosched() // Allow other goroutines to run
+
+	tester.InitServer(fmt.Sprintf(`
+	{
+		http_port     9080
+		https_port    9443
+	}
+	http://localhost:9080 {
+		reverse_proxy {
+			to unix/%s
+	
+			health_uri /health
+			health_interval 2s
+			health_timeout 5s
+		}
+	}
+	`, socketName), "caddyfile")
+
+	tester.AssertGetResponse("http://localhost:9080/", 200, "Hello, World!")
+}

caddytest/integration/stream_test.go

@@ -6,7 +6,6 @@ import (
 	"crypto/rand"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -110,7 +109,7 @@ func TestH2ToH2CStream(t *testing.T) {
 	r, w := io.Pipe()
 	req := &http.Request{
 		Method: "PUT",
-		Body:   ioutil.NopCloser(r),
+		Body:   io.NopCloser(r),
 		URL: &url.URL{
 			Scheme: "https",
 			Host:   "127.0.0.1:9443",
@@ -124,8 +123,8 @@ func TestH2ToH2CStream(t *testing.T) {
 	// Disable any compression method from server.
 	req.Header.Set("Accept-Encoding", "identity")
 
-	resp := tester.AssertResponseCode(req, 200)
-	if 200 != resp.StatusCode {
+	resp := tester.AssertResponseCode(req, http.StatusOK)
+	if resp.StatusCode != http.StatusOK {
 		return
 	}
 	go func() {
@@ -134,7 +133,7 @@ func TestH2ToH2CStream(t *testing.T) {
 	}()
 
 	defer resp.Body.Close()
-	bytes, err := ioutil.ReadAll(resp.Body)
+	bytes, err := io.ReadAll(resp.Body)
 	if err != nil {
 		t.Fatalf("unable to read the response body %s", err)
 	}
@@ -144,7 +143,6 @@ func TestH2ToH2CStream(t *testing.T) {
 	if !strings.Contains(body, expectedBody) {
 		t.Errorf("requesting \"%s\" expected response body \"%s\" but got \"%s\"", req.RequestURI, expectedBody, body)
 	}
-	return
 }
 
 func testH2ToH2CStreamServeH2C(t *testing.T) *http.Server {
@@ -319,7 +317,7 @@ func TestH2ToH1ChunkedResponse(t *testing.T) {
 	r, w := io.Pipe()
 	req := &http.Request{
 		Method: "PUT",
-		Body:   ioutil.NopCloser(r),
+		Body:   io.NopCloser(r),
 		URL: &url.URL{
 			Scheme: "https",
 			Host:   "127.0.0.1:9443",
@@ -336,13 +334,13 @@ func TestH2ToH1ChunkedResponse(t *testing.T) {
 		fmt.Fprint(w, expectedBody)
 		w.Close()
 	}()
-	resp := tester.AssertResponseCode(req, 200)
-	if 200 != resp.StatusCode {
+	resp := tester.AssertResponseCode(req, http.StatusOK)
+	if resp.StatusCode != http.StatusOK {
 		return
 	}
 
 	defer resp.Body.Close()
-	bytes, err := ioutil.ReadAll(resp.Body)
+	bytes, err := io.ReadAll(resp.Body)
 	if err != nil {
 		t.Fatalf("unable to read the response body %s", err)
 	}
@@ -352,7 +350,6 @@ func TestH2ToH1ChunkedResponse(t *testing.T) {
 	if body != expectedBody {
 		t.Errorf("requesting \"%s\" expected response body \"%s\" but got \"%s\"", req.RequestURI, expectedBody, body)
 	}
-	return
 }
 
 func testH2ToH1ChunkedResponseServeH1(t *testing.T) *http.Server {
@@ -370,7 +367,7 @@ func testH2ToH1ChunkedResponseServeH1(t *testing.T) *http.Server {
 		}
 
 		defer r.Body.Close()
-		bytes, err := ioutil.ReadAll(r.Body)
+		bytes, err := io.ReadAll(r.Body)
 		if err != nil {
 			t.Fatalf("unable to read the response body %s", err)
 		}

cmd/caddy/main.go

@@ -24,6 +24,8 @@
 //   3. Run `go mod init caddy`
 //   4. Run `go install` or `go build` - you now have a custom binary!
 //
+// Or you can use xcaddy which does it all for you as a command:
+// https://github.com/caddyserver/xcaddy
 package main
 
 import (

cmd/cobra.go

@@ -0,0 +1,120 @@
+package caddycmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var rootCmd = &cobra.Command{
+	Use: "caddy",
+	Long: `Caddy is an extensible server platform written in Go.
+
+At its core, Caddy merely manages configuration. Modules are plugged
+in statically at compile-time to provide useful functionality. Caddy's
+standard distribution includes common modules to serve HTTP, TLS,
+and PKI applications, including the automation of certificates.
+
+To run Caddy, use:
+
+	- 'caddy run' to run Caddy in the foreground (recommended).
+	- 'caddy start' to start Caddy in the background; only do this
+	  if you will be keeping the terminal window open until you run
+	  'caddy stop' to close the server.
+
+When Caddy is started, it opens a locally-bound administrative socket
+to which configuration can be POSTed via a restful HTTP API (see
+https://caddyserver.com/docs/api).
+
+Caddy's native configuration format is JSON. However, config adapters
+can be used to convert other config formats to JSON when Caddy receives
+its configuration. The Caddyfile is a built-in config adapter that is
+popular for hand-written configurations due to its straightforward
+syntax (see https://caddyserver.com/docs/caddyfile). Many third-party
+adapters are available (see https://caddyserver.com/docs/config-adapters).
+Use 'caddy adapt' to see how a config translates to JSON.
+
+For convenience, the CLI can act as an HTTP client to give Caddy its
+initial configuration for you. If a file named Caddyfile is in the
+current working directory, it will do this automatically. Otherwise,
+you can use the --config flag to specify the path to a config file.
+
+Some special-purpose subcommands build and load a configuration file
+for you directly from command line input; for example:
+
+	- caddy file-server
+	- caddy reverse-proxy
+	- caddy respond
+
+These commands disable the administration endpoint because their
+configuration is specified solely on the command line.
+
+In general, the most common way to run Caddy is simply:
+
+	$ caddy run
+
+Or, with a configuration file:
+
+	$ caddy run --config caddy.json
+
+If running interactively in a terminal, running Caddy in the
+background may be more convenient:
+
+	$ caddy start
+	...
+	$ caddy stop
+
+This allows you to run other commands while Caddy stays running.
+Be sure to stop Caddy before you close the terminal!
+
+Depending on the system, Caddy may need permission to bind to low
+ports. One way to do this on Linux is to use setcap:
+
+	$ sudo setcap cap_net_bind_service=+ep $(which caddy)
+
+Remember to run that command again after replacing the binary.
+
+See the Caddy website for tutorials, configuration structure,
+syntax, and module documentation: https://caddyserver.com/docs/
+
+Custom Caddy builds are available on the Caddy download page at:
+https://caddyserver.com/download
+
+The xcaddy command can be used to build Caddy from source with or
+without additional plugins: https://github.com/caddyserver/xcaddy
+
+Where possible, Caddy should be installed using officially-supported
+package installers: https://caddyserver.com/docs/install
+
+Instructions for running Caddy in production are also available:
+https://caddyserver.com/docs/running
+`,
+	Example: `  $ caddy run
+  $ caddy run --config caddy.json
+  $ caddy reload --config caddy.json
+  $ caddy stop`,
+
+	// kind of annoying to have all the help text printed out if
+	// caddy has an error provisioning its modules, for instance...
+	SilenceUsage: true,
+}
+
+const fullDocsFooter = `Full documentation is available at:
+https://caddyserver.com/docs/command-line`
+
+func init() {
+	rootCmd.SetHelpTemplate(rootCmd.HelpTemplate() + "\n" + fullDocsFooter)
+}
+
+func caddyCmdToCoral(caddyCmd Command) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   caddyCmd.Name,
+		Short: caddyCmd.Short,
+		Long:  caddyCmd.Long,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			fls := cmd.Flags()
+			_, err := caddyCmd.Func(Flags{fls})
+			return err
+		},
+	}
+	cmd.Flags().AddGoFlagSet(caddyCmd.Flags)
+	return cmd
+}

cmd/commandfuncs.go

@@ -19,21 +19,19 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"os/exec"
-	"reflect"
 	"runtime"
 	"runtime/debug"
-	"sort"
 	"strings"
 
+	"github.com/aryann/difflib"
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
@@ -121,7 +119,7 @@ func cmdStart(fl Flags) (int, error) {
 		for {
 			conn, err := ln.Accept()
 			if err != nil {
-				if !strings.Contains(err.Error(), "use of closed network connection") {
+				if !errors.Is(err, net.ErrClosed) {
 					log.Println(err)
 				}
 				break
@@ -182,7 +180,7 @@ func cmdRun(fl Flags) (int, error) {
 	var config []byte
 	var err error
 	if runCmdResumeFlag {
-		config, err = ioutil.ReadFile(caddy.ConfigAutosavePath)
+		config, err = os.ReadFile(caddy.ConfigAutosavePath)
 		if os.IsNotExist(err) {
 			// not a bad error; just can't resume if autosave file doesn't exist
 			caddy.Log().Info("no autosave file exists", zap.String("autosave_file", caddy.ConfigAutosavePath))
@@ -204,7 +202,7 @@ func cmdRun(fl Flags) (int, error) {
 	// we don't use 'else' here since this value might have been changed in 'if' block; i.e. not mutually exclusive
 	var configFile string
 	if !runCmdResumeFlag {
-		config, configFile, err = loadConfig(runCmdConfigFlag, runCmdConfigAdapterFlag)
+		config, configFile, err = LoadConfig(runCmdConfigFlag, runCmdConfigAdapterFlag)
 		if err != nil {
 			return caddy.ExitCodeFailedStartup, err
 		}
@@ -220,7 +218,7 @@ func cmdRun(fl Flags) (int, error) {
 	// if we are to report to another process the successful start
 	// of the server, do so now by echoing back contents of stdin
 	if runCmdPingbackFlag != "" {
-		confirmationBytes, err := ioutil.ReadAll(os.Stdin)
+		confirmationBytes, err := io.ReadAll(os.Stdin)
 		if err != nil {
 			return caddy.ExitCodeFailedStartup,
 				fmt.Errorf("reading confirmation bytes from stdin: %v", err)
@@ -277,25 +275,33 @@ func cmdRun(fl Flags) (int, error) {
 }
 
 func cmdStop(fl Flags) (int, error) {
-	stopCmdAddrFlag := fl.String("address")
+	addrFlag := fl.String("address")
+	configFlag := fl.String("config")
+	configAdapterFlag := fl.String("adapter")
 
-	err := apiRequest(stopCmdAddrFlag, http.MethodPost, "/stop", nil, nil)
+	adminAddr, err := DetermineAdminAPIAddress(addrFlag, nil, configFlag, configAdapterFlag)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("couldn't determine admin API address: %v", err)
+	}
+
+	resp, err := AdminAPIRequest(adminAddr, http.MethodPost, "/stop", nil, nil)
 	if err != nil {
 		caddy.Log().Warn("failed using API to stop instance", zap.Error(err))
 		return caddy.ExitCodeFailedStartup, err
 	}
+	defer resp.Body.Close()
 
 	return caddy.ExitCodeSuccess, nil
 }
 
 func cmdReload(fl Flags) (int, error) {
-	reloadCmdConfigFlag := fl.String("config")
-	reloadCmdConfigAdapterFlag := fl.String("adapter")
-	reloadCmdAddrFlag := fl.String("address")
-	reloadCmdForceFlag := fl.Bool("force")
+	configFlag := fl.String("config")
+	configAdapterFlag := fl.String("adapter")
+	addrFlag := fl.String("address")
+	forceFlag := fl.Bool("force")
 
 	// get the config in caddy's native format
-	config, configFile, err := loadConfig(reloadCmdConfigFlag, reloadCmdConfigAdapterFlag)
+	config, configFile, err := LoadConfig(configFlag, configAdapterFlag)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, err
 	}
@@ -303,65 +309,45 @@ func cmdReload(fl Flags) (int, error) {
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("no config file to load")
 	}
 
-	// get the address of the admin listener; use flag if specified
-	adminAddr := reloadCmdAddrFlag
-	if adminAddr == "" && len(config) > 0 {
-		var tmpStruct struct {
-			Admin caddy.AdminConfig `json:"admin"`
-		}
-		err = json.Unmarshal(config, &tmpStruct)
-		if err != nil {
-			return caddy.ExitCodeFailedStartup,
-				fmt.Errorf("unmarshaling admin listener address from config: %v", err)
-		}
-		adminAddr = tmpStruct.Admin.Listen
+	adminAddr, err := DetermineAdminAPIAddress(addrFlag, config, configFlag, configAdapterFlag)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("couldn't determine admin API address: %v", err)
 	}
 
 	// optionally force a config reload
 	headers := make(http.Header)
-	if reloadCmdForceFlag {
+	if forceFlag {
 		headers.Set("Cache-Control", "must-revalidate")
 	}
 
-	err = apiRequest(adminAddr, http.MethodPost, "/load", headers, bytes.NewReader(config))
+	resp, err := AdminAPIRequest(adminAddr, http.MethodPost, "/load", headers, bytes.NewReader(config))
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("sending configuration to instance: %v", err)
 	}
+	defer resp.Body.Close()
 
 	return caddy.ExitCodeSuccess, nil
 }
 
 func cmdVersion(_ Flags) (int, error) {
-	fmt.Println(caddyVersion())
+	_, full := caddy.Version()
+	fmt.Println(full)
 	return caddy.ExitCodeSuccess, nil
 }
 
-func cmdBuildInfo(fl Flags) (int, error) {
+func cmdBuildInfo(_ Flags) (int, error) {
 	bi, ok := debug.ReadBuildInfo()
 	if !ok {
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("no build information")
 	}
-
-	fmt.Printf("go_version: %s\n", runtime.Version())
-	fmt.Printf("go_os:      %s\n", runtime.GOOS)
-	fmt.Printf("go_arch:    %s\n", runtime.GOARCH)
-	fmt.Printf("path:       %s\n", bi.Path)
-	fmt.Printf("main:       %s %s %s\n", bi.Main.Path, bi.Main.Version, bi.Main.Sum)
-	fmt.Println("dependencies:")
-
-	for _, goMod := range bi.Deps {
-		fmt.Printf("%s %s %s", goMod.Path, goMod.Version, goMod.Sum)
-		if goMod.Replace != nil {
-			fmt.Printf(" => %s %s %s", goMod.Replace.Path, goMod.Replace.Version, goMod.Replace.Sum)
-		}
-		fmt.Println()
-	}
+	fmt.Println(bi)
 	return caddy.ExitCodeSuccess, nil
 }
 
 func cmdListModules(fl Flags) (int, error) {
 	packages := fl.Bool("packages")
 	versions := fl.Bool("versions")
+	skipStandard := fl.Bool("skip-standard")
 
 	printModuleInfo := func(mi moduleInfo) {
 		fmt.Print(mi.caddyModuleID)
@@ -390,14 +376,19 @@ func cmdListModules(fl Flags) (int, error) {
 		return caddy.ExitCodeSuccess, nil
 	}
 
-	if len(standard) > 0 {
-		for _, mod := range standard {
-			printModuleInfo(mod)
-		}
-	}
-	fmt.Printf("\n  Standard modules: %d\n", len(standard))
-	if len(nonstandard) > 0 {
+	// Standard modules (always shipped with Caddy)
+	if !skipStandard {
 		if len(standard) > 0 {
+			for _, mod := range standard {
+				printModuleInfo(mod)
+			}
+		}
+		fmt.Printf("\n  Standard modules: %d\n", len(standard))
+	}
+
+	// Non-standard modules (third party plugins)
+	if len(nonstandard) > 0 {
+		if len(standard) > 0 && !skipStandard {
 			fmt.Println()
 		}
 		for _, mod := range nonstandard {
@@ -405,8 +396,10 @@ func cmdListModules(fl Flags) (int, error) {
 		}
 	}
 	fmt.Printf("\n  Non-standard modules: %d\n", len(nonstandard))
+
+	// Unknown modules (couldn't get Caddy module info)
 	if len(unknown) > 0 {
-		if len(standard) > 0 || len(nonstandard) > 0 {
+		if (len(standard) > 0 && !skipStandard) || len(nonstandard) > 0 {
 			fmt.Println()
 		}
 		for _, mod := range unknown {
@@ -458,13 +451,13 @@ func cmdAdaptConfig(fl Flags) (int, error) {
 			fmt.Errorf("unrecognized config adapter: %s", adaptCmdAdapterFlag)
 	}
 
-	input, err := ioutil.ReadFile(adaptCmdInputFlag)
+	input, err := os.ReadFile(adaptCmdInputFlag)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup,
 			fmt.Errorf("reading input file: %v", err)
 	}
 
-	opts := map[string]interface{}{"filename": adaptCmdInputFlag}
+	opts := map[string]any{"filename": adaptCmdInputFlag}
 
 	adaptedConfig, warnings, err := cfgAdapter.Adapt(input, opts)
 	if err != nil {
@@ -489,7 +482,9 @@ func cmdAdaptConfig(fl Flags) (int, error) {
 		if warn.Directive != "" {
 			msg = fmt.Sprintf("%s: %s", warn.Directive, warn.Message)
 		}
-		fmt.Fprintf(os.Stderr, "[WARNING][%s] %s:%d: %s\n", adaptCmdAdapterFlag, warn.File, warn.Line, msg)
+		caddy.Log().Named(adaptCmdAdapterFlag).Warn(msg,
+			zap.String("file", warn.File),
+			zap.Int("line", warn.Line))
 	}
 
 	// validate output if requested
@@ -512,7 +507,7 @@ func cmdValidateConfig(fl Flags) (int, error) {
 	validateCmdConfigFlag := fl.String("config")
 	validateCmdAdapterFlag := fl.String("adapter")
 
-	input, _, err := loadConfig(validateCmdConfigFlag, validateCmdAdapterFlag)
+	input, _, err := LoadConfig(validateCmdConfigFlag, validateCmdAdapterFlag)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, err
 	}
@@ -542,7 +537,7 @@ func cmdFmt(fl Flags) (int, error) {
 
 	// as a special case, read from stdin if the file name is "-"
 	if formatCmdConfigFile == "-" {
-		input, err := ioutil.ReadAll(os.Stdin)
+		input, err := io.ReadAll(os.Stdin)
 		if err != nil {
 			return caddy.ExitCodeFailedStartup,
 				fmt.Errorf("reading stdin: %v", err)
@@ -551,7 +546,7 @@ func cmdFmt(fl Flags) (int, error) {
 		return caddy.ExitCodeSuccess, nil
 	}
 
-	input, err := ioutil.ReadFile(formatCmdConfigFile)
+	input, err := os.ReadFile(formatCmdConfigFile)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup,
 			fmt.Errorf("reading input file: %v", err)
@@ -560,8 +555,22 @@ func cmdFmt(fl Flags) (int, error) {
 	output := caddyfile.Format(input)
 
 	if fl.Bool("overwrite") {
-		if err := ioutil.WriteFile(formatCmdConfigFile, output, 0600); err != nil {
-			return caddy.ExitCodeFailedStartup, nil
+		if err := os.WriteFile(formatCmdConfigFile, output, 0600); err != nil {
+			return caddy.ExitCodeFailedStartup, fmt.Errorf("overwriting formatted file: %v", err)
+		}
+	} else if fl.Bool("diff") {
+		diff := difflib.Diff(
+			strings.Split(string(input), "\n"),
+			strings.Split(string(output), "\n"))
+		for _, d := range diff {
+			switch d.Delta {
+			case difflib.Common:
+				fmt.Printf("  %s\n", d.Payload)
+			case difflib.LeftOnly:
+				fmt.Printf("- %s\n", d.Payload)
+			case difflib.RightOnly:
+				fmt.Printf("+ %s\n", d.Payload)
+			}
 		}
 	} else {
 		fmt.Print(string(output))
@@ -570,286 +579,25 @@ func cmdFmt(fl Flags) (int, error) {
 	return caddy.ExitCodeSuccess, nil
 }
 
-func cmdUpgrade(_ Flags) (int, error) {
-	l := caddy.Log()
-
-	thisExecPath, err := os.Executable()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("determining current executable path: %v", err)
-	}
-	thisExecStat, err := os.Stat(thisExecPath)
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("retrieving current executable permission bits: %v", err)
-	}
-	l.Info("this executable will be replaced", zap.String("path", thisExecPath))
-
-	// get the list of nonstandard plugins
-	_, nonstandard, _, err := getModules()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err)
-	}
-	pluginPkgs := make(map[string]struct{})
-	for _, mod := range nonstandard {
-		if mod.goModule.Replace != nil {
-			return caddy.ExitCodeFailedStartup, fmt.Errorf("cannot auto-upgrade when Go module has been replaced: %s => %s",
-				mod.goModule.Path, mod.goModule.Replace.Path)
-		}
-		l.Info("found non-standard module",
-			zap.String("id", mod.caddyModuleID),
-			zap.String("package", mod.goModule.Path))
-		pluginPkgs[mod.goModule.Path] = struct{}{}
-	}
-
-	// build the request URL to download this custom build
-	qs := url.Values{
-		"os":   {runtime.GOOS},
-		"arch": {runtime.GOARCH},
-	}
-	for pkg := range pluginPkgs {
-		qs.Add("p", pkg)
-	}
-	urlStr := fmt.Sprintf("https://caddyserver.com/api/download?%s", qs.Encode())
-
-	// initiate the build
-	l.Info("requesting build",
-		zap.String("os", qs.Get("os")),
-		zap.String("arch", qs.Get("arch")),
-		zap.Strings("packages", qs["p"]))
-	resp, err := http.Get(urlStr)
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("secure request failed: %v", err)
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		var details struct {
-			StatusCode int `json:"status_code"`
-			Error      struct {
-				Message string `json:"message"`
-				ID      string `json:"id"`
-			} `json:"error"`
-		}
-		err2 := json.NewDecoder(resp.Body).Decode(&details)
-		if err2 != nil {
-			return caddy.ExitCodeFailedStartup, fmt.Errorf("download and error decoding failed: HTTP %d: %v", resp.StatusCode, err2)
-		}
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download failed: HTTP %d: %s (id=%s)", resp.StatusCode, details.Error.Message, details.Error.ID)
-	}
-
-	// back up the current binary, in case something goes wrong we can replace it
-	backupExecPath := thisExecPath + ".tmp"
-	l.Info("build acquired; backing up current executable",
-		zap.String("current_path", thisExecPath),
-		zap.String("backup_path", backupExecPath))
-	err = os.Rename(thisExecPath, backupExecPath)
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("backing up current binary: %v", err)
-	}
-	defer func() {
-		if err != nil {
-			err2 := os.Rename(backupExecPath, thisExecPath)
-			if err2 != nil {
-				l.Error("restoring original executable failed; will need to be restored manually",
-					zap.String("backup_path", backupExecPath),
-					zap.String("original_path", thisExecPath),
-					zap.Error(err2))
-			}
-		}
-	}()
-
-	// download the file; do this in a closure to close reliably before we execute it
-	writeFile := func() error {
-		destFile, err := os.OpenFile(thisExecPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, thisExecStat.Mode())
-		if err != nil {
-			return fmt.Errorf("unable to open destination file: %v", err)
-		}
-		defer destFile.Close()
-
-		l.Info("downloading binary", zap.String("source", urlStr), zap.String("destination", thisExecPath))
-
-		_, err = io.Copy(destFile, resp.Body)
-		if err != nil {
-			return fmt.Errorf("unable to download file: %v", err)
-		}
-
-		err = destFile.Sync()
-		if err != nil {
-			return fmt.Errorf("syncing downloaded file to device: %v", err)
-		}
-
-		return nil
-	}
-	err = writeFile()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, err
-	}
-
-	l.Info("download successful; displaying new binary details", zap.String("location", thisExecPath))
-
-	// use the new binary to print out version and module info
-	fmt.Print("\nModule versions:\n\n")
-	cmd := exec.Command(thisExecPath, "list-modules", "--versions")
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	err = cmd.Run()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err)
-	}
-	fmt.Println("\nVersion:")
-	cmd = exec.Command(thisExecPath, "version")
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	err = cmd.Run()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err)
-	}
-	fmt.Println()
-
-	// clean up the backup file
-	err = os.Remove(backupExecPath)
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to clean up backup binary: %v", err)
-	}
-
-	l.Info("upgrade successful; please restart any running Caddy instances", zap.String("executable", thisExecPath))
-
-	return caddy.ExitCodeSuccess, nil
-}
-
-func cmdHelp(fl Flags) (int, error) {
-	const fullDocs = `Full documentation is available at:
-https://caddyserver.com/docs/command-line`
-
-	args := fl.Args()
-	if len(args) == 0 {
-		s := `Caddy is an extensible server platform.
-
-usage:
-  caddy <command> [<args...>]
-
-commands:
-`
-		keys := make([]string, 0, len(commands))
-		for k := range commands {
-			keys = append(keys, k)
-		}
-		sort.Strings(keys)
-		for _, k := range keys {
-			cmd := commands[k]
-			short := strings.TrimSuffix(cmd.Short, ".")
-			s += fmt.Sprintf("  %-15s %s\n", cmd.Name, short)
-		}
-
-		s += "\nUse 'caddy help <command>' for more information about a command.\n"
-		s += "\n" + fullDocs + "\n"
-
-		fmt.Print(s)
-
-		return caddy.ExitCodeSuccess, nil
-	} else if len(args) > 1 {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("can only give help with one command")
-	}
-
-	subcommand, ok := commands[args[0]]
-	if !ok {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("unknown command: %s", args[0])
-	}
-
-	helpText := strings.TrimSpace(subcommand.Long)
-	if helpText == "" {
-		helpText = subcommand.Short
-		if !strings.HasSuffix(helpText, ".") {
-			helpText += "."
-		}
-	}
-
-	result := fmt.Sprintf("%s\n\nusage:\n  caddy %s %s\n",
-		helpText,
-		subcommand.Name,
-		strings.TrimSpace(subcommand.Usage),
-	)
-
-	if help := flagHelp(subcommand.Flags); help != "" {
-		result += fmt.Sprintf("\nflags:\n%s", help)
-	}
-
-	result += "\n" + fullDocs + "\n"
-
-	fmt.Print(result)
-
-	return caddy.ExitCodeSuccess, nil
-}
-
-func getModules() (standard, nonstandard, unknown []moduleInfo, err error) {
-	bi, ok := debug.ReadBuildInfo()
-	if !ok {
-		err = fmt.Errorf("no build info")
-		return
-	}
-
-	for _, modID := range caddy.Modules() {
-		modInfo, err := caddy.GetModule(modID)
-		if err != nil {
-			// that's weird, shouldn't happen
-			unknown = append(unknown, moduleInfo{caddyModuleID: modID, err: err})
-			continue
-		}
-
-		// to get the Caddy plugin's version info, we need to know
-		// the package that the Caddy module's value comes from; we
-		// can use reflection but we need a non-pointer value (I'm
-		// not sure why), and since New() should return a pointer
-		// value, we need to dereference it first
-		iface := interface{}(modInfo.New())
-		if rv := reflect.ValueOf(iface); rv.Kind() == reflect.Ptr {
-			iface = reflect.New(reflect.TypeOf(iface).Elem()).Elem().Interface()
-		}
-		modPkgPath := reflect.TypeOf(iface).PkgPath()
-
-		// now we find the Go module that the Caddy module's package
-		// belongs to; we assume the Caddy module package path will
-		// be prefixed by its Go module path, and we will choose the
-		// longest matching prefix in case there are nested modules
-		var matched *debug.Module
-		for _, dep := range bi.Deps {
-			if strings.HasPrefix(modPkgPath, dep.Path) {
-				if matched == nil || len(dep.Path) > len(matched.Path) {
-					matched = dep
-				}
-			}
-		}
-
-		caddyModGoMod := moduleInfo{caddyModuleID: modID, goModule: matched}
-
-		if strings.HasPrefix(modPkgPath, caddy.ImportPath) {
-			standard = append(standard, caddyModGoMod)
-		} else {
-			nonstandard = append(nonstandard, caddyModGoMod)
-		}
-	}
-	return
-}
-
-// apiRequest makes an API request to the endpoint adminAddr with the
-// given HTTP method and request URI. If body is non-nil, it will be
-// assumed to be Content-Type application/json.
-func apiRequest(adminAddr, method, uri string, headers http.Header, body io.Reader) error {
-	// parse the admin address
-	if adminAddr == "" {
-		adminAddr = caddy.DefaultAdminListen
-	}
+// AdminAPIRequest makes an API request according to the CLI flags given,
+// with the given HTTP method and request URI. If body is non-nil, it will
+// be assumed to be Content-Type application/json. The caller should close
+// the response body. Should only be used by Caddy CLI commands which
+// need to interact with a running instance of Caddy via the admin API.
+func AdminAPIRequest(adminAddr, method, uri string, headers http.Header, body io.Reader) (*http.Response, error) {
 	parsedAddr, err := caddy.ParseNetworkAddress(adminAddr)
 	if err != nil || parsedAddr.PortRangeSize() > 1 {
-		return fmt.Errorf("invalid admin address %s: %v", adminAddr, err)
+		return nil, fmt.Errorf("invalid admin address %s: %v", adminAddr, err)
 	}
-	origin := parsedAddr.JoinHostPort(0)
+	origin := "http://" + parsedAddr.JoinHostPort(0)
 	if parsedAddr.IsUnixNetwork() {
-		origin = "unixsocket" // hack so that http.NewRequest() is happy
+		origin = "http://unixsocket" // hack so that http.NewRequest() is happy
 	}
 
 	// form the request
-	req, err := http.NewRequest(method, "http://"+origin+uri, body)
+	req, err := http.NewRequest(method, origin+uri, body)
 	if err != nil {
-		return fmt.Errorf("making request: %v", err)
+		return nil, fmt.Errorf("making request: %v", err)
 	}
 	if parsedAddr.IsUnixNetwork() {
 		// When listening on a unix socket, the admin endpoint doesn't
@@ -889,20 +637,69 @@ func apiRequest(adminAddr, method, uri string, headers http.Header, body io.Read
 
 	resp, err := client.Do(req)
 	if err != nil {
-		return fmt.Errorf("performing request: %v", err)
+		return nil, fmt.Errorf("performing request: %v", err)
 	}
-	defer resp.Body.Close()
 
 	// if it didn't work, let the user know
 	if resp.StatusCode >= 400 {
-		respBody, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1024*10))
+		respBody, err := io.ReadAll(io.LimitReader(resp.Body, 1024*10))
 		if err != nil {
-			return fmt.Errorf("HTTP %d: reading error message: %v", resp.StatusCode, err)
+			return nil, fmt.Errorf("HTTP %d: reading error message: %v", resp.StatusCode, err)
 		}
-		return fmt.Errorf("caddy responded with error: HTTP %d: %s", resp.StatusCode, respBody)
+		return nil, fmt.Errorf("caddy responded with error: HTTP %d: %s", resp.StatusCode, respBody)
 	}
 
-	return nil
+	return resp, nil
+}
+
+// DetermineAdminAPIAddress determines which admin API endpoint address should
+// be used based on the inputs. By priority: if `address` is specified, then
+// it is returned; if `config` is specified, then that config will be used for
+// finding the admin address; if `configFile` (and `configAdapter`) are specified,
+// then that config will be loaded to find the admin address; otherwise, the
+// default admin listen address will be returned.
+func DetermineAdminAPIAddress(address string, config []byte, configFile, configAdapter string) (string, error) {
+	// Prefer the address if specified and non-empty
+	if address != "" {
+		return address, nil
+	}
+
+	// Try to load the config from file if specified, with the given adapter name
+	if configFile != "" {
+		var loadedConfigFile string
+		var err error
+
+		// use the provided loaded config if non-empty
+		// otherwise, load it from the specified file/adapter
+		loadedConfig := config
+		if len(loadedConfig) == 0 {
+			// get the config in caddy's native format
+			loadedConfig, loadedConfigFile, err = LoadConfig(configFile, configAdapter)
+			if err != nil {
+				return "", err
+			}
+			if loadedConfigFile == "" {
+				return "", fmt.Errorf("no config file to load")
+			}
+		}
+
+		// get the address of the admin listener from the config
+		if len(loadedConfig) > 0 {
+			var tmpStruct struct {
+				Admin caddy.AdminConfig `json:"admin"`
+			}
+			err := json.Unmarshal(loadedConfig, &tmpStruct)
+			if err != nil {
+				return "", fmt.Errorf("unmarshaling admin listener address from config: %v", err)
+			}
+			if tmpStruct.Admin.Listen != "" {
+				return tmpStruct.Admin.Listen, nil
+			}
+		}
+	}
+
+	// Fallback to the default listen address otherwise
+	return caddy.DefaultAdminListen, nil
 }
 
 type moduleInfo struct {

cmd/commands.go

@@ -16,7 +16,14 @@ package caddycmd
 
 import (
 	"flag"
+	"fmt"
+	"os"
 	"regexp"
+	"strings"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/spf13/cobra"
+	"github.com/spf13/cobra/doc"
 )
 
 // Command represents a subcommand. Name, Func,
@@ -61,16 +68,15 @@ type Command struct {
 // any error that occurred.
 type CommandFunc func(Flags) (int, error)
 
+// Commands returns a list of commands initialised by
+// RegisterCommand
+func Commands() map[string]Command {
+	return commands
+}
+
 var commands = make(map[string]Command)
 
 func init() {
-	RegisterCommand(Command{
-		Name:  "help",
-		Func:  cmdHelp,
-		Usage: "<command>",
-		Short: "Shows help for a Caddy subcommand",
-	})
-
 	RegisterCommand(Command{
 		Name:  "start",
 		Func:  cmdStart,
@@ -131,8 +137,8 @@ The --resume flag will override the --config flag if there is a config auto-
 save file. It is not an error if --resume is used and no autosave file exists.
 
 If --watch is specified, the config file will be loaded automatically after
-changes. ⚠️ This is dangerous in production! Only use this option in a local
-development environment.`,
+changes. ⚠️ This can make unintentional config changes easier; only use this
+option in a local development environment.`,
 		Flags: func() *flag.FlagSet {
 			fs := flag.NewFlagSet("run", flag.ExitOnError)
 			fs.String("config", "", "Configuration file")
@@ -150,16 +156,19 @@ development environment.`,
 	RegisterCommand(Command{
 		Name:  "stop",
 		Func:  cmdStop,
+		Usage: "[--address <interface>] [--config <path> [--adapter <name>]]",
 		Short: "Gracefully stops a started Caddy process",
 		Long: `
 Stops the background Caddy process as gracefully as possible.
 
 It requires that the admin API is enabled and accessible, since it will
-use the API's /stop endpoint. The address of this request can be
-customized using the --address flag if it is not the default.`,
+use the API's /stop endpoint. The address of this request can be customized
+using the --address flag, or from the given --config, if not the default.`,
 		Flags: func() *flag.FlagSet {
 			fs := flag.NewFlagSet("stop", flag.ExitOnError)
 			fs.String("address", "", "The address to use to reach the admin API endpoint, if not the default")
+			fs.String("config", "", "Configuration file to use to parse the admin address, if --address is not used")
+			fs.String("adapter", "", "Name of config adapter to apply (when --config is used)")
 			return fs
 		}(),
 	})
@@ -191,6 +200,19 @@ config file; otherwise the default is assumed.`,
 		Name:  "version",
 		Func:  cmdVersion,
 		Short: "Prints the version",
+		Long: `
+Prints the version of this Caddy binary.
+
+Version information must be embedded into the binary at compile-time in
+order for Caddy to display anything useful with this command. If Caddy
+is built from within a version control repository, the Go command will
+embed the revision hash if available. However, if Caddy is built in the
+way specified by our online documentation (or by using xcaddy), more
+detailed version information is printed as given by Go modules.
+
+For more details about the full version string, see the Go module
+documentation: https://go.dev/doc/modules/version-numbers
+`,
 	})
 
 	RegisterCommand(Command{
@@ -202,6 +224,7 @@ config file; otherwise the default is assumed.`,
 			fs := flag.NewFlagSet("list-modules", flag.ExitOnError)
 			fs.Bool("packages", false, "Print package paths")
 			fs.Bool("versions", false, "Print version information")
+			fs.Bool("skip-standard", false, "Skip printing standard modules")
 			return fs
 		}(),
 	})
@@ -216,6 +239,24 @@ config file; otherwise the default is assumed.`,
 		Name:  "environ",
 		Func:  cmdEnviron,
 		Short: "Prints the environment",
+		Long: `
+Prints the environment as seen by this Caddy process.
+
+The environment includes variables set in the system. If your Caddy
+configuration uses environment variables (e.g. "{env.VARIABLE}") then
+this command can be useful for verifying that the variables will have
+the values you expect in your config.
+
+Note that environments may be different depending on how you run Caddy.
+Environments for Caddy instances started by service managers such as
+systemd are often different than the environment inherited from your
+shell or terminal.
+
+You can also print the environment the same time you use "caddy run"
+by adding the "--environ" flag.
+
+Environments may contain sensitive data.
+`,
 	})
 
 	RegisterCommand(Command{
@@ -253,7 +294,7 @@ Loads and provisions the provided config, but does not start running it.
 This reveals any errors with the configuration through the loading and
 provisioning stages.`,
 		Flags: func() *flag.FlagSet {
-			fs := flag.NewFlagSet("load", flag.ExitOnError)
+			fs := flag.NewFlagSet("validate", flag.ExitOnError)
 			fs.String("config", "", "Input configuration file")
 			fs.String("adapter", "", "Name of config adapter")
 			return fs
@@ -272,12 +313,18 @@ human readability. It prints the result to stdout.
 If --overwrite is specified, the output will be written to the config file
 directly instead of printing it.
 
+If --diff is specified, the output will be compared against the input, and
+lines will be prefixed with '-' and '+' where they differ. Note that
+unchanged lines are prefixed with two spaces for alignment, and that this
+is not a valid patch format.
+
 If you wish you use stdin instead of a regular file, use - as the path.
 When reading from stdin, the --overwrite flag has no effect: the result
 is always printed to stdout.`,
 		Flags: func() *flag.FlagSet {
-			fs := flag.NewFlagSet("format", flag.ExitOnError)
+			fs := flag.NewFlagSet("fmt", flag.ExitOnError)
 			fs.Bool("overwrite", false, "Overwrite the input file with the results")
+			fs.Bool("diff", false, "Print the differences between the input file and the formatted output")
 			return fs
 		}(),
 	})
@@ -289,18 +336,152 @@ is always printed to stdout.`,
 		Long: `
 Downloads an updated Caddy binary with the same modules/plugins at the
 latest versions. EXPERIMENTAL: May be changed or removed.`,
+		Flags: func() *flag.FlagSet {
+			fs := flag.NewFlagSet("upgrade", flag.ExitOnError)
+			fs.Bool("keep-backup", false, "Keep the backed up binary, instead of deleting it")
+			return fs
+		}(),
 	})
 
+	RegisterCommand(Command{
+		Name:  "add-package",
+		Func:  cmdAddPackage,
+		Usage: "<packages...>",
+		Short: "Adds Caddy packages (EXPERIMENTAL)",
+		Long: `
+Downloads an updated Caddy binary with the specified packages (module/plugin)
+added. Retains existing packages. Returns an error if the any of packages are 
+already included. EXPERIMENTAL: May be changed or removed.
+`,
+		Flags: func() *flag.FlagSet {
+			fs := flag.NewFlagSet("add-package", flag.ExitOnError)
+			fs.Bool("keep-backup", false, "Keep the backed up binary, instead of deleting it")
+			return fs
+		}(),
+	})
+
+	RegisterCommand(Command{
+		Name:  "remove-package",
+		Func:  cmdRemovePackage,
+		Usage: "<packages...>",
+		Short: "Removes Caddy packages (EXPERIMENTAL)",
+		Long: `
+Downloads an updated Caddy binaries without the specified packages (module/plugin). 
+Returns an error if any of the packages are not included. 
+EXPERIMENTAL: May be changed or removed.
+`,
+		Flags: func() *flag.FlagSet {
+			fs := flag.NewFlagSet("remove-package", flag.ExitOnError)
+			fs.Bool("keep-backup", false, "Keep the backed up binary, instead of deleting it")
+			return fs
+		}(),
+	})
+
+	RegisterCommand(Command{
+		Name: "manpage",
+		Func: func(fl Flags) (int, error) {
+			dir := strings.TrimSpace(fl.String("directory"))
+			if dir == "" {
+				return caddy.ExitCodeFailedQuit, fmt.Errorf("designated output directory and specified section are required")
+			}
+			if err := os.MkdirAll(dir, 0755); err != nil {
+				return caddy.ExitCodeFailedQuit, err
+			}
+			if err := doc.GenManTree(rootCmd, &doc.GenManHeader{
+				Title:   "Caddy",
+				Section: "8", // https://en.wikipedia.org/wiki/Man_page#Manual_sections
+			}, dir); err != nil {
+				return caddy.ExitCodeFailedQuit, err
+			}
+			return caddy.ExitCodeSuccess, nil
+		},
+		Usage: "--directory <path>",
+		Short: "Generates the manual pages for Caddy commands",
+		Long: `
+Generates the manual pages for Caddy commands into the designated directory
+tagged into section 8 (System Administration).
+
+The manual page files are generated into the directory specified by the
+argument of --directory. If the directory does not exist, it will be created.
+`,
+		Flags: func() *flag.FlagSet {
+			fs := flag.NewFlagSet("manpage", flag.ExitOnError)
+			fs.String("directory", "", "The output directory where the manpages are generated")
+			return fs
+		}(),
+	})
+
+	// source: https://github.com/spf13/cobra/blob/main/shell_completions.md
+	rootCmd.AddCommand(&cobra.Command{
+		Use:   "completion [bash|zsh|fish|powershell]",
+		Short: "Generate completion script",
+		Long: fmt.Sprintf(`To load completions:
+	
+	Bash:
+	
+	  $ source <(%[1]s completion bash)
+	
+	  # To load completions for each session, execute once:
+	  # Linux:
+	  $ %[1]s completion bash > /etc/bash_completion.d/%[1]s
+	  # macOS:
+	  $ %[1]s completion bash > $(brew --prefix)/etc/bash_completion.d/%[1]s
+	
+	Zsh:
+	
+	  # If shell completion is not already enabled in your environment,
+	  # you will need to enable it.  You can execute the following once:
+	
+	  $ echo "autoload -U compinit; compinit" >> ~/.zshrc
+	
+	  # To load completions for each session, execute once:
+	  $ %[1]s completion zsh > "${fpath[1]}/_%[1]s"
+	
+	  # You will need to start a new shell for this setup to take effect.
+	
+	fish:
+	
+	  $ %[1]s completion fish | source
+	
+	  # To load completions for each session, execute once:
+	  $ %[1]s completion fish > ~/.config/fish/completions/%[1]s.fish
+	
+	PowerShell:
+	
+	  PS> %[1]s completion powershell | Out-String | Invoke-Expression
+	
+	  # To load completions for every new session, run:
+	  PS> %[1]s completion powershell > %[1]s.ps1
+	  # and source this file from your PowerShell profile.
+	`, rootCmd.Root().Name()),
+		DisableFlagsInUseLine: true,
+		ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
+		Args:                  cobra.ExactValidArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			switch args[0] {
+			case "bash":
+				return cmd.Root().GenBashCompletion(os.Stdout)
+			case "zsh":
+				return cmd.Root().GenZshCompletion(os.Stdout)
+			case "fish":
+				return cmd.Root().GenFishCompletion(os.Stdout, true)
+			case "powershell":
+				return cmd.Root().GenPowerShellCompletionWithDesc(os.Stdout)
+			default:
+				return fmt.Errorf("unrecognized shell: %s", args[0])
+			}
+		},
+	})
 }
 
 // RegisterCommand registers the command cmd.
 // cmd.Name must be unique and conform to the
 // following format:
 //
-//    - lowercase
-//    - alphanumeric and hyphen characters only
-//    - cannot start or end with a hyphen
-//    - hyphen cannot be adjacent to another hyphen
+//   - lowercase
+//   - alphanumeric and hyphen characters only
+//   - cannot start or end with a hyphen
+//   - hyphen cannot be adjacent to another hyphen
 //
 // This function panics if the name is already registered,
 // if the name does not meet the described format, or if
@@ -323,7 +504,7 @@ func RegisterCommand(cmd Command) {
 	if !commandNameRegex.MatchString(cmd.Name) {
 		panic("invalid command name")
 	}
-	commands[cmd.Name] = cmd
+	rootCmd.AddCommand(caddyCmdToCoral(cmd))
 }
 
 var commandNameRegex = regexp.MustCompile(`^[a-z0-9]$|^([a-z0-9]+-?[a-z0-9]*)+[a-z0-9]$`)

cmd/main.go

@@ -20,7 +20,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net"
 	"os"
@@ -34,13 +33,14 @@ import (
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"github.com/caddyserver/certmagic"
+	"github.com/spf13/pflag"
 	"go.uber.org/zap"
 )
 
 func init() {
 	// set a fitting User-Agent for ACME requests
-	goModule := caddy.GoModule()
-	cleanModVersion := strings.TrimPrefix(goModule.Version, "v")
+	version, _ := caddy.Version()
+	cleanModVersion := strings.TrimPrefix(version, "v")
 	certmagic.UserAgent = "Caddy/" + cleanModVersion
 
 	// by using Caddy, user indicates agreement to CA terms
@@ -51,50 +51,21 @@ func init() {
 // Main implements the main function of the caddy command.
 // Call this if Caddy is to be the main() of your program.
 func Main() {
-	switch len(os.Args) {
-	case 0:
+	if len(os.Args) == 0 {
 		fmt.Printf("[FATAL] no arguments provided by OS; args[0] must be command\n")
 		os.Exit(caddy.ExitCodeFailedStartup)
-	case 1:
-		os.Args = append(os.Args, "help")
 	}
 
-	subcommandName := os.Args[1]
-	subcommand, ok := commands[subcommandName]
-	if !ok {
-		if strings.HasPrefix(os.Args[1], "-") {
-			// user probably forgot to type the subcommand
-			fmt.Println("[ERROR] first argument must be a subcommand; see 'caddy help'")
-		} else {
-			fmt.Printf("[ERROR] '%s' is not a recognized subcommand; see 'caddy help'\n", os.Args[1])
-		}
-		os.Exit(caddy.ExitCodeFailedStartup)
+	if err := rootCmd.Execute(); err != nil {
+		os.Exit(1)
 	}
-
-	fs := subcommand.Flags
-	if fs == nil {
-		fs = flag.NewFlagSet(subcommand.Name, flag.ExitOnError)
-	}
-
-	err := fs.Parse(os.Args[2:])
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(caddy.ExitCodeFailedStartup)
-	}
-
-	exitCode, err := subcommand.Func(Flags{fs})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s: %v\n", subcommand.Name, err)
-	}
-
-	os.Exit(exitCode)
 }
 
 // handlePingbackConn reads from conn and ensures it matches
 // the bytes in expect, or returns an error if it doesn't.
 func handlePingbackConn(conn net.Conn, expect []byte) error {
 	defer conn.Close()
-	confirmationBytes, err := ioutil.ReadAll(io.LimitReader(conn, 32))
+	confirmationBytes, err := io.ReadAll(io.LimitReader(conn, 32))
 	if err != nil {
 		return err
 	}
@@ -104,15 +75,15 @@ func handlePingbackConn(conn net.Conn, expect []byte) error {
 	return nil
 }
 
-// loadConfig loads the config from configFile and adapts it
+// LoadConfig loads the config from configFile and adapts it
 // using adapterName. If adapterName is specified, configFile
 // must be also. If no configFile is specified, it tries
 // loading a default config file. The lack of a config file is
 // not treated as an error, but false will be returned if
 // there is no config available. It prints any warnings to stderr,
 // and returns the resulting JSON config bytes along with
-// whether a config file was loaded or not.
-func loadConfig(configFile, adapterName string) ([]byte, string, error) {
+// the name of the loaded config file (if any).
+func LoadConfig(configFile, adapterName string) ([]byte, string, error) {
 	// specifying an adapter without a config file is ambiguous
 	if adapterName != "" && configFile == "" {
 		return nil, "", fmt.Errorf("cannot adapt config without config file (use --config)")
@@ -124,9 +95,9 @@ func loadConfig(configFile, adapterName string) ([]byte, string, error) {
 	var err error
 	if configFile != "" {
 		if configFile == "-" {
-			config, err = ioutil.ReadAll(os.Stdin)
+			config, err = io.ReadAll(os.Stdin)
 		} else {
-			config, err = ioutil.ReadFile(configFile)
+			config, err = os.ReadFile(configFile)
 		}
 		if err != nil {
 			return nil, "", fmt.Errorf("reading config file: %v", err)
@@ -140,7 +111,7 @@ func loadConfig(configFile, adapterName string) ([]byte, string, error) {
 		// plugged in, and if so, try using a default Caddyfile
 		cfgAdapter = caddyconfig.GetAdapter("caddyfile")
 		if cfgAdapter != nil {
-			config, err = ioutil.ReadFile("Caddyfile")
+			config, err = os.ReadFile("Caddyfile")
 			if os.IsNotExist(err) {
 				// okay, no default Caddyfile; pretend like this never happened
 				cfgAdapter = nil
@@ -174,7 +145,7 @@ func loadConfig(configFile, adapterName string) ([]byte, string, error) {
 
 	// adapt config
 	if cfgAdapter != nil {
-		adaptedConfig, warnings, err := cfgAdapter.Adapt(config, map[string]interface{}{
+		adaptedConfig, warnings, err := cfgAdapter.Adapt(config, map[string]any{
 			"filename": configFile,
 		})
 		if err != nil {
@@ -263,7 +234,7 @@ func watchConfigFile(filename, adapterName string) {
 		lastModified = info.ModTime()
 
 		// load the contents of the file
-		config, _, err := loadConfig(filename, adapterName)
+		config, _, err := LoadConfig(filename, adapterName)
 		if err != nil {
 			logger().Error("unable to load latest config", zap.Error(err))
 			continue
@@ -281,7 +252,7 @@ func watchConfigFile(filename, adapterName string) {
 // Flags wraps a FlagSet so that typed values
 // from flags can be easily retrieved.
 type Flags struct {
-	*flag.FlagSet
+	*pflag.FlagSet
 }
 
 // String returns the string representation of the
@@ -327,22 +298,6 @@ func (f Flags) Duration(name string) time.Duration {
 	return val
 }
 
-// flagHelp returns the help text for fs.
-func flagHelp(fs *flag.FlagSet) string {
-	if fs == nil {
-		return ""
-	}
-
-	// temporarily redirect output
-	out := fs.Output()
-	defer fs.SetOutput(out)
-
-	buf := new(bytes.Buffer)
-	fs.SetOutput(buf)
-	fs.PrintDefaults()
-	return buf.String()
-}
-
 func loadEnvFromFile(envFile string) error {
 	file, err := os.Open(envFile)
 	if err != nil {
@@ -361,45 +316,73 @@ func loadEnvFromFile(envFile string) error {
 		}
 	}
 
+	// Update the storage paths to ensure they have the proper
+	// value after loading a specified env file.
+	caddy.ConfigAutosavePath = filepath.Join(caddy.AppConfigDir(), "autosave.json")
+	caddy.DefaultStorage = &certmagic.FileStorage{Path: caddy.AppDataDir()}
+
 	return nil
 }
 
+// parseEnvFile parses an env file from KEY=VALUE format.
+// It's pretty naive. Limited value quotation is supported,
+// but variable and command expansions are not supported.
 func parseEnvFile(envInput io.Reader) (map[string]string, error) {
 	envMap := make(map[string]string)
 
 	scanner := bufio.NewScanner(envInput)
-	var line string
-	lineNumber := 0
+	var lineNumber int
 
 	for scanner.Scan() {
-		line = strings.TrimSpace(scanner.Text())
+		line := strings.TrimSpace(scanner.Text())
 		lineNumber++
 
-		// skip lines starting with comment
-		if strings.HasPrefix(line, "#") {
+		// skip empty lines and lines starting with comment
+		if line == "" || strings.HasPrefix(line, "#") {
 			continue
 		}
 
-		// skip empty line
-		if len(line) == 0 {
-			continue
-		}
-
-		fields := strings.SplitN(line, "=", 2)
-		if len(fields) != 2 {
+		// split line into key and value
+		before, after, isCut := strings.Cut(line, "=")
+		if !isCut {
 			return nil, fmt.Errorf("can't parse line %d; line should be in KEY=VALUE format", lineNumber)
 		}
+		key, val := before, after
 
-		if strings.Contains(fields[0], " ") {
-			return nil, fmt.Errorf("bad key on line %d: contains whitespace", lineNumber)
-		}
-
-		key := fields[0]
-		val := fields[1]
+		// sometimes keys are prefixed by "export " so file can be sourced in bash; ignore it here
+		key = strings.TrimPrefix(key, "export ")
 
+		// validate key and value
 		if key == "" {
 			return nil, fmt.Errorf("missing or empty key on line %d", lineNumber)
 		}
+		if strings.Contains(key, " ") {
+			return nil, fmt.Errorf("invalid key on line %d: contains whitespace: %s", lineNumber, key)
+		}
+		if strings.HasPrefix(val, " ") || strings.HasPrefix(val, "\t") {
+			return nil, fmt.Errorf("invalid value on line %d: whitespace before value: '%s'", lineNumber, val)
+		}
+
+		// remove any trailing comment after value
+		if commentStart, _, found := strings.Cut(val, "#"); found {
+			val = strings.TrimRight(commentStart, " \t")
+		}
+
+		// quoted value: support newlines
+		if strings.HasPrefix(val, `"`) {
+			for !(strings.HasSuffix(line, `"`) && !strings.HasSuffix(line, `\"`)) {
+				val = strings.ReplaceAll(val, `\"`, `"`)
+				if !scanner.Scan() {
+					break
+				}
+				lineNumber++
+				line = strings.ReplaceAll(scanner.Text(), `\"`, `"`)
+				val += "\n" + line
+			}
+			val = strings.TrimPrefix(val, `"`)
+			val = strings.TrimSuffix(val, `"`)
+		}
+
 		envMap[key] = val
 	}
 
@@ -411,11 +394,12 @@ func parseEnvFile(envInput io.Reader) (map[string]string, error) {
 }
 
 func printEnvironment() {
+	_, version := caddy.Version()
 	fmt.Printf("caddy.HomeDir=%s\n", caddy.HomeDir())
 	fmt.Printf("caddy.AppDataDir=%s\n", caddy.AppDataDir())
 	fmt.Printf("caddy.AppConfigDir=%s\n", caddy.AppConfigDir())
 	fmt.Printf("caddy.ConfigAutosavePath=%s\n", caddy.ConfigAutosavePath)
-	fmt.Printf("caddy.Version=%s\n", caddyVersion())
+	fmt.Printf("caddy.Version=%s\n", version)
 	fmt.Printf("runtime.GOOS=%s\n", runtime.GOOS)
 	fmt.Printf("runtime.GOARCH=%s\n", runtime.GOARCH)
 	fmt.Printf("runtime.Compiler=%s\n", runtime.Compiler)
@@ -432,21 +416,15 @@ func printEnvironment() {
 	}
 }
 
-// caddyVersion returns a detailed version string, if available.
-func caddyVersion() string {
-	goModule := caddy.GoModule()
-	ver := goModule.Version
-	if goModule.Sum != "" {
-		ver += " " + goModule.Sum
-	}
-	if goModule.Replace != nil {
-		ver += " => " + goModule.Replace.Path
-		if goModule.Replace.Version != "" {
-			ver += "@" + goModule.Replace.Version
-		}
-		if goModule.Replace.Sum != "" {
-			ver += " " + goModule.Replace.Sum
-		}
-	}
-	return ver
+// StringSlice is a flag.Value that enables repeated use of a string flag.
+type StringSlice []string
+
+func (ss StringSlice) String() string { return "[" + strings.Join(ss, ", ") + "]" }
+
+func (ss *StringSlice) Set(value string) error {
+	*ss = append(*ss, value)
+	return nil
 }
+
+// Interface guard
+var _ flag.Value = (*StringSlice)(nil)

cmd/main_test.go

@@ -0,0 +1,170 @@
+package caddycmd
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestParseEnvFile(t *testing.T) {
+	for i, tc := range []struct {
+		input     string
+		expect    map[string]string
+		shouldErr bool
+	}{
+		{
+			input: `KEY=value`,
+			expect: map[string]string{
+				"KEY": "value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				INVALID KEY=asdf
+				OTHER_KEY=Some Value
+			`,
+			shouldErr: true,
+		},
+		{
+			input: `
+				KEY=value
+				SIMPLE_QUOTED="quoted value"
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":           "value",
+				"SIMPLE_QUOTED": "quoted value",
+				"OTHER_KEY":     "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				NEWLINES="foo
+	bar"
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"NEWLINES":  "foo\n\tbar",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				ESCAPED="\"escaped quotes\"
+here"
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"ESCAPED":   "\"escaped quotes\"\nhere",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				export KEY=value
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				=value
+				OTHER_KEY=Some Value
+			`,
+			shouldErr: true,
+		},
+		{
+			input: `
+				EMPTY=
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"EMPTY":     "",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				EMPTY=""
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"EMPTY":     "",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				#OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY": "value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				COMMENT=foo bar  # some comment here
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":       "value",
+				"COMMENT":   "foo bar",
+				"OTHER_KEY": "Some Value",
+			},
+		},
+		{
+			input: `
+				KEY=value
+				WHITESPACE=   foo 
+				OTHER_KEY=Some Value
+			`,
+			shouldErr: true,
+		},
+		{
+			input: `
+				KEY=value
+				WHITESPACE="   foo bar "
+				OTHER_KEY=Some Value
+			`,
+			expect: map[string]string{
+				"KEY":        "value",
+				"WHITESPACE": "   foo bar ",
+				"OTHER_KEY":  "Some Value",
+			},
+		},
+	} {
+		actual, err := parseEnvFile(strings.NewReader(tc.input))
+		if err != nil && !tc.shouldErr {
+			t.Errorf("Test %d: Got error but shouldn't have: %v", i, err)
+		}
+		if err == nil && tc.shouldErr {
+			t.Errorf("Test %d: Did not get error but should have", i)
+		}
+		if tc.shouldErr {
+			continue
+		}
+		if !reflect.DeepEqual(tc.expect, actual) {
+			t.Errorf("Test %d: Expected %v but got %v", i, tc.expect, actual)
+		}
+	}
+}

cmd/packagesfuncs.go

@@ -0,0 +1,303 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddycmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"reflect"
+	"runtime"
+	"runtime/debug"
+	"strings"
+
+	"github.com/caddyserver/caddy/v2"
+	"go.uber.org/zap"
+)
+
+func cmdUpgrade(fl Flags) (int, error) {
+	_, nonstandard, _, err := getModules()
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err)
+	}
+	pluginPkgs, err := getPluginPackages(nonstandard)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	return upgradeBuild(pluginPkgs, fl)
+}
+
+func cmdAddPackage(fl Flags) (int, error) {
+	if len(fl.Args()) == 0 {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("at least one package name must be specified")
+	}
+	_, nonstandard, _, err := getModules()
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err)
+	}
+	pluginPkgs, err := getPluginPackages(nonstandard)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	for _, arg := range fl.Args() {
+		if _, ok := pluginPkgs[arg]; ok {
+			return caddy.ExitCodeFailedStartup, fmt.Errorf("package is already added")
+		}
+		pluginPkgs[arg] = struct{}{}
+	}
+
+	return upgradeBuild(pluginPkgs, fl)
+}
+
+func cmdRemovePackage(fl Flags) (int, error) {
+	if len(fl.Args()) == 0 {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("at least one package name must be specified")
+	}
+	_, nonstandard, _, err := getModules()
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err)
+	}
+	pluginPkgs, err := getPluginPackages(nonstandard)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	for _, arg := range fl.Args() {
+		if _, ok := pluginPkgs[arg]; !ok {
+			// package does not exist
+			return caddy.ExitCodeFailedStartup, fmt.Errorf("package is not added")
+		}
+		delete(pluginPkgs, arg)
+	}
+
+	return upgradeBuild(pluginPkgs, fl)
+}
+
+func upgradeBuild(pluginPkgs map[string]struct{}, fl Flags) (int, error) {
+	l := caddy.Log()
+
+	thisExecPath, err := os.Executable()
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("determining current executable path: %v", err)
+	}
+	thisExecStat, err := os.Stat(thisExecPath)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("retrieving current executable permission bits: %v", err)
+	}
+	l.Info("this executable will be replaced", zap.String("path", thisExecPath))
+
+	// build the request URL to download this custom build
+	qs := url.Values{
+		"os":   {runtime.GOOS},
+		"arch": {runtime.GOARCH},
+	}
+	for pkg := range pluginPkgs {
+		qs.Add("p", pkg)
+	}
+
+	// initiate the build
+	resp, err := downloadBuild(qs)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("download failed: %v", err)
+	}
+	defer resp.Body.Close()
+
+	// back up the current binary, in case something goes wrong we can replace it
+	backupExecPath := thisExecPath + ".tmp"
+	l.Info("build acquired; backing up current executable",
+		zap.String("current_path", thisExecPath),
+		zap.String("backup_path", backupExecPath))
+	err = os.Rename(thisExecPath, backupExecPath)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("backing up current binary: %v", err)
+	}
+	defer func() {
+		if err != nil {
+			err2 := os.Rename(backupExecPath, thisExecPath)
+			if err2 != nil {
+				l.Error("restoring original executable failed; will need to be restored manually",
+					zap.String("backup_path", backupExecPath),
+					zap.String("original_path", thisExecPath),
+					zap.Error(err2))
+			}
+		}
+	}()
+
+	// download the file; do this in a closure to close reliably before we execute it
+	err = writeCaddyBinary(thisExecPath, &resp.Body, thisExecStat)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	l.Info("download successful; displaying new binary details", zap.String("location", thisExecPath))
+
+	// use the new binary to print out version and module info
+	fmt.Print("\nModule versions:\n\n")
+	if err = listModules(thisExecPath); err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute 'caddy list-modules': %v", err)
+	}
+	fmt.Println("\nVersion:")
+	if err = showVersion(thisExecPath); err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute 'caddy version': %v", err)
+	}
+	fmt.Println()
+
+	// clean up the backup file
+	if !fl.Bool("keep-backup") {
+		if err = removeCaddyBinary(backupExecPath); err != nil {
+			return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to clean up backup binary: %v", err)
+		}
+	} else {
+		l.Info("skipped cleaning up the backup file", zap.String("backup_path", backupExecPath))
+	}
+
+	l.Info("upgrade successful; please restart any running Caddy instances", zap.String("executable", thisExecPath))
+
+	return caddy.ExitCodeSuccess, nil
+}
+
+func getModules() (standard, nonstandard, unknown []moduleInfo, err error) {
+	bi, ok := debug.ReadBuildInfo()
+	if !ok {
+		err = fmt.Errorf("no build info")
+		return
+	}
+
+	for _, modID := range caddy.Modules() {
+		modInfo, err := caddy.GetModule(modID)
+		if err != nil {
+			// that's weird, shouldn't happen
+			unknown = append(unknown, moduleInfo{caddyModuleID: modID, err: err})
+			continue
+		}
+
+		// to get the Caddy plugin's version info, we need to know
+		// the package that the Caddy module's value comes from; we
+		// can use reflection but we need a non-pointer value (I'm
+		// not sure why), and since New() should return a pointer
+		// value, we need to dereference it first
+		iface := any(modInfo.New())
+		if rv := reflect.ValueOf(iface); rv.Kind() == reflect.Ptr {
+			iface = reflect.New(reflect.TypeOf(iface).Elem()).Elem().Interface()
+		}
+		modPkgPath := reflect.TypeOf(iface).PkgPath()
+
+		// now we find the Go module that the Caddy module's package
+		// belongs to; we assume the Caddy module package path will
+		// be prefixed by its Go module path, and we will choose the
+		// longest matching prefix in case there are nested modules
+		var matched *debug.Module
+		for _, dep := range bi.Deps {
+			if strings.HasPrefix(modPkgPath, dep.Path) {
+				if matched == nil || len(dep.Path) > len(matched.Path) {
+					matched = dep
+				}
+			}
+		}
+
+		caddyModGoMod := moduleInfo{caddyModuleID: modID, goModule: matched}
+
+		if strings.HasPrefix(modPkgPath, caddy.ImportPath) {
+			standard = append(standard, caddyModGoMod)
+		} else {
+			nonstandard = append(nonstandard, caddyModGoMod)
+		}
+	}
+	return
+}
+
+func listModules(path string) error {
+	cmd := exec.Command(path, "list-modules", "--versions", "--skip-standard")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func showVersion(path string) error {
+	cmd := exec.Command(path, "version")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func downloadBuild(qs url.Values) (*http.Response, error) {
+	l := caddy.Log()
+	l.Info("requesting build",
+		zap.String("os", qs.Get("os")),
+		zap.String("arch", qs.Get("arch")),
+		zap.Strings("packages", qs["p"]))
+	resp, err := http.Get(fmt.Sprintf("%s?%s", downloadPath, qs.Encode()))
+	if err != nil {
+		return nil, fmt.Errorf("secure request failed: %v", err)
+	}
+	if resp.StatusCode >= 400 {
+		var details struct {
+			StatusCode int `json:"status_code"`
+			Error      struct {
+				Message string `json:"message"`
+				ID      string `json:"id"`
+			} `json:"error"`
+		}
+		err2 := json.NewDecoder(resp.Body).Decode(&details)
+		if err2 != nil {
+			return nil, fmt.Errorf("download and error decoding failed: HTTP %d: %v", resp.StatusCode, err2)
+		}
+		return nil, fmt.Errorf("download failed: HTTP %d: %s (id=%s)", resp.StatusCode, details.Error.Message, details.Error.ID)
+	}
+	return resp, nil
+}
+
+func getPluginPackages(modules []moduleInfo) (map[string]struct{}, error) {
+	pluginPkgs := make(map[string]struct{})
+	for _, mod := range modules {
+		if mod.goModule.Replace != nil {
+			return nil, fmt.Errorf("cannot auto-upgrade when Go module has been replaced: %s => %s",
+				mod.goModule.Path, mod.goModule.Replace.Path)
+		}
+		pluginPkgs[mod.goModule.Path] = struct{}{}
+	}
+	return pluginPkgs, nil
+}
+
+func writeCaddyBinary(path string, body *io.ReadCloser, fileInfo os.FileInfo) error {
+	l := caddy.Log()
+	destFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileInfo.Mode())
+	if err != nil {
+		return fmt.Errorf("unable to open destination file: %v", err)
+	}
+	defer destFile.Close()
+
+	l.Info("downloading binary", zap.String("destination", path))
+
+	_, err = io.Copy(destFile, *body)
+	if err != nil {
+		return fmt.Errorf("unable to download file: %v", err)
+	}
+
+	err = destFile.Sync()
+	if err != nil {
+		return fmt.Errorf("syncing downloaded file to device: %v", err)
+	}
+
+	return nil
+}
+
+const downloadPath = "https://caddyserver.com/api/download"