ci: force fetch the upstream tags (#3947)

* metrics: allow disabling OpenMetrics negotiation

Signed-off-by: Dave Henderson <dhenderson@gmail.com>

* fixup! metrics: allow disabling OpenMetrics negotiation

.github/SECURITY.md

@@ -11,7 +11,7 @@ Please note that we consider publicly-registered domain names to be public infor
 | Version | Supported          |
 | ------- | ------------------ |
 | 2.x     | :white_check_mark: |
-| 1.x     | :white_check_mark: (deprecating soon) |
+| 1.x     | :x:                |
 | < 1.x   | :x:                |
 
 ## Reporting a Vulnerability
@@ -22,6 +22,6 @@ We'll need enough information to verify the bug and make a patch. It will speed
 
 Please also understand that due to our nature as an open source project, we do not have a budget to award security bounties. We can only thank you.
 
-If your report is valid and a patch is released, we will not reveal your identity by default. If you wish to be credited, please give us the name to use.
+If your report is valid and a patch is released, we will not reveal your identity by default. If you wish to be credited, please give us the name to use and/or your GitHub username. If you don't provide this we can't credit you.
 
 Thanks for responsibly helping Caddy&mdash;and thousands of websites&mdash;be more secure!

.github/workflows/ci.yml

@@ -1,6 +1,6 @@
 # Used as inspiration: https://github.com/mvdan/github-actions-golang
 
-name: Cross-Platform
+name: Tests
 
 on:
   push:
@@ -17,7 +17,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
-        go-version: [ 1.14.x ]
+        go: [ '1.14', '1.15' ]
 
         # Set some variables per OS, usable via ${{ matrix.VAR }}
         # CADDY_BIN_PATH: the path to the compiled Caddy binary, for artifact publishing
@@ -39,9 +39,9 @@ jobs:
 
     steps:
     - name: Install Go
-      uses: actions/setup-go@v1
+      uses: actions/setup-go@v2
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: ${{ matrix.go }}
 
     - name: Checkout code
       uses: actions/checkout@v2
@@ -69,12 +69,12 @@ jobs:
         echo "::set-output name=go_cache::$(go env GOCACHE)"
 
     - name: Cache the build cache
-      uses: actions/cache@v1
+      uses: actions/cache@v2
       with:
         path: ${{ steps.vars.outputs.go_cache }}
-        key: ${{ runner.os }}-go-ci-${{ hashFiles('**/go.sum') }}
+        key: ${{ runner.os }}-${{ matrix.go }}-go-ci-${{ hashFiles('**/go.sum') }}
         restore-keys: |
-          ${{ runner.os }}-go-ci
+          ${{ runner.os }}-${{ matrix.go }}-go-ci
 
     - name: Get dependencies
       run: |
@@ -91,7 +91,7 @@ jobs:
     - name: Publish Build Artifact
       uses: actions/upload-artifact@v1
       with:
-        name: caddy_v2_${{ runner.os }}_${{ steps.vars.outputs.short_sha }}
+        name: caddy_${{ runner.os }}_go${{ matrix.go }}_${{ steps.vars.outputs.short_sha }}
         path: ${{ matrix.CADDY_BIN_PATH }}
 
     # Commented bits below were useful to allow the job to continue
@@ -124,6 +124,7 @@ jobs:
     name: test (s390x on IBM Z)
     runs-on: ubuntu-latest
     if: github.event.pull_request.head.repo.full_name == github.repository
+    continue-on-error: true  # August 2020: s390x VM is down due to weather and power issues
     steps:
       - name: Checkout code into the Go module directory
         uses: actions/checkout@v2
@@ -147,26 +148,12 @@ jobs:
         env:
           SSH_KEY: ${{ secrets.S390X_SSH_KEY }}
 
-  # From https://github.com/reviewdog/action-golangci-lint
-  golangci-lint:
-    name: runner / golangci-lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Run golangci-lint
-        uses: reviewdog/action-golangci-lint@v1
-        # uses: docker://reviewdog/action-golangci-lint:v1 # pre-build docker image
-        with:
-          github_token: ${{ secrets.github_token }}
-
   goreleaser-check:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
         uses: actions/checkout@v2
-      - uses: goreleaser/goreleaser-action@v1
+      - uses: goreleaser/goreleaser-action@v2
         with:
           version: latest
           args: check

.github/workflows/cross-build.yml

@@ -0,0 +1,60 @@
+name: Cross-Build
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  cross-build-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        goos: ['android', 'linux', 'solaris', 'illumos', 'dragonfly', 'freebsd', 'openbsd', 'plan9', 'windows', 'darwin', 'netbsd']
+        go: [ '1.14', '1.15' ]
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: Print Go version and environment
+        id: vars
+        run: |
+          printf "Using go at: $(which go)\n"
+          printf "Go version: $(go version)\n"
+          printf "\n\nGo environment:\n\n"
+          go env
+          printf "\n\nSystem environment:\n\n"
+          env
+          echo "::set-output name=go_cache::$(go env GOCACHE)"
+
+      - name: Cache the build cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.vars.outputs.go_cache }}
+          key: cross-build-go${{ matrix.go }}-${{ matrix.goos }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            cross-build-go${{ matrix.go }}-${{ matrix.goos }}
+
+      - name: Checkout code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Run Build
+        env:
+          CGO_ENABLED: 0
+          GOOS: ${{ matrix.goos }}
+        shell: bash
+        continue-on-error: true
+        working-directory: ./cmd/caddy
+        run: |
+          GOOS=$GOOS go build -trimpath -o caddy-"$GOOS"-amd64 2> /dev/null
+          if [ $? -ne 0 ]; then
+            echo "::warning ::$GOOS Build Failed"
+            exit 0
+          fi

.github/workflows/fuzzing.yml

@@ -1,73 +0,0 @@
-name: Fuzzing
-
-on:
-  # Daily midnight fuzzing
-  schedule:
-    - cron: '0 0 * * *'
-
-jobs:
-  fuzzing:
-    name: Fuzzing
-
-    strategy:
-      matrix:
-        os: [ ubuntu-latest ]
-        go-version: [ 1.14.x ]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-    - name: Install Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: ${{ matrix.go-version }}
-
-    - name: Checkout code
-      uses: actions/checkout@v2
-
-    - name: Download go-fuzz tools and the Fuzzit CLI, move Fuzzit CLI to GOBIN
-      # If we decide we need to prevent this from running on forks, we can use this line:
-      # if: github.repository == 'caddyserver/caddy'
-      run: |
-
-        go get -v github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
-        wget -q -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.77/fuzzit_Linux_x86_64
-        chmod a+x fuzzit
-        mv fuzzit $(go env GOPATH)/bin
-        echo "::add-path::$(go env GOPATH)/bin"
-
-    - name: Generate fuzzers & submit them to Fuzzit
-      continue-on-error: true
-      env:
-        FUZZIT_API_KEY: ${{ secrets.FUZZIT_API_KEY }}
-        SYSTEM_PULLREQUEST_SOURCEBRANCH: ${{ github.ref }}
-        BUILD_SOURCEVERSION: ${{ github.sha }}
-      run: |
-        # debug
-        echo "PR Source Branch: $SYSTEM_PULLREQUEST_SOURCEBRANCH"
-        echo "Source version: $BUILD_SOURCEVERSION"
-
-        declare -A fuzzers_funcs=(\
-          ["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="FuzzParseAddress" \
-          ["./listeners_fuzz.go"]="FuzzParseNetworkAddress" \
-          ["./replacer_fuzz.go"]="FuzzReplacer" \
-        )
-
-        declare -A fuzzers_targets=(\
-          ["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="parse-address" \
-          ["./listeners_fuzz.go"]="parse-network-address" \
-          ["./replacer_fuzz.go"]="replacer" \
-        )
-
-        fuzz_type="fuzzing"
-
-        for f in $(find . -name \*_fuzz.go); do
-          FUZZER_DIRECTORY=$(dirname "$f")
-          
-          echo "go-fuzz-build func ${fuzzers_funcs[$f]} residing in $f"
-          
-          go-fuzz-build -func "${fuzzers_funcs[$f]}" -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.zip" "$FUZZER_DIRECTORY"
-          
-          fuzzit create job --engine go-fuzz caddyserver/"${fuzzers_targets[$f]}" "$FUZZER_DIRECTORY"/"${fuzzers_targets[$f]}.zip" --api-key "${FUZZIT_API_KEY}" --type "${fuzz_type}" --branch "${SYSTEM_PULLREQUEST_SOURCEBRANCH}" --revision "${BUILD_SOURCEVERSION}"
-          
-          echo "Completed $f"
-        done

.github/workflows/lint.yml

@@ -0,0 +1,23 @@
+name: Lint
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  # From https://github.com/golangci/golangci-lint-action
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: v1.31
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true

.github/workflows/release.yml

@@ -11,21 +11,30 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        go-version: [ 1.14.x ]
+        go: [ '1.15' ]
     runs-on: ${{ matrix.os }}
 
     steps:
     - name: Install Go
-      uses: actions/setup-go@v1
+      uses: actions/setup-go@v2
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: ${{ matrix.go }}
 
     - name: Checkout code
       uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
 
-    # So GoReleaser can generate the changelog properly
+    # Force fetch upstream tags -- because 65 minutes
-    - name: Unshallowify the repo clone
+    # tl;dr: actions/checkout@v2 runs this line:
-      run: git fetch --prune --unshallow
+    #   git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +ebc278ec98bb24f2852b61fde2a9bf2e3d83818b:refs/tags/
+    # which makes its own local lightweight tag, losing all the annotations in the process. Our earlier script ran:
+    #   git fetch --prune --unshallow
+    # which doesn't overwrite that tag because that would be destructive.
+    # Credit to @francislavoie for the investigation.
+    # https://github.com/actions/checkout/issues/290#issuecomment-680260080
+    - name: Force fetch upstream tags
+      run: git fetch --tags --force
 
     # https://github.community/t5/GitHub-Actions/How-to-get-just-the-tag-name/m-p/32167/highlight/true#M1027
     - name: Print Go version and environment
@@ -41,6 +50,9 @@ jobs:
         echo "::set-output name=short_sha::$(git rev-parse --short HEAD)"
         echo "::set-output name=go_cache::$(go env GOCACHE)"
 
+        # Add "pip install" CLI tools to PATH
+        echo ~/.local/bin >> $GITHUB_PATH
+
         # Parse semver
         TAG=${GITHUB_REF/refs\/tags\//}
         SEMVER_RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z\.-]*\)'
@@ -53,17 +65,32 @@ jobs:
         echo "::set-output name=tag_patch::${TAG_PATCH}"
         echo "::set-output name=tag_special::${TAG_SPECIAL}"
 
+    # Cloudsmith CLI tooling for pushing releases
+    # See https://help.cloudsmith.io/docs/cli
+    - name: Install Cloudsmith CLI
+      run: pip install --upgrade cloudsmith-cli
+
+    - name: Validate commits and tag signatures
+      run: |
+        
+        # Import Matt Holt's key
+        curl 'https://github.com/mholt.gpg' | gpg --import
+
+        echo "Verifying the tag: ${{ steps.vars.outputs.version_tag }}"
+        # tags are only accepted if signed by Matt's key
+        git verify-tag "${{ steps.vars.outputs.version_tag }}" || exit 1
+
     - name: Cache the build cache
-      uses: actions/cache@v1
+      uses: actions/cache@v2
       with:
         path: ${{ steps.vars.outputs.go_cache }}
-        key: ${{ runner.os }}-go-release-${{ hashFiles('**/go.sum') }}
+        key: ${{ runner.os }}-go${{ matrix.go }}-release-${{ hashFiles('**/go.sum') }}
         restore-keys: |
-          ${{ runner.os }}-go-release
+          ${{ runner.os }}-go${{ matrix.go }}-release
 
     # GoReleaser will take care of publishing those artifacts into the release
     - name: Run GoReleaser
-      uses: goreleaser/goreleaser-action@v1
+      uses: goreleaser/goreleaser-action@v2
       with:
         version: latest
         args: release --rm-dist
@@ -72,12 +99,59 @@ jobs:
         TAG: ${{ steps.vars.outputs.version_tag }}
 
     # Only publish on non-special tags (e.g. non-beta)
+    # We will continue to push to Gemfury for the forseeable future, although
+    # Cloudsmith is probably better, to not break things for existing users of Gemfury.
+    # See https://gemfury.com/caddy/deb:caddy
     - name: Publish .deb to Gemfury
       if: ${{ steps.vars.outputs.tag_special == '' }}
       env:
         GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
       run: |
         for filename in dist/*.deb; do
+          # armv6 and armv7 are both "armhf" so we can skip the duplicate
+          if [[ "$filename" == *"armv6"* ]]; then
+            echo "Skipping $filename"
+            continue
+          fi
+
           curl -F package=@"$filename" https://${GEMFURY_PUSH_TOKEN}:@push.fury.io/caddy/
         done
 
+    # Publish only special tags (unstable/beta/rc) to the "testing" repo
+    # See https://cloudsmith.io/~caddy/repos/testing/
+    - name: Publish .deb to Cloudsmith (special tags)
+      if: ${{ steps.vars.outputs.tag_special != '' }}
+      env:
+        CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }}
+      run: |
+        for filename in dist/*.deb; do
+          # armv6 and armv7 are both "armhf" so we can skip the duplicate
+          if [[ "$filename" == *"armv6"* ]]; then
+            echo "Skipping $filename"
+            continue
+          fi
+
+          echo "Pushing $filename to 'testing'"
+          cloudsmith push deb caddy/testing/any-distro/any-version $filename
+        done
+
+    # Publish stable tags to Cloudsmith to both repos, "stable" and "testing"
+    # See https://cloudsmith.io/~caddy/repos/stable/
+    - name: Publish .deb to Cloudsmith (stable tags)
+      if: ${{ steps.vars.outputs.tag_special == '' }}
+      env:
+        CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }}
+      run: |
+        for filename in dist/*.deb; do
+          # armv6 and armv7 are both "armhf" so we can skip the duplicate
+          if [[ "$filename" == *"armv6"* ]]; then
+            echo "Skipping $filename"
+            continue
+          fi
+
+          echo "Pushing $filename to 'stable'"
+          cloudsmith push deb caddy/stable/any-distro/any-version $filename
+
+          echo "Pushing $filename to 'testing'"
+          cloudsmith push deb caddy/testing/any-distro/any-version $filename
+        done

.github/workflows/release_published.yml

@@ -30,5 +30,5 @@ jobs:
         token: ${{ secrets.REPO_DISPATCH_TOKEN }}
         repository: caddyserver/caddy-docker
         event-type: release-tagged
-        client-payload: '{"tag": "${{ github.release.tag_name }}"}'
+        client-payload: '{"tag": "${{ github.event.release.tag_name }}"}'
 

.gitignore

@@ -7,9 +7,10 @@ Caddyfile
 *.prof
 *.test
 
-# build artifacts
+# build artifacts and helpers
 cmd/caddy/caddy
 cmd/caddy/caddy.exe
+cmd/caddy/setcap*
 
 # mac specific
 .DS_Store

.golangci.yml

@@ -1,21 +1,68 @@
 linters-settings:
   errcheck:
-    ignore: fmt:.*,io/ioutil:^Read.*,github.com/caddyserver/caddy/v2/caddyconfig:RegisterAdapter,github.com/caddyserver/caddy/v2:RegisterModule
+    ignore: fmt:.*,io/ioutil:^Read.*,go.uber.org/zap/zapcore:^Add.*
     ignoretests: true
-  misspell:
-    locale: US
 
 linters:
+  disable-all: true
   enable:
     - bodyclose
-    - prealloc
+    - deadcode
-    - unconvert
     - errcheck
     - gofmt
     - goimports
     - gosec
+    - gosimple
+    - govet
     - ineffassign
     - misspell
+    - prealloc
+    - staticcheck
+    - structcheck
+    - typecheck
+    - unconvert
+    - unused
+    - varcheck
+  # these are implicitly disabled:
+  # - asciicheck
+  # - depguard
+  # - dogsled
+  # - dupl
+  # - exhaustive
+  # - exportloopref
+  # - funlen
+  # - gci
+  # - gochecknoglobals
+  # - gochecknoinits
+  # - gocognit
+  # - goconst
+  # - gocritic
+  # - gocyclo
+  # - godot
+  # - godox
+  # - goerr113
+  # - gofumpt
+  # - goheader
+  # - golint
+  # - gomnd
+  # - gomodguard
+  # - goprintffuncname
+  # - interfacer
+  # - lll
+  # - maligned
+  # - nakedret
+  # - nestif
+  # - nlreturn
+  # - noctx
+  # - nolintlint
+  # - rowserrcheck
+  # - scopelint
+  # - sqlclosecheck
+  # - stylecheck
+  # - testpackage
+  # - unparam
+  # - whitespace
+  # - wsl
 
 run:
   # default concurrency is a available CPU number.

.goreleaser.yml

@@ -84,13 +84,22 @@ nfpms:
       # - rpm
 
     bindir: /usr/bin
-    files:
+    contents:
-      ./caddy-dist/init/caddy.service: /lib/systemd/system/caddy.service
+      - src: ./caddy-dist/init/caddy.service
-      ./caddy-dist/init/caddy-api.service: /lib/systemd/system/caddy-api.service
+        dst: /lib/systemd/system/caddy.service
-      ./caddy-dist/welcome/index.html: /usr/share/caddy/index.html
+        
-      ./caddy-dist/scripts/completions/bash-completion: /etc/bash_completion.d/caddy
+      - src: ./caddy-dist/init/caddy-api.service
-    config_files:
+        dst: /lib/systemd/system/caddy-api.service
-      ./caddy-dist/config/Caddyfile: /etc/caddy/Caddyfile
+      
+      - src: ./caddy-dist/welcome/index.html
+        dst: /usr/share/caddy/index.html
+      
+      - src: ./caddy-dist/scripts/completions/bash-completion
+        dst: /etc/bash_completion.d/caddy
+    
+      - src: ./caddy-dist/config/Caddyfile
+        dst: /etc/caddy/Caddyfile
+        type: config
 
     scripts:
       postinstall: ./caddy-dist/scripts/postinstall.sh
@@ -112,5 +121,6 @@ changelog:
     - '^chore:'
     - '^ci:'
     - '^docs?:'
+    - '^readme:'
     - '^tests?:'
     - '^\w+\s+' # a hack to remove commit messages without colons thus don't correspond to a package

README.md

@@ -1,21 +1,25 @@
 <p align="center">
 	<a href="https://caddyserver.com"><img src="https://user-images.githubusercontent.com/1128849/36338535-05fb646a-136f-11e8-987b-e6901e717d5a.png" alt="Caddy" width="450"></a>
+	<br>
+	<h3 align="center">a <a href="https://zerossl.com"><img src="https://caddyserver.com/resources/images/zerossl-logo.svg" height="28" valign="middle"></a> project</h3>
 </p>
+<hr>
 <h3 align="center">Every site on HTTPS</h3>
 <p align="center">Caddy is an extensible server platform that uses TLS by default.</p>
 <p align="center">
 	<a href="https://github.com/caddyserver/caddy/actions?query=workflow%3ACross-Platform"><img src="https://github.com/caddyserver/caddy/workflows/Cross-Platform/badge.svg"></a>
-	<a href="https://pkg.go.dev/github.com/caddyserver/caddy/v2"><img src="https://img.shields.io/badge/godoc-reference-blue.svg"></a>
+	<a href="https://pkg.go.dev/github.com/caddyserver/caddy/v2"><img src="https://img.shields.io/badge/godoc-reference-%23007d9c.svg"></a>
-	<a href="https://app.fuzzit.dev/orgs/caddyserver-gh/dashboard"><img src="https://app.fuzzit.dev/badge?org_id=caddyserver-gh"></a>
 	<br>
 	<a href="https://twitter.com/caddyserver" title="@caddyserver on Twitter"><img src="https://img.shields.io/badge/twitter-@caddyserver-55acee.svg" alt="@caddyserver on Twitter"></a>
 	<a href="https://caddy.community" title="Caddy Forum"><img src="https://img.shields.io/badge/community-forum-ff69b4.svg" alt="Caddy Forum"></a>
+	<br>
 	<a href="https://sourcegraph.com/github.com/caddyserver/caddy?badge" title="Caddy on Sourcegraph"><img src="https://sourcegraph.com/github.com/caddyserver/caddy/-/badge.svg" alt="Caddy on Sourcegraph"></a>
+	<a href="https://cloudsmith.io/~caddy/repos/"><img src="https://img.shields.io/badge/OSS%20hosting%20by-cloudsmith-blue?logo=cloudsmith" alt="Cloudsmith"></a>
 </p>
 <p align="center">
-	<a href="https://github.com/caddyserver/caddy/releases">Download</a> ·
+	<a href="https://github.com/caddyserver/caddy/releases">Releases</a> ·
 	<a href="https://caddyserver.com/docs/">Documentation</a> ·
-	<a href="https://caddy.community">Community</a>
+	<a href="https://caddy.community">Get Help</a>
 </p>
 
 
@@ -23,6 +27,7 @@
 ### Menu
 
 - [Features](#features)
+- [Install](#install)
 - [Build from source](#build-from-source)
 	- [For development](#for-development)
 	- [With version information and/or plugins](#with-version-information-andor-plugins)
@@ -39,25 +44,32 @@
 </p>
 
 
-## Features
+## [Features](https://caddyserver.com/v2)
 
 - **Easy configuration** with the [Caddyfile](https://caddyserver.com/docs/caddyfile)
 - **Powerful configuration** with its [native JSON config](https://caddyserver.com/docs/json/)
 - **Dynamic configuration** with the [JSON API](https://caddyserver.com/docs/api)
 - [**Config adapters**](https://caddyserver.com/docs/config-adapters) if you don't like JSON
 - **Automatic HTTPS** by default
-	- [Let's Encrypt](https://letsencrypt.org) for public sites
+	- [ZeroSSL](https://zerossl.com) and [Let's Encrypt](https://letsencrypt.org) for public names
 	- Fully-managed local CA for internal names & IPs
 	- Can coordinate with other Caddy instances in a cluster
+	- Multi-issuer fallback
 - **Stays up when other servers go down** due to TLS/OCSP/certificate-related issues
+- **Production-ready** after serving trillions of requests and managing millions of TLS certificates
+- **Scales to tens of thousands of sites** ... and probably more
 - **HTTP/1.1, HTTP/2, and experimental HTTP/3** support
 - **Highly extensible** [modular architecture](https://caddyserver.com/docs/architecture) lets Caddy do anything without bloat
 - **Runs anywhere** with **no external dependencies** (not even libc)
 - Written in Go, a language with higher **memory safety guarantees** than other servers
 - Actually **fun to use**
-- So, so much more to discover
+- So, so much more to [discover](https://caddyserver.com/v2)
 
+## Install
 
+The simplest, cross-platform way is to download from [GitHub Releases](https://github.com/caddyserver/caddy/releases) and place the executable file in your PATH.
+
+For other install options, see https://caddyserver.com/docs/download.
 
 ## Build from source
 
@@ -67,17 +79,41 @@ Requirements:
 
 ### For development
  
+_**Note:** These steps [will not embed proper version information](https://github.com/golang/go/issues/29228). For that, please follow the instructions in the next section._
+
 ```bash
 $ git clone "https://github.com/caddyserver/caddy.git"
 $ cd caddy/cmd/caddy/
 $ go build
 ```
 
-_**Note:** These steps [will not embed proper version information](https://github.com/golang/go/issues/29228). For that, please follow the instructions below._
+When you run Caddy, it may try to bind to low ports unless otherwise specified in your config. If your OS requires elevated privileges, you will need to give your new binary permission to do so. On Linux, this can be done easily with: `sudo setcap cap_net_bind_service=+ep ./caddy`
+
+If you prefer to use `go run` which creates temporary binaries, you can still do this. Make an executable file called `setcap.sh` (or whatever you want) with these contents:
+
+```bash
+#!/bin/sh
+sudo setcap cap_net_bind_service=+ep "$1"
+"$@"
+```
+
+then you can use `go run` like so:
+
+```bash
+$ go run -exec ./setcap.sh main.go
+```
+
+If you don't want to type your password for `setcap`, use `sudo visudo` to edit your sudoers file and allow your user account to run that command without a password, for example:
+
+```
+username ALL=(ALL:ALL) NOPASSWD: /usr/sbin/setcap
+```
+
+replacing `username` with your actual username. Please be careful and only do this if you know what you are doing! We are only qualified to document how to use Caddy, not Go tooling or your computer, and we are providing these instructions for convenience only; please learn how to use your own computer at your own risk and make any needful adjustments.
 
 ### With version information and/or plugins
 
-Using [our builder tool](https://github.com/caddyserver/xcaddy)...
+Using [our builder tool, `xcaddy`](https://github.com/caddyserver/xcaddy)...
 
 ```
 $ xcaddy build
@@ -89,8 +125,8 @@ $ xcaddy build
 2. Change into it: `cd caddy`
 3. Copy [Caddy's main.go](https://github.com/caddyserver/caddy/blob/master/cmd/caddy/main.go) into the empty folder. Add imports for any custom plugins you want to add.
 4. Initialize a Go module: `go mod init caddy`
-5. (Optional) Pin Caddy version: `go get github.com/caddyserver/caddy/v2@TAG` replacing `TAG` with a git tag or commit.
+5. (Optional) Pin Caddy version: `go get github.com/caddyserver/caddy/v2@version` replacing `version` with a git tag or commit.
-6. (Optional) Add plugins by adding their import: `_ "IMPORT_PATH"`
+6. (Optional) Add plugins by adding their import: `_ "import/path/here"`
 7. Compile: `go build`
 
 
@@ -100,7 +136,7 @@ $ xcaddy build
 
 The [Caddy website](https://caddyserver.com/docs/) has documentation that includes tutorials, quick-start guides, reference, and more.
 
-**We recommend that all users do our [Getting Started](https://caddyserver.com/docs/getting-started) guide to become familiar with using Caddy.**
+**We recommend that all users -- regardless of experience level -- do our [Getting Started](https://caddyserver.com/docs/getting-started) guide to become familiar with using Caddy.**
 
 If you've only got a minute, [the website has several quick-start tutorials](https://caddyserver.com/docs/quick-starts) to choose from! However, after finishing a quick-start tutorial, please read more documentation to understand how the software works. 🙂
 
@@ -119,7 +155,7 @@ The primary way to configure Caddy is through [its API](https://caddyserver.com/
 
 Caddy exposes an unprecedented level of control compared to any web server in existence. In Caddy, you are usually setting the actual values of the initialized types in memory that power everything from your HTTP handlers and TLS handshakes to your storage medium. Caddy is also ridiculously extensible, with a powerful plugin system that makes vast improvements over other web servers.
 
-To wield the power of this design, you need to know how the config document is structured. Please see the [our documentation site](https://caddyserver.com/docs/) for details about [Caddy's config structure](https://caddyserver.com/docs/json/).
+To wield the power of this design, you need to know how the config document is structured. Please see [our documentation site](https://caddyserver.com/docs/) for details about [Caddy's config structure](https://caddyserver.com/docs/json/).
 
 Nearly all of Caddy's configuration is contained in a single config document, rather than being scattered across CLI flags and env variables and a configuration file as with other web servers. This makes managing your server config more straightforward and reduces hidden variables/factors.
 
@@ -138,6 +174,8 @@ The docs are also open source. You can contribute to them here: https://github.c
 
 - We **strongly recommend** that all professionals or companies using Caddy get a support contract through [Ardan Labs](https://www.ardanlabs.com/my/contact-us?dd=caddy) before help is needed.
 
+- A [sponsorship](https://github.com/sponsors/mholt) goes a long way!
+
 - Individuals can exchange help for free on our community forum at https://caddy.community. Remember that people give help out of their spare time and good will. The best way to get help is to give it first!
 
 Please use our [issue tracker](https://github.com/caddyserver/caddy/issues) only for bug reports and feature requests, i.e. actionable development items (support questions will usually be referred to the forums).
@@ -146,7 +184,11 @@ Please use our [issue tracker](https://github.com/caddyserver/caddy/issues) only
 
 ## About
 
-**The name "Caddy" is trademarked.** The name of the software is "Caddy", not "Caddy Server" or "CaddyServer". Please call it "Caddy" or, if you wish to clarify, "the Caddy web server". Caddy is a registered trademark of Light Code Labs, LLC.
+**The name "Caddy" is trademarked.** The name of the software is "Caddy", not "Caddy Server" or "CaddyServer". Please call it "Caddy" or, if you wish to clarify, "the Caddy web server". Caddy is a registered trademark of apilayer GmbH.
 
 - _Project on Twitter: [@caddyserver](https://twitter.com/caddyserver)_
 - _Author on Twitter: [@mholt6](https://twitter.com/mholt6)_
+
+Caddy is a project of [ZeroSSL](https://zerossl.com), an [apilayer](https://apilayer.com) company.
+
+Debian package repository hosting is graciously provided by [Cloudsmith](https://cloudsmith.com). Cloudsmith is the only fully hosted, cloud-native, universal package management solution, that enables your organization to create, store and share packages in any format, to any place, with total confidence.

admin.go

@@ -18,6 +18,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"expvar"
 	"fmt"
 	"io"
@@ -34,6 +35,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
 )
 
@@ -106,34 +108,53 @@ func (admin AdminConfig) newAdminHandler(addr NetworkAddress) adminHandler {
 		mux:            http.NewServeMux(),
 	}
 
+	addRouteWithMetrics := func(pattern string, handlerLabel string, h http.Handler) {
+		labels := prometheus.Labels{"path": pattern, "handler": handlerLabel}
+		h = instrumentHandlerCounter(
+			adminMetrics.requestCount.MustCurryWith(labels),
+			h,
+		)
+		muxWrap.mux.Handle(pattern, h)
+	}
 	// addRoute just calls muxWrap.mux.Handle after
 	// wrapping the handler with error handling
-	addRoute := func(pattern string, h AdminHandler) {
+	addRoute := func(pattern string, handlerLabel string, h AdminHandler) {
 		wrapper := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			err := h.ServeHTTP(w, r)
+			if err != nil {
+				labels := prometheus.Labels{
+					"path":    pattern,
+					"handler": handlerLabel,
+					"method":  strings.ToUpper(r.Method),
+				}
+				adminMetrics.requestErrors.With(labels).Inc()
+			}
 			muxWrap.handleError(w, r, err)
 		})
-		muxWrap.mux.Handle(pattern, wrapper)
+		addRouteWithMetrics(pattern, handlerLabel, wrapper)
 	}
 
+	const handlerLabel = "admin"
+
 	// register standard config control endpoints
-	addRoute("/"+rawConfigKey+"/", AdminHandlerFunc(handleConfig))
+	addRoute("/"+rawConfigKey+"/", handlerLabel, AdminHandlerFunc(handleConfig))
-	addRoute("/id/", AdminHandlerFunc(handleConfigID))
+	addRoute("/id/", handlerLabel, AdminHandlerFunc(handleConfigID))
-	addRoute("/stop", AdminHandlerFunc(handleStop))
+	addRoute("/stop", handlerLabel, AdminHandlerFunc(handleStop))
 
 	// register debugging endpoints
-	muxWrap.mux.HandleFunc("/debug/pprof/", pprof.Index)
+	addRouteWithMetrics("/debug/pprof/", handlerLabel, http.HandlerFunc(pprof.Index))
-	muxWrap.mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+	addRouteWithMetrics("/debug/pprof/cmdline", handlerLabel, http.HandlerFunc(pprof.Cmdline))
-	muxWrap.mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+	addRouteWithMetrics("/debug/pprof/profile", handlerLabel, http.HandlerFunc(pprof.Profile))
-	muxWrap.mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+	addRouteWithMetrics("/debug/pprof/symbol", handlerLabel, http.HandlerFunc(pprof.Symbol))
-	muxWrap.mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+	addRouteWithMetrics("/debug/pprof/trace", handlerLabel, http.HandlerFunc(pprof.Trace))
-	muxWrap.mux.Handle("/debug/vars", expvar.Handler())
+	addRouteWithMetrics("/debug/vars", handlerLabel, expvar.Handler())
 
 	// register third-party module endpoints
 	for _, m := range GetModules("admin.api") {
 		router := m.New().(AdminRouter)
+		handlerLabel := m.ID.Name()
 		for _, route := range router.Routes() {
-			addRoute(route.Pattern, route.Handler)
+			addRoute(route.Pattern, handlerLabel, route.Handler)
 		}
 	}
 
@@ -235,15 +256,20 @@ func replaceAdmin(cfg *Config) error {
 		MaxHeaderBytes:    1024 * 64,
 	}
 
-	go adminServer.Serve(ln)
+	adminLogger := Log().Named("admin")
+	go func() {
+		if err := adminServer.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
+			adminLogger.Error("admin server shutdown for unknown reason", zap.Error(err))
+		}
+	}()
 
-	Log().Named("admin").Info("admin endpoint started",
+	adminLogger.Info("admin endpoint started",
 		zap.String("address", addr.String()),
 		zap.Bool("enforce_origin", adminConfig.EnforceOrigin),
 		zap.Strings("origins", handler.allowedOrigins))
 
 	if !handler.enforceHost {
-		Log().Named("admin").Warn("admin endpoint on open interface; host checking disabled",
+		adminLogger.Warn("admin endpoint on open interface; host checking disabled",
 			zap.String("address", addr.String()))
 	}
 
@@ -285,13 +311,18 @@ type adminHandler struct {
 // ServeHTTP is the external entry point for API requests.
 // It will only be called once per request.
 func (h adminHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	Log().Named("admin.api").Info("received request",
+	log := Log().Named("admin.api").With(
 		zap.String("method", r.Method),
 		zap.String("host", r.Host),
 		zap.String("uri", r.RequestURI),
 		zap.String("remote_addr", r.RemoteAddr),
 		zap.Reflect("headers", r.Header),
 	)
+	if r.RequestURI == "/metrics" {
+		log.Debug("received request")
+	} else {
+		log.Info("received request")
+	}
 	h.serveHTTP(w, r)
 }
 
@@ -367,7 +398,10 @@ func (h adminHandler) handleError(w http.ResponseWriter, r *http.Request, err er
 
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(apiErr.Code)
-	json.NewEncoder(w).Encode(apiErr)
+	encErr := json.NewEncoder(w).Encode(apiErr)
+	if encErr != nil {
+		Log().Named("admin.api").Error("failed to encode error response", zap.Error(encErr))
+	}
 }
 
 // checkHost returns a handler that wraps next such that
@@ -807,7 +841,7 @@ var (
 // will get deleted before the process gracefully exits.
 func PIDFile(filename string) error {
 	pid := []byte(strconv.Itoa(os.Getpid()) + "\n")
-	err := ioutil.WriteFile(filename, pid, 0644)
+	err := ioutil.WriteFile(filename, pid, 0600)
 	if err != nil {
 		return err
 	}

caddy.go

@@ -471,7 +471,7 @@ func stopAndCleanup() error {
 	}
 	certmagic.CleanUpOwnLocks()
 	if pidfile != "" {
-		os.Remove(pidfile)
+		return os.Remove(pidfile)
 	}
 	return nil
 }

caddyconfig/caddyfile/formatter.go

@@ -78,6 +78,8 @@ func Format(input []byte) []byte {
 		if comment {
 			if ch == '\n' {
 				comment = false
+				nextLine()
+				continue
 			} else {
 				write(ch)
 				continue

caddyconfig/caddyfile/formatter_test.go

@@ -310,6 +310,17 @@ baz`,
 			input:       `redir / /some/#/path`,
 			expect:      `redir / /some/#/path`,
 		},
+		{
+			description: "brace does not fold into comment above",
+			input: `# comment
+{
+	foo
+}`,
+			expect: `# comment
+{
+	foo
+}`,
+		},
 	} {
 		// the formatter should output a trailing newline,
 		// even if the tests aren't written to expect that

caddyconfig/caddyfile/lexer.go

@@ -16,6 +16,7 @@ package caddyfile
 
 import (
 	"bufio"
+	"bytes"
 	"io"
 	"unicode"
 )
@@ -168,3 +169,21 @@ func (l *lexer) next() bool {
 		val = append(val, ch)
 	}
 }
+
+// Tokenize takes bytes as input and lexes it into
+// a list of tokens that can be parsed as a Caddyfile.
+// Also takes a filename to fill the token's File as
+// the source of the tokens, which is important to
+// determine relative paths for `import` directives.
+func Tokenize(input []byte, filename string) ([]Token, error) {
+	l := lexer{}
+	if err := l.load(bytes.NewReader(input)); err != nil {
+		return nil, err
+	}
+	var tokens []Token
+	for l.next() {
+		l.token.File = filename
+		tokens = append(tokens, l.token)
+	}
+	return tokens, nil
+}

caddyconfig/caddyfile/lexer_test.go

@@ -15,37 +15,35 @@
 package caddyfile
 
 import (
-	"log"
-	"strings"
 	"testing"
 )
 
 type lexerTestCase struct {
-	input    string
+	input    []byte
 	expected []Token
 }
 
 func TestLexer(t *testing.T) {
 	testCases := []lexerTestCase{
 		{
-			input: `host:123`,
+			input: []byte(`host:123`),
 			expected: []Token{
 				{Line: 1, Text: "host:123"},
 			},
 		},
 		{
-			input: `host:123
+			input: []byte(`host:123
 
-					directive`,
+					directive`),
 			expected: []Token{
 				{Line: 1, Text: "host:123"},
 				{Line: 3, Text: "directive"},
 			},
 		},
 		{
-			input: `host:123 {
+			input: []byte(`host:123 {
 						directive
-					}`,
+					}`),
 			expected: []Token{
 				{Line: 1, Text: "host:123"},
 				{Line: 1, Text: "{"},
@@ -54,7 +52,7 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: `host:123 { directive }`,
+			input: []byte(`host:123 { directive }`),
 			expected: []Token{
 				{Line: 1, Text: "host:123"},
 				{Line: 1, Text: "{"},
@@ -63,12 +61,12 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: `host:123 {
+			input: []byte(`host:123 {
 						#comment
 						directive
 						# comment
 						foobar # another comment
-					}`,
+					}`),
 			expected: []Token{
 				{Line: 1, Text: "host:123"},
 				{Line: 1, Text: "{"},
@@ -78,10 +76,10 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: `host:123 {
+			input: []byte(`host:123 {
 						# hash inside string is not a comment
 						redir / /some/#/path
-					}`,
+					}`),
 			expected: []Token{
 				{Line: 1, Text: "host:123"},
 				{Line: 1, Text: "{"},
@@ -92,14 +90,14 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: "# comment at beginning of file\n# comment at beginning of line\nhost:123",
+			input: []byte("# comment at beginning of file\n# comment at beginning of line\nhost:123"),
 			expected: []Token{
 				{Line: 3, Text: "host:123"},
 			},
 		},
 		{
-			input: `a "quoted value" b
+			input: []byte(`a "quoted value" b
-					foobar`,
+					foobar`),
 			expected: []Token{
 				{Line: 1, Text: "a"},
 				{Line: 1, Text: "quoted value"},
@@ -108,7 +106,7 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: `A "quoted \"value\" inside" B`,
+			input: []byte(`A "quoted \"value\" inside" B`),
 			expected: []Token{
 				{Line: 1, Text: "A"},
 				{Line: 1, Text: `quoted "value" inside`},
@@ -116,7 +114,7 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: "An escaped \"newline\\\ninside\" quotes",
+			input: []byte("An escaped \"newline\\\ninside\" quotes"),
 			expected: []Token{
 				{Line: 1, Text: "An"},
 				{Line: 1, Text: "escaped"},
@@ -125,7 +123,7 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: "An escaped newline\\\noutside quotes",
+			input: []byte("An escaped newline\\\noutside quotes"),
 			expected: []Token{
 				{Line: 1, Text: "An"},
 				{Line: 1, Text: "escaped"},
@@ -135,7 +133,7 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: "line1\\\nescaped\nline2\nline3",
+			input: []byte("line1\\\nescaped\nline2\nline3"),
 			expected: []Token{
 				{Line: 1, Text: "line1"},
 				{Line: 1, Text: "escaped"},
@@ -144,7 +142,7 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: "line1\\\nescaped1\\\nescaped2\nline4\nline5",
+			input: []byte("line1\\\nescaped1\\\nescaped2\nline4\nline5"),
 			expected: []Token{
 				{Line: 1, Text: "line1"},
 				{Line: 1, Text: "escaped1"},
@@ -154,34 +152,34 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: `"unescapable\ in quotes"`,
+			input: []byte(`"unescapable\ in quotes"`),
 			expected: []Token{
 				{Line: 1, Text: `unescapable\ in quotes`},
 			},
 		},
 		{
-			input: `"don't\escape"`,
+			input: []byte(`"don't\escape"`),
 			expected: []Token{
 				{Line: 1, Text: `don't\escape`},
 			},
 		},
 		{
-			input: `"don't\\escape"`,
+			input: []byte(`"don't\\escape"`),
 			expected: []Token{
 				{Line: 1, Text: `don't\\escape`},
 			},
 		},
 		{
-			input: `un\escapable`,
+			input: []byte(`un\escapable`),
 			expected: []Token{
 				{Line: 1, Text: `un\escapable`},
 			},
 		},
 		{
-			input: `A "quoted value with line
+			input: []byte(`A "quoted value with line
 					break inside" {
 						foobar
-					}`,
+					}`),
 			expected: []Token{
 				{Line: 1, Text: "A"},
 				{Line: 1, Text: "quoted value with line\n\t\t\t\t\tbreak inside"},
@@ -191,13 +189,13 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: `"C:\php\php-cgi.exe"`,
+			input: []byte(`"C:\php\php-cgi.exe"`),
 			expected: []Token{
 				{Line: 1, Text: `C:\php\php-cgi.exe`},
 			},
 		},
 		{
-			input: `empty "" string`,
+			input: []byte(`empty "" string`),
 			expected: []Token{
 				{Line: 1, Text: `empty`},
 				{Line: 1, Text: ``},
@@ -205,7 +203,7 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: "skip those\r\nCR characters",
+			input: []byte("skip those\r\nCR characters"),
 			expected: []Token{
 				{Line: 1, Text: "skip"},
 				{Line: 1, Text: "those"},
@@ -214,13 +212,13 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: "\xEF\xBB\xBF:8080", // test with leading byte order mark
+			input: []byte("\xEF\xBB\xBF:8080"), // test with leading byte order mark
 			expected: []Token{
 				{Line: 1, Text: ":8080"},
 			},
 		},
 		{
-			input: "simple `backtick quoted` string",
+			input: []byte("simple `backtick quoted` string"),
 			expected: []Token{
 				{Line: 1, Text: `simple`},
 				{Line: 1, Text: `backtick quoted`},
@@ -228,7 +226,7 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: "multiline `backtick\nquoted\n` string",
+			input: []byte("multiline `backtick\nquoted\n` string"),
 			expected: []Token{
 				{Line: 1, Text: `multiline`},
 				{Line: 1, Text: "backtick\nquoted\n"},
@@ -236,7 +234,7 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: "nested `\"quotes inside\" backticks` string",
+			input: []byte("nested `\"quotes inside\" backticks` string"),
 			expected: []Token{
 				{Line: 1, Text: `nested`},
 				{Line: 1, Text: `"quotes inside" backticks`},
@@ -244,7 +242,7 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			input: "reverse-nested \"`backticks` inside\" quotes",
+			input: []byte("reverse-nested \"`backticks` inside\" quotes"),
 			expected: []Token{
 				{Line: 1, Text: `reverse-nested`},
 				{Line: 1, Text: "`backticks` inside"},
@@ -254,22 +252,14 @@ func TestLexer(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		actual := tokenize(testCase.input)
+		actual, err := Tokenize(testCase.input, "")
+		if err != nil {
+			t.Errorf("%v", err)
+		}
 		lexerCompare(t, i, testCase.expected, actual)
 	}
 }
 
-func tokenize(input string) (tokens []Token) {
-	l := lexer{}
-	if err := l.load(strings.NewReader(input)); err != nil {
-		log.Printf("[ERROR] load failed: %v", err)
-	}
-	for l.next() {
-		tokens = append(tokens, l.token)
-	}
-	return
-}
-
 func lexerCompare(t *testing.T, n int, expected, actual []Token) {
 	if len(expected) != len(actual) {
 		t.Errorf("Test case %d: expected %d token(s) but got %d", n, len(expected), len(actual))

caddyconfig/caddyfile/parse.go

@@ -60,21 +60,31 @@ func replaceEnvVars(input []byte) ([]byte, error) {
 		end += begin + len(spanOpen) // make end relative to input, not begin
 
 		// get the name; if there is no name, skip it
-		envVarName := input[begin+len(spanOpen) : end]
+		envString := input[begin+len(spanOpen) : end]
-		if len(envVarName) == 0 {
+		if len(envString) == 0 {
 			offset = end + len(spanClose)
 			continue
 		}
 
+		// split the string into a key and an optional default
+		envParts := strings.SplitN(string(envString), envVarDefaultDelimiter, 2)
+
+		// do a lookup for the env var, replace with the default if not found
+		envVarValue, found := os.LookupEnv(envParts[0])
+		if !found && len(envParts) == 2 {
+			envVarValue = envParts[1]
+		}
+
 		// get the value of the environment variable
-		envVarValue := []byte(os.ExpandEnv(os.Getenv(string(envVarName))))
+		// note that this causes one-level deep chaining
+		envVarBytes := []byte(envVarValue)
 
 		// splice in the value
 		input = append(input[:begin],
-			append(envVarValue, input[end+len(spanClose):]...)...)
+			append(envVarBytes, input[end+len(spanClose):]...)...)
 
 		// continue at the end of the replacement
-		offset = begin + len(envVarValue)
+		offset = begin + len(envVarBytes)
 	}
 	return input, nil
 }
@@ -87,16 +97,10 @@ func allTokens(filename string, input []byte) ([]Token, error) {
 	if err != nil {
 		return nil, err
 	}
-	l := new(lexer)
+	tokens, err := Tokenize(input, filename)
-	err = l.load(bytes.NewReader(input))
 	if err != nil {
 		return nil, err
 	}
-	var tokens []Token
-	for l.next() {
-		l.token.File = filename
-		tokens = append(tokens, l.token)
-	}
 	return tokens, nil
 }
 
@@ -554,4 +558,7 @@ func (s Segment) Directive() string {
 
 // spanOpen and spanClose are used to bound spans that
 // contain the name of an environment variable.
-var spanOpen, spanClose = []byte{'{', '$'}, []byte{'}'}
+var (
+	spanOpen, spanClose    = []byte{'{', '$'}, []byte{'}'}
+	envVarDefaultDelimiter = ":"
+)

caddyconfig/caddyfile/parse_test.go

@@ -478,6 +478,7 @@ func TestParseAll(t *testing.T) {
 
 func TestEnvironmentReplacement(t *testing.T) {
 	os.Setenv("FOOBAR", "foobar")
+	os.Setenv("CHAINED", "$FOOBAR")
 
 	for i, test := range []struct {
 		input  string
@@ -523,6 +524,22 @@ func TestEnvironmentReplacement(t *testing.T) {
 			input:  "{$FOOBAR}{$FOOBAR}",
 			expect: "foobarfoobar",
 		},
+		{
+			input:  "{$CHAINED}",
+			expect: "$FOOBAR", // should not chain env expands
+		},
+		{
+			input:  "{$FOO:default}",
+			expect: "default",
+		},
+		{
+			input:  "foo{$BAR:bar}baz",
+			expect: "foobarbaz",
+		},
+		{
+			input:  "foo{$BAR:$FOOBAR}baz",
+			expect: "foo$FOOBARbaz", // should not chain env expands
+		},
 		{
 			input:  "{$FOOBAR",
 			expect: "{$FOOBAR",

caddyconfig/httpcaddyfile/addresses.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"net"
 	"reflect"
+	"sort"
 	"strconv"
 	"strings"
 	"unicode"
@@ -163,6 +164,13 @@ func (st *ServerType) consolidateAddrMappings(addrToServerBlocks map[string][]se
 
 		sbaddrs = append(sbaddrs, a)
 	}
+
+	// sort them by their first address (we know there will always be at least one)
+	// to avoid problems with non-deterministic ordering (makes tests flaky)
+	sort.Slice(sbaddrs, func(i, j int) bool {
+		return sbaddrs[i].addresses[0] < sbaddrs[j].addresses[0]
+	})
+
 	return sbaddrs
 }
 

caddyconfig/httpcaddyfile/builtins.go

@@ -29,6 +29,8 @@ import (
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
+	"github.com/caddyserver/certmagic"
+	"github.com/mholt/acmez/acme"
 	"go.uber.org/zap/zapcore"
 )
 
@@ -73,8 +75,10 @@ func parseBind(h Helper) ([]ConfigValue, error) {
 //         load      <paths...>
 //         ca        <acme_ca_endpoint>
 //         ca_root   <pem_file>
-//         dns       <provider_name>
+//         dns       <provider_name> [...]
 //         on_demand
+//         eab    <key_id> <mac_key>
+//         issuer <module_name> [...]
 //     }
 //
 func parseTLS(h Helper) ([]ConfigValue, error) {
@@ -84,6 +88,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 	var certSelector caddytls.CustomCertSelectionPolicy
 	var acmeIssuer *caddytls.ACMEIssuer
 	var internalIssuer *caddytls.InternalIssuer
+	var issuers []certmagic.Issuer
 	var onDemand bool
 
 	for h.Next() {
@@ -262,6 +267,42 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 				}
 				acmeIssuer.CA = arg[0]
 
+			case "eab":
+				arg := h.RemainingArgs()
+				if len(arg) != 2 {
+					return nil, h.ArgErr()
+				}
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
+				acmeIssuer.ExternalAccount = &acme.EAB{
+					KeyID:  arg[0],
+					MACKey: arg[1],
+				}
+
+			case "issuer":
+				if !h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				modName := h.Val()
+				mod, err := caddy.GetModule("tls.issuance." + modName)
+				if err != nil {
+					return nil, h.Errf("getting issuer module '%s': %v", modName, err)
+				}
+				unm, ok := mod.New().(caddyfile.Unmarshaler)
+				if !ok {
+					return nil, h.Errf("issuer module '%s' is not a Caddyfile unmarshaler", mod.ID)
+				}
+				err = unm.UnmarshalCaddyfile(h.NewFromNextSegment())
+				if err != nil {
+					return nil, err
+				}
+				issuer, ok := unm.(certmagic.Issuer)
+				if !ok {
+					return nil, h.Errf("module %s is not a certmagic.Issuer", mod.ID)
+				}
+				issuers = append(issuers, issuer)
+
 			case "dns":
 				if !h.NextArg() {
 					return nil, h.ArgErr()
@@ -315,7 +356,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 	}
 
 	// begin building the final config values
-	var configVals []ConfigValue
+	configVals := []ConfigValue{}
 
 	// certificate loaders
 	if len(fileLoader) > 0 {
@@ -331,28 +372,25 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 		})
 	}
 
-	// issuer
+	if len(issuers) > 0 && (acmeIssuer != nil || internalIssuer != nil) {
-	if acmeIssuer != nil && internalIssuer != nil {
+		// some tls subdirectives are shortcuts that implicitly configure issuers, and the
-		// the logic to support this would be complex
+		// user can also configure issuers explicitly using the issuer subdirective; the
-		return nil, h.Err("cannot use both ACME and internal issuers in same server block")
+		// logic to support both would likely be complex, or at least unintuitive
+		return nil, h.Err("cannot mix issuer subdirective (explicit issuers) with other issuer-specific subdirectives (implicit issuers)")
 	}
-	if acmeIssuer != nil {
+	for _, issuer := range issuers {
-		// fill in global defaults, if configured
-		if email := h.Option("email"); email != nil && acmeIssuer.Email == "" {
-			acmeIssuer.Email = email.(string)
-		}
-		if acmeCA := h.Option("acme_ca"); acmeCA != nil && acmeIssuer.CA == "" {
-			acmeIssuer.CA = acmeCA.(string)
-		}
-		if caPemFile := h.Option("acme_ca_root"); caPemFile != nil {
-			acmeIssuer.TrustedRootsPEMFiles = append(acmeIssuer.TrustedRootsPEMFiles, caPemFile.(string))
-		}
-
 		configVals = append(configVals, ConfigValue{
 			Class: "tls.cert_issuer",
-			Value: acmeIssuer,
+			Value: issuer,
 		})
-	} else if internalIssuer != nil {
+	}
+	if acmeIssuer != nil {
+		configVals = append(configVals, ConfigValue{
+			Class: "tls.cert_issuer",
+			Value: disambiguateACMEIssuer(acmeIssuer),
+		})
+	}
+	if internalIssuer != nil {
 		configVals = append(configVals, ConfigValue{
 			Class: "tls.cert_issuer",
 			Value: internalIssuer,
@@ -466,36 +504,23 @@ func parseRespond(h Helper) (caddyhttp.MiddlewareHandler, error) {
 func parseRoute(h Helper) (caddyhttp.MiddlewareHandler, error) {
 	sr := new(caddyhttp.Subroute)
 
-	for h.Next() {
+	allResults, err := parseSegmentAsConfig(h)
-		for nesting := h.Nesting(); h.NextBlock(nesting); {
+	if err != nil {
-			dir := h.Val()
+		return nil, err
+	}
 
-			dirFunc, ok := registeredDirectives[dir]
+	for _, result := range allResults {
-			if !ok {
+		switch handler := result.Value.(type) {
-				return nil, h.Errf("unrecognized directive: %s", dir)
+		case caddyhttp.Route:
-			}
+			sr.Routes = append(sr.Routes, handler)
-
+		case caddyhttp.Subroute:
-			subHelper := h
+			// directives which return a literal subroute instead of a route
-			subHelper.Dispenser = h.NewFromNextSegment()
+			// means they intend to keep those handlers together without
-
+			// them being reordered; we're doing that anyway since we're in
-			results, err := dirFunc(subHelper)
+			// the route directive, so just append its handlers
-			if err != nil {
+			sr.Routes = append(sr.Routes, handler.Routes...)
-				return nil, h.Errf("parsing caddyfile tokens for '%s': %v", dir, err)
+		default:
-			}
+			return nil, h.Errf("%s directive returned something other than an HTTP route or subroute: %#v (only handler directives can be used in routes)", result.directive, result.Value)
-			for _, result := range results {
-				switch handler := result.Value.(type) {
-				case caddyhttp.Route:
-					sr.Routes = append(sr.Routes, handler)
-				case caddyhttp.Subroute:
-					// directives which return a literal subroute instead of a route
-					// means they intend to keep those handlers together without
-					// them being reordered; we're doing that anyway since we're in
-					// the route directive, so just append its handlers
-					sr.Routes = append(sr.Routes, handler.Routes...)
-				default:
-					return nil, h.Errf("%s directive returned something other than an HTTP route or subroute: %#v (only handler directives can be used in routes)", dir, result.Value)
-				}
-			}
 		}
 	}
 

caddyconfig/httpcaddyfile/directives.go

@@ -41,6 +41,7 @@ var directiveOrder = []string{
 	"root",
 
 	"header",
+	"request_body",
 
 	"redir",
 	"rewrite",
@@ -55,13 +56,15 @@ var directiveOrder = []string{
 	"encode",
 	"templates",
 
-	// special routing directives
+	// special routing & dispatching directives
 	"handle",
 	"handle_path",
 	"route",
+	"push",
 
 	// handlers that typically respond to requests
 	"respond",
+	"metrics",
 	"reverse_proxy",
 	"php_fastcgi",
 	"file_server",
@@ -100,20 +103,11 @@ func RegisterHandlerDirective(dir string, setupFunc UnmarshalHandlerFunc) {
 			return nil, h.ArgErr()
 		}
 
-		matcherSet, ok, err := h.MatcherToken()
+		matcherSet, err := h.ExtractMatcherSet()
 		if err != nil {
 			return nil, err
 		}
-		if ok {
-			// strip matcher token; we don't need to
-			// use the return value here because a
-			// new dispenser should have been made
-			// solely for this directive's tokens,
-			// with no other uses of same slice
-			h.Dispenser.Delete()
-		}
 
-		h.Dispenser.Reset() // pretend this lookahead never happened
 		val, err := setupFunc(h)
 		if err != nil {
 			return nil, err
@@ -198,7 +192,12 @@ func (h Helper) ExtractMatcherSet() (caddy.ModuleMap, error) {
 		return nil, err
 	}
 	if hasMatcher {
-		h.Dispenser.Delete() // strip matcher token
+		// strip matcher token; we don't need to
+		// use the return value here because a
+		// new dispenser should have been made
+		// solely for this directive's tokens,
+		// with no other uses of same slice
+		h.Dispenser.Delete()
 	}
 	h.Dispenser.Reset() // pretend this lookahead never happened
 	return matcherSet, nil
@@ -268,9 +267,26 @@ func (h Helper) NewBindAddresses(addrs []string) []ConfigValue {
 // are themselves treated as directives, from which a subroute is built
 // and returned.
 func ParseSegmentAsSubroute(h Helper) (caddyhttp.MiddlewareHandler, error) {
+	allResults, err := parseSegmentAsConfig(h)
+	if err != nil {
+		return nil, err
+	}
+
+	return buildSubroute(allResults, h.groupCounter)
+}
+
+// parseSegmentAsConfig parses the segment such that its subdirectives
+// are themselves treated as directives, including named matcher definitions,
+// and the raw Config structs are returned.
+func parseSegmentAsConfig(h Helper) ([]ConfigValue, error) {
 	var allResults []ConfigValue
 
 	for h.Next() {
+		// don't allow non-matcher args on the first line
+		if h.NextArg() {
+			return nil, h.ArgErr()
+		}
+
 		// slice the linear list of tokens into top-level segments
 		var segments []caddyfile.Segment
 		for nesting := h.Nesting(); h.NextBlock(nesting); {
@@ -285,13 +301,17 @@ func ParseSegmentAsSubroute(h Helper) (caddyhttp.MiddlewareHandler, error) {
 		}
 
 		// find and extract any embedded matcher definitions in this scope
-		for i, seg := range segments {
+		for i := 0; i < len(segments); i++ {
+			seg := segments[i]
 			if strings.HasPrefix(seg.Directive(), matcherPrefix) {
+				// parse, then add the matcher to matcherDefs
 				err := parseMatcherDefinitions(caddyfile.NewDispenser(seg), matcherDefs)
 				if err != nil {
 					return nil, err
 				}
+				// remove the matcher segment (consumed), then step back the loop
 				segments = append(segments[:i], segments[i+1:]...)
+				i--
 			}
 		}
 
@@ -318,7 +338,7 @@ func ParseSegmentAsSubroute(h Helper) (caddyhttp.MiddlewareHandler, error) {
 		}
 	}
 
-	return buildSubroute(allResults, h.groupCounter)
+	return allResults, nil
 }
 
 // ConfigValue represents a value to be added to the final
@@ -385,6 +405,14 @@ func sortRoutes(routes []ConfigValue) {
 		if len(jPM) > 0 {
 			jPathLen = len(jPM[0])
 		}
+
+		// if both directives have no path matcher, use whichever one
+		// has any kind of matcher defined first.
+		if iPathLen == 0 && jPathLen == 0 {
+			return len(iRoute.MatcherSetsRaw) > 0 && len(jRoute.MatcherSetsRaw) == 0
+		}
+
+		// sort with the most-specific (longest) path first
 		return iPathLen > jPathLen
 	})
 }

caddyconfig/httpcaddyfile/httptype.go

@@ -99,6 +99,7 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		"{tls_client_issuer}", "{http.request.tls.client.issuer}",
 		"{tls_client_serial}", "{http.request.tls.client.serial}",
 		"{tls_client_subject}", "{http.request.tls.client.subject}",
+		"{tls_client_certificate_pem}", "{http.request.tls.client.certificate_pem}",
 	)
 
 	// these are placeholders that allow a user-defined final
@@ -172,6 +173,15 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 			if err != nil {
 				return nil, warnings, fmt.Errorf("parsing caddyfile tokens for '%s': %v", dir, err)
 			}
+
+			// As a special case, we want "handle_path" to be sorted
+			// at the same level as "handle", so we force them to use
+			// the same directive name after their parsing is complete.
+			// See https://github.com/caddyserver/caddy/issues/3675#issuecomment-678042377
+			if dir == "handle_path" {
+				dir = "handle"
+			}
+
 			for _, result := range results {
 				result.directive = dir
 				sb.pile[result.Class] = append(sb.pile[result.Class], result)
@@ -208,13 +218,6 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		return nil, warnings, err
 	}
 
-	// if experimental HTTP/3 is enabled, enable it on each server
-	if enableH3, ok := options["experimental_http3"].(bool); ok && enableH3 {
-		for _, srv := range httpApp.Servers {
-			srv.ExperimentalHTTP3 = true
-		}
-	}
-
 	// extract any custom logs, and enforce configured levels
 	var customLogs []namedCustomLog
 	var hasDefaultLog bool
@@ -261,12 +264,8 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 			storageCvtr.(caddy.Module).CaddyModule().ID.Name(),
 			&warnings)
 	}
-	if adminConfig, ok := options["admin"].(string); ok && adminConfig != "" {
+	if adminConfig, ok := options["admin"].(*caddy.AdminConfig); ok && adminConfig != nil {
-		if adminConfig == "off" {
+		cfg.Admin = adminConfig
-			cfg.Admin = &caddy.AdminConfig{Disabled: true}
-		} else {
-			cfg.Admin = &caddy.AdminConfig{Listen: adminConfig}
-		}
 	}
 	if len(customLogs) > 0 {
 		if cfg.Logging == nil {
@@ -305,23 +304,54 @@ func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options
 	}
 
 	for _, segment := range serverBlocks[0].block.Segments {
-		dir := segment.Directive()
+		opt := segment.Directive()
 		var val interface{}
 		var err error
 		disp := caddyfile.NewDispenser(segment)
 
-		dirFunc, ok := registeredGlobalOptions[dir]
+		optFunc, ok := registeredGlobalOptions[opt]
 		if !ok {
 			tkn := segment[0]
-			return nil, fmt.Errorf("%s:%d: unrecognized global option: %s", tkn.File, tkn.Line, dir)
+			return nil, fmt.Errorf("%s:%d: unrecognized global option: %s", tkn.File, tkn.Line, opt)
 		}
 
-		val, err = dirFunc(disp)
+		val, err = optFunc(disp)
 		if err != nil {
-			return nil, fmt.Errorf("parsing caddyfile tokens for '%s': %v", dir, err)
+			return nil, fmt.Errorf("parsing caddyfile tokens for '%s': %v", opt, err)
 		}
 
-		options[dir] = val
+		// As a special case, fold multiple "servers" options together
+		// in an array instead of overwriting a possible existing value
+		if opt == "servers" {
+			existingOpts, ok := options[opt].([]serverOptions)
+			if !ok {
+				existingOpts = []serverOptions{}
+			}
+			serverOpts, ok := val.(serverOptions)
+			if !ok {
+				return nil, fmt.Errorf("unexpected type from 'servers' global options")
+			}
+			options[opt] = append(existingOpts, serverOpts)
+			continue
+		}
+
+		options[opt] = val
+	}
+
+	// If we got "servers" options, we'll sort them by their listener address
+	if serverOpts, ok := options["servers"].([]serverOptions); ok {
+		sort.Slice(serverOpts, func(i, j int) bool {
+			return len(serverOpts[i].ListenerAddress) > len(serverOpts[j].ListenerAddress)
+		})
+
+		// Reject the config if there are duplicate listener address
+		seen := make(map[string]bool)
+		for _, entry := range serverOpts {
+			if _, alreadySeen := seen[entry.ListenerAddress]; alreadySeen {
+				return nil, fmt.Errorf("cannot have 'servers' global options with duplicate listener addresses: %s", entry.ListenerAddress)
+			}
+			seen[entry.ListenerAddress] = true
+		}
 	}
 
 	return serverBlocks[1:], nil
@@ -420,6 +450,15 @@ func (st *ServerType) serversFromPairings(
 		var hasCatchAllTLSConnPolicy, addressQualifiesForTLS bool
 		autoHTTPSWillAddConnPolicy := autoHTTPS != "off"
 
+		// if a catch-all server block (one which accepts all hostnames) exists in this pairing,
+		// we need to know that so that we can configure logs properly (see #3878)
+		var catchAllSblockExists bool
+		for _, sblock := range p.serverBlocks {
+			if len(sblock.hostsFromKeys(false)) == 0 {
+				catchAllSblockExists = true
+			}
+		}
+
 		// create a subroute for each site in the server block
 		for _, sblock := range p.serverBlocks {
 			matcherSetsEnc, err := st.compileEncodedMatcherSets(sblock)
@@ -450,12 +489,12 @@ func (st *ServerType) serversFromPairings(
 						}
 					} else {
 						cp.DefaultSNI = defaultSNI
-						hasCatchAllTLSConnPolicy = true
 					}
 
 					// only append this policy if it actually changes something
 					if !cp.SettingsEmpty() {
 						srv.TLSConnPolicies = append(srv.TLSConnPolicies, cp)
+						hasCatchAllTLSConnPolicy = len(hosts) == 0
 					}
 				}
 			}
@@ -533,13 +572,13 @@ func (st *ServerType) serversFromPairings(
 				} else {
 					// map each host to the user's desired logger name
 					for _, h := range sblockLogHosts {
-						// if the custom logger name is non-empty, add it to
+						// if the custom logger name is non-empty, add it to the map;
-						// the map; otherwise, only map to an empty logger
+						// otherwise, only map to an empty logger name if this or
-						// name if the server block has a catch-all host (in
+						// another site block on this server has a catch-all host (in
-						// which case only requests with mapped hostnames will
+						// which case only requests with mapped hostnames will be
-						// be access-logged, so it'll be necessary to add them
+						// access-logged, so it'll be necessary to add them to the
-						// to the map even if they use default logger)
+						// map even if they use default logger)
-						if ncl.name != "" || len(hosts) == 0 {
+						if ncl.name != "" || catchAllSblockExists {
 							if srv.Logs.LoggerNames == nil {
 								srv.Logs.LoggerNames = make(map[string]string)
 							}
@@ -596,6 +635,11 @@ func (st *ServerType) serversFromPairings(
 		servers[fmt.Sprintf("srv%d", i)] = srv
 	}
 
+	err := applyServerOptions(servers, options, warnings)
+	if err != nil {
+		return nil, err
+	}
+
 	return servers, nil
 }
 
@@ -657,9 +701,15 @@ func detectConflictingSchemes(srv *caddyhttp.Server, serverBlocks []serverBlock,
 	return nil
 }
 
-// consolidateConnPolicies removes empty TLS connection policies and combines
+// consolidateConnPolicies sorts any catch-all policy to the end, removes empty TLS connection
-// equivalent ones for a cleaner overall output.
+// policies, and combines equivalent ones for a cleaner overall output.
 func consolidateConnPolicies(cps caddytls.ConnectionPolicies) (caddytls.ConnectionPolicies, error) {
+	// catch-all policies (those without any matcher) should be at the
+	// end, otherwise it nullifies any more specific policies
+	sort.SliceStable(cps, func(i, j int) bool {
+		return cps[j].MatchersRaw == nil && cps[i].MatchersRaw != nil
+	})
+
 	for i := 0; i < len(cps); i++ {
 		// compare it to the others
 		for j := 0; j < len(cps); j++ {
@@ -852,7 +902,18 @@ func buildSubroute(routes []ConfigValue, groupCounter counter) (*caddyhttp.Subro
 		// root directives would overwrite previously-matched ones; they should not cascade
 		"root": {},
 	}
-	for meDir, info := range mutuallyExclusiveDirs {
+
+	// we need to deterministically loop over each of these directives
+	// in order to keep the group numbers consistent
+	keys := make([]string, 0, len(mutuallyExclusiveDirs))
+	for k := range mutuallyExclusiveDirs {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, meDir := range keys {
+		info := mutuallyExclusiveDirs[meDir]
+
 		// see how many instances of the directive there are
 		for _, r := range routes {
 			if r.directive == meDir {

caddyconfig/httpcaddyfile/httptype_test.go

@@ -164,6 +164,58 @@ func TestGlobalOptions(t *testing.T) {
 			expectWarn:  false,
 			expectError: true,
 		},
+		{
+			input: `
+				{
+					admin {
+						enforce_origin
+						origins 192.168.1.1:2020 127.0.0.1:2020
+					}
+				}
+				:80
+			`,
+			expectWarn:  false,
+			expectError: false,
+		},
+		{
+			input: `
+				{
+					admin 127.0.0.1:2020 {
+						enforce_origin
+						origins 192.168.1.1:2020 127.0.0.1:2020
+					}
+				}
+				:80
+			`,
+			expectWarn:  false,
+			expectError: false,
+		},
+		{
+			input: `
+				{
+					admin 192.168.1.1:2020 127.0.0.1:2020 {
+						enforce_origin
+						origins 192.168.1.1:2020 127.0.0.1:2020
+					}
+				}
+				:80
+			`,
+			expectWarn:  false,
+			expectError: true,
+		},
+		{
+			input: `
+				{
+					admin off {
+						enforce_origin
+						origins 192.168.1.1:2020 127.0.0.1:2020
+					}
+				}
+				:80
+			`,
+			expectWarn:  false,
+			expectError: true,
+		},
 	} {
 
 		adapter := caddyfile.Adapter{

caddyconfig/httpcaddyfile/options.go

@@ -20,6 +20,8 @@ import (
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
+	"github.com/caddyserver/certmagic"
+	"github.com/mholt/acmez/acme"
 )
 
 func init() {
@@ -34,12 +36,14 @@ func init() {
 	RegisterGlobalOption("acme_ca_root", parseOptSingleString)
 	RegisterGlobalOption("acme_dns", parseOptSingleString)
 	RegisterGlobalOption("acme_eab", parseOptACMEEAB)
+	RegisterGlobalOption("cert_issuer", parseOptCertIssuer)
 	RegisterGlobalOption("email", parseOptSingleString)
 	RegisterGlobalOption("admin", parseOptAdmin)
 	RegisterGlobalOption("on_demand_tls", parseOptOnDemand)
 	RegisterGlobalOption("local_certs", parseOptTrue)
 	RegisterGlobalOption("key_type", parseOptSingleString)
 	RegisterGlobalOption("auto_https", parseOptAutoHTTPS)
+	RegisterGlobalOption("servers", parseServerOptions)
 }
 
 func parseOptTrue(d *caddyfile.Dispenser) (interface{}, error) {
@@ -182,7 +186,7 @@ func parseOptStorage(d *caddyfile.Dispenser) (interface{}, error) {
 }
 
 func parseOptACMEEAB(d *caddyfile.Dispenser) (interface{}, error) {
-	eab := new(caddytls.ExternalAccountBinding)
+	eab := new(acme.EAB)
 	for d.Next() {
 		if d.NextArg() {
 			return nil, d.ArgErr()
@@ -195,11 +199,11 @@ func parseOptACMEEAB(d *caddyfile.Dispenser) (interface{}, error) {
 				}
 				eab.KeyID = d.Val()
 
-			case "hmac":
+			case "mac_key":
 				if !d.NextArg() {
 					return nil, d.ArgErr()
 				}
-				eab.HMAC = d.Val()
+				eab.MACKey = d.Val()
 
 			default:
 				return nil, d.Errf("unrecognized parameter '%s'", d.Val())
@@ -209,6 +213,33 @@ func parseOptACMEEAB(d *caddyfile.Dispenser) (interface{}, error) {
 	return eab, nil
 }
 
+func parseOptCertIssuer(d *caddyfile.Dispenser) (interface{}, error) {
+	if !d.Next() { // consume option name
+		return nil, d.ArgErr()
+	}
+	if !d.Next() { // get issuer module name
+		return nil, d.ArgErr()
+	}
+	modName := d.Val()
+	mod, err := caddy.GetModule("tls.issuance." + modName)
+	if err != nil {
+		return nil, d.Errf("getting issuer module '%s': %v", modName, err)
+	}
+	unm, ok := mod.New().(caddyfile.Unmarshaler)
+	if !ok {
+		return nil, d.Errf("issuer module '%s' is not a Caddyfile unmarshaler", mod.ID)
+	}
+	err = unm.UnmarshalCaddyfile(d.NewFromNextSegment())
+	if err != nil {
+		return nil, err
+	}
+	iss, ok := unm.(certmagic.Issuer)
+	if !ok {
+		return nil, d.Errf("module %s is not a certmagic.Issuer", mod.ID)
+	}
+	return iss, nil
+}
+
 func parseOptSingleString(d *caddyfile.Dispenser) (interface{}, error) {
 	d.Next() // consume parameter name
 	if !d.Next() {
@@ -222,17 +253,39 @@ func parseOptSingleString(d *caddyfile.Dispenser) (interface{}, error) {
 }
 
 func parseOptAdmin(d *caddyfile.Dispenser) (interface{}, error) {
-	if d.Next() {
+	adminCfg := new(caddy.AdminConfig)
-		var listenAddress string
+	for d.Next() {
-		if !d.AllArgs(&listenAddress) {
+		if d.NextArg() {
-			return "", d.ArgErr()
+			listenAddress := d.Val()
+			if listenAddress == "off" {
+				adminCfg.Disabled = true
+				if d.Next() { // Do not accept any remaining options including block
+					return nil, d.Err("No more option is allowed after turning off admin config")
+				}
+			} else {
+				adminCfg.Listen = listenAddress
+				if d.NextArg() { // At most 1 arg is allowed
+					return nil, d.ArgErr()
+				}
+			}
 		}
-		if listenAddress == "" {
+		for nesting := d.Nesting(); d.NextBlock(nesting); {
-			listenAddress = caddy.DefaultAdminListen
+			switch d.Val() {
+			case "enforce_origin":
+				adminCfg.EnforceOrigin = true
+
+			case "origins":
+				adminCfg.Origins = d.RemainingArgs()
+
+			default:
+				return nil, d.Errf("unrecognized parameter '%s'", d.Val())
+			}
 		}
-		return listenAddress, nil
 	}
-	return "", nil
+	if adminCfg.Listen == "" && !adminCfg.Disabled {
+		adminCfg.Listen = caddy.DefaultAdminListen
+	}
+	return adminCfg, nil
 }
 
 func parseOptOnDemand(d *caddyfile.Dispenser) (interface{}, error) {
@@ -309,3 +362,7 @@ func parseOptAutoHTTPS(d *caddyfile.Dispenser) (interface{}, error) {
 	}
 	return val, nil
 }
+
+func parseServerOptions(d *caddyfile.Dispenser) (interface{}, error) {
+	return unmarshalCaddyfileServerOptions(d)
+}

caddyconfig/httpcaddyfile/serveroptions.go

@@ -0,0 +1,235 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpcaddyfile
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"github.com/dustin/go-humanize"
+)
+
+// serverOptions collects server config overrides parsed from Caddyfile global options
+type serverOptions struct {
+	// If set, will only apply these options to servers that contain a
+	// listener address that matches exactly. If empty, will apply to all
+	// servers that were not already matched by another serverOptions.
+	ListenerAddress string
+
+	// These will all map 1:1 to the caddyhttp.Server struct
+	ListenerWrappersRaw []json.RawMessage
+	ReadTimeout         caddy.Duration
+	ReadHeaderTimeout   caddy.Duration
+	WriteTimeout        caddy.Duration
+	IdleTimeout         caddy.Duration
+	MaxHeaderBytes      int
+	AllowH2C            bool
+	ExperimentalHTTP3   bool
+	StrictSNIHost       *bool
+}
+
+func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (interface{}, error) {
+	serverOpts := serverOptions{}
+	for d.Next() {
+		if d.NextArg() {
+			serverOpts.ListenerAddress = d.Val()
+			if d.NextArg() {
+				return nil, d.ArgErr()
+			}
+		}
+		for nesting := d.Nesting(); d.NextBlock(nesting); {
+			switch d.Val() {
+			case "listener_wrappers":
+				for nesting := d.Nesting(); d.NextBlock(nesting); {
+					mod, err := caddy.GetModule("caddy.listeners." + d.Val())
+					if err != nil {
+						return nil, fmt.Errorf("finding listener module '%s': %v", d.Val(), err)
+					}
+					unm, ok := mod.New().(caddyfile.Unmarshaler)
+					if !ok {
+						return nil, fmt.Errorf("listener module '%s' is not a Caddyfile unmarshaler", mod)
+					}
+					err = unm.UnmarshalCaddyfile(d.NewFromNextSegment())
+					if err != nil {
+						return nil, err
+					}
+					listenerWrapper, ok := unm.(caddy.ListenerWrapper)
+					if !ok {
+						return nil, fmt.Errorf("module %s is not a listener wrapper", mod)
+					}
+					jsonListenerWrapper := caddyconfig.JSONModuleObject(
+						listenerWrapper,
+						"wrapper",
+						listenerWrapper.(caddy.Module).CaddyModule().ID.Name(),
+						nil,
+					)
+					serverOpts.ListenerWrappersRaw = append(serverOpts.ListenerWrappersRaw, jsonListenerWrapper)
+				}
+
+			case "timeouts":
+				for nesting := d.Nesting(); d.NextBlock(nesting); {
+					switch d.Val() {
+					case "read_body":
+						if !d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						dur, err := caddy.ParseDuration(d.Val())
+						if err != nil {
+							return nil, d.Errf("parsing read_body timeout duration: %v", err)
+						}
+						serverOpts.ReadTimeout = caddy.Duration(dur)
+
+					case "read_header":
+						if !d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						dur, err := caddy.ParseDuration(d.Val())
+						if err != nil {
+							return nil, d.Errf("parsing read_header timeout duration: %v", err)
+						}
+						serverOpts.ReadHeaderTimeout = caddy.Duration(dur)
+
+					case "write":
+						if !d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						dur, err := caddy.ParseDuration(d.Val())
+						if err != nil {
+							return nil, d.Errf("parsing write timeout duration: %v", err)
+						}
+						serverOpts.WriteTimeout = caddy.Duration(dur)
+
+					case "idle":
+						if !d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						dur, err := caddy.ParseDuration(d.Val())
+						if err != nil {
+							return nil, d.Errf("parsing idle timeout duration: %v", err)
+						}
+						serverOpts.IdleTimeout = caddy.Duration(dur)
+
+					default:
+						return nil, d.Errf("unrecognized timeouts option '%s'", d.Val())
+					}
+				}
+
+			case "max_header_size":
+				var sizeStr string
+				if !d.AllArgs(&sizeStr) {
+					return nil, d.ArgErr()
+				}
+				size, err := humanize.ParseBytes(sizeStr)
+				if err != nil {
+					return nil, d.Errf("parsing max_header_size: %v", err)
+				}
+				serverOpts.MaxHeaderBytes = int(size)
+
+			case "protocol":
+				for nesting := d.Nesting(); d.NextBlock(nesting); {
+					switch d.Val() {
+					case "allow_h2c":
+						if d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						serverOpts.AllowH2C = true
+
+					case "experimental_http3":
+						if d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						serverOpts.ExperimentalHTTP3 = true
+
+					case "strict_sni_host":
+						if d.NextArg() {
+							return nil, d.ArgErr()
+						}
+						trueBool := true
+						serverOpts.StrictSNIHost = &trueBool
+
+					default:
+						return nil, d.Errf("unrecognized protocol option '%s'", d.Val())
+					}
+				}
+
+			default:
+				return nil, d.Errf("unrecognized servers option '%s'", d.Val())
+			}
+		}
+	}
+	return serverOpts, nil
+}
+
+// applyServerOptions sets the server options on the appropriate servers
+func applyServerOptions(
+	servers map[string]*caddyhttp.Server,
+	options map[string]interface{},
+	warnings *[]caddyconfig.Warning,
+) error {
+	// If experimental HTTP/3 is enabled, enable it on each server.
+	// We already know there won't be a conflict with serverOptions because
+	// we validated earlier that "experimental_http3" cannot be set at the same
+	// time as "servers"
+	if enableH3, ok := options["experimental_http3"].(bool); ok && enableH3 {
+		*warnings = append(*warnings, caddyconfig.Warning{Message: "the 'experimental_http3' global option is deprecated, please use the 'servers > protocol > experimental_http3' option instead"})
+		for _, srv := range servers {
+			srv.ExperimentalHTTP3 = true
+		}
+	}
+
+	serverOpts, ok := options["servers"].([]serverOptions)
+	if !ok {
+		return nil
+	}
+
+	for _, server := range servers {
+		// find the options that apply to this server
+		opts := func() *serverOptions {
+			for _, entry := range serverOpts {
+				if entry.ListenerAddress == "" {
+					return &entry
+				}
+				for _, listener := range server.Listen {
+					if entry.ListenerAddress == listener {
+						return &entry
+					}
+				}
+			}
+			return nil
+		}()
+
+		// if none apply, then move to the next server
+		if opts == nil {
+			continue
+		}
+
+		// set all the options
+		server.ListenerWrappersRaw = opts.ListenerWrappersRaw
+		server.ReadTimeout = opts.ReadTimeout
+		server.ReadHeaderTimeout = opts.ReadHeaderTimeout
+		server.WriteTimeout = opts.WriteTimeout
+		server.IdleTimeout = opts.IdleTimeout
+		server.MaxHeaderBytes = opts.MaxHeaderBytes
+		server.AllowH2C = opts.AllowH2C
+		server.ExperimentalHTTP3 = opts.ExperimentalHTTP3
+		server.StrictSNIHost = opts.StrictSNIHost
+	}
+
+	return nil
+}

caddyconfig/httpcaddyfile/tlsapp.go

@@ -21,12 +21,14 @@ import (
 	"reflect"
 	"sort"
 	"strconv"
+	"strings"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
 	"github.com/caddyserver/certmagic"
+	"github.com/mholt/acmez/acme"
 )
 
 func (st ServerType) buildTLSApp(
@@ -74,163 +76,148 @@ func (st ServerType) buildTLSApp(
 		}
 	}
 
+	// a catch-all automation policy is used as a "default" for all subjects that
+	// don't have custom configuration explicitly associated with them; this
+	// is only to add if the global settings or defaults are non-empty
 	catchAllAP, err := newBaseAutomationPolicy(options, warnings, false)
 	if err != nil {
 		return nil, warnings, err
 	}
+	if catchAllAP != nil {
+		if tlsApp.Automation == nil {
+			tlsApp.Automation = new(caddytls.AutomationConfig)
+		}
+		tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, catchAllAP)
+	}
 
 	for _, p := range pairings {
 		for _, sblock := range p.serverBlocks {
 			// get values that populate an automation policy for this block
-			var ap *caddytls.AutomationPolicy
+			ap, err := newBaseAutomationPolicy(options, warnings, true)
+			if err != nil {
+				return nil, warnings, err
+			}
 
 			sblockHosts := sblock.hostsFromKeys(false)
-			if len(sblockHosts) == 0 {
+			if len(sblockHosts) == 0 && catchAllAP != nil {
 				ap = catchAllAP
 			}
 
 			// on-demand tls
 			if _, ok := sblock.pile["tls.on_demand"]; ok {
-				if ap == nil {
-					var err error
-					ap, err = newBaseAutomationPolicy(options, warnings, true)
-					if err != nil {
-						return nil, warnings, err
-					}
-				}
 				ap.OnDemand = true
 			}
 
 			// certificate issuers
 			if issuerVals, ok := sblock.pile["tls.cert_issuer"]; ok {
+				var issuers []certmagic.Issuer
 				for _, issuerVal := range issuerVals {
-					issuer := issuerVal.Value.(certmagic.Issuer)
+					ap.Issuers = append(ap.Issuers, issuerVal.Value.(certmagic.Issuer))
-					if ap == nil {
+				}
-						var err error
+				if ap == catchAllAP && !reflect.DeepEqual(ap.Issuers, issuers) {
-						ap, err = newBaseAutomationPolicy(options, warnings, true)
+					return nil, warnings, fmt.Errorf("automation policy from site block is also default/catch-all policy because of key without hostname, and the two are in conflict: %#v != %#v", ap.Issuers, issuers)
-						if err != nil {
-							return nil, warnings, err
-						}
-					}
-					if ap == catchAllAP && !reflect.DeepEqual(ap.Issuer, issuer) {
-						return nil, warnings, fmt.Errorf("automation policy from site block is also default/catch-all policy because of key without hostname, and the two are in conflict: %#v != %#v", ap.Issuer, issuer)
-					}
-					ap.Issuer = issuer
 				}
 			}
 
 			// custom bind host
 			for _, cfgVal := range sblock.pile["bind"] {
-				// either an existing issuer is already configured (and thus, ap is not
+				for _, iss := range ap.Issuers {
-				// nil), or we need to configure an issuer, so we need ap to be non-nil
+					// if an issuer was already configured and it is NOT an ACME issuer,
-				if ap == nil {
+					// skip, since we intend to adjust only ACME issuers; ensure we
-					ap, err = newBaseAutomationPolicy(options, warnings, true)
+					// include any issuer that embeds/wraps an underlying ACME issuer
-					if err != nil {
+					var acmeIssuer *caddytls.ACMEIssuer
-						return nil, warnings, err
+					if acmeWrapper, ok := iss.(acmeCapable); ok {
+						acmeIssuer = acmeWrapper.GetACMEIssuer()
+					}
+					if acmeIssuer == nil {
+						continue
 					}
-				}
 
-				// if an issuer was already configured and it is NOT an ACME
+					// proceed to configure the ACME issuer's bind host, without
-				// issuer, skip, since we intend to adjust only ACME issuers
+					// overwriting any existing settings
-				var acmeIssuer *caddytls.ACMEIssuer
+					if acmeIssuer.Challenges == nil {
-				if ap.Issuer != nil {
+						acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
-					var ok bool
+					}
-					if acmeIssuer, ok = ap.Issuer.(*caddytls.ACMEIssuer); !ok {
+					if acmeIssuer.Challenges.BindHost == "" {
-						break
+						// only binding to one host is supported
+						var bindHost string
+						if bindHosts, ok := cfgVal.Value.([]string); ok && len(bindHosts) > 0 {
+							bindHost = bindHosts[0]
+						}
+						acmeIssuer.Challenges.BindHost = bindHost
 					}
 				}
-
-				// proceed to configure the ACME issuer's bind host, without
-				// overwriting any existing settings
-				if acmeIssuer == nil {
-					acmeIssuer = new(caddytls.ACMEIssuer)
-				}
-				if acmeIssuer.Challenges == nil {
-					acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
-				}
-				if acmeIssuer.Challenges.BindHost == "" {
-					// only binding to one host is supported
-					var bindHost string
-					if bindHosts, ok := cfgVal.Value.([]string); ok && len(bindHosts) > 0 {
-						bindHost = bindHosts[0]
-					}
-					acmeIssuer.Challenges.BindHost = bindHost
-				}
-				ap.Issuer = acmeIssuer // we'll encode it later
 			}
 
-			if ap != nil {
+			// first make sure this block is allowed to create an automation policy;
-				if ap.Issuer != nil {
+			// doing so is forbidden if it has a key with no host (i.e. ":443")
-					// encode issuer now that it's all set up
+			// and if there is a different server block that also has a key with no
-					issuerName := ap.Issuer.(caddy.Module).CaddyModule().ID.Name()
+			// host -- since a key with no host matches any host, we need its
-					ap.IssuerRaw = caddyconfig.JSONModuleObject(ap.Issuer, "module", issuerName, &warnings)
+			// associated automation policy to have an empty Subjects list, i.e. no
+			// host filter, which is indistinguishable between the two server blocks
+			// because automation is not done in the context of a particular server...
+			// this is an example of a poor mapping from Caddyfile to JSON but that's
+			// the least-leaky abstraction I could figure out
+			if len(sblockHosts) == 0 {
+				if serverBlocksWithTLSHostlessKey > 1 {
+					// this server block and at least one other has a key with no host,
+					// making the two indistinguishable; it is misleading to define such
+					// a policy within one server block since it actually will apply to
+					// others as well
+					return nil, warnings, fmt.Errorf("cannot make a TLS automation policy from a server block that has a host-less address when there are other TLS-enabled server block addresses lacking a host")
 				}
+				if catchAllAP == nil {
+					// this server block has a key with no hosts, but there is not yet
+					// a catch-all automation policy (probably because no global options
+					// were set), so this one becomes it
+					catchAllAP = ap
+				}
+			}
 
-				// first make sure this block is allowed to create an automation policy;
+			// associate our new automation policy with this server block's hosts
-				// doing so is forbidden if it has a key with no host (i.e. ":443")
+			ap.Subjects = sblockHosts
-				// and if there is a different server block that also has a key with no
+			sort.Strings(ap.Subjects) // solely for deterministic test results
-				// host -- since a key with no host matches any host, we need its
+
-				// associated automation policy to have an empty Subjects list, i.e. no
+			// if a combination of public and internal names were given
-				// host filter, which is indistinguishable between the two server blocks
+			// for this same server block and no issuer was specified, we
-				// because automation is not done in the context of a particular server...
+			// need to separate them out in the automation policies so
-				// this is an example of a poor mapping from Caddyfile to JSON but that's
+			// that the internal names can use the internal issuer and
-				// the least-leaky abstraction I could figure out
+			// the other names can use the default/public/ACME issuer
-				if len(sblockHosts) == 0 {
+			var ap2 *caddytls.AutomationPolicy
-					if serverBlocksWithTLSHostlessKey > 1 {
+			if len(ap.Issuers) == 0 {
-						// this server block and at least one other has a key with no host,
+				var internal, external []string
-						// making the two indistinguishable; it is misleading to define such
+				for _, s := range ap.Subjects {
-						// a policy within one server block since it actually will apply to
+					if !certmagic.SubjectQualifiesForCert(s) {
-						// others as well
+						return nil, warnings, fmt.Errorf("subject does not qualify for certificate: '%s'", s)
-						return nil, warnings, fmt.Errorf("cannot make a TLS automation policy from a server block that has a host-less address when there are other TLS-enabled server block addresses lacking a host")
 					}
-					if catchAllAP == nil {
+					// we don't use certmagic.SubjectQualifiesForPublicCert() because of one nuance:
-						// this server block has a key with no hosts, but there is not yet
+					// names like *.*.tld that may not qualify for a public certificate are actually
-						// a catch-all automation policy (probably because no global options
+					// fine when used with OnDemand, since OnDemand (currently) does not obtain
-						// were set), so this one becomes it
+					// wildcards (if it ever does, there will be a separate config option to enable
-						catchAllAP = ap
+					// it that we would need to check here) since the hostname is known at handshake;
+					// and it is unexpected to switch to internal issuer when the user wants to get
+					// regular certificates on-demand for a class of certs like *.*.tld.
+					if !certmagic.SubjectIsIP(s) && !certmagic.SubjectIsInternal(s) && (strings.Count(s, "*.") < 2 || ap.OnDemand) {
+						external = append(external, s)
+					} else {
+						internal = append(internal, s)
 					}
 				}
-
+				if len(external) > 0 && len(internal) > 0 {
-				// associate our new automation policy with this server block's hosts,
+					ap.Subjects = external
-				// unless, of course, the server block has a key with no hosts, in which
+					apCopy := *ap
-				// case its automation policy becomes or blends with the default/global
+					ap2 = &apCopy
-				// automation policy because, of necessity, it applies to all hostnames
+					ap2.Subjects = internal
-				// (i.e. it has no Subjects filter) -- in that case, we'll append it last
+					ap2.IssuersRaw = []json.RawMessage{caddyconfig.JSONModuleObject(caddytls.InternalIssuer{}, "module", "internal", &warnings)}
-				if ap != catchAllAP {
-					ap.Subjects = sblockHosts
-
-					// if a combination of public and internal names were given
-					// for this same server block and no issuer was specified, we
-					// need to separate them out in the automation policies so
-					// that the internal names can use the internal issuer and
-					// the other names can use the default/public/ACME issuer
-					var ap2 *caddytls.AutomationPolicy
-					if ap.Issuer == nil {
-						var internal, external []string
-						for _, s := range ap.Subjects {
-							if certmagic.SubjectQualifiesForPublicCert(s) {
-								external = append(external, s)
-							} else {
-								internal = append(internal, s)
-							}
-						}
-						if len(external) > 0 && len(internal) > 0 {
-							ap.Subjects = external
-							apCopy := *ap
-							ap2 = &apCopy
-							ap2.Subjects = internal
-							ap2.IssuerRaw = caddyconfig.JSONModuleObject(caddytls.InternalIssuer{}, "module", "internal", &warnings)
-						}
-					}
-					if tlsApp.Automation == nil {
-						tlsApp.Automation = new(caddytls.AutomationConfig)
-					}
-					tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, ap)
-					if ap2 != nil {
-						tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, ap2)
-					}
 				}
 			}
+			if tlsApp.Automation == nil {
+				tlsApp.Automation = new(caddytls.AutomationConfig)
+			}
+			tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, ap)
+			if ap2 != nil {
+				tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, ap2)
+			}
 
 			// certificate loaders
 			if clVals, ok := sblock.pile["tls.cert_loader"]; ok {
@@ -286,7 +273,7 @@ func (st ServerType) buildTLSApp(
 	// get internal certificates by default rather than ACME
 	var al caddytls.AutomateLoader
 	internalAP := &caddytls.AutomationPolicy{
-		IssuerRaw: json.RawMessage(`{"module":"internal"}`),
+		IssuersRaw: []json.RawMessage{json.RawMessage(`{"module":"internal"}`)},
 	}
 	for h := range hostsSharedWithHostlessKey {
 		al = append(al, h)
@@ -304,23 +291,54 @@ func (st ServerType) buildTLSApp(
 		tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, internalAP)
 	}
 
-	// if there is a global/catch-all automation policy, ensure it goes last
+	// if there are any global options set for issuers (ACME ones in particular), make sure they
-	if catchAllAP != nil {
+	// take effect in every automation policy that does not have any issuers
-		// first, encode its issuer, if there is one
+	if tlsApp.Automation != nil {
-		if catchAllAP.Issuer != nil {
+		globalEmail := options["email"]
-			issuerName := catchAllAP.Issuer.(caddy.Module).CaddyModule().ID.Name()
+		globalACMECA := options["acme_ca"]
-			catchAllAP.IssuerRaw = caddyconfig.JSONModuleObject(catchAllAP.Issuer, "module", issuerName, &warnings)
+		globalACMECARoot := options["acme_ca_root"]
-		}
+		globalACMEDNS := options["acme_dns"]
+		globalACMEEAB := options["acme_eab"]
+		hasGlobalACMEDefaults := globalEmail != nil || globalACMECA != nil || globalACMECARoot != nil || globalACMEDNS != nil || globalACMEEAB != nil
+		if hasGlobalACMEDefaults {
+			for _, ap := range tlsApp.Automation.Policies {
+				if len(ap.Issuers) == 0 {
+					acme, zerosslACME := new(caddytls.ACMEIssuer), new(caddytls.ACMEIssuer)
+					zerossl := &caddytls.ZeroSSLIssuer{ACMEIssuer: zerosslACME}
+					ap.Issuers = []certmagic.Issuer{acme, zerossl} // TODO: keep this in sync with Caddy's other issuer defaults elsewhere, like in caddytls/automation.go (DefaultIssuers).
 
-		// then append it to the end of the policies list
+					// if a non-ZeroSSL endpoint is specified, we assume we can't use the ZeroSSL issuer successfully
-		if tlsApp.Automation == nil {
+					if globalACMECA != nil && !strings.Contains(globalACMECA.(string), "zerossl") {
-			tlsApp.Automation = new(caddytls.AutomationConfig)
+						ap.Issuers = []certmagic.Issuer{acme}
+					}
+				}
+			}
 		}
-		tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, catchAllAP)
 	}
 
-	// do a little verification & cleanup
+	// finalize and verify policies; do cleanup
 	if tlsApp.Automation != nil {
+		for i, ap := range tlsApp.Automation.Policies {
+			// ensure all issuers have global defaults filled in
+			for j, issuer := range ap.Issuers {
+				err := fillInGlobalACMEDefaults(issuer, options)
+				if err != nil {
+					return nil, warnings, fmt.Errorf("filling in global issuer defaults for AP %d, issuer %d: %v", i, j, err)
+				}
+			}
+
+			// encode all issuer values we created, so they will be rendered in the output
+			if len(ap.Issuers) > 0 && ap.IssuersRaw == nil {
+				for _, iss := range ap.Issuers {
+					issuerName := iss.(caddy.Module).CaddyModule().ID.Name()
+					ap.IssuersRaw = append(ap.IssuersRaw, caddyconfig.JSONModuleObject(iss, "module", issuerName, &warnings))
+				}
+			}
+		}
+
+		// consolidate automation policies that are the exact same
+		tlsApp.Automation.Policies = consolidateAutomationPolicies(tlsApp.Automation.Policies)
+
 		// ensure automation policies don't overlap subjects (this should be
 		// an error at provision-time as well, but catch it in the adapt phase
 		// for convenience)
@@ -334,29 +352,74 @@ func (st ServerType) buildTLSApp(
 			}
 		}
 
-		// consolidate automation policies that are the exact same
+		// if nothing remains, remove any excess values to clean up the resulting config
-		tlsApp.Automation.Policies = consolidateAutomationPolicies(tlsApp.Automation.Policies)
+		if len(tlsApp.Automation.Policies) == 0 {
+			tlsApp.Automation.Policies = nil
+		}
+		if reflect.DeepEqual(tlsApp.Automation, new(caddytls.AutomationConfig)) {
+			tlsApp.Automation = nil
+		}
 	}
 
 	return tlsApp, warnings, nil
 }
 
+type acmeCapable interface{ GetACMEIssuer() *caddytls.ACMEIssuer }
+
+func fillInGlobalACMEDefaults(issuer certmagic.Issuer, options map[string]interface{}) error {
+	acmeWrapper, ok := issuer.(acmeCapable)
+	if !ok {
+		return nil
+	}
+	acmeIssuer := acmeWrapper.GetACMEIssuer()
+	if acmeIssuer == nil {
+		return nil
+	}
+
+	globalEmail := options["email"]
+	globalACMECA := options["acme_ca"]
+	globalACMECARoot := options["acme_ca_root"]
+	globalACMEDNS := options["acme_dns"]
+	globalACMEEAB := options["acme_eab"]
+
+	if globalEmail != nil && acmeIssuer.Email == "" {
+		acmeIssuer.Email = globalEmail.(string)
+	}
+	if globalACMECA != nil && acmeIssuer.CA == "" {
+		acmeIssuer.CA = globalACMECA.(string)
+	}
+	if globalACMECARoot != nil && !sliceContains(acmeIssuer.TrustedRootsPEMFiles, globalACMECARoot.(string)) {
+		acmeIssuer.TrustedRootsPEMFiles = append(acmeIssuer.TrustedRootsPEMFiles, globalACMECARoot.(string))
+	}
+	if globalACMEDNS != nil && (acmeIssuer.Challenges == nil || acmeIssuer.Challenges.DNS == nil) {
+		provName := globalACMEDNS.(string)
+		dnsProvModule, err := caddy.GetModule("dns.providers." + provName)
+		if err != nil {
+			return fmt.Errorf("getting DNS provider module named '%s': %v", provName, err)
+		}
+		acmeIssuer.Challenges = &caddytls.ChallengesConfig{
+			DNS: &caddytls.DNSChallengeConfig{
+				ProviderRaw: caddyconfig.JSONModuleObject(dnsProvModule.New(), "name", provName, nil),
+			},
+		}
+	}
+	if globalACMEEAB != nil && acmeIssuer.ExternalAccount == nil {
+		acmeIssuer.ExternalAccount = globalACMEEAB.(*acme.EAB)
+	}
+	return nil
+}
+
 // newBaseAutomationPolicy returns a new TLS automation policy that gets
 // its values from the global options map. It should be used as the base
 // for any other automation policies. A nil policy (and no error) will be
 // returned if there are no default/global options. However, if always is
 // true, a non-nil value will always be returned (unless there is an error).
 func newBaseAutomationPolicy(options map[string]interface{}, warnings []caddyconfig.Warning, always bool) (*caddytls.AutomationPolicy, error) {
-	acmeCA, hasACMECA := options["acme_ca"]
+	issuer, hasIssuer := options["cert_issuer"]
-	acmeCARoot, hasACMECARoot := options["acme_ca_root"]
+	_, hasLocalCerts := options["local_certs"]
-	acmeDNS, hasACMEDNS := options["acme_dns"]
-	acmeEAB, hasACMEEAB := options["acme_eab"]
-
-	email, hasEmail := options["email"]
-	localCerts, hasLocalCerts := options["local_certs"]
 	keyType, hasKeyType := options["key_type"]
 
-	hasGlobalAutomationOpts := hasACMECA || hasACMECARoot || hasACMEDNS || hasACMEEAB || hasEmail || hasLocalCerts || hasKeyType
+	hasGlobalAutomationOpts := hasIssuer || hasLocalCerts || hasKeyType
 
 	// if there are no global options related to automation policies
 	// set, then we can just return right away
@@ -368,57 +431,64 @@ func newBaseAutomationPolicy(options map[string]interface{}, warnings []caddycon
 	}
 
 	ap := new(caddytls.AutomationPolicy)
+	if hasKeyType {
+		ap.KeyType = keyType.(string)
+	}
 
-	if localCerts != nil {
+	if hasIssuer && hasLocalCerts {
-		// internal issuer enabled trumps any ACME configurations; useful in testing
+		return nil, fmt.Errorf("global options are ambiguous: local_certs is confusing when combined with cert_issuer, because local_certs is also a specific kind of issuer")
-		ap.Issuer = new(caddytls.InternalIssuer) // we'll encode it later
+	}
-	} else {
+
-		if acmeCA == nil {
+	if hasIssuer {
-			acmeCA = ""
+		ap.Issuers = []certmagic.Issuer{issuer.(certmagic.Issuer)}
-		}
+	} else if hasLocalCerts {
-		if email == nil {
+		ap.Issuers = []certmagic.Issuer{new(caddytls.InternalIssuer)}
-			email = ""
-		}
-		mgr := &caddytls.ACMEIssuer{
-			CA:    acmeCA.(string),
-			Email: email.(string),
-		}
-		if acmeDNS != nil {
-			provName := acmeDNS.(string)
-			dnsProvModule, err := caddy.GetModule("dns.providers." + provName)
-			if err != nil {
-				return nil, fmt.Errorf("getting DNS provider module named '%s': %v", provName, err)
-			}
-			mgr.Challenges = &caddytls.ChallengesConfig{
-				DNS: &caddytls.DNSChallengeConfig{
-					ProviderRaw: caddyconfig.JSONModuleObject(dnsProvModule.New(), "name", provName, &warnings),
-				},
-			}
-		}
-		if acmeCARoot != nil {
-			mgr.TrustedRootsPEMFiles = []string{acmeCARoot.(string)}
-		}
-		if acmeEAB != nil {
-			mgr.ExternalAccount = acmeEAB.(*caddytls.ExternalAccountBinding)
-		}
-		if keyType != nil {
-			ap.KeyType = keyType.(string)
-		}
-		ap.Issuer = mgr // we'll encode it later
 	}
 
 	return ap, nil
 }
 
+// disambiguateACMEIssuer returns an issuer based on the properties of acmeIssuer.
+// If acmeIssuer implicitly configures a certain kind of ACMEIssuer (for example,
+// ZeroSSL), the proper wrapper over acmeIssuer will be returned instead.
+func disambiguateACMEIssuer(acmeIssuer *caddytls.ACMEIssuer) certmagic.Issuer {
+	// as a special case, we integrate with ZeroSSL's ACME endpoint if it looks like an
+	// implicit ZeroSSL configuration (this requires a  wrapper type over ACMEIssuer
+	// because of the EAB generation; if EAB is provided, we can use plain ACMEIssuer)
+	if strings.Contains(acmeIssuer.CA, "acme.zerossl.com") && acmeIssuer.ExternalAccount == nil {
+		return &caddytls.ZeroSSLIssuer{ACMEIssuer: acmeIssuer}
+	}
+	return acmeIssuer
+}
+
 // consolidateAutomationPolicies combines automation policies that are the same,
 // for a cleaner overall output.
 func consolidateAutomationPolicies(aps []*caddytls.AutomationPolicy) []*caddytls.AutomationPolicy {
-	for i := 0; i < len(aps); i++ {
+	// sort from most specific to least specific; we depend on this ordering
-		for j := 0; j < len(aps); j++ {
+	sort.SliceStable(aps, func(i, j int) bool {
-			if j == i {
+		if automationPolicyIsSubset(aps[i], aps[j]) {
-				continue
+			return true
-			}
+		}
+		if automationPolicyIsSubset(aps[j], aps[i]) {
+			return false
+		}
+		return len(aps[i].Subjects) > len(aps[j].Subjects)
+	})
 
+	// remove any empty policies (except subjects, of course)
+	emptyAP := new(caddytls.AutomationPolicy)
+	for i := 0; i < len(aps); i++ {
+		emptyAP.Subjects = aps[i].Subjects
+		if reflect.DeepEqual(aps[i], emptyAP) {
+			aps = append(aps[:i], aps[i+1:]...)
+			i--
+		}
+	}
+
+	// remove or combine duplicate policies
+	for i := 0; i < len(aps); i++ {
+		// compare only with next policies; we sorted by specificity so we must not delete earlier policies
+		for j := i + 1; j < len(aps); j++ {
 			// if they're exactly equal in every way, just keep one of them
 			if reflect.DeepEqual(aps[i], aps[j]) {
 				aps = append(aps[:j], aps[j+1:]...)
@@ -432,30 +502,75 @@ func consolidateAutomationPolicies(aps []*caddytls.AutomationPolicy) []*caddytls
 			// otherwise the one without any subjects (a catch-all) would be
 			// eaten up by the one with subjects; and if both have subjects, we
 			// need to combine their lists
-			if bytes.Equal(aps[i].IssuerRaw, aps[j].IssuerRaw) &&
+			if reflect.DeepEqual(aps[i].IssuersRaw, aps[j].IssuersRaw) &&
 				bytes.Equal(aps[i].StorageRaw, aps[j].StorageRaw) &&
 				aps[i].MustStaple == aps[j].MustStaple &&
 				aps[i].KeyType == aps[j].KeyType &&
 				aps[i].OnDemand == aps[j].OnDemand &&
 				aps[i].RenewalWindowRatio == aps[j].RenewalWindowRatio {
-				if len(aps[i].Subjects) == 0 && len(aps[j].Subjects) > 0 {
+				if len(aps[i].Subjects) > 0 && len(aps[j].Subjects) == 0 {
-					aps = append(aps[:j], aps[j+1:]...)
+					// later policy (at j) has no subjects ("catch-all"), so we can
-				} else if len(aps[i].Subjects) > 0 && len(aps[j].Subjects) == 0 {
+					// remove the identical-but-more-specific policy that comes first
-					aps = append(aps[:i], aps[i+1:]...)
+					// AS LONG AS it is not shadowed by another policy before it; e.g.
+					// if policy i is for example.com, policy i+1 is '*.com', and policy
+					// j is catch-all, we cannot remove policy i because that would
+					// cause example.com to be served by the less specific policy for
+					// '*.com', which might be different (yes we've seen this happen)
+					if automationPolicyShadows(i, aps) >= j {
+						aps = append(aps[:i], aps[i+1:]...)
+					}
 				} else {
-					aps[i].Subjects = append(aps[i].Subjects, aps[j].Subjects...)
+					// avoid repeated subjects
+					for _, subj := range aps[j].Subjects {
+						if !sliceContains(aps[i].Subjects, subj) {
+							aps[i].Subjects = append(aps[i].Subjects, subj)
+						}
+					}
 					aps = append(aps[:j], aps[j+1:]...)
+					j--
 				}
-				i--
-				break
 			}
 		}
 	}
 
-	// ensure any catch-all policies go last
-	sort.SliceStable(aps, func(i, j int) bool {
-		return len(aps[i].Subjects) > len(aps[j].Subjects)
-	})
-
 	return aps
 }
+
+// automationPolicyIsSubset returns true if a's subjects are a subset
+// of b's subjects.
+func automationPolicyIsSubset(a, b *caddytls.AutomationPolicy) bool {
+	if len(b.Subjects) == 0 {
+		return true
+	}
+	if len(a.Subjects) == 0 {
+		return false
+	}
+	for _, aSubj := range a.Subjects {
+		var inSuperset bool
+		for _, bSubj := range b.Subjects {
+			if certmagic.MatchWildcard(aSubj, bSubj) {
+				inSuperset = true
+				break
+			}
+		}
+		if !inSuperset {
+			return false
+		}
+	}
+	return true
+}
+
+// automationPolicyShadows returns the index of a policy that aps[i] shadows;
+// in other words, for all policies after position i, if that policy covers
+// the same subjects but is less specific, that policy's position is returned,
+// or -1 if no shadowing is found. For example, if policy i is for
+// "foo.example.com" and policy i+2 is for "*.example.com", then i+2 will be
+// returned, since that policy is shadowed by i, which is in front.
+func automationPolicyShadows(i int, aps []*caddytls.AutomationPolicy) int {
+	for j := i + 1; j < len(aps); j++ {
+		if automationPolicyIsSubset(aps[i], aps[j]) {
+			return j
+		}
+	}
+	return -1
+}

caddyconfig/httpcaddyfile/tlsapp_test.go

@@ -0,0 +1,56 @@
+package httpcaddyfile
+
+import (
+	"testing"
+
+	"github.com/caddyserver/caddy/v2/modules/caddytls"
+)
+
+func TestAutomationPolicyIsSubset(t *testing.T) {
+	for i, test := range []struct {
+		a, b   []string
+		expect bool
+	}{
+		{
+			a:      []string{"example.com"},
+			b:      []string{},
+			expect: true,
+		},
+		{
+			a:      []string{},
+			b:      []string{"example.com"},
+			expect: false,
+		},
+		{
+			a:      []string{"foo.example.com"},
+			b:      []string{"*.example.com"},
+			expect: true,
+		},
+		{
+			a:      []string{"foo.example.com"},
+			b:      []string{"foo.example.com"},
+			expect: true,
+		},
+		{
+			a:      []string{"foo.example.com"},
+			b:      []string{"example.com"},
+			expect: false,
+		},
+		{
+			a:      []string{"example.com", "foo.example.com"},
+			b:      []string{"*.com", "*.*.com"},
+			expect: true,
+		},
+		{
+			a:      []string{"example.com", "foo.example.com"},
+			b:      []string{"*.com"},
+			expect: false,
+		},
+	} {
+		apA := &caddytls.AutomationPolicy{Subjects: test.a}
+		apB := &caddytls.AutomationPolicy{Subjects: test.b}
+		if actual := automationPolicyIsSubset(apA, apB); actual != test.expect {
+			t.Errorf("Test %d: Expected %t but got %t (A: %v  B: %v)", i, test.expect, actual, test.a, test.b)
+		}
+	}
+}

caddytest/caddytest.go

@@ -124,10 +124,10 @@ func (tc *Tester) initServer(rawConfig string, configType string) error {
 				return
 			}
 			defer res.Body.Close()
-			body, err := ioutil.ReadAll(res.Body)
+			body, _ := ioutil.ReadAll(res.Body)
 
 			var out bytes.Buffer
-			json.Indent(&out, body, "", "  ")
+			_ = json.Indent(&out, body, "", "  ")
 			tc.t.Logf("----------- failed with config -----------\n%s", out.String())
 		}
 	})
@@ -221,10 +221,11 @@ func isCaddyAdminRunning() error {
 	client := &http.Client{
 		Timeout: Default.LoadRequestTimeout,
 	}
-	_, err := client.Get(fmt.Sprintf("http://localhost:%d/config/", Default.AdminPort))
+	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/config/", Default.AdminPort))
 	if err != nil {
 		return errors.New("caddy integration test caddy server not running. Expected to be listening on localhost:2019")
 	}
+	resp.Body.Close()
 
 	return nil
 }
@@ -272,7 +273,7 @@ func CreateTestingTransport() *http.Transport {
 		IdleConnTimeout:       90 * time.Second,
 		TLSHandshakeTimeout:   5 * time.Second,
 		ExpectContinueTimeout: 1 * time.Second,
-		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
+		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true}, //nolint:gosec
 	}
 }
 
@@ -313,9 +314,13 @@ func (tc *Tester) AssertRedirect(requestURI string, expectedToLocation string, e
 	if err != nil {
 		tc.t.Errorf("requesting \"%s\" expected location: \"%s\" but got error: %s", requestURI, expectedToLocation, err)
 	}
-
+	if loc == nil && expectedToLocation != "" {
-	if expectedToLocation != loc.String() {
+		tc.t.Errorf("requesting \"%s\" expected a Location header, but didn't get one", requestURI)
-		tc.t.Errorf("requesting \"%s\" expected location: \"%s\" but got \"%s\"", requestURI, expectedToLocation, loc.String())
+	}
+	if loc != nil {
+		if expectedToLocation != loc.String() {
+			tc.t.Errorf("requesting \"%s\" expected location: \"%s\" but got \"%s\"", requestURI, expectedToLocation, loc.String())
+		}
 	}
 
 	return resp
@@ -430,7 +435,7 @@ func (tc *Tester) AssertResponse(req *http.Request, expectedStatusCode int, expe
 
 	body := string(bytes)
 
-	if !strings.Contains(body, expectedBody) {
+	if body != expectedBody {
 		tc.t.Errorf("requesting \"%s\" expected response body \"%s\" but got \"%s\"", req.RequestURI, expectedBody, body)
 	}
 

caddytest/integration/autohttps_test.go

@@ -0,0 +1,82 @@
+package integration
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/caddyserver/caddy/v2/caddytest"
+)
+
+func TestAutoHTTPtoHTTPSRedirectsImplicitPort(t *testing.T) {
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+	{
+		http_port     9080
+		https_port    9443
+	}
+	localhost
+	respond "Yahaha! You found me!"
+  `, "caddyfile")
+
+	tester.AssertRedirect("http://localhost:9080/", "https://localhost/", http.StatusPermanentRedirect)
+}
+
+func TestAutoHTTPtoHTTPSRedirectsExplicitPortSameAsHTTPSPort(t *testing.T) {
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+	{
+		http_port     9080
+		https_port    9443
+	}
+	localhost:9443
+	respond "Yahaha! You found me!"
+  `, "caddyfile")
+
+	tester.AssertRedirect("http://localhost:9080/", "https://localhost/", http.StatusPermanentRedirect)
+}
+
+func TestAutoHTTPtoHTTPSRedirectsExplicitPortDifferentFromHTTPSPort(t *testing.T) {
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+	{
+		http_port     9080
+		https_port    9443
+	}
+	localhost:1234
+	respond "Yahaha! You found me!"
+  `, "caddyfile")
+
+	tester.AssertRedirect("http://localhost:9080/", "https://localhost:1234/", http.StatusPermanentRedirect)
+}
+
+func TestAutoHTTPRedirectsWithHTTPListenerFirstInAddresses(t *testing.T) {
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+{
+  "apps": {
+    "http": {
+      "http_port": 9080,
+      "https_port": 9443,
+      "servers": {
+        "ingress_server": {
+          "listen": [
+            ":9080",
+            ":9443"
+          ],
+          "routes": [
+            {
+              "match": [
+                {
+				  "host": ["localhost"]
+                }
+              ]
+            }
+          ]
+        }
+      }
+    }
+  }
+}
+`, "json")
+	tester.AssertRedirect("http://localhost:9080/", "https://localhost/", http.StatusPermanentRedirect)
+}

caddytest/integration/caddyfile_adapt/global_options.txt

@@ -54,9 +54,12 @@
 			"automation": {
 				"policies": [
 					{
-						"issuer": {
+						"issuers": [
-							"module": "internal"
+							{
-						}
+								"module": "internal"
+							}
+						],
+						"key_type": "ed25519"
 					}
 				],
 				"on_demand": {

caddytest/integration/caddyfile_adapt/global_options_acme.txt

@@ -9,8 +9,8 @@
 	}
 	acme_ca https://example.com
 	acme_eab {
-		key_id 4K2scIVbBpNd-78scadB2g
+		key_id  4K2scIVbBpNd-78scadB2g
-		hmac   abcdefghijklmnopqrstuvwx-abcdefghijklnopqrstuvwxyz12ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefgh
+		mac_key abcdefghijklmnopqrstuvwx-abcdefghijklnopqrstuvwxyz12ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefgh
 	}
 	acme_ca_root /path/to/ca.crt
 	email test@example.com
@@ -57,18 +57,20 @@
 			"automation": {
 				"policies": [
 					{
-						"issuer": {
+						"issuers": [
-							"ca": "https://example.com",
+							{
-							"email": "test@example.com",
+								"ca": "https://example.com",
-							"external_account": {
+								"email": "test@example.com",
-								"hmac": "abcdefghijklmnopqrstuvwx-abcdefghijklnopqrstuvwxyz12ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefgh",
+								"external_account": {
-								"key_id": "4K2scIVbBpNd-78scadB2g"
+									"key_id": "4K2scIVbBpNd-78scadB2g",
-							},
+									"mac_key": "abcdefghijklmnopqrstuvwx-abcdefghijklnopqrstuvwxyz12ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefgh"
-							"module": "acme",
+								},
-							"trusted_roots_pem_files": [
+								"module": "acme",
-								"/path/to/ca.crt"
+								"trusted_roots_pem_files": [
-							]
+									"/path/to/ca.crt"
-						},
+								]
+							}
+						],
 						"key_type": "ed25519"
 					}
 				],

caddytest/integration/caddyfile_adapt/global_options_admin.txt

@@ -0,0 +1,83 @@
+{
+	debug
+	http_port 8080
+	https_port 8443
+	default_sni localhost
+	order root first
+	storage file_system {
+		root /data
+	}
+	acme_ca https://example.com
+	acme_ca_root /path/to/ca.crt
+
+	email test@example.com
+	admin {
+		origins localhost:2019 [::1]:2019 127.0.0.1:2019 192.168.10.128
+	}
+	on_demand_tls {
+		ask https://example.com
+		interval 30s
+		burst 20
+	}
+	local_certs
+	key_type ed25519
+}
+
+:80
+----------
+{
+	"admin": {
+		"listen": "localhost:2019",
+		"origins": [
+			"localhost:2019",
+			"[::1]:2019",
+			"127.0.0.1:2019",
+			"192.168.10.128"
+		]
+	},
+	"logging": {
+		"logs": {
+			"default": {
+				"level": "DEBUG"
+			}
+		}
+	},
+	"storage": {
+		"module": "file_system",
+		"root": "/data"
+	},
+	"apps": {
+		"http": {
+			"http_port": 8080,
+			"https_port": 8443,
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"issuers": [
+							{
+								"module": "internal"
+							}
+						],
+						"key_type": "ed25519"
+					}
+				],
+				"on_demand": {
+					"rate_limit": {
+						"interval": 30000000000,
+						"burst": 20
+					},
+					"ask": "https://example.com"
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_server_options_multi.txt

@@ -0,0 +1,83 @@
+{
+	servers {
+		timeouts {
+			idle 90s
+		}
+	}
+	servers :80 {
+		timeouts {
+			idle 60s
+		}
+	}
+	servers :443 {
+		timeouts {
+			idle 30s
+		}
+	}
+}
+
+foo.com {	
+}
+
+http://bar.com {
+}
+
+:8080 {
+}
+
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"idle_timeout": 30000000000,
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"foo.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				},
+				"srv1": {
+					"listen": [
+						":80"
+					],
+					"idle_timeout": 60000000000,
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"bar.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"automatic_https": {
+						"skip": [
+							"bar.com"
+						]
+					}
+				},
+				"srv2": {
+					"listen": [
+						":8080"
+					],
+					"idle_timeout": 90000000000
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/global_server_options_single.txt

@@ -0,0 +1,62 @@
+{
+	servers {
+		listener_wrappers {
+			tls
+		}
+		timeouts {
+			read_body 30s
+			read_header 30s
+			write 30s
+			idle 30s
+		}
+		max_header_size 100MB
+		protocol {
+			allow_h2c
+			experimental_http3
+			strict_sni_host
+		}
+	}
+}
+
+foo.com {	
+}
+
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"listener_wrappers": [
+						{
+							"wrapper": "tls"
+						}
+					],
+					"read_timeout": 30000000000,
+					"read_header_timeout": 30000000000,
+					"write_timeout": 30000000000,
+					"idle_timeout": 30000000000,
+					"max_header_bytes": 100000000,
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"foo.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"strict_sni_host": true,
+					"experimental_http3": true,
+					"allow_h2c": true
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/handle_path_sorting.txt

@@ -0,0 +1,105 @@
+:80 {
+	handle /api/* {
+		respond "api"
+	}
+
+	handle_path /static/* {
+		respond "static"
+	}
+
+	handle {
+		respond "handle"
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"group": "group3",
+							"match": [
+								{
+									"path": [
+										"/static/*"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "rewrite",
+													"strip_path_prefix": "/static"
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"body": "static",
+													"handler": "static_response"
+												}
+											]
+										}
+									]
+								}
+							]
+						},
+						{
+							"group": "group3",
+							"match": [
+								{
+									"path": [
+										"/api/*"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"body": "api",
+													"handler": "static_response"
+												}
+											]
+										}
+									]
+								}
+							]
+						},
+						{
+							"group": "group3",
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"body": "handle",
+													"handler": "static_response"
+												}
+											]
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/header.txt

@@ -0,0 +1,132 @@
+:80 {
+	header Denis "Ritchie"
+	header +Edsger "Dijkstra"
+	header ?John "von Neumann"
+	header -Wolfram
+	header {
+		Grace: "Hopper"  # some users habitually suffix field names with a colon
+		+Ray "Solomonoff"
+		?Tim "Berners-Lee"
+		defer
+	}
+	@images path /images/*
+	header @images {
+		Cache-Control "public, max-age=3600, stale-while-revalidate=86400"
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"path": [
+										"/images/*"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "headers",
+									"response": {
+										"set": {
+											"Cache-Control": [
+												"public, max-age=3600, stale-while-revalidate=86400"
+											]
+										}
+									}
+								}
+							]
+						},
+						{
+							"handle": [
+								{
+									"handler": "headers",
+									"response": {
+										"set": {
+											"Denis": [
+												"Ritchie"
+											]
+										}
+									}
+								},
+								{
+									"handler": "headers",
+									"response": {
+										"add": {
+											"Edsger": [
+												"Dijkstra"
+											]
+										}
+									}
+								},
+								{
+									"handler": "headers",
+									"response": {
+										"require": {
+											"headers": {
+												"John": null
+											}
+										},
+										"set": {
+											"John": [
+												"von Neumann"
+											]
+										}
+									}
+								},
+								{
+									"handler": "headers",
+									"response": {
+										"deferred": true,
+										"delete": [
+											"Wolfram"
+										]
+									}
+								},
+								{
+									"handler": "headers",
+									"response": {
+										"add": {
+											"Ray": [
+												"Solomonoff"
+											]
+										},
+										"deferred": true,
+										"set": {
+											"Grace": [
+												"Hopper"
+											]
+										}
+									}
+								},
+								{
+									"handler": "headers",
+									"response": {
+										"require": {
+											"headers": {
+												"Tim": null
+											}
+										},
+										"set": {
+											"Tim": [
+												"Berners-Lee"
+											]
+										}
+									}
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/log_except_catchall_blocks.txt

@@ -0,0 +1,78 @@
+http://localhost:2020 {
+	log
+	respond 200
+}
+
+:2020 {
+	respond 418
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":2020"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "static_response",
+													"status_code": 200
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "static_response",
+													"status_code": 418
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"automatic_https": {
+						"skip": [
+							"localhost"
+						]
+					},
+					"logs": {
+						"logger_names": {
+							"localhost:2020": ""
+						},
+						"skip_unmapped_hosts": true
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/log_filters.txt

@@ -0,0 +1,69 @@
+:80
+
+log {
+	output stdout
+	format filter {
+		wrap console
+		fields {
+			request>headers>Authorization delete
+			request>headers>Server delete
+			request>remote_addr ip_mask {
+				ipv4 24
+				ipv6 32
+			}
+		}
+	}
+}
+----------
+{
+	"logging": {
+		"logs": {
+			"default": {
+				"exclude": [
+					"http.log.access.log0"
+				]
+			},
+			"log0": {
+				"writer": {
+					"output": "stdout"
+				},
+				"encoder": {
+					"fields": {
+						"request\u003eheaders\u003eAuthorization": {
+							"filter": "delete"
+						},
+						"request\u003eheaders\u003eServer": {
+							"filter": "delete"
+						},
+						"request\u003eremote_addr": {
+							"filter": "ip_mask",
+							"ipv4_cidr": 24,
+							"ipv6_cidr": 32
+						}
+					},
+					"format": "filter",
+					"wrap": {
+						"format": "console"
+					}
+				},
+				"include": [
+					"http.log.access.log0"
+				]
+			}
+		}
+	},
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"logs": {
+						"default_logger_name": "log0"
+					}
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/matcher_syntax.txt

@@ -9,6 +9,34 @@
 
 	@matcher3 not method PUT
 	respond @matcher3 "not put"
+
+	@matcher4 vars "{http.request.uri}" "/vars-matcher"
+	respond @matcher4 "from vars matcher"
+
+	@matcher5 vars_regexp static "{http.request.uri}" `\.([a-f0-9]{6})\.(css|js)$`
+	respond @matcher5 "from vars_regexp matcher with name"
+
+	@matcher6 vars_regexp "{http.request.uri}" `\.([a-f0-9]{6})\.(css|js)$`
+	respond @matcher6 "from vars_regexp matcher without name"
+
+	@matcher7 {
+		header Foo bar
+		header Foo foobar
+		header Bar foo
+	}
+	respond @matcher7 "header matcher merging values of the same field"
+
+	@matcher8 {
+		query foo=bar foo=baz bar=foo
+		query bar=baz
+	}
+	respond @matcher8 "query matcher merging pairs with the same keys"
+
+	@matcher9 {
+		header !Foo
+		header Bar foo
+	}
+	respond @matcher9 "header matcher with null field matcher"
 }
 ----------
 {
@@ -68,6 +96,117 @@
 									"handler": "static_response"
 								}
 							]
+						},
+						{
+							"match": [
+								{
+									"vars": {
+										"{http.request.uri}": "/vars-matcher"
+									}
+								}
+							],
+							"handle": [
+								{
+									"body": "from vars matcher",
+									"handler": "static_response"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"vars_regexp": {
+										"{http.request.uri}": {
+											"name": "static",
+											"pattern": "\\.([a-f0-9]{6})\\.(css|js)$"
+										}
+									}
+								}
+							],
+							"handle": [
+								{
+									"body": "from vars_regexp matcher with name",
+									"handler": "static_response"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"vars_regexp": {
+										"{http.request.uri}": {
+											"pattern": "\\.([a-f0-9]{6})\\.(css|js)$"
+										}
+									}
+								}
+							],
+							"handle": [
+								{
+									"body": "from vars_regexp matcher without name",
+									"handler": "static_response"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"header": {
+										"Bar": [
+											"foo"
+										],
+										"Foo": [
+											"bar",
+											"foobar"
+										]
+									}
+								}
+							],
+							"handle": [
+								{
+									"body": "header matcher merging values of the same field",
+									"handler": "static_response"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"query": {
+										"bar": [
+											"foo",
+											"baz"
+										],
+										"foo": [
+											"bar",
+											"baz"
+										]
+									}
+								}
+							],
+							"handle": [
+								{
+									"body": "query matcher merging pairs with the same keys",
+									"handler": "static_response"
+								}
+							]
+						},
+						{
+							"match": [
+								{
+									"header": {
+										"Bar": [
+											"foo"
+										],
+										"Foo": null
+									}
+								}
+							],
+							"handle": [
+								{
+									"body": "header matcher with null field matcher",
+									"handler": "static_response"
+								}
+							]
 						}
 					]
 				}

caddytest/integration/caddyfile_adapt/matchers_in_route.txt

@@ -0,0 +1,31 @@
+:80 {
+    route {
+        # unused matchers should not panic
+        # see https://github.com/caddyserver/caddy/issues/3745
+        @matcher1 path /path1
+        @matcher2 path /path2
+    }
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "subroute"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/metrics_disable_om.txt

@@ -0,0 +1,36 @@
+:80 {
+	metrics /metrics {
+		disable_openmetrics
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"path": [
+										"/metrics"
+									]
+								}
+							],
+							"handle": [
+								{
+									"disable_openmetrics": true,
+									"handler": "metrics"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/metrics_syntax.txt

@@ -0,0 +1,33 @@
+:80 {
+	metrics /metrics
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"path": [
+										"/metrics"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "metrics"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/php_fastcgi_expanded_form.txt

@@ -0,0 +1,132 @@
+:8886
+
+route {
+	# Add trailing slash for directory requests
+	@canonicalPath {
+		file {
+			try_files {path}/index.php
+		}
+		not path */
+	}
+	redir @canonicalPath {path}/ 308
+
+	# If the requested file does not exist, try index files
+	@indexFiles {
+		file {
+			try_files {path} {path}/index.php index.php
+			split_path .php
+		}
+	}
+	rewrite @indexFiles {http.matchers.file.relative}
+
+	# Proxy PHP files to the FastCGI responder
+	@phpFiles {
+		path *.php
+	}
+	reverse_proxy @phpFiles 127.0.0.1:9000 {
+		transport fastcgi {
+			split .php
+		}
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8886"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "static_response",
+													"headers": {
+														"Location": [
+															"{http.request.uri.path}/"
+														]
+													},
+													"status_code": 308
+												}
+											],
+											"match": [
+												{
+													"file": {
+														"try_files": [
+															"{http.request.uri.path}/index.php"
+														]
+													},
+													"not": [
+														{
+															"path": [
+																"*/"
+															]
+														}
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"handler": "rewrite",
+													"uri": "{http.matchers.file.relative}"
+												}
+											],
+											"match": [
+												{
+													"file": {
+														"split_path": [
+															".php"
+														],
+														"try_files": [
+															"{http.request.uri.path}",
+															"{http.request.uri.path}/index.php",
+															"index.php"
+														]
+													}
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"handler": "reverse_proxy",
+													"transport": {
+														"protocol": "fastcgi",
+														"split_path": [
+															".php"
+														]
+													},
+													"upstreams": [
+														{
+															"dial": "127.0.0.1:9000"
+														}
+													]
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"*.php"
+													]
+												}
+											]
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/php_fastcgi_index_off.txt

@@ -1,15 +1,18 @@
 :8884
 
 php_fastcgi localhost:9000 {
-    # some php_fastcgi-specific subdirectives
+	# some php_fastcgi-specific subdirectives
-    split .php .php5
+	split .php .php5
-    env VAR1 value1
+	env VAR1 value1
-    env VAR2 value2
+	env VAR2 value2
-    root /var/www
+	root /var/www
-    index off
+	index off
+	dial_timeout 3s
+	read_timeout 10s
+	write_timeout 20s
 
-    # passed through to reverse_proxy (directive order doesn't matter!)
+	# passed through to reverse_proxy (directive order doesn't matter!)
-    lb_policy random
+	lb_policy random
 }
 ----------
 {
@@ -39,16 +42,19 @@ php_fastcgi localhost:9000 {
 										}
 									},
 									"transport": {
+										"dial_timeout": 3000000000,
 										"env": {
 											"VAR1": "value1",
 											"VAR2": "value2"
 										},
 										"protocol": "fastcgi",
+										"read_timeout": 10000000000,
 										"root": "/var/www",
 										"split_path": [
 											".php",
 											".php5"
-										]
+										],
+										"write_timeout": 20000000000
 									},
 									"upstreams": [
 										{

caddytest/integration/caddyfile_adapt/portless_upstream.txt

@@ -0,0 +1,113 @@
+whoami.example.com {
+	reverse_proxy whoami
+}
+
+app.example.com {
+	reverse_proxy app:80
+}
+unix.example.com {
+	reverse_proxy unix//path/to/socket
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"whoami.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "reverse_proxy",
+													"upstreams": [
+														{
+															"dial": "whoami:80"
+														}
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"unix.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "reverse_proxy",
+													"upstreams": [
+														{
+															"dial": "unix//path/to/socket"
+														}
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"app.example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "reverse_proxy",
+													"upstreams": [
+														{
+															"dial": "app:80"
+														}
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/request_body.txt

@@ -0,0 +1,45 @@
+localhost
+request_body {
+  max_size 1MB
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "request_body",
+													"max_size": 1000000
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/reverse_proxy_empty_non_http_transport.txt

@@ -0,0 +1,36 @@
+:8884
+
+reverse_proxy 127.0.0.1:65535 {
+    transport fastcgi
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "reverse_proxy",
+									"transport": {
+										"protocol": "fastcgi"
+									},
+									"upstreams": [
+										{
+											"dial": "127.0.0.1:65535"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/reverse_proxy_h2c_shorthand.txt

@@ -0,0 +1,38 @@
+:8884
+
+reverse_proxy h2c://localhost:8080
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "reverse_proxy",
+									"transport": {
+										"protocol": "http",
+										"versions": [
+											"h2c",
+											"2"
+										]
+									},
+									"upstreams": [
+										{
+											"dial": "localhost:8080"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/reverse_proxy_options.txt

@@ -0,0 +1,119 @@
+
+https://example.com {
+	reverse_proxy /path http://localhost:54321 {
+		header_up Host {host}
+		header_up X-Real-IP {remote}
+		header_up X-Forwarded-For {remote}
+		header_up X-Forwarded-Port {server_port}
+		header_up X-Forwarded-Proto "http"
+
+		buffer_requests
+
+		transport http {
+			read_buffer 10MB
+			write_buffer 20MB
+			max_response_header 30MB
+			dial_timeout 3s
+			dial_fallback_delay 5s
+			response_header_timeout 8s
+			expect_continue_timeout 9s
+
+			versions h2c 2
+			compression off
+			max_conns_per_host 5
+			max_idle_conns_per_host 2
+		}
+	}
+}
+
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"buffer_requests": true,
+													"handler": "reverse_proxy",
+													"headers": {
+														"request": {
+															"set": {
+																"Host": [
+																	"{http.request.host}"
+																],
+																"X-Forwarded-For": [
+																	"{http.request.remote}"
+																],
+																"X-Forwarded-Port": [
+																	"{server_port}"
+																],
+																"X-Forwarded-Proto": [
+																	"http"
+																],
+																"X-Real-Ip": [
+																	"{http.request.remote}"
+																]
+															}
+														}
+													},
+													"transport": {
+														"compression": false,
+														"dial_fallback_delay": 5000000000,
+														"dial_timeout": 3000000000,
+														"expect_continue_timeout": 9000000000,
+														"max_conns_per_host": 5,
+														"max_idle_conns_per_host": 2,
+														"max_response_header_size": 30000000,
+														"protocol": "http",
+														"read_buffer_size": 10000000,
+														"response_header_timeout": 8000000000,
+														"versions": [
+															"h2c",
+															"2"
+														],
+														"write_buffer_size": 20000000
+													},
+													"upstreams": [
+														{
+															"dial": "localhost:54321"
+														}
+													]
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/path"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/sort_directives_with_any_matcher_first.txt

@@ -0,0 +1,51 @@
+:80
+
+respond 200
+
+@untrusted not remote_ip 10.1.1.0/24
+respond @untrusted 401
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"not": [
+										{
+											"remote_ip": {
+												"ranges": [
+													"10.1.1.0/24"
+												]
+											}
+										}
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "static_response",
+									"status_code": 401
+								}
+							]
+						},
+						{
+							"handle": [
+								{
+									"handler": "static_response",
+									"status_code": 200
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_automation_policies.txt

@@ -0,0 +1,86 @@
+{
+	local_certs
+}
+
+*.tld, *.*.tld {
+	tls {
+		on_demand
+	}
+}
+
+foo.tld, www.foo.tld {
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"foo.tld",
+										"www.foo.tld"
+									]
+								}
+							],
+							"terminal": true
+						},
+						{
+							"match": [
+								{
+									"host": [
+										"*.tld",
+										"*.*.tld"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"foo.tld",
+							"www.foo.tld"
+						],
+						"issuers": [
+							{
+								"module": "internal"
+							}
+						]
+					},
+					{
+						"subjects": [
+							"*.*.tld",
+							"*.tld"
+						],
+						"issuers": [
+							{
+								"module": "internal"
+							}
+						],
+						"on_demand": true
+					},
+					{
+						"issuers": [
+							{
+								"module": "internal"
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_conn_policy_consolidate.txt

@@ -0,0 +1,137 @@
+# https://github.com/caddyserver/caddy/issues/3906
+a.a {
+	tls internal
+	respond 403
+}
+
+http://b.b https://b.b:8443 {
+	tls internal
+	respond 404
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"a.a"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "static_response",
+													"status_code": 403
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				},
+				"srv1": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"b.b"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "static_response",
+													"status_code": 404
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"automatic_https": {
+						"skip": [
+							"b.b"
+						]
+					}
+				},
+				"srv2": {
+					"listen": [
+						":8443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"b.b"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "static_response",
+													"status_code": 404
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"a.a",
+							"b.b"
+						],
+						"issuers": [
+							{
+								"module": "internal"
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}

caddytest/integration/handler_test.go

@@ -0,0 +1,28 @@
+package integration
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/caddyserver/caddy/v2/caddytest"
+)
+
+func TestBrowse(t *testing.T) {
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+	{
+		http_port     9080
+		https_port    9443
+	}
+	http://localhost:9080 {
+		file_server browse
+	}
+  `, "caddyfile")
+
+	req, err := http.NewRequest(http.MethodGet, "http://localhost:9080/", nil)
+	if err != nil {
+		t.Fail()
+		return
+	}
+	tester.AssertResponseCode(req, 200)
+}

caddytest/integration/map_test.go

@@ -8,7 +8,6 @@ import (
 )
 
 func TestMap(t *testing.T) {
-
 	// arrange
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`{
@@ -18,25 +17,24 @@ func TestMap(t *testing.T) {
 
 	localhost:9080 {
 
-		map http.request.method dest-name {
+		map {http.request.method} {dest-1} {dest-2} {
-			default unknown
+			default unknown1    unknown2
-			G.T get-called
+			~G.T    get-called
-			POST post-called
+			POST    post-called foobar
 		}
 
 		respond /version 200 {
-			body "hello from localhost {dest-name}"
+			body "hello from localhost {dest-1} {dest-2}"
 		}	
 	}
 	`, "caddyfile")
 
 	// act and assert
-	tester.AssertGetResponse("http://localhost:9080/version", 200, "hello from localhost get-called")
+	tester.AssertGetResponse("http://localhost:9080/version", 200, "hello from localhost get-called unknown2")
-	tester.AssertPostResponseBody("http://localhost:9080/version", []string{}, bytes.NewBuffer([]byte{}), 200, "hello from localhost post-called")
+	tester.AssertPostResponseBody("http://localhost:9080/version", []string{}, bytes.NewBuffer([]byte{}), 200, "hello from localhost post-called foobar")
 }
 
 func TestMapRespondWithDefault(t *testing.T) {
-
 	// arrange
 	tester := caddytest.NewTester(t)
 	tester.InitServer(`{
@@ -46,9 +44,9 @@ func TestMapRespondWithDefault(t *testing.T) {
 		
 		localhost:9080 {
 	
-			map http.request.method dest-name {
+			map {http.request.method} {dest-name} {
 				default unknown
-				GET get-called
+				GET     get-called
 			}
 		
 			respond /version 200 {
@@ -63,80 +61,75 @@ func TestMapRespondWithDefault(t *testing.T) {
 }
 
 func TestMapAsJson(t *testing.T) {
-
 	// arrange
 	tester := caddytest.NewTester(t)
-	tester.InitServer(`{
+	tester.InitServer(`
+	{
 		"apps": {
 			"http": {
-			"http_port": 9080,
+				"http_port": 9080,
-			"https_port": 9443,
+				"https_port": 9443,
-			"servers": {
+				"servers": {
-				"srv0": {
+					"srv0": {
-				"listen": [
+						"listen": [
-					":9080"
+							":9080"
-				],
+						],
-				"routes": [
-					{
-					"handle": [
-						{
-						"handler": "subroute",
 						"routes": [
 							{
-							"handle": [
+								"handle": [
-								{
-								"handler": "map",
-								"source": "http.request.method",
-								"destination": "dest-name",
-								"default": "unknown",
-								"items": [
 									{
-									"expression": "GET",
+										"handler": "subroute",
-									"value": "get-called"
+										"routes": [
-									},
+											{
-									{
+												"handle": [
-									"expression": "POST",
+													{
-									"value": "post-called"
+														"handler": "map",
+														"source": "{http.request.method}",
+														"destinations": ["dest-name"],
+														"defaults": ["unknown"],
+														"mappings": [
+															{
+																"input": "GET",
+																"outputs": ["get-called"]
+															},
+															{
+																"input": "POST",
+																"outputs": ["post-called"]
+															}
+														]
+													}
+												]
+											},
+											{
+												"handle": [
+													{
+														"body": "hello from localhost {dest-name}",
+														"handler": "static_response",
+														"status_code": 200
+													}
+												],
+												"match": [
+													{
+														"path": ["/version"]
+													}
+												]
+											}
+										]
 									}
-								]
+								],
-								}
+								"match": [
-							]
+									{
-							},
+										"host": ["localhost"]
-							{
+									}
-							"handle": [
+								],
-								{
+								"terminal": true
-								"body": "hello from localhost {dest-name}",
-								"handler": "static_response",
-								"status_code": 200
-								}
-							],
-							"match": [
-								{
-								"path": [
-									"/version"
-								]
-								}
-							]
 							}
 						]
-						}
-					],
-					"match": [
-						{
-						"host": [
-							"localhost"
-						]
-						}
-					],
-					"terminal": true
 					}
-				]
 				}
 			}
-			}
 		}
-		}
+	}`, "json")
-		`, "json")
 
 	tester.AssertGetResponse("http://localhost:9080/version", 200, "hello from localhost get-called")
 	tester.AssertPostResponseBody("http://localhost:9080/version", []string{}, bytes.NewBuffer([]byte{}), 200, "hello from localhost post-called")

caddytest/integration/reverseproxy_test.go

@@ -0,0 +1,438 @@
+package integration
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/caddyserver/caddy/v2/caddytest"
+)
+
+func TestSRVReverseProxy(t *testing.T) {
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+	{
+		"apps": {
+		  "http": {
+			"servers": {
+			  "srv0": {
+				"listen": [
+				  ":8080"
+				],
+				"routes": [
+				  {
+					"handle": [
+					  {
+						"handler": "reverse_proxy",
+						"upstreams": [
+						  {
+							"lookup_srv": "srv.host.service.consul"
+						  }
+						]
+					  }
+					]
+				  }
+				]
+			  }
+			}
+		  }
+		}
+	  }
+  `, "json")
+}
+
+func TestSRVWithDial(t *testing.T) {
+	caddytest.AssertLoadError(t, `
+	{
+		"apps": {
+		  "http": {
+			"servers": {
+			  "srv0": {
+				"listen": [
+				  ":8080"
+				],
+				"routes": [
+				  {
+					"handle": [
+					  {
+						"handler": "reverse_proxy",
+						"upstreams": [
+						  {
+							"dial": "tcp/address.to.upstream:80",
+							"lookup_srv": "srv.host.service.consul"
+						  }
+						]
+					  }
+					]
+				  }
+				]
+			  }
+			}
+		  }
+		}
+	  }
+	`, "json", `upstream: specifying dial address is incompatible with lookup_srv: 0: {\"dial\": \"tcp/address.to.upstream:80\", \"lookup_srv\": \"srv.host.service.consul\"}`)
+}
+
+func TestDialWithPlaceholderUnix(t *testing.T) {
+
+	if runtime.GOOS == "windows" {
+		t.SkipNow()
+	}
+
+	f, err := ioutil.TempFile("", "*.sock")
+	if err != nil {
+		t.Errorf("failed to create TempFile: %s", err)
+		return
+	}
+	// a hack to get a file name within a valid path to use as socket
+	socketName := f.Name()
+	os.Remove(f.Name())
+
+	server := http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			w.Write([]byte("Hello, World!"))
+		}),
+	}
+
+	unixListener, err := net.Listen("unix", socketName)
+	if err != nil {
+		t.Errorf("failed to listen on the socket: %s", err)
+		return
+	}
+	go server.Serve(unixListener)
+	t.Cleanup(func() {
+		server.Close()
+	})
+	runtime.Gosched() // Allow other goroutines to run
+
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+	{
+		"apps": {
+		  "http": {
+			"servers": {
+			  "srv0": {
+				"listen": [
+				  ":8080"
+				],
+				"routes": [
+				  {
+					"handle": [
+					  {
+						"handler": "reverse_proxy",
+						"upstreams": [
+						  {
+							"dial": "unix/{http.request.header.X-Caddy-Upstream-Dial}"
+						  }
+						]
+					  }
+					]
+				  }
+				]
+			  }
+			}
+		  }
+		}
+	  }
+	`, "json")
+
+	req, err := http.NewRequest(http.MethodGet, "http://localhost:8080", nil)
+	if err != nil {
+		t.Fail()
+		return
+	}
+	req.Header.Set("X-Caddy-Upstream-Dial", socketName)
+	tester.AssertResponse(req, 200, "Hello, World!")
+}
+
+func TestReverseProxyWithPlaceholderDialAddress(t *testing.T) {
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+	{
+		"apps": {
+			"http": {
+				"servers": {
+					"srv0": {
+						"listen": [
+							":8080"
+						],
+						"routes": [
+							{
+								"match": [
+									{
+										"host": [
+											"localhost"
+										]
+									}
+								],
+								"handle": [
+									{
+										"handler": "static_response",
+										"body": "Hello, World!"
+									}
+								],
+								"terminal": true
+							}
+						],
+						"automatic_https": {
+							"skip": [
+								"localhost"
+							]
+						}
+					},
+					"srv1": {
+						"listen": [
+							":9080"
+						],
+						"routes": [
+							{
+								"match": [
+									{
+										"host": [
+											"localhost"
+										]
+									}
+								],
+								"handle": [
+									{
+	
+										"handler": "reverse_proxy",
+										"upstreams": [
+											{
+												"dial": "{http.request.header.X-Caddy-Upstream-Dial}"
+											}
+										]
+									}
+								],
+								"terminal": true
+							}
+						],
+						"automatic_https": {
+							"skip": [
+								"localhost"
+							]
+						}
+					}
+				}
+			}
+		}
+	}
+  	`, "json")
+
+	req, err := http.NewRequest(http.MethodGet, "http://localhost:9080", nil)
+	if err != nil {
+		t.Fail()
+		return
+	}
+	req.Header.Set("X-Caddy-Upstream-Dial", "localhost:8080")
+	tester.AssertResponse(req, 200, "Hello, World!")
+}
+
+func TestReverseProxyWithPlaceholderTCPDialAddress(t *testing.T) {
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+	{
+		"apps": {
+			"http": {
+				"servers": {
+					"srv0": {
+						"listen": [
+							":8080"
+						],
+						"routes": [
+							{
+								"match": [
+									{
+										"host": [
+											"localhost"
+										]
+									}
+								],
+								"handle": [
+									{
+										"handler": "static_response",
+										"body": "Hello, World!"
+									}
+								],
+								"terminal": true
+							}
+						],
+						"automatic_https": {
+							"skip": [
+								"localhost"
+							]
+						}
+					},
+					"srv1": {
+						"listen": [
+							":9080"
+						],
+						"routes": [
+							{
+								"match": [
+									{
+										"host": [
+											"localhost"
+										]
+									}
+								],
+								"handle": [
+									{
+	
+										"handler": "reverse_proxy",
+										"upstreams": [
+											{
+												"dial": "tcp/{http.request.header.X-Caddy-Upstream-Dial}:8080"
+											}
+										]
+									}
+								],
+								"terminal": true
+							}
+						],
+						"automatic_https": {
+							"skip": [
+								"localhost"
+							]
+						}
+					}
+				}
+			}
+		}
+	}
+  	`, "json")
+
+	req, err := http.NewRequest(http.MethodGet, "http://localhost:9080", nil)
+	if err != nil {
+		t.Fail()
+		return
+	}
+	req.Header.Set("X-Caddy-Upstream-Dial", "localhost")
+	tester.AssertResponse(req, 200, "Hello, World!")
+}
+
+func TestSRVWithActiveHealthcheck(t *testing.T) {
+	caddytest.AssertLoadError(t, `
+	{
+		"apps": {
+		  "http": {
+			"servers": {
+			  "srv0": {
+				"listen": [
+				  ":8080"
+				],
+				"routes": [
+				  {
+					"handle": [
+					  {
+						"handler": "reverse_proxy",
+						"health_checks": {
+							"active": {
+								"path": "/ok"
+							}
+						},
+						"upstreams": [
+						  {
+							"lookup_srv": "srv.host.service.consul"
+						  }
+						]
+					  }
+					]
+				  }
+				]
+			  }
+			}
+		  }
+		}
+	  }
+	`, "json", `upstream: lookup_srv is incompatible with active health checks: 0: {\"dial\": \"\", \"lookup_srv\": \"srv.host.service.consul\"}`)
+}
+
+func TestReverseProxyHealthCheck(t *testing.T) {
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+	{
+		http_port     9080
+		https_port    9443
+	}
+	http://localhost:2020 {
+		respond "Hello, World!"
+	}
+	http://localhost:2021 {
+		respond "ok"
+	}
+	http://localhost:9080 {
+		reverse_proxy {
+			to localhost:2020
+	
+			health_path /health
+			health_port 2021
+			health_interval 2s
+			health_timeout 5s
+		}
+	}
+  `, "caddyfile")
+
+	tester.AssertGetResponse("http://localhost:9080/", 200, "Hello, World!")
+}
+
+func TestReverseProxyHealthCheckUnixSocket(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.SkipNow()
+	}
+	tester := caddytest.NewTester(t)
+	f, err := ioutil.TempFile("", "*.sock")
+	if err != nil {
+		t.Errorf("failed to create TempFile: %s", err)
+		return
+	}
+	// a hack to get a file name within a valid path to use as socket
+	socketName := f.Name()
+	os.Remove(f.Name())
+
+	server := http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			if strings.HasPrefix(req.URL.Path, "/health") {
+				w.Write([]byte("ok"))
+				return
+			}
+			w.Write([]byte("Hello, World!"))
+		}),
+	}
+
+	unixListener, err := net.Listen("unix", socketName)
+	if err != nil {
+		t.Errorf("failed to listen on the socket: %s", err)
+		return
+	}
+	go server.Serve(unixListener)
+	t.Cleanup(func() {
+		server.Close()
+	})
+	runtime.Gosched() // Allow other goroutines to run
+
+	tester.InitServer(fmt.Sprintf(`
+	{
+		http_port     9080
+		https_port    9443
+	}
+	http://localhost:9080 {
+		reverse_proxy {
+			to unix/%s
+	
+			health_path /health
+			health_port 2021
+			health_interval 2s
+			health_timeout 5s
+		}
+	}
+	`, socketName), "caddyfile")
+
+	tester.AssertGetResponse("http://localhost:9080/", 200, "Hello, World!")
+}

caddytest/integration/sni_test.go

@@ -99,7 +99,7 @@ func TestDefaultSNI(t *testing.T) {
 
 	// act and assert
 	// makes a request with no sni
-	tester.AssertGetResponse("https://127.0.0.1:9443/version", 200, "hello from a")
+	tester.AssertGetResponse("https://127.0.0.1:9443/version", 200, "hello from a.caddy.localhost")
 }
 
 func TestDefaultSNIWithNamedHostAndExplicitIP(t *testing.T) {
@@ -204,7 +204,6 @@ func TestDefaultSNIWithNamedHostAndExplicitIP(t *testing.T) {
 }
 
 func TestDefaultSNIWithPortMappingOnly(t *testing.T) {
-
 	// arrange
 	tester := caddytest.NewTester(t)
 	tester.InitServer(` 
@@ -273,7 +272,7 @@ func TestDefaultSNIWithPortMappingOnly(t *testing.T) {
 
 	// act and assert
 	// makes a request with no sni
-	tester.AssertGetResponse("https://127.0.0.1:9443/version", 200, "hello from a")
+	tester.AssertGetResponse("https://127.0.0.1:9443/version", 200, "hello from a.caddy.localhost")
 }
 
 func TestHttpOnlyOnDomainWithSNI(t *testing.T) {

caddytest/integration/stream_test.go

@@ -0,0 +1,437 @@
+package integration
+
+import (
+	"compress/gzip"
+	"context"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/caddyserver/caddy/v2/caddytest"
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
+)
+
+// (see https://github.com/caddyserver/caddy/issues/3556 for use case)
+func TestH2ToH2CStream(t *testing.T) {
+	tester := caddytest.NewTester(t)
+	tester.InitServer(` 
+  {
+    "apps": {
+      "http": {
+        "http_port": 9080,
+        "https_port": 9443,
+        "servers": {
+          "srv0": {
+            "listen": [
+              ":9443"
+            ],
+            "routes": [
+              {
+                "handle": [
+                  {
+                    "handler": "reverse_proxy",
+                    "transport": {
+                      "protocol": "http",
+                      "compression": false,
+                      "versions": [
+                        "h2c",
+                        "2"
+                      ]
+                    },
+                    "upstreams": [
+                      {
+                        "dial": "localhost:54321"
+                      }
+                    ]
+                  }
+                ],
+                "match": [
+                  {
+                    "path": [
+                      "/tov2ray"
+                    ]
+                  }
+                ]
+              }
+            ],
+            "tls_connection_policies": [
+              {
+                "certificate_selection": {
+                  "any_tag": ["cert0"]
+                },
+                "default_sni": "a.caddy.localhost"
+              }
+            ]
+          }
+        }
+      },
+      "tls": {
+        "certificates": {
+          "load_files": [
+            {
+              "certificate": "/a.caddy.localhost.crt",
+              "key": "/a.caddy.localhost.key",
+              "tags": [
+                "cert0"
+              ]
+            }
+          ]
+        }
+      },
+      "pki": {
+        "certificate_authorities" : {
+          "local" : {
+            "install_trust": false
+          }
+        }
+      }
+    }
+  }
+  `, "json")
+
+	expectedBody := "some data to be echoed"
+	// start the server
+	server := testH2ToH2CStreamServeH2C(t)
+	go server.ListenAndServe()
+	defer func() {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond)
+		defer cancel()
+		server.Shutdown(ctx)
+	}()
+
+	r, w := io.Pipe()
+	req := &http.Request{
+		Method: "PUT",
+		Body:   ioutil.NopCloser(r),
+		URL: &url.URL{
+			Scheme: "https",
+			Host:   "127.0.0.1:9443",
+			Path:   "/tov2ray",
+		},
+		Proto:      "HTTP/2",
+		ProtoMajor: 2,
+		ProtoMinor: 0,
+		Header:     make(http.Header),
+	}
+	// Disable any compression method from server.
+	req.Header.Set("Accept-Encoding", "identity")
+
+	resp := tester.AssertResponseCode(req, 200)
+	if 200 != resp.StatusCode {
+		return
+	}
+	go func() {
+		fmt.Fprint(w, expectedBody)
+		w.Close()
+	}()
+
+	defer resp.Body.Close()
+	bytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("unable to read the response body %s", err)
+	}
+
+	body := string(bytes)
+
+	if !strings.Contains(body, expectedBody) {
+		t.Errorf("requesting \"%s\" expected response body \"%s\" but got \"%s\"", req.RequestURI, expectedBody, body)
+	}
+	return
+}
+
+func testH2ToH2CStreamServeH2C(t *testing.T) *http.Server {
+	h2s := &http2.Server{}
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		rstring, err := httputil.DumpRequest(r, false)
+		if err == nil {
+			t.Logf("h2c server received req: %s", rstring)
+		}
+		// We only accept HTTP/2!
+		if r.ProtoMajor != 2 {
+			t.Error("Not a HTTP/2 request, rejected!")
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+
+		if r.Host != "127.0.0.1:9443" {
+			t.Errorf("r.Host doesn't match, %v!", r.Host)
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+
+		if !strings.HasPrefix(r.URL.Path, "/tov2ray") {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+
+		w.Header().Set("Cache-Control", "no-store")
+		w.WriteHeader(200)
+		if f, ok := w.(http.Flusher); ok {
+			f.Flush()
+		}
+
+		buf := make([]byte, 4*1024)
+
+		for {
+			n, err := r.Body.Read(buf)
+			if n > 0 {
+				w.Write(buf[:n])
+			}
+
+			if err != nil {
+				if err == io.EOF {
+					r.Body.Close()
+				}
+				break
+			}
+		}
+	})
+
+	server := &http.Server{
+		Addr:    "127.0.0.1:54321",
+		Handler: h2c.NewHandler(handler, h2s),
+	}
+	return server
+}
+
+// (see https://github.com/caddyserver/caddy/issues/3606 for use case)
+func TestH2ToH1ChunkedResponse(t *testing.T) {
+	tester := caddytest.NewTester(t)
+	tester.InitServer(` 
+{
+  "logging": {
+    "logs": {
+      "default": {
+        "level": "DEBUG"
+      }
+    }
+  },
+  "apps": {
+    "http": {
+      "http_port": 9080,
+      "https_port": 9443,
+      "servers": {
+        "srv0": {
+          "listen": [
+            ":9443"
+          ],
+          "routes": [
+            {
+              "handle": [
+                {
+                  "handler": "subroute",
+                  "routes": [
+                    {
+                      "handle": [
+                        {
+                          "encodings": {
+                            "gzip": {}
+                          },
+                          "handler": "encode"
+                        }
+                      ]
+                    },
+                    {
+                      "handle": [
+                        {
+                          "handler": "reverse_proxy",
+                          "upstreams": [
+                            {
+                              "dial": "localhost:54321"
+                            }
+                          ]
+                        }
+                      ],
+                      "match": [
+                        {
+                          "path": [
+                            "/tov2ray"
+                          ]
+                        }
+                      ]
+                    }
+                  ]
+                }
+              ],
+              "terminal": true
+            }
+          ],
+          "tls_connection_policies": [
+            {
+              "certificate_selection": {
+                "any_tag": [
+                  "cert0"
+                ]
+              },
+              "default_sni": "a.caddy.localhost"
+            }
+          ]
+        }
+      }
+    },
+    "tls": {
+      "certificates": {
+        "load_files": [
+          {
+            "certificate": "/a.caddy.localhost.crt",
+            "key": "/a.caddy.localhost.key",
+            "tags": [
+              "cert0"
+            ]
+          }
+        ]
+      }
+    },
+    "pki": {
+      "certificate_authorities": {
+        "local": {
+          "install_trust": false
+        }
+      }
+    }
+  }
+}
+  `, "json")
+
+	// need a large body here to trigger caddy's compression, larger than gzip.miniLength
+	expectedBody, err := GenerateRandomString(1024)
+	if err != nil {
+		t.Fatalf("generate expected body failed, err: %s", err)
+	}
+
+	// start the server
+	server := testH2ToH1ChunkedResponseServeH1(t)
+	go server.ListenAndServe()
+	defer func() {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond)
+		defer cancel()
+		server.Shutdown(ctx)
+	}()
+
+	r, w := io.Pipe()
+	req := &http.Request{
+		Method: "PUT",
+		Body:   ioutil.NopCloser(r),
+		URL: &url.URL{
+			Scheme: "https",
+			Host:   "127.0.0.1:9443",
+			Path:   "/tov2ray",
+		},
+		Proto:      "HTTP/2",
+		ProtoMajor: 2,
+		ProtoMinor: 0,
+		Header:     make(http.Header),
+	}
+	// underlying transport will automaticlly add gzip
+	// req.Header.Set("Accept-Encoding", "gzip")
+	go func() {
+		fmt.Fprint(w, expectedBody)
+		w.Close()
+	}()
+	resp := tester.AssertResponseCode(req, 200)
+	if 200 != resp.StatusCode {
+		return
+	}
+
+	defer resp.Body.Close()
+	bytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("unable to read the response body %s", err)
+	}
+
+	body := string(bytes)
+
+	if body != expectedBody {
+		t.Errorf("requesting \"%s\" expected response body \"%s\" but got \"%s\"", req.RequestURI, expectedBody, body)
+	}
+	return
+}
+
+func testH2ToH1ChunkedResponseServeH1(t *testing.T) *http.Server {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.Host != "127.0.0.1:9443" {
+			t.Errorf("r.Host doesn't match, %v!", r.Host)
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+
+		if !strings.HasPrefix(r.URL.Path, "/tov2ray") {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+
+		defer r.Body.Close()
+		bytes, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Fatalf("unable to read the response body %s", err)
+		}
+
+		n := len(bytes)
+
+		var writer io.Writer
+		if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+			gw, err := gzip.NewWriterLevel(w, 5)
+			if err != nil {
+				t.Error("can't return gzip data")
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+			defer gw.Close()
+			writer = gw
+			w.Header().Set("Content-Encoding", "gzip")
+			w.Header().Del("Content-Length")
+			w.WriteHeader(200)
+		} else {
+			writer = w
+		}
+		if n > 0 {
+			writer.Write(bytes[:])
+		}
+	})
+
+	server := &http.Server{
+		Addr:    "127.0.0.1:54321",
+		Handler: handler,
+	}
+	return server
+}
+
+// GenerateRandomBytes returns securely generated random bytes.
+// It will return an error if the system's secure random
+// number generator fails to function correctly, in which
+// case the caller should not continue.
+func GenerateRandomBytes(n int) ([]byte, error) {
+	b := make([]byte, n)
+	_, err := rand.Read(b)
+	// Note that err == nil only if we read len(b) bytes.
+	if err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+// GenerateRandomString returns a securely generated random string.
+// It will return an error if the system's secure random
+// number generator fails to function correctly, in which
+// case the caller should not continue.
+func GenerateRandomString(n int) (string, error) {
+	const letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-"
+	bytes, err := GenerateRandomBytes(n)
+	if err != nil {
+		return "", err
+	}
+	for i, b := range bytes {
+		bytes[i] = letters[b%byte(len(letters))]
+	}
+	return string(bytes), nil
+}

cmd/commandfuncs.go

@@ -96,7 +96,7 @@ func cmdStart(fl Flags) (int, error) {
 	// started yet, and writing synchronously would result
 	// in a deadlock
 	go func() {
-		stdinpipe.Write(expect)
+		_, _ = stdinpipe.Write(expect)
 		stdinpipe.Close()
 	}()
 
@@ -533,7 +533,17 @@ func cmdFmt(fl Flags) (int, error) {
 	if formatCmdConfigFile == "" {
 		formatCmdConfigFile = "Caddyfile"
 	}
-	overwrite := fl.Bool("overwrite")
+
+	// as a special case, read from stdin if the file name is "-"
+	if formatCmdConfigFile == "-" {
+		input, err := ioutil.ReadAll(os.Stdin)
+		if err != nil {
+			return caddy.ExitCodeFailedStartup,
+				fmt.Errorf("reading stdin: %v", err)
+		}
+		fmt.Print(string(caddyfile.Format(input)))
+		return caddy.ExitCodeSuccess, nil
+	}
 
 	input, err := ioutil.ReadFile(formatCmdConfigFile)
 	if err != nil {
@@ -543,9 +553,8 @@ func cmdFmt(fl Flags) (int, error) {
 
 	output := caddyfile.Format(input)
 
-	if overwrite {
+	if fl.Bool("overwrite") {
-		err = ioutil.WriteFile(formatCmdConfigFile, output, 0644)
+		if err := ioutil.WriteFile(formatCmdConfigFile, output, 0600); err != nil {
-		if err != nil {
 			return caddy.ExitCodeFailedStartup, nil
 		}
 	} else {

cmd/commands.go

@@ -263,8 +263,12 @@ provisioning stages.`,
 Formats the Caddyfile by adding proper indentation and spaces to improve
 human readability. It prints the result to stdout.
 
-If --write is specified, the output will be written to the config file
+If --overwrite is specified, the output will be written to the config file
-directly instead of printing it.`,
+directly instead of printing it.
+
+If you wish you use stdin instead of a regular file, use - as the path.
+When reading from stdin, the --overwrite flag has no effect: the result
+is always printed to stdout.`,
 		Flags: func() *flag.FlagSet {
 			fs := flag.NewFlagSet("format", flag.ExitOnError)
 			fs.Bool("overwrite", false, "Overwrite the input file with the results")

cmd/main.go

@@ -123,7 +123,11 @@ func loadConfig(configFile, adapterName string) ([]byte, string, error) {
 	var cfgAdapter caddyconfig.Adapter
 	var err error
 	if configFile != "" {
-		config, err = ioutil.ReadFile(configFile)
+		if configFile == "-" {
+			config, err = ioutil.ReadAll(os.Stdin)
+		} else {
+			config, err = ioutil.ReadFile(configFile)
+		}
 		if err != nil {
 			return nil, "", fmt.Errorf("reading config file: %v", err)
 		}
@@ -236,6 +240,7 @@ func watchConfigFile(filename, adapterName string) {
 	}
 
 	// begin poller
+	//nolint:staticcheck
 	for range time.Tick(1 * time.Second) {
 		// get the file info
 		info, err := os.Stat(filename)
@@ -410,6 +415,7 @@ func printEnvironment() {
 	fmt.Printf("caddy.AppDataDir=%s\n", caddy.AppDataDir())
 	fmt.Printf("caddy.AppConfigDir=%s\n", caddy.AppConfigDir())
 	fmt.Printf("caddy.ConfigAutosavePath=%s\n", caddy.ConfigAutosavePath)
+	fmt.Printf("caddy.Version=%s\n", caddy.GoModule().Version)
 	fmt.Printf("runtime.GOOS=%s\n", runtime.GOOS)
 	fmt.Printf("runtime.GOARCH=%s\n", runtime.GOARCH)
 	fmt.Printf("runtime.Compiler=%s\n", runtime.Compiler)

cmd/proc_posix.go

@@ -1,37 +0,0 @@
-// Copyright 2015 Matthew Holt and The Caddy Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build !windows
-
-package caddycmd
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"syscall"
-)
-
-func gracefullyStopProcess(pid int) error {
-	fmt.Print("Graceful stop... ")
-	err := syscall.Kill(pid, syscall.SIGINT)
-	if err != nil {
-		return fmt.Errorf("kill: %v", err)
-	}
-	return nil
-}
-
-func getProcessName() string {
-	return filepath.Base(os.Args[0])
-}

cmd/proc_windows.go

@@ -1,44 +0,0 @@
-// Copyright 2015 Matthew Holt and The Caddy Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package caddycmd
-
-import (
-	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strconv"
-)
-
-func gracefullyStopProcess(pid int) error {
-	fmt.Print("Forceful stop... ")
-	// process on windows will not stop unless forced with /f
-	cmd := exec.Command("taskkill", "/pid", strconv.Itoa(pid), "/f")
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("taskkill: %v", err)
-	}
-	return nil
-}
-
-// On Windows the app name passed in os.Args[0] will match how
-// caddy was started eg will match caddy or caddy.exe.
-// So return appname with .exe for consistency
-func getProcessName() string {
-	base := filepath.Base(os.Args[0])
-	if filepath.Ext(base) == "" {
-		return base + ".exe"
-	}
-	return base
-}

go.mod

@@ -4,31 +4,31 @@ go 1.14
 
 require (
 	github.com/Masterminds/sprig/v3 v3.1.0
-	github.com/alecthomas/chroma v0.7.4-0.20200517063913-500529fd43c1
+	github.com/alecthomas/chroma v0.8.2
 	github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a
-	github.com/caddyserver/certmagic v0.11.2
+	github.com/caddyserver/certmagic v0.12.1-0.20201215190346-201f83a06067
 	github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac
-	github.com/go-acme/lego/v3 v3.7.0
 	github.com/go-chi/chi v4.1.2+incompatible
-	github.com/google/cel-go v0.5.1
+	github.com/google/cel-go v0.6.0
 	github.com/jsternberg/zap-logfmt v1.2.0
-	github.com/klauspost/compress v1.10.10
+	github.com/klauspost/compress v1.11.3
-	github.com/klauspost/cpuid v1.3.0
+	github.com/klauspost/cpuid/v2 v2.0.1
-	github.com/libdns/libdns v0.0.0-20200501023120-186724ffc821
+	github.com/lucas-clemente/quic-go v0.19.3
-	github.com/lucas-clemente/quic-go v0.17.1
+	github.com/mholt/acmez v0.1.1
 	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/naoina/toml v0.1.1
-	github.com/smallstep/certificates v0.15.0-rc.1.0.20200506212953-e855707dc274
+	github.com/prometheus/client_golang v1.9.0
-	github.com/smallstep/cli v0.14.4
+	github.com/smallstep/certificates v0.15.4
-	github.com/smallstep/nosql v0.3.0
+	github.com/smallstep/cli v0.15.2
-	github.com/smallstep/truststore v0.9.5
+	github.com/smallstep/nosql v0.3.0 // cannot upgrade from v0.3.0 until protobuf warning is fixed
-	github.com/yuin/goldmark v1.1.32
+	github.com/smallstep/truststore v0.9.6
+	github.com/yuin/goldmark v1.2.1
 	github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691
-	go.uber.org/zap v1.15.0
+	go.uber.org/zap v1.16.0
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
+	golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de
-	golang.org/x/net v0.0.0-20200625001655-4c5254603344
+	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
-	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013
+	google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98
-	google.golang.org/protobuf v1.25.0
+	google.golang.org/protobuf v1.24.0 // cannot upgrade until warning is fixed
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v2 v2.3.0
 )

listeners.go

@@ -129,9 +129,9 @@ func (fcl *fakeCloseListener) Accept() (net.Conn, error) {
 	if *fcl.deadline {
 		switch ln := fcl.Listener.(type) {
 		case *net.TCPListener:
-			ln.SetDeadline(time.Time{})
+			_ = ln.SetDeadline(time.Time{})
 		case *net.UnixListener:
-			ln.SetDeadline(time.Time{})
+			_ = ln.SetDeadline(time.Time{})
 		}
 		*fcl.deadline = false
 	}
@@ -167,9 +167,9 @@ func (fcl *fakeCloseListener) Close() error {
 		if !*fcl.deadline {
 			switch ln := fcl.Listener.(type) {
 			case *net.TCPListener:
-				ln.SetDeadline(time.Now().Add(-1 * time.Minute))
+				_ = ln.SetDeadline(time.Now().Add(-1 * time.Minute))
 			case *net.UnixListener:
-				ln.SetDeadline(time.Now().Add(-1 * time.Minute))
+				_ = ln.SetDeadline(time.Now().Add(-1 * time.Minute))
 			}
 			*fcl.deadline = true
 		}

logging.go

@@ -458,7 +458,7 @@ func (cl *CustomLog) buildCore() {
 		if cl.Sampling.Thereafter == 0 {
 			cl.Sampling.Thereafter = 100
 		}
-		c = zapcore.NewSampler(c, cl.Sampling.Interval,
+		c = zapcore.NewSamplerWithOptions(c, cl.Sampling.Interval,
 			cl.Sampling.First, cl.Sampling.Thereafter)
 	}
 	cl.core = c

metrics.go

@@ -0,0 +1,77 @@
+package caddy
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+// define and register the metrics used in this package.
+func init() {
+	prometheus.MustRegister(prometheus.NewBuildInfoCollector())
+
+	const ns, sub = "caddy", "admin"
+
+	adminMetrics.requestCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: ns,
+		Subsystem: sub,
+		Name:      "http_requests_total",
+		Help:      "Counter of requests made to the Admin API's HTTP endpoints.",
+	}, []string{"handler", "path", "code", "method"})
+	adminMetrics.requestErrors = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: ns,
+		Subsystem: sub,
+		Name:      "http_request_errors_total",
+		Help:      "Number of requests resulting in middleware errors.",
+	}, []string{"handler", "path", "method"})
+}
+
+// adminMetrics is a collection of metrics that can be tracked for the admin API.
+var adminMetrics = struct {
+	requestCount  *prometheus.CounterVec
+	requestErrors *prometheus.CounterVec
+}{}
+
+// Similar to promhttp.InstrumentHandlerCounter, but upper-cases method names
+// instead of lower-casing them.
+//
+// Unlike promhttp.InstrumentHandlerCounter, this assumes a "code" and "method"
+// label is present, and will panic otherwise.
+func instrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler) http.HandlerFunc {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		d := newDelegator(w)
+		next.ServeHTTP(d, r)
+		counter.With(prometheus.Labels{
+			"code":   sanitizeCode(d.status),
+			"method": strings.ToUpper(r.Method),
+		}).Inc()
+	})
+}
+
+func newDelegator(w http.ResponseWriter) *delegator {
+	return &delegator{
+		ResponseWriter: w,
+	}
+}
+
+type delegator struct {
+	http.ResponseWriter
+	status int
+}
+
+func (d *delegator) WriteHeader(code int) {
+	d.status = code
+	d.ResponseWriter.WriteHeader(code)
+}
+
+func sanitizeCode(s int) string {
+	switch s {
+	case 0, 200:
+		return "200"
+	default:
+		return strconv.Itoa(s)
+	}
+}

modules/caddyhttp/app.go

@@ -47,6 +47,7 @@ func init() {
 //
 // Placeholder | Description
 // ------------|---------------
+// `{http.request.body}` | The request body (⚠️ inefficient; use only for debugging)
 // `{http.request.cookie.*}` | HTTP request cookie
 // `{http.request.header.*}` | Specific request header field
 // `{http.request.host.labels.*}` | Request host labels (0-based from right); e.g. for foo.example.com: 0=com, 1=example, 2=foo
@@ -74,6 +75,7 @@ func init() {
 // `{http.request.tls.client.fingerprint}` | The SHA256 checksum of the client certificate
 // `{http.request.tls.client.public_key}` | The public key of the client certificate.
 // `{http.request.tls.client.public_key_sha256}` | The SHA256 checksum of the client's public key.
+// `{http.request.tls.client.certificate_pem}` | The PEM-encoded value of the certificate.
 // `{http.request.tls.client.issuer}` | The issuer DN of the client certificate
 // `{http.request.tls.client.serial}` | The serial number of the client certificate
 // `{http.request.tls.client.subject}` | The subject DN of the client certificate
@@ -154,6 +156,7 @@ func (app *App) Provision(ctx caddy.Context) error {
 
 	// prepare each server
 	for srvName, srv := range app.Servers {
+		srv.name = srvName
 		srv.tlsApp = app.tlsApp
 		srv.logger = app.logger.Named("log")
 		srv.errorLogger = app.logger.Named("log.error")
@@ -247,6 +250,13 @@ func (app *App) Provision(ctx caddy.Context) error {
 		if err != nil {
 			return fmt.Errorf("server %s: setting up TLS connection policies: %v", srvName, err)
 		}
+
+		// if there is no idle timeout, set a sane default; users have complained
+		// before that aggressive CDNs leave connections open until the server
+		// closes them, so if we don't close them it leads to resource exhaustion
+		if srv.IdleTimeout == 0 {
+			srv.IdleTimeout = defaultIdleTimeout
+		}
 	}
 
 	return nil
@@ -280,6 +290,12 @@ func (app *App) Validate() error {
 // Start runs the app. It finishes automatic HTTPS if enabled,
 // including management of certificates.
 func (app *App) Start() error {
+	// get a logger compatible with http.Server
+	serverLogger, err := zap.NewStdLogAt(app.logger.Named("stdlib"), zap.DebugLevel)
+	if err != nil {
+		return fmt.Errorf("failed to set up server logger: %v", err)
+	}
+
 	for srvName, srv := range app.Servers {
 		s := &http.Server{
 			ReadTimeout:       time.Duration(srv.ReadTimeout),
@@ -288,6 +304,7 @@ func (app *App) Start() error {
 			IdleTimeout:       time.Duration(srv.IdleTimeout),
 			MaxHeaderBytes:    srv.MaxHeaderBytes,
 			Handler:           srv,
+			ErrorLog:          serverLogger,
 		}
 
 		// enable h2c if configured
@@ -343,8 +360,10 @@ func (app *App) Start() error {
 								Addr:      hostport,
 								Handler:   srv,
 								TLSConfig: tlsCfg,
+								ErrorLog:  serverLogger,
 							},
 						}
+						//nolint:errcheck
 						go h3srv.Serve(h3ln)
 						app.h3servers = append(app.h3servers, h3srv)
 						app.h3listeners = append(app.h3listeners, h3ln)
@@ -373,6 +392,7 @@ func (app *App) Start() error {
 					zap.Bool("tls", useTLS),
 				)
 
+				//nolint:errcheck
 				go s.Serve(ln)
 				app.servers = append(app.servers, s)
 			}
@@ -381,7 +401,7 @@ func (app *App) Start() error {
 
 	// finish automatic HTTPS by finally beginning
 	// certificate management
-	err := app.automaticHTTPSPhase2()
+	err = app.automaticHTTPSPhase2()
 	if err != nil {
 		return fmt.Errorf("finalizing automatic HTTPS: %v", err)
 	}
@@ -447,6 +467,12 @@ func (app *App) httpsPort() int {
 	return app.HTTPSPort
 }
 
+// defaultIdleTimeout is the default HTTP server timeout
+// for closing idle connections; useful to avoid resource
+// exhaustion behind hungry CDNs, for example (we've had
+// several complaints without this).
+const defaultIdleTimeout = caddy.Duration(5 * time.Minute)
+
 // Interface guards
 var (
 	_ caddy.App         = (*App)(nil)

modules/caddyhttp/autohttps.go

@@ -241,7 +241,7 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 	// we now have a list of all the unique names for which we need certs;
 	// turn the set into a slice so that phase 2 can use it
 	app.allCertDomains = make([]string, 0, len(uniqueDomainsForCerts))
-	var internal, external []string
+	var internal []string
 uniqueDomainsLoop:
 	for d := range uniqueDomainsForCerts {
 		// whether or not there is already an automation policy for this
@@ -264,15 +264,13 @@ uniqueDomainsLoop:
 
 		// if no automation policy exists for the name yet, we
 		// will associate it with an implicit one
-		if certmagic.SubjectQualifiesForPublicCert(d) {
+		if !certmagic.SubjectQualifiesForPublicCert(d) {
-			external = append(external, d)
-		} else {
 			internal = append(internal, d)
 		}
 	}
 
 	// ensure there is an automation policy to handle these certs
-	err := app.createAutomationPolicies(ctx, external, internal)
+	err := app.createAutomationPolicies(ctx, internal)
 	if err != nil {
 		return err
 	}
@@ -305,31 +303,11 @@ uniqueDomainsLoop:
 			matcherSet = append(matcherSet, MatchHost(domains))
 		}
 
-		// build the address to which to redirect
 		addr, err := caddy.ParseNetworkAddress(addrStr)
 		if err != nil {
 			return err
 		}
-		redirTo := "https://{http.request.host}"
+		redirRoute := app.makeRedirRoute(addr.StartPort, matcherSet)
-		if addr.StartPort != uint(app.httpsPort()) {
-			redirTo += ":" + strconv.Itoa(int(addr.StartPort))
-		}
-		redirTo += "{http.request.uri}"
-
-		// build the route
-		redirRoute := Route{
-			MatcherSets: []MatcherSet{matcherSet},
-			Handlers: []MiddlewareHandler{
-				StaticResponse{
-					StatusCode: WeakString(strconv.Itoa(http.StatusPermanentRedirect)),
-					Headers: http.Header{
-						"Location":   []string{redirTo},
-						"Connection": []string{"close"},
-					},
-					Close: true,
-				},
-			},
-		}
 
 		// use the network/host information from the address,
 		// but change the port to the HTTP port then rebuild
@@ -357,25 +335,7 @@ uniqueDomainsLoop:
 	// it's not something that should be relied on. We can change this
 	// if we want to.
 	appendCatchAll := func(routes []Route) []Route {
-		redirTo := "https://{http.request.host}"
+		return append(routes, app.makeRedirRoute(uint(app.httpsPort()), MatcherSet{MatchProtocol("http")}))
-		if app.httpsPort() != DefaultHTTPSPort {
-			redirTo += ":" + strconv.Itoa(app.httpsPort())
-		}
-		redirTo += "{http.request.uri}"
-		routes = append(routes, Route{
-			MatcherSets: []MatcherSet{{MatchProtocol("http")}},
-			Handlers: []MiddlewareHandler{
-				StaticResponse{
-					StatusCode: WeakString(strconv.Itoa(http.StatusPermanentRedirect)),
-					Headers: http.Header{
-						"Location":   []string{redirTo},
-						"Connection": []string{"close"},
-					},
-					Close: true,
-				},
-			},
-		})
-		return routes
 	}
 
 redirServersLoop:
@@ -424,13 +384,47 @@ redirServersLoop:
 	return nil
 }
 
+func (app *App) makeRedirRoute(redirToPort uint, matcherSet MatcherSet) Route {
+	redirTo := "https://{http.request.host}"
+
+	// since this is an external redirect, we should only append an explicit
+	// port if we know it is not the officially standardized HTTPS port, and,
+	// notably, also not the port that Caddy thinks is the HTTPS port (the
+	// configurable HTTPSPort parameter) - we can't change the standard HTTPS
+	// port externally, so that config parameter is for internal use only;
+	// we also do not append the port if it happens to be the HTTP port as
+	// well, obviously (for example, user defines the HTTP port explicitly
+	// in the list of listen addresses for a server)
+	if redirToPort != uint(app.httpPort()) &&
+		redirToPort != uint(app.httpsPort()) &&
+		redirToPort != DefaultHTTPPort &&
+		redirToPort != DefaultHTTPSPort {
+		redirTo += ":" + strconv.Itoa(int(redirToPort))
+	}
+
+	redirTo += "{http.request.uri}"
+	return Route{
+		MatcherSets: []MatcherSet{matcherSet},
+		Handlers: []MiddlewareHandler{
+			StaticResponse{
+				StatusCode: WeakString(strconv.Itoa(http.StatusPermanentRedirect)),
+				Headers: http.Header{
+					"Location":   []string{redirTo},
+					"Connection": []string{"close"},
+				},
+				Close: true,
+			},
+		},
+	}
+}
+
 // createAutomationPolicy ensures that automated certificates for this
 // app are managed properly. This adds up to two automation policies:
 // one for the public names, and one for the internal names. If a catch-all
 // automation policy exists, it will be shallow-copied and used as the
 // base for the new ones (this is important for preserving behavior the
 // user intends to be "defaults").
-func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, internalNames []string) error {
+func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []string) error {
 	// before we begin, loop through the existing automation policies
 	// and, for any ACMEIssuers we find, make sure they're filled in
 	// with default values that might be specified in our HTTP app; also
@@ -447,16 +441,23 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, interna
 		// set up default issuer -- honestly, this is only
 		// really necessary because the HTTP app is opinionated
 		// and has settings which could be inferred as new
-		// defaults for the ACMEIssuer in the TLS app
+		// defaults for the ACMEIssuer in the TLS app (such as
-		if ap.Issuer == nil {
+		// what the HTTP and HTTPS ports are)
-			ap.Issuer = new(caddytls.ACMEIssuer)
+		if ap.Issuers == nil {
-		}
+			var err error
-		if acmeIssuer, ok := ap.Issuer.(*caddytls.ACMEIssuer); ok {
+			ap.Issuers, err = caddytls.DefaultIssuers(ctx)
-			err := app.fillInACMEIssuer(acmeIssuer)
 			if err != nil {
 				return err
 			}
 		}
+		for _, iss := range ap.Issuers {
+			if acmeIssuer, ok := iss.(acmeCapable); ok {
+				err := app.fillInACMEIssuer(acmeIssuer.GetACMEIssuer())
+				if err != nil {
+					return err
+				}
+			}
+		}
 
 		// while we're here, is this the catch-all/base policy?
 		if !foundBasePolicy && len(ap.Subjects) == 0 {
@@ -470,9 +471,16 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, interna
 		basePolicy = new(caddytls.AutomationPolicy)
 	}
 
-	// if the basePolicy has an existing ACMEIssuer, let's
+	// if the basePolicy has an existing ACMEIssuer (particularly to
-	// use it, otherwise we'll make one
+	// include any type that embeds/wraps an ACMEIssuer), let's use it
-	baseACMEIssuer, _ := basePolicy.Issuer.(*caddytls.ACMEIssuer)
+	// (I guess we just use the first one?), otherwise we'll make one
+	var baseACMEIssuer *caddytls.ACMEIssuer
+	for _, iss := range basePolicy.Issuers {
+		if acmeWrapper, ok := iss.(acmeCapable); ok {
+			baseACMEIssuer = acmeWrapper.GetACMEIssuer()
+			break
+		}
+	}
 	if baseACMEIssuer == nil {
 		// note that this happens if basePolicy.Issuer is nil
 		// OR if it is not nil but is not an ACMEIssuer
@@ -481,7 +489,7 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, interna
 
 	// if there was a base policy to begin with, we already
 	// filled in its issuer's defaults; if there wasn't, we
-	// stil need to do that
+	// still need to do that
 	if !foundBasePolicy {
 		err := app.fillInACMEIssuer(baseACMEIssuer)
 		if err != nil {
@@ -490,8 +498,20 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, interna
 	}
 
 	// never overwrite any other issuer that might already be configured
-	if basePolicy.Issuer == nil {
+	if basePolicy.Issuers == nil {
-		basePolicy.Issuer = baseACMEIssuer
+		var err error
+		basePolicy.Issuers, err = caddytls.DefaultIssuers(ctx)
+		if err != nil {
+			return err
+		}
+		for _, iss := range basePolicy.Issuers {
+			if acmeIssuer, ok := iss.(acmeCapable); ok {
+				err := app.fillInACMEIssuer(acmeIssuer.GetACMEIssuer())
+				if err != nil {
+					return err
+				}
+			}
+		}
 	}
 
 	if !foundBasePolicy {
@@ -499,7 +519,10 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, interna
 		// our base/catch-all policy - this will serve the
 		// public-looking names as well as any other names
 		// that don't match any other policy
-		app.tlsApp.AddAutomationPolicy(basePolicy)
+		err := app.tlsApp.AddAutomationPolicy(basePolicy)
+		if err != nil {
+			return err
+		}
 	} else {
 		// a base policy already existed; we might have
 		// changed it, so re-provision it
@@ -545,8 +568,7 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, interna
 		// of names that would normally use the production API;
 		// anyway, that gets into the weeds a bit...
 		newPolicy.Subjects = internalNames
-		newPolicy.Issuer = internalIssuer
+		newPolicy.Issuers = []certmagic.Issuer{internalIssuer}
-
 		err := app.tlsApp.AddAutomationPolicy(newPolicy)
 		if err != nil {
 			return err
@@ -630,3 +652,5 @@ func (app *App) automaticHTTPSPhase2() error {
 	app.allCertDomains = nil // no longer needed; allow GC to deallocate
 	return nil
 }
+
+type acmeCapable interface{ GetACMEIssuer() *caddytls.ACMEIssuer }

modules/caddyhttp/caddyauth/basicauth.go

@@ -52,11 +52,19 @@ type HTTPBasicAuth struct {
 	// memory for a longer time (this should not be a problem
 	// as long as your machine is not compromised, at which point
 	// all bets are off, since basicauth necessitates plaintext
-	// passwords being received over the wire anyway).
+	// passwords being received over the wire anyway). Note that
+	// a cache hit does not mean it is a valid password.
 	HashCache *Cache `json:"hash_cache,omitempty"`
 
 	Accounts map[string]Account `json:"-"`
 	Hash     Comparer           `json:"-"`
+
+	// fakePassword is used when a given user is not found,
+	// so that timing side-channels can be mitigated: it gives
+	// us something to hash and compare even if the user does
+	// not exist, which should have similar timing as a user
+	// account that does exist.
+	fakePassword []byte
 }
 
 // CaddyModule returns the Caddy module information.
@@ -84,6 +92,14 @@ func (hba *HTTPBasicAuth) Provision(ctx caddy.Context) error {
 		return fmt.Errorf("hash is required")
 	}
 
+	// if supported, generate a fake password we can compare against if needed
+	if hasher, ok := hba.Hash.(Hasher); ok {
+		hba.fakePassword, err = hasher.Hash([]byte("antitiming"), []byte("fakesalt"))
+		if err != nil {
+			return fmt.Errorf("generating anti-timing password hash: %v", err)
+		}
+	}
+
 	repl := caddy.NewReplacer()
 
 	// load account list
@@ -118,7 +134,7 @@ func (hba *HTTPBasicAuth) Provision(ctx caddy.Context) error {
 
 	if hba.HashCache != nil {
 		hba.HashCache.cache = make(map[string]bool)
-		hba.HashCache.mu = new(sync.Mutex)
+		hba.HashCache.mu = new(sync.RWMutex)
 	}
 
 	return nil
@@ -132,16 +148,17 @@ func (hba HTTPBasicAuth) Authenticate(w http.ResponseWriter, req *http.Request)
 	}
 
 	account, accountExists := hba.Accounts[username]
-	// don't return early if account does not exist; we want
+	if !accountExists {
-	// to try to avoid side-channels that leak existence
+		// don't return early if account does not exist; we want
+		// to try to avoid side-channels that leak existence, so
+		// we use a fake password to simulate realistic CPU cycles
+		account.password = hba.fakePassword
+	}
 
 	same, err := hba.correctPassword(account, []byte(plaintextPasswordStr))
-	if err != nil {
+	if err != nil || !same || !accountExists {
 		return hba.promptForCredentials(w, err)
 	}
-	if !same || !accountExists {
-		return hba.promptForCredentials(w, nil)
-	}
 
 	return User{ID: username}, true, nil
 }
@@ -160,13 +177,12 @@ func (hba HTTPBasicAuth) correctPassword(account Account, plaintextPassword []by
 	cacheKey := hex.EncodeToString(append(append(account.password, account.salt...), plaintextPassword...))
 
 	// fast track: if the result of the input is already cached, use it
-	hba.HashCache.mu.Lock()
+	hba.HashCache.mu.RLock()
 	same, ok := hba.HashCache.cache[cacheKey]
+	hba.HashCache.mu.RUnlock()
 	if ok {
-		hba.HashCache.mu.Unlock()
 		return same, nil
 	}
-	hba.HashCache.mu.Unlock()
 
 	// slow track: do the expensive op, then add it to the cache
 	same, err := compare()
@@ -199,7 +215,7 @@ func (hba HTTPBasicAuth) promptForCredentials(w http.ResponseWriter, err error)
 // helpful for secure password hashes which can be expensive to
 // compute on every HTTP request.
 type Cache struct {
-	mu *sync.Mutex
+	mu *sync.RWMutex
 
 	// map of concatenated hashed password + plaintext password + salt, to result
 	cache map[string]bool
@@ -224,6 +240,7 @@ func (c *Cache) makeRoom() {
 		// map with less code, this is a heavily skewed eviction
 		// strategy; generating random numbers is cheap and
 		// ensures a much better distribution.
+		//nolint:gosec
 		rnd := weakrand.Intn(len(c.cache))
 		i := 0
 		for key := range c.cache {
@@ -249,6 +266,16 @@ type Comparer interface {
 	Compare(hashedPassword, plaintextPassword, salt []byte) (bool, error)
 }
 
+// Hasher is a type that can generate a secure hash
+// given a plaintext and optional salt (for algorithms
+// that require a salt). Hashing modules which implement
+// this interface can be used with the hash-password
+// subcommand as well as benefitting from anti-timing
+// features.
+type Hasher interface {
+	Hash(plaintext, salt []byte) ([]byte, error)
+}
+
 // Account contains a username, password, and salt (if applicable).
 type Account struct {
 	// A user's username.

modules/caddyhttp/caddyauth/caddyauth.go

@@ -16,11 +16,11 @@ package caddyauth
 
 import (
 	"fmt"
-	"log"
 	"net/http"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"go.uber.org/zap"
 )
 
 func init() {
@@ -30,6 +30,11 @@ func init() {
 // Authentication is a middleware which provides user authentication.
 // Rejects requests with HTTP 401 if the request is not authenticated.
 //
+// After a successful authentication, the placeholder
+// `{http.auth.user.id}` will be set to the username, and also
+// `{http.auth.user.*}` placeholders may be set for any authentication
+// modules that provide user metadata.
+//
 // Its API is still experimental and may be subject to change.
 type Authentication struct {
 	// A set of authentication providers. If none are specified,
@@ -37,6 +42,8 @@ type Authentication struct {
 	ProvidersRaw caddy.ModuleMap `json:"providers,omitempty" caddy:"namespace=http.authentication.providers"`
 
 	Providers map[string]Authenticator `json:"-"`
+
+	logger *zap.Logger
 }
 
 // CaddyModule returns the Caddy module information.
@@ -49,6 +56,7 @@ func (Authentication) CaddyModule() caddy.ModuleInfo {
 
 // Provision sets up a.
 func (a *Authentication) Provision(ctx caddy.Context) error {
+	a.logger = ctx.Logger(a)
 	a.Providers = make(map[string]Authenticator)
 	mods, err := ctx.LoadModule(a, "ProvidersRaw")
 	if err != nil {
@@ -67,7 +75,9 @@ func (a Authentication) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 	for provName, prov := range a.Providers {
 		user, authed, err = prov.Authenticate(w, r)
 		if err != nil {
-			log.Printf("[ERROR] Authenticating with %s: %v", provName, err)
+			a.logger.Error("auth provider returned error",
+				zap.String("provider", provName),
+				zap.Error(err))
 			continue
 		}
 		if authed {

modules/caddyhttp/caddyauth/command.go

@@ -25,8 +25,6 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	caddycmd "github.com/caddyserver/caddy/v2/cmd"
-	"golang.org/x/crypto/bcrypt"
-	"golang.org/x/crypto/scrypt"
 	"golang.org/x/crypto/ssh/terminal"
 )
 
@@ -72,7 +70,7 @@ func cmdHashPassword(fs caddycmd.Flags) (int, error) {
 		if terminal.IsTerminal(fd) {
 			// ensure the terminal state is restored on SIGINT
 			state, _ := terminal.GetState(fd)
-			c := make(chan os.Signal)
+			c := make(chan os.Signal, 1)
 			signal.Notify(c, os.Interrupt)
 			go func() {
 				<-c
@@ -116,11 +114,11 @@ func cmdHashPassword(fs caddycmd.Flags) (int, error) {
 	var hash []byte
 	switch algorithm {
 	case "bcrypt":
-		hash, err = bcrypt.GenerateFromPassword(plaintext, bcrypt.DefaultCost)
+		hash, err = BcryptHash{}.Hash(plaintext, nil)
 	case "scrypt":
 		def := ScryptHash{}
 		def.SetDefaults()
-		hash, err = scrypt.Key(plaintext, salt, def.N, def.R, def.P, def.KeyLength)
+		hash, err = def.Hash(plaintext, salt)
 	default:
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("unrecognized hash algorithm: %s", algorithm)
 	}

modules/caddyhttp/caddyauth/hashes.go

@@ -50,6 +50,11 @@ func (BcryptHash) Compare(hashed, plaintext, _ []byte) (bool, error) {
 	return true, nil
 }
 
+// Hash hashes plaintext using a random salt.
+func (BcryptHash) Hash(plaintext, _ []byte) ([]byte, error) {
+	return bcrypt.GenerateFromPassword(plaintext, 14)
+}
+
 // ScryptHash implements the scrypt KDF as a hash.
 type ScryptHash struct {
 	// scrypt's N parameter. If unset or 0, a safe default is used.
@@ -113,6 +118,11 @@ func (s ScryptHash) Compare(hashed, plaintext, salt []byte) (bool, error) {
 	return false, nil
 }
 
+// Hash hashes plaintext using the given salt.
+func (s ScryptHash) Hash(plaintext, salt []byte) ([]byte, error) {
+	return scrypt.Key(plaintext, salt, s.N, s.R, s.P, s.KeyLength)
+}
+
 func hashesMatch(pwdHash1, pwdHash2 []byte) bool {
 	return subtle.ConstantTimeCompare(pwdHash1, pwdHash2) == 1
 }
@@ -121,5 +131,7 @@ func hashesMatch(pwdHash1, pwdHash2 []byte) bool {
 var (
 	_ Comparer          = (*BcryptHash)(nil)
 	_ Comparer          = (*ScryptHash)(nil)
+	_ Hasher            = (*BcryptHash)(nil)
+	_ Hasher            = (*ScryptHash)(nil)
 	_ caddy.Provisioner = (*ScryptHash)(nil)
 )

modules/caddyhttp/caddyhttp.go

@@ -18,18 +18,15 @@ import (
 	"bytes"
 	"encoding/json"
 	"io"
-	weakrand "math/rand"
 	"net"
 	"net/http"
 	"strconv"
-	"time"
 
 	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 )
 
 func init() {
-	weakrand.Seed(time.Now().UnixNano())
-
 	caddy.RegisterModule(tlsPlaceholderWrapper{})
 }
 
@@ -235,6 +232,8 @@ func (tlsPlaceholderWrapper) CaddyModule() caddy.ModuleInfo {
 
 func (tlsPlaceholderWrapper) WrapListener(ln net.Listener) net.Listener { return ln }
 
+func (tlsPlaceholderWrapper) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { return nil }
+
 const (
 	// DefaultHTTPPort is the default port for HTTP.
 	DefaultHTTPPort = 80
@@ -245,3 +244,4 @@ const (
 
 // Interface guard
 var _ caddy.ListenerWrapper = (*tlsPlaceholderWrapper)(nil)
+var _ caddyfile.Unmarshaler = (*tlsPlaceholderWrapper)(nil)

modules/caddyhttp/celmatcher.go

@@ -15,6 +15,7 @@
 package caddyhttp
 
 import (
+	"crypto/x509/pkix"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -199,6 +200,27 @@ func (cr celHTTPRequest) Equal(other ref.Val) ref.Val {
 func (celHTTPRequest) Type() ref.Type        { return httpRequestCELType }
 func (cr celHTTPRequest) Value() interface{} { return cr }
 
+var pkixNameCELType = types.NewTypeValue("pkix.Name", traits.ReceiverType)
+
+// celPkixName wraps an pkix.Name with
+// methods to satisfy the ref.Val interface.
+type celPkixName struct{ *pkix.Name }
+
+func (pn celPkixName) ConvertToNative(typeDesc reflect.Type) (interface{}, error) {
+	return pn.Name, nil
+}
+func (celPkixName) ConvertToType(typeVal ref.Type) ref.Val {
+	panic("not implemented")
+}
+func (pn celPkixName) Equal(other ref.Val) ref.Val {
+	if o, ok := other.Value().(string); ok {
+		return types.Bool(pn.Name.String() == o)
+	}
+	return types.ValOrErr(other, "%v is not comparable type", other)
+}
+func (celPkixName) Type() ref.Type        { return pkixNameCELType }
+func (pn celPkixName) Value() interface{} { return pn }
+
 // celTypeAdapter can adapt our custom types to a CEL value.
 type celTypeAdapter struct{}
 
@@ -206,6 +228,8 @@ func (celTypeAdapter) NativeToValue(value interface{}) ref.Val {
 	switch v := value.(type) {
 	case celHTTPRequest:
 		return v
+	case pkix.Name:
+		return celPkixName{&v}
 	case time.Time:
 		// TODO: eliminate direct protobuf dependency, sigh -- just wrap stdlib time.Time instead...
 		return types.Timestamp{Timestamp: &timestamp.Timestamp{Seconds: v.Unix(), Nanos: int32(v.Nanosecond())}}

modules/caddyhttp/celmatcher_test.go

@@ -15,6 +15,11 @@
 package caddyhttp
 
 import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"net/http/httptest"
 	"testing"
 
 	"github.com/caddyserver/caddy/v2"
@@ -27,7 +32,7 @@ func TestMatchExpressionProvision(t *testing.T) {
 		wantErr    bool
 	}{
 		{
-			name: "boolean mtaches succeed",
+			name: "boolean matches succeed",
 			expression: &MatchExpression{
 				Expr: "{http.request.uri.query} != ''",
 			},
@@ -49,3 +54,71 @@ func TestMatchExpressionProvision(t *testing.T) {
 		})
 	}
 }
+
+func TestMatchExpressionMatch(t *testing.T) {
+
+	clientCert := []byte(`-----BEGIN CERTIFICATE-----
+MIIB9jCCAV+gAwIBAgIBAjANBgkqhkiG9w0BAQsFADAYMRYwFAYDVQQDDA1DYWRk
+eSBUZXN0IENBMB4XDTE4MDcyNDIxMzUwNVoXDTI4MDcyMTIxMzUwNVowHTEbMBkG
+A1UEAwwSY2xpZW50LmxvY2FsZG9tYWluMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB
+iQKBgQDFDEpzF0ew68teT3xDzcUxVFaTII+jXH1ftHXxxP4BEYBU4q90qzeKFneF
+z83I0nC0WAQ45ZwHfhLMYHFzHPdxr6+jkvKPASf0J2v2HDJuTM1bHBbik5Ls5eq+
+fVZDP8o/VHKSBKxNs8Goc2NTsr5b07QTIpkRStQK+RJALk4x9QIDAQABo0swSTAJ
+BgNVHRMEAjAAMAsGA1UdDwQEAwIHgDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
+AAEwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDQYJKoZIhvcNAQELBQADgYEANSjz2Sk+
+eqp31wM9il1n+guTNyxJd+FzVAH+hCZE5K+tCgVDdVFUlDEHHbS/wqb2PSIoouLV
+3Q9fgDkiUod+uIK0IynzIKvw+Cjg+3nx6NQ0IM0zo8c7v398RzB4apbXKZyeeqUH
+9fNwfEi+OoXR6s+upSKobCmLGLGi9Na5s5g=
+-----END CERTIFICATE-----`)
+
+	tests := []struct {
+		name              string
+		expression        *MatchExpression
+		wantErr           bool
+		wantResult        bool
+		clientCertificate []byte
+	}{
+		{
+			name: "boolean matches succeed for placeholder http.request.tls.client.subject",
+			expression: &MatchExpression{
+				Expr: "{http.request.tls.client.subject} == 'CN=client.localdomain'",
+			},
+			clientCertificate: clientCert,
+			wantResult:        true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.expression.Provision(caddy.Context{}); (err != nil) != tt.wantErr {
+				t.Errorf("MatchExpression.Provision() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			req := httptest.NewRequest("GET", "https://example.com/foo", nil)
+			repl := caddy.NewReplacer()
+			ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
+			req = req.WithContext(ctx)
+			addHTTPVarsToReplacer(repl, req, httptest.NewRecorder())
+
+			if tt.clientCertificate != nil {
+				block, _ := pem.Decode(clientCert)
+				if block == nil {
+					t.Fatalf("failed to decode PEM certificate")
+				}
+
+				cert, err := x509.ParseCertificate(block.Bytes)
+				if err != nil {
+					t.Fatalf("failed to decode PEM certificate: %v", err)
+				}
+
+				req.TLS = &tls.ConnectionState{
+					PeerCertificates: []*x509.Certificate{cert},
+				}
+			}
+
+			if tt.expression.Match(req) != tt.wantResult {
+				t.Errorf("MatchExpression.Match() expected to return '%t', for expression : '%s'", tt.wantResult, tt.expression)
+			}
+
+		})
+	}
+}

modules/caddyhttp/encode/encode.go

@@ -262,7 +262,7 @@ func acceptedEncodings(r *http.Request) []string {
 		return []string{}
 	}
 
-	var prefs []encodingPreference
+	prefs := []encodingPreference{}
 
 	for _, accepted := range strings.Split(acceptEncHeader, ",") {
 		parts := strings.Split(accepted, ";")

modules/caddyhttp/errors.go

@@ -16,14 +16,19 @@ package caddyhttp
 
 import (
 	"fmt"
-	mathrand "math/rand"
+	weakrand "math/rand"
 	"path"
 	"runtime"
 	"strings"
+	"time"
 
 	"github.com/caddyserver/caddy/v2"
 )
 
+func init() {
+	weakrand.Seed(time.Now().UnixNano())
+}
+
 // Error is a convenient way for a Handler to populate the
 // essential fields of a HandlerError. If err is itself a
 // HandlerError, then any essential fields that are not
@@ -92,7 +97,8 @@ func randString(n int, sameCase bool) string {
 	}
 	b := make([]byte, n)
 	for i := range b {
-		b[i] = dict[mathrand.Int63()%int64(len(dict))]
+		//nolint:gosec
+		b[i] = dict[weakrand.Int63()%int64(len(dict))]
 	}
 	return string(b)
 }

modules/caddyhttp/fileserver/browse.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"go.uber.org/zap"
 )
 
 // Browse configures directory browsing.
@@ -35,12 +36,17 @@ type Browse struct {
 	template *template.Template
 }
 
-func (fsrv *FileServer) serveBrowse(dirPath string, w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
+func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
+	fsrv.logger.Debug("browse enabled; listing directory contents",
+		zap.String("path", dirPath),
+		zap.String("root", root))
+
 	// navigation on the client-side gets messed up if the
 	// URL doesn't end in a trailing slash because hrefs like
 	// "/b/c" on a path like "/a" end up going to "/b/c" instead
 	// of "/a/b/c" - so we have to redirect in this case
 	if !strings.HasSuffix(r.URL.Path, "/") {
+		fsrv.logger.Debug("redirecting to trailing slash to preserve hrefs", zap.String("request_path", r.URL.Path))
 		r.URL.Path += "/"
 		http.Redirect(w, r, r.URL.String(), http.StatusMovedPermanently)
 		return nil
@@ -55,7 +61,7 @@ func (fsrv *FileServer) serveBrowse(dirPath string, w http.ResponseWriter, r *ht
 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
 
 	// calling path.Clean here prevents weird breadcrumbs when URL paths are sketchy like /%2e%2e%2f
-	listing, err := fsrv.loadDirectoryContents(dir, path.Clean(r.URL.Path), repl)
+	listing, err := fsrv.loadDirectoryContents(dir, root, path.Clean(r.URL.Path), repl)
 	switch {
 	case os.IsPermission(err):
 		return caddyhttp.Error(http.StatusForbidden, err)
@@ -82,22 +88,21 @@ func (fsrv *FileServer) serveBrowse(dirPath string, w http.ResponseWriter, r *ht
 		w.Header().Set("Content-Type", "text/html; charset=utf-8")
 	}
 
-	buf.WriteTo(w)
+	_, _ = buf.WriteTo(w)
 
 	return nil
 }
 
-func (fsrv *FileServer) loadDirectoryContents(dir *os.File, urlPath string, repl *caddy.Replacer) (browseListing, error) {
+func (fsrv *FileServer) loadDirectoryContents(dir *os.File, root, urlPath string, repl *caddy.Replacer) (browseListing, error) {
 	files, err := dir.Readdir(-1)
 	if err != nil {
 		return browseListing{}, err
 	}
 
-	// determine if user can browse up another folder
+	// user can presumably browse "up" to parent folder if path is longer than "/"
-	curPathDir := path.Dir(strings.TrimSuffix(urlPath, "/"))
+	canGoUp := len(urlPath) > 1
-	canGoUp := strings.HasPrefix(curPathDir, fsrv.Root)
 
-	return fsrv.directoryListing(files, canGoUp, urlPath, repl), nil
+	return fsrv.directoryListing(files, canGoUp, root, urlPath, repl), nil
 }
 
 // browseApplyQueryParams applies query parameters to the listing.
@@ -106,6 +111,7 @@ func (fsrv *FileServer) browseApplyQueryParams(w http.ResponseWriter, r *http.Re
 	sortParam := r.URL.Query().Get("sort")
 	orderParam := r.URL.Query().Get("order")
 	limitParam := r.URL.Query().Get("limit")
+	offsetParam := r.URL.Query().Get("offset")
 
 	// first figure out what to sort by
 	switch sortParam {
@@ -130,7 +136,7 @@ func (fsrv *FileServer) browseApplyQueryParams(w http.ResponseWriter, r *http.Re
 	}
 
 	// finally, apply the sorting and limiting
-	listing.applySortAndLimit(sortParam, orderParam, limitParam)
+	listing.applySortAndLimit(sortParam, orderParam, limitParam, offsetParam)
 }
 
 func (fsrv *FileServer) browseWriteJSON(listing browseListing) (*bytes.Buffer, error) {

modules/caddyhttp/fileserver/browse_test.go

@@ -11,15 +11,15 @@ func BenchmarkBrowseWriteJSON(b *testing.B) {
 	fsrv := new(FileServer)
 	fsrv.Provision(caddy.Context{})
 	listing := browseListing{
-		Name:           "test",
+		Name:     "test",
-		Path:           "test",
+		Path:     "test",
-		CanGoUp:        false,
+		CanGoUp:  false,
-		Items:          make([]fileInfo, 100),
+		Items:    make([]fileInfo, 100),
-		NumDirs:        42,
+		NumDirs:  42,
-		NumFiles:       420,
+		NumFiles: 420,
-		Sort:           "",
+		Sort:     "",
-		Order:          "",
+		Order:    "",
-		ItemsLimitedTo: 42,
+		Limit:    42,
 	}
 	b.ResetTimer()
 
@@ -36,15 +36,15 @@ func BenchmarkBrowseWriteHTML(b *testing.B) {
 		template:     template.New("test"),
 	}
 	listing := browseListing{
-		Name:           "test",
+		Name:     "test",
-		Path:           "test",
+		Path:     "test",
-		CanGoUp:        false,
+		CanGoUp:  false,
-		Items:          make([]fileInfo, 100),
+		Items:    make([]fileInfo, 100),
-		NumDirs:        42,
+		NumDirs:  42,
-		NumFiles:       420,
+		NumFiles: 420,
-		Sort:           "",
+		Sort:     "",
-		Order:          "",
+		Order:    "",
-		ItemsLimitedTo: 42,
+		Limit:    42,
 	}
 	b.ResetTimer()
 

modules/caddyhttp/fileserver/browselisting.go

@@ -27,13 +27,11 @@ import (
 	"github.com/dustin/go-humanize"
 )
 
-func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, urlPath string, repl *caddy.Replacer) browseListing {
+func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, root, urlPath string, repl *caddy.Replacer) browseListing {
 	filesToHide := fsrv.transformHidePaths(repl)
 
-	var (
+	var dirCount, fileCount int
-		fileInfos           []fileInfo
+	fileInfos := []fileInfo{}
-		dirCount, fileCount int
-	)
 
 	for _, f := range files {
 		name := f.Name()
@@ -42,7 +40,7 @@ func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, urlP
 			continue
 		}
 
-		isDir := f.IsDir() || isSymlinkTargetDir(f, fsrv.Root, urlPath)
+		isDir := f.IsDir() || isSymlinkTargetDir(f, root, urlPath)
 
 		if isDir {
 			name += "/"
@@ -76,40 +74,41 @@ func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, urlP
 
 type browseListing struct {
 	// The name of the directory (the last element of the path).
-	Name string
+	Name string `json:"name"`
 
 	// The full path of the request.
-	Path string
+	Path string `json:"path"`
 
 	// Whether the parent directory is browseable.
-	CanGoUp bool
+	CanGoUp bool `json:"can_go_up"`
 
 	// The items (files and folders) in the path.
-	Items []fileInfo
+	Items []fileInfo `json:"items,omitempty"`
 
-	// The number of directories in the listing.
+	// If ≠0 then Items starting from that many elements.
-	NumDirs int
+	Offset int `json:"offset,omitempty"`
-
-	// The number of files (items that aren't directories) in the listing.
-	NumFiles int
-
-	// Sort column used
-	Sort string
-
-	// Sorting order
-	Order string
 
 	// If ≠0 then Items have been limited to that many elements.
-	ItemsLimitedTo int
+	Limit int `json:"limit,omitempty"`
+
+	// The number of directories in the listing.
+	NumDirs int `json:"num_dirs"`
+
+	// The number of files (items that aren't directories) in the listing.
+	NumFiles int `json:"num_files"`
+
+	// Sort column used
+	Sort string `json:"sort,omitempty"`
+
+	// Sorting order
+	Order string `json:"order,omitempty"`
 }
 
 // Breadcrumbs returns l.Path where every element maps
 // the link to the text to display.
 func (l browseListing) Breadcrumbs() []crumb {
-	var result []crumb
-
 	if len(l.Path) == 0 {
-		return result
+		return []crumb{}
 	}
 
 	// skip trailing slash
@@ -119,19 +118,19 @@ func (l browseListing) Breadcrumbs() []crumb {
 	}
 
 	parts := strings.Split(lpath, "/")
-	for i := range parts {
+	result := make([]crumb, len(parts))
-		txt := parts[i]
+	for i, p := range parts {
-		if i == 0 && parts[i] == "" {
+		if i == 0 && p == "" {
-			txt = "/"
+			p = "/"
 		}
 		lnk := strings.Repeat("../", len(parts)-i-1)
-		result = append(result, crumb{Link: lnk, Text: txt})
+		result[i] = crumb{Link: lnk, Text: p}
 	}
 
 	return result
 }
 
-func (l *browseListing) applySortAndLimit(sortParam, orderParam, limitParam string) {
+func (l *browseListing) applySortAndLimit(sortParam, orderParam, limitParam string, offsetParam string) {
 	l.Sort = sortParam
 	l.Order = orderParam
 
@@ -159,11 +158,20 @@ func (l *browseListing) applySortAndLimit(sortParam, orderParam, limitParam stri
 		}
 	}
 
+	if offsetParam != "" {
+		offset, _ := strconv.Atoi(offsetParam)
+		if offset > 0 && offset <= len(l.Items) {
+			l.Items = l.Items[offset:]
+			l.Offset = offset
+		}
+	}
+
 	if limitParam != "" {
 		limit, _ := strconv.Atoi(limitParam)
+
 		if limit > 0 && limit <= len(l.Items) {
 			l.Items = l.Items[:limit]
-			l.ItemsLimitedTo = limit
+			l.Limit = limit
 		}
 	}
 }

modules/caddyhttp/fileserver/browselisting_test.go

@@ -0,0 +1,40 @@
+package fileserver
+
+import (
+	"testing"
+)
+
+func TestBreadcrumbs(t *testing.T) {
+	testdata := []struct {
+		path     string
+		expected []crumb
+	}{
+		{"", []crumb{}},
+		{"/", []crumb{{Text: "/"}}},
+		{"foo/bar/baz", []crumb{
+			{Link: "../../", Text: "foo"},
+			{Link: "../", Text: "bar"},
+			{Link: "", Text: "baz"},
+		}},
+		{"/qux/quux/corge/", []crumb{
+			{Link: "../../../", Text: "/"},
+			{Link: "../../", Text: "qux"},
+			{Link: "../", Text: "quux"},
+			{Link: "", Text: "corge"},
+		}},
+	}
+
+	for _, d := range testdata {
+		l := browseListing{Path: d.path}
+		actual := l.Breadcrumbs()
+		if len(actual) != len(d.expected) {
+			t.Errorf("wrong size output, got %d elements but expected %d", len(actual), len(d.expected))
+			continue
+		}
+		for i, c := range actual {
+			if c != d.expected[i] {
+				t.Errorf("got %#v but expected %#v at index %d", c, d.expected[i], i)
+			}
+		}
+	}
+}

modules/caddyhttp/fileserver/browsetpl.go

@@ -283,8 +283,8 @@ footer {
 				<div id="summary">
 					<span class="meta-item"><b>{{.NumDirs}}</b> director{{if eq 1 .NumDirs}}y{{else}}ies{{end}}</span>
 					<span class="meta-item"><b>{{.NumFiles}}</b> file{{if ne 1 .NumFiles}}s{{end}}</span>
-					{{- if ne 0 .ItemsLimitedTo}}
+					{{- if ne 0 .Limit}}
-					<span class="meta-item">(of which only <b>{{.ItemsLimitedTo}}</b> are displayed)</span>
+					<span class="meta-item">(of which only <b>{{.Limit}}</b> are displayed)</span>
 					{{- end}}
 					<span class="meta-item"><input type="text" placeholder="filter" id="filter" onkeyup='filter()'></span>
 				</div>
@@ -296,37 +296,37 @@ footer {
 						<th></th>
 						<th>
 							{{- if and (eq .Sort "namedirfirst") (ne .Order "desc")}}
-							<a href="?sort=namedirfirst&order=desc{{if ne 0 .ItemsLimitedTo}}&limit={{.ItemsLimitedTo}}{{end}}" class="icon"><svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#up-arrow"></use></svg></a>
+							<a href="?sort=namedirfirst&order=desc{{if ne 0 .Limit}}&limit={{.Limit}}{{end}}{{if ne 0 .Offset}}&offset={{.Offset}}{{end}}" class="icon"><svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#up-arrow"></use></svg></a>
 							{{- else if and (eq .Sort "namedirfirst") (ne .Order "asc")}}
-							<a href="?sort=namedirfirst&order=asc{{if ne 0 .ItemsLimitedTo}}&limit={{.ItemsLimitedTo}}{{end}}" class="icon"><svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#down-arrow"></use></svg></a>
+							<a href="?sort=namedirfirst&order=asc{{if ne 0 .Limit}}&limit={{.Limit}}{{end}}{{if ne 0 .Offset}}&offset={{.Offset}}{{end}}" class="icon"><svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#down-arrow"></use></svg></a>
 							{{- else}}
-							<a href="?sort=namedirfirst&order=asc{{if ne 0 .ItemsLimitedTo}}&limit={{.ItemsLimitedTo}}{{end}}" class="icon sort"><svg class="top" width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#up-arrow"></use></svg><svg class="bottom" width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#down-arrow"></use></svg></a>
+							<a href="?sort=namedirfirst&order=asc{{if ne 0 .Limit}}&limit={{.Limit}}{{end}}{{if ne 0 .Offset}}&offset={{.Offset}}{{end}}" class="icon sort"><svg class="top" width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#up-arrow"></use></svg><svg class="bottom" width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#down-arrow"></use></svg></a>
 							{{- end}}
 							
 							{{- if and (eq .Sort "name") (ne .Order "desc")}}
-							<a href="?sort=name&order=desc{{if ne 0 .ItemsLimitedTo}}&limit={{.ItemsLimitedTo}}{{end}}">Name <svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#up-arrow"></use></svg></a>
+							<a href="?sort=name&order=desc{{if ne 0 .Limit}}&limit={{.Limit}}{{end}}{{if ne 0 .Offset}}&offset={{.Offset}}{{end}}">Name <svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#up-arrow"></use></svg></a>
 							{{- else if and (eq .Sort "name") (ne .Order "asc")}}
-							<a href="?sort=name&order=asc{{if ne 0 .ItemsLimitedTo}}&limit={{.ItemsLimitedTo}}{{end}}">Name <svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#down-arrow"></use></svg></a>
+							<a href="?sort=name&order=asc{{if ne 0 .Limit}}&limit={{.Limit}}{{end}}{{if ne 0 .Offset}}&offset={{.Offset}}{{end}}">Name <svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#down-arrow"></use></svg></a>
 							{{- else}}
-							<a href="?sort=name&order=asc{{if ne 0 .ItemsLimitedTo}}&limit={{.ItemsLimitedTo}}{{end}}">Name</a>
+							<a href="?sort=name&order=asc{{if ne 0 .Limit}}&limit={{.Limit}}{{end}}{{if ne 0 .Offset}}&offset={{.Offset}}{{end}}">Name</a>
 							{{- end}}
 						</th>
 						<th>
 							{{- if and (eq .Sort "size") (ne .Order "desc")}}
-							<a href="?sort=size&order=desc{{if ne 0 .ItemsLimitedTo}}&limit={{.ItemsLimitedTo}}{{end}}">Size <svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#up-arrow"></use></svg></a>
+							<a href="?sort=size&order=desc{{if ne 0 .Limit}}&limit={{.Limit}}{{end}}{{if ne 0 .Offset}}&offset={{.Offset}}{{end}}">Size <svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#up-arrow"></use></svg></a>
 							{{- else if and (eq .Sort "size") (ne .Order "asc")}}
-							<a href="?sort=size&order=asc{{if ne 0 .ItemsLimitedTo}}&limit={{.ItemsLimitedTo}}{{end}}">Size <svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#down-arrow"></use></svg></a>
+							<a href="?sort=size&order=asc{{if ne 0 .Limit}}&limit={{.Limit}}{{end}}{{if ne 0 .Offset}}&offset={{.Offset}}{{end}}">Size <svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#down-arrow"></use></svg></a>
 							{{- else}}
-							<a href="?sort=size&order=asc{{if ne 0 .ItemsLimitedTo}}&limit={{.ItemsLimitedTo}}{{end}}">Size</a>
+							<a href="?sort=size&order=asc{{if ne 0 .Limit}}&limit={{.Limit}}{{end}}{{if ne 0 .Offset}}&offset={{.Offset}}{{end}}">Size</a>
 							{{- end}}
 						</th>
 						<th class="hideable">
 							{{- if and (eq .Sort "time") (ne .Order "desc")}}
-							<a href="?sort=time&order=desc{{if ne 0 .ItemsLimitedTo}}&limit={{.ItemsLimitedTo}}{{end}}">Modified <svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#up-arrow"></use></svg></a>
+							<a href="?sort=time&order=desc{{if ne 0 .Limit}}&limit={{.Limit}}{{end}}{{if ne 0 .Offset}}&offset={{.Offset}}{{end}}">Modified <svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#up-arrow"></use></svg></a>
 							{{- else if and (eq .Sort "time") (ne .Order "asc")}}
-							<a href="?sort=time&order=asc{{if ne 0 .ItemsLimitedTo}}&limit={{.ItemsLimitedTo}}{{end}}">Modified <svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#down-arrow"></use></svg></a>
+							<a href="?sort=time&order=asc{{if ne 0 .Limit}}&limit={{.Limit}}{{end}}{{if ne 0 .Offset}}&offset={{.Offset}}{{end}}">Modified <svg width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#down-arrow"></use></svg></a>
 							{{- else}}
-							<a href="?sort=time&order=asc{{if ne 0 .ItemsLimitedTo}}&limit={{.ItemsLimitedTo}}{{end}}">Modified</a>
+							<a href="?sort=time&order=asc{{if ne 0 .Limit}}&limit={{.Limit}}{{end}}{{if ne 0 .Offset}}&offset={{.Offset}}{{end}}">Modified</a>
 							{{- end}}
 						</th>
 						<th class="hideable"></th>

modules/caddyhttp/fileserver/caddyfile.go

@@ -15,6 +15,7 @@
 package fileserver
 
 import (
+	"path/filepath"
 	"strings"
 
 	"github.com/caddyserver/caddy/v2"
@@ -85,7 +86,14 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 	// hide the Caddyfile (and any imported Caddyfiles)
 	if configFiles := h.Caddyfiles(); len(configFiles) > 0 {
 		for _, file := range configFiles {
+			file = filepath.Clean(file)
 			if !fileHidden(file, fsrv.Hide) {
+				// if there's no path separator, the file server module will hide all
+				// files by that name, rather than a specific one; but we want to hide
+				// only this specific file, so ensure there's always a path separator
+				if !strings.Contains(file, separator) {
+					file = "." + separator + file
+				}
 				fsrv.Hide = append(fsrv.Hide, file)
 			}
 		}

modules/caddyhttp/fileserver/matcher.go

@@ -34,7 +34,7 @@ func init() {
 // MatchFile is an HTTP request matcher that can match
 // requests based upon file existence.
 //
-// Upon matching, two new placeholders will be made
+// Upon matching, three new placeholders will be made
 // available:
 //
 // - `{http.matchers.file.relative}` The root-relative
@@ -42,6 +42,10 @@ func init() {
 // requests.
 // - `{http.matchers.file.absolute}` The absolute path
 // of the matched file.
+// - `{http.matchers.file.type}` Set to "directory" if
+// the matched file is a directory, "file" otherwise.
+// - `{http.matchers.file.remainder}` Set to the remainder
+// of the path if the path was split by `split_path`.
 type MatchFile struct {
 	// The root directory, used for creating absolute
 	// file paths, and required when working with
@@ -117,11 +121,13 @@ func (m *MatchFile) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 					return d.ArgErr()
 				}
 				m.TryPolicy = d.Val()
-			case "split":
+			case "split_path":
 				m.SplitPath = d.RemainingArgs()
 				if len(m.SplitPath) == 0 {
 					return d.ArgErr()
 				}
+			default:
+				return d.Errf("unrecognized subdirective: %s", d.Val())
 			}
 		}
 	}
@@ -151,25 +157,19 @@ func (m MatchFile) Validate() error {
 }
 
 // Match returns true if r matches m. Returns true
-// if a file was matched. If so, two placeholders
+// if a file was matched. If so, four placeholders
 // will be available:
 //    - http.matchers.file.relative
 //    - http.matchers.file.absolute
+//    - http.matchers.file.type
+//    - http.matchers.file.remainder
 func (m MatchFile) Match(r *http.Request) bool {
-	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
+	return m.selectFile(r)
-	rel, abs, matched := m.selectFile(r)
-	if matched {
-		repl.Set("http.matchers.file.relative", rel)
-		repl.Set("http.matchers.file.absolute", abs)
-	}
-	return matched
 }
 
 // selectFile chooses a file according to m.TryPolicy by appending
 // the paths in m.TryFiles to m.Root, with placeholder replacements.
-// It returns the root-relative path to the matched file, the full
+func (m MatchFile) selectFile(r *http.Request) (matched bool) {
-// or absolute path, and whether a match was made.
-func (m MatchFile) selectFile(r *http.Request) (rel, abs string, matched bool) {
 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
 
 	root := repl.ReplaceAll(m.Root, ".")
@@ -181,13 +181,36 @@ func (m MatchFile) selectFile(r *http.Request) (rel, abs string, matched bool) {
 		m.TryFiles = []string{r.URL.Path}
 	}
 
+	// common preparation of the file into parts
+	prepareFilePath := func(file string) (suffix, fullpath, remainder string) {
+		suffix, remainder = m.firstSplit(path.Clean(repl.ReplaceAll(file, "")))
+		if strings.HasSuffix(file, "/") {
+			suffix += "/"
+		}
+		fullpath = sanitizedPathJoin(root, suffix)
+		return
+	}
+
+	// sets up the placeholders for the matched file
+	setPlaceholders := func(info os.FileInfo, rel string, abs string, remainder string) {
+		repl.Set("http.matchers.file.relative", rel)
+		repl.Set("http.matchers.file.absolute", abs)
+		repl.Set("http.matchers.file.remainder", remainder)
+
+		fileType := "file"
+		if info.IsDir() {
+			fileType = "directory"
+		}
+		repl.Set("http.matchers.file.type", fileType)
+	}
+
 	switch m.TryPolicy {
 	case "", tryPolicyFirstExist:
 		for _, f := range m.TryFiles {
-			suffix := m.firstSplit(path.Clean(repl.ReplaceAll(f, "")))
+			suffix, fullpath, remainder := prepareFilePath(f)
-			fullpath := sanitizedPathJoin(root, suffix)
+			if info, exists := strictFileExists(fullpath); exists {
-			if strictFileExists(fullpath) {
+				setPlaceholders(info, suffix, fullpath, remainder)
-				return suffix, fullpath, true
+				return true
 			}
 		}
 
@@ -195,50 +218,59 @@ func (m MatchFile) selectFile(r *http.Request) (rel, abs string, matched bool) {
 		var largestSize int64
 		var largestFilename string
 		var largestSuffix string
+		var remainder string
+		var info os.FileInfo
 		for _, f := range m.TryFiles {
-			suffix := m.firstSplit(path.Clean(repl.ReplaceAll(f, "")))
+			suffix, fullpath, splitRemainder := prepareFilePath(f)
-			fullpath := sanitizedPathJoin(root, suffix)
 			info, err := os.Stat(fullpath)
 			if err == nil && info.Size() > largestSize {
 				largestSize = info.Size()
 				largestFilename = fullpath
 				largestSuffix = suffix
+				remainder = splitRemainder
 			}
 		}
-		return largestSuffix, largestFilename, true
+		setPlaceholders(info, largestSuffix, largestFilename, remainder)
+		return true
 
 	case tryPolicySmallestSize:
 		var smallestSize int64
 		var smallestFilename string
 		var smallestSuffix string
+		var remainder string
+		var info os.FileInfo
 		for _, f := range m.TryFiles {
-			suffix := m.firstSplit(path.Clean(repl.ReplaceAll(f, "")))
+			suffix, fullpath, splitRemainder := prepareFilePath(f)
-			fullpath := sanitizedPathJoin(root, suffix)
 			info, err := os.Stat(fullpath)
 			if err == nil && (smallestSize == 0 || info.Size() < smallestSize) {
 				smallestSize = info.Size()
 				smallestFilename = fullpath
 				smallestSuffix = suffix
+				remainder = splitRemainder
 			}
 		}
-		return smallestSuffix, smallestFilename, true
+		setPlaceholders(info, smallestSuffix, smallestFilename, remainder)
+		return true
 
 	case tryPolicyMostRecentlyMod:
 		var recentDate time.Time
 		var recentFilename string
 		var recentSuffix string
+		var remainder string
+		var info os.FileInfo
 		for _, f := range m.TryFiles {
-			suffix := m.firstSplit(path.Clean(repl.ReplaceAll(f, "")))
+			suffix, fullpath, splitRemainder := prepareFilePath(f)
-			fullpath := sanitizedPathJoin(root, suffix)
 			info, err := os.Stat(fullpath)
 			if err == nil &&
 				(recentDate.IsZero() || info.ModTime().After(recentDate)) {
 				recentDate = info.ModTime()
 				recentFilename = fullpath
 				recentSuffix = suffix
+				remainder = splitRemainder
 			}
 		}
-		return recentSuffix, recentFilename, true
+		setPlaceholders(info, recentSuffix, recentFilename, remainder)
+		return true
 	}
 
 	return
@@ -250,7 +282,7 @@ func (m MatchFile) selectFile(r *http.Request) (rel, abs string, matched bool) {
 // the file must also be a directory; if it does
 // NOT end in a forward slash, the file must NOT
 // be a directory.
-func strictFileExists(file string) bool {
+func strictFileExists(file string) (os.FileInfo, bool) {
 	stat, err := os.Stat(file)
 	if err != nil {
 		// in reality, this can be any error
@@ -261,36 +293,49 @@ func strictFileExists(file string) bool {
 		// the file exists, so we just treat any
 		// error as if it does not exist; see
 		// https://stackoverflow.com/a/12518877/1048862
-		return false
+		return nil, false
 	}
-	if strings.HasSuffix(file, "/") {
+	if strings.HasSuffix(file, separator) {
 		// by convention, file paths ending
-		// in a slash must be a directory
+		// in a path separator must be a directory
-		return stat.IsDir()
+		return stat, stat.IsDir()
 	}
 	// by convention, file paths NOT ending
-	// in a slash must NOT be a directory
+	// in a path separator must NOT be a directory
-	return !stat.IsDir()
+	return stat, !stat.IsDir()
 }
 
 // firstSplit returns the first result where the path
 // can be split in two by a value in m.SplitPath. The
-// result is the first piece of the path that ends with
+// return values are the first piece of the path that
-// in the split value. Returns the path as-is if the
+// ends with the split substring and the remainder.
-// path cannot be split.
+// If the path cannot be split, the path is returned
-func (m MatchFile) firstSplit(path string) string {
+// as-is (with no remainder).
-	lowerPath := strings.ToLower(path)
+func (m MatchFile) firstSplit(path string) (splitPart, remainder string) {
 	for _, split := range m.SplitPath {
-		if idx := strings.Index(lowerPath, strings.ToLower(split)); idx > -1 {
+		if idx := indexFold(path, split); idx > -1 {
 			pos := idx + len(split)
 			// skip the split if it's not the final part of the filename
 			if pos != len(path) && !strings.HasPrefix(path[pos:], "/") {
 				continue
 			}
-			return path[:pos]
+			return path[:pos], path[pos:]
 		}
 	}
-	return path
+	return path, ""
+}
+
+// There is no strings.IndexFold() function like there is strings.EqualFold(),
+// but we can use strings.EqualFold() to build our own case-insensitive
+// substring search (as of Go 1.14).
+func indexFold(haystack, needle string) int {
+	nlen := len(needle)
+	for i := 0; i+nlen < len(haystack); i++ {
+		if strings.EqualFold(haystack[i:i+nlen], needle) {
+			return i
+		}
+	}
+	return -1
 }
 
 const (

modules/caddyhttp/fileserver/matcher_test.go

@@ -22,69 +22,64 @@ import (
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 )
 
-func TestPhpFileMatcher(t *testing.T) {
+func TestFileMatcher(t *testing.T) {
-
 	for i, tc := range []struct {
-		path string
+		path         string
 		expectedPath string
-		matched bool
+		expectedType string
+		matched      bool
 	}{
 		{
-			path: "/index.php",
+			path:         "/foo.txt",
-			expectedPath: "/index.php",
+			expectedPath: "/foo.txt",
-			matched: true,
+			expectedType: "file",
+			matched:      true,
 		},
 		{
-			path: "/index.php/somewhere",
+			path:         "/foo.txt/",
-			expectedPath: "/index.php",
+			expectedPath: "/foo.txt",
-			matched: true,
+			expectedType: "file",
+			matched:      true,
 		},
 		{
-			path: "/remote.php",
+			path:         "/foodir",
-			expectedPath: "/remote.php",
+			expectedPath: "/foodir/",
-			matched: true,
+			expectedType: "directory",
+			matched:      true,
 		},
 		{
-			path: "/remote.php/somewhere",
+			path:         "/foodir/",
-			expectedPath: "/remote.php",
+			expectedPath: "/foodir/",
-			matched: true,
+			expectedType: "directory",
+			matched:      true,
 		},
 		{
-			path: "/missingfile.php",
+			path:         "/foodir/foo.txt",
+			expectedPath: "/foodir/foo.txt",
+			expectedType: "file",
+			matched:      true,
+		},
+		{
+			path:    "/missingfile.php",
 			matched: false,
 		},
-		{
-			path: "/notphp.php.txt",
-			expectedPath: "/notphp.php.txt",
-			matched: true,
-		},
-		{
-			path: "/notphp.php.txt/",
-			expectedPath: "/notphp.php.txt",
-			matched: true,
-		},
-		{
-			path: "/notphp.php.txt.suffixed",
-			matched: false,
-		},
-		{
-			path: "/foo.php.php/index.php",
-			expectedPath: "/foo.php.php/index.php",
-			matched: true,
-		},
 	} {
 		m := &MatchFile{
-			Root:      "./testdata",
+			Root:     "./testdata",
-			TryFiles:  []string{"{http.request.uri.path}"},
+			TryFiles: []string{"{http.request.uri.path}", "{http.request.uri.path}/"},
-			SplitPath: []string{".php"},
 		}
 
-		req := &http.Request{URL: &url.URL{Path: tc.path}}
+		u, err := url.Parse(tc.path)
+		if err != nil {
+			t.Fatalf("Test %d: parsing path: %v", i, err)
+		}
+
+		req := &http.Request{URL: u}
 		repl := caddyhttp.NewTestReplacer(req)
 
 		result := m.Match(req)
 		if result != tc.matched {
-			t.Fatalf("Test %d: match bool result: %v, expected: %v", i, result, tc.matched)
+			t.Fatalf("Test %d: expected match=%t, got %t", i, tc.matched, result)
 		}
 
 		rel, ok := repl.Get("http.matchers.file.relative")
@@ -98,5 +93,126 @@ func TestPhpFileMatcher(t *testing.T) {
 		if rel != tc.expectedPath {
 			t.Fatalf("Test %d: actual path: %v, expected: %v", i, rel, tc.expectedPath)
 		}
+
+		fileType, ok := repl.Get("http.matchers.file.type")
+		if fileType != tc.expectedType {
+			t.Fatalf("Test %d: actual file type: %v, expected: %v", i, fileType, tc.expectedType)
+		}
+	}
+}
+
+func TestPHPFileMatcher(t *testing.T) {
+	for i, tc := range []struct {
+		path         string
+		expectedPath string
+		expectedType string
+		matched      bool
+	}{
+		{
+			path:         "/index.php",
+			expectedPath: "/index.php",
+			expectedType: "file",
+			matched:      true,
+		},
+		{
+			path:         "/index.php/somewhere",
+			expectedPath: "/index.php",
+			expectedType: "file",
+			matched:      true,
+		},
+		{
+			path:         "/remote.php",
+			expectedPath: "/remote.php",
+			expectedType: "file",
+			matched:      true,
+		},
+		{
+			path:         "/remote.php/somewhere",
+			expectedPath: "/remote.php",
+			expectedType: "file",
+			matched:      true,
+		},
+		{
+			path:    "/missingfile.php",
+			matched: false,
+		},
+		{
+			path:         "/notphp.php.txt",
+			expectedPath: "/notphp.php.txt",
+			expectedType: "file",
+			matched:      true,
+		},
+		{
+			path:         "/notphp.php.txt/",
+			expectedPath: "/notphp.php.txt",
+			expectedType: "file",
+			matched:      true,
+		},
+		{
+			path:    "/notphp.php.txt.suffixed",
+			matched: false,
+		},
+		{
+			path:         "/foo.php.php/index.php",
+			expectedPath: "/foo.php.php/index.php",
+			expectedType: "file",
+			matched:      true,
+		},
+		{
+			// See https://github.com/caddyserver/caddy/issues/3623
+			path:         "/%E2%C3",
+			expectedPath: "/%E2%C3",
+			expectedType: "file",
+			matched:      false,
+		},
+	} {
+		m := &MatchFile{
+			Root:      "./testdata",
+			TryFiles:  []string{"{http.request.uri.path}", "{http.request.uri.path}/index.php"},
+			SplitPath: []string{".php"},
+		}
+
+		u, err := url.Parse(tc.path)
+		if err != nil {
+			t.Fatalf("Test %d: parsing path: %v", i, err)
+		}
+
+		req := &http.Request{URL: u}
+		repl := caddyhttp.NewTestReplacer(req)
+
+		result := m.Match(req)
+		if result != tc.matched {
+			t.Fatalf("Test %d: expected match=%t, got %t", i, tc.matched, result)
+		}
+
+		rel, ok := repl.Get("http.matchers.file.relative")
+		if !ok && result {
+			t.Fatalf("Test %d: expected replacer value", i)
+		}
+		if !result {
+			continue
+		}
+
+		if rel != tc.expectedPath {
+			t.Fatalf("Test %d: actual path: %v, expected: %v", i, rel, tc.expectedPath)
+		}
+
+		fileType, ok := repl.Get("http.matchers.file.type")
+		if fileType != tc.expectedType {
+			t.Fatalf("Test %d: actual file type: %v, expected: %v", i, fileType, tc.expectedType)
+		}
+	}
+}
+
+func TestFirstSplit(t *testing.T) {
+	m := MatchFile{SplitPath: []string{".php"}}
+	actual, remainder := m.firstSplit("index.PHP/somewhere")
+	expected := "index.PHP"
+	expectedRemainder := "/somewhere"
+	if actual != expected {
+		t.Errorf("Expected split %s but got %s", expected, actual)
+	}
+	if remainder != expectedRemainder {
+		t.Errorf("Expected remainder %s but got %s", expectedRemainder, remainder)
 	}
 }

modules/caddyhttp/fileserver/staticfiles.go

@@ -23,7 +23,6 @@ import (
 	"mime"
 	"net/http"
 	"os"
-	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -32,6 +31,7 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"go.uber.org/zap"
 )
 
 func init() {
@@ -47,7 +47,20 @@ type FileServer struct {
 	Root string `json:"root,omitempty"`
 
 	// A list of files or folders to hide; the file server will pretend as if
-	// they don't exist. Accepts globular patterns like "*.hidden" or "/foo/*/bar".
+	// they don't exist. Accepts globular patterns like "*.ext" or "/foo/*/bar"
+	// as well as placeholders. Because site roots can be dynamic, this list
+	// uses file system paths, not request paths. To clarify, the base of
+	// relative paths is the current working directory, NOT the site root.
+	//
+	// Entries without a path separator (`/` or `\` depending on OS) will match
+	// any file or directory of that name regardless of its path. To hide only a
+	// specific file with a name that may not be unique, always use a path
+	// separator. For example, to hide all files or folder trees named "hidden",
+	// put "hidden" in the list. To hide only ./hidden, put "./hidden" in the list.
+	//
+	// When possible, all paths are resolved to their absolute form before
+	// comparisons are made. For maximum clarity and explictness, use complete,
+	// absolute paths; or, for greater portability, use relative paths instead.
 	Hide []string `json:"hide,omitempty"`
 
 	// The names of files to try as index files if a folder is requested.
@@ -65,6 +78,8 @@ type FileServer struct {
 	// it will invoke the next handler in the chain instead of returning
 	// a 404 error. By default, this is false (disabled).
 	PassThru bool `json:"pass_thru,omitempty"`
+
+	logger *zap.Logger
 }
 
 // CaddyModule returns the Caddy module information.
@@ -77,6 +92,8 @@ func (FileServer) CaddyModule() caddy.ModuleInfo {
 
 // Provision sets up the static files responder.
 func (fsrv *FileServer) Provision(ctx caddy.Context) error {
+	fsrv.logger = ctx.Logger(fsrv)
+
 	if fsrv.Root == "" {
 		fsrv.Root = "{http.vars.root}"
 	}
@@ -102,6 +119,16 @@ func (fsrv *FileServer) Provision(ctx caddy.Context) error {
 		fsrv.Browse.template = tpl
 	}
 
+	// for hide paths that are static (i.e. no placeholders), we can transform them into
+	// absolute paths before the server starts for very slight performance improvement
+	for i, h := range fsrv.Hide {
+		if !strings.Contains(h, "{") && strings.Contains(h, separator) {
+			if abs, err := filepath.Abs(h); err == nil {
+				fsrv.Hide[i] = abs
+			}
+		}
+	}
+
 	return nil
 }
 
@@ -114,6 +141,11 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 	suffix := repl.ReplaceAll(r.URL.Path, "")
 	filename := sanitizedPathJoin(root, suffix)
 
+	fsrv.logger.Debug("sanitized path join",
+		zap.String("site_root", root),
+		zap.String("request_path", suffix),
+		zap.String("result", filename))
+
 	// get information about the file
 	info, err := os.Stat(filename)
 	if err != nil {
@@ -135,6 +167,9 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 			indexPath := sanitizedPathJoin(filename, indexPage)
 			if fileHidden(indexPath, filesToHide) {
 				// pretend this file doesn't exist
+				fsrv.logger.Debug("hiding index file",
+					zap.String("filename", indexPath),
+					zap.Strings("files_to_hide", filesToHide))
 				continue
 			}
 
@@ -154,6 +189,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 			info = indexInfo
 			filename = indexPath
 			implicitIndexFile = true
+			fsrv.logger.Debug("located index file", zap.String("filename", filename))
 			break
 		}
 	}
@@ -161,8 +197,11 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 	// if still referencing a directory, delegate
 	// to browse or return an error
 	if info.IsDir() {
+		fsrv.logger.Debug("no index file in directory",
+			zap.String("path", filename),
+			zap.Strings("index_filenames", fsrv.IndexNames))
 		if fsrv.Browse != nil && !fileHidden(filename, filesToHide) {
-			return fsrv.serveBrowse(filename, w, r, next)
+			return fsrv.serveBrowse(root, filename, w, r, next)
 		}
 		return fsrv.notFound(w, r, next)
 	}
@@ -172,6 +211,9 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 	// one last check to ensure the file isn't hidden (we might
 	// have changed the filename from when we last checked)
 	if fileHidden(filename, filesToHide) {
+		fsrv.logger.Debug("hiding file",
+			zap.String("filename", filename),
+			zap.Strings("files_to_hide", filesToHide))
 		return fsrv.notFound(w, r, next)
 	}
 
@@ -181,12 +223,16 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 	// in HTML (see https://github.com/caddyserver/caddy/issues/2741)
 	if fsrv.CanonicalURIs == nil || *fsrv.CanonicalURIs {
 		if implicitIndexFile && !strings.HasSuffix(r.URL.Path, "/") {
+			fsrv.logger.Debug("redirecting to canonical URI (adding trailing slash for directory)", zap.String("path", r.URL.Path))
 			return redirect(w, r, r.URL.Path+"/")
 		} else if !implicitIndexFile && strings.HasSuffix(r.URL.Path, "/") {
+			fsrv.logger.Debug("redirecting to canonical URI (removing trailing slash for file)", zap.String("path", r.URL.Path))
 			return redirect(w, r, r.URL.Path[:len(r.URL.Path)-1])
 		}
 	}
 
+	fsrv.logger.Debug("opening file", zap.String("filename", filename))
+
 	// open the file
 	file, err := fsrv.openFile(filename, w)
 	if err != nil {
@@ -226,8 +272,8 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 			}
 		}
 		w.WriteHeader(statusCode)
-		if r.Method != "HEAD" {
+		if r.Method != http.MethodHead {
-			io.Copy(w, file)
+			_, _ = io.Copy(w, file)
 		}
 		return nil
 	}
@@ -256,6 +302,7 @@ func (fsrv *FileServer) openFile(filename string, w http.ResponseWriter) (*os.Fi
 		}
 		// maybe the server is under load and ran out of file descriptors?
 		// have client wait arbitrary seconds to help prevent a stampede
+		//nolint:gosec
 		backoff := weakrand.Intn(maxBackoff-minBackoff) + minBackoff
 		w.Header().Set("Retry-After", strconv.Itoa(backoff))
 		return nil, caddyhttp.Error(http.StatusServiceUnavailable, err)
@@ -274,12 +321,12 @@ func mapDirOpenError(originalErr error, name string) error {
 		return originalErr
 	}
 
-	parts := strings.Split(name, string(filepath.Separator))
+	parts := strings.Split(name, separator)
 	for i := range parts {
 		if parts[i] == "" {
 			continue
 		}
-		fi, err := os.Stat(strings.Join(parts[:i+1], string(filepath.Separator)))
+		fi, err := os.Stat(strings.Join(parts[:i+1], separator))
 		if err != nil {
 			return originalErr
 		}
@@ -291,12 +338,19 @@ func mapDirOpenError(originalErr error, name string) error {
 	return originalErr
 }
 
-// transformHidePaths performs replacements for all the elements of
+// transformHidePaths performs replacements for all the elements of fsrv.Hide and
-// fsrv.Hide and returns a new list of the transformed values.
+// makes them absolute paths (if they contain a path separator), then returns a
+// new list of the transformed values.
 func (fsrv *FileServer) transformHidePaths(repl *caddy.Replacer) []string {
 	hide := make([]string, len(fsrv.Hide))
 	for i := range fsrv.Hide {
 		hide[i] = repl.ReplaceAll(fsrv.Hide[i], "")
+		if strings.Contains(hide[i], separator) {
+			abs, err := filepath.Abs(hide[i])
+			if err == nil {
+				hide[i] = abs
+			}
+		}
 	}
 	return hide
 }
@@ -323,34 +377,64 @@ func sanitizedPathJoin(root, reqPath string) string {
 	if root == "" {
 		root = "."
 	}
-	return filepath.Join(root, filepath.FromSlash(path.Clean("/"+reqPath)))
+
+	path := filepath.Join(root, filepath.Clean("/"+reqPath))
+
+	// filepath.Join also cleans the path, and cleaning strips
+	// the trailing slash, so we need to re-add it afterwards.
+	// if the length is 1, then it's a path to the root,
+	// and that should return ".", so we don't append the separator.
+	if strings.HasSuffix(reqPath, "/") && len(reqPath) > 1 {
+		path += separator
+	}
+
+	return path
 }
 
-// fileHidden returns true if filename is hidden
+// fileHidden returns true if filename is hidden according to the hide list.
-// according to the hide list.
+// filename must be a relative or absolute file system path, not a request
+// URI path. It is expected that all the paths in the hide list are absolute
+// paths or are singular filenames (without a path separator).
 func fileHidden(filename string, hide []string) bool {
-	nameOnly := filepath.Base(filename)
+	if len(hide) == 0 {
-	sep := string(filepath.Separator)
+		return false
+	}
+
+	// all path comparisons use the complete absolute path if possible
+	filenameAbs, err := filepath.Abs(filename)
+	if err == nil {
+		filename = filenameAbs
+	}
+
+	var components []string
 
 	for _, h := range hide {
-		// assuming h is a glob/shell-like pattern,
+		if !strings.Contains(h, separator) {
-		// use it to compare the whole file path;
+			// if there is no separator in h, then we assume the user
-		// but if there is no separator in h, then
+			// wants to hide any files or folders that match that
-		// just compare against the file's name
+			// name; thus we have to compare against each component
-		compare := filename
+			// of the filename, e.g. hiding "bar" would hide "/bar"
-		if !strings.Contains(h, sep) {
+			// as well as "/foo/bar/baz" but not "/barstool".
-			compare = nameOnly
+			if len(components) == 0 {
-		}
+				components = strings.Split(filename, separator)
-
+			}
-		hidden, err := filepath.Match(h, compare)
+			for _, c := range components {
-		if err != nil {
+				if hidden, _ := filepath.Match(h, c); hidden {
-			// malformed pattern; fallback by checking prefix
+					return true
-			if strings.HasPrefix(filename, h) {
+				}
+			}
+		} else if strings.HasPrefix(filename, h) {
+			// if there is a separator in h, and filename is exactly
+			// prefixed with h, then we can do a prefix match so that
+			// "/foo" matches "/foo/bar" but not "/foobar".
+			withoutPrefix := strings.TrimPrefix(filename, h)
+			if strings.HasPrefix(withoutPrefix, separator) {
 				return true
 			}
 		}
-		if hidden {
+
-			// file name or path matches hide pattern
+		// in the general case, a glob match will suffice
+		if hidden, _ := filepath.Match(h, filename); hidden {
 			return true
 		}
 	}
@@ -395,7 +479,10 @@ var bufPool = sync.Pool{
 	},
 }
 
-const minBackoff, maxBackoff = 2, 5
+const (
+	minBackoff, maxBackoff = 2, 5
+	separator              = string(filepath.Separator)
+)
 
 // Interface guards
 var (

modules/caddyhttp/fileserver/staticfiles_test.go

@@ -17,6 +17,8 @@ package fileserver
 import (
 	"net/url"
 	"path/filepath"
+	"runtime"
+	"strings"
 	"testing"
 )
 
@@ -42,6 +44,10 @@ func TestSanitizedPathJoin(t *testing.T) {
 			inputPath: "/foo",
 			expect:    "foo",
 		},
+		{
+			inputPath: "/foo/",
+			expect:    "foo" + separator,
+		},
 		{
 			inputPath: "/foo/bar",
 			expect:    filepath.Join("foo", "bar"),
@@ -73,7 +79,7 @@ func TestSanitizedPathJoin(t *testing.T) {
 		{
 			inputRoot: "/a/b",
 			inputPath: "/%2e%2e%2f%2e%2e%2f",
-			expect:    filepath.Join("/", "a", "b"),
+			expect:    filepath.Join("/", "a", "b") + separator,
 		},
 		{
 			inputRoot: "C:\\www",
@@ -93,9 +99,116 @@ func TestSanitizedPathJoin(t *testing.T) {
 		}
 		actual := sanitizedPathJoin(tc.inputRoot, u.Path)
 		if actual != tc.expect {
-			t.Errorf("Test %d: [%s %s] => %s (expected %s)", i, tc.inputRoot, tc.inputPath, actual, tc.expect)
+			t.Errorf("Test %d: [%s %s] => %s (expected %s)",
+				i, tc.inputRoot, tc.inputPath, actual, tc.expect)
 		}
 	}
 }
 
-// TODO: test fileHidden
+func TestFileHidden(t *testing.T) {
+	for i, tc := range []struct {
+		inputHide []string
+		inputPath string
+		expect    bool
+	}{
+		{
+			inputHide: nil,
+			inputPath: "",
+			expect:    false,
+		},
+		{
+			inputHide: []string{".gitignore"},
+			inputPath: "/.gitignore",
+			expect:    true,
+		},
+		{
+			inputHide: []string{".git"},
+			inputPath: "/.gitignore",
+			expect:    false,
+		},
+		{
+			inputHide: []string{"/.git"},
+			inputPath: "/.gitignore",
+			expect:    false,
+		},
+		{
+			inputHide: []string{".git"},
+			inputPath: "/.git",
+			expect:    true,
+		},
+		{
+			inputHide: []string{".git"},
+			inputPath: "/.git/foo",
+			expect:    true,
+		},
+		{
+			inputHide: []string{".git"},
+			inputPath: "/foo/.git/bar",
+			expect:    true,
+		},
+		{
+			inputHide: []string{"/prefix"},
+			inputPath: "/prefix/foo",
+			expect:    true,
+		},
+		{
+			inputHide: []string{"/foo/*/bar"},
+			inputPath: "/foo/asdf/bar",
+			expect:    true,
+		},
+		{
+			inputHide: []string{"*.txt"},
+			inputPath: "/foo/bar.txt",
+			expect:    true,
+		},
+		{
+			inputHide: []string{"/foo/bar/*.txt"},
+			inputPath: "/foo/bar/baz.txt",
+			expect:    true,
+		},
+		{
+			inputHide: []string{"/foo/bar/*.txt"},
+			inputPath: "/foo/bar.txt",
+			expect:    false,
+		},
+		{
+			inputHide: []string{"/foo/bar/*.txt"},
+			inputPath: "/foo/bar/index.html",
+			expect:    false,
+		},
+		{
+			inputHide: []string{"/foo"},
+			inputPath: "/foo",
+			expect:    true,
+		},
+		{
+			inputHide: []string{"/foo"},
+			inputPath: "/foobar",
+			expect:    false,
+		},
+		{
+			inputHide: []string{"first", "second"},
+			inputPath: "/second",
+			expect:    true,
+		},
+	} {
+		if runtime.GOOS == "windows" {
+			if strings.HasPrefix(tc.inputPath, "/") {
+				tc.inputPath, _ = filepath.Abs(tc.inputPath)
+			}
+			tc.inputPath = filepath.FromSlash(tc.inputPath)
+			for i := range tc.inputHide {
+				if strings.HasPrefix(tc.inputHide[i], "/") {
+					tc.inputHide[i], _ = filepath.Abs(tc.inputHide[i])
+				}
+				tc.inputHide[i] = filepath.FromSlash(tc.inputHide[i])
+			}
+		}
+
+		actual := fileHidden(tc.inputPath, tc.inputHide)
+		if actual != tc.expect {
+			t.Errorf("Test %d: Does %v hide %s? Got %t but expected %t",
+				i, tc.inputHide, tc.inputPath, actual, tc.expect)
+		}
+	}
+}

modules/caddyhttp/fileserver/testdata/foo.txt

@@ -0,0 +1 @@
+foo.txt

modules/caddyhttp/fileserver/testdata/foodir/foo.txt

@@ -0,0 +1 @@
+foodir/foo.txt

modules/caddyhttp/headers/caddyfile.go

@@ -15,7 +15,9 @@
 package headers
 
 import (
+	"fmt"
 	"net/http"
+	"reflect"
 	"strings"
 
 	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
@@ -23,15 +25,16 @@ import (
 )
 
 func init() {
-	httpcaddyfile.RegisterHandlerDirective("header", parseCaddyfile)
+	httpcaddyfile.RegisterDirective("header", parseCaddyfile)
-	httpcaddyfile.RegisterHandlerDirective("request_header", parseReqHdrCaddyfile)
+	httpcaddyfile.RegisterDirective("request_header", parseReqHdrCaddyfile)
 }
 
 // parseCaddyfile sets up the handler for response headers from
 // Caddyfile tokens. Syntax:
 //
-//     header [<matcher>] [[+|-]<field> [<value|regexp>] [<replacement>]] {
+//     header [<matcher>] [[+|-|?]<field> [<value|regexp>] [<replacement>]] {
 //         [+]<field> [<value|regexp> [<replacement>]]
+//         ?<field> <default_value>
 //         -<field>
 //         [defer]
 //     }
@@ -39,17 +42,27 @@ func init() {
 // Either a block can be opened or a single header field can be configured
 // in the first line, but not both in the same directive. Header operations
 // are deferred to write-time if any headers are being deleted or if the
-// 'defer' subdirective is used.
+// 'defer' subdirective is used. + appends a header value, - deletes a field,
-func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
+// and ? conditionally sets a value only if the header field is not already
-	hdr := new(Handler)
+// set.
+func parseCaddyfile(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error) {
+	if !h.Next() {
+		return nil, h.ArgErr()
+	}
 
-	makeResponseOps := func() {
+	matcherSet, err := h.ExtractMatcherSet()
-		if hdr.Response == nil {
+	if err != nil {
-			hdr.Response = &RespHeaderOps{
+		return nil, err
-				HeaderOps: new(HeaderOps),
+	}
-			}
+
+	makeHandler := func() Handler {
+		return Handler{
+			Response: &RespHeaderOps{
+				HeaderOps: &HeaderOps{},
+			},
 		}
 	}
+	handler, handlerWithRequire := makeHandler(), makeHandler()
 
 	for h.Next() {
 		// first see if headers are in the initial line
@@ -64,10 +77,18 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 			if h.NextArg() {
 				replacement = h.Val()
 			}
-			makeResponseOps()
+			err := applyHeaderOp(
-			CaddyfileHeaderOp(hdr.Response.HeaderOps, field, value, replacement)
+				handler.Response.HeaderOps,
-			if len(hdr.Response.HeaderOps.Delete) > 0 {
+				handler.Response,
-				hdr.Response.Deferred = true
+				field,
+				value,
+				replacement,
+			)
+			if err != nil {
+				return nil, h.Err(err.Error())
+			}
+			if len(handler.Response.HeaderOps.Delete) > 0 {
+				handler.Response.Deferred = true
 			}
 		}
 
@@ -75,12 +96,18 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 		for h.NextBlock(0) {
 			field := h.Val()
 			if field == "defer" {
-				hdr.Response.Deferred = true
+				handler.Response.Deferred = true
 				continue
 			}
 			if hasArgs {
-				return nil, h.Err("cannot specify headers in both arguments and block")
+				return nil, h.Err("cannot specify headers in both arguments and block") // because it would be weird
 			}
+
+			// sometimes it is habitual for users to suffix a field name with a colon,
+			// as if they were writing a curl command or something; see
+			// https://caddy.community/t/v2-reverse-proxy-please-add-cors-example-to-the-docs/7349/19
+			field = strings.TrimSuffix(field, ":")
+
 			var value, replacement string
 			if h.NextArg() {
 				value = h.Val()
@@ -88,15 +115,34 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 			if h.NextArg() {
 				replacement = h.Val()
 			}
-			makeResponseOps()
+
-			CaddyfileHeaderOp(hdr.Response.HeaderOps, field, value, replacement)
+			handlerToUse := handler
-			if len(hdr.Response.HeaderOps.Delete) > 0 {
+			if strings.HasPrefix(field, "?") {
-				hdr.Response.Deferred = true
+				handlerToUse = handlerWithRequire
+			}
+
+			err := applyHeaderOp(
+				handlerToUse.Response.HeaderOps,
+				handlerToUse.Response,
+				field,
+				value,
+				replacement,
+			)
+			if err != nil {
+				return nil, h.Err(err.Error())
 			}
 		}
 	}
 
-	return hdr, nil
+	var configValues []httpcaddyfile.ConfigValue
+	if !reflect.DeepEqual(handler, makeHandler()) {
+		configValues = append(configValues, h.NewRoute(matcherSet, handler)...)
+	}
+	if !reflect.DeepEqual(handlerWithRequire, makeHandler()) {
+		configValues = append(configValues, h.NewRoute(matcherSet, handlerWithRequire)...)
+	}
+
+	return configValues, nil
 }
 
 // parseReqHdrCaddyfile sets up the handler for request headers
@@ -104,17 +150,27 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 //
 //     request_header [<matcher>] [[+|-]<field> [<value|regexp>] [<replacement>]]
 //
-func parseReqHdrCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
+func parseReqHdrCaddyfile(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error) {
-	hdr := new(Handler)
+	matcherSet, err := h.ExtractMatcherSet()
+	if err != nil {
+		return nil, err
+	}
+
+	configValues := []httpcaddyfile.ConfigValue{}
+
 	for h.Next() {
 		if !h.NextArg() {
 			return nil, h.ArgErr()
 		}
 		field := h.Val()
 
+		hdr := Handler{
+			Request: &HeaderOps{},
+		}
+
 		// sometimes it is habitual for users to suffix a field name with a colon,
 		// as if they were writing a curl command or something; see
-		// https://caddy.community/t/v2-reverse-proxy-please-add-cors-example-to-the-docs/7349
+		// https://caddy.community/t/v2-reverse-proxy-please-add-cors-example-to-the-docs/7349/19
 		field = strings.TrimSuffix(field, ":")
 
 		var value, replacement string
@@ -131,13 +187,17 @@ func parseReqHdrCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler,
 		if hdr.Request == nil {
 			hdr.Request = new(HeaderOps)
 		}
-		CaddyfileHeaderOp(hdr.Request, field, value, replacement)
+		if err := CaddyfileHeaderOp(hdr.Request, field, value, replacement); err != nil {
+			return nil, h.Err(err.Error())
+		}
+
+		configValues = append(configValues, h.NewRoute(matcherSet, hdr)...)
 
 		if h.NextArg() {
 			return nil, h.ArgErr()
 		}
 	}
-	return hdr, nil
+	return configValues, nil
 }
 
 // CaddyfileHeaderOp applies a new header operation according to
@@ -148,32 +208,59 @@ func parseReqHdrCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler,
 // will be used to search and then replacement will be used to
 // complete the substring replacement; in that case, any + or -
 // prefix to field will be ignored.
-func CaddyfileHeaderOp(ops *HeaderOps, field, value, replacement string) {
+func CaddyfileHeaderOp(ops *HeaderOps, field, value, replacement string) error {
-	if strings.HasPrefix(field, "+") {
+	return applyHeaderOp(ops, nil, field, value, replacement)
+}
+
+func applyHeaderOp(ops *HeaderOps, respHeaderOps *RespHeaderOps, field, value, replacement string) error {
+	switch {
+	case strings.HasPrefix(field, "+"): // append
 		if ops.Add == nil {
 			ops.Add = make(http.Header)
 		}
 		ops.Add.Set(field[1:], value)
-	} else if strings.HasPrefix(field, "-") {
+
+	case strings.HasPrefix(field, "-"): // delete
 		ops.Delete = append(ops.Delete, field[1:])
-	} else {
+		if respHeaderOps != nil {
-		if replacement == "" {
+			respHeaderOps.Deferred = true
-			if ops.Set == nil {
-				ops.Set = make(http.Header)
-			}
-			ops.Set.Set(field, value)
-		} else {
-			if ops.Replace == nil {
-				ops.Replace = make(map[string][]Replacement)
-			}
-			field = strings.TrimLeft(field, "+-")
-			ops.Replace[field] = append(
-				ops.Replace[field],
-				Replacement{
-					SearchRegexp: value,
-					Replace:      replacement,
-				},
-			)
 		}
+
+	case strings.HasPrefix(field, "?"): // default (conditional on not existing) - response headers only
+		if respHeaderOps == nil {
+			return fmt.Errorf("%v: the default header modifier ('?') can only be used on response headers; for conditional manipulation of request headers, use matchers", field)
+		}
+		if respHeaderOps.Require == nil {
+			respHeaderOps.Require = &caddyhttp.ResponseMatcher{
+				Headers: make(http.Header),
+			}
+		}
+		field = strings.TrimPrefix(field, "?")
+		respHeaderOps.Require.Headers[field] = nil
+		if respHeaderOps.Set == nil {
+			respHeaderOps.Set = make(http.Header)
+		}
+		respHeaderOps.Set.Set(field, value)
+
+	case replacement != "": // replace
+		if ops.Replace == nil {
+			ops.Replace = make(map[string][]Replacement)
+		}
+		field = strings.TrimLeft(field, "+-?")
+		ops.Replace[field] = append(
+			ops.Replace[field],
+			Replacement{
+				SearchRegexp: value,
+				Replace:      replacement,
+			},
+		)
+
+	default: // set (overwrite)
+		if ops.Set == nil {
+			ops.Set = make(http.Header)
+		}
+		ops.Set.Set(field, value)
 	}
+
+	return nil
 }

modules/caddyhttp/headers/headers.go

@@ -54,15 +54,15 @@ func (Handler) CaddyModule() caddy.ModuleInfo {
 }
 
 // Provision sets up h's configuration.
-func (h *Handler) Provision(_ caddy.Context) error {
+func (h *Handler) Provision(ctx caddy.Context) error {
 	if h.Request != nil {
-		err := h.Request.provision()
+		err := h.Request.Provision(ctx)
 		if err != nil {
 			return err
 		}
 	}
 	if h.Response != nil {
-		err := h.Response.provision()
+		err := h.Response.Provision(ctx)
 		if err != nil {
 			return err
 		}
@@ -125,7 +125,8 @@ type HeaderOps struct {
 	Replace map[string][]Replacement `json:"replace,omitempty"`
 }
 
-func (ops *HeaderOps) provision() error {
+// Provision sets up the header operations.
+func (ops *HeaderOps) Provision(_ caddy.Context) error {
 	for fieldName, replacements := range ops.Replace {
 		for i, r := range replacements {
 			if r.SearchRegexp != "" {

modules/caddyhttp/headers/headers_test.go

@@ -14,8 +14,197 @@
 
 package headers
 
-import "testing"
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
 
-func TestReqHeaders(t *testing.T) {
+	"github.com/caddyserver/caddy/v2"
-	// TODO: write tests
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+)
+
+func TestHandler(t *testing.T) {
+	for i, tc := range []struct {
+		handler            Handler
+		reqHeader          http.Header
+		respHeader         http.Header
+		respStatusCode     int
+		expectedReqHeader  http.Header
+		expectedRespHeader http.Header
+	}{
+		{
+			handler: Handler{
+				Request: &HeaderOps{
+					Add: http.Header{
+						"Expose-Secrets": []string{"always"},
+					},
+				},
+			},
+			reqHeader: http.Header{
+				"Expose-Secrets": []string{"i'm serious"},
+			},
+			expectedReqHeader: http.Header{
+				"Expose-Secrets": []string{"i'm serious", "always"},
+			},
+		},
+		{
+			handler: Handler{
+				Request: &HeaderOps{
+					Set: http.Header{
+						"Who-Wins": []string{"batman"},
+					},
+				},
+			},
+			reqHeader: http.Header{
+				"Who-Wins": []string{"joker"},
+			},
+			expectedReqHeader: http.Header{
+				"Who-Wins": []string{"batman"},
+			},
+		},
+		{
+			handler: Handler{
+				Request: &HeaderOps{
+					Delete: []string{"Kick-Me"},
+				},
+			},
+			reqHeader: http.Header{
+				"Kick-Me": []string{"if you can"},
+				"Keep-Me": []string{"i swear i'm innocent"},
+			},
+			expectedReqHeader: http.Header{
+				"Keep-Me": []string{"i swear i'm innocent"},
+			},
+		},
+		{
+			handler: Handler{
+				Request: &HeaderOps{
+					Replace: map[string][]Replacement{
+						"Best-Server": {
+							Replacement{
+								Search:  "NGINX",
+								Replace: "the Caddy web server",
+							},
+							Replacement{
+								SearchRegexp: `Apache(\d+)`,
+								Replace:      "Caddy",
+							},
+						},
+					},
+				},
+			},
+			reqHeader: http.Header{
+				"Best-Server": []string{"it's NGINX, undoubtedly", "I love Apache2"},
+			},
+			expectedReqHeader: http.Header{
+				"Best-Server": []string{"it's the Caddy web server, undoubtedly", "I love Caddy"},
+			},
+		},
+		{
+			handler: Handler{
+				Response: &RespHeaderOps{
+					Require: &caddyhttp.ResponseMatcher{
+						Headers: http.Header{
+							"Cache-Control": nil,
+						},
+					},
+					HeaderOps: &HeaderOps{
+						Add: http.Header{
+							"Cache-Control": []string{"no-cache"},
+						},
+					},
+				},
+			},
+			respHeader: http.Header{},
+			expectedRespHeader: http.Header{
+				"Cache-Control": []string{"no-cache"},
+			},
+		},
+		{
+			handler: Handler{
+				Response: &RespHeaderOps{
+					Require: &caddyhttp.ResponseMatcher{
+						Headers: http.Header{
+							"Cache-Control": []string{"no-cache"},
+						},
+					},
+					HeaderOps: &HeaderOps{
+						Delete: []string{"Cache-Control"},
+					},
+				},
+			},
+			respHeader: http.Header{
+				"Cache-Control": []string{"no-cache"},
+			},
+			expectedRespHeader: http.Header{},
+		},
+		{
+			handler: Handler{
+				Response: &RespHeaderOps{
+					Require: &caddyhttp.ResponseMatcher{
+						StatusCode: []int{5},
+					},
+					HeaderOps: &HeaderOps{
+						Add: http.Header{
+							"Fail-5xx": []string{"true"},
+						},
+					},
+				},
+			},
+			respStatusCode: 503,
+			respHeader:     http.Header{},
+			expectedRespHeader: http.Header{
+				"Fail-5xx": []string{"true"},
+			},
+		},
+	} {
+		rr := httptest.NewRecorder()
+
+		req := &http.Request{Header: tc.reqHeader}
+		repl := caddy.NewReplacer()
+		ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
+		req = req.WithContext(ctx)
+
+		tc.handler.Provision(caddy.Context{})
+
+		next := nextHandler(func(w http.ResponseWriter, r *http.Request) error {
+			for k, hdrs := range tc.respHeader {
+				for _, v := range hdrs {
+					w.Header().Add(k, v)
+				}
+			}
+
+			status := 200
+			if tc.respStatusCode != 0 {
+				status = tc.respStatusCode
+			}
+			w.WriteHeader(status)
+
+			if tc.expectedReqHeader != nil && !reflect.DeepEqual(r.Header, tc.expectedReqHeader) {
+				return fmt.Errorf("expected request header %v, got %v", tc.expectedReqHeader, r.Header)
+			}
+
+			return nil
+		})
+
+		if err := tc.handler.ServeHTTP(rr, req, next); err != nil {
+			t.Errorf("Test %d: %w", i, err)
+			continue
+		}
+
+		actual := rr.Header()
+		if tc.expectedRespHeader != nil && !reflect.DeepEqual(actual, tc.expectedRespHeader) {
+			t.Errorf("Test %d: expected response header %v, got %v", i, tc.expectedRespHeader, actual)
+			continue
+		}
+	}
+}
+
+type nextHandler func(http.ResponseWriter, *http.Request) error
+
+func (f nextHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) error {
+	return f(w, r)
 }

modules/caddyhttp/map/caddyfile.go

@@ -15,6 +15,8 @@
 package maphandler
 
 import (
+	"strings"
+
 	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 )
@@ -23,49 +25,85 @@ func init() {
 	httpcaddyfile.RegisterHandlerDirective("map", parseCaddyfile)
 }
 
-// parseCaddyfile sets up the handler for a map from Caddyfile tokens. Syntax:
+// parseCaddyfile sets up the map handler from Caddyfile tokens. Syntax:
 //
-//     map <source> <dest> {
+//     map [<matcher>] <source> <destinations...> {
-//         [default <default>] - used if not match is found
+//         [~]<input> <outputs...>
-//         [<regexp> <replacement>] - regular expression to match against the source find and the matching replacement value
+//         default    <defaults...>
-//         ...
 //     }
 //
-// The map takes a source variable and maps it into the dest variable. The mapping process
+// If the input value is prefixed with a tilde (~), then the input will be parsed as a
-// will check the source variable for the first successful match against a list of regular expressions.
+// regular expression.
-// If a successful match is found the dest variable will contain the replacement value.
-// If no successful match is found and the default is specified then the dest will contain the default value.
 //
+// The Caddyfile adapter treats outputs that are a literal hyphen (-) as a null/nil
+// value. This is useful if you want to fall back to default for that particular output.
+//
+// The number of outputs for each mapping must not be more than the number of destinations.
+// However, for convenience, there may be fewer outputs than destinations and any missing
+// outputs will be filled in implicitly.
 func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
-	m := new(Handler)
+	var handler Handler
 
 	for h.Next() {
-		// first see if source and dest are configured
+		// source
-		if h.NextArg() {
+		if !h.NextArg() {
-			m.Source = h.Val()
+			return nil, h.ArgErr()
-			if h.NextArg() {
+		}
-				m.Destination = h.Val()
+		handler.Source = h.Val()
-			}
+
+		// destinations
+		handler.Destinations = h.RemainingArgs()
+		if len(handler.Destinations) == 0 {
+			return nil, h.Err("missing destination argument(s)")
 		}
 
-		// load the rules
+		// mappings
 		for h.NextBlock(0) {
-			expression := h.Val()
+			// defaults are a special case
-			if expression == "default" {
+			if h.Val() == "default" {
-				args := h.RemainingArgs()
+				if len(handler.Defaults) > 0 {
-				if len(args) != 1 {
+					return nil, h.Err("defaults already defined")
-					return m, h.ArgErr()
 				}
-				m.Default = args[0]
+				handler.Defaults = h.RemainingArgs()
-			} else {
+				for len(handler.Defaults) < len(handler.Destinations) {
-				args := h.RemainingArgs()
+					handler.Defaults = append(handler.Defaults, "")
-				if len(args) != 1 {
-					return m, h.ArgErr()
 				}
-				m.Items = append(m.Items, Item{Expression: expression, Value: args[0]})
+				continue
 			}
+
+			// every other line maps one input to one or more outputs
+			in := h.Val()
+			var outs []interface{}
+			for _, out := range h.RemainingArgs() {
+				if out == "-" {
+					outs = append(outs, nil)
+				} else {
+					outs = append(outs, out)
+				}
+			}
+
+			// cannot have more outputs than destinations
+			if len(outs) > len(handler.Destinations) {
+				return nil, h.Err("too many outputs")
+			}
+
+			// for convenience, can have fewer outputs than destinations, but the
+			// underlying handler won't accept that, so we fill in nil values
+			for len(outs) < len(handler.Destinations) {
+				outs = append(outs, nil)
+			}
+
+			// create the mapping
+			mapping := Mapping{Outputs: outs}
+			if strings.HasPrefix(in, "~") {
+				mapping.InputRegexp = in[1:]
+			} else {
+				mapping.Input = in
+			}
+
+			handler.Mappings = append(handler.Mappings, mapping)
 		}
 	}
 
-	return m, nil
+	return handler, nil
 }

modules/caddyhttp/map/map.go

@@ -15,8 +15,10 @@
 package maphandler
 
 import (
+	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
@@ -26,27 +28,27 @@ func init() {
 	caddy.RegisterModule(Handler{})
 }
 
-// Handler is a middleware that maps a source placeholder to a destination
+// Handler implements a middleware that maps inputs to outputs. Specifically, it
-// placeholder.
+// compares a source value against the map inputs, and for one that matches, it
-//
+// applies the output values to each destination. Destinations become placeholder
-// The mapping process happens early in the request handling lifecycle so that
+// names.
-// the Destination placeholder is calculated and available for substitution.
-// The Items array contains pairs of regex expressions and values, the
-// Source is matched against the expression, if they match then the destination
-// placeholder is set to the value.
-//
-// The Default is optional, if no Item expression is matched then the value of
-// the Default will be used.
 //
+// Mapped placeholders are not evaluated until they are used, so even for very
+// large mappings, this handler is quite efficient.
 type Handler struct {
-	// Source is a placeholder
+	// Source is the placeholder from which to get the input value.
 	Source string `json:"source,omitempty"`
-	// Destination is a new placeholder
+
-	Destination string `json:"destination,omitempty"`
+	// Destinations are the names of placeholders in which to store the outputs.
-	// Default is an optional value to use if no other was found
+	Destinations []string `json:"destinations,omitempty"`
-	Default string `json:"default,omitempty"`
+
-	// Items is an array of regex expressions and values
+	// Mappings from source values (inputs) to destination values (outputs).
-	Items []Item `json:"items,omitempty"`
+	// The first matching, non-nil mapping will be applied.
+	Mappings []Mapping `json:"mappings,omitempty"`
+
+	// If no mappings match or if the mapped output is null/nil, the associated
+	// default output will be applied (optional).
+	Defaults []string
 }
 
 // CaddyModule returns the Caddy module information.
@@ -57,10 +59,58 @@ func (Handler) CaddyModule() caddy.ModuleInfo {
 	}
 }
 
-// Provision will compile all regular expressions
+// Provision sets up h.
 func (h *Handler) Provision(_ caddy.Context) error {
-	for i := 0; i < len(h.Items); i++ {
+	for j, dest := range h.Destinations {
-		h.Items[i].compiled = regexp.MustCompile(h.Items[i].Expression)
+		h.Destinations[j] = strings.Trim(dest, "{}")
+	}
+
+	for i, m := range h.Mappings {
+		if m.InputRegexp == "" {
+			continue
+		}
+		var err error
+		h.Mappings[i].re, err = regexp.Compile(m.InputRegexp)
+		if err != nil {
+			return fmt.Errorf("compiling regexp for mapping %d: %v", i, err)
+		}
+	}
+
+	// TODO: improve efficiency even further by using an actual map type
+	// for the non-regexp mappings, OR sort them and do a binary search
+
+	return nil
+}
+
+// Validate ensures that h is configured properly.
+func (h *Handler) Validate() error {
+	nDest, nDef := len(h.Destinations), len(h.Defaults)
+	if nDef > 0 && nDef != nDest {
+		return fmt.Errorf("%d destinations != %d defaults", nDest, nDef)
+	}
+
+	seen := make(map[string]int)
+	for i, m := range h.Mappings {
+		// prevent confusing/ambiguous mappings
+		if m.Input != "" && m.InputRegexp != "" {
+			return fmt.Errorf("mapping %d has both input and input_regexp fields specified, which is confusing", i)
+		}
+
+		// prevent duplicate mappings
+		input := m.Input
+		if m.InputRegexp != "" {
+			input = m.InputRegexp
+		}
+		if prev, ok := seen[input]; ok {
+			return fmt.Errorf("mapping %d has a duplicate input '%s' previously used with mapping %d", i, input, prev)
+		}
+		seen[input] = i
+
+		// ensure mappings have 1:1 output-to-destination correspondence
+		nOut := len(m.Outputs)
+		if nOut != nDest {
+			return fmt.Errorf("mapping %d has %d outputs but there are %d destinations defined", i, nOut, nDest)
+		}
 	}
 	return nil
 }
@@ -68,38 +118,80 @@ func (h *Handler) Provision(_ caddy.Context) error {
 func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
 
-	// get the source value, if the source value was not found do no
+	// defer work until a variable is actually evaluated by using replacer's Map callback
-	// replacement.
+	repl.Map(func(key string) (interface{}, bool) {
-	val, ok := repl.GetString(h.Source)
+		// return early if the variable is not even a configured destination
-	if ok {
+		destIdx := h.destinationIndex(key)
-		found := false
+		if destIdx < 0 {
-		for i := 0; i < len(h.Items); i++ {
+			return nil, false
-			if h.Items[i].compiled.MatchString(val) {
+		}
-				found = true
+
-				repl.Set(h.Destination, h.Items[i].Value)
+		input := repl.ReplaceAll(h.Source, "")
-				break
+
+		// find the first mapping matching the input and return
+		// the requested destination/output value
+		for _, m := range h.Mappings {
+			if m.re != nil {
+				if m.re.MatchString(input) {
+					if output := m.Outputs[destIdx]; output == nil {
+						continue
+					} else {
+						return output, true
+					}
+				}
+				continue
+			}
+			if input == m.Input {
+				if output := m.Outputs[destIdx]; output == nil {
+					continue
+				} else {
+					return output, true
+				}
 			}
 		}
 
-		if !found && h.Default != "" {
+		// fall back to default if no match or if matched nil value
-			repl.Set(h.Destination, h.Default)
+		if len(h.Defaults) > destIdx {
+			return h.Defaults[destIdx], true
 		}
-	}
+
+		return nil, true
+	})
+
 	return next.ServeHTTP(w, r)
 }
 
-// Item defines each entry in the map
+// destinationIndex returns the positional index of the destination
-type Item struct {
+// is name is a known destination; otherwise it returns -1.
-	// Expression is the regular expression searched for
+func (h Handler) destinationIndex(name string) int {
-	Expression string `json:"expression,omitempty"`
+	for i, dest := range h.Destinations {
-	// Value to use once the expression has been found
+		if dest == name {
-	Value string `json:"value,omitempty"`
+			return i
-	// compiled expression, internal use
+		}
-	compiled *regexp.Regexp
+	}
+	return -1
+}
+
+// Mapping describes a mapping from input to outputs.
+type Mapping struct {
+	// The input value to match. Must be distinct from other mappings.
+	// Mutually exclusive to input_regexp.
+	Input string `json:"input,omitempty"`
+
+	// The input regular expression to match. Mutually exclusive to input.
+	InputRegexp string `json:"input_regexp,omitempty"`
+
+	// Upon a match with the input, each output is positionally correlated
+	// with each destination of the parent handler. An output that is null
+	// (nil) will be treated as if it was not mapped at all.
+	Outputs []interface{} `json:"outputs,omitempty"`
+
+	re *regexp.Regexp
 }
 
 // Interface guards
 var (
 	_ caddy.Provisioner           = (*Handler)(nil)
+	_ caddy.Validator             = (*Handler)(nil)
 	_ caddyhttp.MiddlewareHandler = (*Handler)(nil)
 )

modules/caddyhttp/marshalers.go

@@ -26,11 +26,11 @@ type LoggableHTTPRequest struct{ *http.Request }
 
 // MarshalLogObject satisfies the zapcore.ObjectMarshaler interface.
 func (r LoggableHTTPRequest) MarshalLogObject(enc zapcore.ObjectEncoder) error {
-	enc.AddString("method", r.Method)
-	enc.AddString("uri", r.RequestURI)
-	enc.AddString("proto", r.Proto)
 	enc.AddString("remote_addr", r.RemoteAddr)
+	enc.AddString("proto", r.Proto)
+	enc.AddString("method", r.Method)
 	enc.AddString("host", r.Host)
+	enc.AddString("uri", r.RequestURI)
 	enc.AddObject("headers", LoggableHTTPHeader(r.Header))
 	if r.TLS != nil {
 		enc.AddObject("tls", LoggableTLSConnState(*r.TLS))
@@ -73,10 +73,15 @@ type LoggableTLSConnState tls.ConnectionState
 func (t LoggableTLSConnState) MarshalLogObject(enc zapcore.ObjectEncoder) error {
 	enc.AddBool("resumed", t.DidResume)
 	enc.AddUint16("version", t.Version)
-	enc.AddUint16("ciphersuite", t.CipherSuite)
+	enc.AddUint16("cipher_suite", t.CipherSuite)
 	enc.AddString("proto", t.NegotiatedProtocol)
-	enc.AddBool("proto_mutual", t.NegotiatedProtocolIsMutual)
+	// NegotiatedProtocolIsMutual is deprecated - it's always true
+	enc.AddBool("proto_mutual", true)
 	enc.AddString("server_name", t.ServerName)
+	if len(t.PeerCertificates) > 0 {
+		enc.AddString("client_common_name", t.PeerCertificates[0].Subject.CommonName)
+		enc.AddString("client_serial", t.PeerCertificates[0].SerialNumber.String())
+	}
 	return nil
 }
 

modules/caddyhttp/matchers.go

@@ -23,6 +23,7 @@ import (
 	"net/url"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
 
 	"github.com/caddyserver/caddy/v2"
@@ -51,6 +52,8 @@ type (
 	//
 	// The wildcard can be useful for matching all subdomains, for example:
 	// `*.example.com` matches `foo.example.com` but not `foo.bar.example.com`.
+	//
+	// Duplicate entries will return an error.
 	MatchHost []string
 
 	// MatchPath matches requests by the URI's path (case-insensitive). Path
@@ -103,8 +106,15 @@ type (
 
 	// MatchRemoteIP matches requests by client IP (or CIDR range).
 	MatchRemoteIP struct {
+		// The IPs or CIDR ranges to match.
 		Ranges []string `json:"ranges,omitempty"`
 
+		// If true, prefer the first IP in the request's X-Forwarded-For
+		// header, if present, rather than the immediate peer's IP, as
+		// the reference IP against which to match. Note that it is easy
+		// to spoof request headers. Default: false
+		Forwarded bool `json:"forwarded,omitempty"`
+
 		cidrs  []*net.IPNet
 		logger *zap.Logger
 	}
@@ -167,6 +177,40 @@ func (m *MatchHost) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	return nil
 }
 
+// Provision sets up and validates m, including making it more efficient for large lists.
+func (m MatchHost) Provision(_ caddy.Context) error {
+	// check for duplicates; they are nonsensical and reduce efficiency
+	// (we could just remove them, but the user should know their config is erroneous)
+	seen := make(map[string]int)
+	for i, h := range m {
+		h = strings.ToLower(h)
+		if firstI, ok := seen[h]; ok {
+			return fmt.Errorf("host at index %d is repeated at index %d: %s", firstI, i, h)
+		}
+		seen[h] = i
+	}
+
+	if m.large() {
+		// sort the slice lexicographically, grouping "fuzzy" entries (wildcards and placeholders)
+		// at the front of the list; this allows us to use binary search for exact matches, which
+		// we have seen from experience is the most common kind of value in large lists; and any
+		// other kinds of values (wildcards and placeholders) are grouped in front so the linear
+		// search should find a match fairly quickly
+		sort.Slice(m, func(i, j int) bool {
+			iInexact, jInexact := m.fuzzy(m[i]), m.fuzzy(m[j])
+			if iInexact && !jInexact {
+				return true
+			}
+			if !iInexact && jInexact {
+				return false
+			}
+			return m[i] < m[j]
+		})
+	}
+
+	return nil
+}
+
 // Match returns true if r matches m.
 func (m MatchHost) Match(r *http.Request) bool {
 	reqHost, _, err := net.SplitHostPort(r.Host)
@@ -179,10 +223,31 @@ func (m MatchHost) Match(r *http.Request) bool {
 		reqHost = strings.TrimSuffix(reqHost, "]")
 	}
 
+	if m.large() {
+		// fast path: locate exact match using binary search (about 100-1000x faster for large lists)
+		pos := sort.Search(len(m), func(i int) bool {
+			if m.fuzzy(m[i]) {
+				return false
+			}
+			return m[i] >= reqHost
+		})
+		if pos < len(m) && m[pos] == reqHost {
+			return true
+		}
+	}
+
 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
 
 outer:
 	for _, host := range m {
+		// fast path: if matcher is large, we already know we don't have an exact
+		// match, so we're only looking for fuzzy match now, which should be at the
+		// front of the list; if we have reached a value that is not fuzzy, there
+		// will be no match and we can short-circuit for efficiency
+		if m.large() && !m.fuzzy(host) {
+			break
+		}
+
 		host = repl.ReplaceAll(host, "")
 		if strings.Contains(host, "*") {
 			patternParts := strings.Split(host, ".")
@@ -207,6 +272,15 @@ outer:
 	return false
 }
 
+// fuzzy returns true if the given hostname h is not a specific
+// hostname, e.g. has placeholders or wildcards.
+func (MatchHost) fuzzy(h string) bool { return strings.ContainsAny(h, "{*") }
+
+// large returns true if m is considered to be large. Optimizing
+// the matcher for smaller lists has diminishing returns.
+// See related benchmark function in test file to conduct experiments.
+func (m MatchHost) large() bool { return len(m) > 100 }
+
 // CaddyModule returns the Caddy module information.
 func (MatchPath) CaddyModule() caddy.ModuleInfo {
 	return caddy.ModuleInfo{
@@ -353,18 +427,16 @@ func (m *MatchQuery) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 		*m = make(map[string][]string)
 	}
 	for d.Next() {
-		var query string
+		for _, query := range d.RemainingArgs() {
-		if !d.Args(&query) {
+			if query == "" {
-			return d.ArgErr()
+				continue
+			}
+			parts := strings.SplitN(query, "=", 2)
+			if len(parts) != 2 {
+				return d.Errf("malformed query matcher token: %s; must be in param=val format", d.Val())
+			}
+			url.Values(*m).Add(parts[0], parts[1])
 		}
-		if query == "" {
-			continue
-		}
-		parts := strings.SplitN(query, "=", 2)
-		if len(parts) != 2 {
-			return d.Errf("malformed query matcher token: %s; must be in param=val format", d.Val())
-		}
-		url.Values(*m).Set(parts[0], parts[1])
 		if d.NextBlock(0) {
 			return d.Err("malformed query matcher: blocks are not supported")
 		}
@@ -405,10 +477,33 @@ func (m *MatchHeader) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	}
 	for d.Next() {
 		var field, val string
-		if !d.Args(&field, &val) {
+		if !d.Args(&field) {
-			return d.Errf("malformed header matcher: expected both field and value")
+			return d.Errf("malformed header matcher: expected field")
 		}
-		http.Header(*m).Set(field, val)
+
+		if strings.HasPrefix(field, "!") {
+			if len(field) == 1 {
+				return d.Errf("malformed header matcher: must have field name following ! character")
+			}
+
+			field = field[1:]
+			headers := *m
+			headers[field] = nil
+			m = &headers
+			if d.NextArg() {
+				return d.Errf("malformed header matcher: null matching headers cannot have a field value")
+			}
+		} else {
+			if !d.NextArg() {
+				return d.Errf("malformed header matcher: expected both field and value")
+			}
+
+			// If multiple header matchers with the same header field are defined,
+			// we want to add the existing to the list of headers (will be OR'ed)
+			val = d.Val()
+			http.Header(*m).Add(field, val)
+		}
+
 		if d.NextBlock(0) {
 			return d.Err("malformed header matcher: blocks are not supported")
 		}
@@ -443,6 +538,10 @@ func matchHeaders(input, against http.Header, host string) bool {
 			// match if the header field exists at all
 			continue
 		}
+		if allowedFieldVals == nil && actualFieldVals == nil {
+			// a nil list means match if the header does not exist at all
+			continue
+		}
 		var match bool
 	fieldVals:
 		for _, actualFieldVal := range actualFieldVals {
@@ -700,7 +799,16 @@ func (MatchRemoteIP) CaddyModule() caddy.ModuleInfo {
 // UnmarshalCaddyfile implements caddyfile.Unmarshaler.
 func (m *MatchRemoteIP) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	for d.Next() {
-		m.Ranges = append(m.Ranges, d.RemainingArgs()...)
+		for d.NextArg() {
+			if d.Val() == "forwarded" {
+				if len(m.Ranges) > 0 {
+					return d.Err("if used, 'forwarded' must be first argument")
+				}
+				m.Forwarded = true
+				continue
+			}
+			m.Ranges = append(m.Ranges, d.Val())
+		}
 		if d.NextBlock(0) {
 			return d.Err("malformed remote_ip matcher: blocks are not supported")
 		}
@@ -734,24 +842,20 @@ func (m *MatchRemoteIP) Provision(ctx caddy.Context) error {
 }
 
 func (m MatchRemoteIP) getClientIP(r *http.Request) (net.IP, error) {
-	var remote string
+	remote := r.RemoteAddr
-	if fwdFor := r.Header.Get("X-Forwarded-For"); fwdFor != "" {
+	if m.Forwarded {
-		remote = strings.TrimSpace(strings.Split(fwdFor, ",")[0])
+		if fwdFor := r.Header.Get("X-Forwarded-For"); fwdFor != "" {
+			remote = strings.TrimSpace(strings.Split(fwdFor, ",")[0])
+		}
 	}
-	if remote == "" {
-		remote = r.RemoteAddr
-	}
-
 	ipStr, _, err := net.SplitHostPort(remote)
 	if err != nil {
 		ipStr = remote // OK; probably didn't have a port
 	}
-
 	ip := net.ParseIP(ipStr)
 	if ip == nil {
 		return nil, fmt.Errorf("invalid client IP address: %s", ipStr)
 	}
-
 	return ip, nil
 }
 
@@ -903,6 +1007,7 @@ const regexpPlaceholderPrefix = "http.regexp"
 // Interface guards
 var (
 	_ RequestMatcher    = (*MatchHost)(nil)
+	_ caddy.Provisioner = (*MatchHost)(nil)
 	_ RequestMatcher    = (*MatchPath)(nil)
 	_ RequestMatcher    = (*MatchPathRE)(nil)
 	_ caddy.Provisioner = (*MatchPathRE)(nil)
@@ -927,6 +1032,8 @@ var (
 	_ caddyfile.Unmarshaler = (*MatchHeaderRE)(nil)
 	_ caddyfile.Unmarshaler = (*MatchProtocol)(nil)
 	_ caddyfile.Unmarshaler = (*MatchRemoteIP)(nil)
+	_ caddyfile.Unmarshaler = (*VarsMatcher)(nil)
+	_ caddyfile.Unmarshaler = (*MatchVarsRE)(nil)
 
 	_ json.Marshaler   = (*MatchNot)(nil)
 	_ json.Unmarshaler = (*MatchNot)(nil)

modules/caddyhttp/matchers_test.go

@@ -122,6 +122,11 @@ func TestHostMatcher(t *testing.T) {
 			input:  "sub.foo.example.net",
 			expect: false,
 		},
+		{
+			match:  MatchHost{"www.*.*"},
+			input:  "www.example.com",
+			expect: true,
+		},
 		{
 			match:  MatchHost{"example.com"},
 			input:  "example.com:5555",
@@ -475,6 +480,16 @@ func TestHeaderMatcher(t *testing.T) {
 			host:   "caddyserver.com",
 			expect: false,
 		},
+		{
+			match:  MatchHeader{"Must-Not-Exist": nil},
+			input:  http.Header{},
+			expect: true,
+		},
+		{
+			match:  MatchHeader{"Must-Not-Exist": nil},
+			input:  http.Header{"Must-Not-Exist": []string{"do not match"}},
+			expect: false,
+		},
 	} {
 		req := &http.Request{Header: tc.input, Host: tc.host}
 		actual := tc.match.Match(req)
@@ -1003,6 +1018,33 @@ func TestNotMatcher(t *testing.T) {
 		}
 	}
 }
+func BenchmarkLargeHostMatcher(b *testing.B) {
+	// this benchmark simulates a large host matcher (thousands of entries) where each
+	// value is an exact hostname (not a placeholder or wildcard) - compare the results
+	// of this with and without the binary search (comment out the various fast path
+	// sections in Match) to conduct experiments
+
+	const n = 10000
+	lastHost := fmt.Sprintf("%d.example.com", n-1)
+	req := &http.Request{Host: lastHost}
+	repl := caddy.NewReplacer()
+	ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
+	req = req.WithContext(ctx)
+
+	matcher := make(MatchHost, n)
+	for i := 0; i < n; i++ {
+		matcher[i] = fmt.Sprintf("%d.example.com", i)
+	}
+	err := matcher.Provision(caddy.Context{})
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		matcher.Match(req)
+	}
+}
 
 func BenchmarkHostMatcherWithoutPlaceholder(b *testing.B) {
 	req := &http.Request{Host: "localhost"}

modules/caddyhttp/metrics.go

@@ -0,0 +1,186 @@
+package caddyhttp
+
+import (
+	"context"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var httpMetrics = struct {
+	init             sync.Once
+	requestInFlight  *prometheus.GaugeVec
+	requestCount     *prometheus.CounterVec
+	requestErrors    *prometheus.CounterVec
+	requestDuration  *prometheus.HistogramVec
+	requestSize      *prometheus.HistogramVec
+	responseSize     *prometheus.HistogramVec
+	responseDuration *prometheus.HistogramVec
+}{
+	init: sync.Once{},
+}
+
+func initHTTPMetrics() {
+	const ns, sub = "caddy", "http"
+
+	basicLabels := []string{"server", "handler"}
+	httpMetrics.requestInFlight = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: ns,
+		Subsystem: sub,
+		Name:      "requests_in_flight",
+		Help:      "Number of requests currently handled by this server.",
+	}, basicLabels)
+	httpMetrics.requestErrors = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: ns,
+		Subsystem: sub,
+		Name:      "request_errors_total",
+		Help:      "Number of requests resulting in middleware errors.",
+	}, basicLabels)
+	httpMetrics.requestCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: ns,
+		Subsystem: sub,
+		Name:      "requests_total",
+		Help:      "Counter of HTTP(S) requests made.",
+	}, basicLabels)
+
+	// TODO: allow these to be customized in the config
+	durationBuckets := prometheus.DefBuckets
+	sizeBuckets := prometheus.ExponentialBuckets(256, 4, 8)
+
+	httpLabels := []string{"server", "handler", "code", "method"}
+	httpMetrics.requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: ns,
+		Subsystem: sub,
+		Name:      "request_duration_seconds",
+		Help:      "Histogram of round-trip request durations.",
+		Buckets:   durationBuckets,
+	}, httpLabels)
+	httpMetrics.requestSize = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: ns,
+		Subsystem: sub,
+		Name:      "request_size_bytes",
+		Help:      "Total size of the request. Includes body",
+		Buckets:   sizeBuckets,
+	}, httpLabels)
+	httpMetrics.responseSize = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: ns,
+		Subsystem: sub,
+		Name:      "response_size_bytes",
+		Help:      "Size of the returned response.",
+		Buckets:   sizeBuckets,
+	}, httpLabels)
+	httpMetrics.responseDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: ns,
+		Subsystem: sub,
+		Name:      "response_duration_seconds",
+		Help:      "Histogram of times to first byte in response bodies.",
+		Buckets:   durationBuckets,
+	}, httpLabels)
+}
+
+// serverNameFromContext extracts the current server name from the context.
+// Returns "UNKNOWN" if none is available (should probably never happen).
+func serverNameFromContext(ctx context.Context) string {
+	srv, ok := ctx.Value(ServerCtxKey).(*Server)
+	if !ok || srv == nil || srv.name == "" {
+		return "UNKNOWN"
+	}
+	return srv.name
+}
+
+type metricsInstrumentedHandler struct {
+	handler string
+	mh      MiddlewareHandler
+}
+
+func newMetricsInstrumentedHandler(handler string, mh MiddlewareHandler) *metricsInstrumentedHandler {
+	httpMetrics.init.Do(func() {
+		initHTTPMetrics()
+	})
+
+	return &metricsInstrumentedHandler{handler, mh}
+}
+
+func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, next Handler) error {
+	server := serverNameFromContext(r.Context())
+	labels := prometheus.Labels{"server": server, "handler": h.handler}
+	method := strings.ToUpper(r.Method)
+	// the "code" value is set later, but initialized here to eliminate the possibility
+	// of a panic
+	statusLabels := prometheus.Labels{"server": server, "handler": h.handler, "method": method, "code": ""}
+
+	inFlight := httpMetrics.requestInFlight.With(labels)
+	inFlight.Inc()
+	defer inFlight.Dec()
+
+	start := time.Now()
+
+	// This is a _bit_ of a hack - it depends on the ShouldBufferFunc always
+	// being called when the headers are written.
+	// Effectively the same behaviour as promhttp.InstrumentHandlerTimeToWriteHeader.
+	writeHeaderRecorder := ShouldBufferFunc(func(status int, header http.Header) bool {
+		statusLabels["code"] = sanitizeCode(status)
+		ttfb := time.Since(start).Seconds()
+		httpMetrics.responseDuration.With(statusLabels).Observe(ttfb)
+		return false
+	})
+	wrec := NewResponseRecorder(w, nil, writeHeaderRecorder)
+	err := h.mh.ServeHTTP(wrec, r, next)
+	dur := time.Since(start).Seconds()
+	httpMetrics.requestCount.With(labels).Inc()
+	if err != nil {
+		httpMetrics.requestErrors.With(labels).Inc()
+		return err
+	}
+
+	// If the code hasn't been set yet, and we didn't encounter an error, we're
+	// probably falling through with an empty handler.
+	if statusLabels["code"] == "" {
+		// we still sanitize it, even though it's likely to be 0. A 200 is
+		// returned on fallthrough so we want to reflect that.
+		statusLabels["code"] = sanitizeCode(wrec.Status())
+	}
+
+	httpMetrics.requestDuration.With(statusLabels).Observe(dur)
+	httpMetrics.requestSize.With(statusLabels).Observe(float64(computeApproximateRequestSize(r)))
+	httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size()))
+
+	return nil
+}
+
+func sanitizeCode(code int) string {
+	if code == 0 {
+		return "200"
+	}
+	return strconv.Itoa(code)
+}
+
+// taken from https://github.com/prometheus/client_golang/blob/6007b2b5cae01203111de55f753e76d8dac1f529/prometheus/promhttp/instrument_server.go#L298
+func computeApproximateRequestSize(r *http.Request) int {
+	s := 0
+	if r.URL != nil {
+		s += len(r.URL.String())
+	}
+
+	s += len(r.Method)
+	s += len(r.Proto)
+	for name, values := range r.Header {
+		s += len(name)
+		for _, value := range values {
+			s += len(value)
+		}
+	}
+	s += len(r.Host)
+
+	// N.B. r.Form and r.MultipartForm are assumed to be included in r.URL.
+
+	if r.ContentLength != -1 {
+		s += int(r.ContentLength)
+	}
+	return s
+}