reverseproxy: Minor tweaks

We'll need that context in v2.1 when the transport can manage its own
client certificates; see #3198

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: [mholt] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/ci.yml

@@ -0,0 +1,126 @@
+# Used as inspiration: https://github.com/mvdan/github-actions-golang
+
+name: Cross-Platform
+
+on:
+  push:
+    branches: 
+      - master
+  pull_request:
+    branches: 
+      - master
+
+jobs:
+  test:
+    strategy:
+      # Default is true, cancels jobs for other platforms in the matrix if one fails
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+        go-version: [ 1.14.x ]
+
+        # Set some variables per OS, usable via ${{ matrix.VAR }}
+        # CADDY_BIN_PATH: the path to the compiled Caddy binary, for artifact publishing
+        # SUCCESS: the typical value for $? per OS (Windows/pwsh returns 'True')
+        include:
+        - os: ubuntu-latest
+          CADDY_BIN_PATH: ./cmd/caddy/caddy
+          SUCCESS: 0
+
+        - os: macos-latest
+          CADDY_BIN_PATH: ./cmd/caddy/caddy
+          SUCCESS: 0
+
+        - os: windows-latest
+          CADDY_BIN_PATH: ./cmd/caddy/caddy.exe
+          SUCCESS: 'True'
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    # These tools would be useful if we later decide to reinvestigate
+    # publishing test/coverage reports to some tool for easier consumption
+    # - name: Install test and coverage analysis tools
+    #   run: |
+    #     go get github.com/axw/gocov/gocov
+    #     go get github.com/AlekSi/gocov-xml
+    #     go get -u github.com/jstemmer/go-junit-report
+    #     echo "::add-path::$(go env GOPATH)/bin"
+
+    - name: Print Go version and environment
+      id: vars
+      run: |
+        printf "Using go at: $(which go)\n"
+        printf "Go version: $(go version)\n"
+        printf "\n\nGo environment:\n\n"
+        go env
+        printf "\n\nSystem environment:\n\n"
+        env
+        # Calculate the short SHA1 hash of the git commit
+        echo "::set-output name=short_sha::$(git rev-parse --short HEAD)"
+
+    - name: Get dependencies
+      run: |
+        go get -v -t -d ./...
+        # mkdir test-results
+
+    - name: Build Caddy
+      working-directory: ./cmd/caddy
+      env:
+        CGO_ENABLED: 0
+      run: |
+        go build -trimpath -a -ldflags="-w -s" -v
+
+    - name: Publish Build Artifact
+      uses: actions/upload-artifact@v1
+      with:
+        name: caddy_v2_${{ runner.os }}_${{ steps.vars.outputs.short_sha }}
+        path: ${{ matrix.CADDY_BIN_PATH }}
+
+    # Commented bits below were useful to allow the job to continue
+    # even if the tests fail, so we can publish the report separately
+    # For info about set-output, see https://stackoverflow.com/questions/57850553/github-actions-check-steps-status
+    - name: Run tests
+      # id: step_test
+      # continue-on-error: true
+      run: |
+        # (go test -v -coverprofile=cover-profile.out -race ./... 2>&1) > test-results/test-result.out
+        go test -v -coverprofile="cover-profile.out" -short -race ./...
+        # echo "::set-output name=status::$?"
+
+    # Relevant step if we reinvestigate publishing test/coverage reports
+    # - name: Prepare coverage reports
+    #   run: |
+    #     mkdir coverage
+    #     gocov convert cover-profile.out > coverage/coverage.json
+    #     # Because Windows doesn't work with input redirection like *nix, but output redirection works.
+    #     (cat ./coverage/coverage.json | gocov-xml) > coverage/coverage.xml
+
+    # To return the correct result even though we set 'continue-on-error: true'
+    # - name: Coerce correct build result
+    #   if: matrix.os != 'windows-latest' && steps.step_test.outputs.status != ${{ matrix.SUCCESS }}
+    #   run: |
+    #     echo "step_test ${{ steps.step_test.outputs.status }}\n"
+    #     exit 1
+
+  # From https://github.com/reviewdog/action-golangci-lint
+  golangci-lint:
+    name: runner / golangci-lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Run golangci-lint
+        uses: reviewdog/action-golangci-lint@v1
+        # uses: docker://reviewdog/action-golangci-lint:v1 # pre-build docker image
+        with:
+          github_token: ${{ secrets.github_token }}

.github/workflows/fuzzing.yml

@@ -0,0 +1,75 @@
+name: Fuzzing
+
+on:
+  # Daily midnight fuzzing
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  fuzzing:
+    name: Fuzzing
+
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        go-version: [ 1.14.x ]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Download go-fuzz tools and the Fuzzit CLI, move Fuzzit CLI to GOBIN
+      # If we decide we need to prevent this from running on forks, we can use this line:
+      # if: github.repository == 'caddyserver/caddy'
+      run: |
+
+        go get -v github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
+        wget -q -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.77/fuzzit_Linux_x86_64
+        chmod a+x fuzzit
+        mv fuzzit $(go env GOPATH)/bin
+        echo "::add-path::$(go env GOPATH)/bin"
+
+    - name: Generate fuzzers & submit them to Fuzzit
+      continue-on-error: true
+      env:
+        FUZZIT_API_KEY: ${{ secrets.FUZZIT_API_KEY }}
+        SYSTEM_PULLREQUEST_SOURCEBRANCH: ${{ github.ref }}
+        BUILD_SOURCEVERSION: ${{ github.sha }}
+      run: |
+        # debug
+        echo "PR Source Branch: $SYSTEM_PULLREQUEST_SOURCEBRANCH"
+        echo "Source version: $BUILD_SOURCEVERSION"
+
+        declare -A fuzzers_funcs=(\
+          ["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="FuzzParseAddress" \
+          ["./caddyconfig/caddyfile/parse_fuzz.go"]="FuzzParseCaddyfile" \
+          ["./listeners_fuzz.go"]="FuzzParseNetworkAddress" \
+          ["./replacer_fuzz.go"]="FuzzReplacer" \
+        )
+
+        declare -A fuzzers_targets=(\
+          ["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="parse-address" \
+          ["./caddyconfig/caddyfile/parse_fuzz.go"]="parse-caddyfile" \
+          ["./listeners_fuzz.go"]="parse-network-address" \
+          ["./replacer_fuzz.go"]="replacer" \
+        )
+
+        fuzz_type="fuzzing"
+
+        for f in $(find . -name \*_fuzz.go); do
+          FUZZER_DIRECTORY=$(dirname "$f")
+          
+          echo "go-fuzz-build func ${fuzzers_funcs[$f]} residing in $f"
+          
+          go-fuzz-build -func "${fuzzers_funcs[$f]}" -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.zip" "$FUZZER_DIRECTORY"
+          
+          fuzzit create job --engine go-fuzz caddyserver/"${fuzzers_targets[$f]}" "$FUZZER_DIRECTORY"/"${fuzzers_targets[$f]}.zip" --api-key "${FUZZIT_API_KEY}" --type "${fuzz_type}" --branch "${SYSTEM_PULLREQUEST_SOURCEBRANCH}" --revision "${BUILD_SOURCEVERSION}"
+          
+          echo "Completed $f"
+        done

.github/workflows/release.yml

@@ -0,0 +1,53 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+
+jobs:
+  release:
+    name: Release
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        go-version: [ 1.14.x ]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    # So GoReleaser can generate the changelog properly
+    - name: Unshallowify the repo clone
+      run: git fetch --prune --unshallow
+
+    - name: Print Go version and environment
+      id: vars
+      run: |
+        printf "Using go at: $(which go)\n"
+        printf "Go version: $(go version)\n"
+        printf "\n\nGo environment:\n\n"
+        go env
+        printf "\n\nSystem environment:\n\n"
+        env
+    
+    # https://github.community/t5/GitHub-Actions/How-to-get-just-the-tag-name/m-p/32167/highlight/true#M1027
+    - name: Get the version
+      id: get_version
+      run: echo "::set-output name=version_tag::${GITHUB_REF/refs\/tags\//}"
+
+    # GoReleaser will take care of publishing those artifacts into the release
+    - name: Run GoReleaser
+      uses: goreleaser/goreleaser-action@v1
+      with:
+        version: latest
+        args: release --rm-dist
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        TAG: ${{ steps.get_version.outputs.version_tag }}

.gitignore

@@ -16,3 +16,8 @@ cmd/caddy/caddy.exe
 
 # go modules
 vendor
+
+# goreleaser artifacts
+dist
+caddy-build
+caddy-dist

.golangci.yml

@@ -8,6 +8,8 @@ linters-settings:
 linters:
   enable:
     - bodyclose
+    - prealloc
+    - unconvert
     - errcheck
     - gofmt
     - goimports

.goreleaser.yml

@@ -0,0 +1,60 @@
+before:
+  hooks:
+    - mkdir -p caddy-build
+    - cp cmd/caddy/main.go caddy-build/main.go
+    - cp ./go.mod caddy-build/go.mod
+    - sed -i.bkp s/github.com\/caddyserver\/caddy\/v2/caddy/g ./caddy-build/go.mod
+    # GoReleaser doesn't seem to offer {{.Tag}} at this stage, so we have to embed it into the env
+    # so we run: TAG=$(git describe --abbrev=0) goreleaser release --rm-dist --skip-publish --skip-validate
+    - go mod edit -require=github.com/caddyserver/caddy/v2@{{.Env.TAG}} ./caddy-build/go.mod
+    - git clone --depth 1 https://github.com/caddyserver/dist caddy-dist
+    - go mod download
+builds:
+- env:
+  - CGO_ENABLED=0
+  - GO111MODULE=on
+  main: main.go
+  dir: ./caddy-build
+  binary: caddy
+  goos:
+  - darwin
+  - linux
+  - windows
+  - freebsd
+  goarch:
+  - amd64
+  - arm
+  - arm64
+  goarm:
+  - 6
+  - 7
+  ignore:
+    - goos: darwin
+      goarch: arm
+  flags:
+  - -trimpath
+  ldflags:
+  - -s -w
+archives:
+  - format_overrides:
+      - goos: windows
+        format: zip
+    replacements:
+      darwin: mac
+checksum:
+  algorithm: sha512
+release:
+  github:
+    owner: caddyserver
+    name: caddy
+  draft: true
+  prerelease: auto
+changelog:
+  sort: asc
+  filters:
+    exclude:
+    - '^chore:'
+    - '^ci:'
+    - '^docs?:'
+    - '^tests?:'
+    - '^\w+\s+' # a hack to remove commit messages without colons thus don't correspond to a package

README.md

@@ -1,21 +1,10 @@
-Caddy 2
-=======
-
-This is the development branch for Caddy 2.
-
-**Caddy 2 is production-ready, but there may be breaking changes before the stable 2.0 release.** Please test it and deploy it as much as you are able, and submit your feedback!
-
-**Caddy 2 is the web server of the Go community.** We are looking for maintainers to represent the community! Please become involved with issues, PRs, [our forum](https://caddy.community), sharing on social media, etc.
-
----
-
 <p align="center">
 	<a href="https://caddyserver.com"><img src="https://user-images.githubusercontent.com/1128849/36338535-05fb646a-136f-11e8-987b-e6901e717d5a.png" alt="Caddy" width="450"></a>
 </p>
 <h3 align="center">Every site on HTTPS</h3>
 <p align="center">Caddy is an extensible server platform that uses TLS by default.</p>
 <p align="center">
-	<a href="https://dev.azure.com/mholt-dev/Caddy/_build/latest?definitionId=5&branchName=v2"><img src="https://dev.azure.com/mholt-dev/Caddy/_apis/build/status/Multiplatform%20Tests?branchName=v2"></a>
+	<a href="https://github.com/caddyserver/caddy/actions?query=workflow%3ACross-Platform"><img src="https://github.com/caddyserver/caddy/workflows/Cross-Platform/badge.svg"></a>
 	<a href="https://pkg.go.dev/github.com/caddyserver/caddy/v2"><img src="https://img.shields.io/badge/godoc-reference-blue.svg"></a>
 	<a href="https://app.fuzzit.dev/orgs/caddyserver-gh/dashboard"><img src="https://app.fuzzit.dev/badge?org_id=caddyserver-gh"></a>
 	<br>
@@ -33,8 +22,10 @@ This is the development branch for Caddy 2.
 
 ### Menu
 
+- [Features](#features)
 - [Build from source](#build-from-source)
-	- [Building with plugins](#building-with-plugins)
+	- [For development](#for-development)
+	- [With version information and/or plugins](#with-version-information-andor-plugins)
 - [Getting started](#getting-started)
 - [Overview](#overview)
 - [Full documentation](#full-documentation)
@@ -44,49 +35,63 @@ This is the development branch for Caddy 2.
 <p align="center">
 	<b>Powered by</b>
 	<br>
-	<a href="https://github.com/mholt/certmagic"><img src="https://user-images.githubusercontent.com/1128849/49704830-49d37200-fbd5-11e8-8385-767e0cd033c3.png" alt="CertMagic" width="250"></a>
+	<a href="https://github.com/caddyserver/certmagic"><img src="https://user-images.githubusercontent.com/1128849/49704830-49d37200-fbd5-11e8-8385-767e0cd033c3.png" alt="CertMagic" width="250"></a>
 </p>
 
-## Build from source
 
-_**Note:** These steps [will not embed proper version information](https://github.com/golang/go/issues/29228). For that, please follow the instructions below for building with plugins (you do not have to add any plugins)._
+## Features
+
+- **Easy configuration** with the [Caddyfile](https://caddyserver.com/docs/caddyfile)
+- **Powerful configuration** with its [native JSON config](https://caddyserver.com/docs/json/)
+- **Dynamic configuration** with the [JSON API](https://caddyserver.com/api)
+- [**Config adapters**](https://caddyserver.com/docs/config-adapters) if you don't like JSON
+- **Automatic HTTPS** by default
+	- [Let's Encrypt](https://letsencrypt.org) for public sites
+	- Fully-managed local CA for internal names & IPs
+	- Can coordinate with other Caddy instances in a cluster
+- **Stays up when other servers go down** due to TLS/OCSP/certificate-related issues
+- **HTTP/1.1, HTTP/2, and experimental HTTP/3** support
+- **Highly extensible** [modular architecture](https://caddyserver.com/docs/extending-caddy) lets Caddy do anything without bloat
+- **Runs anywhere** with **no external dependencies** (not even libc)
+- Written in Go, a language with higher **memory safety guarantees** than other servers
+- Actually **fun to use**
+- So, so much more to discover
+
+
+
+## Build from source
 
 Requirements:
 
 - [Go 1.14 or newer](https://golang.org/dl/)
-- Do NOT disable [Go modules](https://github.com/golang/go/wiki/Modules) (`export GO111MODULE=auto`)
-
-Download the `v2` source code:
-
-```bash
-$ git clone -b v2 "https://github.com/caddyserver/caddy.git"
-```
-
-Build:
+- Do NOT disable [Go modules](https://github.com/golang/go/wiki/Modules) (`export GO111MODULE=on`)
 
+### For development
+ 
 ```bash
+$ git clone "https://github.com/caddyserver/caddy.git"
 $ cd caddy/cmd/caddy/
 $ go build
 ```
 
-That will put a `caddy(.exe)` binary into the current directory.
+_**Note:** These steps [will not embed proper version information](https://github.com/golang/go/issues/29228). For that, please follow the instructions below._
 
-If you encounter any Go-module-related errors, try clearing your Go module cache (`$GOPATH/pkg/mod`) or Go package cache (`$GOPATH/pkg`) and read [the Go wiki page about modules for help](https://github.com/golang/go/wiki/Modules). If you have issues with Go modules, please consult the Go community for help. But if there is an actual error in Caddy, please report it to us.
+### With version information and/or plugins
 
-### Building with plugins
+Using [our builder tool](https://github.com/caddyserver/xcaddy)...
 
-Caddy is extensible with plugins. Plugins are added at compile-time, so all Caddy binaries are static (self-contained) and portable.
+```
+$ xcaddy --version CADDY_VERSION
+```
 
-Instructions for doing this are also given in comments in [cmd/caddy/main.go](https://github.com/caddyserver/caddy/blob/v2/cmd/caddy/main.go) which you can copy and use as a template.
+...the following steps are automated:
 
 1. Create a new folder: `mkdir caddy`
 2. Change into it: `cd caddy`
-3. Copy [Caddy's main.go](https://github.com/caddyserver/caddy/blob/v2/cmd/caddy/main.go) into the empty folder. Add imports for any plugins you want to include.
-4. Run: `go mod init caddy`
-5. Run: `go get github.com/caddyserver/caddy/v2@TAG` replacing `TAG` with the latest v2 tag. (Won't be necessary after stable 2.0 release.)
-6. Run: `go build`
-
-Congrats, you now have a custom Caddy build with proper version information!
+3. Copy [Caddy's main.go](https://github.com/caddyserver/caddy/blob/master/cmd/caddy/main.go) into the empty folder. Add imports for any custom plugins you want to add.
+4. Initialize a Go module: `go mod init caddy`
+5. Pin Caddy version: `go get github.com/caddyserver/caddy/v2@TAG` replacing `TAG` with a git tag or commit. You can also pin any plugin versions similarly.
+6. Compile: `go build`
 
 
 
@@ -97,7 +102,7 @@ The [Caddy website](https://caddyserver.com/docs/) has documentation that includ
 
 **We recommend that all users do our [Getting Started](https://caddyserver.com/docs/getting-started) guide to become familiar with using Caddy.**
 
-If you've only got a few minutes, [the website has several quick-start tutorials](https://caddyserver.com/docs/quick-starts) to choose from! However, after finishing a quick-start tutorial, please read more documentation to understand how the software works. 🙂
+If you've only got a minute, [the website has several quick-start tutorials](https://caddyserver.com/docs/quick-starts) to choose from! However, after finishing a quick-start tutorial, please read more documentation to understand how the software works. 🙂
 
 
 

admin.go

@@ -21,7 +21,6 @@ import (
 	"expvar"
 	"fmt"
 	"io"
-	"mime"
 	"net/http"
 	"net/http/pprof"
 	"net/url"
@@ -33,7 +32,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"go.uber.org/zap"
 )
 
@@ -115,7 +113,6 @@ func (admin AdminConfig) newAdminHandler(listenAddr string) adminHandler {
 	}
 
 	// register standard config control endpoints
-	addRoute("/load", AdminHandlerFunc(handleLoad))
 	addRoute("/"+rawConfigKey+"/", AdminHandlerFunc(handleConfig))
 	addRoute("/id/", AdminHandlerFunc(handleConfigID))
 	addRoute("/stop", AdminHandlerFunc(handleStop))
@@ -152,7 +149,7 @@ func (admin AdminConfig) allowedOrigins(listen string) []string {
 	if admin.Origins == nil {
 		uniqueOrigins[listen] = struct{}{}
 	}
-	var allowed []string
+	allowed := make([]string, 0, len(uniqueOrigins))
 	for origin := range uniqueOrigins {
 		allowed = append(allowed, origin)
 	}
@@ -407,86 +404,6 @@ func (h adminHandler) originAllowed(origin string) bool {
 	return false
 }
 
-func handleLoad(w http.ResponseWriter, r *http.Request) error {
-	if r.Method != http.MethodPost {
-		return APIError{
-			Code: http.StatusMethodNotAllowed,
-			Err:  fmt.Errorf("method not allowed"),
-		}
-	}
-
-	buf := bufPool.Get().(*bytes.Buffer)
-	buf.Reset()
-	defer bufPool.Put(buf)
-
-	_, err := io.Copy(buf, r.Body)
-	if err != nil {
-		return APIError{
-			Code: http.StatusBadRequest,
-			Err:  fmt.Errorf("reading request body: %v", err),
-		}
-	}
-	body := buf.Bytes()
-
-	// if the config is formatted other than Caddy's native
-	// JSON, we need to adapt it before loading it
-	if ctHeader := r.Header.Get("Content-Type"); ctHeader != "" {
-		ct, _, err := mime.ParseMediaType(ctHeader)
-		if err != nil {
-			return APIError{
-				Code: http.StatusBadRequest,
-				Err:  fmt.Errorf("invalid Content-Type: %v", err),
-			}
-		}
-		if !strings.HasSuffix(ct, "/json") {
-			slashIdx := strings.Index(ct, "/")
-			if slashIdx < 0 {
-				return APIError{
-					Code: http.StatusBadRequest,
-					Err:  fmt.Errorf("malformed Content-Type"),
-				}
-			}
-			adapterName := ct[slashIdx+1:]
-			cfgAdapter := caddyconfig.GetAdapter(adapterName)
-			if cfgAdapter == nil {
-				return APIError{
-					Code: http.StatusBadRequest,
-					Err:  fmt.Errorf("unrecognized config adapter '%s'", adapterName),
-				}
-			}
-			result, warnings, err := cfgAdapter.Adapt(body, nil)
-			if err != nil {
-				return APIError{
-					Code: http.StatusBadRequest,
-					Err:  fmt.Errorf("adapting config using %s adapter: %v", adapterName, err),
-				}
-			}
-			if len(warnings) > 0 {
-				respBody, err := json.Marshal(warnings)
-				if err != nil {
-					Log().Named("admin.api.load").Error(err.Error())
-				}
-				w.Write(respBody)
-			}
-			body = result
-		}
-	}
-
-	forceReload := r.Header.Get("Cache-Control") == "must-revalidate"
-
-	err = Load(body, forceReload)
-	if err != nil {
-		return APIError{
-			Code: http.StatusBadRequest,
-			Err:  fmt.Errorf("loading config: %v", err),
-		}
-	}
-
-	Log().Named("admin.api").Info("load complete")
-
-	return nil
-}
-
 func handleConfig(w http.ResponseWriter, r *http.Request) error {
 	switch r.Method {
 	case http.MethodGet:
@@ -859,7 +776,7 @@ var (
 // in the config. It also matches adjacent commas so that syntax
 // can be preserved no matter where in the object the field appears.
 // It supports string and most numeric values.
-var idRegexp = regexp.MustCompile(`(?m),?\s*"` + idKey + `":\s?(-?[0-9]+(\.[0-9]+)?|(?U)".*")\s*,?`)
+var idRegexp = regexp.MustCompile(`(?m),?\s*"` + idKey + `"\s*:\s*(-?[0-9]+(\.[0-9]+)?|(?U)".*")\s*,?`)
 
 const (
 	rawConfigKey = "config"

azure-pipelines.yml

@@ -1,242 +0,0 @@
-# Mutilated beyond recognition from the example at:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/go
-
-trigger:
-  - v2
-
-schedules:
-- cron: "0 0 * * *"
-  displayName: Daily midnight fuzzing
-  branches:
-    include:
-    - v2
-  always: true
-
-variables:
-  GOROOT: $(gorootDir)/go
-  GOPATH: $(system.defaultWorkingDirectory)/gopath
-  GOBIN:  $(GOPATH)/bin
-  modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)'
-
-jobs:
-- job: crossPlatformTest
-  displayName: "Cross-Platform Tests"
-  strategy:
-    matrix:
-      linux:
-        imageName: ubuntu-16.04
-        gorootDir: /usr/local
-      mac:
-        imageName: macos-10.14
-        gorootDir: /usr/local
-      windows:
-        imageName: windows-2019
-        gorootDir: C:\
-  pool:
-    vmImage: $(imageName)
-  
-  steps:
-  - bash: |
-      latestGo=$(curl "https://golang.org/VERSION?m=text")
-      echo "##vso[task.setvariable variable=LATEST_GO]$latestGo"
-      echo "Latest Go version: $latestGo"
-    displayName: "Get latest Go version"
-
-  - bash: |
-      sudo rm -f $(which go)
-      echo '##vso[task.prependpath]$(GOBIN)'
-      echo '##vso[task.prependpath]$(GOROOT)/bin'
-      mkdir -p '$(modulePath)'
-      shopt -s extglob
-      shopt -s dotglob
-      mv !(gopath) '$(modulePath)'
-    displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH
-
-  # Install Go (this varies by platform)
-  - bash: |
-      wget "https://dl.google.com/go/$(LATEST_GO).linux-amd64.tar.gz"
-      sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).linux-amd64.tar.gz"
-    condition: eq( variables['Agent.OS'], 'Linux' )
-    displayName: Install Go on Linux
-
-  - bash: |
-      wget "https://dl.google.com/go/$(LATEST_GO).darwin-amd64.tar.gz"
-      sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).darwin-amd64.tar.gz"
-    condition: eq( variables['Agent.OS'], 'Darwin' )
-    displayName: Install Go on macOS
-
-  # The low performance is partly due to PowerShell's attempt to update the progress bar. Disabling it speeds up the process.
-  # Reference: https://github.com/PowerShell/PowerShell/issues/2138
-  - powershell: |
-      $ProgressPreference = 'SilentlyContinue'
-      Write-Host "Downloading Go..."
-      (New-Object System.Net.WebClient).DownloadFile("https://dl.google.com/go/$(LATEST_GO).windows-amd64.zip", "$(LATEST_GO).windows-amd64.zip")
-      Write-Host "Extracting Go... (I'm slow too)"
-      7z x "$(LATEST_GO).windows-amd64.zip" -o"$(gorootDir)"
-    condition: eq( variables['Agent.OS'], 'Windows_NT' )
-    displayName: Install Go on Windows
-
-  - bash: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.23.6
-    displayName: Install golangci-lint
-
-  - script: |
-      go get github.com/axw/gocov/gocov
-      go get github.com/AlekSi/gocov-xml
-      go get -u github.com/jstemmer/go-junit-report
-    displayName: Install test and coverage analysis tools
-
-  - bash: |
-      printf "Using go at: $(which go)\n"
-      printf "Go version: $(go version)\n"
-      printf "\n\nGo environment:\n\n"
-      go env
-      printf "\n\nSystem environment:\n\n"
-      env
-    displayName: Print Go version and environment
-
-  - script: |
-      go get -v -t -d ./...
-      mkdir test-results
-    workingDirectory: '$(modulePath)'
-    displayName: Get dependencies
-
-  - bash: go build -v
-    workingDirectory: '$(modulePath)/cmd/caddy'
-    displayName: Build Caddy
-
-  # its behavior is governed by .golangci.yml
-  - script: |
-      (golangci-lint run --out-format junit-xml) > test-results/lint-result.xml
-      exit 0
-    workingDirectory: '$(modulePath)'
-    continueOnError: true
-    displayName: Run lint check
-
-  - script: |
-      (go test -v -coverprofile=cover-profile.out -race ./... 2>&1) > test-results/test-result.out
-    workingDirectory: '$(modulePath)'
-    continueOnError: true
-    displayName: Run tests
-
-  - script: |
-      mkdir coverage
-      gocov convert cover-profile.out > coverage/coverage.json
-      # Because Windows doesn't work with input redirection like *nix, but output redirection works.
-      (cat ./coverage/coverage.json | gocov-xml) > coverage/coverage.xml 
-    workingDirectory: '$(modulePath)'
-    displayName: Prepare coverage reports
-
-  - script: |
-      (cat ./test-results/test-result.out | go-junit-report) > test-results/test-result.xml
-    workingDirectory: '$(modulePath)'
-    displayName: Prepare test report
-
-  - task: PublishCodeCoverageResults@1
-    displayName: Publish test coverage report
-    inputs:
-      codeCoverageTool: Cobertura 
-      summaryFileLocation: $(modulePath)/coverage/coverage.xml
-
-  - task: PublishTestResults@2
-    displayName: Publish unit test
-    inputs:
-      testResultsFormat: 'JUnit'
-      testResultsFiles: $(modulePath)/test-results/test-result.xml
-      testRunTitle: $(agent.OS) Unit Test
-      mergeTestResults: false
-
-  - task: PublishTestResults@2
-    displayName: Publish lint results
-    inputs:
-      testResultsFormat: 'JUnit'
-      testResultsFiles: $(modulePath)/test-results/lint-result.xml
-      testRunTitle: $(agent.OS) Lint
-      mergeTestResults: false
-
-  - bash: |
-      exit 1
-    condition: eq(variables['Agent.JobStatus'], 'SucceededWithIssues')
-    displayName: Coerce correct build result
-
-- job: fuzzing
-  displayName: 'Fuzzing'
-  # Only run this job on schedules or PRs for non-forks.
-  condition: or(eq(variables['System.PullRequest.IsFork'], 'False'), eq(variables['Build.Reason'], 'Schedule') )
-  strategy:
-    matrix:
-      linux:
-        imageName: ubuntu-16.04
-        gorootDir: /usr/local
-  pool:
-    vmImage: $(imageName)
-
-  steps:
-  - bash: |
-      latestGo=$(curl "https://golang.org/VERSION?m=text")
-      echo "##vso[task.setvariable variable=LATEST_GO]$latestGo"
-      echo "Latest Go version: $latestGo"
-    displayName: "Get latest Go version"
-
-  - bash: |
-      sudo rm -f $(which go)
-      echo '##vso[task.prependpath]$(GOBIN)'
-      echo '##vso[task.prependpath]$(GOROOT)/bin'
-      mkdir -p '$(modulePath)'
-      shopt -s extglob
-      shopt -s dotglob
-      mv !(gopath) '$(modulePath)'
-    displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH
-
-  - bash: |
-      wget "https://dl.google.com/go/$(LATEST_GO).linux-amd64.tar.gz"
-      sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).linux-amd64.tar.gz"
-    condition: eq( variables['Agent.OS'], 'Linux' )
-    displayName: Install Go on Linux
-
-  - bash: |
-      # Install Clang-7.0 because other versions seem to be missing the file libclang_rt.fuzzer-x86_64.a
-      sudo add-apt-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main"
-      wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
-      sudo apt update && sudo apt install -y clang-7 lldb-7 lld-7
-
-      go get -v github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
-      wget -q -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.77/fuzzit_Linux_x86_64
-      chmod a+x fuzzit
-      mv fuzzit $(GOBIN)
-    displayName: Download go-fuzz tools and the Fuzzit CLI, and move Fuzzit CLI to GOBIN
-    condition: and(eq(variables['System.PullRequest.IsFork'], 'False') , eq( variables['Agent.OS'], 'Linux' ))
-
-  - bash: |
-      declare -A fuzzers_funcs=(\
-        ["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="FuzzParseAddress" \
-        ["./caddyconfig/caddyfile/parse_fuzz.go"]="FuzzParseCaddyfile" \
-        ["./listeners_fuzz.go"]="FuzzParseNetworkAddress" \
-        ["./replacer_fuzz.go"]="FuzzReplacer" \
-      )
-
-      declare -A fuzzers_targets=(\
-        ["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="parse-address" \
-        ["./caddyconfig/caddyfile/parse_fuzz.go"]="parse-caddyfile" \
-        ["./listeners_fuzz.go"]="parse-network-address" \
-        ["./replacer_fuzz.go"]="replacer" \
-      )
-
-      fuzz_type="local-regression"
-      if [[ $(Build.Reason) == "Schedule" ]]; then
-        fuzz_type="fuzzing"
-      fi
-      echo "Fuzzing type: $fuzz_type"
-
-      for f in $(find . -name \*_fuzz.go); do
-        FUZZER_DIRECTORY=$(dirname $f)
-        echo "go-fuzz-build func ${fuzzers_funcs[$f]} residing in $f"
-        go-fuzz-build -func "${fuzzers_funcs[$f]}" -libfuzzer -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" $FUZZER_DIRECTORY
-        echo "Generating fuzzer binary of func ${fuzzers_funcs[$f]} which resides in $f"
-        clang-7 -fsanitize=fuzzer "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}"
-        fuzzit create job caddyserver/${fuzzers_targets[$f]} $FUZZER_DIRECTORY/${fuzzers_targets[$f]} --api-key ${FUZZIT_API_KEY} --type "${fuzz_type}" --branch "${SYSTEM_PULLREQUEST_SOURCEBRANCH}" --revision "${BUILD_SOURCEVERSION}"
-        echo "Completed $f"
-      done
-    env:
-      FUZZIT_API_KEY: $(FUZZIT_API_KEY)
-    workingDirectory: '$(modulePath)'
-    displayName: Generate fuzzers & submit them to Fuzzit

caddy.go

@@ -32,7 +32,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/mholt/certmagic"
+	"github.com/caddyserver/certmagic"
 	"go.uber.org/zap"
 )
 
@@ -372,7 +372,7 @@ func run(newCfg *Config, start bool) error {
 		}
 
 		if newCfg.storage == nil {
-			newCfg.storage = &certmagic.FileStorage{Path: AppDataDir()}
+			newCfg.storage = DefaultStorage
 		}
 		certmagic.Default.Storage = newCfg.storage
 
@@ -382,14 +382,12 @@ func run(newCfg *Config, start bool) error {
 		return err
 	}
 
-	// Load, Provision, Validate each app and their submodules
+	// Load and Provision each app and their submodules
 	err = func() error {
-		appsIface, err := ctx.LoadModule(newCfg, "AppsRaw")
-		if err != nil {
-			return fmt.Errorf("loading app modules: %v", err)
-		}
-		for appName, appIface := range appsIface.(map[string]interface{}) {
-			newCfg.apps[appName] = appIface.(App)
+		for appName := range newCfg.AppsRaw {
+			if _, err := ctx.App(appName); err != nil {
+				return err
+			}
 		}
 		return nil
 	}()

caddyconfig/caddyfile/adapter.go

@@ -68,6 +68,11 @@ func (a Adapter) Adapt(body []byte, options map[string]interface{}) ([]byte, []c
 // into JSON. Caddyfile-unmarshaled values
 // will not be used directly; they will be
 // encoded as JSON and then used from that.
+// Implementations must be able to support
+// multiple segments (instances of their
+// directive or batch of tokens); typically
+// this means wrapping all token logic in
+// a loop: `for d.Next() { ... }`.
 type Unmarshaler interface {
 	UnmarshalCaddyfile(d *Dispenser) error
 }

caddyconfig/caddyfile/dispenser_test.go

@@ -27,7 +27,7 @@ func TestDispenser_Val_Next(t *testing.T) {
 			  dir1 arg1
 			  dir2 arg2 arg3
 			  dir3`
-	d := newTestDispenser(input)
+	d := NewTestDispenser(input)
 
 	if val := d.Val(); val != "" {
 		t.Fatalf("Val(): Should return empty string when no token loaded; got '%s'", val)
@@ -65,7 +65,7 @@ func TestDispenser_NextArg(t *testing.T) {
 	input := `dir1 arg1
 			  dir2 arg2 arg3
 			  dir3`
-	d := newTestDispenser(input)
+	d := NewTestDispenser(input)
 
 	assertNext := func(shouldLoad bool, expectedVal string, expectedCursor int) {
 		if d.Next() != shouldLoad {
@@ -112,7 +112,7 @@ func TestDispenser_NextLine(t *testing.T) {
 	input := `host:port
 			  dir1 arg1
 			  dir2 arg2 arg3`
-	d := newTestDispenser(input)
+	d := NewTestDispenser(input)
 
 	assertNextLine := func(shouldLoad bool, expectedVal string, expectedCursor int) {
 		if d.NextLine() != shouldLoad {
@@ -145,7 +145,7 @@ func TestDispenser_NextBlock(t *testing.T) {
 			  }
 			  foobar2 {
 			  }`
-	d := newTestDispenser(input)
+	d := NewTestDispenser(input)
 
 	assertNextBlock := func(shouldLoad bool, expectedCursor, expectedNesting int) {
 		if loaded := d.NextBlock(0); loaded != shouldLoad {
@@ -175,7 +175,7 @@ func TestDispenser_Args(t *testing.T) {
 			  dir2 arg4 arg5
 			  dir3 arg6 arg7
 			  dir4`
-	d := newTestDispenser(input)
+	d := NewTestDispenser(input)
 
 	d.Next() // dir1
 
@@ -242,7 +242,7 @@ func TestDispenser_RemainingArgs(t *testing.T) {
 			  dir2 arg4 arg5
 			  dir3 arg6 { arg7
 			  dir4`
-	d := newTestDispenser(input)
+	d := NewTestDispenser(input)
 
 	d.Next() // dir1
 
@@ -279,7 +279,7 @@ func TestDispenser_ArgErr_Err(t *testing.T) {
 	input := `dir1 {
 			  }
 			  dir2 arg1 arg2`
-	d := newTestDispenser(input)
+	d := NewTestDispenser(input)
 
 	d.cursor = 1 // {
 
@@ -307,7 +307,9 @@ func TestDispenser_ArgErr_Err(t *testing.T) {
 	}
 }
 
-func newTestDispenser(input string) *Dispenser {
+// NewTestDispenser parses input into tokens and creates a new
+// Disenser for test purposes only; any errors are fatal.
+func NewTestDispenser(input string) *Dispenser {
 	tokens, err := allTokens("Testfile", []byte(input))
 	if err != nil && err != io.EOF {
 		log.Fatalf("getting all tokens from input: %v", err)

caddyconfig/caddyfile/formatter.go

@@ -0,0 +1,216 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddyfile
+
+import (
+	"bytes"
+	"io"
+	"unicode"
+)
+
+// Format formats the input Caddyfile to a standard, nice-looking
+// appearance. It works by reading each rune of the input and taking
+// control over all the bracing and whitespace that is written; otherwise,
+// words, comments, placeholders, and escaped characters are all treated
+// literally and written as they appear in the input.
+func Format(input []byte) []byte {
+	input = bytes.TrimSpace(input)
+
+	out := new(bytes.Buffer)
+	rdr := bytes.NewReader(input)
+
+	var (
+		last rune // the last character that was written to the result
+
+		space           = true // whether current/previous character was whitespace (beginning of input counts as space)
+		beginningOfLine = true // whether we are at beginning of line
+
+		openBrace        bool // whether current word/token is or started with open curly brace
+		openBraceWritten bool // if openBrace, whether that brace was written or not
+		openBraceSpace   bool // whether there was a non-newline space before open brace
+
+		newLines int // count of newlines consumed
+
+		comment bool // whether we're in a comment
+		quoted  bool // whether we're in a quoted segment
+		escaped bool // whether current char is escaped
+
+		nesting int // indentation level
+	)
+
+	write := func(ch rune) {
+		out.WriteRune(ch)
+		last = ch
+	}
+
+	indent := func() {
+		for tabs := nesting; tabs > 0; tabs-- {
+			write('\t')
+		}
+	}
+
+	nextLine := func() {
+		write('\n')
+		beginningOfLine = true
+	}
+
+	for {
+		ch, _, err := rdr.ReadRune()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			panic(err)
+		}
+
+		if comment {
+			if ch == '\n' {
+				comment = false
+			} else {
+				write(ch)
+				continue
+			}
+		}
+
+		if !escaped && ch == '\\' {
+			if space {
+				write(' ')
+				space = false
+			}
+			write(ch)
+			escaped = true
+			continue
+		}
+
+		if escaped {
+			write(ch)
+			escaped = false
+			continue
+		}
+
+		if quoted {
+			if ch == '"' {
+				quoted = false
+			}
+			write(ch)
+			continue
+		}
+
+		if space && ch == '"' {
+			quoted = true
+		}
+
+		if unicode.IsSpace(ch) {
+			space = true
+			if ch == '\n' {
+				newLines++
+			}
+			continue
+		}
+		spacePrior := space
+		space = false
+
+		//////////////////////////////////////////////////////////
+		// I find it helpful to think of the formatting loop in two
+		// main sections; by the time we reach this point, we
+		// know we are in a "regular" part of the file: we know
+		// the character is not a space, not in a literal segment
+		// like a comment or quoted, it's not escaped, etc.
+		//////////////////////////////////////////////////////////
+
+		if ch == '#' {
+			if !spacePrior && !beginningOfLine {
+				write(' ')
+			}
+			comment = true
+		}
+
+		if openBrace && spacePrior && !openBraceWritten {
+			if nesting == 0 && last == '}' {
+				nextLine()
+				nextLine()
+			}
+
+			openBrace = false
+			if beginningOfLine {
+				indent()
+			} else if !openBraceSpace {
+				write(' ')
+			}
+			write('{')
+			openBraceWritten = true
+			nextLine()
+			newLines = 0
+			nesting++
+		}
+
+		switch {
+		case ch == '{':
+			openBrace = true
+			openBraceWritten = false
+			openBraceSpace = spacePrior && !beginningOfLine
+			if openBraceSpace {
+				write(' ')
+			}
+			continue
+
+		case ch == '}' && (spacePrior || !openBrace):
+			if last != '\n' {
+				nextLine()
+			}
+			if nesting > 0 {
+				nesting--
+			}
+			indent()
+			write('}')
+			newLines = 0
+			continue
+		}
+
+		if newLines > 2 {
+			newLines = 2
+		}
+		for i := 0; i < newLines; i++ {
+			nextLine()
+		}
+		newLines = 0
+		if beginningOfLine {
+			indent()
+		}
+		if nesting == 0 && last == '}' && beginningOfLine {
+			nextLine()
+			nextLine()
+		}
+
+		if !beginningOfLine && spacePrior {
+			write(' ')
+		}
+
+		if openBrace && !openBraceWritten {
+			write('{')
+			openBraceWritten = true
+		}
+		write(ch)
+
+		beginningOfLine = false
+	}
+
+	// the Caddyfile does not need any leading or trailing spaces, but...
+	trimmedResult := bytes.TrimSpace(out.Bytes())
+
+	// ...Caddyfiles should, however, end with a newline because
+	// newlines are significant to the syntax of the file
+	return append(trimmedResult, '\n')
+}

caddyconfig/caddyfile/formatter_test.go

@@ -0,0 +1,322 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddyfile
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestFormatter(t *testing.T) {
+	for i, tc := range []struct {
+		description string
+		input       string
+		expect      string
+	}{
+		{
+			description: "very simple",
+			input: `abc   def
+	g hi jkl
+mn`,
+			expect: `abc def
+g hi jkl
+mn`,
+		},
+		{
+			description: "basic indentation, line breaks, and nesting",
+			input: `  a
+b
+
+	c {
+		d
+}
+
+e { f
+}
+
+
+
+g {
+h {
+i
+}
+}
+
+j { k {
+l
+}
+}
+
+m {
+	n { o
+	}
+	p { q r
+s }
+}
+
+	{
+{ t
+		u
+
+	v
+
+w
+}
+}`,
+			expect: `a
+b
+
+c {
+	d
+}
+
+e {
+	f
+}
+
+g {
+	h {
+		i
+	}
+}
+
+j {
+	k {
+		l
+	}
+}
+
+m {
+	n {
+		o
+	}
+	p {
+		q r
+		s
+	}
+}
+
+{
+	{
+		t
+		u
+
+		v
+
+		w
+	}
+}`,
+		},
+		{
+			description: "block spacing",
+			input: `a{
+	b
+}
+
+c{ d
+}`,
+			expect: `a {
+	b
+}
+
+c {
+	d
+}`,
+		},
+		{
+			description: "advanced spacing",
+			input: `abc {
+	def
+}ghi{
+	jkl mno
+pqr}`,
+			expect: `abc {
+	def
+}
+
+ghi {
+	jkl mno
+	pqr
+}`,
+		},
+		{
+			description: "env var placeholders",
+			input: `{$A}
+
+b {
+{$C}
+}
+
+d { {$E}
+}
+
+{ {$F}
+}
+`,
+			expect: `{$A}
+
+b {
+	{$C}
+}
+
+d {
+	{$E}
+}
+
+{
+	{$F}
+}`,
+		},
+		{
+			description: "comments",
+			input: `#a "\n"
+
+ #b {
+	c
+}
+
+d {
+e#f
+# g
+}
+
+h { # i
+}`,
+			expect: `#a "\n"
+
+#b {
+c
+}
+
+d {
+	e #f
+	# g
+}
+
+h {
+	# i
+}`,
+		},
+		{
+			description: "quotes and escaping",
+			input: `"a \"b\" "#c
+	d
+
+e {
+"f"
+}
+
+g { "h"
+}
+
+i {
+	"foo
+bar"
+}
+
+j {
+"\"k\" l m"
+}`,
+			expect: `"a \"b\" " #c
+d
+
+e {
+	"f"
+}
+
+g {
+	"h"
+}
+
+i {
+	"foo
+bar"
+}
+
+j {
+	"\"k\" l m"
+}`,
+		},
+		{
+			description: "bad nesting (too many open)",
+			input: `a
+{
+	{
+}`,
+			expect: `a {
+	{
+	}
+`,
+		},
+		{
+			description: "bad nesting (too many close)",
+			input: `a
+{
+	{
+}}}`,
+			expect: `a {
+	{
+	}
+}
+}
+`,
+		},
+		{
+			description: "json",
+			input: `foo
+bar      "{\"key\":34}"
+`,
+			expect: `foo
+bar "{\"key\":34}"`,
+		},
+		{
+			description: "escaping after spaces",
+			input:       `foo \"literal\"`,
+			expect:      `foo \"literal\"`,
+		},
+		{
+			description: "simple placeholders as standalone tokens",
+			input:       `foo {bar}`,
+			expect:      `foo {bar}`,
+		},
+		{
+			description: "simple placeholders within tokens",
+			input:       `foo{bar} foo{bar}baz`,
+			expect:      `foo{bar} foo{bar}baz`,
+		},
+		{
+			description: "placeholders and malformed braces",
+			input:       `foo{bar} foo{ bar}baz`,
+			expect: `foo{bar} foo {
+	bar
+}
+
+baz`,
+		},
+	} {
+		// the formatter should output a trailing newline,
+		// even if the tests aren't written to expect that
+		if !strings.HasSuffix(tc.expect, "\n") {
+			tc.expect += "\n"
+		}
+
+		actual := Format([]byte(tc.input))
+
+		if string(actual) != tc.expect {
+			t.Errorf("\n[TEST %d: %s]\n====== EXPECTED ======\n%s\n====== ACTUAL ======\n%s^^^^^^^^^^^^^^^^^^^^^",
+				i, tc.description, string(tc.expect), string(actual))
+		}
+	}
+}

caddyconfig/caddyfile/parse_fuzz.go

@@ -13,16 +13,16 @@
 // limitations under the License.
 
 // +build gofuzz
-// +build gofuzz_libfuzzer
 
 package caddyfile
 
-import (
-	"bytes"
-)
+import "bytes"
 
 func FuzzParseCaddyfile(data []byte) (score int) {
-	sb, err := Parse("Caddyfile", bytes.NewReader(data))
+	if bytes.Contains(data, []byte("import")) {
+		return -1
+	}
+	sb, err := Parse("Caddyfile", data)
 	if err != nil {
 		// if both an error is received and some ServerBlocks,
 		// then the parse was able to parse partially. Mark this

caddyconfig/caddyfile/parse_test.go

@@ -568,10 +568,6 @@ func TestSnippets(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	for _, b := range blocks {
-		t.Log(b.Keys)
-		t.Log(b.Segments)
-	}
 	if len(blocks) != 1 {
 		t.Fatalf("Expect exactly one server block. Got %d.", len(blocks))
 	}
@@ -616,10 +612,6 @@ func TestImportedFilesIgnoreNonDirectiveImportTokens(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	for _, b := range blocks {
-		t.Log(b.Keys)
-		t.Log(b.Segments)
-	}
 	auth := blocks[0].Segments[0]
 	line := auth[0].Text + " " + auth[1].Text + " " + auth[2].Text + " " + auth[3].Text
 	if line != "basicauth / import password" {
@@ -651,10 +643,6 @@ func TestSnippetAcrossMultipleFiles(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	for _, b := range blocks {
-		t.Log(b.Keys)
-		t.Log(b.Segments)
-	}
 	if len(blocks) != 1 {
 		t.Fatalf("Expect exactly one server block. Got %d.", len(blocks))
 	}
@@ -670,5 +658,5 @@ func TestSnippetAcrossMultipleFiles(t *testing.T) {
 }
 
 func testParser(input string) parser {
-	return parser{Dispenser: newTestDispenser(input)}
+	return parser{Dispenser: NewTestDispenser(input)}
 }

caddyconfig/configadapters.go

@@ -17,6 +17,8 @@ package caddyconfig
 import (
 	"encoding/json"
 	"fmt"
+
+	"github.com/caddyserver/caddy/v2"
 )
 
 // Adapter is a type which can adapt a configuration to Caddy JSON.
@@ -105,7 +107,7 @@ func RegisterAdapter(name string, adapter Adapter) error {
 		return fmt.Errorf("%s: already registered", name)
 	}
 	configAdapters[name] = adapter
-	return nil
+	return caddy.RegisterModule(adapterModule{name, adapter})
 }
 
 // GetAdapter returns the adapter with the given name,
@@ -114,4 +116,21 @@ func GetAdapter(name string) Adapter {
 	return configAdapters[name]
 }
 
+// adapterModule is a wrapper type that can turn any config
+// adapter into a Caddy module, which has the benefit of being
+// counted with other modules, even though they do not
+// technically extend the Caddy configuration structure.
+// See caddyserver/caddy#3132.
+type adapterModule struct {
+	name string
+	Adapter
+}
+
+func (am adapterModule) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  caddy.ModuleID("caddy.adapters." + am.name),
+		New: func() caddy.Module { return am },
+	}
+}
+
 var configAdapters = make(map[string]Adapter)

caddyconfig/httpcaddyfile/addresses.go

@@ -21,9 +21,10 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
-	"github.com/mholt/certmagic"
+	"github.com/caddyserver/certmagic"
 )
 
 // mapAddressToServerBlocks returns a map of listener address to list of server
@@ -105,12 +106,22 @@ func (st *ServerType) mapAddressToServerBlocks(originalServerBlocks []serverBloc
 		// server block are only the ones which use the address; but
 		// the contents (tokens) are of course the same
 		for addr, keys := range addrToKeys {
+			// parse keys so that we only have to do it once
+			parsedKeys := make([]Address, 0, len(keys))
+			for _, key := range keys {
+				addr, err := ParseAddress(key)
+				if err != nil {
+					return nil, fmt.Errorf("parsing key '%s': %v", key, err)
+				}
+				parsedKeys = append(parsedKeys, addr.Normalize())
+			}
 			sbmap[addr] = append(sbmap[addr], serverBlock{
 				block: caddyfile.ServerBlock{
 					Keys:     keys,
 					Segments: sblock.block.Segments,
 				},
 				pile: sblock.pile,
+				keys: parsedKeys,
 			})
 		}
 	}
@@ -127,7 +138,7 @@ func (st *ServerType) mapAddressToServerBlocks(originalServerBlocks []serverBloc
 // association from multiple addresses to multiple server blocks; i.e. each element of
 // the returned slice) becomes a server definition in the output JSON.
 func (st *ServerType) consolidateAddrMappings(addrToServerBlocks map[string][]serverBlock) []sbAddrAssociation {
-	var sbaddrs []sbAddrAssociation
+	sbaddrs := make([]sbAddrAssociation, 0, len(addrToServerBlocks))
 	for addr, sblocks := range addrToServerBlocks {
 		// we start with knowing that at least this address
 		// maps to these server blocks
@@ -164,7 +175,7 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 
 	// figure out the HTTP and HTTPS ports; either
 	// use defaults, or override with user config
-	httpPort, httpsPort := strconv.Itoa(certmagic.HTTPPort), strconv.Itoa(certmagic.HTTPSPort)
+	httpPort, httpsPort := strconv.Itoa(caddyhttp.DefaultHTTPPort), strconv.Itoa(caddyhttp.DefaultHTTPSPort)
 	if hport, ok := options["http_port"]; ok {
 		httpPort = strconv.Itoa(hport.(int))
 	}
@@ -172,20 +183,14 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 		httpsPort = strconv.Itoa(hsport.(int))
 	}
 
-	lnPort := DefaultPort
+	// default port is the HTTPS port
+	lnPort := httpsPort
 	if addr.Port != "" {
 		// port explicitly defined
 		lnPort = addr.Port
-	} else if addr.Scheme != "" {
+	} else if addr.Scheme == "http" {
 		// port inferred from scheme
-		if addr.Scheme == "http" {
-			lnPort = httpPort
-		} else if addr.Scheme == "https" {
-			lnPort = httpsPort
-		}
-	} else if certmagic.HostQualifies(addr.Host) {
-		// automatic HTTPS
-		lnPort = httpsPort
+		lnPort = httpPort
 	}
 
 	// error if scheme and port combination violate convention
@@ -194,7 +199,7 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 	}
 
 	// the bind directive specifies hosts, but is optional
-	var lnHosts []string
+	lnHosts := make([]string, 0, len(sblock.pile))
 	for _, cfgVal := range sblock.pile["bind"] {
 		lnHosts = append(lnHosts, cfgVal.Value.([]string)...)
 	}
@@ -205,15 +210,19 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 	// use a map to prevent duplication
 	listeners := make(map[string]struct{})
 	for _, host := range lnHosts {
-		listeners[net.JoinHostPort(host, lnPort)] = struct{}{}
+		addr, err := caddy.ParseNetworkAddress(host)
+		if err == nil && addr.IsUnixNetwork() {
+			listeners[host] = struct{}{}
+		} else {
+			listeners[net.JoinHostPort(host, lnPort)] = struct{}{}
+		}
 	}
 
 	// now turn map into list
-	var listenersList []string
+	listenersList := make([]string, 0, len(listeners))
 	for lnStr := range listeners {
 		listenersList = append(listenersList, lnStr)
 	}
-	// sort.Strings(listenersList) // TODO: is sorting necessary?
 
 	return listenersList, nil
 }
@@ -281,8 +290,6 @@ func ParseAddress(str string) (Address, error) {
 	return a, nil
 }
 
-// TODO: which of the methods on Address are even used?
-
 // String returns a human-readable form of a. It will
 // be a cleaned-up and filled-out URL string.
 func (a Address) String() string {
@@ -317,12 +324,9 @@ func (a Address) String() string {
 // Normalize returns a normalized version of a.
 func (a Address) Normalize() Address {
 	path := a.Path
-	if !caseSensitivePath {
-		path = strings.ToLower(path)
-	}
 
 	// ensure host is normalized if it's an IP address
-	host := a.Host
+	host := strings.TrimSpace(a.Host)
 	if ip := net.ParseIP(host); ip != nil {
 		host = ip.String()
 	}
@@ -357,10 +361,3 @@ func (a Address) Key() string {
 	}
 	return res
 }
-
-const (
-	// DefaultPort is the default port to use.
-	DefaultPort = "2015"
-
-	caseSensitivePath = false // TODO: Used?
-)

caddyconfig/httpcaddyfile/addresses_fuzz.go

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 // +build gofuzz
-// +build gofuzz_libfuzzer
 
 package httpcaddyfile
 

caddyconfig/httpcaddyfile/addresses_test.go

@@ -1,7 +1,6 @@
 package httpcaddyfile
 
 import (
-	"strings"
 	"testing"
 )
 
@@ -156,15 +155,8 @@ func TestKeyNormalization(t *testing.T) {
 			t.Errorf("Test %d: Parsing address '%s': %v", i, tc.input, err)
 			continue
 		}
-		expect := tc.expect
-		if !caseSensitivePath {
-			// every other part of the address should be lowercased when normalized,
-			// so simply lower-case the whole thing to do case-insensitive comparison
-			// of the path as well
-			expect = strings.ToLower(expect)
-		}
-		if actual := addr.Normalize().Key(); actual != expect {
-			t.Errorf("Test %d: Normalized key for address '%s' was '%s' but expected '%s'", i, tc.input, actual, expect)
+		if actual := addr.Normalize().Key(); actual != tc.expect {
+			t.Errorf("Test %d: Normalized key for address '%s' was '%s' but expected '%s'", i, tc.input, actual, tc.expect)
 		}
 
 	}

caddyconfig/httpcaddyfile/builtins.go

@@ -15,7 +15,6 @@
 package httpcaddyfile
 
 import (
-	"encoding/json"
 	"fmt"
 	"html"
 	"net/http"
@@ -32,12 +31,12 @@ import (
 
 func init() {
 	RegisterDirective("bind", parseBind)
-	RegisterDirective("root", parseRoot) // TODO: isn't this a handler directive?
 	RegisterDirective("tls", parseTLS)
+	RegisterHandlerDirective("root", parseRoot)
 	RegisterHandlerDirective("redir", parseRedir)
 	RegisterHandlerDirective("respond", parseRespond)
 	RegisterHandlerDirective("route", parseRoute)
-	RegisterHandlerDirective("handle", parseSegmentAsSubroute)
+	RegisterHandlerDirective("handle", parseHandle)
 	RegisterDirective("handle_errors", parseHandleErrors)
 	RegisterDirective("log", parseLog)
 }
@@ -54,75 +53,28 @@ func parseBind(h Helper) ([]ConfigValue, error) {
 	return h.NewBindAddresses(lnHosts), nil
 }
 
-// parseRoot parses the root directive. Syntax:
-//
-//     root [<matcher>] <path>
-//
-func parseRoot(h Helper) ([]ConfigValue, error) {
-	if !h.Next() {
-		return nil, h.ArgErr()
-	}
-
-	matcherSet, ok, err := h.MatcherToken()
-	if err != nil {
-		return nil, err
-	}
-	if !ok {
-		// no matcher token; oops
-		h.Dispenser.Prev()
-	}
-
-	if !h.NextArg() {
-		return nil, h.ArgErr()
-	}
-	root := h.Val()
-	if h.NextArg() {
-		return nil, h.ArgErr()
-	}
-
-	varsHandler := caddyhttp.VarsMiddleware{"root": root}
-	route := caddyhttp.Route{
-		HandlersRaw: []json.RawMessage{
-			caddyconfig.JSONModuleObject(varsHandler, "handler", "vars", nil),
-		},
-	}
-	if matcherSet != nil {
-		route.MatcherSetsRaw = []caddy.ModuleMap{matcherSet}
-	}
-
-	return []ConfigValue{{Class: "route", Value: route}}, nil
-}
-
 // parseTLS parses the tls directive. Syntax:
 //
-//     tls [<email>]|[<cert_file> <key_file>] {
+//     tls [<email>|internal]|[<cert_file> <key_file>] {
 //         protocols <min> [<max>]
 //         ciphers   <cipher_suites...>
 //         curves    <curves...>
 //         alpn      <values...>
 //         load      <paths...>
 //         ca        <acme_ca_endpoint>
+//         ca_root   <pem_file>
 //         dns       <provider_name>
+//         on_demand
 //     }
 //
 func parseTLS(h Helper) ([]ConfigValue, error) {
-	var configVals []ConfigValue
-
-	var cp *caddytls.ConnectionPolicy
+	cp := new(caddytls.ConnectionPolicy)
 	var fileLoader caddytls.FileLoader
 	var folderLoader caddytls.FolderLoader
-	var mgr caddytls.ACMEManagerMaker
-
-	// fill in global defaults, if configured
-	if email := h.Option("email"); email != nil {
-		mgr.Email = email.(string)
-	}
-	if acmeCA := h.Option("acme_ca"); acmeCA != nil {
-		mgr.CA = acmeCA.(string)
-	}
-	if caPemFile := h.Option("acme_ca_root"); caPemFile != nil {
-		mgr.TrustedRootsPEMFiles = append(mgr.TrustedRootsPEMFiles, caPemFile.(string))
-	}
+	var certSelector caddytls.CustomCertSelectionPolicy
+	var acmeIssuer *caddytls.ACMEIssuer
+	var internalIssuer *caddytls.InternalIssuer
+	var onDemand bool
 
 	for h.Next() {
 		// file certificate loader
@@ -130,10 +82,17 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 		switch len(firstLine) {
 		case 0:
 		case 1:
-			if !strings.Contains(firstLine[0], "@") {
-				return nil, h.Err("single argument must be an email address")
+			if firstLine[0] == "internal" {
+				internalIssuer = new(caddytls.InternalIssuer)
+			} else if !strings.Contains(firstLine[0], "@") {
+				return nil, h.Err("single argument must either be 'internal' or an email address")
+			} else {
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
+				acmeIssuer.Email = firstLine[0]
 			}
-			mgr.Email = firstLine[0]
+
 		case 2:
 			certFilename := firstLine[0]
 			keyFilename := firstLine[1]
@@ -143,7 +102,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 			// https://github.com/caddyserver/caddy/issues/2588 ... but we
 			// must be careful about how we do this; being careless will
 			// lead to failed handshakes
-
+			//
 			// we need to remember which cert files we've seen, since we
 			// must load each cert only once; otherwise, they each get a
 			// different tag... since a cert loaded twice has the same
@@ -152,6 +111,18 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 			// policy that is looking for any tag but the last one to be
 			// loaded won't find it, and TLS handshakes will fail (see end)
 			// of issue #3004)
+			//
+			// tlsCertTags maps certificate filenames to their tag.
+			// This is used to remember which tag is used for each
+			// certificate files, since we need to avoid loading
+			// the same certificate files more than once, overwriting
+			// previous tags
+			tlsCertTags, ok := h.State["tlsCertTags"].(map[string]string)
+			if !ok {
+				tlsCertTags = make(map[string]string)
+				h.State["tlsCertTags"] = tlsCertTags
+			}
+
 			tag, ok := tlsCertTags[certFilename]
 			if !ok {
 				// haven't seen this cert file yet, let's give it a tag
@@ -165,12 +136,8 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 				// remember this for next time we see this cert file
 				tlsCertTags[certFilename] = tag
 			}
-			certSelector := caddytls.CustomCertSelectionPolicy{Tag: tag}
-			if cp == nil {
-				cp = new(caddytls.ConnectionPolicy)
-			}
+			certSelector.AnyTag = append(certSelector.AnyTag, tag)
 
-			cp.CertSelection = caddyconfig.JSONModuleObject(certSelector, "policy", "custom", h.warnings)
 		default:
 			return nil, h.ArgErr()
 		}
@@ -180,7 +147,6 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 			hasBlock = true
 
 			switch h.Val() {
-			// connection policy
 			case "protocols":
 				args := h.RemainingArgs()
 				if len(args) == 0 {
@@ -190,83 +156,83 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 					if _, ok := caddytls.SupportedProtocols[args[0]]; !ok {
 						return nil, h.Errf("Wrong protocol name or protocol not supported: '%s'", args[0])
 					}
-					if cp == nil {
-						cp = new(caddytls.ConnectionPolicy)
-					}
 					cp.ProtocolMin = args[0]
 				}
 				if len(args) > 1 {
 					if _, ok := caddytls.SupportedProtocols[args[1]]; !ok {
 						return nil, h.Errf("Wrong protocol name or protocol not supported: '%s'", args[1])
 					}
-					if cp == nil {
-						cp = new(caddytls.ConnectionPolicy)
-					}
 					cp.ProtocolMax = args[1]
 				}
+
 			case "ciphers":
 				for h.NextArg() {
-					if _, ok := caddytls.SupportedCipherSuites[h.Val()]; !ok {
+					if !caddytls.CipherSuiteNameSupported(h.Val()) {
 						return nil, h.Errf("Wrong cipher suite name or cipher suite not supported: '%s'", h.Val())
 					}
-					if cp == nil {
-						cp = new(caddytls.ConnectionPolicy)
-					}
 					cp.CipherSuites = append(cp.CipherSuites, h.Val())
 				}
+
 			case "curves":
 				for h.NextArg() {
 					if _, ok := caddytls.SupportedCurves[h.Val()]; !ok {
 						return nil, h.Errf("Wrong curve name or curve not supported: '%s'", h.Val())
 					}
-					if cp == nil {
-						cp = new(caddytls.ConnectionPolicy)
-					}
 					cp.Curves = append(cp.Curves, h.Val())
 				}
+
 			case "alpn":
 				args := h.RemainingArgs()
 				if len(args) == 0 {
 					return nil, h.ArgErr()
 				}
-				if cp == nil {
-					cp = new(caddytls.ConnectionPolicy)
-				}
 				cp.ALPN = args
 
-			// certificate folder loader
 			case "load":
 				folderLoader = append(folderLoader, h.RemainingArgs()...)
 
-			// automation policy
 			case "ca":
 				arg := h.RemainingArgs()
 				if len(arg) != 1 {
 					return nil, h.ArgErr()
 				}
-				mgr.CA = arg[0]
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
+				acmeIssuer.CA = arg[0]
 
-			// DNS provider for ACME DNS challenge
 			case "dns":
 				if !h.Next() {
 					return nil, h.ArgErr()
 				}
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
 				provName := h.Val()
-				if mgr.Challenges == nil {
-					mgr.Challenges = new(caddytls.ChallengesConfig)
+				if acmeIssuer.Challenges == nil {
+					acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
 				}
 				dnsProvModule, err := caddy.GetModule("tls.dns." + provName)
 				if err != nil {
 					return nil, h.Errf("getting DNS provider module named '%s': %v", provName, err)
 				}
-				mgr.Challenges.DNSRaw = caddyconfig.JSONModuleObject(dnsProvModule.New(), "provider", provName, h.warnings)
+				acmeIssuer.Challenges.DNSRaw = caddyconfig.JSONModuleObject(dnsProvModule.New(), "provider", provName, h.warnings)
 
 			case "ca_root":
 				arg := h.RemainingArgs()
 				if len(arg) != 1 {
 					return nil, h.ArgErr()
 				}
-				mgr.TrustedRootsPEMFiles = append(mgr.TrustedRootsPEMFiles, arg[0])
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
+				acmeIssuer.TrustedRootsPEMFiles = append(acmeIssuer.TrustedRootsPEMFiles, arg[0])
+
+			case "on_demand":
+				if h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				onDemand = true
 
 			default:
 				return nil, h.Errf("unknown subdirective: %s", h.Val())
@@ -279,47 +245,95 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 		}
 	}
 
+	// begin building the final config values
+	var configVals []ConfigValue
+
 	// certificate loaders
 	if len(fileLoader) > 0 {
 		configVals = append(configVals, ConfigValue{
 			Class: "tls.certificate_loader",
 			Value: fileLoader,
 		})
-		// ensure server uses HTTPS by setting non-nil conn policy
-		if cp == nil {
-			cp = new(caddytls.ConnectionPolicy)
-		}
 	}
 	if len(folderLoader) > 0 {
 		configVals = append(configVals, ConfigValue{
 			Class: "tls.certificate_loader",
 			Value: folderLoader,
 		})
-		// ensure server uses HTTPS by setting non-nil conn policy
-		if cp == nil {
-			cp = new(caddytls.ConnectionPolicy)
+	}
+
+	// issuer
+	if acmeIssuer != nil && internalIssuer != nil {
+		// the logic to support this would be complex
+		return nil, h.Err("cannot use both ACME and internal issuers in same server block")
+	}
+	if acmeIssuer != nil {
+		// fill in global defaults, if configured
+		if email := h.Option("email"); email != nil && acmeIssuer.Email == "" {
+			acmeIssuer.Email = email.(string)
+		}
+		if acmeCA := h.Option("acme_ca"); acmeCA != nil && acmeIssuer.CA == "" {
+			acmeIssuer.CA = acmeCA.(string)
+		}
+		if caPemFile := h.Option("acme_ca_root"); caPemFile != nil {
+			acmeIssuer.TrustedRootsPEMFiles = append(acmeIssuer.TrustedRootsPEMFiles, caPemFile.(string))
 		}
-	}
 
-	// connection policy
-	if cp != nil {
 		configVals = append(configVals, ConfigValue{
-			Class: "tls.connection_policy",
-			Value: cp,
+			Class: "tls.cert_issuer",
+			Value: acmeIssuer,
+		})
+	} else if internalIssuer != nil {
+		configVals = append(configVals, ConfigValue{
+			Class: "tls.cert_issuer",
+			Value: internalIssuer,
 		})
 	}
 
-	// automation policy
-	if !reflect.DeepEqual(mgr, caddytls.ACMEManagerMaker{}) {
+	// on-demand TLS
+	if onDemand {
 		configVals = append(configVals, ConfigValue{
-			Class: "tls.automation_manager",
-			Value: mgr,
+			Class: "tls.on_demand",
+			Value: true,
 		})
 	}
 
+	// custom certificate selection
+	if len(certSelector.AnyTag) > 0 {
+		cp.CertSelection = &certSelector
+	}
+
+	// connection policy -- always add one, to ensure that TLS
+	// is enabled, because this directive was used (this is
+	// needed, for instance, when a site block has a key of
+	// just ":5000" - i.e. no hostname, and only on-demand TLS
+	// is enabled)
+	configVals = append(configVals, ConfigValue{
+		Class: "tls.connection_policy",
+		Value: cp,
+	})
+
 	return configVals, nil
 }
 
+// parseRoot parses the root directive. Syntax:
+//
+//     root [<matcher>] <path>
+//
+func parseRoot(h Helper) (caddyhttp.MiddlewareHandler, error) {
+	var root string
+	for h.Next() {
+		if !h.NextArg() {
+			return nil, h.ArgErr()
+		}
+		root = h.Val()
+		if h.NextArg() {
+			return nil, h.ArgErr()
+		}
+	}
+	return caddyhttp.VarsMiddleware{"root": root}, nil
+}
+
 // parseRedir parses the redir directive. Syntax:
 //
 //     redir [<matcher>] <to> [<code>]
@@ -400,11 +414,18 @@ func parseRoute(h Helper) (caddyhttp.MiddlewareHandler, error) {
 				return nil, h.Errf("parsing caddyfile tokens for '%s': %v", dir, err)
 			}
 			for _, result := range results {
-				handler, ok := result.Value.(caddyhttp.Route)
-				if !ok {
-					return nil, h.Errf("%s directive returned something other than an HTTP route: %#v (only handler directives can be used in routes)", dir, result.Value)
+				switch handler := result.Value.(type) {
+				case caddyhttp.Route:
+					sr.Routes = append(sr.Routes, handler)
+				case caddyhttp.Subroute:
+					// directives which return a literal subroute instead of a route
+					// means they intend to keep those handlers together without
+					// them being reordered; we're doing that anyway since we're in
+					// the route directive, so just append its handlers
+					sr.Routes = append(sr.Routes, handler.Routes...)
+				default:
+					return nil, h.Errf("%s directive returned something other than an HTTP route or subroute: %#v (only handler directives can be used in routes)", dir, result.Value)
 				}
-				sr.Routes = append(sr.Routes, handler)
 			}
 		}
 	}
@@ -521,10 +542,15 @@ func parseLog(h Helper) ([]ConfigValue, error) {
 
 		var val namedCustomLog
 		if !reflect.DeepEqual(cl, new(caddy.CustomLog)) {
-			cl.Include = []string{"http.log.access"}
+			logCounter, ok := h.State["logCounter"].(int)
+			if !ok {
+				logCounter = 0
+			}
 			val.name = fmt.Sprintf("log%d", logCounter)
+			cl.Include = []string{"http.log.access." + val.name}
 			val.log = cl
 			logCounter++
+			h.State["logCounter"] = logCounter
 		}
 		configValues = append(configValues, ConfigValue{
 			Class: "custom_log",
@@ -533,12 +559,3 @@ func parseLog(h Helper) ([]ConfigValue, error) {
 	}
 	return configValues, nil
 }
-
-// tlsCertTags maps certificate filenames to their tag.
-// This is used to remember which tag is used for each
-// certificate files, since we need to avoid loading
-// the same certificate files more than once, overwriting
-// previous tags
-var tlsCertTags = make(map[string]string)
-
-var logCounter int

caddyconfig/httpcaddyfile/directives.go

@@ -16,7 +16,9 @@ package httpcaddyfile
 
 import (
 	"encoding/json"
+	"net"
 	"sort"
+	"strconv"
 	"strings"
 
 	"github.com/caddyserver/caddy/v2"
@@ -27,24 +29,32 @@ import (
 
 // directiveOrder specifies the order
 // to apply directives in HTTP routes.
+//
+// The root directive goes first in case rewrites or
+// redirects depend on existence of files, i.e. the
+// file matcher, which must know the root first.
+//
+// The header directive goes second so that headers
+// can be manipulated before doing redirects.
 var directiveOrder = []string{
+	"root",
+
+	"header",
+
 	"redir",
 	"rewrite",
 
-	"root",
-
-	"strip_prefix",
-	"strip_suffix",
-	"uri_replace",
+	// URI manipulation
+	"uri",
 	"try_files",
 
-	// middleware handlers that typically wrap responses
+	// middleware handlers; some wrap responses
 	"basicauth",
-	"header",
 	"request_header",
 	"encode",
 	"templates",
 
+	// special routing directives
 	"handle",
 	"route",
 
@@ -114,6 +124,8 @@ func RegisterHandlerDirective(dir string, setupFunc UnmarshalHandlerFunc) {
 // Caddyfile tokens.
 type Helper struct {
 	*caddyfile.Dispenser
+	// State stores intermediate variables during caddyfile adaptation.
+	State        map[string]interface{}
 	options      map[string]interface{}
 	warnings     *[]caddyconfig.Warning
 	matcherDefs  map[string]caddy.ModuleMap
@@ -283,28 +295,31 @@ func sortRoutes(routes []ConfigValue) {
 				return false
 			}
 
-			if len(iRoute.MatcherSetsRaw) == 1 && len(jRoute.MatcherSetsRaw) == 1 {
-				// use already-decoded matcher, or decode if it's the first time seeing it
-				iPM, jPM := decodedMatchers[i], decodedMatchers[j]
-				if iPM == nil {
-					var pathMatcher caddyhttp.MatchPath
-					_ = json.Unmarshal(iRoute.MatcherSetsRaw[0]["path"], &pathMatcher)
-					decodedMatchers[i] = pathMatcher
-					iPM = pathMatcher
-				}
-				if jPM == nil {
-					var pathMatcher caddyhttp.MatchPath
-					_ = json.Unmarshal(jRoute.MatcherSetsRaw[0]["path"], &pathMatcher)
-					decodedMatchers[j] = pathMatcher
-					jPM = pathMatcher
-				}
-
-				// if there is only one path in the matcher, sort by
-				// longer path (more specific) first
-				if len(iPM) == 1 && len(jPM) == 1 {
-					return len(iPM[0]) > len(jPM[0])
-				}
+			// use already-decoded matcher, or decode if it's the first time seeing it
+			iPM, jPM := decodedMatchers[i], decodedMatchers[j]
+			if iPM == nil && len(iRoute.MatcherSetsRaw) == 1 {
+				var pathMatcher caddyhttp.MatchPath
+				_ = json.Unmarshal(iRoute.MatcherSetsRaw[0]["path"], &pathMatcher)
+				decodedMatchers[i] = pathMatcher
+				iPM = pathMatcher
 			}
+			if jPM == nil && len(jRoute.MatcherSetsRaw) == 1 {
+				var pathMatcher caddyhttp.MatchPath
+				_ = json.Unmarshal(jRoute.MatcherSetsRaw[0]["path"], &pathMatcher)
+				decodedMatchers[j] = pathMatcher
+				jPM = pathMatcher
+			}
+
+			// sort by longer path (more specific) first; missing
+			// path matchers are treated as zero-length paths
+			var iPathLen, jPathLen int
+			if iPM != nil {
+				iPathLen = len(iPM[0])
+			}
+			if jPM != nil {
+				jPathLen = len(jPM[0])
+			}
+			return iPathLen > jPathLen
 		}
 
 		return dirPositions[iDir] < dirPositions[jDir]
@@ -368,12 +383,67 @@ func parseSegmentAsSubroute(h Helper) (caddyhttp.MiddlewareHandler, error) {
 	return buildSubroute(allResults, h.groupCounter)
 }
 
-// serverBlock pairs a Caddyfile server block
-// with a "pile" of config values, keyed by class
-// name.
+// serverBlock pairs a Caddyfile server block with
+// a "pile" of config values, keyed by class name,
+// as well as its parsed keys for convenience.
 type serverBlock struct {
 	block caddyfile.ServerBlock
 	pile  map[string][]ConfigValue // config values obtained from directives
+	keys  []Address
+}
+
+// hostsFromKeys returns a list of all the non-empty hostnames found in
+// the keys of the server block sb. If logger mode is false, a key with
+// an empty hostname portion will return an empty slice, since that
+// server block is interpreted to effectively match all hosts. An empty
+// string is never added to the slice.
+//
+// If loggerMode is true, then the non-standard ports of keys will be
+// joined to the hostnames. This is to effectively match the Host
+// header of requests that come in for that key.
+//
+// The resulting slice is not sorted but will never have duplicates.
+func (sb serverBlock) hostsFromKeys(loggerMode bool) []string {
+	// ensure each entry in our list is unique
+	hostMap := make(map[string]struct{})
+	for _, addr := range sb.keys {
+		if addr.Host == "" {
+			if !loggerMode {
+				// server block contains a key like ":443", i.e. the host portion
+				// is empty / catch-all, which means to match all hosts
+				return []string{}
+			}
+			// never append an empty string
+			continue
+		}
+		if loggerMode &&
+			addr.Port != "" &&
+			addr.Port != strconv.Itoa(caddyhttp.DefaultHTTPPort) &&
+			addr.Port != strconv.Itoa(caddyhttp.DefaultHTTPSPort) {
+			hostMap[net.JoinHostPort(addr.Host, addr.Port)] = struct{}{}
+		} else {
+			hostMap[addr.Host] = struct{}{}
+		}
+	}
+
+	// convert map to slice
+	sblockHosts := make([]string, 0, len(hostMap))
+	for host := range hostMap {
+		sblockHosts = append(sblockHosts, host)
+	}
+
+	return sblockHosts
+}
+
+// hasHostCatchAllKey returns true if sb has a key that
+// omits a host portion, i.e. it "catches all" hosts.
+func (sb serverBlock) hasHostCatchAllKey() bool {
+	for _, addr := range sb.keys {
+		if addr.Host == "" {
+			return true
+		}
+	}
+	return false
 }
 
 type (

caddyconfig/httpcaddyfile/directives_test.go

@@ -0,0 +1,94 @@
+package httpcaddyfile
+
+import (
+	"reflect"
+	"sort"
+	"testing"
+)
+
+func TestHostsFromKeys(t *testing.T) {
+	for i, tc := range []struct {
+		keys             []Address
+		expectNormalMode []string
+		expectLoggerMode []string
+	}{
+		{
+			[]Address{
+				Address{Original: "foo", Host: "foo"},
+			},
+			[]string{"foo"},
+			[]string{"foo"},
+		},
+		{
+			[]Address{
+				Address{Original: "foo", Host: "foo"},
+				Address{Original: "bar", Host: "bar"},
+			},
+			[]string{"bar", "foo"},
+			[]string{"bar", "foo"},
+		},
+		{
+			[]Address{
+				Address{Original: ":2015", Port: "2015"},
+			},
+			[]string{}, []string{},
+		},
+		{
+			[]Address{
+				Address{Original: ":443", Port: "443"},
+			},
+			[]string{}, []string{},
+		},
+		{
+			[]Address{
+				Address{Original: "foo", Host: "foo"},
+				Address{Original: ":2015", Port: "2015"},
+			},
+			[]string{}, []string{"foo"},
+		},
+		{
+			[]Address{
+				Address{Original: "example.com:2015", Host: "example.com", Port: "2015"},
+			},
+			[]string{"example.com"},
+			[]string{"example.com:2015"},
+		},
+		{
+			[]Address{
+				Address{Original: "example.com:80", Host: "example.com", Port: "80"},
+			},
+			[]string{"example.com"},
+			[]string{"example.com"},
+		},
+		{
+			[]Address{
+				Address{Original: "https://:2015/foo", Scheme: "https", Port: "2015", Path: "/foo"},
+			},
+			[]string{},
+			[]string{},
+		},
+		{
+			[]Address{
+				Address{Original: "https://example.com:2015/foo", Scheme: "https", Host: "example.com", Port: "2015", Path: "/foo"},
+			},
+			[]string{"example.com"},
+			[]string{"example.com:2015"},
+		},
+	} {
+		sb := serverBlock{keys: tc.keys}
+
+		// test in normal mode
+		actual := sb.hostsFromKeys(false)
+		sort.Strings(actual)
+		if !reflect.DeepEqual(tc.expectNormalMode, actual) {
+			t.Errorf("Test %d (loggerMode=false): Expected: %v Actual: %v", i, tc.expectNormalMode, actual)
+		}
+
+		// test in logger mode
+		actual = sb.hostsFromKeys(true)
+		sort.Strings(actual)
+		if !reflect.DeepEqual(tc.expectLoggerMode, actual) {
+			t.Errorf("Test %d (loggerMode=true): Expected: %v Actual: %v", i, tc.expectLoggerMode, actual)
+		}
+	}
+}

caddyconfig/httpcaddyfile/httptype.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"reflect"
 	"sort"
+	"strconv"
 	"strings"
 
 	"github.com/caddyserver/caddy/v2"
@@ -26,7 +27,6 @@ import (
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
-	"github.com/mholt/certmagic"
 )
 
 func init() {
@@ -42,6 +42,7 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 	options map[string]interface{}) (*caddy.Config, []caddyconfig.Warning, error) {
 	var warnings []caddyconfig.Warning
 	gc := counter{new(int)}
+	state := make(map[string]interface{})
 
 	// load all the server blocks and associate them with a "pile"
 	// of config values; also prohibit duplicate keys because they
@@ -49,7 +50,7 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 	// chosen to handle a request - we actually will make each
 	// server block's route terminal so that only one will run
 	sbKeys := make(map[string]struct{})
-	var serverBlocks []serverBlock
+	serverBlocks := make([]serverBlock, 0, len(originalServerBlocks))
 	for i, sblock := range originalServerBlocks {
 		for j, k := range sblock.Keys {
 			if _, ok := sbKeys[k]; ok {
@@ -83,12 +84,11 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 			"{method}", "{http.request.method}",
 			"{path}", "{http.request.uri.path}",
 			"{query}", "{http.request.uri.query}",
+			"{remote}", "{http.request.remote}",
 			"{remote_host}", "{http.request.remote.host}",
 			"{remote_port}", "{http.request.remote.port}",
-			"{remote}", "{http.request.remote}",
 			"{scheme}", "{http.request.scheme}",
 			"{uri}", "{http.request.uri}",
-
 			"{tls_cipher}", "{http.request.tls.cipher_suite}",
 			"{tls_version}", "{http.request.tls.version}",
 			"{tls_client_fingerprint}", "{http.request.tls.client.fingerprint}",
@@ -133,14 +133,17 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 				return nil, warnings, fmt.Errorf("%s:%d: unrecognized directive: %s", tkn.File, tkn.Line, dir)
 			}
 
-			results, err := dirFunc(Helper{
+			h := Helper{
 				Dispenser:    caddyfile.NewDispenser(segment),
 				options:      options,
 				warnings:     &warnings,
 				matcherDefs:  matcherDefs,
 				parentBlock:  sb.block,
 				groupCounter: gc,
-			})
+				State:        state,
+			}
+
+			results, err := dirFunc(h)
 			if err != nil {
 				return nil, warnings, fmt.Errorf("parsing caddyfile tokens for '%s': %v", dir, err)
 			}
@@ -169,111 +172,15 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 
 	// now that each server is configured, make the HTTP app
 	httpApp := caddyhttp.App{
-		HTTPPort:   tryInt(options["http_port"], &warnings),
-		HTTPSPort:  tryInt(options["https_port"], &warnings),
-		DefaultSNI: tryString(options["default_sni"], &warnings),
-		Servers:    servers,
+		HTTPPort:  tryInt(options["http_port"], &warnings),
+		HTTPSPort: tryInt(options["https_port"], &warnings),
+		Servers:   servers,
 	}
 
-	// now for the TLS app! (TODO: refactor into own func)
-	tlsApp := caddytls.TLS{CertificatesRaw: make(caddy.ModuleMap)}
-	var certLoaders []caddytls.CertificateLoader
-	for _, p := range pairings {
-		for i, sblock := range p.serverBlocks {
-			// tls automation policies
-			if mmVals, ok := sblock.pile["tls.automation_manager"]; ok {
-				for _, mmVal := range mmVals {
-					mm := mmVal.Value.(caddytls.ManagerMaker)
-					sblockHosts, err := st.autoHTTPSHosts(sblock)
-					if err != nil {
-						return nil, warnings, err
-					}
-					if len(sblockHosts) > 0 {
-						if tlsApp.Automation == nil {
-							tlsApp.Automation = new(caddytls.AutomationConfig)
-						}
-						tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, &caddytls.AutomationPolicy{
-							Hosts:         sblockHosts,
-							ManagementRaw: caddyconfig.JSONModuleObject(mm, "module", mm.(caddy.Module).CaddyModule().ID.Name(), &warnings),
-						})
-					} else {
-						warnings = append(warnings, caddyconfig.Warning{
-							Message: fmt.Sprintf("Server block %d %v has no names that qualify for automatic HTTPS, so no TLS automation policy will be added.", i, sblock.block.Keys),
-						})
-					}
-				}
-			}
-			// tls certificate loaders
-			if clVals, ok := sblock.pile["tls.certificate_loader"]; ok {
-				for _, clVal := range clVals {
-					certLoaders = append(certLoaders, clVal.Value.(caddytls.CertificateLoader))
-				}
-			}
-		}
-	}
-	// group certificate loaders by module name, then add to config
-	if len(certLoaders) > 0 {
-		loadersByName := make(map[string]caddytls.CertificateLoader)
-		for _, cl := range certLoaders {
-			name := caddy.GetModuleName(cl)
-			// ugh... technically, we may have multiple FileLoader and FolderLoader
-			// modules (because the tls directive returns one per occurrence), but
-			// the config structure expects only one instance of each kind of loader
-			// module, so we have to combine them... instead of enumerating each
-			// possible cert loader module in a type switch, we can use reflection,
-			// which works on any cert loaders that are slice types
-			if reflect.TypeOf(cl).Kind() == reflect.Slice {
-				combined := reflect.ValueOf(loadersByName[name])
-				if !combined.IsValid() {
-					combined = reflect.New(reflect.TypeOf(cl)).Elem()
-				}
-				clVal := reflect.ValueOf(cl)
-				for i := 0; i < clVal.Len(); i++ {
-					combined = reflect.Append(reflect.Value(combined), clVal.Index(i))
-				}
-				loadersByName[name] = combined.Interface().(caddytls.CertificateLoader)
-			}
-		}
-		for certLoaderName, loaders := range loadersByName {
-			tlsApp.CertificatesRaw[certLoaderName] = caddyconfig.JSON(loaders, &warnings)
-		}
-	}
-	// if global ACME CA, DNS, or email were set, append a catch-all automation
-	// policy that ensures they will be used if no tls directive was used
-	acmeCA, hasACMECA := options["acme_ca"]
-	acmeDNS, hasACMEDNS := options["acme_dns"]
-	email, hasEmail := options["email"]
-	if hasACMECA || hasACMEDNS || hasEmail {
-		if tlsApp.Automation == nil {
-			tlsApp.Automation = new(caddytls.AutomationConfig)
-		}
-		if !hasACMECA {
-			acmeCA = ""
-		}
-		if !hasEmail {
-			email = ""
-		}
-		mgr := caddytls.ACMEManagerMaker{
-			CA:    acmeCA.(string),
-			Email: email.(string),
-		}
-		if hasACMEDNS {
-			provName := acmeDNS.(string)
-			dnsProvModule, err := caddy.GetModule("tls.dns." + provName)
-			if err != nil {
-				return nil, warnings, fmt.Errorf("getting DNS provider module named '%s': %v", provName, err)
-			}
-			mgr.Challenges = &caddytls.ChallengesConfig{
-				DNSRaw: caddyconfig.JSONModuleObject(dnsProvModule.New(), "provider", provName, &warnings),
-			}
-		}
-		tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, &caddytls.AutomationPolicy{
-			ManagementRaw: caddyconfig.JSONModuleObject(mgr, "module", "acme", &warnings),
-		})
-	}
-	if tlsApp.Automation != nil {
-		// consolidate automation policies that are the exact same
-		tlsApp.Automation.Policies = consolidateAutomationPolicies(tlsApp.Automation.Policies)
+	// then make the TLS app
+	tlsApp, warnings, err := st.buildTLSApp(pairings, options, warnings)
+	if err != nil {
+		return nil, warnings, err
 	}
 
 	// if experimental HTTP/3 is enabled, enable it on each server
@@ -314,10 +221,10 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 
 	// annnd the top-level config, then we're done!
 	cfg := &caddy.Config{AppsRaw: make(caddy.ModuleMap)}
-	if !reflect.DeepEqual(httpApp, caddyhttp.App{}) {
+	if len(httpApp.Servers) > 0 {
 		cfg.AppsRaw["http"] = caddyconfig.JSON(httpApp, &warnings)
 	}
-	if !reflect.DeepEqual(tlsApp, caddytls.TLS{CertificatesRaw: make(caddy.ModuleMap)}) {
+	if !reflect.DeepEqual(tlsApp, &caddytls.TLS{CertificatesRaw: make(caddy.ModuleMap)}) {
 		cfg.AppsRaw["tls"] = caddyconfig.JSON(tlsApp, &warnings)
 	}
 	if storageCvtr, ok := options["storage"].(caddy.StorageConverter); ok {
@@ -345,6 +252,18 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 			}
 		}
 	}
+	if len(customLogs) > 0 {
+		if cfg.Logging == nil {
+			cfg.Logging = &caddy.Logging{
+				Logs: make(map[string]*caddy.CustomLog),
+			}
+		}
+		for _, ncl := range customLogs {
+			if ncl.name != "" {
+				cfg.Logging.Logs[ncl.name] = ncl.log
+			}
+		}
+	}
 
 	return cfg, warnings, nil
 }
@@ -363,8 +282,9 @@ func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options
 		var val interface{}
 		var err error
 		disp := caddyfile.NewDispenser(segment)
-		// TODO: make this switch into a map
 		switch dir {
+		case "debug":
+			val = true
 		case "http_port":
 			val, err = parseOptHTTPPort(disp)
 		case "https_port":
@@ -383,8 +303,12 @@ func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options
 			val, err = parseOptSingleString(disp)
 		case "admin":
 			val, err = parseOptAdmin(disp)
-		case "debug":
-			options["debug"] = true
+		case "on_demand_tls":
+			val, err = parseOptOnDemand(disp)
+		case "local_certs":
+			val = true
+		case "key_type":
+			val, err = parseOptSingleString(disp)
 		default:
 			return nil, fmt.Errorf("unrecognized parameter name: %s", dir)
 		}
@@ -397,33 +321,6 @@ func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options
 	return serverBlocks[1:], nil
 }
 
-// hostsFromServerBlockKeys returns a list of all the
-// hostnames found in the keys of the server block sb.
-// The list may not be in a consistent order.
-func (st *ServerType) hostsFromServerBlockKeys(sb caddyfile.ServerBlock) ([]string, error) {
-	// first get each unique hostname
-	hostMap := make(map[string]struct{})
-	for _, sblockKey := range sb.Keys {
-		addr, err := ParseAddress(sblockKey)
-		if err != nil {
-			return nil, fmt.Errorf("parsing server block key: %v", err)
-		}
-		addr = addr.Normalize()
-		if addr.Host == "" {
-			continue
-		}
-		hostMap[addr.Host] = struct{}{}
-	}
-
-	// convert map to slice
-	sblockHosts := make([]string, 0, len(hostMap))
-	for host := range hostMap {
-		sblockHosts = append(sblockHosts, host)
-	}
-
-	return sblockHosts, nil
-}
-
 // serversFromPairings creates the servers for each pairing of addresses
 // to server blocks. Each pairing is essentially a server definition.
 func (st *ServerType) serversFromPairings(
@@ -433,6 +330,12 @@ func (st *ServerType) serversFromPairings(
 	groupCounter counter,
 ) (map[string]*caddyhttp.Server, error) {
 	servers := make(map[string]*caddyhttp.Server)
+	defaultSNI := tryString(options["default_sni"], warnings)
+
+	httpPort := strconv.Itoa(caddyhttp.DefaultHTTPPort)
+	if hp, ok := options["http_port"].(int); ok {
+		httpPort = strconv.Itoa(hp)
+	}
 
 	for i, p := range pairings {
 		srv := &caddyhttp.Server{
@@ -446,11 +349,10 @@ func (st *ServerType) serversFromPairings(
 		// descending sort by length of host then path
 		sort.SliceStable(p.serverBlocks, func(i, j int) bool {
 			// TODO: we could pre-process the specificities for efficiency,
-			// but I don't expect many blocks will have SO many keys...
+			// but I don't expect many blocks will have THAT many keys...
 			var iLongestPath, jLongestPath string
 			var iLongestHost, jLongestHost string
-			for _, key := range p.serverBlocks[i].block.Keys {
-				addr, _ := ParseAddress(key)
+			for _, addr := range p.serverBlocks[i].keys {
 				if specificity(addr.Host) > specificity(iLongestHost) {
 					iLongestHost = addr.Host
 				}
@@ -458,8 +360,7 @@ func (st *ServerType) serversFromPairings(
 					iLongestPath = addr.Path
 				}
 			}
-			for _, key := range p.serverBlocks[j].block.Keys {
-				addr, _ := ParseAddress(key)
+			for _, addr := range p.serverBlocks[j].keys {
 				if specificity(addr.Host) > specificity(jLongestHost) {
 					jLongestHost = addr.Host
 				}
@@ -473,60 +374,49 @@ func (st *ServerType) serversFromPairings(
 			return specificity(iLongestHost) > specificity(jLongestHost)
 		})
 
-		var hasCatchAllTLSConnPolicy bool
+		var hasCatchAllTLSConnPolicy, usesTLS bool
 
 		// create a subroute for each site in the server block
 		for _, sblock := range p.serverBlocks {
-			matcherSetsEnc, err := st.compileEncodedMatcherSets(sblock.block)
+			matcherSetsEnc, err := st.compileEncodedMatcherSets(sblock)
 			if err != nil {
 				return nil, fmt.Errorf("server block %v: compiling matcher sets: %v", sblock.block.Keys, err)
 			}
 
-			// tls: connection policies and toggle auto HTTPS
-			autoHTTPSQualifiedHosts, err := st.autoHTTPSHosts(sblock)
-			if err != nil {
-				return nil, err
-			}
-			if _, ok := sblock.pile["tls.off"]; ok && len(autoHTTPSQualifiedHosts) > 0 {
-				// tls off: disable TLS (and automatic HTTPS) for server block's names
-				if srv.AutoHTTPS == nil {
-					srv.AutoHTTPS = new(caddyhttp.AutoHTTPSConfig)
-				}
-				srv.AutoHTTPS.Skip = append(srv.AutoHTTPS.Skip, autoHTTPSQualifiedHosts...)
-			} else if cpVals, ok := sblock.pile["tls.connection_policy"]; ok {
+			hosts := sblock.hostsFromKeys(false)
+
+			// tls: connection policies
+			if cpVals, ok := sblock.pile["tls.connection_policy"]; ok {
 				// tls connection policies
 				for _, cpVal := range cpVals {
 					cp := cpVal.Value.(*caddytls.ConnectionPolicy)
 
 					// make sure the policy covers all hostnames from the block
-					hosts, err := st.hostsFromServerBlockKeys(sblock.block)
-					if err != nil {
-						return nil, err
+					for _, h := range hosts {
+						if h == defaultSNI {
+							hosts = append(hosts, "")
+							cp.DefaultSNI = defaultSNI
+							break
+						}
 					}
 
-					// TODO: are matchers needed if every hostname of the resulting config is matched?
 					if len(hosts) > 0 {
 						cp.MatchersRaw = caddy.ModuleMap{
 							"sni": caddyconfig.JSON(hosts, warnings), // make sure to match all hosts, not just auto-HTTPS-qualified ones
 						}
 					} else {
+						cp.DefaultSNI = defaultSNI
 						hasCatchAllTLSConnPolicy = true
 					}
 
 					srv.TLSConnPolicies = append(srv.TLSConnPolicies, cp)
 				}
-				// TODO: consolidate equal conn policies
 			}
 
 			// exclude any hosts that were defined explicitly with
 			// "http://" in the key from automated cert management (issue #2998)
-			for _, key := range sblock.block.Keys {
-				addr, err := ParseAddress(key)
-				if err != nil {
-					return nil, err
-				}
-				addr = addr.Normalize()
-				if addr.Scheme == "http" {
+			for _, addr := range sblock.keys {
+				if addr.Scheme == "http" && addr.Host != "" {
 					if srv.AutoHTTPS == nil {
 						srv.AutoHTTPS = new(caddyhttp.AutoHTTPSConfig)
 					}
@@ -534,6 +424,9 @@ func (st *ServerType) serversFromPairings(
 						srv.AutoHTTPS.Skip = append(srv.AutoHTTPS.Skip, addr.Host)
 					}
 				}
+				if addr.Scheme != "http" && addr.Host != "" && addr.Port != httpPort {
+					usesTLS = true
+				}
 			}
 
 			// set up each handler directive, making sure to honor directive order
@@ -565,18 +458,25 @@ func (st *ServerType) serversFromPairings(
 						LoggerNames: make(map[string]string),
 					}
 				}
-				hosts, err := st.hostsFromServerBlockKeys(sblock.block)
-				if err != nil {
-					return nil, err
-				}
-				for _, h := range hosts {
-					if ncl.name != "" {
-						srv.Logs.LoggerNames[h] = ncl.name
+				if sblock.hasHostCatchAllKey() {
+					srv.Logs.LoggerName = ncl.name
+				} else {
+					for _, h := range sblock.hostsFromKeys(true) {
+						if ncl.name != "" {
+							srv.Logs.LoggerNames[h] = ncl.name
+						}
 					}
 				}
 			}
 		}
 
+		// a server cannot (natively) serve both HTTP and HTTPS at the
+		// same time, so make sure the configuration isn't in conflict
+		err := detectConflictingSchemes(srv, p.serverBlocks, options)
+		if err != nil {
+			return nil, err
+		}
+
 		// a catch-all TLS conn policy is necessary to ensure TLS can
 		// be offered to all hostnames of the server; even though only
 		// one policy is needed to enable TLS for the server, that
@@ -586,10 +486,20 @@ func (st *ServerType) serversFromPairings(
 		// catch-all/default policy if there isn't one already (it's
 		// important that it goes at the end) - see issue #3004:
 		// https://github.com/caddyserver/caddy/issues/3004
-		if len(srv.TLSConnPolicies) > 0 && !hasCatchAllTLSConnPolicy {
-			srv.TLSConnPolicies = append(srv.TLSConnPolicies, new(caddytls.ConnectionPolicy))
+		// TODO: maybe a smarter way to handle this might be to just make the
+		// auto-HTTPS logic at provision-time detect if there is any connection
+		// policy missing for any HTTPS-enabled hosts, if so, add it... maybe?
+		if usesTLS &&
+			!hasCatchAllTLSConnPolicy &&
+			(len(srv.TLSConnPolicies) > 0 || defaultSNI != "") {
+			srv.TLSConnPolicies = append(srv.TLSConnPolicies, &caddytls.ConnectionPolicy{DefaultSNI: defaultSNI})
 		}
 
+		// tidy things up a bit
+		srv.TLSConnPolicies, err = consolidateConnPolicies(srv.TLSConnPolicies)
+		if err != nil {
+			return nil, fmt.Errorf("consolidating TLS connection policies for server %d: %v", i, err)
+		}
 		srv.Routes = consolidateRoutes(srv.Routes)
 
 		servers[fmt.Sprintf("srv%d", i)] = srv
@@ -598,6 +508,193 @@ func (st *ServerType) serversFromPairings(
 	return servers, nil
 }
 
+func detectConflictingSchemes(srv *caddyhttp.Server, serverBlocks []serverBlock, options map[string]interface{}) error {
+	httpPort := strconv.Itoa(caddyhttp.DefaultHTTPPort)
+	if hp, ok := options["http_port"].(int); ok {
+		httpPort = strconv.Itoa(hp)
+	}
+	httpsPort := strconv.Itoa(caddyhttp.DefaultHTTPSPort)
+	if hsp, ok := options["https_port"].(int); ok {
+		httpsPort = strconv.Itoa(hsp)
+	}
+
+	var httpOrHTTPS string
+	checkAndSetHTTP := func(addr Address) error {
+		if httpOrHTTPS == "HTTPS" {
+			errMsg := fmt.Errorf("server listening on %v is configured for HTTPS and cannot natively multiplex HTTP and HTTPS: %s",
+				srv.Listen, addr.Original)
+			if addr.Scheme == "" && addr.Host == "" {
+				errMsg = fmt.Errorf("%s (try specifying https:// in the address)", errMsg)
+			}
+			return errMsg
+		}
+		if len(srv.TLSConnPolicies) > 0 {
+			// any connection policies created for an HTTP server
+			// is a logical conflict, as it would enable HTTPS
+			return fmt.Errorf("server listening on %v is HTTP, but attempts to configure TLS connection policies", srv.Listen)
+		}
+		httpOrHTTPS = "HTTP"
+		return nil
+	}
+	checkAndSetHTTPS := func(addr Address) error {
+		if httpOrHTTPS == "HTTP" {
+			return fmt.Errorf("server listening on %v is configured for HTTP and cannot natively multiplex HTTP and HTTPS: %s",
+				srv.Listen, addr.Original)
+		}
+		httpOrHTTPS = "HTTPS"
+		return nil
+	}
+
+	for _, sblock := range serverBlocks {
+		for _, addr := range sblock.keys {
+			if addr.Scheme == "http" || addr.Port == httpPort {
+				if err := checkAndSetHTTP(addr); err != nil {
+					return err
+				}
+			} else if addr.Scheme == "https" || addr.Port == httpsPort {
+				if err := checkAndSetHTTPS(addr); err != nil {
+					return err
+				}
+			} else if addr.Host == "" {
+				if err := checkAndSetHTTP(addr); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// consolidateConnPolicies removes empty TLS connection policies and combines
+// equivalent ones for a cleaner overall output.
+func consolidateConnPolicies(cps caddytls.ConnectionPolicies) (caddytls.ConnectionPolicies, error) {
+	empty := new(caddytls.ConnectionPolicy)
+
+	for i := 0; i < len(cps); i++ {
+		// if the connection policy is empty or has
+		// only matchers, we can remove it entirely
+		empty.MatchersRaw = cps[i].MatchersRaw
+		if reflect.DeepEqual(empty, cps[i]) {
+			cps = append(cps[:i], cps[i+1:]...)
+			i--
+			continue
+		}
+
+		// compare it to the others
+		for j := 0; j < len(cps); j++ {
+			if j == i {
+				continue
+			}
+
+			// if they're exactly equal in every way, just keep one of them
+			if reflect.DeepEqual(cps[i], cps[j]) {
+				cps = append(cps[:j], cps[j+1:]...)
+				i--
+				break
+			}
+
+			// if they have the same matcher, try to reconcile each field: either they must
+			// be identical, or we have to be able to combine them safely
+			if reflect.DeepEqual(cps[i].MatchersRaw, cps[j].MatchersRaw) {
+				if len(cps[i].ALPN) > 0 &&
+					len(cps[j].ALPN) > 0 &&
+					!reflect.DeepEqual(cps[i].ALPN, cps[j].ALPN) {
+					return nil, fmt.Errorf("two policies with same match criteria have conflicting ALPN: %v vs. %v",
+						cps[i].ALPN, cps[j].ALPN)
+				}
+				if len(cps[i].CipherSuites) > 0 &&
+					len(cps[j].CipherSuites) > 0 &&
+					!reflect.DeepEqual(cps[i].CipherSuites, cps[j].CipherSuites) {
+					return nil, fmt.Errorf("two policies with same match criteria have conflicting cipher suites: %v vs. %v",
+						cps[i].CipherSuites, cps[j].CipherSuites)
+				}
+				if cps[i].ClientAuthentication == nil &&
+					cps[j].ClientAuthentication != nil &&
+					!reflect.DeepEqual(cps[i].ClientAuthentication, cps[j].ClientAuthentication) {
+					return nil, fmt.Errorf("two policies with same match criteria have conflicting client auth configuration: %+v vs. %+v",
+						cps[i].ClientAuthentication, cps[j].ClientAuthentication)
+				}
+				if len(cps[i].Curves) > 0 &&
+					len(cps[j].Curves) > 0 &&
+					!reflect.DeepEqual(cps[i].Curves, cps[j].Curves) {
+					return nil, fmt.Errorf("two policies with same match criteria have conflicting curves: %v vs. %v",
+						cps[i].Curves, cps[j].Curves)
+				}
+				if cps[i].DefaultSNI != "" &&
+					cps[j].DefaultSNI != "" &&
+					cps[i].DefaultSNI != cps[j].DefaultSNI {
+					return nil, fmt.Errorf("two policies with same match criteria have conflicting default SNI: %s vs. %s",
+						cps[i].DefaultSNI, cps[j].DefaultSNI)
+				}
+				if cps[i].ProtocolMin != "" &&
+					cps[j].ProtocolMin != "" &&
+					cps[i].ProtocolMin != cps[j].ProtocolMin {
+					return nil, fmt.Errorf("two policies with same match criteria have conflicting min protocol: %s vs. %s",
+						cps[i].ProtocolMin, cps[j].ProtocolMin)
+				}
+				if cps[i].ProtocolMax != "" &&
+					cps[j].ProtocolMax != "" &&
+					cps[i].ProtocolMax != cps[j].ProtocolMax {
+					return nil, fmt.Errorf("two policies with same match criteria have conflicting max protocol: %s vs. %s",
+						cps[i].ProtocolMax, cps[j].ProtocolMax)
+				}
+				if cps[i].CertSelection != nil && cps[j].CertSelection != nil {
+					// merging fields other than AnyTag is not implemented
+					if !reflect.DeepEqual(cps[i].CertSelection.SerialNumber, cps[j].CertSelection.SerialNumber) ||
+						!reflect.DeepEqual(cps[i].CertSelection.SubjectOrganization, cps[j].CertSelection.SubjectOrganization) ||
+						cps[i].CertSelection.PublicKeyAlgorithm != cps[j].CertSelection.PublicKeyAlgorithm ||
+						!reflect.DeepEqual(cps[i].CertSelection.AllTags, cps[j].CertSelection.AllTags) {
+						return nil, fmt.Errorf("two policies with same match criteria have conflicting cert selections: %+v vs. %+v",
+							cps[i].CertSelection, cps[j].CertSelection)
+					}
+				}
+
+				// by now we've decided that we can merge the two -- we'll keep i and drop j
+
+				if len(cps[i].ALPN) == 0 && len(cps[j].ALPN) > 0 {
+					cps[i].ALPN = cps[j].ALPN
+				}
+				if len(cps[i].CipherSuites) == 0 && len(cps[j].CipherSuites) > 0 {
+					cps[i].CipherSuites = cps[j].CipherSuites
+				}
+				if cps[i].ClientAuthentication == nil && cps[j].ClientAuthentication != nil {
+					cps[i].ClientAuthentication = cps[j].ClientAuthentication
+				}
+				if len(cps[i].Curves) == 0 && len(cps[j].Curves) > 0 {
+					cps[i].Curves = cps[j].Curves
+				}
+				if cps[i].DefaultSNI == "" && cps[j].DefaultSNI != "" {
+					cps[i].DefaultSNI = cps[j].DefaultSNI
+				}
+				if cps[i].ProtocolMin == "" && cps[j].ProtocolMin != "" {
+					cps[i].ProtocolMin = cps[j].ProtocolMin
+				}
+				if cps[i].ProtocolMax == "" && cps[j].ProtocolMax != "" {
+					cps[i].ProtocolMax = cps[j].ProtocolMax
+				}
+
+				if cps[i].CertSelection == nil && cps[j].CertSelection != nil {
+					// if j is the only one with a policy, move it over to i
+					cps[i].CertSelection = cps[j].CertSelection
+				} else if cps[i].CertSelection != nil && cps[j].CertSelection != nil {
+					// if both have one, then combine AnyTag
+					for _, tag := range cps[j].CertSelection.AnyTag {
+						if !sliceContains(cps[i].CertSelection.AnyTag, tag) {
+							cps[i].CertSelection.AnyTag = append(cps[i].CertSelection.AnyTag, tag)
+						}
+					}
+				}
+
+				cps = append(cps[:j], cps[j+1:]...)
+				i--
+				break
+			}
+		}
+	}
+	return cps, nil
+}
+
 // appendSubrouteToRouteList appends the routes in subroute
 // to the routeList, optionally qualified by matchers.
 func appendSubrouteToRouteList(routeList caddyhttp.RouteList,
@@ -605,18 +702,34 @@ func appendSubrouteToRouteList(routeList caddyhttp.RouteList,
 	matcherSetsEnc []caddy.ModuleMap,
 	p sbAddrAssociation,
 	warnings *[]caddyconfig.Warning) caddyhttp.RouteList {
+
+	// nothing to do if... there's nothing to do
+	if len(matcherSetsEnc) == 0 && len(subroute.Routes) == 0 && subroute.Errors == nil {
+		return routeList
+	}
+
 	if len(matcherSetsEnc) == 0 && len(p.serverBlocks) == 1 {
 		// no need to wrap the handlers in a subroute if this is
 		// the only server block and there is no matcher for it
 		routeList = append(routeList, subroute.Routes...)
 	} else {
-		routeList = append(routeList, caddyhttp.Route{
-			MatcherSetsRaw: matcherSetsEnc,
-			HandlersRaw: []json.RawMessage{
+		route := caddyhttp.Route{
+			// the semantics of a site block in the Caddyfile dictate
+			// that only the first matching one is evaluated, since
+			// site blocks do not cascade nor inherit
+			Terminal: true,
+		}
+		if len(matcherSetsEnc) > 0 {
+			route.MatcherSetsRaw = matcherSetsEnc
+		}
+		if len(subroute.Routes) > 0 || subroute.Errors != nil {
+			route.HandlersRaw = []json.RawMessage{
 				caddyconfig.JSONModuleObject(subroute, "handler", "subroute", warnings),
-			},
-			Terminal: true, // only first matching site block should be evaluated
-		})
+			}
+		}
+		if len(route.MatcherSetsRaw) > 0 || len(route.HandlersRaw) > 0 {
+			routeList = append(routeList, route)
+		}
 	}
 	return routeList
 }
@@ -670,7 +783,16 @@ func buildSubroute(routes []ConfigValue, groupCounter counter) (*caddyhttp.Subro
 			}
 		}
 		// if there is more than one, put them in a group
-		if info.count > 1 {
+		// (special case: "rewrite" directive must always be in
+		// its own group--even if there is only one--because we
+		// do not want a rewrite to be consolidated into other
+		// adjacent routes that happen to have the same matcher,
+		// see caddyserver/caddy#3108 - because the implied
+		// intent of rewrite is to do an internal redirect,
+		// we can't assume that the request will continue to
+		// match the same matcher; anyway, giving a route a
+		// unique group name should keep it from consolidating)
+		if info.count > 1 || meDir == "rewrite" {
 			info.groupName = groupCounter.nextGroup()
 		}
 	}
@@ -711,22 +833,6 @@ func buildSubroute(routes []ConfigValue, groupCounter counter) (*caddyhttp.Subro
 	return subroute, nil
 }
 
-func (st ServerType) autoHTTPSHosts(sb serverBlock) ([]string, error) {
-	// get the hosts for this server block...
-	hosts, err := st.hostsFromServerBlockKeys(sb.block)
-	if err != nil {
-		return nil, err
-	}
-	// ...and of those, which ones qualify for auto HTTPS
-	var autoHTTPSQualifiedHosts []string
-	for _, h := range hosts {
-		if certmagic.HostQualifies(h) {
-			autoHTTPSQualifiedHosts = append(autoHTTPSQualifiedHosts, h)
-		}
-	}
-	return autoHTTPSQualifiedHosts, nil
-}
-
 // consolidateRoutes combines routes with the same properties
 // (same matchers, same Terminal and Group settings) for a
 // cleaner overall output.
@@ -744,52 +850,6 @@ func consolidateRoutes(routes caddyhttp.RouteList) caddyhttp.RouteList {
 	return routes
 }
 
-// consolidateAutomationPolicies combines automation policies that are the same,
-// for a cleaner overall output.
-func consolidateAutomationPolicies(aps []*caddytls.AutomationPolicy) []*caddytls.AutomationPolicy {
-	for i := 0; i < len(aps); i++ {
-		for j := 0; j < len(aps); j++ {
-			if j == i {
-				continue
-			}
-
-			// if they're exactly equal in every way, just keep one of them
-			if reflect.DeepEqual(aps[i], aps[j]) {
-				aps = append(aps[:j], aps[j+1:]...)
-				i--
-				break
-			}
-
-			// if the policy is the same, we can keep just one, but we have
-			// to be careful which one we keep; if only one has any hostnames
-			// defined, then we need to keep the one without any hostnames,
-			// otherwise the one without any hosts (a catch-all) would be
-			// eaten up by the one with hosts; and if both have hosts, we
-			// need to combine their lists
-			if reflect.DeepEqual(aps[i].ManagementRaw, aps[j].ManagementRaw) &&
-				aps[i].ManageSync == aps[j].ManageSync {
-				if len(aps[i].Hosts) == 0 && len(aps[j].Hosts) > 0 {
-					aps = append(aps[:j], aps[j+1:]...)
-				} else if len(aps[i].Hosts) > 0 && len(aps[j].Hosts) == 0 {
-					aps = append(aps[:i], aps[i+1:]...)
-				} else {
-					aps[i].Hosts = append(aps[i].Hosts, aps[j].Hosts...)
-					aps = append(aps[:j], aps[j+1:]...)
-				}
-				i--
-				break
-			}
-		}
-	}
-
-	// ensure any catch-all policies go last
-	sort.SliceStable(aps, func(i, j int) bool {
-		return len(aps[i].Hosts) > len(aps[j].Hosts)
-	})
-
-	return aps
-}
-
 func matcherSetFromMatcherToken(
 	tkn caddyfile.Token,
 	matcherDefs map[string]caddy.ModuleMap,
@@ -816,7 +876,7 @@ func matcherSetFromMatcherToken(
 	return nil, false, nil
 }
 
-func (st *ServerType) compileEncodedMatcherSets(sblock caddyfile.ServerBlock) ([]caddy.ModuleMap, error) {
+func (st *ServerType) compileEncodedMatcherSets(sblock serverBlock) ([]caddy.ModuleMap, error) {
 	type hostPathPair struct {
 		hostm caddyhttp.MatchHost
 		pathm caddyhttp.MatchPath
@@ -825,13 +885,8 @@ func (st *ServerType) compileEncodedMatcherSets(sblock caddyfile.ServerBlock) ([
 	// keep routes with common host and path matchers together
 	var matcherPairs []*hostPathPair
 
-	for _, key := range sblock.Keys {
-		addr, err := ParseAddress(key)
-		if err != nil {
-			return nil, fmt.Errorf("server block %v: parsing and standardizing address '%s': %v", sblock.Keys, key, err)
-		}
-		addr = addr.Normalize()
-
+	var catchAllHosts bool
+	for _, addr := range sblock.keys {
 		// choose a matcher pair that should be shared by this
 		// server block; if none exists yet, create one
 		var chosenMatcherPair *hostPathPair
@@ -850,6 +905,17 @@ func (st *ServerType) compileEncodedMatcherSets(sblock caddyfile.ServerBlock) ([
 			matcherPairs = append(matcherPairs, chosenMatcherPair)
 		}
 
+		// if one of the keys has no host (i.e. is a catch-all for
+		// any hostname), then we need to null out the host matcher
+		// entirely so that it matches all hosts
+		if addr.Host == "" && !catchAllHosts {
+			chosenMatcherPair.hostm = nil
+			catchAllHosts = true
+		}
+		if catchAllHosts {
+			continue
+		}
+
 		// add this server block's keys to the matcher
 		// pair if it doesn't already exist
 		if addr.Host != "" {
@@ -883,11 +949,11 @@ func (st *ServerType) compileEncodedMatcherSets(sblock caddyfile.ServerBlock) ([
 	}
 
 	// finally, encode each of the matcher sets
-	var matcherSetsEnc []caddy.ModuleMap
+	matcherSetsEnc := make([]caddy.ModuleMap, 0, len(matcherSets))
 	for _, ms := range matcherSets {
 		msEncoded, err := encodeMatcherSet(ms)
 		if err != nil {
-			return nil, fmt.Errorf("server block %v: %v", sblock.Keys, err)
+			return nil, fmt.Errorf("server block %v: %v", sblock.block.Keys, err)
 		}
 		matcherSetsEnc = append(matcherSetsEnc, msEncoded)
 	}
@@ -1012,11 +1078,6 @@ func (c counter) nextGroup() string {
 	return name
 }
 
-type matcherSetAndTokens struct {
-	matcherSet caddy.ModuleMap
-	tokens     []caddyfile.Token
-}
-
 type namedCustomLog struct {
 	name string
 	log  *caddy.CustomLog

caddyconfig/httpcaddyfile/httptype_test.go

@@ -6,7 +6,7 @@ import (
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 )
 
-func TestServerType(t *testing.T) {
+func TestMatcherSyntax(t *testing.T) {
 	for i, tc := range []struct {
 		input       string
 		expectWarn  bool
@@ -15,7 +15,7 @@ func TestServerType(t *testing.T) {
 		{
 			input: `http://localhost
 			@debug {
-			  query showdebug=1
+				query showdebug=1
 			}
 			`,
 			expectWarn:  false,
@@ -24,12 +24,32 @@ func TestServerType(t *testing.T) {
 		{
 			input: `http://localhost
 			@debug {
-			  query bad format
+				query bad format
 			}
 			`,
 			expectWarn:  false,
 			expectError: true,
 		},
+		{
+			input: `http://localhost
+			@debug {
+				not {
+					path /somepath*
+				}
+			}
+			`,
+			expectWarn:  false,
+			expectError: false,
+		},
+		{
+			input: `http://localhost
+			@debug {
+				not path /somepath*
+			}
+			`,
+			expectWarn:  false,
+			expectError: false,
+		},
 	} {
 
 		adapter := caddyfile.Adapter{
@@ -79,3 +99,71 @@ func TestSpecificity(t *testing.T) {
 		}
 	}
 }
+
+func TestGlobalOptions(t *testing.T) {
+	for i, tc := range []struct {
+		input       string
+		expectWarn  bool
+		expectError bool
+	}{
+		{
+			input: `
+				{
+					email test@example.com
+				}
+				:80
+			`,
+			expectWarn:  false,
+			expectError: false,
+		},
+		{
+			input: `
+				{
+					admin off
+				}
+				:80
+			`,
+			expectWarn:  false,
+			expectError: false,
+		},
+		{
+			input: `
+				{
+					admin 127.0.0.1:2020
+				}
+				:80
+			`,
+			expectWarn:  false,
+			expectError: false,
+		},
+		{
+			input: `
+				{
+					admin {
+						disabled false
+					}
+				}
+				:80
+			`,
+			expectWarn:  false,
+			expectError: true,
+		},
+	} {
+
+		adapter := caddyfile.Adapter{
+			ServerType: ServerType{},
+		}
+
+		_, warnings, err := adapter.Adapt([]byte(tc.input), nil)
+
+		if len(warnings) > 0 != tc.expectWarn {
+			t.Errorf("Test %d warning expectation failed Expected: %v, got %v", i, tc.expectWarn, warnings)
+			continue
+		}
+
+		if err != nil != tc.expectError {
+			t.Errorf("Test %d error expectation failed Expected: %v, got %s", i, tc.expectError, err)
+			continue
+		}
+	}
+}

caddyconfig/httpcaddyfile/options.go

@@ -15,11 +15,12 @@
 package httpcaddyfile
 
 import (
-	"fmt"
 	"strconv"
+	"time"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+	"github.com/caddyserver/caddy/v2/modules/caddytls"
 )
 
 func parseOptHTTPPort(d *caddyfile.Dispenser) (int, error) {
@@ -68,7 +69,7 @@ func parseOptOrder(d *caddyfile.Dispenser) ([]string, error) {
 		}
 		dirName := d.Val()
 		if _, ok := registeredDirectives[dirName]; !ok {
-			return nil, fmt.Errorf("%s is not a registered directive", dirName)
+			return nil, d.Errf("%s is not a registered directive", dirName)
 		}
 
 		// get positional token
@@ -104,7 +105,7 @@ func parseOptOrder(d *caddyfile.Dispenser) ([]string, error) {
 		case "before":
 		case "after":
 		default:
-			return nil, fmt.Errorf("unknown positional '%s'", pos)
+			return nil, d.Errf("unknown positional '%s'", pos)
 		}
 
 		// get name of other directive
@@ -145,11 +146,11 @@ func parseOptStorage(d *caddyfile.Dispenser) (caddy.StorageConverter, error) {
 	modName := args[0]
 	mod, err := caddy.GetModule("caddy.storage." + modName)
 	if err != nil {
-		return nil, fmt.Errorf("getting storage module '%s': %v", modName, err)
+		return nil, d.Errf("getting storage module '%s': %v", modName, err)
 	}
 	unm, ok := mod.New().(caddyfile.Unmarshaler)
 	if !ok {
-		return nil, fmt.Errorf("storage module '%s' is not a Caddyfile unmarshaler", mod.ID)
+		return nil, d.Errf("storage module '%s' is not a Caddyfile unmarshaler", mod.ID)
 	}
 	err = unm.UnmarshalCaddyfile(d.NewFromNextSegment())
 	if err != nil {
@@ -157,7 +158,7 @@ func parseOptStorage(d *caddyfile.Dispenser) (caddy.StorageConverter, error) {
 	}
 	storage, ok := unm.(caddy.StorageConverter)
 	if !ok {
-		return nil, fmt.Errorf("module %s is not a StorageConverter", mod.ID)
+		return nil, d.Errf("module %s is not a StorageConverter", mod.ID)
 	}
 	return storage, nil
 }
@@ -177,7 +178,9 @@ func parseOptSingleString(d *caddyfile.Dispenser) (string, error) {
 func parseOptAdmin(d *caddyfile.Dispenser) (string, error) {
 	if d.Next() {
 		var listenAddress string
-		d.AllArgs(&listenAddress)
+		if !d.AllArgs(&listenAddress) {
+			return "", d.ArgErr()
+		}
 		if listenAddress == "" {
 			listenAddress = caddy.DefaultAdminListen
 		}
@@ -185,3 +188,63 @@ func parseOptAdmin(d *caddyfile.Dispenser) (string, error) {
 	}
 	return "", nil
 }
+
+func parseOptOnDemand(d *caddyfile.Dispenser) (*caddytls.OnDemandConfig, error) {
+	var ond *caddytls.OnDemandConfig
+	for d.Next() {
+		if d.NextArg() {
+			return nil, d.ArgErr()
+		}
+		for nesting := d.Nesting(); d.NextBlock(nesting); {
+			switch d.Val() {
+			case "ask":
+				if !d.NextArg() {
+					return nil, d.ArgErr()
+				}
+				if ond == nil {
+					ond = new(caddytls.OnDemandConfig)
+				}
+				ond.Ask = d.Val()
+
+			case "interval":
+				if !d.NextArg() {
+					return nil, d.ArgErr()
+				}
+				dur, err := time.ParseDuration(d.Val())
+				if err != nil {
+					return nil, err
+				}
+				if ond == nil {
+					ond = new(caddytls.OnDemandConfig)
+				}
+				if ond.RateLimit == nil {
+					ond.RateLimit = new(caddytls.RateLimit)
+				}
+				ond.RateLimit.Interval = caddy.Duration(dur)
+
+			case "burst":
+				if !d.NextArg() {
+					return nil, d.ArgErr()
+				}
+				burst, err := strconv.Atoi(d.Val())
+				if err != nil {
+					return nil, err
+				}
+				if ond == nil {
+					ond = new(caddytls.OnDemandConfig)
+				}
+				if ond.RateLimit == nil {
+					ond.RateLimit = new(caddytls.RateLimit)
+				}
+				ond.RateLimit.Burst = burst
+
+			default:
+				return nil, d.Errf("unrecognized parameter '%s'", d.Val())
+			}
+		}
+	}
+	if ond == nil {
+		return nil, d.Err("expected at least one config parameter for on_demand_tls")
+	}
+	return ond, nil
+}

caddyconfig/httpcaddyfile/tlsapp.go

@@ -0,0 +1,439 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpcaddyfile
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"sort"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig"
+	"github.com/caddyserver/caddy/v2/modules/caddytls"
+	"github.com/caddyserver/certmagic"
+)
+
+func (st ServerType) buildTLSApp(
+	pairings []sbAddrAssociation,
+	options map[string]interface{},
+	warnings []caddyconfig.Warning,
+) (*caddytls.TLS, []caddyconfig.Warning, error) {
+
+	tlsApp := &caddytls.TLS{CertificatesRaw: make(caddy.ModuleMap)}
+	var certLoaders []caddytls.CertificateLoader
+
+	// count how many server blocks have a key with no host,
+	// and find all hosts that share a server block with a
+	// hostless key, so that they don't get forgotten/omitted
+	// by auto-HTTPS (since they won't appear in route matchers)
+	var serverBlocksWithHostlessKey int
+	hostsSharedWithHostlessKey := make(map[string]struct{})
+	for _, pair := range pairings {
+		for _, sb := range pair.serverBlocks {
+			for _, addr := range sb.keys {
+				if addr.Host == "" {
+					serverBlocksWithHostlessKey++
+					// this server block has a hostless key, now
+					// go through and add all the hosts to the set
+					for _, otherAddr := range sb.keys {
+						if otherAddr.Original == addr.Original {
+							continue
+						}
+						if otherAddr.Host != "" {
+							hostsSharedWithHostlessKey[otherAddr.Host] = struct{}{}
+						}
+					}
+					break
+				}
+			}
+		}
+	}
+
+	catchAllAP, err := newBaseAutomationPolicy(options, warnings, false)
+	if err != nil {
+		return nil, warnings, err
+	}
+
+	for _, p := range pairings {
+		for _, sblock := range p.serverBlocks {
+			// get values that populate an automation policy for this block
+			var ap *caddytls.AutomationPolicy
+
+			sblockHosts := sblock.hostsFromKeys(false)
+			if len(sblockHosts) == 0 {
+				ap = catchAllAP
+			}
+
+			// on-demand tls
+			if _, ok := sblock.pile["tls.on_demand"]; ok {
+				if ap == nil {
+					var err error
+					ap, err = newBaseAutomationPolicy(options, warnings, true)
+					if err != nil {
+						return nil, warnings, err
+					}
+				}
+				ap.OnDemand = true
+			}
+
+			// certificate issuers
+			if issuerVals, ok := sblock.pile["tls.cert_issuer"]; ok {
+				for _, issuerVal := range issuerVals {
+					issuer := issuerVal.Value.(certmagic.Issuer)
+					if ap == nil {
+						var err error
+						ap, err = newBaseAutomationPolicy(options, warnings, true)
+						if err != nil {
+							return nil, warnings, err
+						}
+					}
+					if ap == catchAllAP && !reflect.DeepEqual(ap.Issuer, issuer) {
+						return nil, warnings, fmt.Errorf("automation policy from site block is also default/catch-all policy because of key without hostname, and the two are in conflict: %#v != %#v", ap.Issuer, issuer)
+					}
+					ap.Issuer = issuer
+				}
+			}
+
+			// custom bind host
+			for _, cfgVal := range sblock.pile["bind"] {
+				// either an existing issuer is already configured (and thus, ap is not
+				// nil), or we need to configure an issuer, so we need ap to be non-nil
+				if ap == nil {
+					ap, err = newBaseAutomationPolicy(options, warnings, true)
+					if err != nil {
+						return nil, warnings, err
+					}
+				}
+
+				// if an issuer was already configured and it is NOT an ACME
+				// issuer, skip, since we intend to adjust only ACME issuers
+				var acmeIssuer *caddytls.ACMEIssuer
+				if ap.Issuer != nil {
+					var ok bool
+					if acmeIssuer, ok = ap.Issuer.(*caddytls.ACMEIssuer); !ok {
+						break
+					}
+				}
+
+				// proceed to configure the ACME issuer's bind host, without
+				// overwriting any existing settings
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
+				if acmeIssuer.Challenges == nil {
+					acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
+				}
+				if acmeIssuer.Challenges.BindHost == "" {
+					// only binding to one host is supported
+					var bindHost string
+					if bindHosts, ok := cfgVal.Value.([]string); ok && len(bindHosts) > 0 {
+						bindHost = bindHosts[0]
+					}
+					acmeIssuer.Challenges.BindHost = bindHost
+				}
+				ap.Issuer = acmeIssuer // we'll encode it later
+			}
+
+			if ap != nil {
+				// encode issuer now that it's all set up
+				issuerName := ap.Issuer.(caddy.Module).CaddyModule().ID.Name()
+				ap.IssuerRaw = caddyconfig.JSONModuleObject(ap.Issuer, "module", issuerName, &warnings)
+
+				// first make sure this block is allowed to create an automation policy;
+				// doing so is forbidden if it has a key with no host (i.e. ":443")
+				// and if there is a different server block that also has a key with no
+				// host -- since a key with no host matches any host, we need its
+				// associated automation policy to have an empty Subjects list, i.e. no
+				// host filter, which is indistinguishable between the two server blocks
+				// because automation is not done in the context of a particular server...
+				// this is an example of a poor mapping from Caddyfile to JSON but that's
+				// the least-leaky abstraction I could figure out
+				if len(sblockHosts) == 0 {
+					if serverBlocksWithHostlessKey > 1 {
+						// this server block and at least one other has a key with no host,
+						// making the two indistinguishable; it is misleading to define such
+						// a policy within one server block since it actually will apply to
+						// others as well
+						return nil, warnings, fmt.Errorf("cannot make a TLS automation policy from a server block that has a host-less address when there are other server block addresses lacking a host")
+					}
+					if catchAllAP == nil {
+						// this server block has a key with no hosts, but there is not yet
+						// a catch-all automation policy (probably because no global options
+						// were set), so this one becomes it
+						catchAllAP = ap
+					}
+				}
+
+				// associate our new automation policy with this server block's hosts,
+				// unless, of course, the server block has a key with no hosts, in which
+				// case its automation policy becomes or blends with the default/global
+				// automation policy because, of necessity, it applies to all hostnames
+				// (i.e. it has no Subjects filter) -- in that case, we'll append it last
+				if ap != catchAllAP {
+					ap.Subjects = sblockHosts
+
+					// if a combination of public and internal names were given
+					// for this same server block and no issuer was specified, we
+					// need to separate them out in the automation policies so
+					// that the internal names can use the internal issuer and
+					// the other names can use the default/public/ACME issuer
+					var ap2 *caddytls.AutomationPolicy
+					if ap.Issuer == nil {
+						var internal, external []string
+						for _, s := range ap.Subjects {
+							if certmagic.SubjectQualifiesForPublicCert(s) {
+								external = append(external, s)
+							} else {
+								internal = append(internal, s)
+							}
+						}
+						if len(external) > 0 && len(internal) > 0 {
+							ap.Subjects = external
+							apCopy := *ap
+							ap2 = &apCopy
+							ap2.Subjects = internal
+							ap2.IssuerRaw = caddyconfig.JSONModuleObject(caddytls.InternalIssuer{}, "module", "internal", &warnings)
+						}
+					}
+					if tlsApp.Automation == nil {
+						tlsApp.Automation = new(caddytls.AutomationConfig)
+					}
+					tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, ap)
+					if ap2 != nil {
+						tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, ap2)
+					}
+				}
+			}
+
+			// certificate loaders
+			if clVals, ok := sblock.pile["tls.certificate_loader"]; ok {
+				for _, clVal := range clVals {
+					certLoaders = append(certLoaders, clVal.Value.(caddytls.CertificateLoader))
+				}
+			}
+		}
+	}
+
+	// group certificate loaders by module name, then add to config
+	if len(certLoaders) > 0 {
+		loadersByName := make(map[string]caddytls.CertificateLoader)
+		for _, cl := range certLoaders {
+			name := caddy.GetModuleName(cl)
+			// ugh... technically, we may have multiple FileLoader and FolderLoader
+			// modules (because the tls directive returns one per occurrence), but
+			// the config structure expects only one instance of each kind of loader
+			// module, so we have to combine them... instead of enumerating each
+			// possible cert loader module in a type switch, we can use reflection,
+			// which works on any cert loaders that are slice types
+			if reflect.TypeOf(cl).Kind() == reflect.Slice {
+				combined := reflect.ValueOf(loadersByName[name])
+				if !combined.IsValid() {
+					combined = reflect.New(reflect.TypeOf(cl)).Elem()
+				}
+				clVal := reflect.ValueOf(cl)
+				for i := 0; i < clVal.Len(); i++ {
+					combined = reflect.Append(combined, clVal.Index(i))
+				}
+				loadersByName[name] = combined.Interface().(caddytls.CertificateLoader)
+			}
+		}
+		for certLoaderName, loaders := range loadersByName {
+			tlsApp.CertificatesRaw[certLoaderName] = caddyconfig.JSON(loaders, &warnings)
+		}
+	}
+
+	// set any of the on-demand options, for if/when on-demand TLS is enabled
+	if onDemand, ok := options["on_demand_tls"].(*caddytls.OnDemandConfig); ok {
+		if tlsApp.Automation == nil {
+			tlsApp.Automation = new(caddytls.AutomationConfig)
+		}
+		tlsApp.Automation.OnDemand = onDemand
+	}
+
+	// if any hostnames appear on the same server block as a key with
+	// no host, they will not be used with route matchers because the
+	// hostless key matches all hosts, therefore, it wouldn't be
+	// considered for auto-HTTPS, so we need to make sure those hosts
+	// are manually considered for managed certificates; we also need
+	// to make sure that any of these names which are internal-only
+	// get internal certificates by default rather than ACME
+	var al caddytls.AutomateLoader
+	internalAP := &caddytls.AutomationPolicy{
+		IssuerRaw: json.RawMessage(`{"module":"internal"}`),
+	}
+	for h := range hostsSharedWithHostlessKey {
+		al = append(al, h)
+		if !certmagic.SubjectQualifiesForPublicCert(h) {
+			internalAP.Subjects = append(internalAP.Subjects, h)
+		}
+	}
+	if len(al) > 0 {
+		tlsApp.CertificatesRaw["automate"] = caddyconfig.JSON(al, &warnings)
+	}
+	if len(internalAP.Subjects) > 0 {
+		if tlsApp.Automation == nil {
+			tlsApp.Automation = new(caddytls.AutomationConfig)
+		}
+		tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, internalAP)
+	}
+
+	// if there is a global/catch-all automation policy, ensure it goes last
+	if catchAllAP != nil {
+		// first, encode its issuer
+		issuerName := catchAllAP.Issuer.(caddy.Module).CaddyModule().ID.Name()
+		catchAllAP.IssuerRaw = caddyconfig.JSONModuleObject(catchAllAP.Issuer, "module", issuerName, &warnings)
+
+		// then append it to the end of the policies list
+		if tlsApp.Automation == nil {
+			tlsApp.Automation = new(caddytls.AutomationConfig)
+		}
+		tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, catchAllAP)
+	}
+
+	// do a little verification & cleanup
+	if tlsApp.Automation != nil {
+		// ensure automation policies don't overlap subjects (this should be
+		// an error at provision-time as well, but catch it in the adapt phase
+		// for convenience)
+		automationHostSet := make(map[string]struct{})
+		for _, ap := range tlsApp.Automation.Policies {
+			for _, s := range ap.Subjects {
+				if _, ok := automationHostSet[s]; ok {
+					return nil, warnings, fmt.Errorf("hostname appears in more than one automation policy, making certificate management ambiguous: %s", s)
+				}
+				automationHostSet[s] = struct{}{}
+			}
+		}
+
+		// consolidate automation policies that are the exact same
+		tlsApp.Automation.Policies = consolidateAutomationPolicies(tlsApp.Automation.Policies)
+	}
+
+	return tlsApp, warnings, nil
+}
+
+// newBaseAutomationPolicy returns a new TLS automation policy that gets
+// its values from the global options map. It should be used as the base
+// for any other automation policies. A nil policy (and no error) will be
+// returned if there are no default/global options. However, if always is
+// true, a non-nil value will always be returned (unless there is an error).
+func newBaseAutomationPolicy(options map[string]interface{}, warnings []caddyconfig.Warning, always bool) (*caddytls.AutomationPolicy, error) {
+	acmeCA, hasACMECA := options["acme_ca"]
+	acmeDNS, hasACMEDNS := options["acme_dns"]
+	acmeCARoot, hasACMECARoot := options["acme_ca_root"]
+	email, hasEmail := options["email"]
+	localCerts, hasLocalCerts := options["local_certs"]
+	keyType, hasKeyType := options["key_type"]
+
+	hasGlobalAutomationOpts := hasACMECA || hasACMEDNS || hasACMECARoot || hasEmail || hasLocalCerts || hasKeyType
+
+	// if there are no global options related to automation policies
+	// set, then we can just return right away
+	if !hasGlobalAutomationOpts {
+		if always {
+			return new(caddytls.AutomationPolicy), nil
+		}
+		return nil, nil
+	}
+
+	ap := new(caddytls.AutomationPolicy)
+
+	if localCerts != nil {
+		// internal issuer enabled trumps any ACME configurations; useful in testing
+		ap.Issuer = new(caddytls.InternalIssuer) // we'll encode it later
+	} else {
+		if acmeCA == nil {
+			acmeCA = ""
+		}
+		if email == nil {
+			email = ""
+		}
+		mgr := &caddytls.ACMEIssuer{
+			CA:    acmeCA.(string),
+			Email: email.(string),
+		}
+		if acmeDNS != nil {
+			provName := acmeDNS.(string)
+			dnsProvModule, err := caddy.GetModule("tls.dns." + provName)
+			if err != nil {
+				return nil, fmt.Errorf("getting DNS provider module named '%s': %v", provName, err)
+			}
+			mgr.Challenges = &caddytls.ChallengesConfig{
+				DNSRaw: caddyconfig.JSONModuleObject(dnsProvModule.New(), "provider", provName, &warnings),
+			}
+		}
+		if acmeCARoot != nil {
+			mgr.TrustedRootsPEMFiles = []string{acmeCARoot.(string)}
+		}
+		if keyType != nil {
+			ap.KeyType = keyType.(string)
+		}
+		ap.Issuer = mgr // we'll encode it later
+	}
+
+	return ap, nil
+}
+
+// consolidateAutomationPolicies combines automation policies that are the same,
+// for a cleaner overall output.
+func consolidateAutomationPolicies(aps []*caddytls.AutomationPolicy) []*caddytls.AutomationPolicy {
+	for i := 0; i < len(aps); i++ {
+		for j := 0; j < len(aps); j++ {
+			if j == i {
+				continue
+			}
+
+			// if they're exactly equal in every way, just keep one of them
+			if reflect.DeepEqual(aps[i], aps[j]) {
+				aps = append(aps[:j], aps[j+1:]...)
+				i--
+				break
+			}
+
+			// if the policy is the same, we can keep just one, but we have
+			// to be careful which one we keep; if only one has any hostnames
+			// defined, then we need to keep the one without any hostnames,
+			// otherwise the one without any subjects (a catch-all) would be
+			// eaten up by the one with subjects; and if both have subjects, we
+			// need to combine their lists
+			if bytes.Equal(aps[i].IssuerRaw, aps[j].IssuerRaw) &&
+				bytes.Equal(aps[i].StorageRaw, aps[j].StorageRaw) &&
+				aps[i].MustStaple == aps[j].MustStaple &&
+				aps[i].KeyType == aps[j].KeyType &&
+				aps[i].OnDemand == aps[j].OnDemand &&
+				aps[i].RenewalWindowRatio == aps[j].RenewalWindowRatio {
+				if len(aps[i].Subjects) == 0 && len(aps[j].Subjects) > 0 {
+					aps = append(aps[:j], aps[j+1:]...)
+				} else if len(aps[i].Subjects) > 0 && len(aps[j].Subjects) == 0 {
+					aps = append(aps[:i], aps[i+1:]...)
+				} else {
+					aps[i].Subjects = append(aps[i].Subjects, aps[j].Subjects...)
+					aps = append(aps[:j], aps[j+1:]...)
+				}
+				i--
+				break
+			}
+		}
+	}
+
+	// ensure any catch-all policies go last
+	sort.SliceStable(aps, func(i, j int) bool {
+		return len(aps[i].Subjects) > len(aps[j].Subjects)
+	})
+
+	return aps
+}

caddyconfig/json5/json5.go

@@ -1,43 +0,0 @@
-// Copyright 2015 Matthew Holt and The Caddy Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package json5adapter
-
-import (
-	"encoding/json"
-
-	"github.com/caddyserver/caddy/v2/caddyconfig"
-	"github.com/ilibs/json5"
-)
-
-func init() {
-	caddyconfig.RegisterAdapter("json5", Adapter{})
-}
-
-// Adapter adapts JSON5 to Caddy JSON.
-type Adapter struct{}
-
-// Adapt converts the JSON5 config in body to Caddy JSON.
-func (a Adapter) Adapt(body []byte, options map[string]interface{}) (result []byte, warnings []caddyconfig.Warning, err error) {
-	var decoded interface{}
-	err = json5.Unmarshal(body, &decoded)
-	if err != nil {
-		return
-	}
-	result, err = json.Marshal(decoded)
-	return
-}
-
-// Interface guard
-var _ caddyconfig.Adapter = (*Adapter)(nil)

caddyconfig/jsonc/jsonc.go

@@ -1,49 +0,0 @@
-// Copyright 2015 Matthew Holt and The Caddy Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package jsoncadapter
-
-import (
-	"encoding/json"
-
-	"github.com/caddyserver/caddy/v2/caddyconfig"
-	"github.com/muhammadmuzzammil1998/jsonc"
-)
-
-func init() {
-	caddyconfig.RegisterAdapter("jsonc", Adapter{})
-}
-
-// Adapter adapts JSON-C to Caddy JSON.
-type Adapter struct{}
-
-// Adapt converts the JSON-C config in body to Caddy JSON.
-func (a Adapter) Adapt(body []byte, options map[string]interface{}) (result []byte, warnings []caddyconfig.Warning, err error) {
-	result = jsonc.ToJSON(body)
-
-	// any errors in the JSON will be
-	// reported during config load, but
-	// we can at least warn here that
-	// it is not valid JSON
-	if !json.Valid(result) {
-		warnings = append(warnings, caddyconfig.Warning{
-			Message: "Resulting JSON is invalid.",
-		})
-	}
-
-	return
-}
-
-// Interface guard
-var _ caddyconfig.Adapter = (*Adapter)(nil)

caddyconfig/load.go

@@ -0,0 +1,153 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddyconfig
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/caddyserver/caddy/v2"
+)
+
+func init() {
+	caddy.RegisterModule(adminLoad{})
+}
+
+// adminLoad is a module that provides the /load endpoint
+// for the Caddy admin API. The only reason it's not baked
+// into the caddy package directly is because of the import
+// of the caddyconfig package for its GetAdapter function.
+// If the caddy package depends on the caddyconfig package,
+// then the caddyconfig package will not be able to import
+// the caddy package, and it can more easily cause backward
+// edges in the dependency tree (i.e. import cycle).
+// Fortunately, the admin API has first-class support for
+// adding endpoints from modules.
+type adminLoad struct{}
+
+// CaddyModule returns the Caddy module information.
+func (adminLoad) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "admin.api.load",
+		New: func() caddy.Module { return new(adminLoad) },
+	}
+}
+
+// Routes returns a route for the /load endpoint.
+func (al adminLoad) Routes() []caddy.AdminRoute {
+	return []caddy.AdminRoute{
+		{
+			Pattern: "/load",
+			Handler: caddy.AdminHandlerFunc(al.handleLoad),
+		},
+	}
+}
+
+// handleLoad replaces the entire current configuration with
+// a new one provided in the response body. It supports config
+// adapters through the use of the Content-Type header. A
+// config that is identical to the currently-running config
+// will be a no-op unless Cache-Control: must-revalidate is set.
+func (adminLoad) handleLoad(w http.ResponseWriter, r *http.Request) error {
+	if r.Method != http.MethodPost {
+		return caddy.APIError{
+			Code: http.StatusMethodNotAllowed,
+			Err:  fmt.Errorf("method not allowed"),
+		}
+	}
+
+	buf := bufPool.Get().(*bytes.Buffer)
+	buf.Reset()
+	defer bufPool.Put(buf)
+
+	_, err := io.Copy(buf, r.Body)
+	if err != nil {
+		return caddy.APIError{
+			Code: http.StatusBadRequest,
+			Err:  fmt.Errorf("reading request body: %v", err),
+		}
+	}
+	body := buf.Bytes()
+
+	// if the config is formatted other than Caddy's native
+	// JSON, we need to adapt it before loading it
+	if ctHeader := r.Header.Get("Content-Type"); ctHeader != "" {
+		ct, _, err := mime.ParseMediaType(ctHeader)
+		if err != nil {
+			return caddy.APIError{
+				Code: http.StatusBadRequest,
+				Err:  fmt.Errorf("invalid Content-Type: %v", err),
+			}
+		}
+		if !strings.HasSuffix(ct, "/json") {
+			slashIdx := strings.Index(ct, "/")
+			if slashIdx < 0 {
+				return caddy.APIError{
+					Code: http.StatusBadRequest,
+					Err:  fmt.Errorf("malformed Content-Type"),
+				}
+			}
+			adapterName := ct[slashIdx+1:]
+			cfgAdapter := GetAdapter(adapterName)
+			if cfgAdapter == nil {
+				return caddy.APIError{
+					Code: http.StatusBadRequest,
+					Err:  fmt.Errorf("unrecognized config adapter '%s'", adapterName),
+				}
+			}
+			result, warnings, err := cfgAdapter.Adapt(body, nil)
+			if err != nil {
+				return caddy.APIError{
+					Code: http.StatusBadRequest,
+					Err:  fmt.Errorf("adapting config using %s adapter: %v", adapterName, err),
+				}
+			}
+			if len(warnings) > 0 {
+				respBody, err := json.Marshal(warnings)
+				if err != nil {
+					caddy.Log().Named("admin.api.load").Error(err.Error())
+				}
+				_, _ = w.Write(respBody)
+			}
+			body = result
+		}
+	}
+
+	forceReload := r.Header.Get("Cache-Control") == "must-revalidate"
+
+	err = caddy.Load(body, forceReload)
+	if err != nil {
+		return caddy.APIError{
+			Code: http.StatusBadRequest,
+			Err:  fmt.Errorf("loading config: %v", err),
+		}
+	}
+
+	caddy.Log().Named("admin.api").Info("load complete")
+
+	return nil
+}
+
+var bufPool = sync.Pool{
+	New: func() interface{} {
+		return new(bytes.Buffer)
+	},
+}

caddytest/a.caddy.localhost.crt

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID5zCCAs8CFG4+w/pqR5AZQ+aVB330uRRRKMF0MA0GCSqGSIb3DQEBCwUAMIGv
+MQswCQYDVQQGEwJVUzELMAkGA1UECAwCTlkxGzAZBgNVBAoMEkxvY2FsIERldmVs
+b3BlbWVudDEbMBkGA1UEBwwSTG9jYWwgRGV2ZWxvcGVtZW50MRowGAYDVQQDDBFh
+LmNhZGR5LmxvY2FsaG9zdDEbMBkGA1UECwwSTG9jYWwgRGV2ZWxvcGVtZW50MSAw
+HgYJKoZIhvcNAQkBFhFhZG1pbkBjYWRkeS5sb2NhbDAeFw0yMDAzMTMxODUwMTda
+Fw0zMDAzMTExODUwMTdaMIGvMQswCQYDVQQGEwJVUzELMAkGA1UECAwCTlkxGzAZ
+BgNVBAoMEkxvY2FsIERldmVsb3BlbWVudDEbMBkGA1UEBwwSTG9jYWwgRGV2ZWxv
+cGVtZW50MRowGAYDVQQDDBFhLmNhZGR5LmxvY2FsaG9zdDEbMBkGA1UECwwSTG9j
+YWwgRGV2ZWxvcGVtZW50MSAwHgYJKoZIhvcNAQkBFhFhZG1pbkBjYWRkeS5sb2Nh
+bDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMd9pC9wF7j0459FndPs
+Deud/rq41jEZFsVOVtjQgjS1A5ct6NfeMmSlq8i1F7uaTMPZjbOHzY6y6hzLc9+y
+/VWNgyUC543HjXnNTnp9Xug6tBBxOxvRMw5mv2nAyzjBGDePPgN84xKhOXG2Wj3u
+fOZ+VPVISefRNvjKfN87WLJ0B0HI9wplG5ASVdPQsWDY1cndrZgt2sxQ/3fjIno4
+VvrgRWC9Penizgps/a0ZcFZMD/6HJoX/mSZVa1LjopwbMTXvyHCpXkth21E+rBt6
+I9DMHerdioVQcX25CqPmAwePxPZSNGEQo/Qu32kzcmscmYxTtYBhDa+yLuHgGggI
+j7ECAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAP/94KPtkpYtkWADnhtzDmgQ6Q1pH
+SubTUZdCwQtm6/LrvpT+uFNsOj4L3Mv3TVUnIQDmKd5VvR42W2MRBiTN2LQptgEn
+C7g9BB+UA9kjL3DPk1pJMjzxLHohh0uNLi7eh4mAj8eNvjz9Z4qMWPQoVS0y7/ZK
+cCBRKh2GkIqKm34ih6pX7xmMpPEQsFoTVPRHYJfYD1SZ8Iui+EN+7WqLuJWPsPXw
+JM1HuZKn7pZmJU2MZZBsrupHGUvNMbBg2mFJcxt4D1VvU+p+a67PSjpFQ6dJG2re
+pZoF+N1vMGAFkxe6UqhcC/bXDX+ILVQHJ+RNhzDO6DcWf8dRrC2LaJk3WA==
+-----END CERTIFICATE-----

caddytest/a.caddy.localhost.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAx32kL3AXuPTjn0Wd0+wN653+urjWMRkWxU5W2NCCNLUDly3o
+194yZKWryLUXu5pMw9mNs4fNjrLqHMtz37L9VY2DJQLnjceNec1Oen1e6Dq0EHE7
+G9EzDma/acDLOMEYN48+A3zjEqE5cbZaPe585n5U9UhJ59E2+Mp83ztYsnQHQcj3
+CmUbkBJV09CxYNjVyd2tmC3azFD/d+MiejhW+uBFYL096eLOCmz9rRlwVkwP/ocm
+hf+ZJlVrUuOinBsxNe/IcKleS2HbUT6sG3oj0Mwd6t2KhVBxfbkKo+YDB4/E9lI0
+YRCj9C7faTNyaxyZjFO1gGENr7Iu4eAaCAiPsQIDAQABAoIBAQDD/YFIBeWYlifn
+e9risQDAIrp3sk7lb9O6Rwv1+Wxi4hBEABvJsYhq74VFK/3EF4UhyWR5JIvkjYyK
+e6w887oGyoA05ZSe65XoO7fFidSrbbkoikZbPv3dQT7/ZCWEfdkQBNAVVyY0UGeC
+e3hPbjYRsb5AOSQ694X9idqC6uhqcOrBDjITFrctUoP4S6l9A6a+mLSUIwiICcuh
+mrNl+j0lzy7DMXRp/Z5Hyo5kuUlrC0dCLa1UHqtrrK7MR55AVEOihSNp1w+OC+vw
+f0VjE4JUtO7RQEQUmD1tDfLXwNfMFeWaobB2W0WMvRg0IqoitiqPxsPHRm56OxfM
+SRo/Q7QBAoGBAP8DapzBMuaIcJ7cE8Yl07ZGndWWf8buIKIItGF8rkEO3BXhrIke
+EmpOi+ELtpbMOG0APhORZyQ58f4ZOVrqZfneNKtDiEZV4mJZaYUESm1pU+2Y6+y5
+g4bpQSVKN0ow0xR+MH7qDYtSlsmBU7qAOz775L7BmMA1Bnu72aN/H1JBAoGBAMhD
+OzqCSakHOjUbEd22rPwqWmcIyVyo04gaSmcVVT2dHbqR4/t0gX5a9D9U2qwyO6xi
+/R+PXyMd32xIeVR2D/7SQ0x6dK68HXICLV8ofHZ5UQcHbxy5og4v/YxSZVTkN374
+cEsUeyB0s/UPOHLktFU5hpIlON72/Rp7b+pNIwFxAoGAczpq+Qu/YTWzlcSh1r4O
+7OT5uqI3eH7vFehTAV3iKxl4zxZa7NY+wfRd9kFhrr/2myIp6pOgBFl+hC+HoBIc
+JAyIxf5M3GNAWOpH6MfojYmzV7/qktu8l8BcJGplk0t+hVsDtMUze4nFAqZCXBpH
+Kw2M7bjyuZ78H/rgu6TcVUECgYEAo1M5ldE2U/VCApeuLX1TfWDpU8i1uK0zv3d5
+oLKkT1i5KzTak3SEO9HgC1qf8PoS8tfUio26UICHe99rnHehOfivzEq+qNdgyF+A
+M3BoeZMdgzcL5oh640k+Zte4LtDlddcWdhUhCepD7iPYrNNbQ3pkBwL2a9lRuOxc
+7OC2IPECgYBH8f3OrwXjDltIG1dDvuDPNljxLZbFEFbQyVzMePYNftgZknAyGEdh
+NW/LuWeTzstnmz/s6RE3jN5ZrrMa4sW77VA9+yU9QW2dkHqFyukQ4sfuNg6kDDNZ
++lqZYMCLw0M5P9fIbmnIYwey7tXkHfmzoCpnYHGQDN6hL0Bh0zGwmg==
+-----END RSA PRIVATE KEY-----

caddytest/caddy.localhost.crt

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID5zCCAs8CFFmAAFKV79uhzxc5qXbUw3oBNsYXMA0GCSqGSIb3DQEBCwUAMIGv
+MQswCQYDVQQGEwJVUzELMAkGA1UECAwCTlkxGzAZBgNVBAoMEkxvY2FsIERldmVs
+b3BlbWVudDEbMBkGA1UEBwwSTG9jYWwgRGV2ZWxvcGVtZW50MRowGAYDVQQDDBEq
+LmNhZGR5LmxvY2FsaG9zdDEbMBkGA1UECwwSTG9jYWwgRGV2ZWxvcGVtZW50MSAw
+HgYJKoZIhvcNAQkBFhFhZG1pbkBjYWRkeS5sb2NhbDAeFw0yMDAzMDIwODAxMTZa
+Fw0zMDAyMjgwODAxMTZaMIGvMQswCQYDVQQGEwJVUzELMAkGA1UECAwCTlkxGzAZ
+BgNVBAoMEkxvY2FsIERldmVsb3BlbWVudDEbMBkGA1UEBwwSTG9jYWwgRGV2ZWxv
+cGVtZW50MRowGAYDVQQDDBEqLmNhZGR5LmxvY2FsaG9zdDEbMBkGA1UECwwSTG9j
+YWwgRGV2ZWxvcGVtZW50MSAwHgYJKoZIhvcNAQkBFhFhZG1pbkBjYWRkeS5sb2Nh
+bDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJngfeirQkWaU8ihgIC5
+SKpRQX/3koRjljDK/oCbhLs+wg592kIwVv06l7+mn7NSaNBloabjuA1GqyLRsNLL
+ptrv0HvXa5qLx28+icsb2Ny3dJnQaj9w9PwjxQ1qZqEJfWRH1D8Vz9AmB+QSV/Gu
+8e8alGFewlYZVfH1kbxoTT6QorF37TeA3bh1fgKFtzsGYKswcaZNdDBBHzLunCKZ
+HU6U6L45hm+yLADj3mmDLafUeiVOt6MRLLoSD1eLRVSXGrNo+brJ87zkZntI9+W1
+JxOBoXtZCwka7k2DlAtLihsrmBZA2ZC9yVeu/SQy3qb3iCNnTFTCyAnWeTCr6Tcq
+6w8CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAOWfXqpAmD4C3wGiMeZAeaaS4hDAR
++JmN+avPDA6F6Bq7DB4NJuIwVUlaDL2s07w5VJJtW52aZVKoBlgHR5yG/XUli6J7
+YUJRmdQJvHUSu26cmKvyoOaTrEYbmvtGICWtZc8uTlMf9wQZbJA4KyxTgEQJDXsZ
+B2XFe+wVdhAgEpobYDROi+l/p8TL5z3U24LpwVTcJy5sEZVv7Wfs886IyxU8ORt8
+VZNcDiH6V53OIGeiufIhia/mPe6jbLntfGZfIFxtCcow4IA/lTy1ned7K5fmvNNb
+ZilxOQUk+wVK8genjdrZVAnAxsYLHJIb5yf9O7rr6fWciVMF3a0k5uNK1w==
+-----END CERTIFICATE-----

caddytest/caddy.localhost.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAmeB96KtCRZpTyKGAgLlIqlFBf/eShGOWMMr+gJuEuz7CDn3a
+QjBW/TqXv6afs1Jo0GWhpuO4DUarItGw0sum2u/Qe9drmovHbz6JyxvY3Ld0mdBq
+P3D0/CPFDWpmoQl9ZEfUPxXP0CYH5BJX8a7x7xqUYV7CVhlV8fWRvGhNPpCisXft
+N4DduHV+AoW3OwZgqzBxpk10MEEfMu6cIpkdTpTovjmGb7IsAOPeaYMtp9R6JU63
+oxEsuhIPV4tFVJcas2j5usnzvORme0j35bUnE4Ghe1kLCRruTYOUC0uKGyuYFkDZ
+kL3JV679JDLepveII2dMVMLICdZ5MKvpNyrrDwIDAQABAoIBAFcPK01zb6hfm12c
++k5aBiHOnUdgc/YRPg1XHEz5MEycQkDetZjTLrRQ7UBSbnKPgpu9lIsOtbhVLkgh
+6XAqJroiCou2oruqr+hhsqZGmBiwdvj7cNF6ADGTr05az7v22YneFdinZ481pStF
+sZocx+bm2+KHMV5zMSwXKyA0xtdJLxs2yklniDBxSZRppgppq1pDPprP5DkgKPfe
+3ekUmbQd5bHmivhW8ItbJLuf82XSsMBZ9ZhKiKIlWlbKAgiSV3SqnUQb5fi7l8hG
+yYZxbuCUIGFwKmEpUBBt/nyxrOlMiNtDh9JhrPmijTV3slq70pCLwLL/Ai2aeear
+EVA5VhkCgYEAyAmxfPqc2P7BsDAp67/sA7OEPso9qM4WyuWiVdlX2gb9TLNLYbPX
+Kk/UmpAIVzpoTAGY5Zp3wkvdD/ou8uUQsE8ioNn4S1a4G9XURH1wVhcEbUiAKI1S
+QVBH9B/Pj3eIp5OTKwob0Wj7DNdxoH7ed/Eok0EaTWzOA8pCWADKv/MCgYEAxOzY
+YsX7Nl+eyZr2+9unKyeAK/D1DCT/o99UUAHx72/xaBVP/06cfzpvKBNcF9iYc+fq
+R1yIUIrDRoSmYKBq+Kb3+nOg1nrqih/NBTokbTiI4Q+/30OQt0Al1e7y9iNKqV8H
+jYZItzluGNrWKedZbATwBwbVCY2jnNl6RMDnS3UCgYBxj3cwQUHLuoyQjjcuO80r
+qLzZvIxWiXDNDKIk5HcIMlGYOmz/8U2kGp/SgxQJGQJeq8V2C0QTjGfaCyieAcaA
+oNxCvptDgd6RBsoze5bLeNOtiqwe2WOp6n5+q5R0mOJ+Z7vzghCayGNFPgWmnH+F
+TeW/+wSIkc0+v5L8TK7NWwKBgBrlWlyLO9deUfqpHqihhICBYaEexOlGuF+yZfqT
+eW7BdFBJ8OYm33sFCR+JHV/oZlIWT8o1Wizd9vPPtEWoQ1P4wg/D8Si6GwSIeWEI
+YudD/HX4x7T/rmlI6qIAg9CYW18sqoRq3c2gm2fro6qPfYgiWIItLbWjUcBfd7Ki
+QjTtAoGARKdRv3jMWL84rlEx1nBRgL3pe9Dt+Uxzde2xT3ZeF+5Hp9NfU01qE6M6
+1I6H64smqpetlsXmCEVKwBemP3pJa6avLKgIYiQvHAD/v4rs9mqgy1RTqtYyGNhR
+1A/6dKkbiZ6wzePLLPasXVZxSKEviXf5gJooqumQVSVhCswyCZ0=
+-----END RSA PRIVATE KEY-----

caddytest/caddytest.go

@@ -0,0 +1,373 @@
+package caddytest
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"regexp"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aryann/difflib"
+	"github.com/caddyserver/caddy/v2/caddyconfig"
+	caddycmd "github.com/caddyserver/caddy/v2/cmd"
+
+	// plug in Caddy modules here
+	_ "github.com/caddyserver/caddy/v2/modules/standard"
+)
+
+// Defaults store any configuration required to make the tests run
+type Defaults struct {
+	// Port we expect caddy to listening on
+	AdminPort int
+	// Certificates we expect to be loaded before attempting to run the tests
+	Certifcates []string
+}
+
+// Default testing values
+var Default = Defaults{
+	AdminPort:   2019,
+	Certifcates: []string{"/caddy.localhost.crt", "/caddy.localhost.key"},
+}
+
+var (
+	matchKey  = regexp.MustCompile(`(/[\w\d\.]+\.key)`)
+	matchCert = regexp.MustCompile(`(/[\w\d\.]+\.crt)`)
+)
+
+type configLoadError struct {
+	Response string
+}
+
+func (e configLoadError) Error() string { return e.Response }
+
+func timeElapsed(start time.Time, name string) {
+	elapsed := time.Since(start)
+	log.Printf("%s took %s", name, elapsed)
+}
+
+// InitServer this will configure the server with a configurion of a specific
+// type. The configType must be either "json" or the adapter type.
+func InitServer(t *testing.T, rawConfig string, configType string) {
+
+	if err := initServer(t, rawConfig, configType); err != nil {
+		t.Logf("failed to load config: %s", err)
+		t.Fail()
+	}
+}
+
+// InitServer this will configure the server with a configurion of a specific
+// type. The configType must be either "json" or the adapter type.
+func initServer(t *testing.T, rawConfig string, configType string) error {
+
+	if testing.Short() {
+		t.SkipNow()
+		return nil
+	}
+
+	err := validateTestPrerequisites()
+	if err != nil {
+		t.Skipf("skipping tests as failed integration prerequisites. %s", err)
+		return nil
+	}
+
+	t.Cleanup(func() {
+		if t.Failed() {
+			res, err := http.Get(fmt.Sprintf("http://localhost:%d/config/", Default.AdminPort))
+			if err != nil {
+				t.Log("unable to read the current config")
+			}
+			defer res.Body.Close()
+			body, err := ioutil.ReadAll(res.Body)
+
+			var out bytes.Buffer
+			json.Indent(&out, body, "", "  ")
+			t.Logf("----------- failed with config -----------\n%s", out.String())
+		}
+	})
+
+	rawConfig = prependCaddyFilePath(rawConfig)
+	client := &http.Client{
+		Timeout: time.Second * 2,
+	}
+
+	start := time.Now()
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/load", Default.AdminPort), strings.NewReader(rawConfig))
+	if err != nil {
+		t.Errorf("failed to create request. %s", err)
+		return err
+	}
+
+	if configType == "json" {
+		req.Header.Add("Content-Type", "application/json")
+	} else {
+		req.Header.Add("Content-Type", "text/"+configType)
+	}
+
+	res, err := client.Do(req)
+	if err != nil {
+		t.Errorf("unable to contact caddy server. %s", err)
+		return err
+	}
+	timeElapsed(start, "caddytest: config load time")
+
+	defer res.Body.Close()
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		t.Errorf("unable to read response. %s", err)
+		return err
+	}
+
+	if res.StatusCode != 200 {
+		return configLoadError{Response: string(body)}
+	}
+
+	return nil
+}
+
+var hasValidated bool
+var arePrerequisitesValid bool
+
+func validateTestPrerequisites() error {
+
+	if hasValidated {
+		if !arePrerequisitesValid {
+			return errors.New("caddy integration prerequisites failed. see first error")
+		}
+		return nil
+	}
+
+	hasValidated = true
+	arePrerequisitesValid = false
+
+	// check certificates are found
+	for _, certName := range Default.Certifcates {
+		if _, err := os.Stat(getIntegrationDir() + certName); os.IsNotExist(err) {
+			return fmt.Errorf("caddy integration test certificates (%s) not found", certName)
+		}
+	}
+
+	if isCaddyAdminRunning() != nil {
+		// start inprocess caddy server
+		os.Args = []string{"caddy", "run"}
+		go func() {
+			caddycmd.Main()
+		}()
+
+		// wait for caddy to start
+		retries := 4
+		for ; retries > 0 && isCaddyAdminRunning() != nil; retries-- {
+			time.Sleep(10 * time.Millisecond)
+		}
+	}
+
+	// assert that caddy is running
+	if err := isCaddyAdminRunning(); err != nil {
+		return err
+	}
+
+	arePrerequisitesValid = true
+	return nil
+}
+
+func isCaddyAdminRunning() error {
+	// assert that caddy is running
+	client := &http.Client{
+		Timeout: time.Second * 2,
+	}
+	_, err := client.Get(fmt.Sprintf("http://localhost:%d/config/", Default.AdminPort))
+	if err != nil {
+		return errors.New("caddy integration test caddy server not running. Expected to be listening on localhost:2019")
+	}
+
+	return nil
+}
+
+func getIntegrationDir() string {
+
+	_, filename, _, ok := runtime.Caller(1)
+	if !ok {
+		panic("unable to determine the current file path")
+	}
+
+	return path.Dir(filename)
+}
+
+// use the convention to replace /[certificatename].[crt|key] with the full path
+// this helps reduce the noise in test configurations and also allow this
+// to run in any path
+func prependCaddyFilePath(rawConfig string) string {
+	r := matchKey.ReplaceAllString(rawConfig, getIntegrationDir()+"$1")
+	r = matchCert.ReplaceAllString(r, getIntegrationDir()+"$1")
+	return r
+}
+
+// creates a testing transport that forces call dialing connections to happen locally
+func createTestingTransport() *http.Transport {
+
+	dialer := net.Dialer{
+		Timeout:   5 * time.Second,
+		KeepAlive: 5 * time.Second,
+		DualStack: true,
+	}
+
+	dialContext := func(ctx context.Context, network, addr string) (net.Conn, error) {
+		parts := strings.Split(addr, ":")
+		destAddr := fmt.Sprintf("127.0.0.1:%s", parts[1])
+		log.Printf("caddytest: redirecting the dialer from %s to %s", addr, destAddr)
+		return dialer.DialContext(ctx, network, destAddr)
+	}
+
+	return &http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		DialContext:           dialContext,
+		ForceAttemptHTTP2:     true,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   5 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
+	}
+}
+
+// AssertLoadError will load a config and expect an error
+func AssertLoadError(t *testing.T, rawConfig string, configType string, expectedError string) {
+	err := initServer(t, rawConfig, configType)
+	if !strings.Contains(err.Error(), expectedError) {
+		t.Errorf("expected error \"%s\" but got \"%s\"", expectedError, err.Error())
+	}
+}
+
+// AssertGetResponse request a URI and assert the status code and the body contains a string
+func AssertGetResponse(t *testing.T, requestURI string, statusCode int, expectedBody string) (*http.Response, string) {
+	resp, body := AssertGetResponseBody(t, requestURI, statusCode)
+	if !strings.Contains(body, expectedBody) {
+		t.Errorf("requesting \"%s\" expected response body \"%s\" but got \"%s\"", requestURI, expectedBody, body)
+	}
+	return resp, body
+}
+
+// AssertGetResponseBody request a URI and assert the status code matches
+func AssertGetResponseBody(t *testing.T, requestURI string, expectedStatusCode int) (*http.Response, string) {
+
+	client := &http.Client{
+		Transport: createTestingTransport(),
+	}
+
+	resp, err := client.Get(requestURI)
+	if err != nil {
+		t.Errorf("failed to call server %s", err)
+		return nil, ""
+	}
+
+	defer resp.Body.Close()
+
+	if expectedStatusCode != resp.StatusCode {
+		t.Errorf("requesting \"%s\" expected status code: %d but got %d", requestURI, expectedStatusCode, resp.StatusCode)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Errorf("unable to read the response body %s", err)
+		return nil, ""
+	}
+
+	return resp, string(body)
+}
+
+// AssertRedirect makes a request and asserts the redirection happens
+func AssertRedirect(t *testing.T, requestURI string, expectedToLocation string, expectedStatusCode int) *http.Response {
+
+	redirectPolicyFunc := func(req *http.Request, via []*http.Request) error {
+		return http.ErrUseLastResponse
+	}
+
+	client := &http.Client{
+		CheckRedirect: redirectPolicyFunc,
+		Transport:     createTestingTransport(),
+	}
+
+	resp, err := client.Get(requestURI)
+	if err != nil {
+		t.Errorf("failed to call server %s", err)
+		return nil
+	}
+
+	if expectedStatusCode != resp.StatusCode {
+		t.Errorf("requesting \"%s\" expected status code: %d but got %d", requestURI, expectedStatusCode, resp.StatusCode)
+	}
+
+	loc, err := resp.Location()
+	if err != nil {
+		t.Errorf("requesting \"%s\" expected location: \"%s\" but got error: %s", requestURI, expectedToLocation, err)
+	}
+
+	if expectedToLocation != loc.String() {
+		t.Errorf("requesting \"%s\" expected location: \"%s\" but got \"%s\"", requestURI, expectedToLocation, loc.String())
+	}
+
+	return resp
+}
+
+// AssertAdapt adapts a config and then tests it against an expected result
+func AssertAdapt(t *testing.T, rawConfig string, adapterName string, expectedResponse string) {
+
+	cfgAdapter := caddyconfig.GetAdapter(adapterName)
+	if cfgAdapter == nil {
+		t.Errorf("unrecognized config adapter '%s'", adapterName)
+		return
+	}
+
+	options := make(map[string]interface{})
+	options["pretty"] = "true"
+
+	result, warnings, err := cfgAdapter.Adapt([]byte(rawConfig), options)
+	if err != nil {
+		t.Errorf("adapting config using %s adapter: %v", adapterName, err)
+		return
+	}
+
+	if len(warnings) > 0 {
+		for _, w := range warnings {
+			t.Logf("warning: directive: %s : %s", w.Directive, w.Message)
+		}
+	}
+
+	diff := difflib.Diff(
+		strings.Split(expectedResponse, "\n"),
+		strings.Split(string(result), "\n"))
+
+	// scan for failure
+	failed := false
+	for _, d := range diff {
+		if d.Delta != difflib.Common {
+			failed = true
+			break
+		}
+	}
+
+	if failed {
+		for _, d := range diff {
+			switch d.Delta {
+			case difflib.Common:
+				fmt.Printf("  %s\n", d.Payload)
+			case difflib.LeftOnly:
+				fmt.Printf(" - %s\n", d.Payload)
+			case difflib.RightOnly:
+				fmt.Printf(" + %s\n", d.Payload)
+			}
+		}
+		t.Fail()
+	}
+}

caddytest/caddytest_test.go

@@ -0,0 +1,33 @@
+package caddytest
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestReplaceCertificatePaths(t *testing.T) {
+	rawConfig := `a.caddy.localhost:9443 {
+		tls /caddy.localhost.crt /caddy.localhost.key {
+		}
+
+		redir / https://b.caddy.localhost:9443/version 301
+    
+		respond /version 200 {
+		  body "hello from a.caddy.localhost"
+		}	
+	  }`
+
+	r := prependCaddyFilePath(rawConfig)
+
+	if !strings.Contains(r, getIntegrationDir()+"/caddy.localhost.crt") {
+		t.Error("expected the /caddy.localhost.crt to be expanded to include the full path")
+	}
+
+	if !strings.Contains(r, getIntegrationDir()+"/caddy.localhost.key") {
+		t.Error("expected the /caddy.localhost.crt to be expanded to include the full path")
+	}
+
+	if !strings.Contains(r, "https://b.caddy.localhost:9443/version") {
+		t.Error("expected redirect uri to be unchanged")
+	}
+}

caddytest/integration/caddyfile_adapt_test.go

@@ -0,0 +1,285 @@
+package integration
+
+import (
+	"testing"
+
+	"github.com/caddyserver/caddy/v2/caddytest"
+)
+
+func TestHttpOnlyOnLocalhost(t *testing.T) {
+	caddytest.AssertAdapt(t, ` 
+	localhost:80 {
+		respond /version 200 {
+			body "hello from localhost"
+		}
+	}
+  `, "caddyfile", `{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"body": "hello from localhost",
+													"handler": "static_response",
+													"status_code": 200
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/version"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}`)
+}
+
+func TestHttpOnlyOnAnyAddress(t *testing.T) {
+	caddytest.AssertAdapt(t, ` 
+	:80 {
+		respond /version 200 {
+			body "hello from localhost"
+		}
+	}
+  `, "caddyfile", `{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"path": [
+										"/version"
+									]
+								}
+							],
+							"handle": [
+								{
+									"body": "hello from localhost",
+									"handler": "static_response",
+									"status_code": 200
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}`)
+}
+
+func TestHttpsOnDomain(t *testing.T) {
+	caddytest.AssertAdapt(t, ` 
+	a.caddy.localhost {
+		respond /version 200 {
+			body "hello from localhost"
+		}
+	}
+  `, "caddyfile", `{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"a.caddy.localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"body": "hello from localhost",
+													"handler": "static_response",
+													"status_code": 200
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/version"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}`)
+}
+
+func TestHttpOnlyOnDomain(t *testing.T) {
+	caddytest.AssertAdapt(t, ` 
+	http://a.caddy.localhost {
+		respond /version 200 {
+			body "hello from localhost"
+		}
+	}
+  `, "caddyfile", `{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"a.caddy.localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"body": "hello from localhost",
+													"handler": "static_response",
+													"status_code": 200
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/version"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"automatic_https": {
+						"skip": [
+							"a.caddy.localhost"
+						]
+					}
+				}
+			}
+		}
+	}
+}`)
+}
+
+func TestHttpOnlyOnNonStandardPort(t *testing.T) {
+	caddytest.AssertAdapt(t, ` 
+	http://a.caddy.localhost:81 {
+		respond /version 200 {
+			body "hello from localhost"
+		}
+	}
+  `, "caddyfile", `{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":81"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"a.caddy.localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"body": "hello from localhost",
+													"handler": "static_response",
+													"status_code": 200
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/version"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"automatic_https": {
+						"skip": [
+							"a.caddy.localhost"
+						]
+					}
+				}
+			}
+		}
+	}
+}`)
+}

caddytest/integration/caddyfile_test.go

@@ -0,0 +1,68 @@
+package integration
+
+import (
+	"testing"
+
+	"github.com/caddyserver/caddy/v2/caddytest"
+)
+
+func TestRespond(t *testing.T) {
+
+	// arrange
+	caddytest.InitServer(t, ` 
+  {
+    http_port     9080
+    https_port    9443
+  }
+  
+  localhost:9080 {
+    respond /version 200 {
+      body "hello from localhost"
+    }	
+    }
+  `, "caddyfile")
+
+	// act and assert
+	caddytest.AssertGetResponse(t, "http://localhost:9080/version", 200, "hello from localhost")
+}
+
+func TestRedirect(t *testing.T) {
+
+	// arrange
+	caddytest.InitServer(t, `
+  {
+    http_port     9080
+    https_port    9443
+  }
+  
+  localhost:9080 {
+    
+    redir / http://localhost:9080/hello 301
+    
+    respond /hello 200 {
+      body "hello from localhost"
+    }	
+    }
+  `, "caddyfile")
+
+	// act and assert
+	caddytest.AssertRedirect(t, "http://localhost:9080/", "http://localhost:9080/hello", 301)
+
+	// follow redirect
+	caddytest.AssertGetResponse(t, "http://localhost:9080/", 200, "hello from localhost")
+}
+
+func TestDuplicateHosts(t *testing.T) {
+
+	// act and assert
+	caddytest.AssertLoadError(t,
+		`
+    localhost:9080 {
+    }
+  
+    localhost:9080 { 
+    }
+    `,
+		"caddyfile",
+		"duplicate site address not allowed")
+}

caddytest/integration/sni_test.go

@@ -0,0 +1,317 @@
+package integration
+
+import (
+	"testing"
+
+	"github.com/caddyserver/caddy/v2/caddytest"
+)
+
+func TestDefaultSNI(t *testing.T) {
+
+	// arrange
+	caddytest.InitServer(t, `{
+    "apps": {
+      "http": {
+        "http_port": 9080,
+        "https_port": 9443,
+        "servers": {
+          "srv0": {
+            "listen": [
+              ":9443"
+            ],
+            "routes": [
+              {
+                "handle": [
+                  {
+                    "handler": "subroute",
+                    "routes": [
+                      {
+                        "handle": [
+                          {
+                            "body": "hello from a.caddy.localhost",
+                            "handler": "static_response",
+                            "status_code": 200
+                          }
+                        ],
+                        "match": [
+                          {
+                            "path": [
+                              "/version"
+                            ]
+                          }
+                        ]
+                      }
+                    ]
+                  }
+                ],
+                "match": [
+                  {
+                    "host": [
+                      "127.0.0.1"
+                    ]
+                  }
+                ],
+                "terminal": true
+              }
+            ],
+            "tls_connection_policies": [
+              {
+                "certificate_selection": {
+                  "any_tag": ["cert0"]
+                },
+                "match": {
+                  "sni": [
+                    "127.0.0.1"
+                  ]
+                }
+              },
+              {
+                "default_sni": "*.caddy.localhost"
+              }
+            ]
+          }
+        }
+      },
+      "tls": {
+        "certificates": {
+          "load_files": [
+            {
+              "certificate": "/caddy.localhost.crt",
+              "key": "/caddy.localhost.key",
+              "tags": [
+                "cert0"
+              ]
+            }
+          ]
+        }
+      },
+      "pki": {
+        "certificate_authorities" : {
+          "local" : {
+            "install_trust": false
+          }
+        }
+      }
+    }
+  }
+  `, "json")
+
+	// act and assert
+	// makes a request with no sni
+	caddytest.AssertGetResponse(t, "https://127.0.0.1:9443/version", 200, "hello from a")
+}
+
+func TestDefaultSNIWithNamedHostAndExplicitIP(t *testing.T) {
+
+	// arrange
+	caddytest.InitServer(t, ` 
+  {
+    "apps": {
+      "http": {
+        "http_port": 9080,
+        "https_port": 9443,
+        "servers": {
+          "srv0": {
+            "listen": [
+              ":9443"
+            ],
+            "routes": [
+              {
+                "handle": [
+                  {
+                    "handler": "subroute",
+                    "routes": [
+                      {
+                        "handle": [
+                          {
+                            "body": "hello from a",
+                            "handler": "static_response",
+                            "status_code": 200
+                          }
+                        ],
+                        "match": [
+                          {
+                            "path": [
+                              "/version"
+                            ]
+                          }
+                        ]
+                      }
+                    ]
+                  }
+                ],
+                "match": [
+                  {
+                    "host": [
+                      "a.caddy.localhost",
+                      "127.0.0.1"
+                    ]
+                  }
+                ],
+                "terminal": true
+              }
+            ],
+            "tls_connection_policies": [
+              {
+                "certificate_selection": {
+                  "any_tag": ["cert0"]
+                },
+                "default_sni": "a.caddy.localhost",
+                "match": {
+                  "sni": [
+                    "a.caddy.localhost",
+                    "127.0.0.1",
+                    ""
+                  ]
+                }
+              },
+              {
+                "default_sni": "a.caddy.localhost"
+              }
+            ]
+          }
+        }
+      },
+      "tls": {
+        "certificates": {
+          "load_files": [
+            {
+              "certificate": "/a.caddy.localhost.crt",
+              "key": "/a.caddy.localhost.key",
+              "tags": [
+                "cert0"
+              ]
+            }
+          ]
+        }
+      },
+      "pki": {
+        "certificate_authorities" : {
+          "local" : {
+            "install_trust": false
+          }
+        }
+      }
+    }
+  }
+  `, "json")
+
+	// act and assert
+	// makes a request with no sni
+	caddytest.AssertGetResponse(t, "https://127.0.0.1:9443/version", 200, "hello from a")
+}
+
+func TestDefaultSNIWithPortMappingOnly(t *testing.T) {
+
+	// arrange
+	caddytest.InitServer(t, ` 
+  {
+    "apps": {
+      "http": {
+        "http_port": 9080,
+        "https_port": 9443,
+        "servers": {
+          "srv0": {
+            "listen": [
+              ":9443"
+            ],
+            "routes": [
+              {
+                "handle": [
+                  {
+                    "body": "hello from a.caddy.localhost",
+                    "handler": "static_response",
+                    "status_code": 200
+                  }
+                ],
+                "match": [
+                  {
+                    "path": [
+                      "/version"
+                    ]
+                  }
+                ]
+              }
+            ],
+            "tls_connection_policies": [
+              {
+                "certificate_selection": {
+                  "any_tag": ["cert0"]
+                },
+                "default_sni": "a.caddy.localhost"
+              }
+            ]
+          }
+        }
+      },
+      "tls": {
+        "certificates": {
+          "load_files": [
+            {
+              "certificate": "/a.caddy.localhost.crt",
+              "key": "/a.caddy.localhost.key",
+              "tags": [
+                "cert0"
+              ]
+            }
+          ]
+        }
+      },
+      "pki": {
+        "certificate_authorities" : {
+          "local" : {
+            "install_trust": false
+          }
+        }
+      }
+    }
+  }
+  `, "json")
+
+	// act and assert
+	// makes a request with no sni
+	caddytest.AssertGetResponse(t, "https://127.0.0.1:9443/version", 200, "hello from a")
+}
+
+func TestHttpOnlyOnDomainWithSNI(t *testing.T) {
+	caddytest.AssertAdapt(t, `
+	{
+		default_sni a.caddy.localhost
+	}
+	:80 {
+		respond /version 200 {
+			body "hello from localhost"
+		}
+	}
+	`, "caddyfile", `{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"path": [
+										"/version"
+									]
+								}
+							],
+							"handle": [
+								{
+									"body": "hello from localhost",
+									"handler": "static_response",
+									"status_code": 200
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}`)
+}

cmd/commandfuncs.go

@@ -34,13 +34,14 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
-	"github.com/mholt/certmagic"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"go.uber.org/zap"
 )
 
 func cmdStart(fl Flags) (int, error) {
 	startCmdConfigFlag := fl.String("config")
 	startCmdConfigAdapterFlag := fl.String("adapter")
+	startCmdWatchFlag := fl.Bool("watch")
 
 	// open a listener to which the child process will connect when
 	// it is ready to confirm that it has successfully started
@@ -67,6 +68,9 @@ func cmdStart(fl Flags) (int, error) {
 	if startCmdConfigAdapterFlag != "" {
 		cmd.Args = append(cmd.Args, "--adapter", startCmdConfigAdapterFlag)
 	}
+	if startCmdWatchFlag {
+		cmd.Args = append(cmd.Args, "--watch")
+	}
 	stdinpipe, err := cmd.StdinPipe()
 	if err != nil {
 		return caddy.ExitCodeFailedStartup,
@@ -130,7 +134,7 @@ func cmdStart(fl Flags) (int, error) {
 	// when one of the goroutines unblocks, we're done and can exit
 	select {
 	case <-success:
-		fmt.Printf("Successfully started Caddy (pid=%d)\n", cmd.Process.Pid)
+		fmt.Printf("Successfully started Caddy (pid=%d) - Caddy is running in the background\n", cmd.Process.Pid)
 	case err := <-exit:
 		return caddy.ExitCodeFailedStartup,
 			fmt.Errorf("caddy process exited with error: %v", err)
@@ -144,6 +148,7 @@ func cmdRun(fl Flags) (int, error) {
 	runCmdConfigAdapterFlag := fl.String("adapter")
 	runCmdResumeFlag := fl.Bool("resume")
 	runCmdPrintEnvFlag := fl.Bool("environ")
+	runCmdWatchFlag := fl.Bool("watch")
 	runCmdPingbackFlag := fl.String("pingback")
 
 	// if we are supposed to print the environment, do that first
@@ -166,22 +171,26 @@ func cmdRun(fl Flags) (int, error) {
 		} else if err != nil {
 			return caddy.ExitCodeFailedStartup, err
 		} else {
-			caddy.Log().Info("resuming from last configuration", zap.String("autosave_file", caddy.ConfigAutosavePath))
+			if runCmdConfigFlag == "" {
+				caddy.Log().Info("resuming from last configuration",
+					zap.String("autosave_file", caddy.ConfigAutosavePath))
+			} else {
+				// if they also specified a config file, user should be aware that we're not
+				// using it (doing so could lead to data/config loss by overwriting!)
+				caddy.Log().Warn("--config and --resume flags were used together; ignoring --config and resuming from last configuration",
+					zap.String("autosave_file", caddy.ConfigAutosavePath))
+			}
 		}
 	}
 	// we don't use 'else' here since this value might have been changed in 'if' block; i.e. not mutually exclusive
+	var configFile string
 	if !runCmdResumeFlag {
-		config, _, err = loadConfig(runCmdConfigFlag, runCmdConfigAdapterFlag)
+		config, configFile, err = loadConfig(runCmdConfigFlag, runCmdConfigAdapterFlag)
 		if err != nil {
 			return caddy.ExitCodeFailedStartup, err
 		}
 	}
 
-	// set a fitting User-Agent for ACME requests
-	goModule := caddy.GoModule()
-	cleanModVersion := strings.TrimPrefix(goModule.Version, "v")
-	certmagic.UserAgent = "Caddy/" + cleanModVersion
-
 	// run the initial config
 	err = caddy.Load(config, true)
 	if err != nil {
@@ -210,6 +219,12 @@ func cmdRun(fl Flags) (int, error) {
 		}
 	}
 
+	// if enabled, reload config file automatically on changes
+	// (this better only be used in dev!)
+	if runCmdWatchFlag {
+		go watchConfigFile(configFile, runCmdConfigAdapterFlag)
+	}
+
 	// warn if the environment does not provide enough information about the disk
 	hasXDG := os.Getenv("XDG_DATA_HOME") != "" &&
 		os.Getenv("XDG_CONFIG_HOME") != "" &&
@@ -265,11 +280,11 @@ func cmdReload(fl Flags) (int, error) {
 	reloadCmdAddrFlag := fl.String("address")
 
 	// get the config in caddy's native format
-	config, hasConfig, err := loadConfig(reloadCmdConfigFlag, reloadCmdConfigAdapterFlag)
+	config, configFile, err := loadConfig(reloadCmdConfigFlag, reloadCmdConfigAdapterFlag)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, err
 	}
-	if !hasConfig {
+	if configFile == "" {
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("no config file to load")
 	}
 
@@ -491,35 +506,10 @@ func cmdValidateConfig(fl Flags) (int, error) {
 	validateCmdConfigFlag := fl.String("config")
 	validateCmdAdapterFlag := fl.String("adapter")
 
-	input, err := ioutil.ReadFile(validateCmdConfigFlag)
+	input, _, err := loadConfig(validateCmdConfigFlag, validateCmdAdapterFlag)
 	if err != nil {
-		return caddy.ExitCodeFailedStartup,
-			fmt.Errorf("reading input file: %v", err)
+		return caddy.ExitCodeFailedStartup, err
 	}
-
-	if validateCmdAdapterFlag != "" {
-		cfgAdapter := caddyconfig.GetAdapter(validateCmdAdapterFlag)
-		if cfgAdapter == nil {
-			return caddy.ExitCodeFailedStartup,
-				fmt.Errorf("unrecognized config adapter: %s", validateCmdAdapterFlag)
-		}
-
-		adaptedConfig, warnings, err := cfgAdapter.Adapt(input, nil)
-		if err != nil {
-			return caddy.ExitCodeFailedStartup, err
-		}
-		// print warnings to stderr
-		for _, warn := range warnings {
-			msg := warn.Message
-			if warn.Directive != "" {
-				msg = fmt.Sprintf("%s: %s", warn.Directive, warn.Message)
-			}
-			fmt.Fprintf(os.Stderr, "[WARNING][%s] %s:%d: %s\n", validateCmdAdapterFlag, warn.File, warn.Line, msg)
-		}
-
-		input = adaptedConfig
-	}
-
 	input = caddy.RemoveMetaFields(input)
 
 	var cfg *caddy.Config
@@ -538,6 +528,33 @@ func cmdValidateConfig(fl Flags) (int, error) {
 	return caddy.ExitCodeSuccess, nil
 }
 
+func cmdFmt(fl Flags) (int, error) {
+	formatCmdConfigFile := fl.Arg(0)
+	if formatCmdConfigFile == "" {
+		formatCmdConfigFile = "Caddyfile"
+	}
+	overwrite := fl.Bool("overwrite")
+
+	input, err := ioutil.ReadFile(formatCmdConfigFile)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup,
+			fmt.Errorf("reading input file: %v", err)
+	}
+
+	output := caddyfile.Format(input)
+
+	if overwrite {
+		err = ioutil.WriteFile(formatCmdConfigFile, output, 0644)
+		if err != nil {
+			return caddy.ExitCodeFailedStartup, nil
+		}
+	} else {
+		fmt.Print(string(output))
+	}
+
+	return caddy.ExitCodeSuccess, nil
+}
+
 func cmdHelp(fl Flags) (int, error) {
 	const fullDocs = `Full documentation is available at:
 https://caddyserver.com/docs/command-line`

cmd/commands.go

@@ -74,7 +74,7 @@ func init() {
 	RegisterCommand(Command{
 		Name:  "start",
 		Func:  cmdStart,
-		Usage: "[--config <path> [[--adapter <name>]]",
+		Usage: "[--config <path> [--adapter <name>]] [--watch]",
 		Short: "Starts the Caddy process in the background and then returns",
 		Long: `
 Starts the Caddy process, optionally bootstrapped with an initial config file.
@@ -87,6 +87,7 @@ using 'caddy run' instead to keep it in the foreground.`,
 			fs := flag.NewFlagSet("start", flag.ExitOnError)
 			fs.String("config", "", "Configuration file")
 			fs.String("adapter", "", "Name of config adapter to apply")
+			fs.Bool("watch", false, "Reload changed config file automatically")
 			return fs
 		}(),
 	})
@@ -94,7 +95,7 @@ using 'caddy run' instead to keep it in the foreground.`,
 	RegisterCommand(Command{
 		Name:  "run",
 		Func:  cmdRun,
-		Usage: "[--config <path> [--adapter <name>]] [--environ]",
+		Usage: "[--config <path> [--adapter <name>]] [--environ] [--watch]",
 		Short: `Starts the Caddy process and blocks indefinitely`,
 		Long: `
 Starts the Caddy process, optionally bootstrapped with an initial config file,
@@ -119,13 +120,18 @@ be printed before starting. This is the same as the environ command but does
 not quit after printing, and can be useful for troubleshooting.
 
 The --resume flag will override the --config flag if there is a config auto-
-save file. It is not an error if --resume is used and no autosave file exists.`,
+save file. It is not an error if --resume is used and no autosave file exists.
+
+If --watch is specified, the config file will be loaded automatically after
+changes. ⚠️ This is dangerous in production! Only use this option in a local
+development environment.`,
 		Flags: func() *flag.FlagSet {
 			fs := flag.NewFlagSet("run", flag.ExitOnError)
 			fs.String("config", "", "Configuration file")
 			fs.String("adapter", "", "Name of config adapter to apply")
-			fs.Bool("resume", false, "Use saved config, if any (and prefer over --config file)")
 			fs.Bool("environ", false, "Print environment")
+			fs.Bool("resume", false, "Use saved config, if any (and prefer over --config file)")
+			fs.Bool("watch", false, "Watch config file for changes and reload it automatically")
 			fs.String("pingback", "", "Echo confirmation bytes to this address on success")
 			return fs
 		}(),
@@ -242,6 +248,24 @@ provisioning stages.`,
 		}(),
 	})
 
+	RegisterCommand(Command{
+		Name:  "fmt",
+		Func:  cmdFmt,
+		Usage: "[--overwrite] [<path>]",
+		Short: "Formats a Caddyfile",
+		Long: `
+Formats the Caddyfile by adding proper indentation and spaces to improve
+human readability. It prints the result to stdout.
+
+If --write is specified, the output will be written to the config file
+directly instead of printing it.`,
+		Flags: func() *flag.FlagSet {
+			fs := flag.NewFlagSet("format", flag.ExitOnError)
+			fs.Bool("overwrite", false, "Overwrite the input file with the results")
+			return fs
+		}(),
+	})
+
 }
 
 // RegisterCommand registers the command cmd.
@@ -256,6 +280,8 @@ provisioning stages.`,
 // This function panics if the name is already registered,
 // if the name does not meet the described format, or if
 // any of the fields are missing from cmd.
+//
+// This function should be used in init().
 func RegisterCommand(cmd Command) {
 	if cmd.Name == "" {
 		panic("command name is required")

cmd/main.go

@@ -30,9 +30,21 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
+	"github.com/caddyserver/certmagic"
 	"go.uber.org/zap"
 )
 
+func init() {
+	// set a fitting User-Agent for ACME requests
+	goModule := caddy.GoModule()
+	cleanModVersion := strings.TrimPrefix(goModule.Version, "v")
+	certmagic.UserAgent = "Caddy/" + cleanModVersion
+
+	// by using Caddy, user indicates agreement to CA terms
+	// (very important, or ACME account creation will fail!)
+	certmagic.DefaultACME.Agreed = true
+}
+
 // Main implements the main function of the caddy command.
 // Call this if Caddy is to be the main() if your program.
 func Main() {
@@ -99,10 +111,10 @@ func handlePingbackConn(conn net.Conn, expect []byte) error {
 // there is no config available. It prints any warnings to stderr,
 // and returns the resulting JSON config bytes along with
 // whether a config file was loaded or not.
-func loadConfig(configFile, adapterName string) ([]byte, bool, error) {
+func loadConfig(configFile, adapterName string) ([]byte, string, error) {
 	// specifying an adapter without a config file is ambiguous
 	if adapterName != "" && configFile == "" {
-		return nil, false, fmt.Errorf("cannot adapt config without config file (use --config)")
+		return nil, "", fmt.Errorf("cannot adapt config without config file (use --config)")
 	}
 
 	// load initial config and adapter
@@ -112,7 +124,7 @@ func loadConfig(configFile, adapterName string) ([]byte, bool, error) {
 	if configFile != "" {
 		config, err = ioutil.ReadFile(configFile)
 		if err != nil {
-			return nil, false, fmt.Errorf("reading config file: %v", err)
+			return nil, "", fmt.Errorf("reading config file: %v", err)
 		}
 		caddy.Log().Info("using provided configuration",
 			zap.String("config_file", configFile),
@@ -129,7 +141,7 @@ func loadConfig(configFile, adapterName string) ([]byte, bool, error) {
 				cfgAdapter = nil
 			} else if err != nil {
 				// default Caddyfile exists, but error reading it
-				return nil, false, fmt.Errorf("reading default Caddyfile: %v", err)
+				return nil, "", fmt.Errorf("reading default Caddyfile: %v", err)
 			} else {
 				// success reading default Caddyfile
 				configFile = "Caddyfile"
@@ -151,7 +163,7 @@ func loadConfig(configFile, adapterName string) ([]byte, bool, error) {
 	if adapterName != "" {
 		cfgAdapter = caddyconfig.GetAdapter(adapterName)
 		if cfgAdapter == nil {
-			return nil, false, fmt.Errorf("unrecognized config adapter: %s", adapterName)
+			return nil, "", fmt.Errorf("unrecognized config adapter: %s", adapterName)
 		}
 	}
 
@@ -161,7 +173,7 @@ func loadConfig(configFile, adapterName string) ([]byte, bool, error) {
 			"filename": configFile,
 		})
 		if err != nil {
-			return nil, false, fmt.Errorf("adapting config using %s: %v", adapterName, err)
+			return nil, "", fmt.Errorf("adapting config using %s: %v", adapterName, err)
 		}
 		for _, warn := range warnings {
 			msg := warn.Message
@@ -173,7 +185,85 @@ func loadConfig(configFile, adapterName string) ([]byte, bool, error) {
 		config = adaptedConfig
 	}
 
-	return config, configFile != "", nil
+	return config, configFile, nil
+}
+
+// watchConfigFile watches the config file at filename for changes
+// and reloads the config if the file was updated. This function
+// blocks indefinitely; it only quits if the poller has errors for
+// long enough time. The filename passed in must be the actual
+// config file used, not one to be discovered.
+func watchConfigFile(filename, adapterName string) {
+	// make our logger; since config reloads can change the
+	// default logger, we need to get it dynamically each time
+	logger := func() *zap.Logger {
+		return caddy.Log().
+			Named("watcher").
+			With(zap.String("config_file", filename))
+	}
+
+	// get the initial timestamp on the config file
+	info, err := os.Stat(filename)
+	if err != nil {
+		logger().Error("cannot watch config file", zap.Error(err))
+		return
+	}
+	lastModified := info.ModTime()
+
+	logger().Info("watching config file for changes")
+
+	// if the file disappears or something, we can
+	// stop polling if the error lasts long enough
+	var lastErr time.Time
+	finalError := func(err error) bool {
+		if lastErr.IsZero() {
+			lastErr = time.Now()
+			return false
+		}
+		if time.Since(lastErr) > 30*time.Second {
+			logger().Error("giving up watching config file; too many errors",
+				zap.Error(err))
+			return true
+		}
+		return false
+	}
+
+	// begin poller
+	for range time.Tick(1 * time.Second) {
+		// get the file info
+		info, err := os.Stat(filename)
+		if err != nil {
+			if finalError(err) {
+				return
+			}
+			continue
+		}
+		lastErr = time.Time{} // no error, so clear any memory of one
+
+		// if it hasn't changed, nothing to do
+		if !info.ModTime().After(lastModified) {
+			continue
+		}
+
+		logger().Info("config file changed; reloading")
+
+		// remember this timestamp
+		lastModified = info.ModTime()
+
+		// load the contents of the file
+		config, _, err := loadConfig(filename, adapterName)
+		if err != nil {
+			logger().Error("unable to load latest config", zap.Error(err))
+			continue
+		}
+
+		// apply the updated config
+		err = caddy.Load(config, false)
+		if err != nil {
+			logger().Error("applying latest config", zap.Error(err))
+			continue
+		}
+	}
 }
 
 // Flags wraps a FlagSet so that typed values

context.go

@@ -21,7 +21,7 @@ import (
 	"log"
 	"reflect"
 
-	"github.com/mholt/certmagic"
+	"github.com/caddyserver/certmagic"
 	"go.uber.org/zap"
 )
 
@@ -377,16 +377,25 @@ func (ctx Context) loadModuleInline(moduleNameKey, moduleScope string, raw json.
 	return val, nil
 }
 
-// App returns the configured app named name. If no app with
-// that name is currently configured, a new empty one will be
-// instantiated. (The app module must still be registered.)
+// App returns the configured app named name. If that app has
+// not yet been loaded and provisioned, it will be immediately
+// loaded and provisioned. If no app with that name is
+// configured, a new empty one will be instantiated instead.
+// (The app module must still be registered.) This must not be
+// called during the Provision/Validate phase to reference a
+// module's own host app (since the parent app module is still
+// in the process of being provisioned, it is not yet ready).
 func (ctx Context) App(name string) (interface{}, error) {
 	if app, ok := ctx.cfg.apps[name]; ok {
 		return app, nil
 	}
-	modVal, err := ctx.LoadModuleByID(name, nil)
+	appRaw := ctx.cfg.AppsRaw[name]
+	modVal, err := ctx.LoadModuleByID(name, appRaw)
 	if err != nil {
-		return nil, fmt.Errorf("instantiating new module %s: %v", name, err)
+		return nil, fmt.Errorf("loading %s app module: %v", name, err)
+	}
+	if appRaw != nil {
+		ctx.cfg.AppsRaw[name] = nil // allow GC to deallocate
 	}
 	ctx.cfg.apps[name] = modVal.(App)
 	return modVal, nil

go.mod

@@ -3,36 +3,36 @@ module github.com/caddyserver/caddy/v2
 go 1.14
 
 require (
-	github.com/Masterminds/sprig/v3 v3.0.0
-	github.com/alecthomas/chroma v0.7.0
-	github.com/andybalholm/brotli v0.0.0-20190821151343-b60f0d972eeb
-	github.com/cenkalti/backoff/v3 v3.1.1 // indirect
+	github.com/Masterminds/sprig/v3 v3.0.2
+	github.com/alecthomas/chroma v0.7.2-0.20200305040604-4f3623dce67a
+	github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a
+	github.com/caddyserver/certmagic v0.10.11
+	github.com/cenkalti/backoff/v4 v4.0.2 // indirect
 	github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac
-	github.com/go-acme/lego/v3 v3.3.0
-	github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc
-	github.com/ilibs/json5 v1.0.1
-	github.com/imdario/mergo v0.3.8 // indirect
+	github.com/go-acme/lego/v3 v3.5.0
+	github.com/gogo/protobuf v1.3.1
+	github.com/google/cel-go v0.4.1
+	github.com/imdario/mergo v0.3.9 // indirect
 	github.com/jsternberg/zap-logfmt v1.2.0
-	github.com/klauspost/compress v1.8.6
-	github.com/klauspost/cpuid v1.2.2
-	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lucas-clemente/quic-go v0.14.4
-	github.com/mholt/certmagic v0.9.3
-	github.com/miekg/dns v1.1.25 // indirect
-	github.com/muhammadmuzzammil1998/jsonc v0.0.0-20190906142622-1265e9b150c6
+	github.com/klauspost/compress v1.10.4
+	github.com/klauspost/cpuid v1.2.3
+	github.com/lucas-clemente/quic-go v0.15.3
+	github.com/manifoldco/promptui v0.7.0 // indirect
+	github.com/miekg/dns v1.1.29 // indirect
 	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/naoina/toml v0.1.1
-	github.com/onsi/ginkgo v1.8.0 // indirect
-	github.com/onsi/gomega v1.5.0 // indirect
-	github.com/vulcand/oxy v1.0.0
-	github.com/yuin/goldmark v1.1.17
-	github.com/yuin/goldmark-highlighting v0.0.0-20191202084645-78f32c8dd6d5
-	go.uber.org/multierr v1.2.0 // indirect
-	go.uber.org/zap v1.10.0
-	golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413
-	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553
-	golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect
+	github.com/smallstep/certificates v0.14.1
+	github.com/smallstep/cli v0.14.1
+	github.com/smallstep/truststore v0.9.5
+	github.com/vulcand/oxy v1.1.0
+	github.com/yuin/goldmark v1.1.27
+	github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691
+	go.uber.org/zap v1.14.1
+	golang.org/x/crypto v0.0.0-20200406173513-056763e48d71
+	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e
+	golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa // indirect
+	google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/square/go-jose.v2 v2.4.1 // indirect
-	gopkg.in/yaml.v2 v2.2.2
+	gopkg.in/yaml.v2 v2.2.8
 )

listeners.go

@@ -254,49 +254,49 @@ type globalListener struct {
 	pc         net.PacketConn
 }
 
-// ParsedAddress contains the individual components
+// NetworkAddress contains the individual components
 // for a parsed network address of the form accepted
 // by ParseNetworkAddress(). Network should be a
 // network value accepted by Go's net package. Port
 // ranges are given by [StartPort, EndPort].
-type ParsedAddress struct {
+type NetworkAddress struct {
 	Network   string
 	Host      string
 	StartPort uint
 	EndPort   uint
 }
 
-// IsUnixNetwork returns true if pa.Network is
+// IsUnixNetwork returns true if na.Network is
 // unix, unixgram, or unixpacket.
-func (pa ParsedAddress) IsUnixNetwork() bool {
-	return isUnixNetwork(pa.Network)
+func (na NetworkAddress) IsUnixNetwork() bool {
+	return isUnixNetwork(na.Network)
 }
 
 // JoinHostPort is like net.JoinHostPort, but where the port
 // is StartPort + offset.
-func (pa ParsedAddress) JoinHostPort(offset uint) string {
-	if pa.IsUnixNetwork() {
-		return pa.Host
+func (na NetworkAddress) JoinHostPort(offset uint) string {
+	if na.IsUnixNetwork() {
+		return na.Host
 	}
-	return net.JoinHostPort(pa.Host, strconv.Itoa(int(pa.StartPort+offset)))
+	return net.JoinHostPort(na.Host, strconv.Itoa(int(na.StartPort+offset)))
 }
 
 // PortRangeSize returns how many ports are in
 // pa's port range. Port ranges are inclusive,
 // so the size is the difference of start and
 // end ports plus one.
-func (pa ParsedAddress) PortRangeSize() uint {
-	return (pa.EndPort - pa.StartPort) + 1
+func (na NetworkAddress) PortRangeSize() uint {
+	return (na.EndPort - na.StartPort) + 1
 }
 
 // String reconstructs the address string to the form expected
 // by ParseNetworkAddress().
-func (pa ParsedAddress) String() string {
-	port := strconv.FormatUint(uint64(pa.StartPort), 10)
-	if pa.StartPort != pa.EndPort {
-		port += "-" + strconv.FormatUint(uint64(pa.EndPort), 10)
+func (na NetworkAddress) String() string {
+	port := strconv.FormatUint(uint64(na.StartPort), 10)
+	if na.StartPort != na.EndPort {
+		port += "-" + strconv.FormatUint(uint64(na.EndPort), 10)
 	}
-	return JoinNetworkAddress(pa.Network, pa.Host, port)
+	return JoinNetworkAddress(na.Network, na.Host, port)
 }
 
 func isUnixNetwork(netw string) bool {
@@ -311,17 +311,17 @@ func isUnixNetwork(netw string) bool {
 //
 // Network addresses are distinct from URLs and do not
 // use URL syntax.
-func ParseNetworkAddress(addr string) (ParsedAddress, error) {
+func ParseNetworkAddress(addr string) (NetworkAddress, error) {
 	var host, port string
 	network, host, port, err := SplitNetworkAddress(addr)
 	if network == "" {
 		network = "tcp"
 	}
 	if err != nil {
-		return ParsedAddress{}, err
+		return NetworkAddress{}, err
 	}
 	if isUnixNetwork(network) {
-		return ParsedAddress{
+		return NetworkAddress{
 			Network: network,
 			Host:    host,
 		}, nil
@@ -333,19 +333,19 @@ func ParseNetworkAddress(addr string) (ParsedAddress, error) {
 	var start, end uint64
 	start, err = strconv.ParseUint(ports[0], 10, 16)
 	if err != nil {
-		return ParsedAddress{}, fmt.Errorf("invalid start port: %v", err)
+		return NetworkAddress{}, fmt.Errorf("invalid start port: %v", err)
 	}
 	end, err = strconv.ParseUint(ports[1], 10, 16)
 	if err != nil {
-		return ParsedAddress{}, fmt.Errorf("invalid end port: %v", err)
+		return NetworkAddress{}, fmt.Errorf("invalid end port: %v", err)
 	}
 	if end < start {
-		return ParsedAddress{}, fmt.Errorf("end port must not be less than start port")
+		return NetworkAddress{}, fmt.Errorf("end port must not be less than start port")
 	}
 	if (end - start) > maxPortSpan {
-		return ParsedAddress{}, fmt.Errorf("port range exceeds %d ports", maxPortSpan)
+		return NetworkAddress{}, fmt.Errorf("port range exceeds %d ports", maxPortSpan)
 	}
-	return ParsedAddress{
+	return NetworkAddress{
 		Network:   network,
 		Host:      host,
 		StartPort: uint(start),
@@ -386,6 +386,19 @@ func JoinNetworkAddress(network, host, port string) string {
 	return a
 }
 
+// ListenerWrapper is a type that wraps a listener
+// so it can modify the input listener's methods.
+// Modules that implement this interface are found
+// in the caddy.listeners namespace. Usually, to
+// wrap a listener, you will define your own struct
+// type that embeds the input listener, then
+// implement your own methods that you want to wrap,
+// calling the underlying listener's methods where
+// appropriate.
+type ListenerWrapper interface {
+	WrapListener(net.Listener) net.Listener
+}
+
 var (
 	listeners   = make(map[string]*globalListener)
 	listenersMu sync.Mutex

listeners_fuzz.go

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 // +build gofuzz
-// +build gofuzz_libfuzzer
 
 package caddy
 

listeners_test.go

@@ -153,7 +153,7 @@ func TestJoinNetworkAddress(t *testing.T) {
 func TestParseNetworkAddress(t *testing.T) {
 	for i, tc := range []struct {
 		input      string
-		expectAddr ParsedAddress
+		expectAddr NetworkAddress
 		expectErr  bool
 	}{
 		{
@@ -166,7 +166,7 @@ func TestParseNetworkAddress(t *testing.T) {
 		},
 		{
 			input: ":1234",
-			expectAddr: ParsedAddress{
+			expectAddr: NetworkAddress{
 				Network:   "tcp",
 				Host:      "",
 				StartPort: 1234,
@@ -175,7 +175,7 @@ func TestParseNetworkAddress(t *testing.T) {
 		},
 		{
 			input: "tcp/:1234",
-			expectAddr: ParsedAddress{
+			expectAddr: NetworkAddress{
 				Network:   "tcp",
 				Host:      "",
 				StartPort: 1234,
@@ -184,7 +184,7 @@ func TestParseNetworkAddress(t *testing.T) {
 		},
 		{
 			input: "tcp6/:1234",
-			expectAddr: ParsedAddress{
+			expectAddr: NetworkAddress{
 				Network:   "tcp6",
 				Host:      "",
 				StartPort: 1234,
@@ -193,7 +193,7 @@ func TestParseNetworkAddress(t *testing.T) {
 		},
 		{
 			input: "tcp4/localhost:1234",
-			expectAddr: ParsedAddress{
+			expectAddr: NetworkAddress{
 				Network:   "tcp4",
 				Host:      "localhost",
 				StartPort: 1234,
@@ -202,14 +202,14 @@ func TestParseNetworkAddress(t *testing.T) {
 		},
 		{
 			input: "unix//foo/bar",
-			expectAddr: ParsedAddress{
+			expectAddr: NetworkAddress{
 				Network: "unix",
 				Host:    "/foo/bar",
 			},
 		},
 		{
 			input: "localhost:1234-1234",
-			expectAddr: ParsedAddress{
+			expectAddr: NetworkAddress{
 				Network:   "tcp",
 				Host:      "localhost",
 				StartPort: 1234,
@@ -222,7 +222,7 @@ func TestParseNetworkAddress(t *testing.T) {
 		},
 		{
 			input: "localhost:0",
-			expectAddr: ParsedAddress{
+			expectAddr: NetworkAddress{
 				Network:   "tcp",
 				Host:      "localhost",
 				StartPort: 0,
@@ -253,12 +253,12 @@ func TestParseNetworkAddress(t *testing.T) {
 
 func TestJoinHostPort(t *testing.T) {
 	for i, tc := range []struct {
-		pa     ParsedAddress
+		pa     NetworkAddress
 		offset uint
 		expect string
 	}{
 		{
-			pa: ParsedAddress{
+			pa: NetworkAddress{
 				Network:   "tcp",
 				Host:      "localhost",
 				StartPort: 1234,
@@ -267,7 +267,7 @@ func TestJoinHostPort(t *testing.T) {
 			expect: "localhost:1234",
 		},
 		{
-			pa: ParsedAddress{
+			pa: NetworkAddress{
 				Network:   "tcp",
 				Host:      "localhost",
 				StartPort: 1234,
@@ -276,7 +276,7 @@ func TestJoinHostPort(t *testing.T) {
 			expect: "localhost:1234",
 		},
 		{
-			pa: ParsedAddress{
+			pa: NetworkAddress{
 				Network:   "tcp",
 				Host:      "localhost",
 				StartPort: 1234,
@@ -286,7 +286,7 @@ func TestJoinHostPort(t *testing.T) {
 			expect: "localhost:1235",
 		},
 		{
-			pa: ParsedAddress{
+			pa: NetworkAddress{
 				Network: "unix",
 				Host:    "/run/php/php7.3-fpm.sock",
 			},

logging.go

@@ -27,6 +27,7 @@ import (
 
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
+	"golang.org/x/crypto/ssh/terminal"
 )
 
 func init() {
@@ -35,12 +36,14 @@ func init() {
 	RegisterModule(DiscardWriter{})
 }
 
-// Logging facilitates logging within Caddy.
+// Logging facilitates logging within Caddy. The default log is
+// called "default" and you can customize it. You can also define
+// additional logs.
 //
 // By default, all logs at INFO level and higher are written to
 // standard error ("stderr" writer) in a human-readable format
-// ("console" encoder). The default log is called "default" and
-// you can customize it. You can also define additional logs.
+// ("console" encoder if stdout is an interactive terminal, "json"
+// encoder otherwise).
 //
 // All defined logs accept all log entries by default, but you
 // can filter by level and module/logger names. A logger's name
@@ -50,10 +53,10 @@ func init() {
 // "http.handlers", because all HTTP handler module names have
 // that prefix.
 //
-// Caddy logs (except the sink) are mostly zero-allocation, so
-// they are very high-performing in terms of memory and CPU time.
-// Enabling sampling can further increase throughput on extremely
-// high-load servers.
+// Caddy logs (except the sink) are zero-allocation, so they are
+// very high-performing in terms of memory and CPU time. Enabling
+// sampling can further increase throughput on extremely high-load
+// servers.
 type Logging struct {
 	// Sink is the destination for all unstructured logs emitted
 	// from Go's standard library logger. These logs are common
@@ -214,7 +217,7 @@ func (logging *Logging) Logger(mod Module) *zap.Logger {
 
 	multiCore := zapcore.NewTee(cores...)
 
-	return zap.New(multiCore).Named(string(modID))
+	return zap.New(multiCore).Named(modID)
 }
 
 // openWriter opens a writer using opener, and returns true if
@@ -393,17 +396,6 @@ func (cl *CustomLog) provision(ctx Context, logging *Logging) error {
 		}
 	}
 
-	if cl.EncoderRaw != nil {
-		mod, err := ctx.LoadModule(cl, "EncoderRaw")
-		if err != nil {
-			return fmt.Errorf("loading log encoder module: %v", err)
-		}
-		cl.encoder = mod.(zapcore.Encoder)
-	}
-	if cl.encoder == nil {
-		cl.encoder = newDefaultProductionLogEncoder()
-	}
-
 	if cl.WriterRaw != nil {
 		mod, err := ctx.LoadModule(cl, "WriterRaw")
 		if err != nil {
@@ -420,6 +412,24 @@ func (cl *CustomLog) provision(ctx Context, logging *Logging) error {
 		return fmt.Errorf("opening log writer using %#v: %v", cl.writerOpener, err)
 	}
 
+	if cl.EncoderRaw != nil {
+		mod, err := ctx.LoadModule(cl, "EncoderRaw")
+		if err != nil {
+			return fmt.Errorf("loading log encoder module: %v", err)
+		}
+		cl.encoder = mod.(zapcore.Encoder)
+	}
+	if cl.encoder == nil {
+		// only allow colorized output if this log is going to stdout or stderr
+		var colorize bool
+		switch cl.writerOpener.(type) {
+		case StdoutWriter, StderrWriter,
+			*StdoutWriter, *StderrWriter:
+			colorize = true
+		}
+		cl.encoder = newDefaultProductionLogEncoder(colorize)
+	}
+
 	cl.buildCore()
 
 	return nil
@@ -455,7 +465,7 @@ func (cl *CustomLog) buildCore() {
 }
 
 func (cl *CustomLog) matchesModule(moduleID string) bool {
-	return cl.loggerAllowed(string(moduleID), true)
+	return cl.loggerAllowed(moduleID, true)
 }
 
 // loggerAllowed returns true if name is allowed to emit
@@ -647,7 +657,7 @@ func newDefaultProductionLog() (*defaultCustomLog, error) {
 	if err != nil {
 		return nil, err
 	}
-	cl.encoder = newDefaultProductionLogEncoder()
+	cl.encoder = newDefaultProductionLogEncoder(true)
 	cl.levelEnabler = zapcore.InfoLevel
 
 	cl.buildCore()
@@ -658,13 +668,19 @@ func newDefaultProductionLog() (*defaultCustomLog, error) {
 	}, nil
 }
 
-func newDefaultProductionLogEncoder() zapcore.Encoder {
+func newDefaultProductionLogEncoder(colorize bool) zapcore.Encoder {
 	encCfg := zap.NewProductionEncoderConfig()
-	encCfg.EncodeLevel = zapcore.CapitalColorLevelEncoder
-	encCfg.EncodeTime = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) {
-		encoder.AppendString(ts.UTC().Format("2006/01/02 15:04:05.000"))
+	if terminal.IsTerminal(int(os.Stdout.Fd())) {
+		// if interactive terminal, make output more human-readable by default
+		encCfg.EncodeTime = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) {
+			encoder.AppendString(ts.UTC().Format("2006/01/02 15:04:05.000"))
+		}
+		if colorize {
+			encCfg.EncodeLevel = zapcore.CapitalColorLevelEncoder
+		}
+		return zapcore.NewConsoleEncoder(encCfg)
 	}
-	return zapcore.NewConsoleEncoder(encCfg)
+	return zapcore.NewJSONEncoder(encCfg)
 }
 
 // Log returns the current default logger.

modules.go

@@ -210,7 +210,7 @@ func GetModules(scope string) []ModuleInfo {
 	var mods []ModuleInfo
 iterateModules:
 	for id, m := range modules {
-		modParts := strings.Split(string(id), ".")
+		modParts := strings.Split(id, ".")
 
 		// match only the next level of nesting
 		if len(modParts) != len(scopeParts)+1 {
@@ -241,9 +241,9 @@ func Modules() []string {
 	modulesMu.RLock()
 	defer modulesMu.RUnlock()
 
-	var names []string
+	names := make([]string, 0, len(modules))
 	for name := range modules {
-		names = append(names, string(name))
+		names = append(names, name)
 	}
 
 	sort.Strings(names)
@@ -294,8 +294,8 @@ type Provisioner interface {
 // Validator is implemented by modules which can verify that their
 // configurations are valid. This method will be called after
 // Provision() (if implemented). Validation should always be fast
-// (imperceptible running time) and an error should be returned only
-// if the value's configuration is invalid.
+// (imperceptible running time) and an error must be returned if
+// the module's configuration is invalid.
 type Validator interface {
 	Validate() error
 }

modules/caddyhttp/app.go

@@ -0,0 +1,442 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddyhttp
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddytls"
+	"github.com/lucas-clemente/quic-go/http3"
+	"go.uber.org/zap"
+)
+
+func init() {
+	err := caddy.RegisterModule(App{})
+	if err != nil {
+		caddy.Log().Fatal(err.Error())
+	}
+}
+
+// App is a robust, production-ready HTTP server.
+//
+// HTTPS is enabled by default if host matchers with qualifying names are used
+// in any of routes; certificates are automatically provisioned and renewed.
+// Additionally, automatic HTTPS will also enable HTTPS for servers that listen
+// only on the HTTPS port but which do not have any TLS connection policies
+// defined by adding a good, default TLS connection policy.
+//
+// In HTTP routes, additional placeholders are available (replace any `*`):
+//
+// Placeholder | Description
+// ------------|---------------
+// `{http.request.cookie.*}` | HTTP request cookie
+// `{http.request.header.*}` | Specific request header field
+// `{http.request.host.labels.*}` | Request host labels (0-based from right); e.g. for foo.example.com: 0=com, 1=example, 2=foo
+// `{http.request.host}` | The host part of the request's Host header
+// `{http.request.hostport}` | The host and port from the request's Host header
+// `{http.request.method}` | The request method
+// `{http.request.orig_method}` | The request's original method
+// `{http.request.orig_uri.path.dir}` | The request's original directory
+// `{http.request.orig_uri.path.file}` | The request's original filename
+// `{http.request.orig_uri.path}` | The request's original path
+// `{http.request.orig_uri.query}` | The request's original query string (without `?`)
+// `{http.request.orig_uri}` | The request's original URI
+// `{http.request.port}` | The port part of the request's Host header
+// `{http.request.proto}` | The protocol of the request
+// `{http.request.remote.host}` | The host part of the remote client's address
+// `{http.request.remote.port}` | The port part of the remote client's address
+// `{http.request.remote}` | The address of the remote client
+// `{http.request.scheme}` | The request scheme
+// `{http.request.tls.version}` | The TLS version name
+// `{http.request.tls.cipher_suite}` | The TLS cipher suite
+// `{http.request.tls.resumed}` | The TLS connection resumed a previous connection
+// `{http.request.tls.proto}` | The negotiated next protocol
+// `{http.request.tls.proto_mutual}` | The negotiated next protocol was advertised by the server
+// `{http.request.tls.server_name}` | The server name requested by the client, if any
+// `{http.request.tls.client.fingerprint}` | The SHA256 checksum of the client certificate
+// `{http.request.tls.client.issuer}` | The issuer DN of the client certificate
+// `{http.request.tls.client.serial}` | The serial number of the client certificate
+// `{http.request.tls.client.subject}` | The subject DN of the client certificate
+// `{http.request.uri.path.*}` | Parts of the path, split by `/` (0-based from left)
+// `{http.request.uri.path.dir}` | The directory, excluding leaf filename
+// `{http.request.uri.path.file}` | The filename of the path, excluding directory
+// `{http.request.uri.path}` | The path component of the request URI
+// `{http.request.uri.query.*}` | Individual query string value
+// `{http.request.uri.query}` | The query string (without `?`)
+// `{http.request.uri}` | The full request URI
+// `{http.response.header.*}` | Specific response header field
+// `{http.vars.*}` | Custom variables in the HTTP handler chain
+type App struct {
+	// HTTPPort specifies the port to use for HTTP (as opposed to HTTPS),
+	// which is used when setting up HTTP->HTTPS redirects or ACME HTTP
+	// challenge solvers. Default: 80.
+	HTTPPort int `json:"http_port,omitempty"`
+
+	// HTTPSPort specifies the port to use for HTTPS, which is used when
+	// solving the ACME TLS-ALPN challenges, or whenever HTTPS is needed
+	// but no specific port number is given. Default: 443.
+	HTTPSPort int `json:"https_port,omitempty"`
+
+	// GracePeriod is how long to wait for active connections when shutting
+	// down the server. Once the grace period is over, connections will
+	// be forcefully closed.
+	GracePeriod caddy.Duration `json:"grace_period,omitempty"`
+
+	// Servers is the list of servers, keyed by arbitrary names chosen
+	// at your discretion for your own convenience; the keys do not
+	// affect functionality.
+	Servers map[string]*Server `json:"servers,omitempty"`
+
+	servers     []*http.Server
+	h3servers   []*http3.Server
+	h3listeners []net.PacketConn
+
+	ctx    caddy.Context
+	logger *zap.Logger
+	tlsApp *caddytls.TLS
+
+	// used temporarily between phases 1 and 2 of auto HTTPS
+	allCertDomains []string
+}
+
+// CaddyModule returns the Caddy module information.
+func (App) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "http",
+		New: func() caddy.Module { return new(App) },
+	}
+}
+
+// Provision sets up the app.
+func (app *App) Provision(ctx caddy.Context) error {
+	// store some references
+	tlsAppIface, err := ctx.App("tls")
+	if err != nil {
+		return fmt.Errorf("getting tls app: %v", err)
+	}
+	app.tlsApp = tlsAppIface.(*caddytls.TLS)
+	app.ctx = ctx
+	app.logger = ctx.Logger(app)
+
+	repl := caddy.NewReplacer()
+
+	// this provisions the matchers for each route,
+	// and prepares auto HTTP->HTTPS redirects, and
+	// is required before we provision each server
+	err = app.automaticHTTPSPhase1(ctx, repl)
+	if err != nil {
+		return err
+	}
+
+	// prepare each server
+	for srvName, srv := range app.Servers {
+		srv.tlsApp = app.tlsApp
+		srv.logger = app.logger.Named("log")
+		srv.errorLogger = app.logger.Named("log.error")
+
+		// only enable access logs if configured
+		if srv.Logs != nil {
+			srv.accessLogger = app.logger.Named("log.access")
+		}
+
+		// if not explicitly configured by the user, disallow TLS
+		// client auth bypass (domain fronting) which could
+		// otherwise be exploited by sending an unprotected SNI
+		// value during a TLS handshake, then putting a protected
+		// domain in the Host header after establishing connection;
+		// this is a safe default, but we allow users to override
+		// it for example in the case of running a proxy where
+		// domain fronting is desired and access is not restricted
+		// based on hostname
+		if srv.StrictSNIHost == nil && srv.hasTLSClientAuth() {
+			app.logger.Info("enabling strict SNI-Host matching because TLS client auth is configured",
+				zap.String("server_name", srvName),
+			)
+			trueBool := true
+			srv.StrictSNIHost = &trueBool
+		}
+
+		// process each listener address
+		for i := range srv.Listen {
+			lnOut, err := repl.ReplaceOrErr(srv.Listen[i], true, true)
+			if err != nil {
+				return fmt.Errorf("server %s, listener %d: %v",
+					srvName, i, err)
+			}
+			srv.Listen[i] = lnOut
+		}
+
+		// set up each listener modifier
+		if srv.ListenerWrappersRaw != nil {
+			vals, err := ctx.LoadModule(srv, "ListenerWrappersRaw")
+			if err != nil {
+				return fmt.Errorf("loading listener wrapper modules: %v", err)
+			}
+			var hasTLSPlaceholder bool
+			for i, val := range vals.([]interface{}) {
+				if _, ok := val.(*tlsPlaceholderWrapper); ok {
+					if i == 0 {
+						// putting the tls placeholder wrapper first is nonsensical because
+						// that is the default, implicit setting: without it, all wrappers
+						// will go after the TLS listener anyway
+						return fmt.Errorf("it is unnecessary to specify the TLS listener wrapper in the first position because that is the default")
+					}
+					if hasTLSPlaceholder {
+						return fmt.Errorf("TLS listener wrapper can only be specified once")
+					}
+					hasTLSPlaceholder = true
+				}
+				srv.listenerWrappers = append(srv.listenerWrappers, val.(caddy.ListenerWrapper))
+			}
+			// if any wrappers were configured but the TLS placeholder wrapper is
+			// absent, prepend it so all defined wrappers come after the TLS
+			// handshake; this simplifies logic when starting the server, since we
+			// can simply assume the TLS placeholder will always be there
+			if !hasTLSPlaceholder && len(srv.listenerWrappers) > 0 {
+				srv.listenerWrappers = append([]caddy.ListenerWrapper{new(tlsPlaceholderWrapper)}, srv.listenerWrappers...)
+			}
+		}
+
+		// pre-compile the primary handler chain, and be sure to wrap it in our
+		// route handler so that important security checks are done, etc.
+		primaryRoute := emptyHandler
+		if srv.Routes != nil {
+			err := srv.Routes.ProvisionHandlers(ctx)
+			if err != nil {
+				return fmt.Errorf("server %s: setting up route handlers: %v", srvName, err)
+			}
+			primaryRoute = srv.Routes.Compile(emptyHandler)
+		}
+		srv.primaryHandlerChain = srv.wrapPrimaryRoute(primaryRoute)
+
+		// pre-compile the error handler chain
+		if srv.Errors != nil {
+			err := srv.Errors.Routes.Provision(ctx)
+			if err != nil {
+				return fmt.Errorf("server %s: setting up server error handling routes: %v", srvName, err)
+			}
+			srv.errorHandlerChain = srv.Errors.Routes.Compile(errorEmptyHandler)
+		}
+
+		// prepare the TLS connection policies
+		err = srv.TLSConnPolicies.Provision(ctx)
+		if err != nil {
+			return fmt.Errorf("server %s: setting up TLS connection policies: %v", srvName, err)
+		}
+	}
+
+	return nil
+}
+
+// Validate ensures the app's configuration is valid.
+func (app *App) Validate() error {
+	// each server must use distinct listener addresses
+	lnAddrs := make(map[string]string)
+	for srvName, srv := range app.Servers {
+		for _, addr := range srv.Listen {
+			listenAddr, err := caddy.ParseNetworkAddress(addr)
+			if err != nil {
+				return fmt.Errorf("invalid listener address '%s': %v", addr, err)
+			}
+			// check that every address in the port range is unique to this server;
+			// we do not use <= here because PortRangeSize() adds 1 to EndPort for us
+			for i := uint(0); i < listenAddr.PortRangeSize(); i++ {
+				addr := caddy.JoinNetworkAddress(listenAddr.Network, listenAddr.Host, strconv.Itoa(int(listenAddr.StartPort+i)))
+				if sn, ok := lnAddrs[addr]; ok {
+					return fmt.Errorf("server %s: listener address repeated: %s (already claimed by server '%s')", srvName, addr, sn)
+				}
+				lnAddrs[addr] = srvName
+			}
+		}
+	}
+
+	return nil
+}
+
+// Start runs the app. It finishes automatic HTTPS if enabled,
+// including management of certificates.
+func (app *App) Start() error {
+	for srvName, srv := range app.Servers {
+		s := &http.Server{
+			ReadTimeout:       time.Duration(srv.ReadTimeout),
+			ReadHeaderTimeout: time.Duration(srv.ReadHeaderTimeout),
+			WriteTimeout:      time.Duration(srv.WriteTimeout),
+			IdleTimeout:       time.Duration(srv.IdleTimeout),
+			MaxHeaderBytes:    srv.MaxHeaderBytes,
+			Handler:           srv,
+		}
+
+		for _, lnAddr := range srv.Listen {
+			listenAddr, err := caddy.ParseNetworkAddress(lnAddr)
+			if err != nil {
+				return fmt.Errorf("%s: parsing listen address '%s': %v", srvName, lnAddr, err)
+			}
+			for portOffset := uint(0); portOffset < listenAddr.PortRangeSize(); portOffset++ {
+				// create the listener for this socket
+				hostport := listenAddr.JoinHostPort(portOffset)
+				ln, err := caddy.Listen(listenAddr.Network, hostport)
+				if err != nil {
+					return fmt.Errorf("%s: listening on %s: %v", listenAddr.Network, hostport, err)
+				}
+
+				// wrap listener before TLS (up to the TLS placeholder wrapper)
+				var lnWrapperIdx int
+				for i, lnWrapper := range srv.listenerWrappers {
+					if _, ok := lnWrapper.(*tlsPlaceholderWrapper); ok {
+						lnWrapperIdx = i + 1 // mark the next wrapper's spot
+						break
+					}
+					ln = lnWrapper.WrapListener(ln)
+				}
+
+				// enable TLS if there is a policy and if this is not the HTTP port
+				useTLS := len(srv.TLSConnPolicies) > 0 && int(listenAddr.StartPort+portOffset) != app.httpPort()
+				if useTLS {
+					// create TLS listener
+					tlsCfg := srv.TLSConnPolicies.TLSConfig(app.ctx)
+					ln = tls.NewListener(ln, tlsCfg)
+
+					/////////
+					// TODO: HTTP/3 support is experimental for now
+					if srv.ExperimentalHTTP3 {
+						app.logger.Info("enabling experimental HTTP/3 listener",
+							zap.String("addr", hostport),
+						)
+						h3ln, err := caddy.ListenPacket("udp", hostport)
+						if err != nil {
+							return fmt.Errorf("getting HTTP/3 UDP listener: %v", err)
+						}
+						h3srv := &http3.Server{
+							Server: &http.Server{
+								Addr:      hostport,
+								Handler:   srv,
+								TLSConfig: tlsCfg,
+							},
+						}
+						go h3srv.Serve(h3ln)
+						app.h3servers = append(app.h3servers, h3srv)
+						app.h3listeners = append(app.h3listeners, h3ln)
+						srv.h3server = h3srv
+					}
+					/////////
+				}
+
+				// finish wrapping listener where we left off before TLS
+				for i := lnWrapperIdx; i < len(srv.listenerWrappers); i++ {
+					ln = srv.listenerWrappers[i].WrapListener(ln)
+				}
+
+				// if binding to port 0, the OS chooses a port for us;
+				// but the user won't know the port unless we print it
+				if listenAddr.StartPort == 0 && listenAddr.EndPort == 0 {
+					app.logger.Info("port 0 listener",
+						zap.String("input_address", lnAddr),
+						zap.String("actual_address", ln.Addr().String()),
+					)
+				}
+
+				app.logger.Debug("starting server loop",
+					zap.String("address", ln.Addr().String()),
+					zap.Bool("http3", srv.ExperimentalHTTP3),
+					zap.Bool("tls", useTLS),
+				)
+
+				go s.Serve(ln)
+				app.servers = append(app.servers, s)
+			}
+		}
+	}
+
+	// finish automatic HTTPS by finally beginning
+	// certificate management
+	err := app.automaticHTTPSPhase2()
+	if err != nil {
+		return fmt.Errorf("finalizing automatic HTTPS: %v", err)
+	}
+
+	return nil
+}
+
+// Stop gracefully shuts down the HTTP server.
+func (app *App) Stop() error {
+	ctx := context.Background()
+	if app.GracePeriod > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, time.Duration(app.GracePeriod))
+		defer cancel()
+	}
+	for _, s := range app.servers {
+		err := s.Shutdown(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	// close the http3 servers; it's unclear whether the bug reported in
+	// https://github.com/caddyserver/caddy/pull/2727#issuecomment-526856566
+	// was ever truly fixed, since it seemed racey/nondeterministic; but
+	// recent tests in 2020 were unable to replicate the issue again after
+	// repeated attempts (the bug manifested after a config reload; i.e.
+	// reusing a http3 server or listener was problematic), but it seems
+	// to be working fine now
+	for _, s := range app.h3servers {
+		// TODO: CloseGracefully, once implemented upstream
+		// (see https://github.com/lucas-clemente/quic-go/issues/2103)
+		err := s.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	// closing an http3.Server does not close their underlying listeners
+	// since apparently the listener can be used both by servers and
+	// clients at the same time; so we need to manually call Close()
+	// on the underlying h3 listeners (see lucas-clemente/quic-go#2103)
+	for _, pc := range app.h3listeners {
+		err := pc.Close()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (app *App) httpPort() int {
+	if app.HTTPPort == 0 {
+		return DefaultHTTPPort
+	}
+	return app.HTTPPort
+}
+
+func (app *App) httpsPort() int {
+	if app.HTTPSPort == 0 {
+		return DefaultHTTPSPort
+	}
+	return app.HTTPSPort
+}
+
+// Interface guards
+var (
+	_ caddy.App         = (*App)(nil)
+	_ caddy.Provisioner = (*App)(nil)
+	_ caddy.Validator   = (*App)(nil)
+)

modules/caddyhttp/autohttps.go

@@ -1,3 +1,17 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package caddyhttp
 
 import (
@@ -8,7 +22,7 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
-	"github.com/mholt/certmagic"
+	"github.com/caddyserver/certmagic"
 	"go.uber.org/zap"
 )
 
@@ -42,12 +56,10 @@ type AutoHTTPSConfig struct {
 	// enabled. To force automated certificate management
 	// regardless of loaded certificates, set this to true.
 	IgnoreLoadedCerts bool `json:"ignore_loaded_certificates,omitempty"`
-
-	domainSet map[string]struct{}
 }
 
 // Skipped returns true if name is in skipSlice, which
-// should be one of the Skip* fields on ahc.
+// should be either the Skip or SkipCerts field on ahc.
 func (ahc AutoHTTPSConfig) Skipped(name string, skipSlice []string) bool {
 	for _, n := range skipSlice {
 		if name == n {
@@ -64,9 +76,13 @@ func (ahc AutoHTTPSConfig) Skipped(name string, skipSlice []string) bool {
 // even servers to the app, which still need to be set up with the
 // rest of them during provisioning.
 func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) error {
-	// this map will store associations of HTTP listener
-	// addresses to the routes that do HTTP->HTTPS redirects
-	lnAddrRedirRoutes := make(map[string]Route)
+	// this map acts as a set to store the domain names
+	// for which we will manage certificates automatically
+	uniqueDomainsForCerts := make(map[string]struct{})
+
+	// this maps domain names for automatic HTTP->HTTPS
+	// redirects to their destination server address
+	redirDomains := make(map[string]caddy.NetworkAddress)
 
 	for srvName, srv := range app.Servers {
 		// as a prerequisite, provision route matchers; this is
@@ -99,10 +115,6 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 			continue
 		}
 
-		defaultConnPolicies := caddytls.ConnectionPolicies{
-			&caddytls.ConnectionPolicy{ALPN: defaultALPN},
-		}
-
 		// if all listeners are on the HTTPS port, make sure
 		// there is at least one TLS connection policy; it
 		// should be obvious that they want to use TLS without
@@ -113,11 +125,11 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 				zap.String("server_name", srvName),
 				zap.Int("https_port", app.httpsPort()),
 			)
-			srv.TLSConnPolicies = defaultConnPolicies
+			srv.TLSConnPolicies = caddytls.ConnectionPolicies{new(caddytls.ConnectionPolicy)}
 		}
 
-		// find all qualifying domain names in this server
-		srv.AutoHTTPS.domainSet = make(map[string]struct{})
+		// find all qualifying domain names (deduplicated) in this server
+		serverDomainSet := make(map[string]struct{})
 		for routeIdx, route := range srv.Routes {
 			for matcherSetIdx, matcherSet := range route.MatcherSets {
 				for matcherIdx, m := range matcherSet {
@@ -129,9 +141,8 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 								return fmt.Errorf("%s: route %d, matcher set %d, matcher %d, host matcher %d: %v",
 									srvName, routeIdx, matcherSetIdx, matcherIdx, hostMatcherIdx, err)
 							}
-							if certmagic.HostQualifies(d) &&
-								!srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.Skip) {
-								srv.AutoHTTPS.domainSet[d] = struct{}{}
+							if !srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.Skip) {
+								serverDomainSet[d] = struct{}{}
 							}
 						}
 					}
@@ -141,13 +152,42 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 
 		// nothing more to do here if there are no
 		// domains that qualify for automatic HTTPS
-		if len(srv.AutoHTTPS.domainSet) == 0 {
+		if len(serverDomainSet) == 0 {
 			continue
 		}
 
+		// for all the hostnames we found, filter them so we have
+		// a deduplicated list of names for which to obtain certs
+		for d := range serverDomainSet {
+			if certmagic.SubjectQualifiesForCert(d) &&
+				!srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.SkipCerts) {
+				// if a certificate for this name is already loaded,
+				// don't obtain another one for it, unless we are
+				// supposed to ignore loaded certificates
+				if !srv.AutoHTTPS.IgnoreLoadedCerts &&
+					len(app.tlsApp.AllMatchingCertificates(d)) > 0 {
+					app.logger.Info("skipping automatic certificate management because one or more matching certificates are already loaded",
+						zap.String("domain", d),
+						zap.String("server_name", srvName),
+					)
+					continue
+				}
+
+				// most clients don't accept wildcards like *.tld... we
+				// can handle that, but as a courtesy, warn the user
+				if strings.Contains(d, "*") &&
+					strings.Count(strings.Trim(d, "."), ".") == 1 {
+					app.logger.Warn("most clients do not trust second-level wildcard certificates (*.tld)",
+						zap.String("domain", d))
+				}
+
+				uniqueDomainsForCerts[d] = struct{}{}
+			}
+		}
+
 		// tell the server to use TLS if it is not already doing so
 		if srv.TLSConnPolicies == nil {
-			srv.TLSConnPolicies = defaultConnPolicies
+			srv.TLSConnPolicies = caddytls.ConnectionPolicies{new(caddytls.ConnectionPolicy)}
 		}
 
 		// nothing left to do if auto redirects are disabled
@@ -161,125 +201,388 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 
 		// create HTTP->HTTPS redirects
 		for _, addr := range srv.Listen {
-			netw, host, port, err := caddy.SplitNetworkAddress(addr)
+			// figure out the address we will redirect to...
+			addr, err := caddy.ParseNetworkAddress(addr)
 			if err != nil {
 				return fmt.Errorf("%s: invalid listener address: %v", srvName, addr)
 			}
 
-			if parts := strings.SplitN(port, "-", 2); len(parts) == 2 {
-				port = parts[0]
-			}
-			redirTo := "https://{http.request.host}"
-
-			if port != strconv.Itoa(app.httpsPort()) {
-				redirTo += ":" + port
-			}
-			redirTo += "{http.request.uri}"
-
-			// build the plaintext HTTP variant of this address
-			httpRedirLnAddr := caddy.JoinNetworkAddress(netw, host, strconv.Itoa(app.httpPort()))
-
-			// build the matcher set for this redirect route
-			// (note that we happen to bypass Provision and
-			// Validate steps for these matcher modules)
-			matcherSet := MatcherSet{MatchProtocol("http")}
-			if len(srv.AutoHTTPS.Skip) > 0 {
-				matcherSet = append(matcherSet, MatchNegate{
-					Matchers: MatcherSet{MatchHost(srv.AutoHTTPS.Skip)},
-				})
-			}
-
-			// create the route that does the redirect and associate
-			// it with the listener address it will be served from
-			// (note that we happen to bypass any Provision or Validate
-			// steps on the handler modules created here)
-			lnAddrRedirRoutes[httpRedirLnAddr] = Route{
-				MatcherSets: []MatcherSet{matcherSet},
-				Handlers: []MiddlewareHandler{
-					StaticResponse{
-						StatusCode: WeakString(strconv.Itoa(http.StatusPermanentRedirect)),
-						Headers: http.Header{
-							"Location":   []string{redirTo},
-							"Connection": []string{"close"},
-						},
-						Close: true,
-					},
-				},
+			// ...and associate it with each domain in this server
+			for d := range serverDomainSet {
+				// if this domain is used on more than one HTTPS-enabled
+				// port, we'll have to choose one, so prefer the HTTPS port
+				if _, ok := redirDomains[d]; !ok ||
+					addr.StartPort == uint(app.httpsPort()) {
+					redirDomains[d] = addr
+				}
 			}
 		}
 	}
 
-	// if there are HTTP->HTTPS redirects to add, do so now
-	if len(lnAddrRedirRoutes) == 0 {
+	// we now have a list of all the unique names for which we need certs;
+	// turn the set into a slice so that phase 2 can use it
+	app.allCertDomains = make([]string, 0, len(uniqueDomainsForCerts))
+	var internal, external []string
+uniqueDomainsLoop:
+	for d := range uniqueDomainsForCerts {
+		// whether or not there is already an automation policy for this
+		// name, we should add it to the list to manage a cert for it
+		app.allCertDomains = append(app.allCertDomains, d)
+
+		// some names we've found might already have automation policies
+		// explicitly specified for them; we should exclude those from
+		// our hidden/implicit policy, since applying a name to more than
+		// one automation policy would be confusing and an error
+		if app.tlsApp.Automation != nil {
+			for _, ap := range app.tlsApp.Automation.Policies {
+				for _, apHost := range ap.Subjects {
+					if apHost == d {
+						continue uniqueDomainsLoop
+					}
+				}
+			}
+		}
+
+		// if no automation policy exists for the name yet, we
+		// will associate it with an implicit one
+		if certmagic.SubjectQualifiesForPublicCert(d) {
+			external = append(external, d)
+		} else {
+			internal = append(internal, d)
+		}
+	}
+
+	// ensure there is an automation policy to handle these certs
+	err := app.createAutomationPolicies(ctx, external, internal)
+	if err != nil {
+		return err
+	}
+
+	// we're done if there are no HTTP->HTTPS redirects to add
+	if len(redirDomains) == 0 {
 		return nil
 	}
 
-	var redirServerAddrs []string
+	// we need to reduce the mapping, i.e. group domains by address
+	// since new routes are appended to servers by their address
+	domainsByAddr := make(map[string][]string)
+	for domain, addr := range redirDomains {
+		addrStr := addr.String()
+		domainsByAddr[addrStr] = append(domainsByAddr[addrStr], domain)
+	}
+
+	// these keep track of the redirect server address(es)
+	// and the routes for those servers which actually
+	// respond with the redirects
+	redirServerAddrs := make(map[string]struct{})
 	var redirRoutes RouteList
 
-	// for each redirect listener, see if there's already a
-	// server configured to listen on that exact address; if so,
-	// simply add the redirect route to the end of its route
-	// list; otherwise, we'll create a new server for all the
-	// listener addresses that are unused and serve the
-	// remaining redirects from it
-redirRoutesLoop:
-	for addr, redirRoute := range lnAddrRedirRoutes {
+	redirServers := make(map[string][]Route)
+
+	for addrStr, domains := range domainsByAddr {
+		// build the matcher set for this redirect route
+		// (note that we happen to bypass Provision and
+		// Validate steps for these matcher modules)
+		matcherSet := MatcherSet{
+			MatchProtocol("http"),
+			MatchHost(domains),
+		}
+
+		// build the address to which to redirect
+		addr, err := caddy.ParseNetworkAddress(addrStr)
+		if err != nil {
+			return err
+		}
+		redirTo := "https://{http.request.host}"
+		if addr.StartPort != uint(app.httpsPort()) {
+			redirTo += ":" + strconv.Itoa(int(addr.StartPort))
+		}
+		redirTo += "{http.request.uri}"
+
+		// build the route
+		redirRoute := Route{
+			MatcherSets: []MatcherSet{matcherSet},
+			Handlers: []MiddlewareHandler{
+				StaticResponse{
+					StatusCode: WeakString(strconv.Itoa(http.StatusPermanentRedirect)),
+					Headers: http.Header{
+						"Location":   []string{redirTo},
+						"Connection": []string{"close"},
+					},
+					Close: true,
+				},
+			},
+		}
+
+		// use the network/host information from the address,
+		// but change the port to the HTTP port then rebuild
+		redirAddr := addr
+		redirAddr.StartPort = uint(app.httpPort())
+		redirAddr.EndPort = redirAddr.StartPort
+		redirAddrStr := redirAddr.String()
+
+		redirServers[redirAddrStr] = append(redirServers[redirAddrStr], redirRoute)
+	}
+
+	// on-demand TLS means that hostnames may be used which are not
+	// explicitly defined in the config, and we still need to redirect
+	// those; so we can append a single catch-all route (notice there
+	// is no Host matcher) after the other redirect routes which will
+	// allow us to handle unexpected/new hostnames... however, it's
+	// not entirely clear what the redirect destination should be,
+	// so I'm going to just hard-code the app's HTTPS port and call
+	// it good for now...
+	appendCatchAll := func(routes []Route) []Route {
+		redirTo := "https://{http.request.host}"
+		if app.httpsPort() != DefaultHTTPSPort {
+			redirTo += ":" + strconv.Itoa(app.httpsPort())
+		}
+		redirTo += "{http.request.uri}"
+		routes = append(routes, Route{
+			MatcherSets: []MatcherSet{{MatchProtocol("http")}},
+			Handlers: []MiddlewareHandler{
+				StaticResponse{
+					StatusCode: WeakString(strconv.Itoa(http.StatusPermanentRedirect)),
+					Headers: http.Header{
+						"Location":   []string{redirTo},
+						"Connection": []string{"close"},
+					},
+					Close: true,
+				},
+			},
+		})
+		return routes
+	}
+
+redirServersLoop:
+	for redirServerAddr, routes := range redirServers {
+		// for each redirect listener, see if there's already a
+		// server configured to listen on that exact address; if so,
+		// simply add the redirect route to the end of its route
+		// list; otherwise, we'll create a new server for all the
+		// listener addresses that are unused and serve the
+		// remaining redirects from it
 		for srvName, srv := range app.Servers {
-			if srv.hasListenerAddress(addr) {
+			if srv.hasListenerAddress(redirServerAddr) {
 				// user has configured a server for the same address
 				// that the redirect runs from; simply append our
 				// redirect route to the existing routes, with a
 				// caveat that their config might override ours
-				app.logger.Warn("server is listening on same interface as redirects, so automatic HTTP->HTTPS redirects might be overridden by your own configuration",
+				app.logger.Warn("user server is listening on same interface as automatic HTTP->HTTPS redirects; user-configured routes might override these redirects",
 					zap.String("server_name", srvName),
-					zap.String("interface", addr),
+					zap.String("interface", redirServerAddr),
 				)
-				srv.Routes = append(srv.Routes, redirRoute)
-				continue redirRoutesLoop
+				srv.Routes = append(srv.Routes, appendCatchAll(routes)...)
+				continue redirServersLoop
 			}
 		}
+
 		// no server with this listener address exists;
 		// save this address and route for custom server
-		redirServerAddrs = append(redirServerAddrs, addr)
-		redirRoutes = append(redirRoutes, redirRoute)
+		redirServerAddrs[redirServerAddr] = struct{}{}
+		redirRoutes = append(redirRoutes, routes...)
 	}
 
 	// if there are routes remaining which do not belong
 	// in any existing server, make our own to serve the
 	// rest of the redirects
 	if len(redirServerAddrs) > 0 {
+		redirServerAddrsList := make([]string, 0, len(redirServerAddrs))
+		for a := range redirServerAddrs {
+			redirServerAddrsList = append(redirServerAddrsList, a)
+		}
 		app.Servers["remaining_auto_https_redirects"] = &Server{
-			Listen: redirServerAddrs,
-			Routes: redirRoutes,
+			Listen: redirServerAddrsList,
+			Routes: appendCatchAll(redirRoutes),
 		}
 	}
 
 	return nil
 }
 
-// automaticHTTPSPhase2 attaches a TLS app pointer to each
-// server. This phase must occur after provisioning, and
-// at the beginning of the app start, before starting each
-// of the servers.
-func (app *App) automaticHTTPSPhase2() error {
-	tlsAppIface, err := app.ctx.App("tls")
-	if err != nil {
-		return fmt.Errorf("getting tls app: %v", err)
+// createAutomationPolicy ensures that automated certificates for this
+// app are managed properly. This adds up to two automation policies:
+// one for the public names, and one for the internal names. If a catch-all
+// automation policy exists, it will be shallow-copied and used as the
+// base for the new ones (this is important for preserving behavior the
+// user intends to be "defaults").
+func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, internalNames []string) error {
+	// before we begin, loop through the existing automation policies
+	// and, for any ACMEIssuers we find, make sure they're filled in
+	// with default values that might be specified in our HTTP app; also
+	// look for a base (or "catch-all" / default) automation policy,
+	// which we're going to essentially require, to make sure it has
+	// those defaults, too
+	var basePolicy *caddytls.AutomationPolicy
+	var foundBasePolicy bool
+	if app.tlsApp.Automation == nil {
+		// we will expect this to not be nil from now on
+		app.tlsApp.Automation = new(caddytls.AutomationConfig)
 	}
-	tlsApp := tlsAppIface.(*caddytls.TLS)
+	for _, ap := range app.tlsApp.Automation.Policies {
+		// set up default issuer -- honestly, this is only
+		// really necessary because the HTTP app is opinionated
+		// and has settings which could be inferred as new
+		// defaults for the ACMEIssuer in the TLS app
+		if ap.Issuer == nil {
+			ap.Issuer = new(caddytls.ACMEIssuer)
+		}
+		if acmeIssuer, ok := ap.Issuer.(*caddytls.ACMEIssuer); ok {
+			err := app.fillInACMEIssuer(acmeIssuer)
+			if err != nil {
+				return err
+			}
+		}
 
-	// set the tlsApp pointer before starting any
-	// challenges, since it is required to solve
-	// the ACME HTTP challenge
-	for _, srv := range app.Servers {
-		srv.tlsApp = tlsApp
+		// while we're here, is this the catch-all/base policy?
+		if !foundBasePolicy && len(ap.Subjects) == 0 {
+			basePolicy = ap
+			foundBasePolicy = true
+		}
+	}
+
+	if basePolicy == nil {
+		// no base policy found, we will make one!
+		basePolicy = new(caddytls.AutomationPolicy)
+	}
+
+	// if the basePolicy has an existing ACMEIssuer, let's
+	// use it, otherwise we'll make one
+	baseACMEIssuer, _ := basePolicy.Issuer.(*caddytls.ACMEIssuer)
+	if baseACMEIssuer == nil {
+		// note that this happens if basePolicy.Issuer is nil
+		// OR if it is not nil but is not an ACMEIssuer
+		baseACMEIssuer = new(caddytls.ACMEIssuer)
+	}
+
+	// if there was a base policy to begin with, we already
+	// filled in its issuer's defaults; if there wasn't, we
+	// stil need to do that
+	if !foundBasePolicy {
+		err := app.fillInACMEIssuer(baseACMEIssuer)
+		if err != nil {
+			return err
+		}
+	}
+
+	// never overwrite any other issuer that might already be configured
+	if basePolicy.Issuer == nil {
+		basePolicy.Issuer = baseACMEIssuer
+	}
+
+	if !foundBasePolicy {
+		// there was no base policy to begin with, so add
+		// our base/catch-all policy - this will serve the
+		// public-looking names as well as any other names
+		// that don't match any other policy
+		app.tlsApp.AddAutomationPolicy(basePolicy)
+	} else {
+		// a base policy already existed; we might have
+		// changed it, so re-provision it
+		err := basePolicy.Provision(app.tlsApp)
+		if err != nil {
+			return err
+		}
+	}
+
+	// public names will be taken care of by the base (catch-all)
+	// policy, which we've ensured exists if not already specified;
+	// internal names, however, need to be handled by an internal
+	// issuer, which we need to make a new policy for, scoped to
+	// just those names (yes, this logic is a bit asymmetric, but
+	// it works, because our assumed/natural default issuer is an
+	// ACME issuer)
+	if len(internalNames) > 0 {
+		internalIssuer := new(caddytls.InternalIssuer)
+
+		// shallow-copy the base policy; we want to inherit
+		// from it, not replace it... this takes two lines to
+		// overrule compiler optimizations
+		policyCopy := *basePolicy
+		newPolicy := &policyCopy
+
+		// very important to provision the issuer, since we
+		// are bypassing the JSON-unmarshaling step
+		if err := internalIssuer.Provision(ctx); err != nil {
+			return err
+		}
+
+		// this policy should apply only to the given names
+		// and should use our issuer -- yes, this overrides
+		// any issuer that may have been set in the base
+		// policy, but we do this because these names do not
+		// already have a policy associated with them, which
+		// is easy to do; consider the case of a Caddyfile
+		// that has only "localhost" as a name, but sets the
+		// default/global ACME CA to the Let's Encrypt staging
+		// endpoint... they probably don't intend to change the
+		// fundamental set of names that setting applies to,
+		// rather they just want to change the CA for the set
+		// of names that would normally use the production API;
+		// anyway, that gets into the weeds a bit...
+		newPolicy.Subjects = internalNames
+		newPolicy.Issuer = internalIssuer
+
+		err := app.tlsApp.AddAutomationPolicy(newPolicy)
+		if err != nil {
+			return err
+		}
+	}
+
+	// we just changed a lot of stuff, so double-check that it's all good
+	err := app.tlsApp.Validate()
+	if err != nil {
+		return err
 	}
 
 	return nil
 }
 
-// automaticHTTPSPhase3 begins certificate management for
+// fillInACMEIssuer fills in default values into acmeIssuer that
+// are defined in app; these values at time of writing are just
+// app.HTTPPort and app.HTTPSPort, which are used by ACMEIssuer.
+// Sure, we could just use the global/CertMagic defaults, but if
+// a user has configured those ports in the HTTP app, it makes
+// sense to use them in the TLS app too, even if they forgot (or
+// were too lazy, like me) to set it in each automation policy
+// that uses it -- this just makes things a little less tedious
+// for the user, so they don't have to repeat those ports in
+// potentially many places. This function never steps on existing
+// config values. If any changes are made, acmeIssuer is
+// reprovisioned. acmeIssuer must not be nil.
+func (app *App) fillInACMEIssuer(acmeIssuer *caddytls.ACMEIssuer) error {
+	if app.HTTPPort > 0 || app.HTTPSPort > 0 {
+		if acmeIssuer.Challenges == nil {
+			acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
+		}
+	}
+	if app.HTTPPort > 0 {
+		if acmeIssuer.Challenges.HTTP == nil {
+			acmeIssuer.Challenges.HTTP = new(caddytls.HTTPChallengeConfig)
+		}
+		// don't overwrite existing explicit config
+		if acmeIssuer.Challenges.HTTP.AlternatePort == 0 {
+			acmeIssuer.Challenges.HTTP.AlternatePort = app.HTTPPort
+		}
+	}
+	if app.HTTPSPort > 0 {
+		if acmeIssuer.Challenges.TLSALPN == nil {
+			acmeIssuer.Challenges.TLSALPN = new(caddytls.TLSALPNChallengeConfig)
+		}
+		// don't overwrite existing explicit config
+		if acmeIssuer.Challenges.TLSALPN.AlternatePort == 0 {
+			acmeIssuer.Challenges.TLSALPN.AlternatePort = app.HTTPSPort
+		}
+	}
+	// we must provision all ACME issuers, even if nothing
+	// was changed, because we don't know if they are new
+	// and haven't been provisioned yet; if an ACME issuer
+	// never gets provisioned, its Agree field stays false,
+	// which leads to, um, problems later on
+	return acmeIssuer.Provision(app.ctx)
+}
+
+// automaticHTTPSPhase2 begins certificate management for
 // all names in the qualifying domain set for each server.
 // This phase must occur after provisioning and at the end
 // of app start, after all the servers have been started.
@@ -289,72 +592,17 @@ func (app *App) automaticHTTPSPhase2() error {
 // first, then our servers would fail to bind to them,
 // which would be bad, since CertMagic's bindings are
 // temporary and don't serve the user's sites!).
-func (app *App) automaticHTTPSPhase3() error {
-	// begin managing certificates for enabled servers
-	for srvName, srv := range app.Servers {
-		if srv.AutoHTTPS == nil ||
-			srv.AutoHTTPS.Disabled ||
-			len(srv.AutoHTTPS.domainSet) == 0 {
-			continue
-		}
-
-		// marshal the domains into a slice
-		var domains, domainsForCerts []string
-		for d := range srv.AutoHTTPS.domainSet {
-			domains = append(domains, d)
-			if !srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.SkipCerts) {
-				// if a certificate for this name is already loaded,
-				// don't obtain another one for it, unless we are
-				// supposed to ignore loaded certificates
-				if !srv.AutoHTTPS.IgnoreLoadedCerts &&
-					len(srv.tlsApp.AllMatchingCertificates(d)) > 0 {
-					app.logger.Info("skipping automatic certificate management because one or more matching certificates are already loaded",
-						zap.String("domain", d),
-						zap.String("server_name", srvName),
-					)
-					continue
-				}
-				domainsForCerts = append(domainsForCerts, d)
-			}
-		}
-
-		// ensure that these certificates are managed properly;
-		// for example, it's implied that the HTTPPort should also
-		// be the port the HTTP challenge is solved on, and so
-		// for HTTPS port and TLS-ALPN challenge also - we need
-		// to tell the TLS app to manage these certs by honoring
-		// those port configurations
-		acmeManager := &caddytls.ACMEManagerMaker{
-			Challenges: &caddytls.ChallengesConfig{
-				HTTP: &caddytls.HTTPChallengeConfig{
-					AlternatePort: app.HTTPPort, // we specifically want the user-configured port, if any
-				},
-				TLSALPN: &caddytls.TLSALPNChallengeConfig{
-					AlternatePort: app.HTTPSPort, // we specifically want the user-configured port, if any
-				},
-			},
-		}
-		if srv.tlsApp.Automation == nil {
-			srv.tlsApp.Automation = new(caddytls.AutomationConfig)
-		}
-		srv.tlsApp.Automation.Policies = append(srv.tlsApp.Automation.Policies,
-			&caddytls.AutomationPolicy{
-				Hosts:      domainsForCerts,
-				Management: acmeManager,
-			})
-
-		// manage their certificates
-		app.logger.Info("enabling automatic TLS certificate management",
-			zap.Strings("domains", domainsForCerts),
-		)
-		err := srv.tlsApp.Manage(domainsForCerts)
-		if err != nil {
-			return fmt.Errorf("%s: managing certificate for %s: %s", srvName, domains, err)
-		}
-
-		// no longer needed; allow GC to deallocate
-		srv.AutoHTTPS.domainSet = nil
+func (app *App) automaticHTTPSPhase2() error {
+	if len(app.allCertDomains) == 0 {
+		return nil
 	}
-
+	app.logger.Info("enabling automatic TLS certificate management",
+		zap.Strings("domains", app.allCertDomains),
+	)
+	err := app.tlsApp.Manage(app.allCertDomains)
+	if err != nil {
+		return fmt.Errorf("managing certificates for %v: %s", app.allCertDomains, err)
+	}
+	app.allCertDomains = nil // no longer needed; allow GC to deallocate
 	return nil
 }

modules/caddyhttp/caddyauth/basicauth.go

@@ -77,8 +77,8 @@ func (hba *HTTPBasicAuth) Provision(ctx caddy.Context) error {
 		}
 
 		acct.Username = repl.ReplaceAll(acct.Username, "")
-		acct.Password = repl.ReplaceAll(string(acct.Password), "")
-		acct.Salt = repl.ReplaceAll(string(acct.Salt), "")
+		acct.Password = repl.ReplaceAll(acct.Password, "")
+		acct.Salt = repl.ReplaceAll(acct.Salt, "")
 
 		if acct.Username == "" || acct.Password == "" {
 			return fmt.Errorf("account %d: username and password are required", i)
@@ -105,20 +105,8 @@ func (hba *HTTPBasicAuth) Provision(ctx caddy.Context) error {
 // Authenticate validates the user credentials in req and returns the user, if valid.
 func (hba HTTPBasicAuth) Authenticate(w http.ResponseWriter, req *http.Request) (User, bool, error) {
 	username, plaintextPasswordStr, ok := req.BasicAuth()
-
-	// if basic auth is missing or invalid, prompt for credentials
 	if !ok {
-		// browsers show a message that says something like:
-		// "The website says: <realm>"
-		// which is kinda dumb, but whatever.
-		realm := hba.Realm
-		if realm == "" {
-			realm = "restricted"
-		}
-
-		w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Basic realm="%s"`, realm))
-
-		return User{}, false, nil
+		return hba.promptForCredentials(w, nil)
 	}
 
 	plaintextPassword := []byte(plaintextPasswordStr)
@@ -129,15 +117,27 @@ func (hba HTTPBasicAuth) Authenticate(w http.ResponseWriter, req *http.Request)
 
 	same, err := hba.Hash.Compare(account.password, plaintextPassword, account.salt)
 	if err != nil {
-		return User{}, false, err
+		return hba.promptForCredentials(w, err)
 	}
 	if !same || !accountExists {
-		return User{}, false, nil
+		return hba.promptForCredentials(w, nil)
 	}
 
 	return User{ID: username}, true, nil
 }
 
+func (hba HTTPBasicAuth) promptForCredentials(w http.ResponseWriter, err error) (User, bool, error) {
+	// browsers show a message that says something like:
+	// "The website says: <realm>"
+	// which is kinda dumb, but whatever.
+	realm := hba.Realm
+	if realm == "" {
+		realm = "restricted"
+	}
+	w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Basic realm="%s"`, realm))
+	return User{}, false, err
+}
+
 // Comparer is a type that can securely compare
 // a plaintext password with a hashed password
 // in constant-time. Comparers should hash the

modules/caddyhttp/caddyauth/caddyauth.go

@@ -77,7 +77,10 @@ func (a Authentication) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 	}
 
 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
-	repl.Set("http.authentication.user.id", user.ID)
+	repl.Set("http.auth.user.id", user.ID)
+	for k, v := range user.Metadata {
+		repl.Set("http.auth.user."+k, v)
+	}
 
 	return next.ServeHTTP(w, r)
 }
@@ -92,7 +95,15 @@ type Authenticator interface {
 
 // User represents an authenticated user.
 type User struct {
+	// The ID of the authenticated user.
 	ID string
+
+	// Any other relevant data about this
+	// user. Keys should be adhere to Caddy
+	// conventions (snake_casing), as all
+	// keys will be made available as
+	// placeholders.
+	Metadata map[string]string
 }
 
 // Interface guards

modules/caddyhttp/caddyauth/command.go

@@ -76,7 +76,7 @@ func cmdHashPassword(fs caddycmd.Flags) (int, error) {
 		return caddy.ExitCodeFailedStartup, err
 	}
 
-	hashBase64 := base64.StdEncoding.EncodeToString([]byte(hash))
+	hashBase64 := base64.StdEncoding.EncodeToString(hash)
 
 	fmt.Println(hashBase64)
 

modules/caddyhttp/caddyhttp.go

@@ -16,10 +16,7 @@ package caddyhttp
 
 import (
 	"bytes"
-	"context"
-	"crypto/tls"
 	"encoding/json"
-	"fmt"
 	"io"
 	weakrand "math/rand"
 	"net"
@@ -28,364 +25,17 @@ import (
 	"time"
 
 	"github.com/caddyserver/caddy/v2"
-	"github.com/lucas-clemente/quic-go/http3"
-	"github.com/mholt/certmagic"
-	"go.uber.org/zap"
 )
 
 func init() {
 	weakrand.Seed(time.Now().UnixNano())
 
-	err := caddy.RegisterModule(App{})
+	err := caddy.RegisterModule(tlsPlaceholderWrapper{})
 	if err != nil {
 		caddy.Log().Fatal(err.Error())
 	}
 }
 
-// App is a robust, production-ready HTTP server.
-//
-// HTTPS is enabled by default if host matchers with qualifying names are used
-// in any of routes; certificates are automatically provisioned and renewed.
-// Additionally, automatic HTTPS will also enable HTTPS for servers that listen
-// only on the HTTPS port but which do not have any TLS connection policies
-// defined by adding a good, default TLS connection policy.
-//
-// In HTTP routes, additional placeholders are available (replace any `*`):
-//
-// Placeholder | Description
-// ------------|---------------
-// `{http.request.cookie.*}` | HTTP request cookie
-// `{http.request.header.*}` | Specific request header field
-// `{http.request.host.labels.*}` | Request host labels (0-based from right); e.g. for foo.example.com: 0=com, 1=example, 2=foo
-// `{http.request.host}` | The host part of the request's Host header
-// `{http.request.hostport}` | The host and port from the request's Host header
-// `{http.request.method}` | The request method
-// `{http.request.orig_method}` | The request's original method
-// `{http.request.orig_uri.path.dir}` | The request's original directory
-// `{http.request.orig_uri.path.file}` | The request's original filename
-// `{http.request.orig_uri.path}` | The request's original path
-// `{http.request.orig_uri.query}` | The request's original query string (without `?`)
-// `{http.request.orig_uri}` | The request's original URI
-// `{http.request.port}` | The port part of the request's Host header
-// `{http.request.proto}` | The protocol of the request
-// `{http.request.remote.host}` | The host part of the remote client's address
-// `{http.request.remote.port}` | The port part of the remote client's address
-// `{http.request.remote}` | The address of the remote client
-// `{http.request.scheme}` | The request scheme
-// `{http.request.tls.version}` | The TLS version name
-// `{http.request.tls.cipher_suite}` | The TLS cipher suite
-// `{http.request.tls.resumed}` | The TLS connection resumed a previous connection
-// `{http.request.tls.proto}` | The negotiated next protocol
-// `{http.request.tls.proto_mutual}` | The negotiated next protocol was advertised by the server
-// `{http.request.tls.server_name}` | The server name requested by the client, if any
-// `{http.request.tls.client.fingerprint}` | The SHA256 checksum of the client certificate
-// `{http.request.tls.client.issuer}` | The issuer DN of the client certificate
-// `{http.request.tls.client.serial}` | The serial number of the client certificate
-// `{http.request.tls.client.subject}` | The subject DN of the client certificate
-// `{http.request.uri.path.*}` | Parts of the path, split by `/` (0-based from left)
-// `{http.request.uri.path.dir}` | The directory, excluding leaf filename
-// `{http.request.uri.path.file}` | The filename of the path, excluding directory
-// `{http.request.uri.path}` | The path component of the request URI
-// `{http.request.uri.query.*}` | Individual query string value
-// `{http.request.uri.query}` | The query string (without `?`)
-// `{http.request.uri}` | The full request URI
-// `{http.response.header.*}` | Specific response header field
-// `{http.vars.*}` | Custom variables in the HTTP handler chain
-type App struct {
-	// HTTPPort specifies the port to use for HTTP (as opposed to HTTPS),
-	// which is used when setting up HTTP->HTTPS redirects or ACME HTTP
-	// challenge solvers. Default: 80.
-	HTTPPort int `json:"http_port,omitempty"`
-
-	// HTTPSPort specifies the port to use for HTTPS, which is used when
-	// solving the ACME TLS-ALPN challenges, or whenever HTTPS is needed
-	// but no specific port number is given. Default: 443.
-	HTTPSPort int `json:"https_port,omitempty"`
-
-	// GracePeriod is how long to wait for active connections when shutting
-	// down the server. Once the grace period is over, connections will
-	// be forcefully closed.
-	GracePeriod caddy.Duration `json:"grace_period,omitempty"`
-
-	// Servers is the list of servers, keyed by arbitrary names chosen
-	// at your discretion for your own convenience; the keys do not
-	// affect functionality.
-	Servers map[string]*Server `json:"servers,omitempty"`
-
-	// DefaultSNI if set configures all certificate lookups to fallback to use
-	// this SNI name if a more specific certificate could not be found
-	DefaultSNI string `json:"default_sni,omitempty"`
-
-	servers     []*http.Server
-	h3servers   []*http3.Server
-	h3listeners []net.PacketConn
-
-	ctx    caddy.Context
-	logger *zap.Logger
-}
-
-// CaddyModule returns the Caddy module information.
-func (App) CaddyModule() caddy.ModuleInfo {
-	return caddy.ModuleInfo{
-		ID:  "http",
-		New: func() caddy.Module { return new(App) },
-	}
-}
-
-// Provision sets up the app.
-func (app *App) Provision(ctx caddy.Context) error {
-	app.ctx = ctx
-	app.logger = ctx.Logger(app)
-
-	repl := caddy.NewReplacer()
-
-	certmagic.Default.DefaultServerName = app.DefaultSNI
-
-	// this provisions the matchers for each route,
-	// and prepares auto HTTP->HTTPS redirects, and
-	// is required before we provision each server
-	err := app.automaticHTTPSPhase1(ctx, repl)
-	if err != nil {
-		return err
-	}
-
-	for srvName, srv := range app.Servers {
-		srv.logger = app.logger.Named("log")
-		srv.errorLogger = app.logger.Named("log.error")
-
-		// only enable access logs if configured
-		if srv.Logs != nil {
-			srv.accessLogger = app.logger.Named("log.access")
-		}
-
-		// if not explicitly configured by the user, disallow TLS
-		// client auth bypass (domain fronting) which could
-		// otherwise be exploited by sending an unprotected SNI
-		// value during a TLS handshake, then putting a protected
-		// domain in the Host header after establishing connection;
-		// this is a safe default, but we allow users to override
-		// it for example in the case of running a proxy where
-		// domain fronting is desired and access is not restricted
-		// based on hostname
-		if srv.StrictSNIHost == nil && srv.hasTLSClientAuth() {
-			app.logger.Info("enabling strict SNI-Host matching because TLS client auth is configured",
-				zap.String("server_name", srvName),
-			)
-			trueBool := true
-			srv.StrictSNIHost = &trueBool
-		}
-
-		for i := range srv.Listen {
-			lnOut, err := repl.ReplaceOrErr(srv.Listen[i], true, true)
-			if err != nil {
-				return fmt.Errorf("server %s, listener %d: %v",
-					srvName, i, err)
-			}
-			srv.Listen[i] = lnOut
-		}
-
-		// pre-compile the primary handler chain, and be sure to wrap it in our
-		// route handler so that important security checks are done, etc.
-		primaryRoute := emptyHandler
-		if srv.Routes != nil {
-			err := srv.Routes.ProvisionHandlers(ctx)
-			if err != nil {
-				return fmt.Errorf("server %s: setting up route handlers: %v", srvName, err)
-			}
-			primaryRoute = srv.Routes.Compile(emptyHandler)
-		}
-		srv.primaryHandlerChain = srv.wrapPrimaryRoute(primaryRoute)
-
-		// pre-compile the error handler chain
-		if srv.Errors != nil {
-			err := srv.Errors.Routes.Provision(ctx)
-			if err != nil {
-				return fmt.Errorf("server %s: setting up server error handling routes: %v", srvName, err)
-			}
-
-			srv.errorHandlerChain = srv.Errors.Routes.Compile(errorEmptyHandler)
-		}
-	}
-
-	return nil
-}
-
-// Validate ensures the app's configuration is valid.
-func (app *App) Validate() error {
-	// each server must use distinct listener addresses
-	lnAddrs := make(map[string]string)
-	for srvName, srv := range app.Servers {
-		for _, addr := range srv.Listen {
-			listenAddr, err := caddy.ParseNetworkAddress(addr)
-			if err != nil {
-				return fmt.Errorf("invalid listener address '%s': %v", addr, err)
-			}
-			// check that every address in the port range is unique to this server;
-			// we do not use <= here because PortRangeSize() adds 1 to EndPort for us
-			for i := uint(0); i < listenAddr.PortRangeSize(); i++ {
-				addr := caddy.JoinNetworkAddress(listenAddr.Network, listenAddr.Host, strconv.Itoa(int(listenAddr.StartPort+i)))
-				if sn, ok := lnAddrs[addr]; ok {
-					return fmt.Errorf("server %s: listener address repeated: %s (already claimed by server '%s')", srvName, addr, sn)
-				}
-				lnAddrs[addr] = srvName
-			}
-		}
-	}
-
-	return nil
-}
-
-// Start runs the app. It finishes automatic HTTPS if enabled,
-// including management of certificates.
-func (app *App) Start() error {
-	// give each server a pointer to the TLS app;
-	// this is required before they are started so
-	// they can solve ACME challenges
-	err := app.automaticHTTPSPhase2()
-	if err != nil {
-		return fmt.Errorf("enabling automatic HTTPS, phase 2: %v", err)
-	}
-
-	for srvName, srv := range app.Servers {
-		s := &http.Server{
-			ReadTimeout:       time.Duration(srv.ReadTimeout),
-			ReadHeaderTimeout: time.Duration(srv.ReadHeaderTimeout),
-			WriteTimeout:      time.Duration(srv.WriteTimeout),
-			IdleTimeout:       time.Duration(srv.IdleTimeout),
-			MaxHeaderBytes:    srv.MaxHeaderBytes,
-			Handler:           srv,
-		}
-
-		for _, lnAddr := range srv.Listen {
-			listenAddr, err := caddy.ParseNetworkAddress(lnAddr)
-			if err != nil {
-				return fmt.Errorf("%s: parsing listen address '%s': %v", srvName, lnAddr, err)
-			}
-			for portOffset := uint(0); portOffset < listenAddr.PortRangeSize(); portOffset++ {
-				hostport := listenAddr.JoinHostPort(portOffset)
-				ln, err := caddy.Listen(listenAddr.Network, hostport)
-				if err != nil {
-					return fmt.Errorf("%s: listening on %s: %v", listenAddr.Network, hostport, err)
-				}
-
-				// enable HTTP/2 by default
-				for _, pol := range srv.TLSConnPolicies {
-					if len(pol.ALPN) == 0 {
-						pol.ALPN = append(pol.ALPN, defaultALPN...)
-					}
-				}
-
-				// enable TLS if there is a policy and if this is not the HTTP port
-				if len(srv.TLSConnPolicies) > 0 &&
-					int(listenAddr.StartPort+portOffset) != app.httpPort() {
-					// create TLS listener
-					tlsCfg, err := srv.TLSConnPolicies.TLSConfig(app.ctx)
-					if err != nil {
-						return fmt.Errorf("%s/%s: making TLS configuration: %v", listenAddr.Network, hostport, err)
-					}
-					ln = tls.NewListener(ln, tlsCfg)
-
-					/////////
-					// TODO: HTTP/3 support is experimental for now
-					if srv.ExperimentalHTTP3 {
-						app.logger.Info("enabling experimental HTTP/3 listener",
-							zap.String("addr", hostport),
-						)
-						h3ln, err := caddy.ListenPacket("udp", hostport)
-						if err != nil {
-							return fmt.Errorf("getting HTTP/3 UDP listener: %v", err)
-						}
-						h3srv := &http3.Server{
-							Server: &http.Server{
-								Addr:      hostport,
-								Handler:   srv,
-								TLSConfig: tlsCfg,
-							},
-						}
-						go h3srv.Serve(h3ln)
-						app.h3servers = append(app.h3servers, h3srv)
-						app.h3listeners = append(app.h3listeners, h3ln)
-						srv.h3server = h3srv
-					}
-					/////////
-				}
-
-				go s.Serve(ln)
-				app.servers = append(app.servers, s)
-			}
-		}
-	}
-
-	// finish automatic HTTPS by finally beginning
-	// certificate management
-	err = app.automaticHTTPSPhase3()
-	if err != nil {
-		return fmt.Errorf("finalizing automatic HTTPS: %v", err)
-	}
-
-	return nil
-}
-
-// Stop gracefully shuts down the HTTP server.
-func (app *App) Stop() error {
-	ctx := context.Background()
-	if app.GracePeriod > 0 {
-		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(ctx, time.Duration(app.GracePeriod))
-		defer cancel()
-	}
-	for _, s := range app.servers {
-		err := s.Shutdown(ctx)
-		if err != nil {
-			return err
-		}
-	}
-
-	// close the http3 servers; it's unclear whether the bug reported in
-	// https://github.com/caddyserver/caddy/pull/2727#issuecomment-526856566
-	// was ever truly fixed, since it seemed racey/nondeterministic; but
-	// recent tests in 2020 were unable to replicate the issue again after
-	// repeated attempts (the bug manifested after a config reload; i.e.
-	// reusing a http3 server or listener was problematic), but it seems
-	// to be working fine now
-	for _, s := range app.h3servers {
-		// TODO: CloseGracefully, once implemented upstream
-		// (see https://github.com/lucas-clemente/quic-go/issues/2103)
-		err := s.Close()
-		if err != nil {
-			return err
-		}
-	}
-
-	// closing an http3.Server does not close their underlying listeners
-	// since apparently the listener can be used both by servers and
-	// clients at the same time; so we need to manually call Close()
-	// on the underlying h3 listeners (see lucas-clemente/quic-go#2103)
-	for _, pc := range app.h3listeners {
-		err := pc.Close()
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (app *App) httpPort() int {
-	if app.HTTPPort == 0 {
-		return DefaultHTTPPort
-	}
-	return app.HTTPPort
-}
-
-func (app *App) httpsPort() int {
-	if app.HTTPSPort == 0 {
-		return DefaultHTTPSPort
-	}
-	return app.HTTPSPort
-}
-
-var defaultALPN = []string{"h2", "http/1.1"}
-
 // RequestMatcher is a type that can match to a request.
 // A route matcher MUST NOT modify the request, with the
 // only exception being its context.
@@ -547,6 +197,21 @@ func StatusCodeMatches(actual, configured int) bool {
 	return false
 }
 
+// tlsPlaceholderWrapper is a no-op listener wrapper that marks
+// where the TLS listener should be in a chain of listener wrappers.
+// It should only be used if another listener wrapper must be placed
+// in front of the TLS handshake.
+type tlsPlaceholderWrapper struct{}
+
+func (tlsPlaceholderWrapper) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "caddy.listeners.tls",
+		New: func() caddy.Module { return new(tlsPlaceholderWrapper) },
+	}
+}
+
+func (tlsPlaceholderWrapper) WrapListener(ln net.Listener) net.Listener { return ln }
+
 const (
 	// DefaultHTTPPort is the default port for HTTP.
 	DefaultHTTPPort = 80
@@ -555,9 +220,5 @@ const (
 	DefaultHTTPSPort = 443
 )
 
-// Interface guards
-var (
-	_ caddy.App         = (*App)(nil)
-	_ caddy.Provisioner = (*App)(nil)
-	_ caddy.Validator   = (*App)(nil)
-)
+// Interface guard
+var _ caddy.ListenerWrapper = (*tlsPlaceholderWrapper)(nil)

modules/caddyhttp/celmatcher.go

@@ -0,0 +1,228 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddyhttp
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"reflect"
+	"regexp"
+	"strings"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+	"github.com/gogo/protobuf/proto"
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/checker/decls"
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/common/types/ref"
+	"github.com/google/cel-go/common/types/traits"
+	"github.com/google/cel-go/ext"
+	"github.com/google/cel-go/interpreter/functions"
+	exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
+)
+
+func init() {
+	caddy.RegisterModule(MatchExpression{})
+}
+
+// MatchExpression matches requests by evaluating a
+// [CEL](https://github.com/google/cel-spec) expression.
+// This enables complex logic to be expressed using a comfortable,
+// familiar syntax.
+//
+// COMPATIBILITY NOTE: This module is still experimental and is not
+// subject to Caddy's compatibility guarantee.
+type MatchExpression struct {
+	// The CEL expression to evaluate. Any Caddy placeholders
+	// will be expanded and situated into proper CEL function
+	// calls before evaluating.
+	Expr string
+
+	expandedExpr string
+	prg          cel.Program
+	ta           ref.TypeAdapter
+}
+
+// CaddyModule returns the Caddy module information.
+func (MatchExpression) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "http.matchers.expression",
+		New: func() caddy.Module { return new(MatchExpression) },
+	}
+}
+
+// MarshalJSON marshals m's expression.
+func (m MatchExpression) MarshalJSON() ([]byte, error) {
+	return json.Marshal(m.Expr)
+}
+
+// UnmarshalJSON unmarshals m's expression.
+func (m *MatchExpression) UnmarshalJSON(data []byte) error {
+	return json.Unmarshal(data, &m.Expr)
+}
+
+// Provision sets ups m.
+func (m *MatchExpression) Provision(_ caddy.Context) error {
+	// replace placeholders with a function call - this is just some
+	// light (and possibly naïve) syntactic sugar
+	m.expandedExpr = placeholderRegexp.ReplaceAllString(m.Expr, placeholderExpansion)
+
+	// our type adapter expands CEL's standard type support
+	m.ta = celTypeAdapter{}
+
+	// create the CEL environment
+	env, err := cel.NewEnv(
+		cel.Declarations(
+			decls.NewIdent("request", httpRequestObjectType, nil),
+			decls.NewFunction(placeholderFuncName,
+				decls.NewOverload(placeholderFuncName+"_httpRequest_string",
+					[]*exprpb.Type{httpRequestObjectType, decls.String},
+					decls.Any)),
+		),
+		cel.CustomTypeAdapter(m.ta),
+		ext.Strings(),
+	)
+	if err != nil {
+		return fmt.Errorf("setting up CEL environment: %v", err)
+	}
+
+	// parse and type-check the expression
+	checked, issues := env.Compile(m.expandedExpr)
+	if issues != nil && issues.Err() != nil {
+		return fmt.Errorf("compiling CEL program: %s", issues.Err())
+	}
+
+	// request matching is a boolean operation, so we don't really know
+	// what to do if the expression returns a non-boolean type
+	if !proto.Equal(checked.ResultType(), decls.Bool) {
+		return fmt.Errorf("CEL request matcher expects return type of bool, not %s", checked.ResultType())
+	}
+
+	// compile the "program"
+	m.prg, err = env.Program(checked,
+		cel.Functions(
+			&functions.Overload{
+				Operator: placeholderFuncName,
+				Binary:   m.caddyPlaceholderFunc,
+			},
+		),
+	)
+
+	if err != nil {
+		return fmt.Errorf("compiling CEL program: %s", err)
+	}
+	return nil
+}
+
+// Match returns true if r matches m.
+func (m MatchExpression) Match(r *http.Request) bool {
+	out, _, _ := m.prg.Eval(map[string]interface{}{
+		"request": celHTTPRequest{r},
+	})
+	if outBool, ok := out.Value().(bool); ok {
+		return outBool
+	}
+	return false
+
+}
+
+// UnmarshalCaddyfile implements caddyfile.Unmarshaler.
+func (m *MatchExpression) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	for d.Next() {
+		m.Expr = strings.Join(d.RemainingArgs(), " ")
+	}
+	return nil
+}
+
+// caddyPlaceholderFunc implements the custom CEL function that accesses the
+// Replacer on a request and gets values from it.
+func (m MatchExpression) caddyPlaceholderFunc(lhs, rhs ref.Val) ref.Val {
+	celReq, ok := lhs.(celHTTPRequest)
+	if !ok {
+		return types.NewErr(
+			"invalid request of type '%v' to "+placeholderFuncName+"(request, placeholderVarName)",
+			lhs.Type())
+	}
+	phStr, ok := rhs.(types.String)
+	if !ok {
+		return types.NewErr(
+			"invalid placeholder variable name of type '%v' to "+placeholderFuncName+"(request, placeholderVarName)",
+			rhs.Type())
+	}
+
+	repl := celReq.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
+	val, _ := repl.Get(string(phStr))
+
+	return m.ta.NativeToValue(val)
+}
+
+// httpRequestCELType is the type representation of a native HTTP request.
+var httpRequestCELType = types.NewTypeValue("http.Request", traits.ReceiverType)
+
+// cellHTTPRequest wraps an http.Request with
+// methods to satisfy the ref.Val interface.
+type celHTTPRequest struct{ *http.Request }
+
+func (cr celHTTPRequest) ConvertToNative(typeDesc reflect.Type) (interface{}, error) {
+	return cr.Request, nil
+}
+func (celHTTPRequest) ConvertToType(typeVal ref.Type) ref.Val {
+	panic("not implemented")
+}
+func (cr celHTTPRequest) Equal(other ref.Val) ref.Val {
+	if o, ok := other.Value().(celHTTPRequest); ok {
+		return types.Bool(o.Request == cr.Request)
+	}
+	return types.ValOrErr(other, "%v is not comparable type", other)
+}
+func (celHTTPRequest) Type() ref.Type        { return httpRequestCELType }
+func (cr celHTTPRequest) Value() interface{} { return cr }
+
+// celTypeAdapter can adapt our custom types to a CEL value.
+type celTypeAdapter struct{}
+
+func (celTypeAdapter) NativeToValue(value interface{}) ref.Val {
+	switch v := value.(type) {
+	case celHTTPRequest:
+		return v
+	case error:
+		types.NewErr(v.Error())
+	}
+	return types.DefaultTypeAdapter.NativeToValue(value)
+}
+
+// Variables used for replacing Caddy placeholders in CEL
+// expressions with a proper CEL function call; this is
+// just for syntactic sugar.
+var (
+	placeholderRegexp    = regexp.MustCompile(`{([\w.-]+)}`)
+	placeholderExpansion = `caddyPlaceholder(request, "${1}")`
+)
+
+var httpRequestObjectType = decls.NewObjectType("http.Request")
+
+// The name of the CEL function which accesses Replacer values.
+const placeholderFuncName = "caddyPlaceholder"
+
+// Interface guards
+var (
+	_ caddy.Provisioner     = (*MatchExpression)(nil)
+	_ RequestMatcher        = (*MatchExpression)(nil)
+	_ caddyfile.Unmarshaler = (*MatchExpression)(nil)
+	_ json.Marshaler        = (*MatchExpression)(nil)
+	_ json.Unmarshaler      = (*MatchExpression)(nil)
+)

modules/caddyhttp/encode/brotli/brotli.go

@@ -1,95 +0,0 @@
-// Copyright 2015 Matthew Holt and The Caddy Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package caddybrotli
-
-import (
-	"fmt"
-	"strconv"
-
-	"github.com/andybalholm/brotli"
-	"github.com/caddyserver/caddy/v2"
-	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
-	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
-)
-
-func init() {
-	caddy.RegisterModule(Brotli{})
-}
-
-// Brotli can create brotli encoders. Note that brotli
-// is not known for great encoding performance, and
-// its use during requests is discouraged; instead,
-// pre-compress the content instead.
-type Brotli struct {
-	Quality *int `json:"quality,omitempty"`
-}
-
-// CaddyModule returns the Caddy module information.
-func (Brotli) CaddyModule() caddy.ModuleInfo {
-	return caddy.ModuleInfo{
-		ID:  "http.encoders.brotli",
-		New: func() caddy.Module { return new(Brotli) },
-	}
-}
-
-// UnmarshalCaddyfile sets up the handler from Caddyfile tokens.
-func (b *Brotli) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
-	for d.Next() {
-		if !d.NextArg() {
-			continue
-		}
-		qualityStr := d.Val()
-		quality, err := strconv.Atoi(qualityStr)
-		if err != nil {
-			return err
-		}
-		b.Quality = &quality
-	}
-	return nil
-}
-
-// Validate validates b's configuration.
-func (b Brotli) Validate() error {
-	if b.Quality != nil {
-		quality := *b.Quality
-		if quality < brotli.BestSpeed {
-			return fmt.Errorf("quality too low; must be >= %d", brotli.BestSpeed)
-		}
-		if quality > brotli.BestCompression {
-			return fmt.Errorf("quality too high; must be <= %d", brotli.BestCompression)
-		}
-	}
-	return nil
-}
-
-// AcceptEncoding returns the name of the encoding as
-// used in the Accept-Encoding request headers.
-func (Brotli) AcceptEncoding() string { return "br" }
-
-// NewEncoder returns a new brotli writer.
-func (b Brotli) NewEncoder() encode.Encoder {
-	quality := brotli.DefaultCompression
-	if b.Quality != nil {
-		quality = *b.Quality
-	}
-	return brotli.NewWriterLevel(nil, quality)
-}
-
-// Interface guards
-var (
-	_ encode.Encoding       = (*Brotli)(nil)
-	_ caddy.Validator       = (*Brotli)(nil)
-	_ caddyfile.Unmarshaler = (*Brotli)(nil)
-)

modules/caddyhttp/encode/caddyfile.go

@@ -42,7 +42,6 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 //     encode [<matcher>] <formats...> {
 //         gzip [<level>]
 //         zstd
-//         brotli [<quality>]
 //     }
 //
 // Specifying the formats on the first line will use those formats' defaults.

modules/caddyhttp/encode/gzip/gzip.go

@@ -16,13 +16,13 @@ package caddygzip
 
 import (
 	"compress/flate"
-	"compress/gzip" // TODO: consider using https://github.com/klauspost/compress/gzip
 	"fmt"
 	"strconv"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
+	"github.com/klauspost/compress/gzip"
 )
 
 func init() {

modules/caddyhttp/fileserver/caddyfile.go

@@ -63,7 +63,7 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 				}
 			case "index":
 				fsrv.IndexNames = h.RemainingArgs()
-				if len(fsrv.Hide) == 0 {
+				if len(fsrv.IndexNames) == 0 {
 					return nil, h.ArgErr()
 				}
 			case "root":
@@ -146,7 +146,7 @@ func parseTryFiles(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error)
 	// if there are query strings in the list, we have to split into
 	// a separate route for each item with a query string, because
 	// the rewrite is different for that item
-	var try []string
+	try := make([]string, 0, len(tryFiles))
 	for _, item := range tryFiles {
 		if idx := strings.Index(item, "?"); idx >= 0 {
 			if len(try) > 0 {

modules/caddyhttp/fileserver/command.go

@@ -23,10 +23,10 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
-	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
 	caddycmd "github.com/caddyserver/caddy/v2/cmd"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
-	"github.com/mholt/certmagic"
+	caddytpl "github.com/caddyserver/caddy/v2/modules/caddyhttp/templates"
+	"github.com/caddyserver/certmagic"
 )
 
 func init() {
@@ -41,10 +41,10 @@ demos, and development.
 
 The listener's socket address can be customized with the --listen flag.
 
-If a qualifying hostname is specified with --domain, the default listener
-address will be changed to the HTTPS port and the server will use HTTPS
-if domain validation succeeds. Ensure A/AAAA records are properly
-configured before using this option.
+If a domain name is specified with --domain, the default listener address
+will be changed to the HTTPS port and the server will use HTTPS. If using
+a public domain, ensure A/AAAA records are properly configured before
+using this option.
 
 If --browse is enabled, requests for folders without an index file will
 respond with a file listing.`,
@@ -53,7 +53,8 @@ respond with a file listing.`,
 			fs.String("domain", "", "Domain name at which to serve the files")
 			fs.String("root", "", "The path to the root of the site")
 			fs.String("listen", "", "The address to which to bind the listener")
-			fs.Bool("browse", false, "Whether to enable directory browsing")
+			fs.Bool("browse", false, "Enable directory browsing")
+			fs.Bool("templates", false, "Enable template rendering")
 			return fs
 		}(),
 	})
@@ -64,17 +65,24 @@ func cmdFileServer(fs caddycmd.Flags) (int, error) {
 	root := fs.String("root")
 	listen := fs.String("listen")
 	browse := fs.Bool("browse")
+	templates := fs.Bool("templates")
+
+	var handlers []json.RawMessage
+
+	if templates {
+		handler := caddytpl.Templates{FileRoot: root}
+		handlers = append(handlers, caddyconfig.JSONModuleObject(handler, "handler", "templates", nil))
+	}
 
 	handler := FileServer{Root: root}
 	if browse {
 		handler.Browse = new(Browse)
 	}
 
-	route := caddyhttp.Route{
-		HandlersRaw: []json.RawMessage{
-			caddyconfig.JSONModuleObject(handler, "handler", "file_server", nil),
-		},
-	}
+	handlers = append(handlers, caddyconfig.JSONModuleObject(handler, "handler", "file_server", nil))
+
+	route := caddyhttp.Route{HandlersRaw: handlers}
+
 	if domain != "" {
 		route.MatcherSetsRaw = []caddy.ModuleMap{
 			caddy.ModuleMap{
@@ -90,10 +98,10 @@ func cmdFileServer(fs caddycmd.Flags) (int, error) {
 		Routes:            caddyhttp.RouteList{route},
 	}
 	if listen == "" {
-		if certmagic.HostQualifies(domain) {
-			listen = ":" + strconv.Itoa(certmagic.HTTPSPort)
+		if domain == "" {
+			listen = ":80"
 		} else {
-			listen = ":" + httpcaddyfile.DefaultPort
+			listen = ":" + strconv.Itoa(certmagic.HTTPSPort)
 		}
 	}
 	server.Listen = []string{listen}

modules/caddyhttp/headers/caddyfile.go

@@ -111,6 +111,12 @@ func parseReqHdrCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler,
 			return nil, h.ArgErr()
 		}
 		field := h.Val()
+
+		// sometimes it is habitual for users to suffix a field name with a colon,
+		// as if they were writing a curl command or something; see
+		// https://caddy.community/t/v2-reverse-proxy-please-add-cors-example-to-the-docs/7349
+		field = strings.TrimSuffix(field, ":")
+
 		var value, replacement string
 		if h.NextArg() {
 			value = h.Val()

modules/caddyhttp/httpcache/httpcache.go

@@ -1,236 +0,0 @@
-// Copyright 2015 Matthew Holt and The Caddy Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package httpcache
-
-import (
-	"bytes"
-	"encoding/gob"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"sync"
-
-	"github.com/caddyserver/caddy/v2"
-	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
-	"github.com/golang/groupcache"
-)
-
-func init() {
-	caddy.RegisterModule(Cache{})
-}
-
-// Cache implements a simple distributed cache.
-//
-// NOTE: This module is a work-in-progress. It is
-// not finished and is NOT ready for production use.
-// [We need your help to finish it! Please volunteer
-// in this issue.](https://github.com/caddyserver/caddy/issues/2820)
-// Until it is finished, this module is subject to
-// breaking changes.
-//
-// Caches only GET and HEAD requests. Honors the Cache-Control: no-cache header.
-//
-// Still TODO:
-//
-// - Eviction policies and API
-// - Use single cache per-process
-// - Preserve cache through config reloads
-// - More control over what gets cached
-type Cache struct {
-	// The network address of this cache instance; required.
-	Self string `json:"self,omitempty"`
-
-	// A list of network addresses of cache instances in the group.
-	Peers []string `json:"peers,omitempty"`
-
-	// Maximum size of the cache, in bytes. Default is 512 MB.
-	MaxSize int64 `json:"max_size,omitempty"`
-
-	group *groupcache.Group
-}
-
-// CaddyModule returns the Caddy module information.
-func (Cache) CaddyModule() caddy.ModuleInfo {
-	return caddy.ModuleInfo{
-		ID:  "http.handlers.cache",
-		New: func() caddy.Module { return new(Cache) },
-	}
-}
-
-// Provision provisions c.
-func (c *Cache) Provision(ctx caddy.Context) error {
-	// TODO: use UsagePool so that cache survives config reloads - TODO: a single cache for whole process?
-	maxSize := c.MaxSize
-	if maxSize == 0 {
-		const maxMB = 512
-		maxSize = int64(maxMB << 20)
-	}
-	poolMu.Lock()
-	if pool == nil {
-		pool = groupcache.NewHTTPPool(c.Self)
-		c.group = groupcache.NewGroup(groupName, maxSize, groupcache.GetterFunc(c.getter))
-	} else {
-		c.group = groupcache.GetGroup(groupName)
-	}
-	pool.Set(append(c.Peers, c.Self)...)
-	poolMu.Unlock()
-
-	return nil
-}
-
-// Validate validates c.
-func (c *Cache) Validate() error {
-	if c.Self == "" {
-		return fmt.Errorf("address of this instance (self) is required")
-	}
-	if c.MaxSize < 0 {
-		return fmt.Errorf("size must be greater than 0")
-	}
-	return nil
-}
-
-func (c *Cache) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
-	// TODO: proper RFC implementation of cache control headers...
-	if r.Header.Get("Cache-Control") == "no-cache" || (r.Method != "GET" && r.Method != "HEAD") {
-		return next.ServeHTTP(w, r)
-	}
-
-	ctx := getterContext{w, r, next}
-
-	// TODO: rigorous performance testing
-
-	// TODO: pretty much everything else to handle the nuances of HTTP caching...
-
-	// TODO: groupcache has no explicit cache eviction, so we need to embed
-	// all information related to expiring cache entries into the key; right
-	// now we just use the request URI as a proof-of-concept
-	key := r.RequestURI
-
-	var cachedBytes []byte
-	err := c.group.Get(ctx, key, groupcache.AllocatingByteSliceSink(&cachedBytes))
-	if err == errUncacheable {
-		return nil
-	}
-	if err != nil {
-		return err
-	}
-
-	// the cached bytes consists of two parts: first a
-	// gob encoding of the status and header, immediately
-	// followed by the raw bytes of the response body
-	rdr := bytes.NewReader(cachedBytes)
-
-	// read the header and status first
-	var hs headerAndStatus
-	err = gob.NewDecoder(rdr).Decode(&hs)
-	if err != nil {
-		return err
-	}
-
-	// set and write the cached headers
-	for k, v := range hs.Header {
-		w.Header()[k] = v
-	}
-	w.WriteHeader(hs.Status)
-
-	// write the cached response body
-	io.Copy(w, rdr)
-
-	return nil
-}
-
-func (c *Cache) getter(ctx groupcache.Context, key string, dest groupcache.Sink) error {
-	combo := ctx.(getterContext)
-
-	// the buffer will store the gob-encoded header, then the body
-	buf := bufPool.Get().(*bytes.Buffer)
-	buf.Reset()
-	defer bufPool.Put(buf)
-
-	// we need to record the response if we are to cache it; only cache if
-	// request is successful (TODO: there's probably much more nuance needed here)
-	rr := caddyhttp.NewResponseRecorder(combo.rw, buf, func(status int, header http.Header) bool {
-		shouldBuf := status < 300
-
-		if shouldBuf {
-			// store the header before the body, so we can efficiently
-			// and conveniently use a single buffer for both; gob
-			// decoder will only read up to end of gob message, and
-			// the rest will be the body, which will be written
-			// implicitly for us by the recorder
-			err := gob.NewEncoder(buf).Encode(headerAndStatus{
-				Header: header,
-				Status: status,
-			})
-			if err != nil {
-				log.Printf("[ERROR] Encoding headers for cache entry: %v; not caching this request", err)
-				return false
-			}
-		}
-
-		return shouldBuf
-	})
-
-	// execute next handlers in chain
-	err := combo.next.ServeHTTP(rr, combo.req)
-	if err != nil {
-		return err
-	}
-
-	// if response body was not buffered, response was
-	// already written and we are unable to cache
-	if !rr.Buffered() {
-		return errUncacheable
-	}
-
-	// add to cache
-	dest.SetBytes(buf.Bytes())
-
-	return nil
-}
-
-type headerAndStatus struct {
-	Header http.Header
-	Status int
-}
-
-type getterContext struct {
-	rw   http.ResponseWriter
-	req  *http.Request
-	next caddyhttp.Handler
-}
-
-var bufPool = sync.Pool{
-	New: func() interface{} {
-		return new(bytes.Buffer)
-	},
-}
-
-var (
-	pool   *groupcache.HTTPPool
-	poolMu sync.Mutex
-)
-
-var errUncacheable = fmt.Errorf("uncacheable")
-
-const groupName = "http_requests"
-
-// Interface guards
-var (
-	_ caddy.Provisioner           = (*Cache)(nil)
-	_ caddy.Validator             = (*Cache)(nil)
-	_ caddyhttp.MiddlewareHandler = (*Cache)(nil)
-)

modules/caddyhttp/matchers.go

@@ -41,7 +41,16 @@ type (
 	// especially A/AAAA pointed at your server.
 	//
 	// Automatic HTTPS can be
-	// [customized or disabled](/docs/json/apps/http/servers/automatic_https/).
+	// [customized or disabled](/docs/modules/http#servers/automatic_https).
+	//
+	// Wildcards (`*`) may be used to represent exactly one label of the
+	// hostname, in accordance with RFC 1034 (because host matchers are also
+	// used for automatic HTTPS which influences TLS certificates). Thus,
+	// a host of `*` matches hosts like `localhost` or `internal` but not
+	// `example.com`. To catch all hosts, omit the host matcher entirely.
+	//
+	// The wildcard can be useful for matching all subdomains, for example:
+	// `*.example.com` matches `foo.example.com` but not `foo.bar.example.com`.
 	MatchHost []string
 
 	// MatchPath matches requests by the URI's path (case-insensitive). Path
@@ -99,17 +108,30 @@ type (
 		cidrs []*net.IPNet
 	}
 
-	// MatchNegate matches requests by negating its matchers' results.
-	// To use, simply specify a set of matchers like you normally would;
-	// the only difference is that their result will be negated.
-	MatchNegate struct {
-		MatchersRaw caddy.ModuleMap `json:"-" caddy:"namespace=http.matchers"`
-
-		Matchers MatcherSet `json:"-"`
+	// MatchNot matches requests by negating the results of its matcher
+	// sets. A single "not" matcher takes one or more matcher sets. Each
+	// matcher set is OR'ed; in other words, if any matcher set returns
+	// true, the final result of the "not" matcher is false. Individual
+	// matchers within a set work the same (i.e. different matchers in
+	// the same set are AND'ed).
+	//
+	// Note that the generated docs which describe the structure of
+	// this module are wrong because of how this type unmarshals JSON
+	// in a custom way. The correct structure is:
+	//
+	// ```json
+	// [
+	// 	{},
+	// 	{}
+	// ]
+	// ```
+	//
+	// where each of the array elements is a matcher set, i.e. an
+	// object keyed by matcher name.
+	MatchNot struct {
+		MatcherSetsRaw []caddy.ModuleMap `json:"-" caddy:"namespace=http.matchers"`
+		MatcherSets    []MatcherSet      `json:"-"`
 	}
-
-	// MatchTable matches requests by values in the table.
-	MatchTable string // TODO: finish implementing
 )
 
 func init() {
@@ -122,7 +144,7 @@ func init() {
 	caddy.RegisterModule(MatchHeaderRE{})
 	caddy.RegisterModule(new(MatchProtocol))
 	caddy.RegisterModule(MatchRemoteIP{})
-	caddy.RegisterModule(MatchNegate{})
+	caddy.RegisterModule(MatchNot{})
 }
 
 // CaddyModule returns the Caddy module information.
@@ -135,7 +157,12 @@ func (MatchHost) CaddyModule() caddy.ModuleInfo {
 
 // UnmarshalCaddyfile implements caddyfile.Unmarshaler.
 func (m *MatchHost) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
-	*m = d.RemainingArgs()
+	for d.Next() {
+		*m = append(*m, d.RemainingArgs()...)
+		if d.NextBlock(0) {
+			return d.Err("malformed host matcher: blocks are not supported")
+		}
+	}
 	return nil
 }
 
@@ -211,9 +238,17 @@ func (m MatchPath) Match(r *http.Request) bool {
 	for _, matchPath := range m {
 		matchPath = repl.ReplaceAll(matchPath, "")
 
+		// special case: whole path is wildcard; this is unnecessary
+		// as it matches all requests, which is the same as no matcher
+		if matchPath == "*" {
+			return true
+		}
+
 		// special case: first and last characters are wildcard,
 		// treat it as a fast substring match
-		if strings.HasPrefix(matchPath, "*") && strings.HasSuffix(matchPath, "*") {
+		if len(matchPath) > 1 &&
+			strings.HasPrefix(matchPath, "*") &&
+			strings.HasSuffix(matchPath, "*") {
 			if strings.Contains(lowerPath, matchPath[1:len(matchPath)-1]) {
 				return true
 			}
@@ -252,7 +287,10 @@ func (m MatchPath) Match(r *http.Request) bool {
 // UnmarshalCaddyfile implements caddyfile.Unmarshaler.
 func (m *MatchPath) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	for d.Next() {
-		*m = d.RemainingArgs()
+		*m = append(*m, d.RemainingArgs()...)
+		if d.NextBlock(0) {
+			return d.Err("malformed path matcher: blocks are not supported")
+		}
 	}
 	return nil
 }
@@ -282,7 +320,10 @@ func (MatchMethod) CaddyModule() caddy.ModuleInfo {
 // UnmarshalCaddyfile implements caddyfile.Unmarshaler.
 func (m *MatchMethod) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	for d.Next() {
-		*m = d.RemainingArgs()
+		*m = append(*m, d.RemainingArgs()...)
+		if d.NextBlock(0) {
+			return d.Err("malformed method matcher: blocks are not supported")
+		}
 	}
 	return nil
 }
@@ -321,6 +362,9 @@ func (m *MatchQuery) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 			return d.Errf("malformed query matcher token: %s; must be in param=val format", d.Val())
 		}
 		url.Values(*m).Set(parts[0], parts[1])
+		if d.NextBlock(0) {
+			return d.Err("malformed query matcher: blocks are not supported")
+		}
 	}
 	return nil
 }
@@ -356,9 +400,12 @@ func (m *MatchHeader) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	for d.Next() {
 		var field, val string
 		if !d.Args(&field, &val) {
-			return d.Errf("expected both field and value")
+			return d.Errf("malformed header matcher: expected both field and value")
 		}
 		http.Header(*m).Set(field, val)
+		if d.NextBlock(0) {
+			return d.Err("malformed header matcher: blocks are not supported")
+		}
 	}
 	return nil
 }
@@ -443,6 +490,10 @@ func (m *MatchHeaderRE) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 		}
 
 		(*m)[field] = &MatchRegexp{Pattern: val, Name: name}
+
+		if d.NextBlock(0) {
+			return d.Err("malformed header_regexp matcher: blocks are not supported")
+		}
 	}
 	return nil
 }
@@ -524,33 +575,24 @@ func (m *MatchProtocol) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 }
 
 // CaddyModule returns the Caddy module information.
-func (MatchNegate) CaddyModule() caddy.ModuleInfo {
+func (MatchNot) CaddyModule() caddy.ModuleInfo {
 	return caddy.ModuleInfo{
 		ID:  "http.matchers.not",
-		New: func() caddy.Module { return new(MatchNegate) },
+		New: func() caddy.Module { return new(MatchNot) },
 	}
 }
 
-// UnmarshalJSON unmarshals data into m's unexported map field.
-// This is done because we cannot embed the map directly into
-// the struct, but we need a struct because we need another
-// field just for the provisioned modules.
-func (m *MatchNegate) UnmarshalJSON(data []byte) error {
-	return json.Unmarshal(data, &m.MatchersRaw)
-}
-
-// MarshalJSON marshals m's matchers.
-func (m MatchNegate) MarshalJSON() ([]byte, error) {
-	return json.Marshal(m.MatchersRaw)
-}
-
 // UnmarshalCaddyfile implements caddyfile.Unmarshaler.
-func (m *MatchNegate) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+func (m *MatchNot) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	// first, unmarshal each matcher in the set from its tokens
-
-	matcherMap := make(map[string]RequestMatcher)
+	type matcherPair struct {
+		raw     caddy.ModuleMap
+		decoded MatcherSet
+	}
 	for d.Next() {
-		for d.NextBlock(0) {
+		var mp matcherPair
+		matcherMap := make(map[string]RequestMatcher)
+		for d.NextArg() || d.NextBlock(0) {
 			matcherName := d.Val()
 			mod, err := caddy.GetModule("http.matchers." + matcherName)
 			if err != nil {
@@ -565,42 +607,64 @@ func (m *MatchNegate) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 				return err
 			}
 			rm := unm.(RequestMatcher)
-			m.Matchers = append(m.Matchers, rm)
 			matcherMap[matcherName] = rm
+			mp.decoded = append(mp.decoded, rm)
 		}
-	}
 
-	// we should now be functional, but we also need
-	// to be able to marshal as JSON, otherwise config
-	// adaptation won't work properly
-	m.MatchersRaw = make(caddy.ModuleMap)
-	for name, matchers := range matcherMap {
-		jsonBytes, err := json.Marshal(matchers)
-		if err != nil {
-			return fmt.Errorf("marshaling matcher %s: %v", name, err)
+		// we should now have a functional 'not' matcher, but we also
+		// need to be able to marshal as JSON, otherwise config
+		// adaptation will be missing the matchers!
+		mp.raw = make(caddy.ModuleMap)
+		for name, matcher := range matcherMap {
+			jsonBytes, err := json.Marshal(matcher)
+			if err != nil {
+				return fmt.Errorf("marshaling %T matcher: %v", matcher, err)
+			}
+			mp.raw[name] = jsonBytes
 		}
-		m.MatchersRaw[name] = jsonBytes
+		m.MatcherSetsRaw = append(m.MatcherSetsRaw, mp.raw)
 	}
-
 	return nil
 }
 
+// UnmarshalJSON satisfies json.Unmarshaler. It puts the JSON
+// bytes directly into m's MatcherSetsRaw field.
+func (m *MatchNot) UnmarshalJSON(data []byte) error {
+	return json.Unmarshal(data, &m.MatcherSetsRaw)
+}
+
+// MarshalJSON satisfies json.Marshaler by marshaling
+// m's raw matcher sets.
+func (m MatchNot) MarshalJSON() ([]byte, error) {
+	return json.Marshal(m.MatcherSetsRaw)
+}
+
 // Provision loads the matcher modules to be negated.
-func (m *MatchNegate) Provision(ctx caddy.Context) error {
-	mods, err := ctx.LoadModule(m, "MatchersRaw")
+func (m *MatchNot) Provision(ctx caddy.Context) error {
+	matcherSets, err := ctx.LoadModule(m, "MatcherSetsRaw")
 	if err != nil {
-		return fmt.Errorf("loading matchers: %v", err)
+		return fmt.Errorf("loading matcher sets: %v", err)
 	}
-	for _, modIface := range mods.(map[string]interface{}) {
-		m.Matchers = append(m.Matchers, modIface.(RequestMatcher))
+	for _, modMap := range matcherSets.([]map[string]interface{}) {
+		var ms MatcherSet
+		for _, modIface := range modMap {
+			ms = append(ms, modIface.(RequestMatcher))
+		}
+		m.MatcherSets = append(m.MatcherSets, ms)
 	}
 	return nil
 }
 
-// Match returns true if r matches m. Since this matcher negates the
-// embedded matchers, false is returned if any of its matchers match.
-func (m MatchNegate) Match(r *http.Request) bool {
-	return !m.Matchers.Match(r)
+// Match returns true if r matches m. Since this matcher negates
+// the embedded matchers, false is returned if any of its matcher
+// sets return true.
+func (m MatchNot) Match(r *http.Request) bool {
+	for _, ms := range m.MatcherSets {
+		if ms.Match(r) {
+			return false
+		}
+	}
+	return true
 }
 
 // CaddyModule returns the Caddy module information.
@@ -614,7 +678,10 @@ func (MatchRemoteIP) CaddyModule() caddy.ModuleInfo {
 // UnmarshalCaddyfile implements caddyfile.Unmarshaler.
 func (m *MatchRemoteIP) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	for d.Next() {
-		m.Ranges = d.RemainingArgs()
+		m.Ranges = append(m.Ranges, d.RemainingArgs()...)
+		if d.NextBlock(0) {
+			return d.Err("malformed remote_ip matcher: blocks are not supported")
+		}
 	}
 	return nil
 }
@@ -765,6 +832,9 @@ func (mre *MatchRegexp) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 		default:
 			return d.ArgErr()
 		}
+		if d.NextBlock(0) {
+			return d.Err("malformed path_regexp matcher: blocks are not supported")
+		}
 	}
 	return nil
 }
@@ -844,8 +914,8 @@ var (
 	_ RequestMatcher    = (*MatchProtocol)(nil)
 	_ RequestMatcher    = (*MatchRemoteIP)(nil)
 	_ caddy.Provisioner = (*MatchRemoteIP)(nil)
-	_ RequestMatcher    = (*MatchNegate)(nil)
-	_ caddy.Provisioner = (*MatchNegate)(nil)
+	_ RequestMatcher    = (*MatchNot)(nil)
+	_ caddy.Provisioner = (*MatchNot)(nil)
 	_ caddy.Provisioner = (*MatchRegexp)(nil)
 
 	_ caddyfile.Unmarshaler = (*MatchHost)(nil)
@@ -858,6 +928,6 @@ var (
 	_ caddyfile.Unmarshaler = (*MatchProtocol)(nil)
 	_ caddyfile.Unmarshaler = (*MatchRemoteIP)(nil)
 
-	_ json.Marshaler   = (*MatchNegate)(nil)
-	_ json.Unmarshaler = (*MatchNegate)(nil)
+	_ json.Marshaler   = (*MatchNot)(nil)
+	_ json.Unmarshaler = (*MatchNot)(nil)
 )

modules/caddyhttp/matchers_test.go

@@ -252,6 +252,26 @@ func TestPathMatcher(t *testing.T) {
 			input:  "/foo/BAR.txt",
 			expect: true,
 		},
+		{
+			match:  MatchPath{"*"},
+			input:  "/",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"*"},
+			input:  "/foo/bar",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"**"},
+			input:  "/",
+			expect: true,
+		},
+		{
+			match:  MatchPath{"**"},
+			input:  "/foo/bar",
+			expect: true,
+		},
 	} {
 		req := &http.Request{URL: &url.URL{Path: tc.input}}
 		repl := caddy.NewReplacer()
@@ -803,6 +823,119 @@ func TestResponseMatcher(t *testing.T) {
 	}
 }
 
+func TestNotMatcher(t *testing.T) {
+	for i, tc := range []struct {
+		host, path string
+		match      MatchNot
+		expect     bool
+	}{
+		{
+			host: "example.com", path: "/",
+			match:  MatchNot{},
+			expect: true,
+		},
+		{
+			host: "example.com", path: "/foo",
+			match: MatchNot{
+				MatcherSets: []MatcherSet{
+					{
+						MatchPath{"/foo"},
+					},
+				},
+			},
+			expect: false,
+		},
+		{
+			host: "example.com", path: "/bar",
+			match: MatchNot{
+				MatcherSets: []MatcherSet{
+					{
+						MatchPath{"/foo"},
+					},
+				},
+			},
+			expect: true,
+		},
+		{
+			host: "example.com", path: "/bar",
+			match: MatchNot{
+				MatcherSets: []MatcherSet{
+					{
+						MatchPath{"/foo"},
+					},
+					{
+						MatchHost{"example.com"},
+					},
+				},
+			},
+			expect: false,
+		},
+		{
+			host: "example.com", path: "/bar",
+			match: MatchNot{
+				MatcherSets: []MatcherSet{
+					{
+						MatchPath{"/bar"},
+					},
+					{
+						MatchHost{"example.com"},
+					},
+				},
+			},
+			expect: false,
+		},
+		{
+			host: "example.com", path: "/foo",
+			match: MatchNot{
+				MatcherSets: []MatcherSet{
+					{
+						MatchPath{"/bar"},
+					},
+					{
+						MatchHost{"sub.example.com"},
+					},
+				},
+			},
+			expect: true,
+		},
+		{
+			host: "example.com", path: "/foo",
+			match: MatchNot{
+				MatcherSets: []MatcherSet{
+					{
+						MatchPath{"/foo"},
+						MatchHost{"example.com"},
+					},
+				},
+			},
+			expect: false,
+		},
+		{
+			host: "example.com", path: "/foo",
+			match: MatchNot{
+				MatcherSets: []MatcherSet{
+					{
+						MatchPath{"/bar"},
+						MatchHost{"example.com"},
+					},
+				},
+			},
+			expect: true,
+		},
+	} {
+		req := &http.Request{Host: tc.host, URL: &url.URL{Path: tc.path}}
+		repl := caddy.NewReplacer()
+		ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
+		req = req.WithContext(ctx)
+
+		actual := tc.match.Match(req)
+		if actual != tc.expect {
+			t.Errorf("Test %d %+v: Expected %t, got %t for: host=%s path=%s'", i, tc.match, tc.expect, actual, tc.host, tc.path)
+			continue
+		}
+	}
+}
+
 func BenchmarkHostMatcherWithoutPlaceholder(b *testing.B) {
 	req := &http.Request{Host: "localhost"}
 	repl := caddy.NewReplacer()

modules/caddyhttp/replacer.go

@@ -31,7 +31,7 @@ import (
 )
 
 func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.ResponseWriter) {
-	httpVars := func(key string) (string, bool) {
+	httpVars := func(key string) (interface{}, bool) {
 		if req != nil {
 			// query string parameters
 			if strings.HasPrefix(key, reqURIQueryReplPrefix) {
@@ -62,7 +62,7 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo
 				}
 			}
 
-			// http.request.tls.
+			// http.request.tls.*
 			if strings.HasPrefix(key, reqTLSReplPrefix) {
 				return getReqTLSReplacement(req, key)
 			}
@@ -85,6 +85,9 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo
 				return host, true
 			case "http.request.port":
 				_, port, _ := net.SplitHostPort(req.Host)
+				if portNum, err := strconv.Atoi(port); err == nil {
+					return portNum, true
+				}
 				return port, true
 			case "http.request.hostport":
 				return req.Host, true
@@ -98,6 +101,9 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo
 				return host, true
 			case "http.request.remote.port":
 				_, port, _ := net.SplitHostPort(req.RemoteAddr)
+				if portNum, err := strconv.Atoi(port); err == nil {
+					return portNum, true
+				}
 				return port, true
 
 			// current URI, including any internal rewrites
@@ -182,21 +188,10 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo
 			if strings.HasPrefix(key, varsReplPrefix) {
 				varName := key[len(varsReplPrefix):]
 				tbl := req.Context().Value(VarsCtxKey).(map[string]interface{})
-				raw, ok := tbl[varName]
-				if !ok {
-					// variables can be dynamic, so always return true
-					// even when it may not be set; treat as empty
-					return "", true
-				}
-				// do our best to convert it to a string efficiently
-				switch val := raw.(type) {
-				case string:
-					return val, true
-				case fmt.Stringer:
-					return val.String(), true
-				default:
-					return fmt.Sprintf("%s", val), true
-				}
+				raw := tbl[varName]
+				// variables can be dynamic, so always return true
+				// even when it may not be set; treat as empty then
+				return raw, true
 			}
 		}
 
@@ -211,19 +206,19 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo
 			}
 		}
 
-		return "", false
+		return nil, false
 	}
 
 	repl.Map(httpVars)
 }
 
-func getReqTLSReplacement(req *http.Request, key string) (string, bool) {
+func getReqTLSReplacement(req *http.Request, key string) (interface{}, bool) {
 	if req == nil || req.TLS == nil {
-		return "", false
+		return nil, false
 	}
 
 	if len(key) < len(reqTLSReplPrefix) {
-		return "", false
+		return nil, false
 	}
 
 	field := strings.ToLower(key[len(reqTLSReplPrefix):])
@@ -231,20 +226,20 @@ func getReqTLSReplacement(req *http.Request, key string) (string, bool) {
 	if strings.HasPrefix(field, "client.") {
 		cert := getTLSPeerCert(req.TLS)
 		if cert == nil {
-			return "", false
+			return nil, false
 		}
 
 		switch field {
 		case "client.fingerprint":
 			return fmt.Sprintf("%x", sha256.Sum256(cert.Raw)), true
 		case "client.issuer":
-			return cert.Issuer.String(), true
+			return cert.Issuer, true
 		case "client.serial":
-			return fmt.Sprintf("%x", cert.SerialNumber), true
+			return cert.SerialNumber, true
 		case "client.subject":
-			return cert.Subject.String(), true
+			return cert.Subject, true
 		default:
-			return "", false
+			return nil, false
 		}
 	}
 
@@ -254,22 +249,15 @@ func getReqTLSReplacement(req *http.Request, key string) (string, bool) {
 	case "cipher_suite":
 		return tls.CipherSuiteName(req.TLS.CipherSuite), true
 	case "resumed":
-		if req.TLS.DidResume {
-			return "true", true
-		}
-		return "false", true
+		return req.TLS.DidResume, true
 	case "proto":
 		return req.TLS.NegotiatedProtocol, true
 	case "proto_mutual":
-		if req.TLS.NegotiatedProtocolIsMutual {
-			return "true", true
-		}
-		return "false", true
+		return req.TLS.NegotiatedProtocolIsMutual, true
 	case "server_name":
 		return req.TLS.ServerName, true
-	default:
-		return "", false
 	}
+	return nil, false
 }
 
 // getTLSPeerCert retrieves the first peer certificate from a TLS session.

modules/caddyhttp/requestbody/requestbody.go

@@ -34,7 +34,7 @@ type RequestBody struct {
 // CaddyModule returns the Caddy module information.
 func (RequestBody) CaddyModule() caddy.ModuleInfo {
 	return caddy.ModuleInfo{
-		ID:  "http.handlers.request_body", // TODO: better name for this?
+		ID:  "http.handlers.request_body",
 		New: func() caddy.Module { return new(RequestBody) },
 	}
 }

modules/caddyhttp/responsewriter.go

@@ -76,13 +76,13 @@ var ErrNotImplemented = fmt.Errorf("method not implemented")
 
 type responseRecorder struct {
 	*ResponseWriterWrapper
-	wroteHeader  bool
 	statusCode   int
 	buf          *bytes.Buffer
 	shouldBuffer ShouldBufferFunc
-	stream       bool
 	size         int
 	header       http.Header
+	wroteHeader  bool
+	stream       bool
 }
 
 // NewResponseRecorder returns a new ResponseRecorder that can be

modules/caddyhttp/reverseproxy/caddyfile.go

@@ -101,78 +101,112 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	// TODO: the logic in this function is kind of sensitive, we need
 	// to write tests before making any more changes to it
 	upstreamDialAddress := func(upstreamAddr string) (string, error) {
-		// slight hack, to ensure a non-URL parses correctly (simplifies our code paths)
-		const undefinedScheme = "undefined"
-		if !strings.Contains(upstreamAddr, "://") {
-			upstreamAddr = undefinedScheme + "://" + upstreamAddr
-		}
+		var network, scheme, host, port string
 
-		// convenient way to get desired scheme, host, and port
-		toURL, err := url.Parse(upstreamAddr)
-		if err != nil {
-			return "", d.Errf("parsing upstream address: %v", err)
-		}
-		if toURL.Scheme == undefinedScheme {
-			toURL.Scheme = ""
-		}
-
-		// there is currently no way to perform a URL rewrite between choosing
-		// a backend and proxying to it, so we cannot allow extra components
-		// in backend URLs
-		if toURL.Path != "" || toURL.RawQuery != "" || toURL.Fragment != "" {
-			return "", d.Err("for now, URLs for proxy upstreams only support scheme, host, and port components")
-		}
-
-		// ensure the port and scheme aren't in conflict
-		urlPort := toURL.Port()
-		if toURL.Scheme == "http" && urlPort == "443" {
-			return "", d.Err("upstream address has conflicting scheme (http://) and port (:443, the HTTPS port)")
-		}
-		if toURL.Scheme == "https" && urlPort == "80" {
-			return "", d.Err("upstream address has conflicting scheme (https://) and port (:80, the HTTP port)")
-		}
-
-		// dial addresses always need a port, so if no port was
-		// specified, assume the default ports for HTTP(S)
-		if urlPort == "" {
-			var toPort string
-			if toURL.Scheme == "" {
-				// if no port or scheme is specified, we assume HTTP
-				toPort = "80"
-			} else if toURL.Scheme == "https" {
-				toPort = "443"
+		if strings.Contains(upstreamAddr, "://") {
+			toURL, err := url.Parse(upstreamAddr)
+			if err != nil {
+				return "", d.Errf("parsing upstream URL: %v", err)
+			}
+
+			// there is currently no way to perform a URL rewrite between choosing
+			// a backend and proxying to it, so we cannot allow extra components
+			// in backend URLs
+			if toURL.Path != "" || toURL.RawQuery != "" || toURL.Fragment != "" {
+				return "", d.Err("for now, URLs for proxy upstreams only support scheme, host, and port components")
+			}
+
+			// ensure the port and scheme aren't in conflict
+			urlPort := toURL.Port()
+			if toURL.Scheme == "http" && urlPort == "443" {
+				return "", d.Err("upstream address has conflicting scheme (http://) and port (:443, the HTTPS port)")
+			}
+			if toURL.Scheme == "https" && urlPort == "80" {
+				return "", d.Err("upstream address has conflicting scheme (https://) and port (:80, the HTTP port)")
+			}
+
+			// if port is missing, attempt to infer from scheme
+			if toURL.Port() == "" {
+				var toPort string
+				switch toURL.Scheme {
+				case "", "http":
+					toPort = "80"
+				case "https":
+					toPort = "443"
+				}
+				toURL.Host = net.JoinHostPort(toURL.Hostname(), toPort)
+			}
+
+			scheme, host, port = toURL.Scheme, toURL.Hostname(), toURL.Port()
+		} else {
+			// extract network manually, since caddy.ParseNetworkAddress() will always add one
+			if idx := strings.Index(upstreamAddr, "/"); idx >= 0 {
+				network = strings.ToLower(strings.TrimSpace(upstreamAddr[:idx]))
+				upstreamAddr = upstreamAddr[idx+1:]
+			}
+			var err error
+			host, port, err = net.SplitHostPort(upstreamAddr)
+			if err != nil {
+				host = upstreamAddr
 			}
-			toURL.Host = net.JoinHostPort(toURL.Host, toPort)
 		}
 
-		// if port is known and scheme is not, set the scheme
-		if toURL.Scheme == "" {
-			if urlPort == "80" {
-				toURL.Scheme = "http"
-			} else if urlPort == "443" {
-				toURL.Scheme = "https"
+		// if scheme is not set, we may be able to infer it from a known port
+		if scheme == "" {
+			if port == "80" {
+				scheme = "http"
+			} else if port == "443" {
+				scheme = "https"
 			}
 		}
 
 		// the underlying JSON does not yet support different
 		// transports (protocols or schemes) to each backend,
 		// so we remember the last one we see and compare them
-		if commonScheme != "" && toURL.Scheme != commonScheme {
+		if commonScheme != "" && scheme != commonScheme {
 			return "", d.Errf("for now, all proxy upstreams must use the same scheme (transport protocol); expecting '%s://' but got '%s://'",
-				commonScheme, toURL.Scheme)
+				commonScheme, scheme)
 		}
-		commonScheme = toURL.Scheme
+		commonScheme = scheme
 
-		return toURL.Host, nil
+		// for simplest possible config, we only need to include
+		// the network portion if the user specified one
+		if network != "" {
+			return caddy.JoinNetworkAddress(network, host, port), nil
+		}
+		return net.JoinHostPort(host, port), nil
+	}
+
+	// appendUpstream creates an upstream for address and adds
+	// it to the list. If the address starts with "srv+" it is
+	// treated as a SRV-based upstream, and any port will be
+	// dropped.
+	appendUpstream := func(address string) error {
+		isSRV := strings.HasPrefix(address, "srv+")
+		if isSRV {
+			address = strings.TrimPrefix(address, "srv+")
+		}
+		dialAddr, err := upstreamDialAddress(address)
+		if err != nil {
+			return err
+		}
+		if isSRV {
+			if host, _, err := net.SplitHostPort(dialAddr); err == nil {
+				dialAddr = host
+			}
+			h.Upstreams = append(h.Upstreams, &Upstream{LookupSRV: dialAddr})
+		} else {
+			h.Upstreams = append(h.Upstreams, &Upstream{Dial: dialAddr})
+		}
+		return nil
 	}
 
 	for d.Next() {
 		for _, up := range d.RemainingArgs() {
-			dialAddr, err := upstreamDialAddress(up)
+			err := appendUpstream(up)
 			if err != nil {
 				return err
 			}
-			h.Upstreams = append(h.Upstreams, &Upstream{Dial: dialAddr})
 		}
 
 		for d.NextBlock(0) {
@@ -183,11 +217,10 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 					return d.ArgErr()
 				}
 				for _, up := range args {
-					dialAddr, err := upstreamDialAddress(up)
+					err := appendUpstream(up)
 					if err != nil {
 						return err
 					}
-					h.Upstreams = append(h.Upstreams, &Upstream{Dial: dialAddr})
 				}
 
 			case "lb_policy":
@@ -518,26 +551,20 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 
 	// verify transport configuration, and finally encode it
 	if transport != nil {
-		// TODO: these two cases are identical, but I don't know how to reuse the code
-		switch ht := transport.(type) {
-		case *HTTPTransport:
-			if commonScheme == "https" && ht.TLS == nil {
-				ht.TLS = new(TLSConfig)
+		if te, ok := transport.(TLSTransport); ok {
+			if commonScheme == "https" && !te.TLSEnabled() {
+				err := te.EnableTLS(new(TLSConfig))
+				if err != nil {
+					return err
+				}
 			}
-			if ht.TLS != nil && commonScheme == "http" {
-				return d.Errf("upstream address scheme is HTTP but transport is configured for HTTP+TLS (HTTPS)")
-			}
-
-		case *NTLMTransport:
-			if commonScheme == "https" && ht.TLS == nil {
-				ht.TLS = new(TLSConfig)
-			}
-			if ht.TLS != nil && commonScheme == "http" {
+			if commonScheme == "http" && te.TLSEnabled() {
 				return d.Errf("upstream address scheme is HTTP but transport is configured for HTTP+TLS (HTTPS)")
 			}
+		} else if commonScheme == "https" {
+			return d.Errf("upstreams are configured for HTTPS but transport module does not support TLS: %T", transport)
 		}
-
-		if !reflect.DeepEqual(transport, new(HTTPTransport)) {
+		if !reflect.DeepEqual(transport, reflect.New(reflect.TypeOf(transport).Elem()).Interface()) {
 			h.TransportRaw = caddyconfig.JSONModuleObject(transport, "protocol", transportModuleName, nil)
 		}
 	}

modules/caddyhttp/reverseproxy/circuitbreaker.go

@@ -24,12 +24,12 @@ import (
 )
 
 func init() {
-	caddy.RegisterModule(localCircuitBreaker{})
+	caddy.RegisterModule(internalCircuitBreaker{})
 }
 
-// localCircuitBreaker implements circuit breaking functionality
+// internalCircuitBreaker implements circuit breaking functionality
 // for requests within this process over a sliding time window.
-type localCircuitBreaker struct {
+type internalCircuitBreaker struct {
 	tripped   int32
 	cbFactor  int32
 	threshold float64
@@ -39,15 +39,15 @@ type localCircuitBreaker struct {
 }
 
 // CaddyModule returns the Caddy module information.
-func (localCircuitBreaker) CaddyModule() caddy.ModuleInfo {
+func (internalCircuitBreaker) CaddyModule() caddy.ModuleInfo {
 	return caddy.ModuleInfo{
-		ID:  "http.reverse_proxy.circuit_breakers.local",
-		New: func() caddy.Module { return new(localCircuitBreaker) },
+		ID:  "http.reverse_proxy.circuit_breakers.internal",
+		New: func() caddy.Module { return new(internalCircuitBreaker) },
 	}
 }
 
 // Provision sets up a configured circuit breaker.
-func (c *localCircuitBreaker) Provision(ctx caddy.Context) error {
+func (c *internalCircuitBreaker) Provision(ctx caddy.Context) error {
 	f, ok := typeCB[c.Factor]
 	if !ok {
 		return fmt.Errorf("type is not defined")
@@ -77,19 +77,19 @@ func (c *localCircuitBreaker) Provision(ctx caddy.Context) error {
 }
 
 // Ok returns whether the circuit breaker is tripped or not.
-func (c *localCircuitBreaker) Ok() bool {
+func (c *internalCircuitBreaker) Ok() bool {
 	tripped := atomic.LoadInt32(&c.tripped)
 	return tripped == 0
 }
 
 // RecordMetric records a response status code and execution time of a request. This function should be run in a separate goroutine.
-func (c *localCircuitBreaker) RecordMetric(statusCode int, latency time.Duration) {
+func (c *internalCircuitBreaker) RecordMetric(statusCode int, latency time.Duration) {
 	c.metrics.Record(statusCode, latency)
 	c.checkAndSet()
 }
 
 // Ok checks our metrics to see if we should trip our circuit breaker, or if the fallback duration has completed.
-func (c *localCircuitBreaker) checkAndSet() {
+func (c *internalCircuitBreaker) checkAndSet() {
 	var isTripped bool
 
 	switch c.cbFactor {

modules/caddyhttp/reverseproxy/command.go

@@ -25,11 +25,9 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
-	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
 	caddycmd "github.com/caddyserver/caddy/v2/cmd"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp/headers"
-	"github.com/mholt/certmagic"
 )
 
 func init() {
@@ -53,7 +51,7 @@ default, all incoming headers are passed through unmodified.)
 `,
 		Flags: func() *flag.FlagSet {
 			fs := flag.NewFlagSet("file-server", flag.ExitOnError)
-			fs.String("from", "", "Address on which to receive traffic")
+			fs.String("from", "localhost:443", "Address on which to receive traffic")
 			fs.String("to", "", "Upstream address to which to to proxy traffic")
 			fs.Bool("change-host-header", false, "Set upstream Host header to address of upstream")
 			return fs
@@ -67,7 +65,7 @@ func cmdReverseProxy(fs caddycmd.Flags) (int, error) {
 	changeHost := fs.Bool("change-host-header")
 
 	if from == "" {
-		from = "localhost:" + httpcaddyfile.DefaultPort
+		from = "localhost:443"
 	}
 
 	// URLs need a scheme in order to parse successfully
@@ -123,17 +121,15 @@ func cmdReverseProxy(fs caddycmd.Flags) (int, error) {
 	urlHost := fromURL.Hostname()
 	if urlHost != "" {
 		route.MatcherSetsRaw = []caddy.ModuleMap{
-			caddy.ModuleMap{
+			{
 				"host": caddyconfig.JSON(caddyhttp.MatchHost{urlHost}, nil),
 			},
 		}
 	}
 
-	listen := ":80"
+	listen := ":443"
 	if urlPort := fromURL.Port(); urlPort != "" {
 		listen = ":" + urlPort
-	} else if certmagic.HostQualifies(urlHost) {
-		listen = ":443"
 	}
 
 	server := &caddyhttp.Server{

modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go

@@ -51,10 +51,10 @@ func (t *Transport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 				t.Root = d.Val()
 
 			case "split":
-				if !d.NextArg() {
+				t.SplitPath = d.RemainingArgs()
+				if len(t.SplitPath) == 0 {
 					return d.ArgErr()
 				}
-				t.SplitPath = d.Val()
 
 			case "env":
 				args := d.RemainingArgs()
@@ -127,9 +127,11 @@ func parsePHPFastCGI(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error
 		"file": h.JSON(fileserver.MatchFile{
 			TryFiles: []string{"{http.request.uri.path}/index.php"},
 		}),
-		"not": h.JSON(caddyhttp.MatchNegate{
-			MatchersRaw: caddy.ModuleMap{
-				"path": h.JSON(caddyhttp.MatchPath{"*/"}),
+		"not": h.JSON(caddyhttp.MatchNot{
+			MatcherSetsRaw: []caddy.ModuleMap{
+				{
+					"path": h.JSON(caddyhttp.MatchPath{"*/"}),
+				},
 			},
 		}),
 	}
@@ -173,7 +175,7 @@ func parsePHPFastCGI(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error
 	}
 
 	// set up the transport for FastCGI, and specifically PHP
-	fcgiTransport := Transport{SplitPath: ".php"}
+	fcgiTransport := Transport{SplitPath: []string{".php"}}
 
 	// create the reverse proxy handler which uses our FastCGI transport
 	rpHandler := &reverseproxy.Handler{

modules/caddyhttp/reverseproxy/fastcgi/fastcgi.go

@@ -29,6 +29,7 @@ import (
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
+	"go.uber.org/zap"
 
 	"github.com/caddyserver/caddy/v2"
 )
@@ -47,10 +48,11 @@ type Transport struct {
 	// with the value of SplitPath. The first piece will be assumed as the
 	// actual resource (CGI script) name, and the second piece will be set to
 	// PATH_INFO for the CGI script to use.
+	//
 	// Future enhancements should be careful to avoid CVE-2019-11043,
 	// which can be mitigated with use of a try_files-like behavior
-	// that 404's if the fastcgi path info is not found.
-	SplitPath string `json:"split_path,omitempty"`
+	// that 404s if the fastcgi path info is not found.
+	SplitPath []string `json:"split_path,omitempty"`
 
 	// Extra environment variables.
 	EnvVars map[string]string `json:"env,omitempty"`
@@ -65,6 +67,7 @@ type Transport struct {
 	WriteTimeout caddy.Duration `json:"write_timeout,omitempty"`
 
 	serverSoftware string
+	logger         *zap.Logger
 }
 
 // CaddyModule returns the Caddy module information.
@@ -76,7 +79,8 @@ func (Transport) CaddyModule() caddy.ModuleInfo {
 }
 
 // Provision sets up t.
-func (t *Transport) Provision(_ caddy.Context) error {
+func (t *Transport) Provision(ctx caddy.Context) error {
+	t.logger = ctx.Logger(t)
 	if t.Root == "" {
 		t.Root = "{http.vars.root}"
 	}
@@ -109,6 +113,12 @@ func (t Transport) RoundTrip(r *http.Request) (*http.Response, error) {
 		address = dialInfo.Address
 	}
 
+	t.logger.Debug("roundtrip",
+		zap.Object("request", caddyhttp.LoggableHTTPRequest{Request: r}),
+		zap.String("dial", address),
+		zap.Any("env", env), // TODO: this uses reflection I think
+	)
+
 	fcgiBackend, err := DialContext(ctx, network, address)
 	if err != nil {
 		// TODO: wrap in a special error type if the dial failed, so retries can happen if enabled
@@ -163,17 +173,22 @@ func (t Transport) buildEnv(r *http.Request) (map[string]string, error) {
 	ip = strings.Replace(ip, "[", "", 1)
 	ip = strings.Replace(ip, "]", "", 1)
 
-	root := repl.ReplaceAll(t.Root, ".")
+	// make sure file root is absolute
+	root, err := filepath.Abs(repl.ReplaceAll(t.Root, "."))
+	if err != nil {
+		return nil, err
+	}
+
 	fpath := r.URL.Path
 
-	// Split path in preparation for env variables.
-	// Previous canSplit checks ensure this can never be -1.
-	// TODO: I haven't brought over canSplit; make sure this doesn't break
-	splitPos := t.splitPos(fpath)
-
-	// Request has the extension; path was split successfully
-	docURI := fpath[:splitPos+len(t.SplitPath)]
-	pathInfo := fpath[splitPos+len(t.SplitPath):]
+	// split "actual path" from "path info" if configured
+	var docURI, pathInfo string
+	if splitPos := t.splitPos(fpath); splitPos > -1 {
+		docURI = fpath[:splitPos]
+		pathInfo = fpath[splitPos:]
+	} else {
+		docURI = fpath
+	}
 	scriptName := fpath
 
 	// Strip PATH_INFO from SCRIPT_NAME
@@ -259,9 +274,9 @@ func (t Transport) buildEnv(r *http.Request) (map[string]string, error) {
 			env["SSL_PROTOCOL"] = v
 		}
 		// and pass the cipher suite in a manner compatible with apache's mod_ssl
-		for k, v := range caddytls.SupportedCipherSuites {
-			if v == r.TLS.CipherSuite {
-				env["SSL_CIPHER"] = k
+		for _, cs := range caddytls.SupportedCipherSuites() {
+			if cs.ID == r.TLS.CipherSuite {
+				env["SSL_CIPHER"] = cs.Name
 				break
 			}
 		}
@@ -284,14 +299,19 @@ func (t Transport) buildEnv(r *http.Request) (map[string]string, error) {
 // splitPos returns the index where path should
 // be split based on t.SplitPath.
 func (t Transport) splitPos(path string) int {
-	// TODO:
+	// TODO: from v1...
 	// if httpserver.CaseSensitivePath {
 	// 	return strings.Index(path, r.SplitPath)
 	// }
-	return strings.Index(strings.ToLower(path), strings.ToLower(t.SplitPath))
+	lowerPath := strings.ToLower(path)
+	for _, split := range t.SplitPath {
+		if idx := strings.Index(lowerPath, strings.ToLower(split)); idx > -1 {
+			return idx + len(split)
+		}
+	}
+	return -1
 }
 
-// TODO:
 // Map of supported protocols to Apache ssl_mod format
 // Note that these are slightly different from SupportedProtocols in caddytls/config.go
 var tlsProtocolStrings = map[uint16]string{

modules/caddyhttp/reverseproxy/hosts.go

@@ -17,6 +17,8 @@ package reverseproxy
 import (
 	"context"
 	"fmt"
+	"net"
+	"net/http"
 	"strconv"
 	"sync/atomic"
 
@@ -63,10 +65,10 @@ type UpstreamPool []*Upstream
 type Upstream struct {
 	Host `json:"-"`
 
-	// The [network address](/docs/json/apps/http/#servers/listen)
+	// The [network address](/docs/conventions#network-addresses)
 	// to dial to connect to the upstream. Must represent precisely
 	// one socket (i.e. no port ranges). A valid network address
-	// either has a host and port, or is a unix socket address.
+	// either has a host and port or is a unix socket address.
 	//
 	// Placeholders may be used to make the upstream dynamic, but be
 	// aware of the health check implications of this: a single
@@ -75,6 +77,11 @@ type Upstream struct {
 	// backends is down. Also be aware of open proxy vulnerabilities.
 	Dial string `json:"dial,omitempty"`
 
+	// If DNS SRV records are used for service discovery with this
+	// upstream, specify the DNS name for which to look up SRV
+	// records here, instead of specifying a dial address.
+	LookupSRV string `json:"lookup_srv,omitempty"`
+
 	// The maximum number of simultaneous requests to allow to
 	// this upstream. If set, overrides the global passive health
 	// check UnhealthyRequestCount value.
@@ -89,6 +96,13 @@ type Upstream struct {
 	cb                CircuitBreaker
 }
 
+func (u Upstream) String() string {
+	if u.LookupSRV != "" {
+		return u.LookupSRV
+	}
+	return u.Dial
+}
+
 // Available returns true if the remote host
 // is available to receive requests. This is
 // the method that should be used by selection
@@ -118,6 +132,47 @@ func (u *Upstream) Full() bool {
 	return u.MaxRequests > 0 && u.Host.NumRequests() >= u.MaxRequests
 }
 
+// fillDialInfo returns a filled DialInfo for upstream u, using the request
+// context. If the upstream has a SRV lookup configured, that is done and a
+// returned address is chosen; otherwise, the upstream's regular dial address
+// field is used. Note that the returned value is not a pointer.
+func (u *Upstream) fillDialInfo(r *http.Request) (DialInfo, error) {
+	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
+	var addr caddy.NetworkAddress
+
+	if u.LookupSRV != "" {
+		// perform DNS lookup for SRV records and choose one
+		srvName := repl.ReplaceAll(u.LookupSRV, "")
+		_, records, err := net.DefaultResolver.LookupSRV(r.Context(), "", "", srvName)
+		if err != nil {
+			return DialInfo{}, err
+		}
+		addr.Network = "tcp"
+		addr.Host = records[0].Target
+		addr.StartPort, addr.EndPort = uint(records[0].Port), uint(records[0].Port)
+	} else {
+		// use provided dial address
+		var err error
+		dial := repl.ReplaceAll(u.Dial, "")
+		addr, err = caddy.ParseNetworkAddress(dial)
+		if err != nil {
+			return DialInfo{}, fmt.Errorf("upstream %s: invalid dial address %s: %v", u.Dial, dial, err)
+		}
+		if numPorts := addr.PortRangeSize(); numPorts != 1 {
+			return DialInfo{}, fmt.Errorf("upstream %s: dial address must represent precisely one socket: %s represents %d",
+				u.Dial, dial, numPorts)
+		}
+	}
+
+	return DialInfo{
+		Upstream: u,
+		Network:  addr.Network,
+		Address:  addr.JoinHostPort(0),
+		Host:     addr.Host,
+		Port:     strconv.Itoa(int(addr.StartPort)),
+	}, nil
+}
+
 // upstreamHost is the basic, in-memory representation
 // of the state of a remote host. It implements the
 // Host interface.
@@ -204,27 +259,6 @@ func (di DialInfo) String() string {
 	return caddy.JoinNetworkAddress(di.Network, di.Host, di.Port)
 }
 
-// fillDialInfo returns a filled DialInfo for the given upstream, using
-// the given Replacer. Note that the returned value is not a pointer.
-func fillDialInfo(upstream *Upstream, repl *caddy.Replacer) (DialInfo, error) {
-	dial := repl.ReplaceAll(upstream.Dial, "")
-	addr, err := caddy.ParseNetworkAddress(dial)
-	if err != nil {
-		return DialInfo{}, fmt.Errorf("upstream %s: invalid dial address %s: %v", upstream.Dial, dial, err)
-	}
-	if numPorts := addr.PortRangeSize(); numPorts != 1 {
-		return DialInfo{}, fmt.Errorf("upstream %s: dial address must represent precisely one socket: %s represents %d",
-			upstream.Dial, dial, numPorts)
-	}
-	return DialInfo{
-		Upstream: upstream,
-		Network:  addr.Network,
-		Address:  addr.JoinHostPort(0),
-		Host:     addr.Host,
-		Port:     strconv.Itoa(int(addr.StartPort)),
-	}, nil
-}
-
 // GetDialInfo gets the upstream dialing info out of the context,
 // and returns true if there was a valid value; false otherwise.
 func GetDialInfo(ctx context.Context) (DialInfo, bool) {

modules/caddyhttp/reverseproxy/httptransport.go

@@ -42,19 +42,50 @@ type HTTPTransport struct {
 	// able to borrow/use at least some of these config fields; if so,
 	// maybe move them into a type called CommonTransport and embed it?
 
-	TLS                   *TLSConfig     `json:"tls,omitempty"`
-	KeepAlive             *KeepAlive     `json:"keep_alive,omitempty"`
-	Compression           *bool          `json:"compression,omitempty"`
-	MaxConnsPerHost       int            `json:"max_conns_per_host,omitempty"`
-	DialTimeout           caddy.Duration `json:"dial_timeout,omitempty"`
-	FallbackDelay         caddy.Duration `json:"dial_fallback_delay,omitempty"`
-	ResponseHeaderTimeout caddy.Duration `json:"response_header_timeout,omitempty"`
-	ExpectContinueTimeout caddy.Duration `json:"expect_continue_timeout,omitempty"`
-	MaxResponseHeaderSize int64          `json:"max_response_header_size,omitempty"`
-	WriteBufferSize       int            `json:"write_buffer_size,omitempty"`
-	ReadBufferSize        int            `json:"read_buffer_size,omitempty"`
-	Versions              []string       `json:"versions,omitempty"`
+	// Configures TLS to the upstream. Setting this to an empty struct
+	// is sufficient to enable TLS with reasonable defaults.
+	TLS *TLSConfig `json:"tls,omitempty"`
 
+	// Configures HTTP Keep-Alive (enabled by default). Should only be
+	// necessary if rigorous testing has shown that tuning this helps
+	// improve performance.
+	KeepAlive *KeepAlive `json:"keep_alive,omitempty"`
+
+	// Whether to enable compression to upstream. Default: true
+	Compression *bool `json:"compression,omitempty"`
+
+	// Maximum number of connections per host. Default: 0 (no limit)
+	MaxConnsPerHost int `json:"max_conns_per_host,omitempty"`
+
+	// How long to wait before timing out trying to connect to
+	// an upstream.
+	DialTimeout caddy.Duration `json:"dial_timeout,omitempty"`
+
+	// How long to wait before spawning an RFC 6555 Fast Fallback
+	// connection. A negative value disables this.
+	FallbackDelay caddy.Duration `json:"dial_fallback_delay,omitempty"`
+
+	// How long to wait for reading response headers from server.
+	ResponseHeaderTimeout caddy.Duration `json:"response_header_timeout,omitempty"`
+
+	// The length of time to wait for a server's first response
+	// headers after fully writing the request headers if the
+	// request has a header "Expect: 100-continue".
+	ExpectContinueTimeout caddy.Duration `json:"expect_continue_timeout,omitempty"`
+
+	// The maximum bytes to read from response headers.
+	MaxResponseHeaderSize int64 `json:"max_response_header_size,omitempty"`
+
+	// The size of the write buffer in bytes.
+	WriteBufferSize int `json:"write_buffer_size,omitempty"`
+
+	// The size of the read buffer in bytes.
+	ReadBufferSize int `json:"read_buffer_size,omitempty"`
+
+	// The versions of HTTP to support. Default: ["1.1", "2"]
+	Versions []string `json:"versions,omitempty"`
+
+	// The pre-configured underlying HTTP transport.
 	Transport *http.Transport `json:"-"`
 }
 
@@ -68,12 +99,12 @@ func (HTTPTransport) CaddyModule() caddy.ModuleInfo {
 
 // Provision sets up h.Transport with a *http.Transport
 // that is ready to use.
-func (h *HTTPTransport) Provision(_ caddy.Context) error {
+func (h *HTTPTransport) Provision(ctx caddy.Context) error {
 	if len(h.Versions) == 0 {
 		h.Versions = []string{"1.1", "2"}
 	}
 
-	rt, err := h.newTransport()
+	rt, err := h.NewTransport(ctx)
 	if err != nil {
 		return err
 	}
@@ -82,7 +113,9 @@ func (h *HTTPTransport) Provision(_ caddy.Context) error {
 	return nil
 }
 
-func (h *HTTPTransport) newTransport() (*http.Transport, error) {
+// NewTransport builds a standard-lib-compatible
+// http.Transport value from h.
+func (h *HTTPTransport) NewTransport(_ caddy.Context) (*http.Transport, error) {
 	dialer := &net.Dialer{
 		Timeout:       time.Duration(h.DialTimeout),
 		FallbackDelay: time.Duration(h.FallbackDelay),
@@ -148,14 +181,14 @@ func (h *HTTPTransport) newTransport() (*http.Transport, error) {
 
 // RoundTrip implements http.RoundTripper.
 func (h *HTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	h.setScheme(req)
+	h.SetScheme(req)
 	return h.Transport.RoundTrip(req)
 }
 
-// setScheme ensures that the outbound request req
+// SetScheme ensures that the outbound request req
 // has the scheme set in its URL; the underlying
 // http.Transport requires a scheme to be set.
-func (h *HTTPTransport) setScheme(req *http.Request) {
+func (h *HTTPTransport) SetScheme(req *http.Request) {
 	if req.URL.Scheme == "" {
 		req.URL.Scheme = "http"
 		if h.TLS != nil {
@@ -164,6 +197,17 @@ func (h *HTTPTransport) setScheme(req *http.Request) {
 	}
 }
 
+// TLSEnabled returns true if TLS is enabled.
+func (h HTTPTransport) TLSEnabled() bool {
+	return h.TLS != nil
+}
+
+// EnableTLS enables TLS on the transport.
+func (h *HTTPTransport) EnableTLS(base *TLSConfig) error {
+	h.TLS = base
+	return nil
+}
+
 // Cleanup implements caddy.CleanerUpper and closes any idle connections.
 func (h HTTPTransport) Cleanup() error {
 	if h.Transport == nil {
@@ -173,18 +217,32 @@ func (h HTTPTransport) Cleanup() error {
 	return nil
 }
 
-// TLSConfig holds configuration related to the
-// TLS configuration for the transport/client.
+// TLSConfig holds configuration related to the TLS configuration for the
+// transport/client.
 type TLSConfig struct {
+	// Optional list of base64-encoded DER-encoded CA certificates to trust.
 	RootCAPool []string `json:"root_ca_pool,omitempty"`
-	// Added to the same pool as above, but brought in from files
+
+	// List of PEM-encoded CA certificate files to add to the same trust
+	// store as RootCAPool (or root_ca_pool in the JSON).
 	RootCAPEMFiles []string `json:"root_ca_pem_files,omitempty"`
-	// TODO: Should the client cert+key config use caddytls.CertificateLoader modules?
-	ClientCertificateFile    string         `json:"client_certificate_file,omitempty"`
-	ClientCertificateKeyFile string         `json:"client_certificate_key_file,omitempty"`
-	InsecureSkipVerify       bool           `json:"insecure_skip_verify,omitempty"`
-	HandshakeTimeout         caddy.Duration `json:"handshake_timeout,omitempty"`
-	ServerName               string         `json:"server_name,omitempty"`
+
+	// PEM-encoded client certificate filename to present to servers.
+	ClientCertificateFile string `json:"client_certificate_file,omitempty"`
+
+	// PEM-encoded key to use with the client certificate.
+	ClientCertificateKeyFile string `json:"client_certificate_key_file,omitempty"`
+
+	// If true, TLS verification of server certificates will be disabled.
+	// This is insecure and may be removed in the future. Do not use this
+	// option except in testing or local development environments.
+	InsecureSkipVerify bool `json:"insecure_skip_verify,omitempty"`
+
+	// The duration to allow a TLS handshake to a server.
+	HandshakeTimeout caddy.Duration `json:"handshake_timeout,omitempty"`
+
+	// The server name (SNI) to use in TLS handshakes.
+	ServerName string `json:"server_name,omitempty"`
 }
 
 // MakeTLSClientConfig returns a tls.Config usable by a client to a backend.
@@ -244,11 +302,20 @@ func (t TLSConfig) MakeTLSClientConfig() (*tls.Config, error) {
 
 // KeepAlive holds configuration pertaining to HTTP Keep-Alive.
 type KeepAlive struct {
-	Enabled             *bool          `json:"enabled,omitempty"`
-	ProbeInterval       caddy.Duration `json:"probe_interval,omitempty"`
-	MaxIdleConns        int            `json:"max_idle_conns,omitempty"`
-	MaxIdleConnsPerHost int            `json:"max_idle_conns_per_host,omitempty"`
-	IdleConnTimeout     caddy.Duration `json:"idle_timeout,omitempty"` // how long should connections be kept alive when idle
+	// Whether HTTP Keep-Alive is enabled. Default: true
+	Enabled *bool `json:"enabled,omitempty"`
+
+	// How often to probe for liveness.
+	ProbeInterval caddy.Duration `json:"probe_interval,omitempty"`
+
+	// Maximum number of idle connections.
+	MaxIdleConns int `json:"max_idle_conns,omitempty"`
+
+	// Maximum number of idle connections per upstream host.
+	MaxIdleConnsPerHost int `json:"max_idle_conns_per_host,omitempty"`
+
+	// How long connections should be kept alive when idle.
+	IdleConnTimeout caddy.Duration `json:"idle_timeout,omitempty"`
 }
 
 // decodeBase64DERCert base64-decodes, then DER-decodes, certStr.
@@ -278,4 +345,5 @@ var (
 	_ caddy.Provisioner  = (*HTTPTransport)(nil)
 	_ http.RoundTripper  = (*HTTPTransport)(nil)
 	_ caddy.CleanerUpper = (*HTTPTransport)(nil)
+	_ TLSTransport       = (*HTTPTransport)(nil)
 )

modules/caddyhttp/reverseproxy/ntlm.go

@@ -1,240 +0,0 @@
-// Copyright 2015 Matthew Holt and The Caddy Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package reverseproxy
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"net/http"
-	"strings"
-	"sync"
-
-	"github.com/caddyserver/caddy/v2"
-	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
-)
-
-func init() {
-	caddy.RegisterModule(NTLMTransport{})
-}
-
-// NTLMTransport proxies HTTP with NTLM authentication.
-// It basically wraps HTTPTransport so that it is compatible with
-// NTLM's HTTP-hostile requirements. Specifically, it will use
-// HTTPTransport's single, default *http.Transport for all requests
-// (unless the client's connection is already mapped to a different
-// transport) until a request comes in with an Authorization header
-// that has "NTLM" or "Negotiate"; when that happens, NTLMTransport
-// maps the client's connection (by its address, req.RemoteAddr)
-// to a new transport that is used only by that downstream conn.
-// When the upstream connection is closed, the mapping is deleted.
-// This preserves NTLM authentication contexts by ensuring that
-// client connections use the same upstream connection. It does
-// hurt performance a bit, but that's NTLM for you.
-//
-// This transport also forces HTTP/1.1 and Keep-Alives in order
-// for NTLM to succeed.
-//
-// It is basically the same thing as
-// [nginx's paid ntlm directive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#ntlm)
-// (but is free in Caddy!).
-type NTLMTransport struct {
-	*HTTPTransport
-
-	transports   map[string]*http.Transport
-	transportsMu *sync.RWMutex
-}
-
-// CaddyModule returns the Caddy module information.
-func (NTLMTransport) CaddyModule() caddy.ModuleInfo {
-	return caddy.ModuleInfo{
-		ID:  "http.reverse_proxy.transport.http_ntlm",
-		New: func() caddy.Module { return new(NTLMTransport) },
-	}
-}
-
-// Provision sets up the transport module.
-func (n *NTLMTransport) Provision(ctx caddy.Context) error {
-	n.transports = make(map[string]*http.Transport)
-	n.transportsMu = new(sync.RWMutex)
-
-	if n.HTTPTransport == nil {
-		n.HTTPTransport = new(HTTPTransport)
-	}
-
-	// NTLM requires HTTP/1.1
-	n.HTTPTransport.Versions = []string{"1.1"}
-
-	// NLTM requires keep-alive
-	if n.HTTPTransport.KeepAlive != nil {
-		enabled := true
-		n.HTTPTransport.KeepAlive.Enabled = &enabled
-	}
-
-	// set up the underlying transport, since we
-	// rely on it for the heavy lifting
-	err := n.HTTPTransport.Provision(ctx)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// RoundTrip implements http.RoundTripper. It basically wraps
-// the underlying HTTPTransport.Transport in a way that preserves
-// NTLM context by mapping transports/connections. Note that this
-// method does not call n.HTTPTransport.RoundTrip (our own method),
-// but the underlying n.HTTPTransport.Transport.RoundTrip (standard
-// library's method).
-func (n *NTLMTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	n.HTTPTransport.setScheme(req)
-
-	// when the upstream connection is closed, make sure
-	// we close the downstream connection with the client
-	// when this request is done; we only do this if
-	// using a bound transport
-	closeDownstreamIfClosedUpstream := func() {
-		n.transportsMu.Lock()
-		if _, ok := n.transports[req.RemoteAddr]; !ok {
-			req.Close = true
-		}
-		n.transportsMu.Unlock()
-	}
-
-	// first, see if this downstream connection is
-	// already bound to a particular transport
-	// (transports are abstractions over connections
-	// to our upstream, and NTLM auth requires
-	// preserving authentication state for separate
-	// connections over multiple roundtrips, sigh)
-	n.transportsMu.Lock()
-	transport, ok := n.transports[req.RemoteAddr]
-	if ok {
-		n.transportsMu.Unlock()
-		defer closeDownstreamIfClosedUpstream()
-		return transport.RoundTrip(req)
-	}
-
-	// otherwise, start by assuming we will use
-	// the default transport that carries all
-	// normal/non-NTLM-authenticated requests
-	transport = n.HTTPTransport.Transport
-
-	// but if this request begins the NTLM authentication
-	// process, we need to pin it to a specific transport
-	if requestHasAuth(req) {
-		var err error
-		transport, err = n.newTransport()
-		if err != nil {
-			return nil, fmt.Errorf("making new transport for %s: %v", req.RemoteAddr, err)
-		}
-		n.transports[req.RemoteAddr] = transport
-		defer closeDownstreamIfClosedUpstream()
-	}
-	n.transportsMu.Unlock()
-
-	// finally, do the roundtrip with the transport we selected
-	return transport.RoundTrip(req)
-}
-
-// newTransport makes an NTLM-compatible transport.
-func (n *NTLMTransport) newTransport() (*http.Transport, error) {
-	// start with a regular HTTP transport
-	transport, err := n.HTTPTransport.newTransport()
-	if err != nil {
-		return nil, err
-	}
-
-	// we need to wrap upstream connections so we can
-	// clean up in two ways when that connection is
-	// closed: 1) destroy the transport that housed
-	// this connection, and 2) use that as a signal
-	// to close the connection to the downstream.
-	wrappedDialContext := transport.DialContext
-
-	transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
-		conn2, err := wrappedDialContext(ctx, network, address)
-		if err != nil {
-			return nil, err
-		}
-		req := ctx.Value(caddyhttp.OriginalRequestCtxKey).(http.Request)
-		conn := &unbinderConn{Conn: conn2, ntlm: n, clientAddr: req.RemoteAddr}
-		return conn, nil
-	}
-
-	return transport, nil
-}
-
-// Cleanup implements caddy.CleanerUpper and closes any idle connections.
-func (n *NTLMTransport) Cleanup() error {
-	if err := n.HTTPTransport.Cleanup(); err != nil {
-		return err
-	}
-
-	n.transportsMu.Lock()
-	for _, t := range n.transports {
-		t.CloseIdleConnections()
-	}
-	n.transports = make(map[string]*http.Transport)
-	n.transportsMu.Unlock()
-
-	return nil
-}
-
-// deleteTransportsForClient deletes (unmaps) transports that are
-// associated with clientAddr (a req.RemoteAddr value).
-func (n *NTLMTransport) deleteTransportsForClient(clientAddr string) {
-	n.transportsMu.Lock()
-	for key := range n.transports {
-		if key == clientAddr {
-			delete(n.transports, key)
-		}
-	}
-	n.transportsMu.Unlock()
-}
-
-// requestHasAuth returns true if req has an Authorization
-// header with values "NTLM" or "Negotiate".
-func requestHasAuth(req *http.Request) bool {
-	for _, val := range req.Header["Authorization"] {
-		if strings.HasPrefix(val, "NTLM") ||
-			strings.HasPrefix(val, "Negotiate") {
-			return true
-		}
-	}
-	return false
-}
-
-// unbinderConn is used to wrap upstream connections
-// so that we know when they are closed and can clean
-// up after that.
-type unbinderConn struct {
-	net.Conn
-	clientAddr string
-	ntlm       *NTLMTransport
-}
-
-func (uc *unbinderConn) Close() error {
-	uc.ntlm.deleteTransportsForClient(uc.clientAddr)
-	return uc.Conn.Close()
-}
-
-// Interface guards
-var (
-	_ caddy.Provisioner  = (*NTLMTransport)(nil)
-	_ http.RoundTripper  = (*NTLMTransport)(nil)
-	_ caddy.CleanerUpper = (*NTLMTransport)(nil)
-)

modules/caddyhttp/reverseproxy/reverseproxy.go

@@ -24,7 +24,6 @@ import (
 	"net"
 	"net/http"
 	"regexp"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -172,7 +171,7 @@ func (h *Handler) Provision(ctx caddy.Context) error {
 	for _, upstream := range h.Upstreams {
 		// create or get the host representation for this upstream
 		var host Host = new(upstreamHost)
-		existingHost, loaded := hosts.LoadOrStore(upstream.Dial, host)
+		existingHost, loaded := hosts.LoadOrStore(upstream.String(), host)
 		if loaded {
 			host = existingHost.(Host)
 		}
@@ -252,7 +251,7 @@ func (h *Handler) Cleanup() error {
 
 	// remove hosts from our config from the pool
 	for _, upstream := range h.Upstreams {
-		hosts.Delete(upstream.Dial)
+		hosts.Delete(upstream.String())
 	}
 
 	return nil
@@ -313,7 +312,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht
 		// the dial address may vary per-request if placeholders are
 		// used, so perform those replacements here; the resulting
 		// DialInfo struct should have valid network address syntax
-		dialInfo, err := fillDialInfo(upstream, repl)
+		dialInfo, err := upstream.fillDialInfo(r)
 		if err != nil {
 			return fmt.Errorf("making dial info: %v", err)
 		}
@@ -328,9 +327,9 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht
 		repl.Set("http.reverse_proxy.upstream.hostport", dialInfo.Address)
 		repl.Set("http.reverse_proxy.upstream.host", dialInfo.Host)
 		repl.Set("http.reverse_proxy.upstream.port", dialInfo.Port)
-		repl.Set("http.reverse_proxy.upstream.requests", strconv.Itoa(upstream.Host.NumRequests()))
-		repl.Set("http.reverse_proxy.upstream.max_requests", strconv.Itoa(upstream.MaxRequests))
-		repl.Set("http.reverse_proxy.upstream.fails", strconv.Itoa(upstream.Host.Fails()))
+		repl.Set("http.reverse_proxy.upstream.requests", upstream.Host.NumRequests())
+		repl.Set("http.reverse_proxy.upstream.max_requests", upstream.MaxRequests)
+		repl.Set("http.reverse_proxy.upstream.fails", upstream.Host.Fails())
 
 		// mutate request headers according to this upstream;
 		// because we're in a retry loop, we have to copy
@@ -446,6 +445,7 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, di Dia
 	}
 
 	h.logger.Debug("upstream roundtrip",
+		zap.String("upstream", di.Upstream.String()),
 		zap.Object("request", caddyhttp.LoggableHTTPRequest{Request: req}),
 		zap.Object("headers", caddyhttp.LoggableHTTPHeader(res.Header)),
 		zap.Duration("duration", duration),
@@ -725,6 +725,7 @@ type Selector interface {
 // obsoleted RFC 2616 (section 13.5.1) and are used for backward
 // compatibility.
 var hopHeaders = []string{
+	"Alt-Svc",
 	"Connection",
 	"Proxy-Connection", // non-standard but still sent by libcurl and rejected by e.g. google
 	"Keep-Alive",
@@ -738,8 +739,19 @@ var hopHeaders = []string{
 
 // DialError is an error that specifically occurs
 // in a call to Dial or DialContext.
-type DialError struct {
-	error
+type DialError struct{ error }
+
+// TLSTransport is implemented by transports
+// that are capable of using TLS.
+type TLSTransport interface {
+	// TLSEnabled returns true if the transport
+	// has TLS enabled, false otherwise.
+	TLSEnabled() bool
+
+	// EnableTLS enables TLS within the transport
+	// if it is not already, using the provided
+	// value as a basis for the TLS config.
+	EnableTLS(base *TLSConfig) error
 }
 
 var bufPool = sync.Pool{

modules/caddyhttp/reverseproxy/selectionpolicies.go

@@ -74,6 +74,16 @@ func (r RandomSelection) Select(pool UpstreamPool, request *http.Request) *Upstr
 	return randomHost
 }
 
+// UnmarshalCaddyfile sets up the module from Caddyfile tokens.
+func (r *RandomSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	for d.Next() {
+		if d.NextArg() {
+			return d.ArgErr()
+		}
+	}
+	return nil
+}
+
 // RandomChoiceSelection is a policy that selects
 // two or more available hosts at random, then
 // chooses the one with the least load.
@@ -192,6 +202,16 @@ func (LeastConnSelection) Select(pool UpstreamPool, _ *http.Request) *Upstream {
 	return bestHost
 }
 
+// UnmarshalCaddyfile sets up the module from Caddyfile tokens.
+func (r *LeastConnSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	for d.Next() {
+		if d.NextArg() {
+			return d.ArgErr()
+		}
+	}
+	return nil
+}
+
 // RoundRobinSelection is a policy that selects
 // a host based on round-robin ordering.
 type RoundRobinSelection struct {
@@ -222,6 +242,16 @@ func (r *RoundRobinSelection) Select(pool UpstreamPool, _ *http.Request) *Upstre
 	return nil
 }
 
+// UnmarshalCaddyfile sets up the module from Caddyfile tokens.
+func (r *RoundRobinSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	for d.Next() {
+		if d.NextArg() {
+			return d.ArgErr()
+		}
+	}
+	return nil
+}
+
 // FirstSelection is a policy that selects
 // the first available host.
 type FirstSelection struct{}
@@ -244,6 +274,16 @@ func (FirstSelection) Select(pool UpstreamPool, _ *http.Request) *Upstream {
 	return nil
 }
 
+// UnmarshalCaddyfile sets up the module from Caddyfile tokens.
+func (r *FirstSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	for d.Next() {
+		if d.NextArg() {
+			return d.ArgErr()
+		}
+	}
+	return nil
+}
+
 // IPHashSelection is a policy that selects a host
 // based on hashing the remote IP of the request.
 type IPHashSelection struct{}
@@ -265,6 +305,16 @@ func (IPHashSelection) Select(pool UpstreamPool, req *http.Request) *Upstream {
 	return hostByHashing(pool, clientIP)
 }
 
+// UnmarshalCaddyfile sets up the module from Caddyfile tokens.
+func (r *IPHashSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	for d.Next() {
+		if d.NextArg() {
+			return d.ArgErr()
+		}
+	}
+	return nil
+}
+
 // URIHashSelection is a policy that selects a
 // host by hashing the request URI.
 type URIHashSelection struct{}
@@ -282,6 +332,16 @@ func (URIHashSelection) Select(pool UpstreamPool, req *http.Request) *Upstream {
 	return hostByHashing(pool, req.RequestURI)
 }
 
+// UnmarshalCaddyfile sets up the module from Caddyfile tokens.
+func (r *URIHashSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	for d.Next() {
+		if d.NextArg() {
+			return d.ArgErr()
+		}
+	}
+	return nil
+}
+
 // HeaderHashSelection is a policy that selects
 // a host based on a given request header.
 type HeaderHashSelection struct {
@@ -309,6 +369,17 @@ func (s HeaderHashSelection) Select(pool UpstreamPool, req *http.Request) *Upstr
 	return hostByHashing(pool, val)
 }
 
+// UnmarshalCaddyfile sets up the module from Caddyfile tokens.
+func (s *HeaderHashSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	for d.Next() {
+		if !d.NextArg() {
+			return d.ArgErr()
+		}
+		s.Field = d.Val()
+	}
+	return nil
+}
+
 // leastRequests returns the host with the
 // least number of active requests to it.
 // If more than one host has the same

modules/caddyhttp/rewrite/caddyfile.go

@@ -24,9 +24,7 @@ import (
 
 func init() {
 	httpcaddyfile.RegisterHandlerDirective("rewrite", parseCaddyfileRewrite)
-	httpcaddyfile.RegisterHandlerDirective("strip_prefix", parseCaddyfileStripPrefix)
-	httpcaddyfile.RegisterHandlerDirective("strip_suffix", parseCaddyfileStripSuffix)
-	httpcaddyfile.RegisterHandlerDirective("uri_replace", parseCaddyfileURIReplace)
+	httpcaddyfile.RegisterHandlerDirective("uri", parseCaddyfileURI)
 }
 
 // parseCaddyfileRewrite sets up a basic rewrite handler from Caddyfile tokens. Syntax:
@@ -49,89 +47,66 @@ func parseCaddyfileRewrite(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler,
 	return rewr, nil
 }
 
-// parseCaddyfileStripPrefix sets up a handler from Caddyfile tokens. Syntax:
+// parseCaddyfileURI sets up a handler for manipulating (but not "rewriting") the
+// URI from Caddyfile tokens. Syntax:
 //
-//     strip_prefix [<matcher>] <prefix>
+//     uri [<matcher>] strip_prefix|strip_suffix|replace <target> [<replacement> [<limit>]]
 //
-// The request path will be stripped the given prefix.
-func parseCaddyfileStripPrefix(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
+// If strip_prefix or strip_suffix are used, then <target> will be stripped
+// only if it is the beginning or the end, respectively, of the URI path. If
+// replace is used, then <target> will be replaced with <replacement> across
+// the whole URI, up to <limit> times (or unlimited if unspecified).
+func parseCaddyfileURI(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
 	var rewr Rewrite
-	for h.Next() {
-		if !h.NextArg() {
-			return nil, h.ArgErr()
-		}
-		rewr.StripPathPrefix = h.Val()
-		if !strings.HasPrefix(rewr.StripPathPrefix, "/") {
-			rewr.StripPathPrefix = "/" + rewr.StripPathPrefix
-		}
-		if h.NextArg() {
-			return nil, h.ArgErr()
-		}
-	}
-	return rewr, nil
-}
-
-// parseCaddyfileStripSuffix sets up a handler from Caddyfile tokens. Syntax:
-//
-//     strip_suffix [<matcher>] <suffix>
-//
-// The request path will be stripped the given suffix.
-func parseCaddyfileStripSuffix(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
-	var rewr Rewrite
-	for h.Next() {
-		if !h.NextArg() {
-			return nil, h.ArgErr()
-		}
-		rewr.StripPathSuffix = h.Val()
-		if h.NextArg() {
-			return nil, h.ArgErr()
-		}
-	}
-	return rewr, nil
-}
-
-// parseCaddyfileURIReplace sets up a handler from Caddyfile tokens. Syntax:
-//
-//     uri_replace [<matcher>] <find> <replace> [<limit>]
-//
-// Substring replacements will be performed on the request URI up to the
-// number specified by limit, if any (default = 0, or no limit).
-func parseCaddyfileURIReplace(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
-	var rewr Rewrite
-
-	var repls []replacer
-
 	for h.Next() {
 		args := h.RemainingArgs()
-		var find, replace, lim string
-		switch len(args) {
-		case 3:
-			lim = args[2]
-			fallthrough
-		case 2:
-			find = args[0]
-			replace = args[1]
-		default:
+		if len(args) < 2 {
 			return nil, h.ArgErr()
 		}
-
-		var limInt int
-		if lim != "" {
-			var err error
-			limInt, err = strconv.Atoi(lim)
-			if err != nil {
-				return nil, h.Errf("limit must be an integer; invalid: %v", err)
+		switch args[0] {
+		case "strip_prefix":
+			if len(args) > 2 {
+				return nil, h.ArgErr()
 			}
+			rewr.StripPathPrefix = args[1]
+			if !strings.HasPrefix(rewr.StripPathPrefix, "/") {
+				rewr.StripPathPrefix = "/" + rewr.StripPathPrefix
+			}
+		case "strip_suffix":
+			if len(args) > 2 {
+				return nil, h.ArgErr()
+			}
+			rewr.StripPathSuffix = args[1]
+		case "replace":
+			var find, replace, lim string
+			switch len(args) {
+			case 4:
+				lim = args[3]
+				fallthrough
+			case 3:
+				find = args[1]
+				replace = args[2]
+			default:
+				return nil, h.ArgErr()
+			}
+
+			var limInt int
+			if lim != "" {
+				var err error
+				limInt, err = strconv.Atoi(lim)
+				if err != nil {
+					return nil, h.Errf("limit must be an integer; invalid: %v", err)
+				}
+			}
+
+			rewr.URISubstring = append(rewr.URISubstring, replacer{
+				Find:    find,
+				Replace: replace,
+				Limit:   limInt,
+			})
+		default:
+			return nil, h.Errf("unrecognized URI manipulation '%s'", args[0])
 		}
-
-		repls = append(repls, replacer{
-			Find:    find,
-			Replace: replace,
-			Limit:   limInt,
-		})
 	}
-
-	rewr.URISubstring = repls
-
 	return rewr, nil
 }

modules/caddyhttp/rewrite/rewrite.go

@@ -15,8 +15,10 @@
 package rewrite
 
 import (
+	"fmt"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 
 	"github.com/caddyserver/caddy/v2"
@@ -112,7 +114,7 @@ func (rewr Rewrite) rewrite(r *http.Request, repl *caddy.Replacer, logger *zap.L
 		r.Method = strings.ToUpper(repl.ReplaceAll(rewr.Method, ""))
 	}
 
-	// uri (path, query string, and fragment... because why not)
+	// uri (path, query string and... fragment, because why not)
 	if uri := rewr.URI; uri != "" {
 		// find the bounds of each part of the URI that exist
 		pathStart, qsStart, fragStart := -1, -1, -1
@@ -134,18 +136,43 @@ func (rewr Rewrite) rewrite(r *http.Request, repl *caddy.Replacer, logger *zap.L
 			qsEnd = len(uri)
 		}
 
+		// isolate the three main components of the URI
+		var path, query, frag string
+		if pathStart > -1 {
+			path = uri[pathStart:pathEnd]
+		}
+		if qsStart > -1 {
+			query = uri[qsStart:qsEnd]
+		}
+		if fragStart > -1 {
+			frag = uri[fragStart:]
+		}
+
 		// build components which are specified, and store them
 		// in a temporary variable so that they all read the
 		// same version of the URI
 		var newPath, newQuery, newFrag string
-		if pathStart >= 0 {
-			newPath = repl.ReplaceAll(uri[pathStart:pathEnd], "")
+		if path != "" {
+			newPath = repl.ReplaceAll(path, "")
 		}
-		if qsStart >= 0 {
-			newQuery = buildQueryString(uri[qsStart:qsEnd], repl)
+
+		// before continuing, we need to check if a query string
+		// snuck into the path component during replacements
+		if quPos := strings.Index(newPath, "?"); quPos > -1 {
+			// recompute; new path contains a query string
+			var injectedQuery string
+			newPath, injectedQuery = newPath[:quPos], newPath[quPos+1:]
+			// don't overwrite explicitly-configured query string
+			if query == "" {
+				query = injectedQuery
+			}
 		}
-		if fragStart >= 0 {
-			newFrag = repl.ReplaceAll(uri[fragStart:], "")
+
+		if query != "" {
+			newQuery = buildQueryString(query, repl)
+		}
+		if frag != "" {
+			newFrag = repl.ReplaceAll(frag, "")
 		}
 
 		// update the URI with the new components
@@ -208,11 +235,22 @@ func buildQueryString(qs string, repl *caddy.Replacer) string {
 
 		// consume the component and write the result
 		comp := qs[:end]
-		comp, _ = repl.ReplaceFunc(comp, func(name, val string) (string, error) {
+		comp, _ = repl.ReplaceFunc(comp, func(name string, val interface{}) (interface{}, error) {
 			if name == "http.request.uri.query" && wroteVal {
 				return val, nil // already escaped
 			}
-			return url.QueryEscape(val), nil
+			var valStr string
+			switch v := val.(type) {
+			case string:
+				valStr = v
+			case fmt.Stringer:
+				valStr = v.String()
+			case int:
+				valStr = strconv.Itoa(v)
+			default:
+				valStr = fmt.Sprintf("%+v", v)
+			}
+			return url.QueryEscape(valStr), nil
 		})
 		if end < len(qs) {
 			end++ // consume delimiter

modules/caddyhttp/rewrite/rewrite_test.go

@@ -158,6 +158,36 @@ func TestRewrite(t *testing.T) {
 			input:  newRequest(t, "GET", "/foo/bar?a=b"),
 			expect: newRequest(t, "GET", "/foo?a=b#frag"),
 		},
+		{
+			rule:   Rewrite{URI: "/foo{http.request.uri}"},
+			input:  newRequest(t, "GET", "/bar?a=b"),
+			expect: newRequest(t, "GET", "/foo/bar?a=b"),
+		},
+		{
+			rule:   Rewrite{URI: "/foo{http.request.uri}"},
+			input:  newRequest(t, "GET", "/bar"),
+			expect: newRequest(t, "GET", "/foo/bar"),
+		},
+		{
+			rule:   Rewrite{URI: "/foo{http.request.uri}?c=d"},
+			input:  newRequest(t, "GET", "/bar?a=b"),
+			expect: newRequest(t, "GET", "/foo/bar?c=d"),
+		},
+		{
+			rule:   Rewrite{URI: "/foo{http.request.uri}?{http.request.uri.query}&c=d"},
+			input:  newRequest(t, "GET", "/bar?a=b"),
+			expect: newRequest(t, "GET", "/foo/bar?a=b&c=d"),
+		},
+		{
+			rule:   Rewrite{URI: "{http.request.uri}"},
+			input:  newRequest(t, "GET", "/bar?a=b"),
+			expect: newRequest(t, "GET", "/bar?a=b"),
+		},
+		{
+			rule:   Rewrite{URI: "{http.request.uri.path}bar?c=d"},
+			input:  newRequest(t, "GET", "/foo/?a=b"),
+			expect: newRequest(t, "GET", "/foo/bar?c=d"),
+		},
 
 		{
 			rule:   Rewrite{StripPathPrefix: "/prefix"},
@@ -211,6 +241,7 @@ func TestRewrite(t *testing.T) {
 		}
 
 		// populate the replacer just enough for our tests
+		repl.Set("http.request.uri", tc.input.RequestURI)
 		repl.Set("http.request.uri.path", tc.input.URL.Path)
 		repl.Set("http.request.uri.query", tc.input.URL.RawQuery)
 

modules/caddyhttp/routes.go

@@ -167,7 +167,7 @@ func (routes RouteList) ProvisionHandlers(ctx caddy.Context) error {
 // This should only be done once: after all the routes have
 // been provisioned, and before serving requests.
 func (routes RouteList) Compile(next Handler) Handler {
-	var mid []Middleware
+	mid := make([]Middleware, 0, len(routes))
 	for _, route := range routes {
 		mid = append(mid, wrapRoute(route))
 	}

modules/caddyhttp/server.go

@@ -16,11 +16,11 @@ package caddyhttp
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"net/http"
 	"net/url"
-	"strconv"
 	"strings"
 	"time"
 
@@ -38,6 +38,10 @@ type Server struct {
 	// that may include port ranges.
 	Listen []string `json:"listen,omitempty"`
 
+	// A list of listener wrapper modules, which can modify the behavior
+	// of the base listener. They are applied in the given order.
+	ListenerWrappersRaw []json.RawMessage `json:"listener_wrappers,omitempty" caddy:"namespace=caddy.listeners inline_key=wrapper"`
+
 	// How long to allow a read from a client's upload. Setting this
 	// to a short, non-zero value can mitigate slowloris attacks, but
 	// may also affect legitimately slow clients.
@@ -106,6 +110,7 @@ type Server struct {
 
 	primaryHandlerChain Handler
 	errorHandlerChain   Handler
+	listenerWrappers    []caddy.ListenerWrapper
 
 	tlsApp       *caddytls.TLS
 	logger       *zap.Logger
@@ -160,13 +165,13 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		defer func() {
 			latency := time.Since(start)
 
-			repl.Set("http.response.status", strconv.Itoa(wrec.Status()))
-			repl.Set("http.response.size", strconv.Itoa(wrec.Size()))
-			repl.Set("http.response.latency", latency.String())
+			repl.Set("http.response.status", wrec.Status())
+			repl.Set("http.response.size", wrec.Size())
+			repl.Set("http.response.latency", latency)
 
 			logger := accLog
-			if s.Logs != nil && s.Logs.LoggerNames != nil {
-				logger = logger.Named(s.Logs.LoggerNames[r.Host])
+			if loggerName := s.Logs.getLoggerName(r.Host); loggerName != "" {
+				logger = logger.Named(loggerName)
 			}
 
 			log := logger.Info
@@ -195,8 +200,8 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		// prepare the error log
 		logger := errLog
-		if s.Logs != nil && s.Logs.LoggerNames != nil {
-			logger = logger.Named(s.Logs.LoggerNames[r.Host])
+		if loggerName := s.Logs.getLoggerName(r.Host); loggerName != "" {
+			logger = logger.Named(loggerName)
 		}
 
 		// get the values that will be used to log the error
@@ -349,9 +354,9 @@ func (*HTTPErrorConfig) WithError(r *http.Request, err error) *http.Request {
 
 	// add error values to the replacer
 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
-	repl.Set("http.error", err.Error())
+	repl.Set("http.error", err)
 	if handlerErr, ok := err.(HandlerError); ok {
-		repl.Set("http.error.status_code", strconv.Itoa(handlerErr.StatusCode))
+		repl.Set("http.error.status_code", handlerErr.StatusCode)
 		repl.Set("http.error.status_text", http.StatusText(handlerErr.StatusCode))
 		repl.Set("http.error.trace", handlerErr.Trace)
 		repl.Set("http.error.id", handlerErr.ID)
@@ -362,6 +367,10 @@ func (*HTTPErrorConfig) WithError(r *http.Request, err error) *http.Request {
 
 // ServerLogConfig describes a server's logging configuration.
 type ServerLogConfig struct {
+	// The logger name for all logs emitted by this server unless
+	// the hostname is found in the LoggerNames (logger_names) map.
+	LoggerName string `json:"log_name,omitempty"`
+
 	// LoggerNames maps request hostnames to a custom logger name.
 	// For example, a mapping of "example.com" to "example" would
 	// cause access logs from requests with a Host of example.com
@@ -369,6 +378,13 @@ type ServerLogConfig struct {
 	LoggerNames map[string]string `json:"logger_names,omitempty"`
 }
 
+func (slc ServerLogConfig) getLoggerName(host string) string {
+	if loggerName, ok := slc.LoggerNames[host]; ok {
+		return loggerName
+	}
+	return slc.LoggerName
+}
+
 // errLogValues inspects err and returns the status code
 // to use, the error log message, and any extra fields.
 // If err is a HandlerError, the returned values will

modules/caddyhttp/standard/imports.go

@@ -5,12 +5,10 @@ import (
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyauth"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
-	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/brotli"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/gzip"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/zstd"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/fileserver"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/headers"
-	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/httpcache"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/requestbody"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/fastcgi"

modules/caddyhttp/templates/frontmatter.go

@@ -33,7 +33,7 @@ func extractFrontMatter(input string) (map[string]interface{}, string, error) {
 	// see what kind of front matter there is, if any
 	var closingFence string
 	var fmParser func([]byte) (map[string]interface{}, error)
-	switch string(firstLine) {
+	switch firstLine {
 	case yamlFrontMatterFenceOpen:
 		fmParser = yamlFrontMatter
 		closingFence = yamlFrontMatterFenceClose

modules/caddyhttp/templates/templates.go

@@ -33,12 +33,14 @@ func init() {
 // The syntax is documented in the Go standard library's
 // [text/template package](https://golang.org/pkg/text/template/).
 //
+// ⚠️ Template functions/actions are still experimental, so they are subject to change.
+//
 // [All Sprig functions](https://masterminds.github.io/sprig/) are supported.
 //
 // In addition to the standard functions and Sprig functions, Caddy adds
 // extra functions and data that are available to a template:
 //
-// ##### **`.Args`**
+// ##### `.Args`
 //
 // Access arguments passed to this page/context, for example as the result of a `include`.
 //
@@ -54,6 +56,14 @@ func init() {
 // {{.Cookie "cookiename"}}
 // ```
 //
+// ##### `env`
+//
+// Gets an environment variable.
+//
+// ```
+// {{env "VAR_NAME"}}
+// ```
+//
 // ##### `.Host`
 //
 // Returns the hostname portion (no port) of the Host header of the HTTP request.

modules/caddyhttp/templates/tplcontext.go

@@ -17,14 +17,15 @@ package templates
 import (
 	"bytes"
 	"fmt"
-	"html/template"
 	"io"
 	"net"
 	"net/http"
+	"os"
 	"path"
 	"strconv"
 	"strings"
 	"sync"
+	"text/template"
 
 	"github.com/Masterminds/sprig/v3"
 	"github.com/alecthomas/chroma/formatters/html"
@@ -57,7 +58,7 @@ func (c templateContext) OriginalReq() http.Request {
 // Note that included files are NOT escaped, so you should only include
 // trusted files. If it is not trusted, be sure to use escaping functions
 // in your template.
-func (c templateContext) funcInclude(filename string, args ...interface{}) (template.HTML, error) {
+func (c templateContext) funcInclude(filename string, args ...interface{}) (string, error) {
 	if c.Root == nil {
 		return "", fmt.Errorf("root file system not specified")
 	}
@@ -84,14 +85,14 @@ func (c templateContext) funcInclude(filename string, args ...interface{}) (temp
 		return "", err
 	}
 
-	return template.HTML(bodyBuf.String()), nil
+	return bodyBuf.String(), nil
 }
 
 // funcHTTPInclude returns the body of a virtual (lightweight) request
 // to the given URI on the same server. Note that included bodies
 // are NOT escaped, so you should only include trusted resources.
 // If it is not trusted, be sure to use escaping functions yourself.
-func (c templateContext) funcHTTPInclude(uri string) (template.HTML, error) {
+func (c templateContext) funcHTTPInclude(uri string) (string, error) {
 	// prevent virtual request loops by counting how many levels
 	// deep we are; and if we get too deep, return an error
 	recursionCount := 1
@@ -132,7 +133,7 @@ func (c templateContext) funcHTTPInclude(uri string) (template.HTML, error) {
 		return "", err
 	}
 
-	return template.HTML(buf.String()), nil
+	return buf.String(), nil
 }
 
 func (c templateContext) executeTemplateInBuffer(tplName string, buf *bytes.Buffer) error {
@@ -150,6 +151,7 @@ func (c templateContext) executeTemplateInBuffer(tplName string, buf *bytes.Buff
 		"markdown":         c.funcMarkdown,
 		"splitFrontMatter": c.funcSplitFrontMatter,
 		"listFiles":        c.funcListFiles,
+		"env":              c.funcEnv,
 	})
 
 	parsedTpl, err := tpl.Parse(buf.String())
@@ -162,6 +164,10 @@ func (c templateContext) executeTemplateInBuffer(tplName string, buf *bytes.Buff
 	return parsedTpl.Execute(buf, c)
 }
 
+func (templateContext) funcEnv(varName string) string {
+	return os.Getenv(varName)
+}
+
 // Cookie gets the value of a cookie with name name.
 func (c templateContext) Cookie(name string) string {
 	cookies := c.Req.Cookies()
@@ -198,7 +204,7 @@ func (c templateContext) Host() (string, error) {
 
 // funcStripHTML returns s without HTML tags. It is fairly naive
 // but works with most valid HTML inputs.
-func (c templateContext) funcStripHTML(s string) string {
+func (templateContext) funcStripHTML(s string) string {
 	var buf bytes.Buffer
 	var inTag, inQuotes bool
 	var tagStart int
@@ -231,13 +237,13 @@ func (c templateContext) funcStripHTML(s string) string {
 
 // funcMarkdown renders the markdown body as HTML. The resulting
 // HTML is NOT escaped so that it can be rendered as HTML.
-func (c templateContext) funcMarkdown(input interface{}) (template.HTML, error) {
+func (templateContext) funcMarkdown(input interface{}) (string, error) {
 	inputStr := toString(input)
 
 	md := goldmark.New(
 		goldmark.WithExtensions(
 			extension.GFM,
-			extension.Table,
+			extension.Footnote,
 			highlighting.NewHighlighting(
 				highlighting.WithFormatOptions(
 					html.WithClasses(true),
@@ -259,13 +265,13 @@ func (c templateContext) funcMarkdown(input interface{}) (template.HTML, error)
 
 	md.Convert([]byte(inputStr), buf)
 
-	return template.HTML(buf.String()), nil
+	return buf.String(), nil
 }
 
 // splitFrontMatter parses front matter out from the beginning of input,
 // and returns the separated key-value pairs and the body/content. input
 // must be a "stringy" value.
-func (c templateContext) funcSplitFrontMatter(input interface{}) (parsedMarkdownDoc, error) {
+func (templateContext) funcSplitFrontMatter(input interface{}) (parsedMarkdownDoc, error) {
 	meta, body, err := extractFrontMatter(toString(input))
 	if err != nil {
 		return parsedMarkdownDoc{}, err
@@ -338,14 +344,12 @@ func toString(input interface{}) string {
 	switch v := input.(type) {
 	case string:
 		return v
-	case template.HTML:
-		return string(v)
 	case fmt.Stringer:
 		return v.String()
 	case error:
 		return v.Error()
 	default:
-		return fmt.Sprintf("%s", input)
+		return fmt.Sprintf("%v", input)
 	}
 }
 
@@ -357,6 +361,6 @@ var bufPool = sync.Pool{
 
 // at time of writing, sprig.FuncMap() makes a copy, thus
 // involves iterating the whole map, so do it just once
-var sprigFuncMap = sprig.FuncMap()
+var sprigFuncMap = sprig.TxtFuncMap()
 
 const recursionPreventionHeader = "Caddy-Templates-Include"

modules/caddyhttp/templates/tplcontext_test.go

@@ -31,7 +31,6 @@ package templates
 import (
 	"bytes"
 	"fmt"
-	"html/template"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -48,7 +47,7 @@ func TestMarkdown(t *testing.T) {
 
 	for i, test := range []struct {
 		body   string
-		expect template.HTML
+		expect string
 	}{
 		{
 			body:   "- str1\n- str2\n",

modules/caddypki/ca.go

@@ -0,0 +1,356 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddypki
+
+import (
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"path"
+	"sync"
+	"time"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/certmagic"
+	"github.com/smallstep/truststore"
+	"go.uber.org/zap"
+)
+
+// CA describes a certificate authority, which consists of
+// root/signing certificates and various settings pertaining
+// to the issuance of certificates and trusting them.
+type CA struct {
+	// The user-facing name of the certificate authority.
+	Name string `json:"name,omitempty"`
+
+	// The name to put in the CommonName field of the
+	// root certificate.
+	RootCommonName string `json:"root_common_name,omitempty"`
+
+	// The name to put in the CommonName field of the
+	// intermediate certificates.
+	IntermediateCommonName string `json:"intermediate_common_name,omitempty"`
+
+	// Whether Caddy will attempt to install the CA's root
+	// into the system trust store, as well as into Java
+	// and Mozilla Firefox trust stores. Default: true.
+	InstallTrust *bool `json:"install_trust,omitempty"`
+
+	Root         *KeyPair `json:"root,omitempty"`
+	Intermediate *KeyPair `json:"intermediate,omitempty"`
+
+	// Optionally configure a separate storage module associated with this
+	// issuer, instead of using Caddy's global/default-configured storage.
+	// This can be useful if you want to keep your signing keys in a
+	// separate location from your leaf certificates.
+	StorageRaw json.RawMessage `json:"storage,omitempty" caddy:"namespace=caddy.storage inline_key=module"`
+
+	id          string
+	storage     certmagic.Storage
+	root, inter *x509.Certificate
+	interKey    interface{} // TODO: should we just store these as crypto.Signer?
+	mu          *sync.RWMutex
+
+	rootCertPath string // mainly used for logging purposes if trusting
+	log          *zap.Logger
+}
+
+// Provision sets up the CA.
+func (ca *CA) Provision(ctx caddy.Context, id string, log *zap.Logger) error {
+	ca.mu = new(sync.RWMutex)
+	ca.log = log.Named("ca." + id)
+
+	if id == "" {
+		return fmt.Errorf("CA ID is required (use 'local' for the default CA)")
+	}
+	ca.mu.Lock()
+	ca.id = id
+	ca.mu.Unlock()
+
+	if ca.StorageRaw != nil {
+		val, err := ctx.LoadModule(ca, "StorageRaw")
+		if err != nil {
+			return fmt.Errorf("loading storage module: %v", err)
+		}
+		cmStorage, err := val.(caddy.StorageConverter).CertMagicStorage()
+		if err != nil {
+			return fmt.Errorf("creating storage configuration: %v", err)
+		}
+		ca.storage = cmStorage
+	}
+	if ca.storage == nil {
+		ca.storage = ctx.Storage()
+	}
+
+	if ca.Name == "" {
+		ca.Name = defaultCAName
+	}
+	if ca.RootCommonName == "" {
+		ca.RootCommonName = defaultRootCommonName
+	}
+	if ca.IntermediateCommonName == "" {
+		ca.IntermediateCommonName = defaultIntermediateCommonName
+	}
+
+	// load the certs and key that will be used for signing
+	var rootCert, interCert *x509.Certificate
+	var rootKey, interKey interface{}
+	var err error
+	if ca.Root != nil {
+		if ca.Root.Format == "" || ca.Root.Format == "pem_file" {
+			ca.rootCertPath = ca.Root.Certificate
+		}
+		rootCert, rootKey, err = ca.Root.Load()
+	} else {
+		ca.rootCertPath = "storage:" + ca.storageKeyRootCert()
+		rootCert, rootKey, err = ca.loadOrGenRoot()
+	}
+	if err != nil {
+		return err
+	}
+	if ca.Intermediate != nil {
+		interCert, interKey, err = ca.Intermediate.Load()
+	} else {
+		interCert, interKey, err = ca.loadOrGenIntermediate(rootCert, rootKey)
+	}
+	if err != nil {
+		return err
+	}
+
+	ca.mu.Lock()
+	ca.root, ca.inter, ca.interKey = rootCert, interCert, interKey
+	ca.mu.Unlock()
+
+	return nil
+}
+
+// ID returns the CA's ID, as given by the user in the config.
+func (ca CA) ID() string {
+	return ca.id
+}
+
+// RootCertificate returns the CA's root certificate (public key).
+func (ca CA) RootCertificate() *x509.Certificate {
+	ca.mu.RLock()
+	defer ca.mu.RUnlock()
+	return ca.root
+}
+
+// RootKey returns the CA's root private key. Since the root key is
+// not cached in memory long-term, it needs to be loaded from storage,
+// which could yield an error.
+func (ca CA) RootKey() (interface{}, error) {
+	_, rootKey, err := ca.loadOrGenRoot()
+	return rootKey, err
+}
+
+// IntermediateCertificate returns the CA's intermediate
+// certificate (public key).
+func (ca CA) IntermediateCertificate() *x509.Certificate {
+	ca.mu.RLock()
+	defer ca.mu.RUnlock()
+	return ca.inter
+}
+
+// IntermediateKey returns the CA's intermediate private key.
+func (ca CA) IntermediateKey() interface{} {
+	ca.mu.RLock()
+	defer ca.mu.RUnlock()
+	return ca.interKey
+}
+
+func (ca CA) loadOrGenRoot() (rootCert *x509.Certificate, rootKey interface{}, err error) {
+	rootCertPEM, err := ca.storage.Load(ca.storageKeyRootCert())
+	if err != nil {
+		if _, ok := err.(certmagic.ErrNotExist); !ok {
+			return nil, nil, fmt.Errorf("loading root cert: %v", err)
+		}
+
+		// TODO: should we require that all or none of the assets are required before overwriting anything?
+		rootCert, rootKey, err = ca.genRoot()
+		if err != nil {
+			return nil, nil, fmt.Errorf("generating root: %v", err)
+		}
+	}
+
+	if rootCert == nil {
+		rootCert, err = pemDecodeSingleCert(rootCertPEM)
+		if err != nil {
+			return nil, nil, fmt.Errorf("parsing root certificate PEM: %v", err)
+		}
+	}
+	if rootKey == nil {
+		rootKeyPEM, err := ca.storage.Load(ca.storageKeyRootKey())
+		if err != nil {
+			return nil, nil, fmt.Errorf("loading root key: %v", err)
+		}
+		rootKey, err = pemDecodePrivateKey(rootKeyPEM)
+		if err != nil {
+			return nil, nil, fmt.Errorf("decoding root key: %v", err)
+		}
+	}
+
+	return rootCert, rootKey, nil
+}
+
+func (ca CA) genRoot() (rootCert *x509.Certificate, rootKey interface{}, err error) {
+	repl := ca.newReplacer()
+
+	rootCert, rootKey, err = generateRoot(repl.ReplaceAll(ca.RootCommonName, ""))
+	if err != nil {
+		return nil, nil, fmt.Errorf("generating CA root: %v", err)
+	}
+	rootCertPEM, err := pemEncodeCert(rootCert.Raw)
+	if err != nil {
+		return nil, nil, fmt.Errorf("encoding root certificate: %v", err)
+	}
+	err = ca.storage.Store(ca.storageKeyRootCert(), rootCertPEM)
+	if err != nil {
+		return nil, nil, fmt.Errorf("saving root certificate: %v", err)
+	}
+	rootKeyPEM, err := pemEncodePrivateKey(rootKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("encoding root key: %v", err)
+	}
+	err = ca.storage.Store(ca.storageKeyRootKey(), rootKeyPEM)
+	if err != nil {
+		return nil, nil, fmt.Errorf("saving root key: %v", err)
+	}
+
+	return rootCert, rootKey, nil
+}
+
+func (ca CA) loadOrGenIntermediate(rootCert *x509.Certificate, rootKey interface{}) (interCert *x509.Certificate, interKey interface{}, err error) {
+	interCertPEM, err := ca.storage.Load(ca.storageKeyIntermediateCert())
+	if err != nil {
+		if _, ok := err.(certmagic.ErrNotExist); !ok {
+			return nil, nil, fmt.Errorf("loading intermediate cert: %v", err)
+		}
+
+		// TODO: should we require that all or none of the assets are required before overwriting anything?
+		interCert, interKey, err = ca.genIntermediate(rootCert, rootKey)
+		if err != nil {
+			return nil, nil, fmt.Errorf("generating new intermediate cert: %v", err)
+		}
+	}
+
+	if interCert == nil {
+		interCert, err = pemDecodeSingleCert(interCertPEM)
+		if err != nil {
+			return nil, nil, fmt.Errorf("decoding intermediate certificate PEM: %v", err)
+		}
+	}
+
+	if interKey == nil {
+		interKeyPEM, err := ca.storage.Load(ca.storageKeyIntermediateKey())
+		if err != nil {
+			return nil, nil, fmt.Errorf("loading intermediate key: %v", err)
+		}
+		interKey, err = pemDecodePrivateKey(interKeyPEM)
+		if err != nil {
+			return nil, nil, fmt.Errorf("decoding intermediate key: %v", err)
+		}
+	}
+
+	return interCert, interKey, nil
+}
+
+func (ca CA) genIntermediate(rootCert *x509.Certificate, rootKey interface{}) (interCert *x509.Certificate, interKey interface{}, err error) {
+	repl := ca.newReplacer()
+
+	rootKeyPEM, err := ca.storage.Load(ca.storageKeyRootKey())
+	if err != nil {
+		return nil, nil, fmt.Errorf("loading root key to sign new intermediate: %v", err)
+	}
+	rootKey, err = pemDecodePrivateKey(rootKeyPEM)
+	if err != nil {
+		return nil, nil, fmt.Errorf("decoding root key: %v", err)
+	}
+	interCert, interKey, err = generateIntermediate(repl.ReplaceAll(ca.IntermediateCommonName, ""), rootCert, rootKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("generating CA intermediate: %v", err)
+	}
+	interCertPEM, err := pemEncodeCert(interCert.Raw)
+	if err != nil {
+		return nil, nil, fmt.Errorf("encoding intermediate certificate: %v", err)
+	}
+	err = ca.storage.Store(ca.storageKeyIntermediateCert(), interCertPEM)
+	if err != nil {
+		return nil, nil, fmt.Errorf("saving intermediate certificate: %v", err)
+	}
+	interKeyPEM, err := pemEncodePrivateKey(interKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("encoding intermediate key: %v", err)
+	}
+	err = ca.storage.Store(ca.storageKeyIntermediateKey(), interKeyPEM)
+	if err != nil {
+		return nil, nil, fmt.Errorf("saving intermediate key: %v", err)
+	}
+
+	return interCert, interKey, nil
+}
+
+func (ca CA) storageKeyCAPrefix() string {
+	return path.Join("pki", "authorities", certmagic.StorageKeys.Safe(ca.id))
+}
+func (ca CA) storageKeyRootCert() string {
+	return path.Join(ca.storageKeyCAPrefix(), "root.crt")
+}
+func (ca CA) storageKeyRootKey() string {
+	return path.Join(ca.storageKeyCAPrefix(), "root.key")
+}
+func (ca CA) storageKeyIntermediateCert() string {
+	return path.Join(ca.storageKeyCAPrefix(), "intermediate.crt")
+}
+func (ca CA) storageKeyIntermediateKey() string {
+	return path.Join(ca.storageKeyCAPrefix(), "intermediate.key")
+}
+
+func (ca CA) newReplacer() *caddy.Replacer {
+	repl := caddy.NewReplacer()
+	repl.Set("pki.ca.name", ca.Name)
+	return repl
+}
+
+// installRoot installs this CA's root certificate into the
+// local trust store(s) if it is not already trusted. The CA
+// must already be provisioned.
+func (ca CA) installRoot() error {
+	// avoid password prompt if already trusted
+	if trusted(ca.root) {
+		ca.log.Info("root certificate is already trusted by system",
+			zap.String("path", ca.rootCertPath))
+		return nil
+	}
+
+	ca.log.Warn("installing root certificate (you might be prompted for password)",
+		zap.String("path", ca.rootCertPath))
+
+	return truststore.Install(ca.root,
+		truststore.WithDebug(),
+		truststore.WithFirefox(),
+		truststore.WithJava(),
+	)
+}
+
+const (
+	defaultCAID                   = "local"
+	defaultCAName                 = "Caddy Local Authority"
+	defaultRootCommonName         = "{pki.ca.name} - {time.now.year} ECC Root"
+	defaultIntermediateCommonName = "{pki.ca.name} - ECC Intermediate"
+
+	defaultRootLifetime         = 24 * time.Hour * 30 * 12 * 10
+	defaultIntermediateLifetime = 24 * time.Hour * 7
+)

modules/caddypki/certificates.go

@@ -0,0 +1,50 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddypki
+
+import (
+	"crypto/x509"
+	"time"
+
+	"github.com/smallstep/cli/crypto/x509util"
+)
+
+func generateRoot(commonName string) (rootCrt *x509.Certificate, privateKey interface{}, err error) {
+	rootProfile, err := x509util.NewRootProfile(commonName)
+	if err != nil {
+		return
+	}
+	rootProfile.Subject().NotAfter = time.Now().Add(defaultRootLifetime) // TODO: make configurable
+	return newCert(rootProfile)
+}
+
+func generateIntermediate(commonName string, rootCrt *x509.Certificate, rootKey interface{}) (cert *x509.Certificate, privateKey interface{}, err error) {
+	interProfile, err := x509util.NewIntermediateProfile(commonName, rootCrt, rootKey)
+	if err != nil {
+		return
+	}
+	interProfile.Subject().NotAfter = time.Now().Add(defaultIntermediateLifetime) // TODO: make configurable
+	return newCert(interProfile)
+}
+
+func newCert(profile x509util.Profile) (cert *x509.Certificate, privateKey interface{}, err error) {
+	certBytes, err := profile.CreateCertificate()
+	if err != nil {
+		return
+	}
+	privateKey = profile.SubjectPrivateKey()
+	cert, err = x509.ParseCertificate(certBytes)
+	return
+}

modules/caddypki/command.go

@@ -0,0 +1,133 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddypki
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/caddyserver/caddy/v2"
+	caddycmd "github.com/caddyserver/caddy/v2/cmd"
+	"github.com/smallstep/truststore"
+)
+
+func init() {
+	caddycmd.RegisterCommand(caddycmd.Command{
+		Name:  "trust",
+		Func:  cmdTrust,
+		Short: "Installs a CA certificate into local trust stores",
+		Long: `
+Adds a root certificate into the local trust stores. Intended for
+development environments only.
+
+Since Caddy will install its root certificates into the local trust
+stores automatically when they are first generated, this command is
+only necessary if you need to pre-install the certificates before
+using them; for example, if you have elevated privileges at one
+point but not later, you will want to use this command so that a
+password prompt is not required later.
+
+This command installs the root certificate only for Caddy's
+default CA.`,
+	})
+
+	caddycmd.RegisterCommand(caddycmd.Command{
+		Name:  "untrust",
+		Func:  cmdUntrust,
+		Usage: "[--ca <id> | --cert <path>]",
+		Short: "Untrusts a locally-trusted CA certificate",
+		Long: `
+Untrusts a root certificate from the local trust store(s). Intended
+for development environments only.
+
+This command uninstalls trust; it does not necessarily delete the
+root certificate from trust stores entirely. Thus, repeatedly
+trusting and untrusting new certificates can fill up trust databases.
+
+This command does not delete or modify certificate files.
+
+Specify which certificate to untrust either by the ID of its CA with
+the --ca flag, or the direct path to the certificate file with the
+--cert flag. If the --ca flag is used, only the default storage paths
+are assumed (i.e. using --ca flag with custom storage backends or file
+paths will not work).
+
+If no flags are specified, --ca=local is assumed.`,
+		Flags: func() *flag.FlagSet {
+			fs := flag.NewFlagSet("untrust", flag.ExitOnError)
+			fs.String("ca", "", "The ID of the CA to untrust")
+			fs.String("cert", "", "The path to the CA certificate to untrust")
+			return fs
+		}(),
+	})
+}
+
+func cmdTrust(fs caddycmd.Flags) (int, error) {
+	// we have to create a sort of dummy context so that
+	// the CA can provision itself...
+	ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
+	defer cancel()
+
+	// provision the CA, which generates and stores a root
+	// certificate if one doesn't already exist in storage
+	ca := CA{
+		storage: caddy.DefaultStorage,
+	}
+	err := ca.Provision(ctx, defaultCAID, caddy.Log())
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	err = ca.installRoot()
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	return caddy.ExitCodeSuccess, nil
+}
+
+func cmdUntrust(fs caddycmd.Flags) (int, error) {
+	ca := fs.String("ca")
+	cert := fs.String("cert")
+
+	if ca != "" && cert != "" {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("conflicting command line arguments")
+	}
+	if ca == "" && cert == "" {
+		ca = defaultCAID
+	}
+	if ca != "" {
+		cert = filepath.Join(caddy.AppDataDir(), "pki", "authorities", ca, "root.crt")
+	}
+
+	// sanity check, make sure cert file exists first
+	_, err := os.Stat(cert)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("accessing certificate file: %v", err)
+	}
+
+	err = truststore.UninstallFile(cert,
+		truststore.WithDebug(),
+		truststore.WithFirefox(),
+		truststore.WithJava())
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	return caddy.ExitCodeSuccess, nil
+}

modules/caddypki/crypto.go

@@ -0,0 +1,155 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddypki
+
+import (
+	"bytes"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/ed25519"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"strings"
+)
+
+func pemDecodeSingleCert(pemDER []byte) (*x509.Certificate, error) {
+	pemBlock, remaining := pem.Decode(pemDER)
+	if pemBlock == nil {
+		return nil, fmt.Errorf("no PEM block found")
+	}
+	if len(remaining) > 0 {
+		return nil, fmt.Errorf("input contained more than a single PEM block")
+	}
+	if pemBlock.Type != "CERTIFICATE" {
+		return nil, fmt.Errorf("expected PEM block type to be CERTIFICATE, but got '%s'", pemBlock.Type)
+	}
+	return x509.ParseCertificate(pemBlock.Bytes)
+}
+
+func pemEncodeCert(der []byte) ([]byte, error) {
+	return pemEncode("CERTIFICATE", der)
+}
+
+// pemEncodePrivateKey marshals a EC or RSA private key into a PEM-encoded array of bytes.
+// TODO: this is the same thing as in certmagic. Should we reuse that code somehow? It's unexported.
+func pemEncodePrivateKey(key crypto.PrivateKey) ([]byte, error) {
+	var pemType string
+	var keyBytes []byte
+	switch key := key.(type) {
+	case *ecdsa.PrivateKey:
+		var err error
+		pemType = "EC"
+		keyBytes, err = x509.MarshalECPrivateKey(key)
+		if err != nil {
+			return nil, err
+		}
+	case *rsa.PrivateKey:
+		pemType = "RSA"
+		keyBytes = x509.MarshalPKCS1PrivateKey(key)
+	case *ed25519.PrivateKey:
+		var err error
+		pemType = "ED25519"
+		keyBytes, err = x509.MarshalPKCS8PrivateKey(key)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("unsupported key type: %T", key)
+	}
+	return pemEncode(pemType+" PRIVATE KEY", keyBytes)
+}
+
+// pemDecodePrivateKey loads a PEM-encoded ECC/RSA private key from an array of bytes.
+// Borrowed from Go standard library, to handle various private key and PEM block types.
+// https://github.com/golang/go/blob/693748e9fa385f1e2c3b91ca9acbb6c0ad2d133d/src/crypto/tls/tls.go#L291-L308
+// https://github.com/golang/go/blob/693748e9fa385f1e2c3b91ca9acbb6c0ad2d133d/src/crypto/tls/tls.go#L238)
+// TODO: this is the same thing as in certmagic. Should we reuse that code somehow? It's unexported.
+func pemDecodePrivateKey(keyPEMBytes []byte) (crypto.PrivateKey, error) {
+	keyBlockDER, _ := pem.Decode(keyPEMBytes)
+
+	if keyBlockDER.Type != "PRIVATE KEY" && !strings.HasSuffix(keyBlockDER.Type, " PRIVATE KEY") {
+		return nil, fmt.Errorf("unknown PEM header %q", keyBlockDER.Type)
+	}
+
+	if key, err := x509.ParsePKCS1PrivateKey(keyBlockDER.Bytes); err == nil {
+		return key, nil
+	}
+
+	if key, err := x509.ParsePKCS8PrivateKey(keyBlockDER.Bytes); err == nil {
+		switch key := key.(type) {
+		case *rsa.PrivateKey, *ecdsa.PrivateKey, ed25519.PrivateKey:
+			return key, nil
+		default:
+			return nil, fmt.Errorf("found unknown private key type in PKCS#8 wrapping: %T", key)
+		}
+	}
+
+	if key, err := x509.ParseECPrivateKey(keyBlockDER.Bytes); err == nil {
+		return key, nil
+	}
+
+	return nil, fmt.Errorf("unknown private key type")
+}
+
+func pemEncode(blockType string, b []byte) ([]byte, error) {
+	var buf bytes.Buffer
+	err := pem.Encode(&buf, &pem.Block{Type: blockType, Bytes: b})
+	return buf.Bytes(), err
+}
+
+func trusted(cert *x509.Certificate) bool {
+	chains, err := cert.Verify(x509.VerifyOptions{})
+	return len(chains) > 0 && err == nil
+}
+
+// KeyPair represents a public-private key pair, where the
+// public key is also called a certificate.
+type KeyPair struct {
+	Certificate string `json:"certificate,omitempty"`
+	PrivateKey  string `json:"private_key,omitempty"`
+	Format      string `json:"format,omitempty"`
+}
+
+// Load loads the certificate and key.
+func (kp KeyPair) Load() (*x509.Certificate, interface{}, error) {
+	switch kp.Format {
+	case "", "pem_file":
+		certData, err := ioutil.ReadFile(kp.Certificate)
+		if err != nil {
+			return nil, nil, err
+		}
+		keyData, err := ioutil.ReadFile(kp.PrivateKey)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		cert, err := pemDecodeSingleCert(certData)
+		if err != nil {
+			return nil, nil, err
+		}
+		key, err := pemDecodePrivateKey(keyData)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		return cert, key, nil
+
+	default:
+		return nil, nil, fmt.Errorf("unsupported format: %s", kp.Format)
+	}
+}

modules/caddypki/maintain.go

@@ -0,0 +1,99 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddypki
+
+import (
+	"crypto/x509"
+	"fmt"
+	"time"
+
+	"go.uber.org/zap"
+)
+
+func (p *PKI) maintenance() {
+	ticker := time.NewTicker(10 * time.Minute) // TODO: make configurable
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			p.renewCerts()
+		case <-p.ctx.Done():
+			return
+		}
+	}
+}
+
+func (p *PKI) renewCerts() {
+	for _, ca := range p.CAs {
+		err := p.renewCertsForCA(ca)
+		if err != nil {
+			p.log.Error("renewing intermediate certificates",
+				zap.Error(err),
+				zap.String("ca", ca.id))
+		}
+	}
+}
+
+func (p *PKI) renewCertsForCA(ca *CA) error {
+	ca.mu.Lock()
+	defer ca.mu.Unlock()
+
+	log := p.log.With(zap.String("ca", ca.id))
+
+	// only maintain the root if it's not manually provided in the config
+	if ca.Root == nil {
+		if needsRenewal(ca.root) {
+			// TODO: implement root renewal (use same key)
+			log.Warn("root certificate expiring soon (FIXME: ROOT RENEWAL NOT YET IMPLEMENTED)",
+				zap.Duration("time_remaining", time.Until(ca.inter.NotAfter)),
+			)
+		}
+	}
+
+	// only maintain the intermediate if it's not manually provided in the config
+	if ca.Intermediate == nil {
+		if needsRenewal(ca.inter) {
+			log.Info("intermediate expires soon; renewing",
+				zap.Duration("time_remaining", time.Until(ca.inter.NotAfter)),
+			)
+
+			rootCert, rootKey, err := ca.loadOrGenRoot()
+			if err != nil {
+				return fmt.Errorf("loading root key: %v", err)
+			}
+			interCert, interKey, err := ca.genIntermediate(rootCert, rootKey)
+			if err != nil {
+				return fmt.Errorf("generating new certificate: %v", err)
+			}
+			ca.inter, ca.interKey = interCert, interKey
+
+			log.Info("renewed intermediate",
+				zap.Time("new_expiration", ca.inter.NotAfter),
+			)
+		}
+	}
+
+	return nil
+}
+
+func needsRenewal(cert *x509.Certificate) bool {
+	lifetime := cert.NotAfter.Sub(cert.NotBefore)
+	renewalWindow := time.Duration(float64(lifetime) * renewalWindowRatio)
+	renewalWindowStart := cert.NotAfter.Add(-renewalWindow)
+	return time.Now().After(renewalWindowStart)
+}
+
+const renewalWindowRatio = 0.2 // TODO: make configurable

modules/caddypki/pki.go

@@ -0,0 +1,105 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddypki
+
+import (
+	"fmt"
+
+	"github.com/caddyserver/caddy/v2"
+	"go.uber.org/zap"
+)
+
+func init() {
+	caddy.RegisterModule(PKI{})
+}
+
+// PKI provides Public Key Infrastructure facilities for Caddy.
+type PKI struct {
+	// The CAs to manage. Each CA is keyed by an ID that is used
+	// to uniquely identify it from other CAs. The default CA ID
+	// is "local".
+	CAs map[string]*CA `json:"certificate_authorities,omitempty"`
+
+	ctx caddy.Context
+	log *zap.Logger
+}
+
+// CaddyModule returns the Caddy module information.
+func (PKI) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "pki",
+		New: func() caddy.Module { return new(PKI) },
+	}
+}
+
+// Provision sets up the configuration for the PKI app.
+func (p *PKI) Provision(ctx caddy.Context) error {
+	p.ctx = ctx
+	p.log = ctx.Logger(p)
+
+	// if this app is initialized at all, ensure there's
+	// at least a default CA that can be used
+	if len(p.CAs) == 0 {
+		p.CAs = map[string]*CA{defaultCAID: new(CA)}
+	}
+
+	for caID, ca := range p.CAs {
+		err := ca.Provision(ctx, caID, p.log)
+		if err != nil {
+			return fmt.Errorf("provisioning CA '%s': %v", caID, err)
+		}
+	}
+
+	return nil
+}
+
+// Start starts the PKI app.
+func (p *PKI) Start() error {
+	// install roots to trust store, if not disabled
+	for _, ca := range p.CAs {
+		if ca.InstallTrust != nil && !*ca.InstallTrust {
+			ca.log.Warn("root certificate trust store installation disabled; unconfigured clients may show warnings",
+				zap.String("path", ca.rootCertPath))
+			continue
+		}
+
+		if err := ca.installRoot(); err != nil {
+			// could be some system dependencies that are missing;
+			// shouldn't totally prevent startup, but we should log it
+			ca.log.Error("failed to install root certificate",
+				zap.Error(err),
+				zap.String("certificate_file", ca.rootCertPath))
+		}
+	}
+
+	// see if root/intermediates need renewal...
+	p.renewCerts()
+
+	// ...and keep them renewed
+	go p.maintenance()
+
+	return nil
+}
+
+// Stop stops the PKI app.
+func (p *PKI) Stop() error {
+	return nil
+}
+
+// Interface guards
+var (
+	_ caddy.Provisioner = (*PKI)(nil)
+	_ caddy.App         = (*PKI)(nil)
+)

modules/caddytls/acmeissuer.go

@@ -0,0 +1,245 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddytls
+
+import (
+	"context"
+	"crypto/x509"
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"time"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/certmagic"
+	"github.com/go-acme/lego/v3/challenge"
+)
+
+func init() {
+	caddy.RegisterModule(ACMEIssuer{})
+}
+
+// ACMEIssuer makes an ACME manager
+// for managing certificates using ACME.
+//
+// TODO: support multiple ACME endpoints (probably
+// requires an array of these structs) - caddy would
+// also have to load certs from the backup CAs if the
+// first one is expired...
+type ACMEIssuer struct {
+	// The URL to the CA's ACME directory endpoint.
+	CA string `json:"ca,omitempty"`
+
+	// The URL to the test CA's ACME directory endpoint.
+	// This endpoint is only used during retries if there
+	// is a failure using the primary CA.
+	TestCA string `json:"test_ca,omitempty"`
+
+	// Your email address, so the CA can contact you if necessary.
+	// Not required, but strongly recommended to provide one so
+	// you can be reached if there is a problem. Your email is
+	// not sent to any Caddy mothership or used for any purpose
+	// other than ACME transactions.
+	Email string `json:"email,omitempty"`
+
+	// If using an ACME CA that requires an external account
+	// binding, specify the CA-provided credentials here.
+	ExternalAccount *ExternalAccountBinding `json:"external_account,omitempty"`
+
+	// Time to wait before timing out an ACME operation.
+	ACMETimeout caddy.Duration `json:"acme_timeout,omitempty"`
+
+	// Configures the various ACME challenge types.
+	Challenges *ChallengesConfig `json:"challenges,omitempty"`
+
+	// An array of files of CA certificates to accept when connecting to the
+	// ACME CA. Generally, you should only use this if the ACME CA endpoint
+	// is internal or for development/testing purposes.
+	TrustedRootsPEMFiles []string `json:"trusted_roots_pem_files,omitempty"`
+
+	rootPool *x509.CertPool
+	template certmagic.ACMEManager
+	magic    *certmagic.Config
+}
+
+// CaddyModule returns the Caddy module information.
+func (ACMEIssuer) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "tls.issuance.acme",
+		New: func() caddy.Module { return new(ACMEIssuer) },
+	}
+}
+
+// Provision sets up m.
+func (m *ACMEIssuer) Provision(ctx caddy.Context) error {
+	// DNS providers
+	if m.Challenges != nil && m.Challenges.DNSRaw != nil {
+		val, err := ctx.LoadModule(m.Challenges, "DNSRaw")
+		if err != nil {
+			return fmt.Errorf("loading DNS provider module: %v", err)
+		}
+		prov, err := val.(DNSProviderMaker).NewDNSProvider()
+		if err != nil {
+			return fmt.Errorf("making DNS provider: %v", err)
+		}
+		m.Challenges.DNS = prov
+	}
+
+	// add any custom CAs to trust store
+	if len(m.TrustedRootsPEMFiles) > 0 {
+		m.rootPool = x509.NewCertPool()
+		for _, pemFile := range m.TrustedRootsPEMFiles {
+			pemData, err := ioutil.ReadFile(pemFile)
+			if err != nil {
+				return fmt.Errorf("loading trusted root CA's PEM file: %s: %v", pemFile, err)
+			}
+			if !m.rootPool.AppendCertsFromPEM(pemData) {
+				return fmt.Errorf("unable to add %s to trust pool: %v", pemFile, err)
+			}
+		}
+	}
+
+	var err error
+	m.template, err = m.makeIssuerTemplate()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *ACMEIssuer) makeIssuerTemplate() (certmagic.ACMEManager, error) {
+	template := certmagic.ACMEManager{
+		CA:                m.CA,
+		Email:             m.Email,
+		CertObtainTimeout: time.Duration(m.ACMETimeout),
+		TrustedRoots:      m.rootPool,
+	}
+
+	if m.ExternalAccount != nil {
+		hmac, err := base64.StdEncoding.DecodeString(m.ExternalAccount.EncodedHMAC)
+		if err != nil {
+			return template, err
+		}
+		if m.ExternalAccount.KeyID == "" || len(hmac) == 0 {
+			return template, fmt.Errorf("when an external account binding is specified, both key ID and HMAC are required")
+		}
+		template.ExternalAccount = &certmagic.ExternalAccountBinding{
+			KeyID: m.ExternalAccount.KeyID,
+			HMAC:  hmac,
+		}
+	}
+
+	if m.Challenges != nil {
+		if m.Challenges.HTTP != nil {
+			template.DisableHTTPChallenge = m.Challenges.HTTP.Disabled
+			template.AltHTTPPort = m.Challenges.HTTP.AlternatePort
+		}
+		if m.Challenges.TLSALPN != nil {
+			template.DisableTLSALPNChallenge = m.Challenges.TLSALPN.Disabled
+			template.AltTLSALPNPort = m.Challenges.TLSALPN.AlternatePort
+		}
+		template.DNSProvider = m.Challenges.DNS
+		template.ListenHost = m.Challenges.BindHost
+	}
+
+	return template, nil
+}
+
+// SetConfig sets the associated certmagic config for this issuer.
+// This is required because ACME needs values from the config in
+// order to solve the challenges during issuance. This implements
+// the ConfigSetter interface.
+func (m *ACMEIssuer) SetConfig(cfg *certmagic.Config) {
+	m.magic = cfg
+}
+
+// TODO: I kind of hate how each call to these methods needs to
+// make a new ACME manager to fill in defaults before using; can
+// we find the right place to do that just once and then re-use?
+
+// PreCheck implements the certmagic.PreChecker interface.
+func (m *ACMEIssuer) PreCheck(names []string, interactive bool) error {
+	return certmagic.NewACMEManager(m.magic, m.template).PreCheck(names, interactive)
+}
+
+// Issue obtains a certificate for the given csr.
+func (m *ACMEIssuer) Issue(ctx context.Context, csr *x509.CertificateRequest) (*certmagic.IssuedCertificate, error) {
+	return certmagic.NewACMEManager(m.magic, m.template).Issue(ctx, csr)
+}
+
+// IssuerKey returns the unique issuer key for the configured CA endpoint.
+func (m *ACMEIssuer) IssuerKey() string {
+	return certmagic.NewACMEManager(m.magic, m.template).IssuerKey()
+}
+
+// Revoke revokes the given certificate.
+func (m *ACMEIssuer) Revoke(ctx context.Context, cert certmagic.CertificateResource) error {
+	return certmagic.NewACMEManager(m.magic, m.template).Revoke(ctx, cert)
+}
+
+// onDemandAskRequest makes a request to the ask URL
+// to see if a certificate can be obtained for name.
+// The certificate request should be denied if this
+// returns an error.
+func onDemandAskRequest(ask string, name string) error {
+	askURL, err := url.Parse(ask)
+	if err != nil {
+		return fmt.Errorf("parsing ask URL: %v", err)
+	}
+	qs := askURL.Query()
+	qs.Set("domain", name)
+	askURL.RawQuery = qs.Encode()
+
+	resp, err := onDemandAskClient.Get(askURL.String())
+	if err != nil {
+		return fmt.Errorf("error checking %v to deterine if certificate for hostname '%s' should be allowed: %v",
+			ask, name, err)
+	}
+	resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return fmt.Errorf("certificate for hostname '%s' not allowed; non-2xx status code %d returned from %v",
+			name, resp.StatusCode, ask)
+	}
+
+	return nil
+}
+
+// DNSProviderMaker is a type that can create a new DNS provider.
+// Modules in the tls.dns namespace should implement this interface.
+type DNSProviderMaker interface {
+	NewDNSProvider() (challenge.Provider, error)
+}
+
+// ExternalAccountBinding contains information for
+// binding an external account to an ACME account.
+type ExternalAccountBinding struct {
+	// The key identifier.
+	KeyID string `json:"key_id,omitempty"`
+
+	// The base64-encoded HMAC.
+	EncodedHMAC string `json:"hmac,omitempty"`
+}
+
+// Interface guards
+var (
+	_ certmagic.PreChecker = (*ACMEIssuer)(nil)
+	_ certmagic.Issuer     = (*ACMEIssuer)(nil)
+	_ certmagic.Revoker    = (*ACMEIssuer)(nil)
+	_ caddy.Provisioner    = (*ACMEIssuer)(nil)
+	_ ConfigSetter         = (*ACMEIssuer)(nil)
+)

modules/caddytls/acmemanager.go

@@ -1,252 +0,0 @@
-// Copyright 2015 Matthew Holt and The Caddy Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package caddytls
-
-import (
-	"crypto/x509"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/url"
-	"time"
-
-	"github.com/caddyserver/caddy/v2"
-	"github.com/go-acme/lego/v3/challenge"
-	"github.com/mholt/certmagic"
-)
-
-func init() {
-	caddy.RegisterModule(ACMEManagerMaker{})
-}
-
-// ACMEManagerMaker makes an ACME manager
-// for managing certificates using ACME.
-// If crafting one manually rather than
-// through the config-unmarshal process
-// (provisioning), be sure to call
-// SetDefaults to ensure sane defaults
-// after you have configured this struct
-// to your liking.
-type ACMEManagerMaker struct {
-	// The URL to the CA's ACME directory endpoint.
-	CA string `json:"ca,omitempty"`
-
-	// Your email address, so the CA can contact you if necessary.
-	// Not required, but strongly recommended to provide one so
-	// you can be reached if there is a problem. Your email is
-	// not sent to any Caddy mothership or used for any purpose
-	// other than ACME transactions.
-	Email string `json:"email,omitempty"`
-
-	// How long before a certificate's expiration to try renewing it.
-	// Should usually be about 1/3 of certificate lifetime, but long
-	// enough to give yourself time to troubleshoot problems before
-	// expiration. Default: 30d
-	RenewAhead caddy.Duration `json:"renew_ahead,omitempty"`
-
-	// The type of key to generate for the certificate.
-	// Supported values: `rsa2048`, `rsa4096`, `p256`, `p384`.
-	KeyType string `json:"key_type,omitempty"`
-
-	// Time to wait before timing out an ACME operation.
-	ACMETimeout caddy.Duration `json:"acme_timeout,omitempty"`
-
-	// If true, certificates will be requested with MustStaple. Not all
-	// CAs support this, and there are potentially serious consequences
-	// of enabling this feature without proper threat modeling.
-	MustStaple bool `json:"must_staple,omitempty"`
-
-	// Configures the various ACME challenge types.
-	Challenges *ChallengesConfig `json:"challenges,omitempty"`
-
-	// If true, certificates will be managed "on demand", that is, during
-	// TLS handshakes or when needed, as opposed to at startup or config
-	// load.
-	OnDemand bool `json:"on_demand,omitempty"`
-
-	// Optionally configure a separate storage module associated with this
-	// manager, instead of using Caddy's global/default-configured storage.
-	Storage json.RawMessage `json:"storage,omitempty" caddy:"namespace=caddy.storage inline_key=module"`
-
-	// An array of files of CA certificates to accept when connecting to the
-	// ACME CA. Generally, you should only use this if the ACME CA endpoint
-	// is internal or for development/testing purposes.
-	TrustedRootsPEMFiles []string `json:"trusted_roots_pem_files,omitempty"`
-
-	storage  certmagic.Storage
-	rootPool *x509.CertPool
-}
-
-// CaddyModule returns the Caddy module information.
-func (ACMEManagerMaker) CaddyModule() caddy.ModuleInfo {
-	return caddy.ModuleInfo{
-		ID:  "tls.management.acme",
-		New: func() caddy.Module { return new(ACMEManagerMaker) },
-	}
-}
-
-// NewManager is a no-op to satisfy the ManagerMaker interface,
-// because this manager type is a special case.
-func (m ACMEManagerMaker) NewManager(interactive bool) (certmagic.Manager, error) {
-	return nil, nil
-}
-
-// Provision sets up m.
-func (m *ACMEManagerMaker) Provision(ctx caddy.Context) error {
-	// DNS providers
-	if m.Challenges != nil && m.Challenges.DNSRaw != nil {
-		val, err := ctx.LoadModule(m.Challenges, "DNSRaw")
-		if err != nil {
-			return fmt.Errorf("loading DNS provider module: %v", err)
-		}
-		prov, err := val.(DNSProviderMaker).NewDNSProvider()
-		if err != nil {
-			return fmt.Errorf("making DNS provider: %v", err)
-		}
-		m.Challenges.DNS = prov
-	}
-
-	// policy-specific storage implementation
-	if m.Storage != nil {
-		val, err := ctx.LoadModule(m, "Storage")
-		if err != nil {
-			return fmt.Errorf("loading TLS storage module: %v", err)
-		}
-		cmStorage, err := val.(caddy.StorageConverter).CertMagicStorage()
-		if err != nil {
-			return fmt.Errorf("creating TLS storage configuration: %v", err)
-		}
-		m.storage = cmStorage
-	}
-
-	// add any custom CAs to trust store
-	if len(m.TrustedRootsPEMFiles) > 0 {
-		m.rootPool = x509.NewCertPool()
-		for _, pemFile := range m.TrustedRootsPEMFiles {
-			pemData, err := ioutil.ReadFile(pemFile)
-			if err != nil {
-				return fmt.Errorf("loading trusted root CA's PEM file: %s: %v", pemFile, err)
-			}
-			if !m.rootPool.AppendCertsFromPEM(pemData) {
-				return fmt.Errorf("unable to add %s to trust pool: %v", pemFile, err)
-			}
-		}
-	}
-
-	return nil
-}
-
-// makeCertMagicConfig converts m into a certmagic.Config, because
-// this is a special case where the default manager is the certmagic
-// Config and not a separate manager.
-func (m *ACMEManagerMaker) makeCertMagicConfig(ctx caddy.Context) certmagic.Config {
-	storage := m.storage
-	if storage == nil {
-		storage = ctx.Storage()
-	}
-
-	var ond *certmagic.OnDemandConfig
-	if m.OnDemand {
-		var onDemand *OnDemandConfig
-		appVal, err := ctx.App("tls")
-		if err == nil && appVal.(*TLS).Automation != nil {
-			onDemand = appVal.(*TLS).Automation.OnDemand
-		}
-
-		ond = &certmagic.OnDemandConfig{
-			DecisionFunc: func(name string) error {
-				if onDemand != nil {
-					if onDemand.Ask != "" {
-						err := onDemandAskRequest(onDemand.Ask, name)
-						if err != nil {
-							return err
-						}
-					}
-					// check the rate limiter last because
-					// doing so makes a reservation
-					if !onDemandRateLimiter.Allow() {
-						return fmt.Errorf("on-demand rate limit exceeded")
-					}
-				}
-				return nil
-			},
-		}
-	}
-
-	cfg := certmagic.Config{
-		CA:                  m.CA,
-		Email:               m.Email,
-		Agreed:              true,
-		RenewDurationBefore: time.Duration(m.RenewAhead),
-		KeyType:             supportedCertKeyTypes[m.KeyType],
-		CertObtainTimeout:   time.Duration(m.ACMETimeout),
-		OnDemand:            ond,
-		MustStaple:          m.MustStaple,
-		Storage:             storage,
-		TrustedRoots:        m.rootPool,
-		// TODO: listenHost
-	}
-
-	if m.Challenges != nil {
-		if m.Challenges.HTTP != nil {
-			cfg.DisableHTTPChallenge = m.Challenges.HTTP.Disabled
-			cfg.AltHTTPPort = m.Challenges.HTTP.AlternatePort
-		}
-		if m.Challenges.TLSALPN != nil {
-			cfg.DisableTLSALPNChallenge = m.Challenges.TLSALPN.Disabled
-			cfg.AltTLSALPNPort = m.Challenges.TLSALPN.AlternatePort
-		}
-		cfg.DNSProvider = m.Challenges.DNS
-	}
-
-	return cfg
-}
-
-// onDemandAskRequest makes a request to the ask URL
-// to see if a certificate can be obtained for name.
-// The certificate request should be denied if this
-// returns an error.
-func onDemandAskRequest(ask string, name string) error {
-	askURL, err := url.Parse(ask)
-	if err != nil {
-		return fmt.Errorf("parsing ask URL: %v", err)
-	}
-	qs := askURL.Query()
-	qs.Set("domain", name)
-	askURL.RawQuery = qs.Encode()
-
-	resp, err := onDemandAskClient.Get(askURL.String())
-	if err != nil {
-		return fmt.Errorf("error checking %v to determine if certificate for hostname '%s' should be allowed: %v",
-			ask, name, err)
-	}
-	resp.Body.Close()
-
-	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		return fmt.Errorf("certificate for hostname '%s' not allowed; non-2xx status code %d returned from %v",
-			name, resp.StatusCode, ask)
-	}
-
-	return nil
-}
-
-// DNSProviderMaker is a type that can create a new DNS provider.
-// Modules in the tls.dns namespace should implement this interface.
-type DNSProviderMaker interface {
-	NewDNSProvider() (challenge.Provider, error)
-}
-
-// Interface guard
-var _ ManagerMaker = (*ACMEManagerMaker)(nil)

modules/caddytls/automation.go

@@ -0,0 +1,328 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddytls
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/certmagic"
+	"github.com/go-acme/lego/v3/challenge"
+	"go.uber.org/zap"
+)
+
+// AutomationConfig designates configuration for the
+// construction and use of ACME clients.
+type AutomationConfig struct {
+	// The list of automation policies. The first matching
+	// policy will be applied for a given certificate/name.
+	Policies []*AutomationPolicy `json:"policies,omitempty"`
+
+	// On-Demand TLS defers certificate operations to the
+	// moment they are needed, e.g. during a TLS handshake.
+	// Useful when you don't know all the hostnames up front.
+	// Caddy was the first web server to deploy this technology.
+	OnDemand *OnDemandConfig `json:"on_demand,omitempty"`
+
+	// Caddy staples OCSP (and caches the response) for all
+	// qualifying certificates by default. This setting
+	// changes how often it scans responses for freshness,
+	// and updates them if they are getting stale.
+	OCSPCheckInterval caddy.Duration `json:"ocsp_interval,omitempty"`
+
+	// Every so often, Caddy will scan all loaded, managed
+	// certificates for expiration. This setting changes how
+	// frequently the scan for expiring certificates is
+	// performed. If your certificate lifetimes are very
+	// short (less than ~24 hours), you should set this to
+	// a low value.
+	RenewCheckInterval caddy.Duration `json:"renew_interval,omitempty"`
+
+	defaultPublicAutomationPolicy   *AutomationPolicy
+	defaultInternalAutomationPolicy *AutomationPolicy // only initialized if necessary
+}
+
+// AutomationPolicy designates the policy for automating the
+// management (obtaining, renewal, and revocation) of managed
+// TLS certificates.
+//
+// An AutomationPolicy value is not valid until it has been
+// provisioned; use the `AddAutomationPolicy()` method on the
+// TLS app to properly provision a new policy.
+type AutomationPolicy struct {
+	// Which subjects (hostnames or IP addresses) this policy applies to.
+	Subjects []string `json:"subjects,omitempty"`
+
+	// The module that will issue certificates. Default: internal if all
+	// subjects do not qualify for public certificates; othewise acme.
+	IssuerRaw json.RawMessage `json:"issuer,omitempty" caddy:"namespace=tls.issuance inline_key=module"`
+
+	// If true, certificates will be requested with MustStaple. Not all
+	// CAs support this, and there are potentially serious consequences
+	// of enabling this feature without proper threat modeling.
+	MustStaple bool `json:"must_staple,omitempty"`
+
+	// How long before a certificate's expiration to try renewing it,
+	// as a function of its total lifetime. As a general and conservative
+	// rule, it is a good idea to renew a certificate when it has about
+	// 1/3 of its total lifetime remaining. This utilizes the majority
+	// of the certificate's lifetime while still saving time to
+	// troubleshoot problems. However, for extremely short-lived certs,
+	// you may wish to increase the ratio to ~1/2.
+	RenewalWindowRatio float64 `json:"renewal_window_ratio,omitempty"`
+
+	// The type of key to generate for certificates.
+	// Supported values: `ed25519`, `p256`, `p384`, `rsa2048`, `rsa4096`.
+	KeyType string `json:"key_type,omitempty"`
+
+	// Optionally configure a separate storage module associated with this
+	// manager, instead of using Caddy's global/default-configured storage.
+	StorageRaw json.RawMessage `json:"storage,omitempty" caddy:"namespace=caddy.storage inline_key=module"`
+
+	// If true, certificates will be managed "on demand"; that is, during
+	// TLS handshakes or when needed, as opposed to at startup or config
+	// load.
+	OnDemand bool `json:"on_demand,omitempty"`
+
+	// Issuer stores the decoded issuer parameters. This is only
+	// used to populate an underlying certmagic.Config's Issuer
+	// field; it is not referenced thereafter.
+	Issuer certmagic.Issuer `json:"-"`
+
+	magic   *certmagic.Config
+	storage certmagic.Storage
+}
+
+// Provision sets up ap and builds its underlying CertMagic config.
+func (ap *AutomationPolicy) Provision(tlsApp *TLS) error {
+	// policy-specific storage implementation
+	if ap.StorageRaw != nil {
+		val, err := tlsApp.ctx.LoadModule(ap, "StorageRaw")
+		if err != nil {
+			return fmt.Errorf("loading TLS storage module: %v", err)
+		}
+		cmStorage, err := val.(caddy.StorageConverter).CertMagicStorage()
+		if err != nil {
+			return fmt.Errorf("creating TLS storage configuration: %v", err)
+		}
+		ap.storage = cmStorage
+	}
+
+	var ond *certmagic.OnDemandConfig
+	if ap.OnDemand {
+		ond = &certmagic.OnDemandConfig{
+			DecisionFunc: func(name string) error {
+				// if an "ask" endpoint was defined, consult it first
+				if tlsApp.Automation != nil &&
+					tlsApp.Automation.OnDemand != nil &&
+					tlsApp.Automation.OnDemand.Ask != "" {
+					err := onDemandAskRequest(tlsApp.Automation.OnDemand.Ask, name)
+					if err != nil {
+						return err
+					}
+				}
+				// check the rate limiter last because
+				// doing so makes a reservation
+				if !onDemandRateLimiter.Allow() {
+					return fmt.Errorf("on-demand rate limit exceeded")
+				}
+				return nil
+			},
+		}
+	}
+
+	// if this automation policy has no Issuer defined, and
+	// none of the subjects qualify for a public certificate,
+	// set the issuer to internal so that these names can all
+	// get certificates; critically, we can only do this if an
+	// issuer is not explicitly configured (IssuerRaw, vs. just
+	// Issuer) AND if the list of subjects is non-empty
+	if ap.IssuerRaw == nil && len(ap.Subjects) > 0 {
+		var anyPublic bool
+		for _, s := range ap.Subjects {
+			if certmagic.SubjectQualifiesForPublicCert(s) {
+				anyPublic = true
+				break
+			}
+		}
+		if !anyPublic {
+			tlsApp.logger.Info("setting internal issuer for automation policy that has only internal subjects but no issuer configured",
+				zap.Strings("subjects", ap.Subjects))
+			ap.IssuerRaw = json.RawMessage(`{"module":"internal"}`)
+		}
+	}
+
+	// load and provision any explicitly-configured issuer module
+	if ap.IssuerRaw != nil {
+		val, err := tlsApp.ctx.LoadModule(ap, "IssuerRaw")
+		if err != nil {
+			return fmt.Errorf("loading TLS automation management module: %s", err)
+		}
+		ap.Issuer = val.(certmagic.Issuer)
+	}
+
+	keyType := ap.KeyType
+	if keyType != "" {
+		var err error
+		keyType, err = caddy.NewReplacer().ReplaceOrErr(ap.KeyType, true, true)
+		if err != nil {
+			return fmt.Errorf("invalid key type %s: %s", ap.KeyType, err)
+		}
+		if _, ok := supportedCertKeyTypes[keyType]; !ok {
+			return fmt.Errorf("unrecognized key type: %s", keyType)
+		}
+	}
+	keySource := certmagic.StandardKeyGenerator{
+		KeyType: supportedCertKeyTypes[keyType],
+	}
+
+	storage := ap.storage
+	if storage == nil {
+		storage = tlsApp.ctx.Storage()
+	}
+
+	template := certmagic.Config{
+		MustStaple:         ap.MustStaple,
+		RenewalWindowRatio: ap.RenewalWindowRatio,
+		KeySource:          keySource,
+		OnDemand:           ond,
+		Storage:            storage,
+		Issuer:             ap.Issuer, // if nil, certmagic.New() will create one
+	}
+	if rev, ok := ap.Issuer.(certmagic.Revoker); ok {
+		template.Revoker = rev
+	}
+	ap.magic = certmagic.New(tlsApp.certCache, template)
+
+	// sometimes issuers may need the parent certmagic.Config in
+	// order to function properly (for example, ACMEIssuer needs
+	// access to the correct storage and cache so it can solve
+	// ACME challenges -- it's an annoying, inelegant circular
+	// dependency that I don't know how to resolve nicely!)
+	if annoying, ok := ap.Issuer.(ConfigSetter); ok {
+		annoying.SetConfig(ap.magic)
+	}
+
+	return nil
+}
+
+// ChallengesConfig configures the ACME challenges.
+type ChallengesConfig struct {
+	// HTTP configures the ACME HTTP challenge. This
+	// challenge is enabled and used automatically
+	// and by default.
+	HTTP *HTTPChallengeConfig `json:"http,omitempty"`
+
+	// TLSALPN configures the ACME TLS-ALPN challenge.
+	// This challenge is enabled and used automatically
+	// and by default.
+	TLSALPN *TLSALPNChallengeConfig `json:"tls-alpn,omitempty"`
+
+	// Configures the ACME DNS challenge. Because this
+	// challenge typically requires credentials for
+	// interfacing with a DNS provider, this challenge is
+	// not enabled by default. This is the only challenge
+	// type which does not require a direct connection
+	// to Caddy from an external server.
+	DNSRaw json.RawMessage `json:"dns,omitempty" caddy:"namespace=tls.dns inline_key=provider"`
+
+	// Optionally customize the host to which a listener
+	// is bound if required for solving a challenge.
+	BindHost string `json:"bind_host,omitempty"`
+
+	DNS challenge.Provider `json:"-"`
+}
+
+// HTTPChallengeConfig configures the ACME HTTP challenge.
+type HTTPChallengeConfig struct {
+	// If true, the HTTP challenge will be disabled.
+	Disabled bool `json:"disabled,omitempty"`
+
+	// An alternate port on which to service this
+	// challenge. Note that the HTTP challenge port is
+	// hard-coded into the spec and cannot be changed,
+	// so you would have to forward packets from the
+	// standard HTTP challenge port to this one.
+	AlternatePort int `json:"alternate_port,omitempty"`
+}
+
+// TLSALPNChallengeConfig configures the ACME TLS-ALPN challenge.
+type TLSALPNChallengeConfig struct {
+	// If true, the TLS-ALPN challenge will be disabled.
+	Disabled bool `json:"disabled,omitempty"`
+
+	// An alternate port on which to service this
+	// challenge. Note that the TLS-ALPN challenge port
+	// is hard-coded into the spec and cannot be changed,
+	// so you would have to forward packets from the
+	// standard TLS-ALPN challenge port to this one.
+	AlternatePort int `json:"alternate_port,omitempty"`
+}
+
+// OnDemandConfig configures on-demand TLS, for obtaining
+// needed certificates at handshake-time. Because this
+// feature can easily be abused, you should set up rate
+// limits and/or an internal endpoint that Caddy can
+// "ask" if it should be allowed to manage certificates
+// for a given hostname.
+type OnDemandConfig struct {
+	// An optional rate limit to throttle the
+	// issuance of certificates from handshakes.
+	RateLimit *RateLimit `json:"rate_limit,omitempty"`
+
+	// If Caddy needs to obtain or renew a certificate
+	// during a TLS handshake, it will perform a quick
+	// HTTP request to this URL to check if it should be
+	// allowed to try to get a certificate for the name
+	// in the "domain" query string parameter, like so:
+	// `?domain=example.com`. The endpoint must return a
+	// 200 OK status if a certificate is allowed;
+	// anything else will cause it to be denied.
+	// Redirects are not followed.
+	Ask string `json:"ask,omitempty"`
+}
+
+// RateLimit specifies an interval with optional burst size.
+type RateLimit struct {
+	// A duration value. A certificate may be obtained 'burst'
+	// times during this interval.
+	Interval caddy.Duration `json:"interval,omitempty"`
+
+	// How many times during an interval a certificate can be obtained.
+	Burst int `json:"burst,omitempty"`
+}
+
+// ConfigSetter is implemented by certmagic.Issuers that
+// need access to a parent certmagic.Config as part of
+// their provisioning phase. For example, the ACMEIssuer
+// requires a config so it can access storage and the
+// cache to solve ACME challenges.
+type ConfigSetter interface {
+	SetConfig(cfg *certmagic.Config)
+}
+
+// These perpetual values are used for on-demand TLS.
+var (
+	onDemandRateLimiter = certmagic.NewRateLimiter(0, 0)
+	onDemandAskClient   = &http.Client{
+		Timeout: 10 * time.Second,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return fmt.Errorf("following http redirects is not allowed")
+		},
+	}
+)

modules/caddytls/certselection.go

@@ -1,71 +1,145 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package caddytls
 
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/json"
 	"fmt"
 	"math/big"
 
-	"github.com/caddyserver/caddy/v2"
-	"github.com/mholt/certmagic"
+	"github.com/caddyserver/certmagic"
 )
 
-func init() {
-	caddy.RegisterModule(CustomCertSelectionPolicy{})
-}
-
 // CustomCertSelectionPolicy represents a policy for selecting the certificate
 // used to complete a handshake when there may be multiple options. All fields
 // specified must match the candidate certificate for it to be chosen.
 // This was needed to solve https://github.com/caddyserver/caddy/issues/2588.
 type CustomCertSelectionPolicy struct {
-	SerialNumber        *big.Int           `json:"serial_number,omitempty"`
-	SubjectOrganization string             `json:"subject_organization,omitempty"`
-	PublicKeyAlgorithm  PublicKeyAlgorithm `json:"public_key_algorithm,omitempty"`
-	Tag                 string             `json:"tag,omitempty"`
+	// The certificate must have one of these serial numbers.
+	SerialNumber []bigInt `json:"serial_number,omitempty"`
+
+	// The certificate must have one of these organization names.
+	SubjectOrganization []string `json:"subject_organization,omitempty"`
+
+	// The certificate must use this public key algorithm.
+	PublicKeyAlgorithm PublicKeyAlgorithm `json:"public_key_algorithm,omitempty"`
+
+	// The certificate must have at least one of the tags in the list.
+	AnyTag []string `json:"any_tag,omitempty"`
+
+	// The certificate must have all of the tags in the list.
+	AllTags []string `json:"all_tags,omitempty"`
 }
 
-// CaddyModule returns the Caddy module information.
-func (CustomCertSelectionPolicy) CaddyModule() caddy.ModuleInfo {
-	return caddy.ModuleInfo{
-		ID:  "tls.certificate_selection.custom",
-		New: func() caddy.Module { return new(CustomCertSelectionPolicy) },
-	}
-}
+// SelectCertificate implements certmagic.CertificateSelector. It
+// only chooses a certificate that at least meets the criteria in
+// p. It then chooses the first non-expired certificate that is
+// compatible with the client. If none are valid, it chooses the
+// first viable candidate anyway.
+func (p CustomCertSelectionPolicy) SelectCertificate(hello *tls.ClientHelloInfo, choices []certmagic.Certificate) (certmagic.Certificate, error) {
+	viable := make([]certmagic.Certificate, 0, len(choices))
 
-// SelectCertificate implements certmagic.CertificateSelector.
-func (p CustomCertSelectionPolicy) SelectCertificate(_ *tls.ClientHelloInfo, choices []certmagic.Certificate) (certmagic.Certificate, error) {
+nextChoice:
 	for _, cert := range choices {
-		if p.SerialNumber != nil && cert.SerialNumber.Cmp(p.SerialNumber) != 0 {
-			continue
-		}
-
-		if p.PublicKeyAlgorithm != PublicKeyAlgorithm(x509.UnknownPublicKeyAlgorithm) &&
-			PublicKeyAlgorithm(cert.PublicKeyAlgorithm) != p.PublicKeyAlgorithm {
-			continue
-		}
-
-		if p.SubjectOrganization != "" {
-			var matchOrg bool
-			for _, org := range cert.Subject.Organization {
-				if p.SubjectOrganization == org {
-					matchOrg = true
+		if len(p.SerialNumber) > 0 {
+			var found bool
+			for _, sn := range p.SerialNumber {
+				if cert.Leaf.SerialNumber.Cmp(&sn.Int) == 0 {
+					found = true
 					break
 				}
 			}
-			if !matchOrg {
+			if !found {
 				continue
 			}
 		}
 
-		if p.Tag != "" && !cert.HasTag(p.Tag) {
+		if len(p.SubjectOrganization) > 0 {
+			var found bool
+			for _, subjOrg := range p.SubjectOrganization {
+				for _, org := range cert.Leaf.Subject.Organization {
+					if subjOrg == org {
+						found = true
+						break
+					}
+				}
+			}
+			if !found {
+				continue
+			}
+		}
+
+		if p.PublicKeyAlgorithm != PublicKeyAlgorithm(x509.UnknownPublicKeyAlgorithm) &&
+			PublicKeyAlgorithm(cert.Leaf.PublicKeyAlgorithm) != p.PublicKeyAlgorithm {
 			continue
 		}
 
-		return cert, nil
+		if len(p.AnyTag) > 0 {
+			var found bool
+			for _, tag := range p.AnyTag {
+				if cert.HasTag(tag) {
+					found = true
+					break
+				}
+			}
+			if !found {
+				continue
+			}
+		}
+
+		if len(p.AllTags) > 0 {
+			for _, tag := range p.AllTags {
+				if !cert.HasTag(tag) {
+					continue nextChoice
+				}
+			}
+		}
+
+		// this certificate at least meets the policy's requirements,
+		// but we still have to check expiration and compatibility
+		viable = append(viable, cert)
 	}
-	return certmagic.Certificate{}, fmt.Errorf("no certificates matched custom selection policy")
+
+	if len(viable) == 0 {
+		return certmagic.Certificate{}, fmt.Errorf("no certificates matched custom selection policy")
+	}
+
+	return certmagic.DefaultCertificateSelector(hello, viable)
 }
 
-// Interface guard
-var _ certmagic.CertificateSelector = (*CustomCertSelectionPolicy)(nil)
+// bigInt is a big.Int type that interops with JSON encodings as a string.
+type bigInt struct{ big.Int }
+
+func (bi bigInt) MarshalJSON() ([]byte, error) {
+	return json.Marshal(bi.String())
+}
+
+func (bi *bigInt) UnmarshalJSON(p []byte) error {
+	if string(p) == "null" {
+		return nil
+	}
+	var stringRep string
+	err := json.Unmarshal(p, &stringRep)
+	if err != nil {
+		return err
+	}
+	_, ok := bi.SetString(stringRep, 10)
+	if !ok {
+		return fmt.Errorf("not a valid big integer: %s", p)
+	}
+	return nil
+}