tls: add system and combined CA pool modules (#7406)

* feat: add system and combined CA pool modules

* fix: combining pools using `CertificateProvider`

* fix: lint issue

* chore: caddyfiletests
* doing it for first time, so not sure if its right.

* fix: use `x509` native addCert

* chore: explicit err handling

* Apply suggestion from @mohammed90

---------

Co-authored-by: Mohammed Al Sahaf <mohammed@caffeinatedwonders.com>

caddytest/integration/caddyfile_adapt/tls_client_auth_combined_trust_pool.caddyfiletest

@@ -0,0 +1,87 @@
+localhost
+
+respond "hello from localhost"
+tls {
+	client_auth {
+		mode request
+		trust_pool combined {
+			source inline {
+				trust_der MIIDSzCCAjOgAwIBAgIUfIRObjWNUA4jxQ/0x8BOCvE2Vw4wDQYJKoZIhvcNAQELBQAwFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwHhcNMTkwODI4MTYyNTU5WhcNMjkwODI1MTYyNTU5WjAWMRQwEgYDVQQDDAtFYXN5LVJTQSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK5m5elxhQfMp/3aVJ4JnpN9PUSz6LlP6LePAPFU7gqohVVFVtDkChJAG3FNkNQNlieVTja/bgH9IcC6oKbROwdY1h0MvNV8AHHigvl03WuJD8g2ReVFXXwsnrPmKXCFzQyMI6TYk3m2gYrXsZOU1GLnfMRC3KAMRgE2F45twOs9hqG169YJ6mM2eQjzjCHWI6S2/iUYvYxRkCOlYUbLsMD/AhgAf1plzg6LPqNxtdlwxZnA0ytgkmhK67HtzJu0+ovUCsMv0RwcMhsEo9T8nyFAGt9XLZ63X5WpBCTUApaAUhnG0XnerjmUWb6eUWw4zev54sEfY5F3x002iQaW6cECAwEAAaOBkDCBjTAdBgNVHQ4EFgQU4CBUbZsS2GaNIkGRz/cBsD5ivjswUQYDVR0jBEowSIAU4CBUbZsS2GaNIkGRz/cBsD5ivjuhGqQYMBYxFDASBgNVBAMMC0Vhc3ktUlNBIENBghR8hE5uNY1QDiPFD/THwE4K8TZXDjAMBgNVHRMEBTADAQH/MAsGA1UdDwQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAQEAKB3V4HIzoiO/Ch6WMj9bLJ2FGbpkMrcb/Eq01hT5zcfKD66lVS1MlK+cRL446Z2b2KDP1oFyVs+qmrmtdwrWgD+nfe2sBmmIHo9m9KygMkEOfG3MghGTEcS+0cTKEcoHYWYyOqQh6jnedXY8Cdm4GM1hAc9MiL3/sqV8YCVSLNnkoNysmr06/rZ0MCUZPGUtRmfd0heWhrfzAKw2HLgX+RAmpOE2MZqWcjvqKGyaRiaZks4nJkP6521aC2Lgp0HhCz1j8/uQ5ldoDszCnu/iro0NAsNtudTMD+YoLQxLqdleIh6CW+illc2VdXwj7mn6J04yns9jfE2jRjW/yTLFuQ==
+			}
+			source file {
+				pem_file ../caddy.ca.cer
+			}
+		}
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"body": "hello from localhost",
+													"handler": "static_response"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"tls_connection_policies": [
+						{
+							"match": {
+								"sni": [
+									"localhost"
+								]
+							},
+							"client_authentication": {
+								"ca": {
+									"provider": "combined",
+									"sources": [
+										{
+											"provider": "inline",
+											"trusted_ca_certs": [
+												"MIIDSzCCAjOgAwIBAgIUfIRObjWNUA4jxQ/0x8BOCvE2Vw4wDQYJKoZIhvcNAQELBQAwFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwHhcNMTkwODI4MTYyNTU5WhcNMjkwODI1MTYyNTU5WjAWMRQwEgYDVQQDDAtFYXN5LVJTQSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK5m5elxhQfMp/3aVJ4JnpN9PUSz6LlP6LePAPFU7gqohVVFVtDkChJAG3FNkNQNlieVTja/bgH9IcC6oKbROwdY1h0MvNV8AHHigvl03WuJD8g2ReVFXXwsnrPmKXCFzQyMI6TYk3m2gYrXsZOU1GLnfMRC3KAMRgE2F45twOs9hqG169YJ6mM2eQjzjCHWI6S2/iUYvYxRkCOlYUbLsMD/AhgAf1plzg6LPqNxtdlwxZnA0ytgkmhK67HtzJu0+ovUCsMv0RwcMhsEo9T8nyFAGt9XLZ63X5WpBCTUApaAUhnG0XnerjmUWb6eUWw4zev54sEfY5F3x002iQaW6cECAwEAAaOBkDCBjTAdBgNVHQ4EFgQU4CBUbZsS2GaNIkGRz/cBsD5ivjswUQYDVR0jBEowSIAU4CBUbZsS2GaNIkGRz/cBsD5ivjuhGqQYMBYxFDASBgNVBAMMC0Vhc3ktUlNBIENBghR8hE5uNY1QDiPFD/THwE4K8TZXDjAMBgNVHRMEBTADAQH/MAsGA1UdDwQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAQEAKB3V4HIzoiO/Ch6WMj9bLJ2FGbpkMrcb/Eq01hT5zcfKD66lVS1MlK+cRL446Z2b2KDP1oFyVs+qmrmtdwrWgD+nfe2sBmmIHo9m9KygMkEOfG3MghGTEcS+0cTKEcoHYWYyOqQh6jnedXY8Cdm4GM1hAc9MiL3/sqV8YCVSLNnkoNysmr06/rZ0MCUZPGUtRmfd0heWhrfzAKw2HLgX+RAmpOE2MZqWcjvqKGyaRiaZks4nJkP6521aC2Lgp0HhCz1j8/uQ5ldoDszCnu/iro0NAsNtudTMD+YoLQxLqdleIh6CW+illc2VdXwj7mn6J04yns9jfE2jRjW/yTLFuQ=="
+											]
+										},
+										{
+											"pem_files": [
+												"../caddy.ca.cer"
+											],
+											"provider": "file"
+										}
+									]
+								},
+								"mode": "request"
+							}
+						},
+						{}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_client_auth_combined_trust_pool_pki.caddyfiletest

@@ -0,0 +1,87 @@
+localhost
+
+respond "hello from localhost"
+tls {
+	client_auth {
+		mode require_and_verify
+		trust_pool combined {
+			source inline {
+				trust_der MIIDSzCCAjOgAwIBAgIUfIRObjWNUA4jxQ/0x8BOCvE2Vw4wDQYJKoZIhvcNAQELBQAwFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwHhcNMTkwODI4MTYyNTU5WhcNMjkwODI1MTYyNTU5WjAWMRQwEgYDVQQDDAtFYXN5LVJTQSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK5m5elxhQfMp/3aVJ4JnpN9PUSz6LlP6LePAPFU7gqohVVFVtDkChJAG3FNkNQNlieVTja/bgH9IcC6oKbROwdY1h0MvNV8AHHigvl03WuJD8g2ReVFXXwsnrPmKXCFzQyMI6TYk3m2gYrXsZOU1GLnfMRC3KAMRgE2F45twOs9hqG169YJ6mM2eQjzjCHWI6S2/iUYvYxRkCOlYUbLsMD/AhgAf1plzg6LPqNxtdlwxZnA0ytgkmhK67HtzJu0+ovUCsMv0RwcMhsEo9T8nyFAGt9XLZ63X5WpBCTUApaAUhnG0XnerjmUWb6eUWw4zev54sEfY5F3x002iQaW6cECAwEAAaOBkDCBjTAdBgNVHQ4EFgQU4CBUbZsS2GaNIkGRz/cBsD5ivjswUQYDVR0jBEowSIAU4CBUbZsS2GaNIkGRz/cBsD5ivjuhGqQYMBYxFDASBgNVBAMMC0Vhc3ktUlNBIENBghR8hE5uNY1QDiPFD/THwE4K8TZXDjAMBgNVHRMEBTADAQH/MAsGA1UdDwQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAQEAKB3V4HIzoiO/Ch6WMj9bLJ2FGbpkMrcb/Eq01hT5zcfKD66lVS1MlK+cRL446Z2b2KDP1oFyVs+qmrmtdwrWgD+nfe2sBmmIHo9m9KygMkEOfG3MghGTEcS+0cTKEcoHYWYyOqQh6jnedXY8Cdm4GM1hAc9MiL3/sqV8YCVSLNnkoNysmr06/rZ0MCUZPGUtRmfd0heWhrfzAKw2HLgX+RAmpOE2MZqWcjvqKGyaRiaZks4nJkP6521aC2Lgp0HhCz1j8/uQ5ldoDszCnu/iro0NAsNtudTMD+YoLQxLqdleIh6CW+illc2VdXwj7mn6J04yns9jfE2jRjW/yTLFuQ==
+			}
+			source pki_root {
+				authority local
+			}
+		}
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"body": "hello from localhost",
+													"handler": "static_response"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"tls_connection_policies": [
+						{
+							"match": {
+								"sni": [
+									"localhost"
+								]
+							},
+							"client_authentication": {
+								"ca": {
+									"provider": "combined",
+									"sources": [
+										{
+											"provider": "inline",
+											"trusted_ca_certs": [
+												"MIIDSzCCAjOgAwIBAgIUfIRObjWNUA4jxQ/0x8BOCvE2Vw4wDQYJKoZIhvcNAQELBQAwFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwHhcNMTkwODI4MTYyNTU5WhcNMjkwODI1MTYyNTU5WjAWMRQwEgYDVQQDDAtFYXN5LVJTQSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK5m5elxhQfMp/3aVJ4JnpN9PUSz6LlP6LePAPFU7gqohVVFVtDkChJAG3FNkNQNlieVTja/bgH9IcC6oKbROwdY1h0MvNV8AHHigvl03WuJD8g2ReVFXXwsnrPmKXCFzQyMI6TYk3m2gYrXsZOU1GLnfMRC3KAMRgE2F45twOs9hqG169YJ6mM2eQjzjCHWI6S2/iUYvYxRkCOlYUbLsMD/AhgAf1plzg6LPqNxtdlwxZnA0ytgkmhK67HtzJu0+ovUCsMv0RwcMhsEo9T8nyFAGt9XLZ63X5WpBCTUApaAUhnG0XnerjmUWb6eUWw4zev54sEfY5F3x002iQaW6cECAwEAAaOBkDCBjTAdBgNVHQ4EFgQU4CBUbZsS2GaNIkGRz/cBsD5ivjswUQYDVR0jBEowSIAU4CBUbZsS2GaNIkGRz/cBsD5ivjuhGqQYMBYxFDASBgNVBAMMC0Vhc3ktUlNBIENBghR8hE5uNY1QDiPFD/THwE4K8TZXDjAMBgNVHRMEBTADAQH/MAsGA1UdDwQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAQEAKB3V4HIzoiO/Ch6WMj9bLJ2FGbpkMrcb/Eq01hT5zcfKD66lVS1MlK+cRL446Z2b2KDP1oFyVs+qmrmtdwrWgD+nfe2sBmmIHo9m9KygMkEOfG3MghGTEcS+0cTKEcoHYWYyOqQh6jnedXY8Cdm4GM1hAc9MiL3/sqV8YCVSLNnkoNysmr06/rZ0MCUZPGUtRmfd0heWhrfzAKw2HLgX+RAmpOE2MZqWcjvqKGyaRiaZks4nJkP6521aC2Lgp0HhCz1j8/uQ5ldoDszCnu/iro0NAsNtudTMD+YoLQxLqdleIh6CW+illc2VdXwj7mn6J04yns9jfE2jRjW/yTLFuQ=="
+											]
+										},
+										{
+											"authority": [
+												"local"
+											],
+											"provider": "pki_root"
+										}
+									]
+								},
+								"mode": "require_and_verify"
+							}
+						},
+						{}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/tls_client_auth_system_trust_pool.caddyfiletest

@@ -0,0 +1,66 @@
+localhost
+
+respond "hello from localhost"
+tls {
+	client_auth {
+		mode request
+		trust_pool system
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"body": "hello from localhost",
+													"handler": "static_response"
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"tls_connection_policies": [
+						{
+							"match": {
+								"sni": [
+									"localhost"
+								]
+							},
+							"client_authentication": {
+								"ca": {
+									"provider": "system"
+								},
+								"mode": "request"
+							}
+						},
+						{}
+					]
+				}
+			}
+		}
+	}
+}

modules/caddytls/capools.go

@@ -4,6 +4,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
+	"encoding/pem"
 	"errors"
 	"fmt"
 	"io"
@@ -27,6 +28,8 @@ func init() {
 	caddy.RegisterModule(PKIIntermediateCAPool{})
 	caddy.RegisterModule(StoragePool{})
 	caddy.RegisterModule(HTTPCertPool{})
+	caddy.RegisterModule(SystemCAPool{})
+	caddy.RegisterModule(CombinedCAPool{})
 }
 
 // The interface to be implemented by all guest modules part of
@@ -35,6 +38,12 @@ type CA interface {
 	CertPool() *x509.CertPool
 }
 
+// CertificateProvider is an optional interface that CA pool sources
+// can implement to expose their underlying certificates for combining.
+type CertificateProvider interface {
+	Certificates() []*x509.Certificate
+}
+
 // InlineCAPool is a certificate authority pool provider coming from
 // a DER-encoded certificates in the config
 type InlineCAPool struct {
@@ -44,7 +53,8 @@ type InlineCAPool struct {
 	// these CAs will be rejected.
 	TrustedCACerts []string `json:"trusted_ca_certs,omitempty"`
 
-	pool *x509.CertPool
+	pool  *x509.CertPool
+	certs []*x509.Certificate
 }
 
 // CaddyModule implements caddy.Module.
@@ -60,14 +70,17 @@ func (icp InlineCAPool) CaddyModule() caddy.ModuleInfo {
 // Provision implements caddy.Provisioner.
 func (icp *InlineCAPool) Provision(ctx caddy.Context) error {
 	caPool := x509.NewCertPool()
+	var certs []*x509.Certificate
 	for i, clientCAString := range icp.TrustedCACerts {
 		clientCA, err := decodeBase64DERCert(clientCAString)
 		if err != nil {
 			return fmt.Errorf("parsing certificate at index %d: %v", i, err)
 		}
 		caPool.AddCert(clientCA)
+		certs = append(certs, clientCA)
 	}
 	icp.pool = caPool
+	icp.certs = certs
 
 	return nil
 }
@@ -103,6 +116,11 @@ func (icp InlineCAPool) CertPool() *x509.CertPool {
 	return icp.pool
 }
 
+// Certificates implements CertificateProvider.
+func (icp InlineCAPool) Certificates() []*x509.Certificate {
+	return icp.certs
+}
+
 // FileCAPool generates trusted root certificates pool from the designated DER and PEM file
 type FileCAPool struct {
 	// TrustedCACertPEMFiles is a list of PEM file names
@@ -111,7 +129,8 @@ type FileCAPool struct {
 	// these CA certificates will be rejected.
 	TrustedCACertPEMFiles []string `json:"pem_files,omitempty"`
 
-	pool *x509.CertPool
+	pool  *x509.CertPool
+	certs []*x509.Certificate
 }
 
 // CaddyModule implements caddy.Module.
@@ -127,14 +146,32 @@ func (FileCAPool) CaddyModule() caddy.ModuleInfo {
 // Loads and decodes the DER and pem files to generate the certificate pool
 func (f *FileCAPool) Provision(ctx caddy.Context) error {
 	caPool := x509.NewCertPool()
+	var certs []*x509.Certificate
 	for _, pemFile := range f.TrustedCACertPEMFiles {
 		pemContents, err := os.ReadFile(pemFile)
 		if err != nil {
 			return fmt.Errorf("reading %s: %v", pemFile, err)
 		}
-		caPool.AppendCertsFromPEM(pemContents)
+		// Parse PEM to extract certificates
+		for len(pemContents) > 0 {
+			var block *pem.Block
+			block, pemContents = pem.Decode(pemContents)
+			if block == nil {
+				break
+			}
+			if block.Type != "CERTIFICATE" {
+				continue
+			}
+			cert, err := x509.ParseCertificate(block.Bytes)
+			if err != nil {
+				return fmt.Errorf("parsing certificate in %s: %v", pemFile, err)
+			}
+			caPool.AddCert(cert)
+			certs = append(certs, cert)
+		}
 	}
 	f.pool = caPool
+	f.certs = certs
 	return nil
 }
 
@@ -166,13 +203,19 @@ func (f FileCAPool) CertPool() *x509.CertPool {
 	return f.pool
 }
 
+// Certificates implements CertificateProvider.
+func (f FileCAPool) Certificates() []*x509.Certificate {
+	return f.certs
+}
+
 // PKIRootCAPool extracts the trusted root certificates from Caddy's native 'pki' app
 type PKIRootCAPool struct {
 	// List of the Authority names that are configured in the `pki` app whose root certificates are trusted
 	Authority []string `json:"authority,omitempty"`
 
-	ca   []*caddypki.CA
-	pool *x509.CertPool
+	ca    []*caddypki.CA
+	pool  *x509.CertPool
+	certs []*x509.Certificate
 }
 
 // CaddyModule implements caddy.Module.
@@ -201,10 +244,17 @@ func (p *PKIRootCAPool) Provision(ctx caddy.Context) error {
 	}
 
 	caPool := x509.NewCertPool()
+	var certs []*x509.Certificate
 	for _, ca := range p.ca {
-		caPool.AddCert(ca.RootCertificate())
+		rootCert := ca.RootCertificate()
+		if rootCert == nil {
+			return fmt.Errorf("CA %s has no root certificate", ca.ID)
+		}
+		caPool.AddCert(rootCert)
+		certs = append(certs, rootCert)
 	}
 	p.pool = caPool
+	p.certs = certs
 
 	return nil
 }
@@ -238,13 +288,19 @@ func (p PKIRootCAPool) CertPool() *x509.CertPool {
 	return p.pool
 }
 
+// Certificates implements CertificateProvider.
+func (p PKIRootCAPool) Certificates() []*x509.Certificate {
+	return p.certs
+}
+
 // PKIIntermediateCAPool extracts the trusted intermediate certificates from Caddy's native 'pki' app
 type PKIIntermediateCAPool struct {
 	// List of the Authority names that are configured in the `pki` app whose intermediate certificates are trusted
 	Authority []string `json:"authority,omitempty"`
 
-	ca   []*caddypki.CA
-	pool *x509.CertPool
+	ca    []*caddypki.CA
+	pool  *x509.CertPool
+	certs []*x509.Certificate
 }
 
 // CaddyModule implements caddy.Module.
@@ -273,12 +329,18 @@ func (p *PKIIntermediateCAPool) Provision(ctx caddy.Context) error {
 	}
 
 	caPool := x509.NewCertPool()
+	var certs []*x509.Certificate
 	for _, ca := range p.ca {
 		for _, c := range ca.IntermediateCertificateChain() {
+			if c == nil {
+				return fmt.Errorf("CA %s has a nil certificate in its intermediate chain", ca.ID)
+			}
 			caPool.AddCert(c)
+			certs = append(certs, c)
 		}
 	}
 	p.pool = caPool
+	p.certs = certs
 	return nil
 }
 
@@ -311,6 +373,11 @@ func (p PKIIntermediateCAPool) CertPool() *x509.CertPool {
 	return p.pool
 }
 
+// Certificates implements CertificateProvider.
+func (p PKIIntermediateCAPool) Certificates() []*x509.Certificate {
+	return p.certs
+}
+
 // StoragePool extracts the trusted certificates root from Caddy storage
 type StoragePool struct {
 	// The storage module where the trusted root certificates are stored. Absent
@@ -322,6 +389,7 @@ type StoragePool struct {
 
 	storage certmagic.Storage
 	pool    *x509.CertPool
+	certs   []*x509.Certificate
 }
 
 // CaddyModule implements caddy.Module.
@@ -354,16 +422,33 @@ func (ca *StoragePool) Provision(ctx caddy.Context) error {
 		return fmt.Errorf("no PEM keys specified")
 	}
 	caPool := x509.NewCertPool()
+	var certs []*x509.Certificate
 	for _, caID := range ca.PEMKeys {
 		bs, err := ca.storage.Load(ctx, caID)
 		if err != nil {
 			return fmt.Errorf("error loading cert '%s' from storage: %s", caID, err)
 		}
-		if !caPool.AppendCertsFromPEM(bs) {
-			return fmt.Errorf("failed to add certificate '%s' to pool", caID)
+		// Parse PEM to extract certificates
+		pemData := bs
+		for len(pemData) > 0 {
+			var block *pem.Block
+			block, pemData = pem.Decode(pemData)
+			if block == nil {
+				break
+			}
+			if block.Type != "CERTIFICATE" {
+				continue
+			}
+			cert, err := x509.ParseCertificate(block.Bytes)
+			if err != nil {
+				return fmt.Errorf("parsing certificate '%s': %v", caID, err)
+			}
+			caPool.AddCert(cert)
+			certs = append(certs, cert)
 		}
 	}
 	ca.pool = caPool
+	ca.certs = certs
 
 	return nil
 }
@@ -413,9 +498,13 @@ func (p StoragePool) CertPool() *x509.CertPool {
 	return p.pool
 }
 
+// Certificates implements CertificateProvider.
+func (p StoragePool) Certificates() []*x509.Certificate {
+	return p.certs
+}
+
 // TLSConfig holds configuration related to the TLS configuration for the
 // transport/client.
-// copied from with minor modifications: modules/caddyhttp/reverseproxy/httptransport.go
 type TLSConfig struct {
 	// Provides the guest module that provides the trusted certificate authority (CA) certificates
 	CARaw json.RawMessage `json:"ca,omitempty" caddy:"namespace=tls.ca_pool.source inline_key=provider"`
@@ -500,7 +589,6 @@ func (t *TLSConfig) unmarshalCaddyfile(d *caddyfile.Dispenser) error {
 
 // MakeTLSClientConfig returns a tls.Config usable by a client to a backend.
 // If there is no custom TLS configuration, a nil config may be returned.
-// copied from with minor modifications: modules/caddyhttp/reverseproxy/httptransport.go
 func (t *TLSConfig) makeTLSClientConfig(ctx caddy.Context) (*tls.Config, error) {
 	repl, ok := ctx.Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
 	if !ok || repl == nil {
@@ -554,7 +642,8 @@ type HTTPCertPool struct {
 	// Customize the TLS connection knobs to used during the HTTP call
 	TLS *TLSConfig `json:"tls,omitempty"`
 
-	pool *x509.CertPool
+	pool  *x509.CertPool
+	certs []*x509.Certificate
 }
 
 // CaddyModule implements caddy.Module.
@@ -570,6 +659,7 @@ func (HTTPCertPool) CaddyModule() caddy.ModuleInfo {
 // Provision implements caddy.Provisioner.
 func (hcp *HTTPCertPool) Provision(ctx caddy.Context) error {
 	caPool := x509.NewCertPool()
+	var certs []*x509.Certificate
 
 	customTransport := http.DefaultTransport.(*http.Transport).Clone()
 	if hcp.TLS != nil {
@@ -597,11 +687,30 @@ func (hcp *HTTPCertPool) Provision(ctx caddy.Context) error {
 		if err != nil {
 			return err
 		}
-		if !caPool.AppendCertsFromPEM(pembs) {
-			return fmt.Errorf("failed to add certs from URL: %s", uri)
+		if res.StatusCode < 200 || res.StatusCode >= 300 {
+			return fmt.Errorf("HTTP %d fetching CA certificate bundle from %s", res.StatusCode, uri)
+		}
+		// Parse PEM to extract certificates
+		pemData := pembs
+		for len(pemData) > 0 {
+			var block *pem.Block
+			block, pemData = pem.Decode(pemData)
+			if block == nil {
+				break
+			}
+			if block.Type != "CERTIFICATE" {
+				continue
+			}
+			cert, err := x509.ParseCertificate(block.Bytes)
+			if err != nil {
+				return fmt.Errorf("parsing certificate from URL %s: %v", uri, err)
+			}
+			caPool.AddCert(cert)
+			certs = append(certs, cert)
 		}
 	}
 	hcp.pool = caPool
+	hcp.certs = certs
 	return nil
 }
 
@@ -665,6 +774,179 @@ func (hcp HTTPCertPool) CertPool() *x509.CertPool {
 	return hcp.pool
 }
 
+// Certificates implements CertificateProvider.
+func (hcp HTTPCertPool) Certificates() []*x509.Certificate {
+	return hcp.certs
+}
+
+// SystemCAPool obtains the trusted root certificates from the system's
+// certificate pool using x509.SystemCertPool()
+type SystemCAPool struct {
+	pool *x509.CertPool
+}
+
+// CaddyModule implements caddy.Module.
+func (SystemCAPool) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID: "tls.ca_pool.source.system",
+		New: func() caddy.Module {
+			return new(SystemCAPool)
+		},
+	}
+}
+
+// Provision implements caddy.Provisioner.
+func (scp *SystemCAPool) Provision(ctx caddy.Context) error {
+	pool, err := x509.SystemCertPool()
+	if err != nil {
+		return fmt.Errorf("failed to load system cert pool: %v", err)
+	}
+	scp.pool = pool
+	return nil
+}
+
+func (scp *SystemCAPool) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	d.Next() // consume module name
+	if d.CountRemainingArgs() > 0 {
+		return d.ArgErr()
+	}
+	if d.NextBlock(0) {
+		return d.Err("system trust pool does not support any configuration")
+	}
+	return nil
+}
+
+// CertPool implements CA.
+func (scp SystemCAPool) CertPool() *x509.CertPool {
+	return scp.pool
+}
+
+// The `combined` pool type merges multiple pools. The `sources` pools must implement the
+// `CertificateProvider` interface, which allows them to export their certificate set.
+//
+// Note: SystemCAPool does not implement CertificateProvider because
+// x509.SystemCertPool() doesn't expose its certificates, so it cannot
+// be used as a source in CombinedCAPool.
+type CombinedCAPool struct {
+	// The CA pool sources to combine. Each source is a CA pool provider module.
+	SourcesRaw []json.RawMessage `json:"sources,omitempty" caddy:"namespace=tls.ca_pool.source inline_key=provider"`
+
+	sources []CA
+	pool    *x509.CertPool
+	certs   []*x509.Certificate
+}
+
+// CaddyModule implements caddy.Module.
+func (CombinedCAPool) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID: "tls.ca_pool.source.combined",
+		New: func() caddy.Module {
+			return new(CombinedCAPool)
+		},
+	}
+}
+
+// Provision implements caddy.Provisioner.
+func (ccp *CombinedCAPool) Provision(ctx caddy.Context) error {
+	if len(ccp.SourcesRaw) == 0 {
+		return fmt.Errorf("no sources specified for combined CA pool")
+	}
+
+	// Load all source modules
+	sources, err := ctx.LoadModule(ccp, "SourcesRaw")
+	if err != nil {
+		return fmt.Errorf("loading CA pool sources: %v", err)
+	}
+
+	caPool := x509.NewCertPool()
+	var allCerts []*x509.Certificate
+
+	for _, src := range sources.([]any) {
+		ca, ok := src.(CA)
+		if !ok {
+			return fmt.Errorf("source module is not a CA pool provider")
+		}
+		ccp.sources = append(ccp.sources, ca)
+
+		certProvider, ok := ca.(CertificateProvider)
+		if !ok {
+			return fmt.Errorf("source %T does not implement CertificateProvider (required for combining)", ca)
+		}
+
+		certs := certProvider.Certificates()
+		if certs == nil {
+			return fmt.Errorf("source %T returned nil certificates", ca)
+		}
+		for _, cert := range certs {
+			if cert == nil {
+				return fmt.Errorf("source %T returned a nil certificate", ca)
+			}
+			caPool.AddCert(cert)
+			allCerts = append(allCerts, cert)
+		}
+	}
+
+	ccp.pool = caPool
+	ccp.certs = allCerts
+
+	return nil
+}
+
+// Syntax:
+//
+//	trust_pool combined {
+//		source <module_name> {
+//			<module_config>
+//		}
+//	}
+//
+// The 'source' directive can be specified multiple times. Sources that
+// don't implement CertificateProvider (like 'system') cannot be combined.
+func (ccp *CombinedCAPool) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	d.Next() // consume module name
+	if d.CountRemainingArgs() > 0 {
+		return d.ArgErr()
+	}
+
+	for nesting := d.Nesting(); d.NextBlock(nesting); {
+		switch d.Val() {
+		case "source":
+			if !d.NextArg() {
+				return d.ArgErr()
+			}
+			modStem := d.Val()
+			modID := "tls.ca_pool.source." + modStem
+			unm, err := caddyfile.UnmarshalModule(d, modID)
+			if err != nil {
+				return err
+			}
+			ca, ok := unm.(CA)
+			if !ok {
+				return d.Errf("module %s is not a CA pool provider", modID)
+			}
+			ccp.SourcesRaw = append(ccp.SourcesRaw, caddyconfig.JSONModuleObject(ca, "provider", modStem, nil))
+		default:
+			return d.Errf("unrecognized directive: %s", d.Val())
+		}
+	}
+
+	if len(ccp.SourcesRaw) == 0 {
+		return d.Err("no sources specified")
+	}
+
+	return nil
+}
+
+// CertPool implements CA.
+func (ccp CombinedCAPool) CertPool() *x509.CertPool {
+	return ccp.pool
+}
+
+// Certificates implements CertificateProvider.
+func (ccp CombinedCAPool) Certificates() []*x509.Certificate {
+	return ccp.certs
+}
+
 var (
 	_ caddy.Module          = (*InlineCAPool)(nil)
 	_ caddy.Provisioner     = (*InlineCAPool)(nil)
@@ -696,4 +978,14 @@ var (
 	_ caddy.Validator       = (*HTTPCertPool)(nil)
 	_ CA                    = (*HTTPCertPool)(nil)
 	_ caddyfile.Unmarshaler = (*HTTPCertPool)(nil)
+
+	_ caddy.Module          = (*SystemCAPool)(nil)
+	_ caddy.Provisioner     = (*SystemCAPool)(nil)
+	_ CA                    = (*SystemCAPool)(nil)
+	_ caddyfile.Unmarshaler = (*SystemCAPool)(nil)
+
+	_ caddy.Module          = (*CombinedCAPool)(nil)
+	_ caddy.Provisioner     = (*CombinedCAPool)(nil)
+	_ CA                    = (*CombinedCAPool)(nil)
+	_ caddyfile.Unmarshaler = (*CombinedCAPool)(nil)
 )

modules/caddytls/capools_test.go

@@ -1,6 +1,7 @@
 package caddytls
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"reflect"
@@ -776,3 +777,219 @@ func TestHTTPCertPoolUnmarshalCaddyfile(t *testing.T) {
 		})
 	}
 }
+
+func TestSystemCAPoolUnmarshalCaddyfile(t *testing.T) {
+	type args struct {
+		d *caddyfile.Dispenser
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "basic system pool configuration",
+			args: args{
+				d: caddyfile.NewTestDispenser(`system`),
+			},
+			wantErr: false,
+		},
+		{
+			name: "system pool with arguments produces error",
+			args: args{
+				d: caddyfile.NewTestDispenser(`system foo`),
+			},
+			wantErr: true,
+		},
+		{
+			name: "system pool with block produces error",
+			args: args{
+				d: caddyfile.NewTestDispenser(`system {
+					foo bar
+				}`),
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			scp := &SystemCAPool{}
+			if err := scp.UnmarshalCaddyfile(tt.args.d); (err != nil) != tt.wantErr {
+				t.Errorf("SystemCAPool.UnmarshalCaddyfile() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestCombinedCAPoolUnmarshalCaddyfile(t *testing.T) {
+	type args struct {
+		d *caddyfile.Dispenser
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "empty block produces error",
+			args: args{
+				d: caddyfile.NewTestDispenser(`combined {
+				}`),
+			},
+			wantErr: true,
+		},
+		{
+			name: "arguments on same line as module name produces error",
+			args: args{
+				d: caddyfile.NewTestDispenser(`combined foo`),
+			},
+			wantErr: true,
+		},
+		{
+			name: "single source - system",
+			args: args{
+				d: caddyfile.NewTestDispenser(`combined {
+					source system
+				}`),
+			},
+			wantErr: false,
+		},
+		{
+			name: "single source - inline with config",
+			args: args{
+				d: caddyfile.NewTestDispenser(fmt.Sprintf(`combined {
+					source inline {
+						trust_der %s
+					}
+				}`, test_der_1)),
+			},
+			wantErr: false,
+		},
+		{
+			name: "multiple sources produces error due to limitation",
+			args: args{
+				d: caddyfile.NewTestDispenser(fmt.Sprintf(`combined {
+					source system
+					source inline {
+						trust_der %s
+					}
+				}`, test_der_1)),
+			},
+			wantErr: false, // UnmarshalCaddyfile succeeds, but Provision will fail
+		},
+		{
+			name: "source without module name produces error",
+			args: args{
+				d: caddyfile.NewTestDispenser(`combined {
+					source
+				}`),
+			},
+			wantErr: true,
+		},
+		{
+			name: "invalid directive produces error",
+			args: args{
+				d: caddyfile.NewTestDispenser(`combined {
+					invalid_directive foo
+				}`),
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ccp := &CombinedCAPool{}
+			if err := ccp.UnmarshalCaddyfile(tt.args.d); (err != nil) != tt.wantErr {
+				t.Errorf("CombinedCAPool.UnmarshalCaddyfile() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if !tt.wantErr && len(ccp.SourcesRaw) == 0 {
+				t.Errorf("CombinedCAPool.UnmarshalCaddyfile() produced no sources")
+			}
+		})
+	}
+}
+
+func TestSystemCAPoolProvision(t *testing.T) {
+	scp := &SystemCAPool{}
+	ctx := caddy.Context{Context: context.Background()}
+
+	err := scp.Provision(ctx)
+	if err != nil {
+		t.Errorf("SystemCAPool.Provision() error = %v", err)
+	}
+
+	if scp.pool == nil {
+		t.Error("SystemCAPool.Provision() did not create a cert pool")
+	}
+
+	pool := scp.CertPool()
+	if pool == nil {
+		t.Error("SystemCAPool.CertPool() returned nil")
+	}
+}
+
+func TestCombinedCAPoolProvisionWithSystemFails(t *testing.T) {
+	// Test that combining system pool fails during Provision
+	// because SystemCAPool doesn't implement CertificateProvider
+	ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
+	defer cancel()
+
+	// Create a combined pool with system source
+	ccp := &CombinedCAPool{
+		SourcesRaw: []json.RawMessage{
+			json.RawMessage(`{"provider":"system"}`),
+		},
+	}
+
+	err := ccp.Provision(ctx)
+	if err == nil {
+		t.Error("CombinedCAPool.Provision() with system source should fail, but succeeded")
+	}
+
+	// Verify error message mentions CertificateProvider
+	if err != nil && !contains(err.Error(), "CertificateProvider") {
+		t.Errorf("Expected error to mention CertificateProvider, got: %v", err)
+	}
+}
+
+func TestCombinedCAPoolProvisionWithInlineSucceeds(t *testing.T) {
+	// Test that combining inline pools works
+	ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
+	defer cancel()
+
+	// Create a combined pool with inline source
+	ccp := &CombinedCAPool{
+		SourcesRaw: []json.RawMessage{
+			json.RawMessage(fmt.Sprintf(`{"provider":"inline","trusted_ca_certs":["%s"]}`, test_der_1)),
+		},
+	}
+
+	err := ccp.Provision(ctx)
+	if err != nil {
+		t.Errorf("CombinedCAPool.Provision() with inline source failed: %v", err)
+	}
+
+	if ccp.pool == nil {
+		t.Error("CombinedCAPool.Provision() did not create a cert pool")
+	}
+
+	pool := ccp.CertPool()
+	if pool == nil {
+		t.Error("CombinedCAPool.CertPool() returned nil")
+	}
+}
+
+// Helper function for string contains check
+func contains(s, substr string) bool {
+	return len(s) >= len(substr) && (s == substr || len(substr) == 0 ||
+		(len(s) > 0 && len(substr) > 0 && findSubstring(s, substr)))
+}
+
+func findSubstring(s, substr string) bool {
+	for i := 0; i <= len(s)-len(substr); i++ {
+		if s[i:i+len(substr)] == substr {
+			return true
+		}
+	}
+	return false
+}