diff --git a/context.go b/context.go
index c665ab388..4b9e8bcbb 100644
--- a/context.go
+++ b/context.go
@@ -21,7 +21,6 @@ import (
 	"log"
 	"log/slog"
 	"reflect"
-	"sync"
 
 	"github.com/caddyserver/certmagic"
 	"github.com/prometheus/client_golang/prometheus"
@@ -63,7 +62,7 @@ type Context struct {
 // See standard library context package's documentation.
 func NewContext(ctx Context) (Context, context.CancelFunc) {
 	r := prometheus.NewPedanticRegistry()
-	newCtx := Context{moduleInstances: make(map[string][]Module), cfg: ctx.cfg, metricsRegistry: &registryGatherer{registry: r, gatherer: r, tracker: make(map[string]*sync.Once)}}
+	newCtx := Context{moduleInstances: make(map[string][]Module), cfg: ctx.cfg, metricsRegistry: &registryGatherer{registry: r, gatherer: r}}
 	c, cancel := context.WithCancel(ctx.Context)
 	wrappedCancel := func() {
 		cancel()
@@ -105,10 +104,7 @@ func (ctx *Context) FileSystems() FileSystems {
 
 // Returns the active metrics registry for the context
 // EXPERIMENTAL: This API is subject to change.
-func (ctx *Context) GetMetricsRegistry() RegistererGatherer {
-	if ctx.Module() != nil {
-		ctx.metricsRegistry.callerModule = ctx.Module().CaddyModule().String()
-	}
+func (ctx *Context) GetMetricsRegistry() MetricsRegistererGatherer {
 	return ctx.metricsRegistry
 }
 
diff --git a/metrics.go b/metrics.go
index 464cce05f..41154067c 100644
--- a/metrics.go
+++ b/metrics.go
@@ -1,8 +1,8 @@
 package caddy
 
 import (
+	"errors"
 	"net/http"
-	"sync"
 
 	"github.com/prometheus/client_golang/prometheus"
 	io_prometheus_client "github.com/prometheus/client_model/go"
@@ -85,16 +85,13 @@ func (d *delegator) Unwrap() http.ResponseWriter {
 	return d.ResponseWriter
 }
 
-type RegistererGatherer interface {
+type MetricsRegistererGatherer interface {
 	prometheus.Registerer
 	prometheus.Gatherer
 }
 type registryGatherer struct {
 	registry prometheus.Registerer
 	gatherer prometheus.Gatherer
-	tracker  map[string]*sync.Once
-
-	callerModule string
 }
 
 // Gather implements prometheus.Gatherer.
@@ -102,31 +99,45 @@ func (r *registryGatherer) Gather() ([]*io_prometheus_client.MetricFamily, error
 	return r.gatherer.Gather()
 }
 
-// MustRegister implements prometheus.Registerer.
+// MustRegister calls `MustRegister` on the backing registry one collector
+// at a time to capture the module at which the call may have panicked. Panics
+// of duplicate registration are ignored.
 func (r *registryGatherer) MustRegister(cs ...prometheus.Collector) {
-	if _, ok := r.tracker[r.callerModule]; !ok {
-		r.tracker[r.callerModule] = &sync.Once{}
+	var current prometheus.Collector
+	defer func() {
+		if r := recover(); r != nil {
+			err, ok := r.(error)
+			if !ok {
+				panic(r)
+			}
+			if !errors.Is(err, prometheus.AlreadyRegisteredError{
+				ExistingCollector: current,
+				NewCollector:      current,
+			}) {
+				panic(err)
+			}
+		}
+	}()
+	for _, current = range cs {
+		r.registry.MustRegister(current)
 	}
-	r.tracker[r.callerModule].Do(func() {
-		r.registry.MustRegister(cs...)
-	})
 }
 
-// Register implements prometheus.Registerer.
+// Register implements prometheus.Registerer. Errors of duplicate registration
+// are ignored.
 func (r *registryGatherer) Register(c prometheus.Collector) error {
-	var err error
-	if _, ok := r.tracker[r.callerModule]; !ok {
-		r.tracker[r.callerModule] = &sync.Once{}
+	if err := r.registry.Register(c); err != nil &&
+		!errors.Is(err, prometheus.AlreadyRegisteredError{
+			ExistingCollector: c,
+			NewCollector:      c,
+		}) {
+		return err
 	}
-	r.tracker[r.callerModule].Do(func() {
-		err = r.registry.Register(c)
-	})
-	return err
+	return nil
 }
 
 // Unregister implements prometheus.Registerer.
 func (r *registryGatherer) Unregister(c prometheus.Collector) bool {
-	delete(r.tracker, r.callerModule)
 	return r.registry.Unregister(c)
 }
 
diff --git a/modules/metrics/adminmetrics.go b/modules/metrics/adminmetrics.go
index aae301ca0..6f990187c 100644
--- a/modules/metrics/adminmetrics.go
+++ b/modules/metrics/adminmetrics.go
@@ -31,7 +31,7 @@ func init() {
 // See the Metrics module for a configurable endpoint that is usable if the
 // Admin API is disabled.
 type AdminMetrics struct {
-	registry caddy.RegistererGatherer
+	registry caddy.MetricsRegistererGatherer
 
 	metricsHandler http.Handler
 }
diff --git a/modules/metrics/metrics.go b/modules/metrics/metrics.go
index 69d10a359..184f57f90 100644
--- a/modules/metrics/metrics.go
+++ b/modules/metrics/metrics.go
@@ -107,7 +107,7 @@ var (
 	_ caddyfile.Unmarshaler       = (*Metrics)(nil)
 )
 
-func createMetricsHandler(logger promhttp.Logger, enableOpenMetrics bool, registry caddy.RegistererGatherer) http.Handler {
+func createMetricsHandler(logger promhttp.Logger, enableOpenMetrics bool, registry caddy.MetricsRegistererGatherer) http.Handler {
 	return promhttp.InstrumentMetricHandler(registry,
 		promhttp.HandlerFor(registry, promhttp.HandlerOpts{
 			// will only log errors if logger is non-nil