caddyhttp: Free up quic listener when stopping (#7177)

listeners.go

@@ -430,6 +430,7 @@ func JoinNetworkAddress(network, host, port string) string {
 // address instead.
 //
 // NOTE: This API is EXPERIMENTAL and may be changed or removed.
+// NOTE: user should close the returned listener twice, once to stop accepting new connections, the second time to free up the packet conn.
 func (na NetworkAddress) ListenQUIC(ctx context.Context, portOffset uint, config net.ListenConfig, tlsConf *tls.Config) (http3.QUICListener, error) {
 	lnKey := listenerKey("quic"+na.Network, na.JoinHostPort(portOffset))
 
@@ -626,6 +627,7 @@ func (fcql *fakeCloseQuicListener) Accept(_ context.Context) (*quic.Conn, error)
 func (fcql *fakeCloseQuicListener) Close() error {
 	if atomic.CompareAndSwapInt32(&fcql.closed, 0, 1) {
 		fcql.contextCancel()
+	} else if atomic.CompareAndSwapInt32(&fcql.closed, 1, 2) {
 		_, _ = listenerPool.Delete(fcql.sharedQuicListener.key)
 	}
 	return nil

modules/caddyhttp/app.go

@@ -722,11 +722,29 @@ func (app *App) Stop() error {
 			return
 		}
 
+		// closing quic listeners won't affect accepted connections now
+		// so like stdlib, close listeners first, but keep the net.PacketConns open
+		for _, h3ln := range server.quicListeners {
+			if err := h3ln.Close(); err != nil {
+				app.logger.Error("http3 listener close",
+					zap.Error(err))
+			}
+		}
+
 		if err := server.h3server.Shutdown(ctx); err != nil {
 			app.logger.Error("HTTP/3 server shutdown",
 				zap.Error(err),
 				zap.Strings("addresses", server.Listen))
 		}
+
+		// close the underlying net.PacketConns now
+		// see the comment for ListenQUIC
+		for _, h3ln := range server.quicListeners {
+			if err := h3ln.Close(); err != nil {
+				app.logger.Error("http3 listener close socket",
+					zap.Error(err))
+			}
+		}
 	}
 	stopH2Listener := func(server *Server) {
 		defer finishedShutdown.Done()

modules/caddyhttp/server.go

@@ -235,7 +235,8 @@ type Server struct {
 	primaryHandlerChain Handler
 	errorHandlerChain   Handler
 	listenerWrappers    []caddy.ListenerWrapper
-	listeners           []net.Listener
+	listeners           []net.Listener       // stdlib http.Server will close these
+	quicListeners       []http3.QUICListener // http3 now leave the quic.Listener management to us
 
 	tlsApp       *caddytls.TLS
 	events       *caddyevents.App
@@ -626,6 +627,8 @@ func (s *Server) serveHTTP3(addr caddy.NetworkAddress, tlsCfg *tls.Config) error
 		}
 	}
 
+	s.quicListeners = append(s.quicListeners, h3ln)
+
 	//nolint:errcheck
 	go s.h3server.ServeListener(h3ln)