From 3b2fd5d47153a2137a8aa841beec950f58218bca Mon Sep 17 00:00:00 2001 From: fanyang Date: Thu, 28 May 2026 23:31:26 +0800 Subject: [PATCH] fix(gateway): route local virtual IP proxy targets to loopback Avoid proxy loops when the target is this node's virtual IP. - Normalize local virtual IP destinations to loopback - Apply the behavior to TCP, KCP, and QUIC proxy paths - Add macOS utun repro tests and normalization coverage --- easytier/src/gateway/kcp_proxy.rs | 8 +- easytier/src/gateway/quic_proxy.rs | 6 +- easytier/src/gateway/tcp_proxy.rs | 143 +++++++++++++++++++++++++++-- 3 files changed, 141 insertions(+), 16 deletions(-) diff --git a/easytier/src/gateway/kcp_proxy.rs b/easytier/src/gateway/kcp_proxy.rs index b1dceacc..e0018e6b 100644 --- a/easytier/src/gateway/kcp_proxy.rs +++ b/easytier/src/gateway/kcp_proxy.rs @@ -19,7 +19,9 @@ use tokio::task::JoinSet; use super::{ CidrSet, - tcp_proxy::{NatDstConnector, NatDstTcpConnector, TcpProxy}, + tcp_proxy::{ + NatDstConnector, NatDstTcpConnector, TcpProxy, normalize_dst_for_local_virtual_ip, + }, }; use crate::utils::task::HedgeExt; use crate::{ @@ -369,9 +371,7 @@ impl KcpProxyDst { } let send_to_self = global_ctx.is_ip_local_virtual_ip(&dst_ip); - if send_to_self && global_ctx.no_tun() { - dst_socket = format!("127.0.0.1:{}", dst_socket.port()).parse().unwrap(); - } + dst_socket = normalize_dst_for_local_virtual_ip(&global_ctx, dst_socket); let acl_handler = ProxyAclHandler { acl_filter: global_ctx.get_acl_filter().clone(), diff --git a/easytier/src/gateway/quic_proxy.rs b/easytier/src/gateway/quic_proxy.rs index 7e0767aa..30e6e107 100644 --- a/easytier/src/gateway/quic_proxy.rs +++ b/easytier/src/gateway/quic_proxy.rs @@ -2,7 +2,7 @@ use crate::common::PeerId; use crate::common::acl_processor::PacketInfo; use crate::common::global_ctx::{ArcGlobalCtx, GlobalCtx}; use crate::gateway::CidrSet; -use crate::gateway::tcp_proxy::{NatDstConnector, TcpProxy}; +use crate::gateway::tcp_proxy::{NatDstConnector, TcpProxy, normalize_dst_for_local_virtual_ip}; use crate::gateway::wrapped_proxy::{ProxyAclHandler, TcpProxyForWrappedSrcTrait}; use crate::peers::PeerPacketFilter; use crate::peers::peer_manager::PeerManager; @@ -748,9 +748,7 @@ impl QuicStreamReceiver { } let send_to_self = global_ctx.is_ip_local_virtual_ip(&dst_ip); - if send_to_self && global_ctx.no_tun() { - dst_socket = format!("127.0.0.1:{}", dst_socket.port()).parse()?; - } + dst_socket = normalize_dst_for_local_virtual_ip(&global_ctx, dst_socket); let acl_handler = ProxyAclHandler { acl_filter: global_ctx.get_acl_filter().clone(), diff --git a/easytier/src/gateway/tcp_proxy.rs b/easytier/src/gateway/tcp_proxy.rs index 6e252268..2636ffac 100644 --- a/easytier/src/gateway/tcp_proxy.rs +++ b/easytier/src/gateway/tcp_proxy.rs @@ -9,7 +9,7 @@ use pnet::packet::ip::IpNextHeaderProtocols; use pnet::packet::ipv4::{Ipv4Packet, MutableIpv4Packet}; use pnet::packet::tcp::{MutableTcpPacket, TcpPacket, ipv4_checksum}; use socket2::{SockRef, TcpKeepalive}; -use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4}; use std::sync::atomic::{AtomicBool, AtomicU16}; use std::sync::{Arc, Weak}; use std::time::{Duration, Instant}; @@ -40,6 +40,20 @@ use super::CidrSet; #[cfg(feature = "smoltcp")] use super::tokio_smoltcp::{self, Net, NetConfig, channel_device}; +pub(crate) fn normalize_dst_for_local_virtual_ip( + global_ctx: &GlobalCtx, + dst: SocketAddr, +) -> SocketAddr { + if !global_ctx.is_ip_local_virtual_ip(&dst.ip()) { + return dst; + } + + match dst { + SocketAddr::V4(addr) => SocketAddr::new(Ipv4Addr::LOCALHOST.into(), addr.port()), + SocketAddr::V6(addr) => SocketAddr::new(Ipv6Addr::LOCALHOST.into(), addr.port()), + } +} + #[async_trait::async_trait] pub(crate) trait NatDstConnector: Send + Sync + Clone + 'static { type DstStream: AsyncRead + AsyncWrite + Unpin + Send; @@ -762,13 +776,7 @@ impl TcpProxy { return; } - let nat_dst = if global_ctx.is_ip_local_virtual_ip(&nat_entry.real_dst.ip()) { - format!("127.0.0.1:{}", nat_entry.real_dst.port()) - .parse() - .unwrap() - } else { - nat_entry.real_dst - }; + let nat_dst = normalize_dst_for_local_virtual_ip(&global_ctx, nat_entry.real_dst); global_ctx .stats_manager() @@ -1033,3 +1041,122 @@ impl TcpProxyRpcService { } } } + +#[cfg(test)] +mod tests { + use std::net::{Ipv4Addr, SocketAddr}; + + use super::normalize_dst_for_local_virtual_ip; + + #[tokio::test] + async fn normalize_dst_for_local_virtual_ip_maps_to_loopback() { + let global_ctx = crate::common::global_ctx::tests::get_mock_global_ctx(); + global_ctx.set_ipv4(Some("10.254.229.6/24".parse().unwrap())); + + let local_virtual = SocketAddr::from(([10, 254, 229, 6], 22)); + let normalized = normalize_dst_for_local_virtual_ip(&global_ctx, local_virtual); + assert_eq!(normalized, SocketAddr::from((Ipv4Addr::LOCALHOST, 22))); + + let remote_virtual = SocketAddr::from(([10, 254, 229, 7], 22)); + assert_eq!( + normalize_dst_for_local_virtual_ip(&global_ctx, remote_virtual), + remote_virtual + ); + } +} + +#[cfg(all(test, target_os = "macos", feature = "tun"))] +mod macos_utun_tests { + use std::{net::SocketAddr, sync::Arc, time::Duration}; + + use tokio::net::{TcpListener, TcpStream}; + + use super::{NatDstConnector as _, NatDstTcpConnector, normalize_dst_for_local_virtual_ip}; + use crate::{ + common::config::{ConfigLoader, TomlConfigLoader}, + instance::instance::Instance, + }; + + async fn run_instance_with_utun(ipv4: &str, enable_kcp: bool, enable_quic: bool) -> Instance { + let config = TomlConfigLoader::default(); + config.set_inst_name(format!("macos-utun-tcp-proxy-repro-{ipv4}")); + config.set_ipv4(Some(ipv4.parse().unwrap())); + config.set_ipv6(None); + config.set_listeners(Vec::new()); + + let mut flags = config.get_flags(); + flags.enable_kcp_proxy = enable_kcp; + flags.enable_quic_proxy = enable_quic; + flags.use_smoltcp = false; + config.set_flags(flags); + + let mut instance = Instance::new(config); + instance.run().await.expect( + "failed to create macOS utun device; run this ignored reproducer with root privileges", + ); + instance + } + + async fn assert_wildcard_listener_reachable_via_local_virtual_ip( + ipv4: &str, + enable_kcp: bool, + enable_quic: bool, + ) { + let mut instance = run_instance_with_utun(ipv4, enable_kcp, enable_quic).await; + let virtual_ip = instance.get_global_ctx().get_ipv4().unwrap().address(); + + let listener = Arc::new(TcpListener::bind("0.0.0.0:0").await.unwrap()); + let port = listener.local_addr().unwrap().port(); + + let baseline_listener = listener.clone(); + let baseline_accept = tokio::spawn(async move { baseline_listener.accept().await }); + let baseline_connect = TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, port)); + let (baseline_connect, baseline_accept) = tokio::join!(baseline_connect, baseline_accept); + baseline_connect.unwrap(); + baseline_accept.unwrap().unwrap(); + + let test_listener = listener.clone(); + let mut accept = tokio::spawn(async move { test_listener.accept().await }); + let dst = SocketAddr::new(virtual_ip.into(), port); + let dst = normalize_dst_for_local_virtual_ip(&instance.get_global_ctx(), dst); + assert_eq!(dst.ip(), std::net::Ipv4Addr::LOCALHOST); + let connect = NatDstTcpConnector {}.connect("0.0.0.0:0".parse().unwrap(), dst); + + let connect = tokio::time::timeout(Duration::from_secs(3), connect).await; + let accept_result = tokio::time::timeout(Duration::from_secs(1), &mut accept).await; + + if accept_result.is_err() { + accept.abort(); + } + instance.clear_resources().await; + + match connect { + Ok(Ok(_)) => {} + Ok(Err(error)) => panic!("connect to local EasyTier virtual IP failed: {error:?}"), + Err(_) => panic!("connect to local EasyTier virtual IP timed out"), + } + + match accept_result { + Ok(Ok(Ok(_))) => {} + Ok(Ok(Err(error))) => panic!("listener accept failed: {error:?}"), + Ok(Err(error)) => panic!("listener task failed: {error:?}"), + Err(_) => panic!( + "listener did not accept connection to local EasyTier virtual IP {virtual_ip}:{port}" + ), + } + } + + #[tokio::test] + #[ignore = "requires root and a real macOS utun device; covers issue #2296"] + async fn macos_utun_kcp_proxy_dst_local_virtual_ip_reaches_wildcard_listener() { + assert_wildcard_listener_reachable_via_local_virtual_ip("10.254.229.6/24", true, false) + .await; + } + + #[tokio::test] + #[ignore = "requires root and a real macOS utun device; covers issue #2296"] + async fn macos_utun_quic_proxy_dst_local_virtual_ip_reaches_wildcard_listener() { + assert_wildcard_listener_reachable_via_local_virtual_ip("10.254.230.6/24", false, true) + .await; + } +}