feat: support allocating public IPv6 addresses from a provider (#2162)

* feat: support allocating public IPv6 addresses from a provider

Add a provider/leaser architecture for public IPv6 address allocation
between nodes in the same network:

- A node with `--ipv6-public-addr-provider` advertises a delegable
  public IPv6 prefix (auto-detected from kernel routes or manually
  configured via `--ipv6-public-addr-prefix`).
- Other nodes with `--ipv6-public-addr-auto` request a /128 lease from
  the selected provider via a new RPC service (PublicIpv6AddrRpc).
- Leases have a 30s TTL, renewed every 10s by the client routine.
- The provider allocates addresses deterministically from its prefix
  using instance-UUID-based hashing to prefer stable assignments.
- Routes to peer leases are installed on the TUN device, and each
  client's own /128 is assigned as its IPv6 address.

Also includes netlink IPv6 route table inspection, integration tests,
and event-driven route/address reconciliation.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
KKRainbow
2026-04-26 21:37:34 +08:00
committed by GitHub
parent b20075e3dc
commit 8f862997eb
30 changed files with 3973 additions and 69 deletions
+69
View File
@@ -170,6 +170,15 @@ pub trait ConfigLoader: Send + Sync {
fn get_ipv6(&self) -> Option<cidr::Ipv6Inet>;
fn set_ipv6(&self, addr: Option<cidr::Ipv6Inet>);
fn get_ipv6_public_addr_provider(&self) -> bool;
fn set_ipv6_public_addr_provider(&self, enabled: bool);
fn get_ipv6_public_addr_auto(&self) -> bool;
fn set_ipv6_public_addr_auto(&self, enabled: bool);
fn get_ipv6_public_addr_prefix(&self) -> Option<cidr::Ipv6Cidr>;
fn set_ipv6_public_addr_prefix(&self, prefix: Option<cidr::Ipv6Cidr>);
fn get_dhcp(&self) -> bool;
fn set_dhcp(&self, dhcp: bool);
@@ -519,6 +528,9 @@ struct Config {
instance_id: Option<uuid::Uuid>,
ipv4: Option<String>,
ipv6: Option<String>,
ipv6_public_addr_provider: Option<bool>,
ipv6_public_addr_auto: Option<bool>,
ipv6_public_addr_prefix: Option<String>,
dhcp: Option<bool>,
network_identity: Option<NetworkIdentity>,
listeners: Option<Vec<url::Url>>,
@@ -700,6 +712,43 @@ impl ConfigLoader for TomlConfigLoader {
self.config.lock().unwrap().ipv6 = addr.map(|addr| addr.to_string());
}
fn get_ipv6_public_addr_provider(&self) -> bool {
self.config
.lock()
.unwrap()
.ipv6_public_addr_provider
.unwrap_or_default()
}
fn set_ipv6_public_addr_provider(&self, enabled: bool) {
self.config.lock().unwrap().ipv6_public_addr_provider = Some(enabled);
}
fn get_ipv6_public_addr_auto(&self) -> bool {
self.config
.lock()
.unwrap()
.ipv6_public_addr_auto
.unwrap_or_default()
}
fn set_ipv6_public_addr_auto(&self, enabled: bool) {
self.config.lock().unwrap().ipv6_public_addr_auto = Some(enabled);
}
fn get_ipv6_public_addr_prefix(&self) -> Option<cidr::Ipv6Cidr> {
let locked_config = self.config.lock().unwrap();
locked_config
.ipv6_public_addr_prefix
.as_ref()
.and_then(|s| s.parse().ok())
}
fn set_ipv6_public_addr_prefix(&self, prefix: Option<cidr::Ipv6Cidr>) {
self.config.lock().unwrap().ipv6_public_addr_prefix =
prefix.map(|prefix| prefix.to_string());
}
fn get_dhcp(&self) -> bool {
self.config.lock().unwrap().dhcp.unwrap_or_default()
}
@@ -1312,6 +1361,26 @@ source = "user"
assert!(!explicit_user.dump().contains("[source]"));
}
#[test]
fn test_ipv6_public_addr_config_roundtrip() {
let config = TomlConfigLoader::default();
let prefix: cidr::Ipv6Cidr = "2001:db8:100::/64".parse().unwrap();
config.set_ipv6_public_addr_provider(true);
config.set_ipv6_public_addr_auto(true);
config.set_ipv6_public_addr_prefix(Some(prefix));
assert!(config.get_ipv6_public_addr_provider());
assert!(config.get_ipv6_public_addr_auto());
assert_eq!(config.get_ipv6_public_addr_prefix(), Some(prefix));
let dumped = config.dump();
let loaded = TomlConfigLoader::new_from_str(&dumped).unwrap();
assert!(loaded.get_ipv6_public_addr_provider());
assert!(loaded.get_ipv6_public_addr_auto());
assert_eq!(loaded.get_ipv6_public_addr_prefix(), Some(prefix));
}
#[tokio::test]
async fn full_example_test() {
let config_str = r#"
+77 -6
View File
@@ -68,6 +68,8 @@ pub enum GlobalCtxEvent {
DhcpIpv4Changed(Option<cidr::Ipv4Inet>, Option<cidr::Ipv4Inet>), // (old, new)
DhcpIpv4Conflicted(Option<cidr::Ipv4Inet>),
PublicIpv6Changed(Option<cidr::Ipv6Inet>, Option<cidr::Ipv6Inet>), // (old, new)
PublicIpv6RoutesUpdated(Vec<cidr::Ipv6Inet>, Vec<cidr::Ipv6Inet>), // (added, removed)
PortForwardAdded(PortForwardConfigPb),
@@ -200,6 +202,7 @@ pub struct GlobalCtx {
cached_ipv4: AtomicCell<Option<cidr::Ipv4Inet>>,
cached_ipv6: AtomicCell<Option<cidr::Ipv6Inet>>,
public_ipv6_lease: AtomicCell<Option<cidr::Ipv6Inet>>,
cached_proxy_cidrs: AtomicCell<Option<Vec<ProxyNetworkConfig>>>,
ip_collector: Mutex<Option<Arc<IPCollector>>>,
@@ -209,6 +212,7 @@ pub struct GlobalCtx {
stun_info_collection: Mutex<Arc<dyn StunInfoCollectorTrait>>,
running_listeners: Mutex<Vec<url::Url>>,
advertised_ipv6_public_addr_prefix: Mutex<Option<cidr::Ipv6Cidr>>,
flags: ArcSwap<Flags>,
@@ -295,6 +299,7 @@ impl GlobalCtx {
event_bus,
cached_ipv4: AtomicCell::new(None),
cached_ipv6: AtomicCell::new(None),
public_ipv6_lease: AtomicCell::new(None),
cached_proxy_cidrs: AtomicCell::new(None),
ip_collector: Mutex::new(Some(Arc::new(IPCollector::new(
@@ -307,6 +312,7 @@ impl GlobalCtx {
stun_info_collection: Mutex::new(stun_info_collector),
running_listeners: Mutex::new(Vec::new()),
advertised_ipv6_public_addr_prefix: Mutex::new(None),
flags: ArcSwap::new(Arc::new(flags)),
@@ -381,6 +387,36 @@ impl GlobalCtx {
self.cached_ipv6.store(None);
}
pub fn get_public_ipv6_lease(&self) -> Option<cidr::Ipv6Inet> {
self.public_ipv6_lease.load()
}
pub fn set_public_ipv6_lease(&self, addr: Option<cidr::Ipv6Inet>) {
self.public_ipv6_lease.store(addr);
}
pub fn is_ip_local_ipv6(&self, ip: &std::net::Ipv6Addr) -> bool {
self.get_ipv6().map(|x| x.address() == *ip).unwrap_or(false)
|| self
.get_public_ipv6_lease()
.map(|x| x.address() == *ip)
.unwrap_or(false)
}
pub fn get_advertised_ipv6_public_addr_prefix(&self) -> Option<cidr::Ipv6Cidr> {
*self.advertised_ipv6_public_addr_prefix.lock().unwrap()
}
pub fn set_advertised_ipv6_public_addr_prefix(&self, prefix: Option<cidr::Ipv6Cidr>) -> bool {
let mut guard = self.advertised_ipv6_public_addr_prefix.lock().unwrap();
if *guard == prefix {
return false;
}
*guard = prefix;
true
}
pub fn get_id(&self) -> uuid::Uuid {
self.config.get_id()
}
@@ -395,7 +431,7 @@ impl GlobalCtx {
pub fn is_ip_local_virtual_ip(&self, ip: &IpAddr) -> bool {
match ip {
IpAddr::V4(v4) => self.get_ipv4().map(|x| x.address() == *v4).unwrap_or(false),
IpAddr::V6(v6) => self.get_ipv6().map(|x| x.address() == *v6).unwrap_or(false),
IpAddr::V6(v6) => self.is_ip_local_ipv6(v6),
}
}
@@ -645,23 +681,23 @@ impl GlobalCtx {
pub fn should_deny_proxy(&self, dst_addr: &SocketAddr, is_udp: bool) -> bool {
let _g = self.net_ns.guard();
let ip = dst_addr.ip();
// first check if ip is virtual ip
// first check if ip is an EasyTier-managed local address
// then try bind this ip, if succ means it is local ip
let dst_is_local_virtual_ip = self.is_ip_local_virtual_ip(&ip);
let dst_is_local_et_ip = self.is_ip_local_virtual_ip(&ip);
// this is an expensive operation, should be called sparingly
// 1. tcp/kcp/quic call this only after proxy conn is established
// 2. udp cache the result in nat entry
let dst_is_local_phy_ip = std::net::UdpSocket::bind(format!("{}:0", ip)).is_ok();
tracing::trace!(
"check should_deny_proxy: dst_addr={}, dst_is_local_virtual_ip={}, dst_is_local_phy_ip={}, is_udp={}",
"check should_deny_proxy: dst_addr={}, dst_is_local_et_ip={}, dst_is_local_phy_ip={}, is_udp={}",
dst_addr,
dst_is_local_virtual_ip,
dst_is_local_et_ip,
dst_is_local_phy_ip,
is_udp
);
if dst_is_local_virtual_ip || dst_is_local_phy_ip {
if dst_is_local_et_ip || dst_is_local_phy_ip {
// if is local ip, make sure the port is not one of the listening ports
self.is_port_in_running_listeners(dst_addr.port(), is_udp)
|| (!is_udp && protected_port::is_protected_tcp_port(dst_addr.port()))
@@ -770,6 +806,7 @@ pub mod tests {
assert!(feature_flags.support_conn_list_sync);
assert!(feature_flags.avoid_relay_data);
assert!(feature_flags.is_public_server);
assert!(!feature_flags.ipv6_public_addr_provider);
}
#[tokio::test]
@@ -789,6 +826,40 @@ pub mod tests {
protected_port::clear_protected_tcp_ports_for_test();
}
#[tokio::test]
async fn virtual_ipv6_and_public_ipv6_lease_are_stored_separately() {
let config = TomlConfigLoader::default();
let global_ctx = GlobalCtx::new(config);
let virtual_ipv6 = "fd00::1/64".parse().unwrap();
let public_ipv6 = "2001:db8::2/64".parse().unwrap();
global_ctx.set_ipv6(Some(virtual_ipv6));
global_ctx.set_public_ipv6_lease(Some(public_ipv6));
assert_eq!(global_ctx.get_ipv6(), Some(virtual_ipv6));
assert_eq!(global_ctx.get_public_ipv6_lease(), Some(public_ipv6));
}
#[tokio::test]
async fn public_ipv6_lease_is_treated_as_local_ip() {
protected_port::clear_protected_tcp_ports_for_test();
let config = TomlConfigLoader::default();
let global_ctx = GlobalCtx::new(config);
let public_ipv6 = "2001:db8::2/64".parse().unwrap();
let listener: url::Url = "tcp://[2001:db8::2]:11010".parse().unwrap();
global_ctx.set_public_ipv6_lease(Some(public_ipv6));
global_ctx.add_running_listener(listener);
let ip = std::net::IpAddr::V6(public_ipv6.address());
let socket = SocketAddr::from((public_ipv6.address(), 11010));
assert!(global_ctx.is_ip_local_virtual_ip(&ip));
assert!(global_ctx.should_deny_proxy(&socket, false));
protected_port::clear_protected_tcp_ports_for_test();
}
pub fn get_mock_global_ctx_with_network(
network_identy: Option<NetworkIdentity>,
) -> ArcGlobalCtx {
+11
View File
@@ -166,3 +166,14 @@ pub type IfConfiger = DummyIfConfiger;
#[cfg(target_os = "windows")]
pub use windows::RegistryManager;
#[cfg(target_os = "linux")]
pub(crate) fn list_ipv6_route_messages()
-> Result<Vec<netlink_packet_route::route::RouteMessage>, Error> {
netlink::NetlinkIfConfiger::list_ipv6_route_messages()
}
#[cfg(target_os = "linux")]
pub(crate) fn get_interface_index(name: &str) -> Result<u32, Error> {
netlink::NetlinkIfConfiger::get_interface_index(name)
}
+200 -16
View File
@@ -160,7 +160,7 @@ impl From<RouteMessage> for Route {
pub struct NetlinkIfConfiger {}
impl NetlinkIfConfiger {
fn get_interface_index(name: &str) -> Result<u32, Error> {
pub(crate) fn get_interface_index(name: &str) -> Result<u32, Error> {
let name = CString::new(name).with_context(|| "failed to convert interface name")?;
match unsafe { libc::if_nametoindex(name.as_ptr()) } {
0 => Err(std::io::Error::last_os_error().into()),
@@ -311,7 +311,7 @@ impl NetlinkIfConfiger {
Self::set_flags_op(name, SIOCGIFFLAGS, InterfaceFlags::empty())
}
fn list_routes() -> Result<Vec<RouteMessage>, Error> {
fn list_route_messages(address_family: AddressFamily) -> Result<Vec<RouteMessage>, Error> {
let mut message = RouteMessage::default();
message.header.table = RouteHeader::RT_TABLE_UNSPEC;
@@ -320,7 +320,7 @@ impl NetlinkIfConfiger {
message.header.scope = RouteScope::Universe;
message.header.kind = RouteType::Unicast;
message.header.address_family = AddressFamily::Inet;
message.header.address_family = address_family;
message.header.destination_prefix_length = 0;
message.header.source_prefix_length = 0;
@@ -367,6 +367,14 @@ impl NetlinkIfConfiger {
Ok(ret_vec)
}
fn list_routes() -> Result<Vec<RouteMessage>, Error> {
Self::list_route_messages(AddressFamily::Inet)
}
pub(crate) fn list_ipv6_route_messages() -> Result<Vec<RouteMessage>, Error> {
Self::list_route_messages(AddressFamily::Inet6)
}
}
#[async_trait]
@@ -551,12 +559,9 @@ impl IfConfiguerTrait for NetlinkIfConfiger {
message.header.scope = RouteScope::Universe;
message.header.kind = RouteType::Unicast;
// Add metric (cost) if specified
if let Some(cost) = cost {
message
.attributes
.push(RouteAttribute::Priority(cost as u32));
}
message
.attributes
.push(RouteAttribute::Priority(cost.unwrap_or(65535) as u32));
message
.attributes
@@ -564,9 +569,11 @@ impl IfConfiguerTrait for NetlinkIfConfiger {
name,
)?));
message
.attributes
.push(RouteAttribute::Destination(RouteAddress::Inet6(address)));
if cidr_prefix != 0 {
message
.attributes
.push(RouteAttribute::Destination(RouteAddress::Inet6(address)));
}
send_netlink_req_and_wait_one_resp(RouteNetlinkMessage::NewRoute(message), false)
}
@@ -577,7 +584,7 @@ impl IfConfiguerTrait for NetlinkIfConfiger {
address: std::net::Ipv6Addr,
cidr_prefix: u8,
) -> Result<(), Error> {
let routes = Self::list_routes()?;
let routes = Self::list_route_messages(AddressFamily::Inet6)?;
let ifidx = NetlinkIfConfiger::get_interface_index(name)?;
for msg in routes {
@@ -598,29 +605,82 @@ impl IfConfiguerTrait for NetlinkIfConfiger {
#[cfg(test)]
mod tests {
use super::*;
use std::process::Command;
const DUMMY_IFACE_NAME: &str = "dummy";
fn run_cmd(cmd: &str) -> String {
let output = std::process::Command::new("sh")
let output = Command::new("sh")
.arg("-c")
.arg(cmd)
.output()
.expect("failed to execute process");
assert!(
output.status.success(),
"command failed: {cmd}\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
);
String::from_utf8(output.stdout).unwrap()
}
fn run_ip(args: &[&str]) {
let output = Command::new("ip")
.args(args)
.output()
.expect("failed to execute ip process");
assert!(
output.status.success(),
"ip command failed: {:?}\nstdout: {}\nstderr: {}",
args,
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
);
}
fn test_iface_name(tag: &str) -> String {
format!("et{}{:x}", tag, std::process::id() & 0xffff)
}
struct ScopedDummyLink {
name: String,
}
impl ScopedDummyLink {
fn new(name: &str) -> Self {
let _ = Command::new("ip").args(["link", "del", name]).output();
run_ip(&["link", "add", name, "type", "dummy"]);
run_ip(&["link", "set", name, "up"]);
Self {
name: name.to_string(),
}
}
}
impl Drop for ScopedDummyLink {
fn drop(&mut self) {
let _ = Command::new("ip")
.args(["link", "del", &self.name])
.output();
}
}
struct PrepareEnv {}
impl PrepareEnv {
fn new() -> Self {
let _ = run_cmd(&format!("sudo ip link add {} type dummy", DUMMY_IFACE_NAME));
let _ = Command::new("ip")
.args(["link", "del", DUMMY_IFACE_NAME])
.output();
let _ = run_cmd(&format!("ip link add {} type dummy", DUMMY_IFACE_NAME));
PrepareEnv {}
}
}
impl Drop for PrepareEnv {
fn drop(&mut self) {
let _ = run_cmd(&format!("sudo ip link del {}", DUMMY_IFACE_NAME));
let _ = Command::new("ip")
.args(["link", "del", DUMMY_IFACE_NAME])
.output();
}
}
@@ -701,4 +761,128 @@ mod tests {
.collect::<Vec<_>>();
assert!(!routes.contains(&IpAddr::V4("10.5.5.0".parse().unwrap())));
}
#[serial_test::serial]
#[tokio::test]
async fn ipv6_addr_readback_test() {
let iface = test_iface_name("a");
let _link = ScopedDummyLink::new(&iface);
run_ip(&["-6", "addr", "add", "2001:db8:1234::2/64", "dev", &iface]);
let addrs = NetlinkIfConfiger::list_addresses(&iface).unwrap();
assert!(addrs.iter().any(|addr| {
addr.address() == IpAddr::V6("2001:db8:1234::2".parse().unwrap())
&& addr.network_length() == 64
}));
}
#[serial_test::serial]
#[tokio::test]
async fn ipv6_route_readback_test() {
let wan_if = test_iface_name("rw");
let lan_if = test_iface_name("rl");
let _wan = ScopedDummyLink::new(&wan_if);
let _lan = ScopedDummyLink::new(&lan_if);
run_ip(&[
"-6",
"addr",
"add",
"2001:db8:100:ffff::2/64",
"dev",
&wan_if,
]);
run_ip(&[
"-6",
"route",
"add",
"default",
"from",
"2001:db8:100::/56",
"dev",
&wan_if,
]);
run_ip(&["-6", "route", "add", "2001:db8:100::/56", "dev", &lan_if]);
let wan_ifindex = NetlinkIfConfiger::get_interface_index(&wan_if).unwrap();
let lan_ifindex = NetlinkIfConfiger::get_interface_index(&lan_if).unwrap();
let routes = NetlinkIfConfiger::list_ipv6_route_messages().unwrap();
assert!(routes.iter().any(|route| {
route.header.kind == RouteType::Unicast
&& route.header.source_prefix_length == 56
&& route.attributes.iter().any(|attr| {
matches!(
attr,
RouteAttribute::Source(RouteAddress::Inet6(addr))
if *addr == "2001:db8:100::".parse::<std::net::Ipv6Addr>().unwrap()
)
})
&& route
.attributes
.iter()
.any(|attr| matches!(attr, RouteAttribute::Oif(index) if *index == wan_ifindex))
&& !route
.attributes
.iter()
.any(|attr| matches!(attr, RouteAttribute::Destination(_)))
}));
assert!(routes.iter().any(|route| {
route.header.kind == RouteType::Unicast
&& route.header.destination_prefix_length == 56
&& route.attributes.iter().any(|attr| {
matches!(
attr,
RouteAttribute::Destination(RouteAddress::Inet6(addr))
if *addr == "2001:db8:100::".parse::<std::net::Ipv6Addr>().unwrap()
)
})
&& route
.attributes
.iter()
.any(|attr| matches!(attr, RouteAttribute::Oif(index) if *index == lan_ifindex))
}));
}
#[serial_test::serial]
#[tokio::test]
async fn ipv6_route_remove_test() {
let iface = test_iface_name("rr");
let _link = ScopedDummyLink::new(&iface);
let ifcfg = NetlinkIfConfiger {};
let route_addr = "2001:db8:200::".parse::<std::net::Ipv6Addr>().unwrap();
ifcfg
.add_ipv6_route(&iface, route_addr, 56, None)
.await
.unwrap();
let ifindex = NetlinkIfConfiger::get_interface_index(&iface).unwrap();
let has_route = |routes: &[RouteMessage]| {
routes.iter().any(|route| {
route.header.destination_prefix_length == 56
&& route.attributes.iter().any(|attr| {
matches!(
attr,
RouteAttribute::Destination(RouteAddress::Inet6(addr)) if *addr == route_addr
)
})
&& route
.attributes
.iter()
.any(|attr| matches!(attr, RouteAttribute::Oif(index) if *index == ifindex))
})
};
let routes = NetlinkIfConfiger::list_ipv6_route_messages().unwrap();
assert!(has_route(&routes));
ifcfg
.remove_ipv6_route(&iface, route_addr, 56)
.await
.unwrap();
let routes = NetlinkIfConfiger::list_ipv6_route_messages().unwrap();
assert!(!has_route(&routes));
}
}