diff --git a/Cargo.lock b/Cargo.lock index ce04bd43..056d071c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2353,7 +2353,6 @@ dependencies = [ "rand 0.8.5", "rcgen", "regex", - "ring", "rstest", "rust-i18n", "rustls", @@ -2440,6 +2439,7 @@ dependencies = [ "hyper", "hyper-util", "idna 1.0.3", + "openssl", "ordered_hash_map", "parking_lot", "percent-encoding", @@ -2451,6 +2451,7 @@ dependencies = [ "prost-types 0.14.3", "quanta", "rand 0.8.5", + "ring", "rustls", "serde", "serde_json", @@ -5995,6 +5996,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "300.6.1+3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.103" @@ -6003,6 +6013,7 @@ checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] diff --git a/easytier-core/Cargo.toml b/easytier-core/Cargo.toml index 6989d924..96d6aca4 100644 --- a/easytier-core/Cargo.toml +++ b/easytier-core/Cargo.toml @@ -50,6 +50,7 @@ prost = "0.14.3" prost-types = "0.14.3" rand = "0.8.5" quanta = "0.12" +ring = { version = "0.17", optional = true } rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"], optional = true } serde = { version = "1.0", features = ["derive"] } serde_json = "1" @@ -79,11 +80,14 @@ zerocopy = { version = "0.7.32", features = ["derive", "simd"] } zstd = { version = "0.13", optional = true } aes-gcm = { version = "0.10.3", optional = true } chacha20poly1305 = { version = "0.10.1", optional = true } +openssl = { version = "0.10", optional = true, features = ["vendored"] } [features] default = ["aes-gcm", "endpoint-discovery", "extended-services", "management", "tcp-hole-punch"] aes-gcm = ["dep:aes-gcm"] chacha20 = ["dep:chacha20poly1305"] +openssl-crypto = ["dep:openssl"] +ring-crypto = ["dep:ring"] config-write = [] endpoint-discovery = [ "dep:http-body-util", diff --git a/easytier-core/src/gateway/dataplane/flow.rs b/easytier-core/src/gateway/dataplane/flow.rs index b1fd1c32..d51e5de3 100644 --- a/easytier-core/src/gateway/dataplane/flow.rs +++ b/easytier-core/src/gateway/dataplane/flow.rs @@ -117,10 +117,21 @@ impl FlowTable { self.count.load(Ordering::Relaxed) } + /// Returns whether no flow is visible or being published. + /// + /// New entries reserve their count before they become visible, while + /// removals release their count after the entry is gone. Consequently a + /// zero observed here is a safe fast-path signal without inspecting every + /// DashMap shard. + pub fn is_idle(&self) -> bool { + self.count.load(Ordering::Acquire) == 0 + } + pub fn len(&self) -> usize { self.entries.len() } + #[cfg(test)] pub fn is_empty(&self) -> bool { self.entries.is_empty() } @@ -241,13 +252,13 @@ impl FlowTable { fn increment_count(&self) -> FlowCountChange { let previous = self .count - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |count| { + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { count.checked_add(1) }) - .unwrap_or_else(|count| count); + .expect("flow count overflow"); FlowCountChange { previous, - current: previous.saturating_add(1), + current: previous + 1, } } @@ -283,13 +294,13 @@ impl FlowTable { let previous = self .count - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |count| { - Some(count.saturating_sub(delta)) + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { + count.checked_sub(delta) }) - .unwrap_or_else(|count| count); + .expect("flow count underflow"); FlowCountChange { previous, - current: previous.saturating_sub(delta), + current: previous - delta, } } } @@ -334,6 +345,7 @@ mod tests { assert!(!inserted.replaced); assert_eq!(inserted.count.previous, 0); assert_eq!(inserted.count.current, 1); + assert!(!table.is_idle()); assert_eq!(table.with_entry(&entry, |value| *value), Some("first")); let replaced = table.insert(entry.clone(), "second"); @@ -346,6 +358,7 @@ mod tests { assert!(removed.removed); assert_eq!(removed.count.previous, 1); assert_eq!(removed.count.current, 0); + assert!(table.is_idle()); let missing = table.remove(&entry); assert!(!missing.removed); diff --git a/easytier-core/src/gateway/dataplane/mod.rs b/easytier-core/src/gateway/dataplane/mod.rs index 2d882edf..661f0430 100644 --- a/easytier-core/src/gateway/dataplane/mod.rs +++ b/easytier-core/src/gateway/dataplane/mod.rs @@ -208,8 +208,7 @@ where H: VirtualTcpSocketFactory + VirtualTcpListenerFactory + VirtualUdpSocketFactory, { async fn try_process_packet_from_peer(&self, packet: ZCPacket) -> Option { - let entry_count = self.entries.count(); - if entry_count == 0 && self.entries.is_empty() { + if self.entries.is_idle() { if tracing::enabled!(tracing::Level::TRACE) && let Some(hdr) = packet.peer_manager_header() && matches!( @@ -244,7 +243,7 @@ where ?tcp_src_port, ?tcp_dst_port, ?tcp_flags, - entry_count, + entry_count = 0, "data plane fast gate passed packet from peer" ); } else { @@ -252,7 +251,7 @@ where packet_type = hdr.packet_type, from_peer_id = hdr.from_peer_id.get(), to_peer_id = hdr.to_peer_id.get(), - entry_count, + entry_count = 0, "data plane fast gate passed non-ipv4 packet from peer" ); } diff --git a/easytier-core/src/gateway/dataplane/tests.rs b/easytier-core/src/gateway/dataplane/tests.rs index b706d55e..d56908b0 100644 --- a/easytier-core/src/gateway/dataplane/tests.rs +++ b/easytier-core/src/gateway/dataplane/tests.rs @@ -12,10 +12,11 @@ use super::*; use crate::{ config::peers::PeerRuntimeSnapshot, config::{IpPrefix, NetworkIdentity}, - host::testkit::TestHost, - peers::{ - PacketRecvChanReceiver, create_packet_recv_chan, peer_manager::PortablePeerManagerConfig, + host::{ + packet::{HostPacketReceiver, host_packet_channel}, + testkit::TestHost, }, + peers::peer_manager::PortablePeerManagerConfig, tunnel::ring::RingTunnelRegistry, }; @@ -49,7 +50,7 @@ fn test_gateway() -> Arc> { struct DataPlaneEndpoint { gateway: Arc>, peer_manager: Arc, - _packet_receiver: PacketRecvChanReceiver, + _packet_receiver: HostPacketReceiver, ip: cidr::Ipv4Inet, } @@ -73,7 +74,7 @@ fn data_plane_endpoint(host: Arc, ip: cidr::Ipv4Inet) -> DataPlaneEndp crate::config::runtime::CoreRuntimeConfig::default(), Arc::new(peer_config.snapshot.clone()), ); - let (packet_sender, packet_receiver) = create_packet_recv_chan(); + let (packet_sender, packet_receiver) = host_packet_channel(); let peer_manager = Arc::new( PeerManagerCore::new_portable_for_test(peer_config, packet_sender) .expect("build portable peer manager"), diff --git a/easytier-core/src/gateway/proxy/tcp_proxy_service.rs b/easytier-core/src/gateway/proxy/tcp_proxy_service.rs index d41bff2b..7a8a1ac6 100644 --- a/easytier-core/src/gateway/proxy/tcp_proxy_service.rs +++ b/easytier-core/src/gateway/proxy/tcp_proxy_service.rs @@ -471,7 +471,12 @@ impl Arc { - let (packet_tx, _packet_rx) = create_packet_recv_chan(); + let (packet_tx, _packet_rx) = crate::host::packet::host_packet_channel(); Arc::new( PeerManagerCore::new_portable_for_test( PortablePeerManagerConfig::new(PeerRuntimeConfig { diff --git a/easytier-core/src/host/packet.rs b/easytier-core/src/host/packet.rs index 85273e80..6b7888e8 100644 --- a/easytier-core/src/host/packet.rs +++ b/easytier-core/src/host/packet.rs @@ -1,24 +1,143 @@ -use std::{io, sync::Arc, task::Poll}; +use std::{fmt, io, sync::Arc, task::Poll}; use async_trait::async_trait; use tokio::sync::mpsc; +use crate::packet::{ZCPacket, ZCPacketType}; + use super::socket::{HostOperationId, HostSocketRuntime}; -/// Receives raw IP packet bytes leaving the EasyTier peer graph. +/// An owned raw IP packet crossing the Host packet seam. +/// +/// The backing allocation and any core-private headroom stay opaque. Native +/// adapters can move the packet without copying its payload, while ABI adapters +/// can borrow [`Self::payload`] at the point where a copy is unavoidable. +pub struct HostPacket { + inner: ZCPacket, +} + +pub(crate) type HostPacketSender = mpsc::Sender; + +/// The single bounded receive side for packets leaving a core instance. +/// +/// Hosts own this receiver for the instance lifetime. Keeping the channel +/// opaque prevents platform adapters from depending on core packet headers or +/// replacing the bounded backpressure contract. +pub struct HostPacketReceiver { + receiver: mpsc::Receiver, +} + +impl HostPacketReceiver { + /// Wraps a Host-owned bounded packet channel. + /// + /// This is intended for native adapters that already receive owned + /// [`HostPacket`] values from an in-process packet sink. + pub fn new(receiver: mpsc::Receiver) -> Self { + Self { receiver } + } + + pub async fn recv(&mut self) -> Option { + self.receiver.recv().await + } +} + +pub(crate) fn host_packet_channel() -> (HostPacketSender, HostPacketReceiver) { + let (sender, receiver) = mpsc::channel(128); + (sender, HostPacketReceiver::new(receiver)) +} + +impl HostPacket { + /// Copies raw IP bytes into core-owned packet storage. + /// + /// Native packet devices should use [`Self::from_tun_packet`] instead. + pub fn copy_from_payload(payload: &[u8]) -> Self { + Self { + inner: ZCPacket::new_with_payload(payload), + } + } + + /// Moves a packet read from a native TUN adapter into the Host seam. + pub fn from_tun_packet(packet: ZCPacket) -> Self { + debug_assert_eq!(packet.packet_type(), ZCPacketType::NIC); + Self { inner: packet } + } + + pub fn payload(&self) -> &[u8] { + self.inner.payload() + } + + pub fn len(&self) -> usize { + self.inner.payload_len() + } + + pub fn is_empty(&self) -> bool { + self.inner.payload().is_empty() + } + + /// Moves a packet into a native TUN adapter without copying its payload. + /// + /// Core-private headers are cleared before the backing allocation crosses + /// the seam. The returned packet retains its original payload offset so the + /// native writer can reuse the existing headroom for packet-info framing. + pub fn into_tun_packet(mut self) -> ZCPacket { + let payload_offset = self.inner.payload_offset(); + self.inner.mut_inner()[..payload_offset].fill(0); + self.inner + } + + pub(crate) fn from_core_packet(packet: ZCPacket) -> Self { + Self { inner: packet } + } + + pub(crate) fn into_core_packet(self) -> ZCPacket { + self.inner + } +} + +impl fmt::Debug for HostPacket { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("HostPacket") + .field("len", &self.len()) + .finish() + } +} + +/// Receives owned IP packets leaving the EasyTier peer graph. /// /// The host decides whether packets go to a TUN device, a Go callback, or a -/// different packet backend. Core's internal packet headers never cross this -/// boundary, and core never performs platform I/O directly. +/// different packet backend. Core's internal packet headers remain hidden, and +/// core never performs platform I/O directly. #[async_trait] pub trait PacketSink: Send + Sync + 'static { - async fn write_packet(&self, packet: Vec) -> anyhow::Result<()>; + async fn write_packet(&self, packet: HostPacket) -> anyhow::Result<()>; } #[async_trait] impl PacketSink for mpsc::Sender> { - async fn write_packet(&self, packet: Vec) -> anyhow::Result<()> { - self.send(packet) + async fn write_packet(&self, packet: HostPacket) -> anyhow::Result<()> { + self.send(packet.payload().to_vec()) + .await + .map_err(|_| anyhow::anyhow!("packet sink channel is closed")) + } +} + +/// Ownership-preserving in-process adapter for native packet runtimes. +pub struct HostPacketChannelSink { + sender: mpsc::Sender, +} + +impl HostPacketChannelSink { + pub fn new(sender: mpsc::Sender) -> Self { + Self { sender } + } +} + +#[async_trait] +impl PacketSink for HostPacketChannelSink { + async fn write_packet(&self, packet: HostPacket) -> anyhow::Result<()> { + self.sender + .send(packet) .await .map_err(|_| anyhow::anyhow!("packet sink channel is closed")) } @@ -97,9 +216,9 @@ impl PacketSink for HostPacketSink where I: HostPacketIo, { - async fn write_packet(&self, packet: Vec) -> anyhow::Result<()> { + async fn write_packet(&self, packet: HostPacket) -> anyhow::Result<()> { loop { - match self.io.try_write_packet(self.handle, &packet) { + match self.io.try_write_packet(self.handle, packet.payload()) { Ok(()) => return Ok(()), Err(error) if error.kind() == io::ErrorKind::WouldBlock => { self.wait_writable().await?; @@ -202,7 +321,9 @@ mod tests { #[tokio::test] async fn admits_complete_packet_without_readiness_wait() { let (_runtime, io, sink) = test_sink(true); - sink.write_packet(vec![1, 2, 3, 4]).await.unwrap(); + sink.write_packet(HostPacket::copy_from_payload(&[1, 2, 3, 4])) + .await + .unwrap(); let state = io.state.lock().unwrap(); assert_eq!( @@ -215,7 +336,10 @@ mod tests { #[tokio::test] async fn waits_for_capacity_then_admits_packet_once() { let (runtime, io, sink) = test_sink(false); - let task = tokio::spawn(async move { sink.write_packet(vec![5, 6, 7]).await }); + let task = tokio::spawn(async move { + sink.write_packet(HostPacket::copy_from_payload(&[5, 6, 7])) + .await + }); tokio::task::yield_now().await; assert!(io.state.lock().unwrap().packets.is_empty()); assert_eq!(runtime.inner.wakers.len(), 1); @@ -238,7 +362,7 @@ mod tests { async fn dropping_pending_waiter_removes_waker_and_host_state() { let (runtime, io, sink) = test_sink(false); let operation = { - let mut write = Box::pin(sink.write_packet(vec![7, 8])); + let mut write = Box::pin(sink.write_packet(HostPacket::copy_from_payload(&[7, 8]))); assert!(futures::poll!(&mut write).is_pending()); assert_eq!(runtime.inner.wakers.len(), 1); let operation = io.waiter(); @@ -257,7 +381,7 @@ mod tests { async fn dropping_ready_waiter_does_not_admit_packet() { let (runtime, io, sink) = test_sink(false); let operation = { - let mut write = Box::pin(sink.write_packet(vec![8, 9])); + let mut write = Box::pin(sink.write_packet(HostPacket::copy_from_payload(&[8, 9]))); assert!(futures::poll!(&mut write).is_pending()); let operation = io.waiter(); io.set_writable(); @@ -272,10 +396,46 @@ mod tests { assert!(state.waiters.is_empty()); assert_eq!(state.cancelled, vec![operation]); } - sink.write_packet(vec![8, 9]).await.unwrap(); + sink.write_packet(HostPacket::copy_from_payload(&[8, 9])) + .await + .unwrap(); assert_eq!( io.state.lock().unwrap().packets, vec![(HostPacketSinkHandle(41), vec![8, 9])] ); } + + #[test] + fn native_packet_round_trip_preserves_allocation_and_hides_headers() { + let mut core_packet = ZCPacket::new_with_payload(b"owned payload"); + let payload_offset = core_packet.payload_offset(); + core_packet.mut_inner()[..payload_offset].fill(0xa5); + let payload_pointer = core_packet.payload().as_ptr(); + + let host_packet = HostPacket::from_core_packet(core_packet); + assert_eq!(host_packet.payload().as_ptr(), payload_pointer); + assert_eq!(host_packet.payload(), b"owned payload"); + + let tun_packet = host_packet.into_tun_packet(); + assert_eq!(tun_packet.payload().as_ptr(), payload_pointer); + assert_eq!(tun_packet.payload(), b"owned payload"); + assert!( + tun_packet.inner()[..payload_offset] + .iter() + .all(|byte| *byte == 0) + ); + } + + #[test] + fn tun_packet_moves_into_core_without_copying_payload() { + let tun_packet = ZCPacket::new_with_payload(b"from tun"); + let payload_pointer = tun_packet.payload().as_ptr(); + + let host_packet = HostPacket::from_tun_packet(tun_packet); + assert_eq!(host_packet.payload().as_ptr(), payload_pointer); + let core_packet = host_packet.into_core_packet(); + + assert_eq!(core_packet.payload().as_ptr(), payload_pointer); + assert_eq!(core_packet.payload(), b"from tun"); + } } diff --git a/easytier-core/src/instance/lifecycle.rs b/easytier-core/src/instance/lifecycle.rs index 65b3c06f..e622d251 100644 --- a/easytier-core/src/instance/lifecycle.rs +++ b/easytier-core/src/instance/lifecycle.rs @@ -103,9 +103,13 @@ where .await; self.start_listener().await?; - if let Some(packet_egress) = &self.packet_egress { - packet_egress.start()?; - } + let packet_receiver = self + .packet_receiver + .lock() + .await + .take() + .ok_or_else(|| anyhow::anyhow!("packet egress is one-shot and already started"))?; + self.packet_egress.start(packet_receiver).await?; self.peer_manager.run().await.map_err(anyhow::Error::from)?; self.direct.run(); #[cfg(feature = "tcp-hole-punch")] @@ -184,9 +188,7 @@ where // before clearing PeerManager resources. self.instance_runtime.shutdown().await; self.peer_manager.clear_resources().await; - if let Some(packet_egress) = &self.packet_egress { - packet_egress.stop().await; - } + self.packet_egress.stop().await; } /// Starts the complete instance through one serial composition path. diff --git a/easytier-core/src/instance/mod.rs b/easytier-core/src/instance/mod.rs index c92a7120..1451aba9 100644 --- a/easytier-core/src/instance/mod.rs +++ b/easytier-core/src/instance/mod.rs @@ -60,7 +60,10 @@ use crate::{ }, events::CoreEventSink, gateway::dhcp::DhcpIpv4Host, - host::dns::{DnsRecordResolver, DnsResolver}, + host::{ + dns::{DnsRecordResolver, DnsResolver}, + packet::{HostPacketReceiver, PacketSink, host_packet_channel}, + }, listener::{ AcceptedSocketHandler, ExternalListenerFactory, ExternalListenerRequest, ListenerFactory, RunningListenerRegistry, @@ -74,7 +77,6 @@ use crate::{ peers::{ admission::{PeerAcceptedTunnelHandler, RawAcceptedTransportHandler}, context::PeerStunInfoSource, - create_packet_recv_chan, credential_manager::CredentialStorage, peer_manager::{PeerManagerCore, PortablePeerManagerConfig}, public_ipv6::{CorePublicIpv6Runtime, PublicIpv6Host}, @@ -108,12 +110,12 @@ use crate::gateway::vpn_portal::VpnPortalModule; use crate::gateway::{ DataPlaneRuntime, DataPlaneSession, PortForwardAdapter, Socks5GatewayAdapter, }; -use crate::host::packet::PacketSink; #[cfg(feature = "public-ipv6-provider")] use crate::peers::public_ipv6::provider::PublicIpv6ProviderRuntime; pub use config::CoreInstanceHostConfig; use management_state::ManagementState; -use packet_io::PacketEgress; +pub use packet_io::PacketEgressHost; +use packet_io::PacketSinkEgress; pub use packet_plane::CorePacketPlane; /// Complete Host capability set required by one portable core instance. @@ -277,7 +279,7 @@ where stun_override: Option::Socket>>>, dns: Arc, process_runtime: Arc, - packet_sink: Arc, + pub packet_egress: Arc, pub instance_runtime: Arc, pub events: Arc, pub credential_storage: Option>, @@ -313,6 +315,22 @@ where dns: Arc, packet_sink: Arc, process_runtime: Arc, + ) -> Self { + Self::new_with_packet_egress( + host, + dns, + Arc::new(PacketSinkEgress::new(packet_sink)), + process_runtime, + ) + } + + /// Creates a host bundle whose packet runtime owns the core's single + /// bounded egress receiver directly. + pub fn new_with_packet_egress( + host: Arc, + dns: Arc, + packet_egress: Arc, + process_runtime: Arc, ) -> Self { Self { host, @@ -321,7 +339,7 @@ where stun_override: None, dns, process_runtime, - packet_sink, + packet_egress, instance_runtime: Arc::new(()), events: Arc::new(()), credential_storage: None, @@ -395,7 +413,8 @@ where proxy_cidr_monitor: ProxyCidrMonitorRuntime, #[cfg(feature = "dhcp-ipv4")] dhcp_ipv4: DhcpIpv4Runtime, - pub(super) packet_egress: Option, + pub(super) packet_egress: Arc, + pub(super) packet_receiver: Mutex>, pub(super) peer_center: Arc, #[cfg(feature = "public-ipv6-provider")] public_ipv6_provider: PublicIpv6ProviderRuntime, @@ -463,7 +482,7 @@ where ) -> anyhow::Result> { let initial_acl = validate_core_instance_config(&config)?; let instance_name = config.instance_name; - let (packet_tx, packet_rx) = create_packet_recv_chan(); + let (packet_tx, packet_rx) = host_packet_channel(); let runtime_config = CoreRuntimeConfigStore::new( config.connectivity.runtime.clone(), Arc::new(config.peer.snapshot.clone()), @@ -516,7 +535,7 @@ where stun_override: _, dns, process_runtime, - packet_sink, + packet_egress, instance_runtime, events, credential_storage: _, @@ -779,7 +798,8 @@ where proxy_cidr_monitor, #[cfg(feature = "dhcp-ipv4")] dhcp_ipv4: DhcpIpv4Runtime::new(), - packet_egress: Some(PacketEgress::new(packet_rx, packet_sink)), + packet_egress, + packet_receiver: Mutex::new(Some(packet_rx)), peer_center, #[cfg(feature = "public-ipv6-provider")] public_ipv6_provider, @@ -887,6 +907,7 @@ where fn drop(&mut self) { self.cancel.cancel(); self.instance_runtime.request_shutdown(); + self.packet_egress.request_stop(); } } diff --git a/easytier-core/src/instance/packet_io.rs b/easytier-core/src/instance/packet_io.rs index e5e49a10..ebb21e2d 100644 --- a/easytier-core/src/instance/packet_io.rs +++ b/easytier-core/src/instance/packet_io.rs @@ -3,10 +3,9 @@ use std::{ sync::{Arc, Mutex}, }; -use tokio::{sync::mpsc, task::JoinHandle}; +use tokio::task::JoinHandle; -use crate::host::packet::PacketSink; -use crate::packet::ZCPacket; +use crate::host::packet::{HostPacketReceiver, PacketSink}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct IpPacketMeta { @@ -64,32 +63,41 @@ fn parse_ipv6_packet(packet: &[u8]) -> anyhow::Result { }) } -pub(crate) struct PacketEgress { - receiver: Mutex>>, - sink: Arc, - task: Mutex>>, +#[async_trait::async_trait] +pub trait PacketEgressHost: Send + Sync + 'static { + async fn start(&self, receiver: HostPacketReceiver) -> anyhow::Result<()>; + + async fn stop(&self); + + fn request_stop(&self) {} } -impl PacketEgress { - pub(crate) fn new(receiver: mpsc::Receiver, sink: Arc) -> Self { +pub(crate) struct PacketSinkEgress { + sink: Arc, + task: Mutex>>, + started: std::sync::atomic::AtomicBool, +} + +impl PacketSinkEgress { + pub(crate) fn new(sink: Arc) -> Self { Self { - receiver: Mutex::new(Some(receiver)), sink, task: Mutex::new(None), + started: std::sync::atomic::AtomicBool::new(false), } } +} - pub(crate) fn start(&self) -> anyhow::Result<()> { - let mut receiver = self - .receiver - .lock() - .unwrap() - .take() - .ok_or_else(|| anyhow::anyhow!("packet egress is one-shot and already started"))?; +#[async_trait::async_trait] +impl PacketEgressHost for PacketSinkEgress { + async fn start(&self, mut receiver: HostPacketReceiver) -> anyhow::Result<()> { + if self.started.swap(true, std::sync::atomic::Ordering::AcqRel) { + anyhow::bail!("packet egress is one-shot and already started"); + } let sink = self.sink.clone(); let task = tokio::spawn(async move { while let Some(packet) = receiver.recv().await { - if let Err(error) = sink.write_packet(packet.payload().to_vec()).await { + if let Err(error) = sink.write_packet(packet).await { tracing::warn!(?error, "host packet sink rejected an egress packet"); } } @@ -98,17 +106,22 @@ impl PacketEgress { Ok(()) } - pub(crate) async fn stop(&self) { + async fn stop(&self) { let task = self.task.lock().unwrap().take(); if let Some(task) = task { task.abort(); let _ = task.await; } - self.receiver.lock().unwrap().take(); + } + + fn request_stop(&self) { + if let Some(task) = self.task.lock().unwrap().take() { + task.abort(); + } } } -impl Drop for PacketEgress { +impl Drop for PacketSinkEgress { fn drop(&mut self) { if let Some(task) = self.task.lock().unwrap().take() { task.abort(); @@ -119,6 +132,7 @@ impl Drop for PacketEgress { #[cfg(test)] mod tests { use crate::foundation::time::{Duration, timeout}; + use crate::host::packet::{HostPacket, host_packet_channel}; use super::*; @@ -169,13 +183,13 @@ mod tests { #[tokio::test] async fn packet_egress_forwards_to_host_sink_and_joins_on_stop() { - let (core_tx, core_rx) = mpsc::channel(1); - let (host_tx, mut host_rx) = mpsc::channel(1); - let egress = PacketEgress::new(core_rx, Arc::new(host_tx)); - egress.start().unwrap(); + let (core_tx, core_rx) = host_packet_channel(); + let (host_tx, mut host_rx) = tokio::sync::mpsc::channel(1); + let egress = PacketSinkEgress::new(Arc::new(host_tx)); + egress.start(core_rx).await.unwrap(); core_tx - .send(ZCPacket::new_with_payload(b"packet")) + .send(HostPacket::copy_from_payload(b"packet")) .await .unwrap(); let packet = timeout(Duration::from_secs(1), host_rx.recv()) @@ -185,6 +199,7 @@ mod tests { assert_eq!(packet, b"packet"); egress.stop().await; - assert!(egress.start().is_err()); + let (_core_tx, core_rx) = host_packet_channel(); + assert!(egress.start(core_rx).await.is_err()); } } diff --git a/easytier-core/src/instance/packet_plane.rs b/easytier-core/src/instance/packet_plane.rs index 26c3bffc..f9a81947 100644 --- a/easytier-core/src/instance/packet_plane.rs +++ b/easytier-core/src/instance/packet_plane.rs @@ -6,6 +6,7 @@ use crate::{ config::runtime::CoreRuntimeConfigStore, gateway::magic_dns::{MagicDnsRouteSnapshot, MagicDnsRouteSource}, gateway::proxy::cidr_monitor::{ProxyCidrDiff, collect_proxy_cidr_diff}, + host::packet::HostPacket, peers::peer_manager::PeerManagerCore, }; @@ -40,30 +41,22 @@ impl CorePacketPlane { } } - pub async fn send_ip_packet(&self, packet: Vec) -> anyhow::Result<()> { - let meta = parse_ip_packet(&packet)?; + pub async fn send_ip_packet(&self, packet: HostPacket) -> anyhow::Result<()> { + let meta = parse_ip_packet(packet.payload())?; let source_is_local = self.peer_manager.is_local_virtual_ip(&meta.source); if matches!(meta.source, IpAddr::V6(ip) if ip.is_unicast_link_local()) && !source_is_local { return Ok(()); } self.peer_manager - .send_msg_by_ip( - crate::packet::ZCPacket::new_with_payload(&packet), - meta.destination, - source_is_local, - ) + .send_msg_by_ip(packet.into_core_packet(), meta.destination, source_is_local) .await .map_err(Into::into) } - pub async fn send_local_ip_packet(&self, packet: Vec) -> anyhow::Result<()> { - let destination = parse_ip_packet(&packet)?.destination; + pub async fn send_local_ip_packet(&self, packet: HostPacket) -> anyhow::Result<()> { + let destination = parse_ip_packet(packet.payload())?.destination; self.peer_manager - .send_msg_by_ip( - crate::packet::ZCPacket::new_with_payload(&packet), - destination, - true, - ) + .send_msg_by_ip(packet.into_core_packet(), destination, true) .await .map_err(Into::into) } diff --git a/easytier-core/src/packet/mod.rs b/easytier-core/src/packet/mod.rs index 6905689c..739222e2 100644 --- a/easytier-core/src/packet/mod.rs +++ b/easytier-core/src/packet/mod.rs @@ -663,6 +663,16 @@ impl ZCPacket { + UDP_TUNNEL_HEADER_SIZE..] } + pub fn udp_payload_bytes(mut self) -> BytesMut { + self.inner.advance( + self.packet_type + .get_packet_offsets() + .udp_tunnel_header_offset + + UDP_TUNNEL_HEADER_SIZE, + ); + self.inner + } + pub fn payload_len(&self) -> usize { self.inner.len() - self.payload_offset() } diff --git a/easytier-core/src/peers/conn/peer_session.rs b/easytier-core/src/peers/conn/peer_session.rs index 93dba786..2c822f75 100644 --- a/easytier-core/src/peers/conn/peer_session.rs +++ b/easytier-core/src/peers/conn/peer_session.rs @@ -421,7 +421,18 @@ mod tests { } #[test] - #[cfg(all(feature = "aes-gcm", feature = "chacha20"))] + #[cfg(all( + any( + feature = "aes-gcm", + feature = "openssl-crypto", + feature = "ring-crypto" + ), + any( + feature = "chacha20", + feature = "openssl-crypto", + feature = "ring-crypto" + ) + ))] fn peer_session_supports_asymmetric_algorithms() { let a: PeerId = 10; let b: PeerId = 20; diff --git a/easytier-core/src/peers/context.rs b/easytier-core/src/peers/context.rs index db891259..cd63a0c6 100644 --- a/easytier-core/src/peers/context.rs +++ b/easytier-core/src/peers/context.rs @@ -528,6 +528,27 @@ impl Default for TrustedKeyMapManager { /// `PeerContext` is intentionally scoped to `easytier-core::peers`; other core /// modules should depend on their own narrow DTOs or traits instead of treating /// this as a core-wide global context. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct PeerPacketPolicy { + pub(crate) disable_relay_data: bool, + pub(crate) p2p_only: bool, + pub(crate) latency_first: bool, + pub(crate) disable_p2p: bool, + pub(crate) lazy_p2p: bool, +} + +impl PeerPacketPolicy { + fn from_flags(flags: &FlagsInConfig) -> Self { + Self { + disable_relay_data: flags.disable_relay_data, + p2p_only: flags.p2p_only, + latency_first: flags.latency_first && !flags.p2p_only, + disable_p2p: flags.disable_p2p, + lazy_p2p: flags.lazy_p2p, + } + } +} + pub(crate) trait PeerContext: Send + Sync { fn host_routing_policy(&self) -> HostRoutingPolicy { HostRoutingPolicy::default() @@ -543,8 +564,12 @@ pub(crate) trait PeerContext: Send + Sync { FlagsInConfig::default() } + fn packet_policy(&self) -> PeerPacketPolicy { + PeerPacketPolicy::from_flags(&self.flags()) + } + fn disable_relay_data(&self) -> bool { - self.flags().disable_relay_data + self.packet_policy().disable_relay_data } fn secure_mode(&self) -> Option { @@ -583,15 +608,6 @@ pub(crate) trait PeerContext: Send + Sync { } } - fn p2p_only(&self) -> bool { - self.flags().p2p_only - } - - fn latency_first(&self) -> bool { - let flags = self.flags(); - flags.latency_first && !flags.p2p_only - } - fn proxy_cidrs(&self) -> Vec { Vec::new() } @@ -746,6 +762,10 @@ impl PeerContext for CorePeerContext { self.snapshot().flags.clone() } + fn packet_policy(&self) -> PeerPacketPolicy { + PeerPacketPolicy::from_flags(&self.snapshot().flags) + } + fn host_routing_policy(&self) -> HostRoutingPolicy { self.snapshot().runtime.host_routing } diff --git a/easytier-core/src/peers/peer_manager.rs b/easytier-core/src/peers/peer_manager.rs index e12d50f5..73befbd6 100644 --- a/easytier-core/src/peers/peer_manager.rs +++ b/easytier-core/src/peers/peer_manager.rs @@ -9,8 +9,8 @@ use std::{ }; use anyhow::Context; +use arc_swap::ArcSwap; use dashmap::DashMap; -use parking_lot::RwLock as SyncRwLock; use quanta::Instant; use serde::{Deserialize, Serialize}; use tokio::sync::{ @@ -26,6 +26,7 @@ use crate::{ config::{P2pPolicyFlags, PeerId, ProxyNetworkConfig}, events::CoreEventSink, foundation::task::ExternalTaskSignal, + host::packet::{HostPacket, HostPacketSender}, packet::{ CompressorAlgo, PacketType, ZCPacket, compressor::{Compressor as _, DefaultCompressor}, @@ -42,8 +43,7 @@ use crate::{ }; use super::{ - BoxNicPacketFilter, BoxPeerPacketFilter, PacketRecvChan, PacketRecvChanReceiver, - PeerPacketFilter, + BoxNicPacketFilter, BoxPeerPacketFilter, PacketRecvChanReceiver, PeerPacketFilter, acl::AclFilter, conn::{ peer_conn::{PeerConn, PeerConnId}, @@ -52,7 +52,7 @@ use super::{ }, context::{ ArcPeerContext, CorePeerContext, CorePeerContextAdapters, NetworkIdentity, PeerContext, - PeerStunInfoSource, + PeerPacketPolicy, PeerStunInfoSource, }, credential_manager::{CredentialManager, CredentialStorage}, error::Error, @@ -401,7 +401,7 @@ pub(crate) async fn close_untrusted_credential_peers( } struct NicPacketProcessor { - nic_channel: PacketRecvChan, + nic_channel: HostPacketSender, } #[async_trait::async_trait] @@ -420,7 +420,10 @@ impl PeerPacketFilter for NicPacketProcessor { return None; } tracing::trace!(?packet, "send packet to nic channel"); - let _ = self.nic_channel.send(packet).await; + let _ = self + .nic_channel + .send(HostPacket::from_core_packet(packet)) + .await; None } else { Some(packet) @@ -448,26 +451,58 @@ impl PeerPacketFilter for PeerRpcPacketProcessor { } } +#[derive(Clone)] pub(crate) struct PeerPipelineEntry { - active: Arc, - filter: Arc>>>, + active: Option>, + filter: Arc, } +#[derive(Clone)] pub(crate) struct NicPipelineEntry { - active: Arc, - filter: Arc>>>, + active: Option>, + filter: Arc, +} + +type PeerPacketPipeline = Arc>>; +type NicPacketPipeline = Arc>>; + +impl PeerPipelineEntry { + fn filter_if_active(&self) -> Option<&Arc> { + if self + .active + .as_ref() + .is_some_and(|active| !active.load(Ordering::Acquire)) + { + return None; + } + Some(&self.filter) + } +} + +impl NicPipelineEntry { + fn filter_if_active(&self) -> Option<&Arc> { + if self + .active + .as_ref() + .is_some_and(|active| !active.load(Ordering::Acquire)) + { + return None; + } + Some(&self.filter) + } } #[derive(Clone)] pub(crate) struct PipelineRegistrationGuard { active: Arc, - release_filter: Arc, + unregister: Arc, } impl PipelineRegistrationGuard { pub fn close(&self) { - self.active.store(false, Ordering::Release); - (self.release_filter)(); + if self.active.swap(false, Ordering::AcqRel) { + (self.unregister)(); + } } } @@ -477,36 +512,52 @@ impl Drop for PipelineRegistrationGuard { } } -fn permanent_peer_pipeline_entry(filter: BoxPeerPacketFilter) -> Arc { - Arc::new(PeerPipelineEntry { - active: Arc::new(AtomicBool::new(true)), - filter: Arc::new(SyncRwLock::new(Some(Arc::from(filter)))), - }) +fn permanent_peer_pipeline_entry(filter: BoxPeerPacketFilter) -> PeerPipelineEntry { + PeerPipelineEntry { + active: None, + filter: Arc::from(filter), + } } -fn permanent_nic_pipeline_entry(filter: BoxNicPacketFilter) -> Arc { - Arc::new(NicPipelineEntry { - active: Arc::new(AtomicBool::new(true)), - filter: Arc::new(SyncRwLock::new(Some(Arc::from(filter)))), - }) +fn permanent_nic_pipeline_entry(filter: BoxNicPacketFilter) -> NicPipelineEntry { + NicPipelineEntry { + active: None, + filter: Arc::from(filter), + } } fn managed_peer_pipeline_entry( filter: BoxPeerPacketFilter, -) -> (Arc, PipelineRegistrationGuard) { + pipeline: &PeerPacketPipeline, +) -> (PeerPipelineEntry, PipelineRegistrationGuard) { let active = Arc::new(AtomicBool::new(true)); - let filter = Arc::new(SyncRwLock::new(Some(Arc::from(filter)))); - let release_filter = filter.clone(); + let weak_pipeline = Arc::downgrade(pipeline); + let registration = active.clone(); ( - Arc::new(PeerPipelineEntry { - active: active.clone(), - filter, - }), + PeerPipelineEntry { + active: Some(active.clone()), + filter: Arc::from(filter), + }, PipelineRegistrationGuard { active, - release_filter: Arc::new(move || { - let filter = release_filter.write().take(); - drop(filter); + unregister: Arc::new(move || { + let Some(pipeline) = weak_pipeline.upgrade() else { + return; + }; + pipeline.rcu(|current| { + Arc::new( + current + .iter() + .filter(|entry| { + entry + .active + .as_ref() + .is_none_or(|active| !Arc::ptr_eq(active, ®istration)) + }) + .cloned() + .collect(), + ) + }); }), }, ) @@ -515,63 +566,101 @@ fn managed_peer_pipeline_entry( #[cfg(any(feature = "proxy-packet", test))] fn managed_nic_pipeline_entry( filter: BoxNicPacketFilter, -) -> (Arc, PipelineRegistrationGuard) { + pipeline: &NicPacketPipeline, +) -> (NicPipelineEntry, PipelineRegistrationGuard) { let active = Arc::new(AtomicBool::new(true)); - let filter = Arc::new(SyncRwLock::new(Some(Arc::from(filter)))); - let release_filter = filter.clone(); + let weak_pipeline = Arc::downgrade(pipeline); + let registration = active.clone(); ( - Arc::new(NicPipelineEntry { - active: active.clone(), - filter, - }), + NicPipelineEntry { + active: Some(active.clone()), + filter: Arc::from(filter), + }, PipelineRegistrationGuard { active, - release_filter: Arc::new(move || { - let filter = release_filter.write().take(); - drop(filter); + unregister: Arc::new(move || { + let Some(pipeline) = weak_pipeline.upgrade() else { + return; + }; + pipeline.rcu(|current| { + Arc::new( + current + .iter() + .filter(|entry| { + entry + .active + .as_ref() + .is_none_or(|active| !Arc::ptr_eq(active, ®istration)) + }) + .cloned() + .collect(), + ) + }); }), }, ) } -#[cfg(any(feature = "proxy-packet", test))] -async fn remove_managed_nic_pipeline_entry( - pipeline: &RwLock>>, - registration: &PipelineRegistrationGuard, -) { - registration.close(); - pipeline - .write() - .await - .retain(|entry| !Arc::ptr_eq(&entry.active, ®istration.active)); +fn append_peer_pipeline(pipeline: &PeerPacketPipeline, entry: PeerPipelineEntry) { + pipeline.rcu(|current| { + let mut next = Vec::with_capacity(current.len() + 1); + next.extend( + current + .iter() + .filter(|entry| { + entry + .active + .as_ref() + .is_none_or(|active| active.load(Ordering::Acquire)) + }) + .cloned(), + ); + next.push(entry.clone()); + Arc::new(next) + }); +} + +fn append_nic_pipeline(pipeline: &NicPacketPipeline, entry: NicPipelineEntry) { + pipeline.rcu(|current| { + let mut next = Vec::with_capacity(current.len() + 1); + next.extend( + current + .iter() + .filter(|entry| { + entry + .active + .as_ref() + .is_none_or(|active| active.load(Ordering::Acquire)) + }) + .cloned(), + ); + next.push(entry.clone()); + Arc::new(next) + }); } async fn init_packet_process_pipeline( - peer_packet_process_pipeline: &RwLock>>, - nic_channel: PacketRecvChan, + peer_packet_process_pipeline: &PeerPacketPipeline, + nic_channel: HostPacketSender, peer_rpc_tspt_sender: UnboundedSender, ) { // for tun/tap ip/eth packet. - peer_packet_process_pipeline - .write() - .await - .push(permanent_peer_pipeline_entry(Box::new( - NicPacketProcessor { nic_channel }, - ))); + append_peer_pipeline( + peer_packet_process_pipeline, + permanent_peer_pipeline_entry(Box::new(NicPacketProcessor { nic_channel })), + ); // for peer rpc packet - peer_packet_process_pipeline - .write() - .await - .push(permanent_peer_pipeline_entry(Box::new( - PeerRpcPacketProcessor { - peer_rpc_tspt_sender, - }, - ))); + append_peer_pipeline( + peer_packet_process_pipeline, + permanent_peer_pipeline_entry(Box::new(PeerRpcPacketProcessor { + peer_rpc_tspt_sender, + })), + ); } async fn add_route( - peer_packet_process_pipeline: &RwLock>>, + peer_packet_process_pipeline: &PeerPacketPipeline, peers: Arc, foreign_network_client: Arc, foreign_network_manager: Arc, @@ -581,10 +670,10 @@ async fn add_route( T: Route + PeerPacketFilter + Send + Sync + 'static, { // for route - peer_packet_process_pipeline - .write() - .await - .push(permanent_peer_pipeline_entry(Box::new(route.clone()))); + append_peer_pipeline( + peer_packet_process_pipeline, + permanent_peer_pipeline_entry(Box::new(route.clone())), + ); let _route_id = route .open(Box::new(PeerManagerRouteInterface { @@ -614,9 +703,9 @@ pub struct PeerManagerCore { peers: Arc, peer_rpc_mgr: Arc, peer_rpc_tspt: Arc, - peer_packet_process_pipeline: Arc>>>, - nic_packet_process_pipeline: Arc>>>, - nic_channel: PacketRecvChan, + peer_packet_process_pipeline: PeerPacketPipeline, + nic_packet_process_pipeline: NicPacketPipeline, + nic_channel: HostPacketSender, route_algo_inst: RouteAlgoInst, foreign_network_client: Arc, foreign_network_manager: Arc, @@ -671,7 +760,7 @@ impl PeerManagerCore { mut config: PortablePeerManagerConfig, runtime_config: CoreRuntimeConfigStore, stun_info_source: Arc, - nic_channel: PacketRecvChan, + nic_channel: HostPacketSender, public_ipv6_runtime: Arc, events: Arc, credential_storage: Option>, @@ -800,7 +889,7 @@ impl PeerManagerCore { my_peer_id: PeerId, core_context: Arc, public_ipv6_runtime: Arc, - nic_channel: PacketRecvChan, + nic_channel: HostPacketSender, encryptor: Arc, is_secure_mode_enabled: bool, data_compress_algo: CompressorAlgo, @@ -928,8 +1017,8 @@ impl PeerManagerCore { } }, )); - let peer_packet_process_pipeline = Arc::new(RwLock::new(Vec::new())); - let nic_packet_process_pipeline = Arc::new(RwLock::new(Vec::new())); + let peer_packet_process_pipeline = Arc::new(ArcSwap::from_pointee(Vec::new())); + let nic_packet_process_pipeline = Arc::new(ArcSwap::from_pointee(Vec::new())); let exit_nodes = Arc::new(RwLock::new(exit_nodes)); let relay_peer_map = super::relay_peer_map::new_relay_peer_map( peers.clone(), @@ -1171,7 +1260,7 @@ impl PeerManagerCore { self.peer_session_store.clone() } - pub fn get_nic_channel(&self) -> PacketRecvChan { + pub(crate) fn get_nic_channel(&self) -> HostPacketSender { self.nic_channel.clone() } @@ -1271,28 +1360,27 @@ impl PeerManagerCore { pub async fn add_packet_process_pipeline(&self, pipeline: BoxPeerPacketFilter) { // newest pipeline will be executed first - self.peer_packet_process_pipeline - .write() - .await - .push(permanent_peer_pipeline_entry(pipeline)); + append_peer_pipeline( + &self.peer_packet_process_pipeline, + permanent_peer_pipeline_entry(pipeline), + ); } pub async fn add_nic_packet_process_pipeline(&self, pipeline: BoxNicPacketFilter) { // newest pipeline will be executed first - self.nic_packet_process_pipeline - .write() - .await - .push(permanent_nic_pipeline_entry(pipeline)); + append_nic_pipeline( + &self.nic_packet_process_pipeline, + permanent_nic_pipeline_entry(pipeline), + ); } pub(crate) async fn add_managed_packet_process_pipeline( &self, pipeline: BoxPeerPacketFilter, ) -> PipelineRegistrationGuard { - let (entry, guard) = managed_peer_pipeline_entry(pipeline); - let mut pipelines = self.peer_packet_process_pipeline.write().await; - pipelines.retain(|pipeline| pipeline.active.load(Ordering::Acquire)); - pipelines.push(entry); + let (entry, guard) = + managed_peer_pipeline_entry(pipeline, &self.peer_packet_process_pipeline); + append_peer_pipeline(&self.peer_packet_process_pipeline, entry); guard } @@ -1301,10 +1389,9 @@ impl PeerManagerCore { &self, pipeline: BoxNicPacketFilter, ) -> PipelineRegistrationGuard { - let (entry, guard) = managed_nic_pipeline_entry(pipeline); - let mut pipelines = self.nic_packet_process_pipeline.write().await; - pipelines.retain(|pipeline| pipeline.active.load(Ordering::Acquire)); - pipelines.push(entry); + let (entry, guard) = + managed_nic_pipeline_entry(pipeline, &self.nic_packet_process_pipeline); + append_nic_pipeline(&self.nic_packet_process_pipeline, entry); guard } @@ -1313,7 +1400,7 @@ impl PeerManagerCore { &self, registration: &PipelineRegistrationGuard, ) { - remove_managed_nic_pipeline_entry(&self.nic_packet_process_pipeline, registration).await; + registration.close(); } pub async fn add_route(&self, route: Arc) @@ -1321,7 +1408,7 @@ impl PeerManagerCore { T: Route + PeerPacketFilter + Send + Sync + 'static, { add_route( - self.peer_packet_process_pipeline.as_ref(), + &self.peer_packet_process_pipeline, self.peers.clone(), self.foreign_network_client.clone(), self.foreign_network_manager.clone(), @@ -1332,16 +1419,28 @@ impl PeerManagerCore { } pub async fn remove_nic_packet_process_pipeline(&self, id: String) -> Result<(), Error> { - let mut pipelines = self.nic_packet_process_pipeline.write().await; - if let Some(pos) = pipelines.iter().position(|pipeline| { - let filter = pipeline.filter.read().clone(); - filter.is_some_and(|filter| filter.id() == id) - }) { - pipelines.remove(pos); - Ok(()) - } else { - Err(Error::NotFound) - } + let snapshot = self.nic_packet_process_pipeline.load_full(); + let Some(target) = snapshot + .iter() + .find(|entry| { + entry + .filter_if_active() + .is_some_and(|filter| filter.id() == id) + }) + .map(|entry| entry.filter.clone()) + else { + return Err(Error::NotFound); + }; + self.nic_packet_process_pipeline.rcu(|current| { + Arc::new( + current + .iter() + .filter(|entry| !Arc::ptr_eq(&entry.filter, &target)) + .cloned() + .collect(), + ) + }); + Ok(()) } pub async fn send_msg_for_proxy( @@ -1428,10 +1527,9 @@ impl PeerManagerCore { pub(crate) async fn clear_resources(&self) { self.stop().await; - let mut peer_pipeline = self.peer_packet_process_pipeline.write().await; - peer_pipeline.clear(); - let mut nic_pipeline = self.nic_packet_process_pipeline.write().await; - nic_pipeline.clear(); + self.peer_packet_process_pipeline + .store(Arc::new(Vec::new())); + self.nic_packet_process_pipeline.store(Arc::new(Vec::new())); self.peer_rpc_mgr.rpc_server().registry().unregister_all(); } @@ -1490,7 +1588,7 @@ impl PeerManagerCore { } init_packet_process_pipeline( - self.peer_packet_process_pipeline.as_ref(), + &self.peer_packet_process_pipeline, self.nic_channel.clone(), self.peer_rpc_tspt.packet_sender(), ) @@ -2061,7 +2159,7 @@ pub(crate) struct PeerOutboundPacketRouter { route: ArcRoute, foreign_network_client: Arc, relay_peer_map: Arc, - nic_packet_process_pipeline: Arc>>>, + nic_packet_process_pipeline: NicPacketPipeline, encryptor: Arc, data_compress_algo: CompressorAlgo, exit_nodes: Arc>>, @@ -2081,7 +2179,7 @@ impl PeerOutboundPacketRouter { route: ArcRoute, foreign_network_client: Arc, relay_peer_map: Arc, - nic_packet_process_pipeline: Arc>>>, + nic_packet_process_pipeline: NicPacketPipeline, encryptor: Arc, data_compress_algo: CompressorAlgo, exit_nodes: Arc>>, @@ -2128,12 +2226,17 @@ impl PeerOutboundPacketRouter { } } - fn mark_recent_traffic(&self, dst_peer_id: PeerId) { - let flags = self.context.flags(); - self.recent_traffic - .mark(dst_peer_id, flags.disable_p2p, flags.lazy_p2p, |peer_id| { - self.has_directly_connected_conn(peer_id) - }); + fn mark_recent_traffic_with_policy( + &self, + dst_peer_id: PeerId, + packet_policy: PeerPacketPolicy, + ) { + self.recent_traffic.mark( + dst_peer_id, + packet_policy.disable_p2p, + packet_policy.lazy_p2p, + |peer_id| self.has_directly_connected_conn(peer_id), + ); } async fn run_nic_packet_process_pipeline(&self, data: &mut ZCPacket) -> bool { @@ -2148,12 +2251,9 @@ impl PeerOutboundPacketRouter { return false; } - for pipeline in self.nic_packet_process_pipeline.read().await.iter().rev() { - if !pipeline.active.load(Ordering::Acquire) { - continue; - } - let filter = pipeline.filter.read().clone(); - if let Some(filter) = filter { + let pipelines = self.nic_packet_process_pipeline.load_full(); + for pipeline in pipelines.iter().rev() { + if let Some(filter) = pipeline.filter_if_active() { let _ = filter.try_process_packet_from_nic(data).await; } } @@ -2161,8 +2261,12 @@ impl PeerOutboundPacketRouter { true } - fn check_p2p_only_before_send(&self, dst_peer_id: PeerId) -> Result<(), Error> { - if self.context.p2p_only() && !self.peers.has_peer(dst_peer_id) { + fn check_p2p_only_before_send( + &self, + dst_peer_id: PeerId, + packet_policy: PeerPacketPolicy, + ) -> Result<(), Error> { + if packet_policy.p2p_only && !self.peers.has_peer(dst_peer_id) { return Err(Error::RouteError(None)); } Ok(()) @@ -2235,8 +2339,9 @@ impl PeerOutboundPacketRouter { mut msg: ZCPacket, dst_peer_id: PeerId, ) -> Result<(), Error> { - self.mark_recent_traffic(dst_peer_id); - self.check_p2p_only_before_send(dst_peer_id)?; + let packet_policy = self.context.packet_policy(); + self.mark_recent_traffic_with_policy(dst_peer_id, packet_policy); + self.check_p2p_only_before_send(dst_peer_id, packet_policy)?; self.counters .compress_tx_bytes_before @@ -2397,9 +2502,10 @@ impl PeerOutboundPacketRouter { if !self.run_nic_packet_process_pipeline(&mut msg).await { return Ok(()); } + let packet_policy = self.context.packet_policy(); let cur_to_peer_id = msg.peer_manager_header().unwrap().to_peer_id.into(); if cur_to_peer_id != 0 { - self.mark_recent_traffic(cur_to_peer_id); + self.mark_recent_traffic_with_policy(cur_to_peer_id, packet_policy); return send_msg_internal( self.peers.as_ref(), &self.foreign_network_client, @@ -2437,10 +2543,9 @@ impl PeerOutboundPacketRouter { .compress_tx_bytes_after .add(msg.buf_len() as u64); - let is_latency_first = self.context.latency_first(); msg.mut_peer_manager_header() .unwrap() - .set_latency_first(is_latency_first) + .set_latency_first(packet_policy.latency_first) .set_exit_node(is_exit_node); let mut errs: Vec = vec![]; @@ -2449,9 +2554,9 @@ impl PeerOutboundPacketRouter { let should_mark_recent_traffic = should_mark_recent_traffic_for_fanout(total_dst_peers); for (i, peer_id) in dst_peers.iter().enumerate() { if should_mark_recent_traffic { - self.mark_recent_traffic(*peer_id); + self.mark_recent_traffic_with_policy(*peer_id, packet_policy); } - if let Err(e) = self.check_p2p_only_before_send(*peer_id) { + if let Err(e) = self.check_p2p_only_before_send(*peer_id, packet_policy) { errs.push(e); continue; } @@ -2523,7 +2628,7 @@ pub(crate) struct PeerPacketRouter { packet_recv: PacketRecvChanReceiver, my_peer_id: PeerId, peers: Arc, - peer_packet_process_pipeline: Arc>>>, + peer_packet_process_pipeline: PeerPacketPipeline, foreign_client: Arc, relay_peer_map: Arc, foreign_network_manager: Arc, @@ -2545,7 +2650,7 @@ impl PeerPacketRouter { packet_recv: PacketRecvChanReceiver, my_peer_id: PeerId, peers: Arc, - peer_packet_process_pipeline: Arc>>>, + peer_packet_process_pipeline: PeerPacketPipeline, foreign_client: Arc, relay_peer_map: Arc, foreign_network_manager: Arc, @@ -2786,12 +2891,9 @@ impl PeerPacketRouter { let mut processed = false; let mut zc_packet = Some(ret); tracing::trace!(?zc_packet, "try_process_packet_from_peer"); - for pipeline in self.peer_packet_process_pipeline.read().await.iter().rev() { - if !pipeline.active.load(Ordering::Acquire) { - continue; - } - let filter = pipeline.filter.read().clone(); - if let Some(filter) = filter { + let pipelines = self.peer_packet_process_pipeline.load_full(); + for pipeline in pipelines.iter().rev() { + if let Some(filter) = pipeline.filter_if_active() { zc_packet = filter .try_process_packet_from_peer(zc_packet.unwrap()) .await; @@ -3099,17 +3201,15 @@ mod tests { use crate::{ config::runtime::CoreRuntimeConfig, config::{CoreConfig, IpPrefix, NetworkIdentity, NodeConfig, ProxyNetworkConfig}, - peers::{ - context::{PeerContext, PeerEvent}, - create_packet_recv_chan, - }, + host::packet::{HostPacketSender, host_packet_channel}, + peers::context::{PeerContext, PeerEvent}, proto::common::{PeerFeatureFlag, StunInfo}, }; impl PeerManagerCore { pub(crate) fn new_portable_for_test( config: PortablePeerManagerConfig, - nic_channel: PacketRecvChan, + nic_channel: HostPacketSender, ) -> anyhow::Result { let runtime_config = CoreRuntimeConfigStore::new( CoreRuntimeConfig::default(), @@ -3182,40 +3282,35 @@ mod tests { } } - #[tokio::test] - async fn managed_nic_pipeline_removal_waits_for_readers_and_drops_filter() { + #[test] + fn managed_nic_pipeline_removal_preserves_in_flight_snapshot() { let drops = Arc::new(AtomicUsize::new(0)); + let pipeline = Arc::new(ArcSwap::from_pointee(Vec::new())); let (entry, registration) = - managed_nic_pipeline_entry(Box::new(DropCountingNicFilter(drops.clone()))); - let pipeline = Arc::new(RwLock::new(vec![entry])); - let reader = pipeline.read().await; - let active_filter = reader[0].filter.read().clone().unwrap(); - let remove_pipeline = pipeline.clone(); - let remove_registration = registration.clone(); - let removal = tokio::spawn(async move { - remove_managed_nic_pipeline_entry(&remove_pipeline, &remove_registration).await; - }); + managed_nic_pipeline_entry(Box::new(DropCountingNicFilter(drops.clone())), &pipeline); + append_nic_pipeline(&pipeline, entry); + let reader = pipeline.load_full(); - tokio::task::yield_now().await; - assert!(!removal.is_finished()); + registration.close(); + + assert!(pipeline.load().is_empty()); assert_eq!(drops.load(Ordering::Relaxed), 0); drop(reader); - drop(active_filter); - removal.await.unwrap(); - assert!(pipeline.read().await.is_empty()); assert_eq!(drops.load(Ordering::Relaxed), 1); } #[test] fn managed_pipeline_guard_releases_filter_without_a_runtime() { let drops = Arc::new(AtomicUsize::new(0)); + let pipeline = Arc::new(ArcSwap::from_pointee(Vec::new())); let (entry, registration) = - managed_nic_pipeline_entry(Box::new(DropCountingNicFilter(drops.clone()))); + managed_nic_pipeline_entry(Box::new(DropCountingNicFilter(drops.clone())), &pipeline); + append_nic_pipeline(&pipeline, entry); drop(registration); - assert!(entry.filter.read().is_none()); + assert!(pipeline.load().is_empty()); assert_eq!(drops.load(Ordering::Relaxed), 1); } @@ -3258,7 +3353,7 @@ mod tests { fn build_portable_config_for_test( config: PortablePeerManagerConfig, ) -> anyhow::Result { - let (packet_tx, _packet_rx) = create_packet_recv_chan(); + let (packet_tx, _packet_rx) = host_packet_channel(); PeerManagerCore::new_portable_for_test(config, packet_tx) } @@ -3305,7 +3400,11 @@ mod tests { ); } - #[cfg(not(feature = "aes-gcm"))] + #[cfg(not(any( + feature = "aes-gcm", + feature = "openssl-crypto", + feature = "ring-crypto" + )))] #[tokio::test] async fn portable_peer_manager_rejects_requested_unavailable_aes() { let mut config = PortablePeerManagerConfig::new(portable_runtime_config("portable-net")); @@ -3382,7 +3481,7 @@ mod tests { let public_ipv6_runtime = CorePublicIpv6Runtime::new(runtime_config.clone(), Arc::new(()), Arc::new(())); let events = Arc::new(CountingPeerEventSink::default()); - let (packet_tx, _packet_rx) = create_packet_recv_chan(); + let (packet_tx, _packet_rx) = host_packet_channel(); let core = PeerManagerCore::new( config, @@ -3417,7 +3516,7 @@ mod tests { }), }; config.snapshot.set_acl_groups(Some(&acl)); - let (packet_tx, _packet_rx) = create_packet_recv_chan(); + let (packet_tx, _packet_rx) = host_packet_channel(); let core = PeerManagerCore::new_portable_for_test(config, packet_tx).unwrap(); @@ -3563,7 +3662,7 @@ mod tests { async fn portable_peer_manager_rejects_inconsistent_network_names() { let mut runtime = portable_runtime_config("identity-net"); runtime.core.node.network_name = "node-net".to_owned(); - let (packet_tx, _packet_rx) = create_packet_recv_chan(); + let (packet_tx, _packet_rx) = host_packet_channel(); let result = PeerManagerCore::new_portable_for_test( PortablePeerManagerConfig::new(runtime), diff --git a/easytier-core/src/peers/traffic_metrics.rs b/easytier-core/src/peers/traffic_metrics.rs index 810ab797..be1f3238 100644 --- a/easytier-core/src/peers/traffic_metrics.rs +++ b/easytier-core/src/peers/traffic_metrics.rs @@ -37,6 +37,13 @@ enum CachedPeerTrafficCounters { } impl CachedPeerTrafficCounters { + fn add_sample(&self, bytes: u64) { + match self { + CachedPeerTrafficCounters::Unknown(counters) + | CachedPeerTrafficCounters::Resolved(counters) => counters.add_sample(bytes), + } + } + fn counters(&self) -> TrafficCounters { match self { CachedPeerTrafficCounters::Unknown(counters) @@ -95,7 +102,7 @@ impl LogicalTrafficMetrics { if let Some(entry) = self.per_peer.get(&peer_id) && entry.value().is_resolved() { - entry.value().counters().add_sample(bytes); + entry.value().add_sample(bytes); return; } diff --git a/easytier-core/src/socket/tcp.rs b/easytier-core/src/socket/tcp.rs index 52588b12..57a5e3c9 100644 --- a/easytier-core/src/socket/tcp.rs +++ b/easytier-core/src/socket/tcp.rs @@ -6,12 +6,28 @@ use tokio::io::{AsyncRead, AsyncWrite}; use crate::socket::{IpVersion, SocketContext, SocketListener}; +pub type VirtualTcpReadHalf = Box; +pub type VirtualTcpWriteHalf = Box; +pub type VirtualTcpSplit = (VirtualTcpReadHalf, VirtualTcpWriteHalf); + /// A core-visible TCP stream endpoint. /// /// Implementations are runtime adapters over concrete TCP stream types. This /// trait deliberately stays below tunnel framing: it only exposes stream I/O and /// socket addresses. pub trait VirtualTcpSocket: AsyncRead + AsyncWrite + Unpin + Send + 'static { + /// Consumes the stream into independently owned read and write halves. + /// + /// Portable adapters may use the generic shared split. Native adapters + /// should override this when their runtime provides lock-free owned halves. + fn into_split(self) -> VirtualTcpSplit + where + Self: Sized, + { + let (reader, writer) = tokio::io::split(self); + (Box::new(reader), Box::new(writer)) + } + fn local_addr(&self) -> io::Result; fn peer_addr(&self) -> io::Result; diff --git a/easytier-core/src/socket/udp/layer.rs b/easytier-core/src/socket/udp/layer.rs index 9aec1ea6..a5279139 100644 --- a/easytier-core/src/socket/udp/layer.rs +++ b/easytier-core/src/socket/udp/layer.rs @@ -18,8 +18,9 @@ use super::{ UDP_SESSION_CONNECT_TIMEOUT, UDP_SESSION_QUEUE_CAPACITY, UDP_SESSION_RESEND_INTERVAL, packet::{ EasyTierUdpPacketKind, UdpDatagramClassification, UdpSessionPacketKind, - classify_udp_datagram, extract_dst_addr_from_v4_hole_punch_packet, - extract_v6_hole_punch_packet, new_sack_packet, new_syn_packet, + classify_session_udp_datagram, classify_udp_datagram, + extract_dst_addr_from_v4_hole_punch_packet, extract_v6_hole_punch_packet, new_sack_packet, + new_syn_packet, }, session::{ ClassifiedUdpSessionAccept, ClassifiedUdpSessionAccepts, ClassifiedUdpSessionKey, @@ -32,8 +33,9 @@ use super::{ dispatch_payload_to_session, udp_session_registry_entry, }, virtual_socket::{ - NoopUdpSessionStunResponder, PreferredIpv6Source, UdpSessionStunResponder, - UdpSocketRecvMeta, UdpSocketSendMeta, VirtualUdpSocket, VirtualUdpSocketFactory, + MAX_UDP_SESSION_DATAGRAM_SIZE, NoopUdpSessionStunResponder, PreferredIpv6Source, + UdpSessionStunResponder, UdpSocketRecvMeta, UdpSocketSendMeta, VirtualUdpSocket, + VirtualUdpSocketFactory, }, }; @@ -443,11 +445,10 @@ pub(super) async fn udp_session_layer_recv_task( S: VirtualUdpSocket, R: UdpSessionStunResponder, { - let mut buf = [0u8; 65535]; let control_permits = Arc::new(Semaphore::new(UDP_SESSION_QUEUE_CAPACITY)); loop { - let (len, remote_addr, recv_meta) = match socket.recv_from_with_meta(&mut buf).await { - Ok(ret) => ret, + let datagram = match socket.recv_datagram().await { + Ok(datagram) => datagram, Err(err) => { tracing::debug!(?err, "udp session recv loop stopped"); let _ = session_shutdown_tx.send(true); @@ -457,12 +458,25 @@ pub(super) async fn udp_session_layer_recv_task( break; } }; - - let payload = BytesMut::from(&buf[..len]); - let datagram = UdpSessionDatagram::new(payload.clone(), recv_meta); + let payload = datagram.payload; + let remote_addr = datagram.remote_addr; + let recv_meta = datagram.meta; + if payload.len() > MAX_UDP_SESSION_DATAGRAM_SIZE { + tracing::debug!( + datagram_len = payload.len(), + max_datagram_len = MAX_UDP_SESSION_DATAGRAM_SIZE, + ?remote_addr, + "dropping oversized udp session datagram" + ); + continue; + } let quic_key = ClassifiedUdpSessionKey::new(UdpSessionProtocol::Quic, remote_addr); if classified_sessions.contains_key(&quic_key) { - dispatch_existing_classified_udp_datagram(&classified_sessions, quic_key, datagram); + dispatch_existing_classified_udp_datagram( + &classified_sessions, + quic_key, + UdpSessionDatagram::new(payload, recv_meta), + ); continue; } match classify_udp_datagram(payload) { @@ -500,23 +514,24 @@ pub(super) async fn udp_session_layer_recv_task( kind, conn_id, packet, - fallback, } => { - let consumed = dispatch_easy_tier_udp_datagram( - socket.clone(), + let unconsumed = dispatch_easy_tier_udp_datagram( + &socket, &sessions, &pending_connects, &mux_accepted, &control, - control_permits.clone(), + &control_permits, remote_addr, kind, conn_id, - &packet, + packet, recv_meta, - session_shutdown_tx.subscribe(), + &session_shutdown_tx, ); - if !consumed { + if let Some(packet) = unconsumed { + let datagram = BytesMut::from(packet.into_bytes()); + let fallback = classify_session_udp_datagram(&datagram); dispatch_session_udp_datagram( socket.clone(), &classified_sessions, @@ -524,7 +539,7 @@ pub(super) async fn udp_session_layer_recv_task( session_shutdown_tx.subscribe(), remote_addr, fallback, - UdpSessionDatagram::new(packet.into_bytes().into(), recv_meta), + UdpSessionDatagram::new(datagram, recv_meta), ); } } @@ -537,14 +552,14 @@ fn dispatch_existing_classified_udp_datagram( key: ClassifiedUdpSessionKey, datagram: UdpSessionDatagram, ) { - let Some(entry) = classified_sessions - .get(&key) - .map(|entry| entry.value().clone()) - else { + let Some(entry) = classified_sessions.get(&key) else { return; }; - if !dispatch_payload_to_session(&entry.incoming, datagram, UdpSessionEnqueuePolicy::Reliable) { + let dispatched = + dispatch_payload_to_session(&entry.incoming, datagram, UdpSessionEnqueuePolicy::Reliable); + drop(entry); + if !dispatched { close_classified_udp_session(classified_sessions, key); tracing::debug!(?key, "classified udp session data queue closed"); } @@ -552,73 +567,84 @@ fn dispatch_existing_classified_udp_datagram( #[allow(clippy::too_many_arguments)] fn dispatch_easy_tier_udp_datagram( - socket: Arc, + socket: &Arc, sessions: &Arc, pending_connects: &Arc, mux_accepted: &mpsc::Sender, control: &mpsc::Sender, - control_permits: Arc, + control_permits: &Arc, remote_addr: SocketAddr, kind: EasyTierUdpPacketKind, conn_id: u32, - packet: &ZCPacket, + packet: ZCPacket, recv_meta: UdpSocketRecvMeta, - session_shutdown: watch::Receiver, -) -> bool + session_shutdown: &watch::Sender, +) -> Option where S: VirtualUdpSocket, { - match kind { + let consumed = match kind { EasyTierUdpPacketKind::Data => { - dispatch_data_packet(sessions, remote_addr, conn_id, packet, recv_meta) + return dispatch_data_packet(sessions, remote_addr, conn_id, packet, recv_meta).err(); } EasyTierUdpPacketKind::Syn => handle_new_easy_tier_mux_connect( - socket, + socket.clone(), sessions.clone(), mux_accepted.clone(), remote_addr, conn_id, - packet, - session_shutdown, + &packet, + session_shutdown.subscribe(), ), EasyTierUdpPacketKind::Sack => { - dispatch_sack_packet(sessions, pending_connects, remote_addr, conn_id, packet) + dispatch_sack_packet(sessions, pending_connects, remote_addr, conn_id, &packet) } EasyTierUdpPacketKind::HolePunch => { dispatch_hole_punch_packet(pending_connects, remote_addr) } - EasyTierUdpPacketKind::V4HolePunch => { - dispatch_v4_hole_punch_control(socket, control_permits, control, remote_addr, packet) - } - EasyTierUdpPacketKind::V6HolePunch => { - dispatch_v6_hole_punch_control(socket, control_permits, control, remote_addr, packet) - } - } + EasyTierUdpPacketKind::V4HolePunch => dispatch_v4_hole_punch_control( + socket.clone(), + control_permits.clone(), + control, + remote_addr, + &packet, + ), + EasyTierUdpPacketKind::V6HolePunch => dispatch_v6_hole_punch_control( + socket.clone(), + control_permits.clone(), + control, + remote_addr, + &packet, + ), + }; + if consumed { None } else { Some(packet) } } pub(super) fn dispatch_data_packet( sessions: &UdpSessionRegistry, peer_addr: SocketAddr, conn_id: u32, - packet: &ZCPacket, + packet: ZCPacket, recv_meta: UdpSocketRecvMeta, -) -> bool { +) -> Result<(), ZCPacket> { let key = UdpSessionKey::new(peer_addr, conn_id); - let Some(entry) = sessions.get(&key).map(|entry| entry.value().clone()) else { - return false; + let Some(entry) = sessions.get(&key) else { + return Err(packet); }; - let payload = UdpSessionDatagram::new(BytesMut::from(packet.udp_payload()), recv_meta); let policy = if packet.is_lossy() { UdpSessionEnqueuePolicy::Lossy } else { UdpSessionEnqueuePolicy::Reliable }; - if !dispatch_payload_to_session(&entry.incoming, payload, policy) { + let payload = UdpSessionDatagram::from_easytier_packet(packet, recv_meta); + let dispatched = dispatch_payload_to_session(&entry.incoming, payload, policy); + drop(entry); + if !dispatched { close_udp_session(sessions, key); tracing::debug!(?key, "udp session data queue closed"); } - true + Ok(()) } fn dispatch_session_udp_datagram( @@ -660,15 +686,14 @@ fn dispatch_classified_udp_datagram( S: VirtualUdpSocket, { let key = ClassifiedUdpSessionKey::new(protocol, remote_addr); - if let Some(entry) = classified_sessions - .get(&key) - .map(|entry| entry.value().clone()) - { - if !dispatch_payload_to_session( + if let Some(entry) = classified_sessions.get(&key) { + let dispatched = dispatch_payload_to_session( &entry.incoming, datagram, UdpSessionEnqueuePolicy::Reliable, - ) { + ); + drop(entry); + if !dispatched { close_classified_udp_session(classified_sessions, key); tracing::debug!(?key, "classified udp session data queue closed"); } @@ -712,11 +737,12 @@ fn dispatch_classified_udp_datagram( } dashmap::mapref::entry::Entry::Occupied(entry) => { let entry = entry.get().clone(); - if !dispatch_payload_to_session( + let dispatched = dispatch_payload_to_session( &entry.incoming, datagram, UdpSessionEnqueuePolicy::Reliable, - ) { + ); + if !dispatched { close_classified_udp_session(classified_sessions, key); tracing::debug!(?key, "classified udp session data queue closed"); } diff --git a/easytier-core/src/socket/udp/mod.rs b/easytier-core/src/socket/udp/mod.rs index cf181c02..dd708c98 100644 --- a/easytier-core/src/socket/udp/mod.rs +++ b/easytier-core/src/socket/udp/mod.rs @@ -29,7 +29,8 @@ pub(crate) use session::{ UdpSessionTunnelParts, }; pub use virtual_socket::{ - NoopUdpSessionStunResponder, PreferredIpv6Source, UdpBindOptions, UdpSessionStunResponder, + MAX_UDP_DATAGRAM_SIZE, MAX_UDP_SESSION_DATAGRAM_SIZE, NoopUdpSessionStunResponder, + PreferredIpv6Source, UdpBindOptions, UdpSessionStunResponder, UdpSocketDatagram, UdpSocketPurpose, UdpSocketRecvMeta, UdpSocketSendMeta, VirtualUdpSocket, VirtualUdpSocketFactory, send_v4_hole_punch_control_packet, send_v6_hole_punch_control_packet, }; diff --git a/easytier-core/src/socket/udp/packet.rs b/easytier-core/src/socket/udp/packet.rs index 8b101a46..33f4d5ef 100644 --- a/easytier-core/src/socket/udp/packet.rs +++ b/easytier-core/src/socket/udp/packet.rs @@ -161,7 +161,6 @@ pub(super) enum UdpDatagramClassification { kind: EasyTierUdpPacketKind, conn_id: u32, packet: ZCPacket, - fallback: UdpSessionPacketKind, }, SessionPacket { kind: UdpSessionPacketKind, @@ -216,7 +215,7 @@ pub(super) enum EasyTierUdpDatagramInspectError { }, } -fn classify_session_udp_datagram(data: &[u8]) -> UdpSessionPacketKind { +pub(super) fn classify_session_udp_datagram(data: &[u8]) -> UdpSessionPacketKind { if is_wireguard_packet(data) { UdpSessionPacketKind::Classified(UdpSessionProtocol::WireGuard) } else if is_quic_packet(data) { @@ -335,12 +334,11 @@ pub(super) fn classify_udp_datagram(datagram: BytesMut) -> UdpDatagramClassifica return UdpDatagramClassification::Stun(datagram); } - let fallback = classify_session_udp_datagram(&datagram); let easytier = match inspect_easytier_udp_datagram(&datagram) { Ok(Some(easytier)) => easytier, Ok(None) => { return UdpDatagramClassification::SessionPacket { - kind: fallback, + kind: classify_session_udp_datagram(&datagram), datagram, }; } @@ -361,7 +359,7 @@ pub(super) fn classify_udp_datagram(datagram: BytesMut) -> UdpDatagramClassifica } } return UdpDatagramClassification::SessionPacket { - kind: fallback, + kind: classify_session_udp_datagram(&datagram), datagram, }; } @@ -372,7 +370,6 @@ pub(super) fn classify_udp_datagram(datagram: BytesMut) -> UdpDatagramClassifica kind: easytier.kind, conn_id: easytier.conn_id, packet, - fallback, } } diff --git a/easytier-core/src/socket/udp/session.rs b/easytier-core/src/socket/udp/session.rs index 36874ab8..4bc3f523 100644 --- a/easytier-core/src/socket/udp/session.rs +++ b/easytier-core/src/socket/udp/session.rs @@ -14,33 +14,63 @@ use tokio::{ task::JoinHandle, }; -use crate::socket::ring::{RingSocket, RingSocketReceiver, RingSocketSendError, RingSocketSender}; +use crate::{ + packet::{UdpPacketType, ZCPacket, ZCPacketType}, + socket::ring::{RingSocket, RingSocketReceiver, RingSocketSendError, RingSocketSender}, +}; use super::{ - UDP_SESSION_QUEUE_CAPACITY, + MAX_UDP_SESSION_DATAGRAM_SIZE, UDP_SESSION_QUEUE_CAPACITY, packet::{new_data_packet, udp_session_payload_len}, virtual_socket::{PreferredIpv6Source, UdpBindOptions, UdpSocketRecvMeta, VirtualUdpSocket}, }; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug)] pub(crate) struct UdpSessionDatagram { - pub(crate) payload: BytesMut, + payload: UdpSessionDatagramPayload, pub(crate) dst_ip: Option, } +#[derive(Debug)] +enum UdpSessionDatagramPayload { + Bytes(BytesMut), + EasyTierPacket(ZCPacket), +} + impl UdpSessionDatagram { pub(crate) fn new(payload: BytesMut, meta: UdpSocketRecvMeta) -> Self { Self { - payload, + payload: UdpSessionDatagramPayload::Bytes(payload), dst_ip: meta.dst_ip, } } + + pub(crate) fn from_easytier_packet(packet: ZCPacket, meta: UdpSocketRecvMeta) -> Self { + Self { + payload: UdpSessionDatagramPayload::EasyTierPacket(packet), + dst_ip: meta.dst_ip, + } + } + + pub(crate) fn payload(&self) -> &[u8] { + match &self.payload { + UdpSessionDatagramPayload::Bytes(payload) => payload, + UdpSessionDatagramPayload::EasyTierPacket(packet) => packet.udp_payload(), + } + } + + pub(crate) fn into_tunnel_packet(self) -> Result { + match self.payload { + UdpSessionDatagramPayload::Bytes(payload) => Err(payload), + UdpSessionDatagramPayload::EasyTierPacket(packet) => Ok(packet), + } + } } impl From for UdpSessionDatagram { fn from(payload: BytesMut) -> Self { Self { - payload, + payload: UdpSessionDatagramPayload::Bytes(payload), dst_ip: None, } } @@ -245,9 +275,12 @@ pub struct UdpSession { pub(super) _cleanup: UdpSessionCleanup, } -pub(crate) struct UdpSessionOutbound { - pub(crate) payload: BytesMut, - pub(crate) completion: oneshot::Sender>, +pub(crate) enum UdpSessionOutbound { + Datagram { + payload: BytesMut, + completion: oneshot::Sender>, + }, + TunnelPacket(ZCPacket), } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -258,13 +291,38 @@ pub(crate) enum UdpSessionCodec { impl UdpSessionCodec { pub(crate) fn validate_payload(&self, payload: &[u8]) -> io::Result<()> { + self.validate_datagram_size(payload.len())?; if matches!(self, Self::EasyTierData { .. }) { udp_session_payload_len(payload)?; } Ok(()) } + fn validate_datagram_size(&self, payload_len: usize) -> io::Result<()> { + let header_len = match self { + Self::EasyTierData { .. } => crate::packet::UDP_TUNNEL_HEADER_SIZE, + Self::Identity => 0, + }; + let datagram_len = payload_len.checked_add(header_len).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "udp session datagram size overflow", + ) + })?; + if datagram_len > MAX_UDP_SESSION_DATAGRAM_SIZE { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "udp session datagram too large: {datagram_len}, max: \ + {MAX_UDP_SESSION_DATAGRAM_SIZE}" + ), + )); + } + Ok(()) + } + fn encode(&self, payload: &[u8]) -> io::Result { + self.validate_payload(payload)?; match self { Self::EasyTierData { conn_id } => { Ok(new_data_packet(*conn_id, payload)?.into_bytes().into()) @@ -272,6 +330,23 @@ impl UdpSessionCodec { Self::Identity => Ok(BytesMut::from(payload)), } } + + fn encode_tunnel_packet(&self, packet: ZCPacket) -> io::Result { + let mut packet = packet.convert_type(ZCPacketType::UDP); + self.validate_datagram_size(packet.udp_payload().len())?; + match self { + Self::EasyTierData { conn_id } => { + let payload_len = udp_session_payload_len(packet.udp_payload())?; + let header = packet.mut_udp_tunnel_header().unwrap(); + header.conn_id.set(*conn_id); + header.msg_type = UdpPacketType::Data as u8; + header.padding = 0; + header.len.set(payload_len); + Ok(packet.into_bytes()) + } + Self::Identity => Ok(packet.udp_payload_bytes().freeze()), + } + } } #[derive(Clone)] @@ -387,8 +462,12 @@ impl UdpSession { peer_addr, codec, rings.session_send_rx, + close.clone(), + )); + let shutdown_task = tokio::spawn(close_udp_session_on_shutdown( shutdown, close.clone(), + send_task.abort_handle(), )); Self { @@ -402,7 +481,7 @@ impl UdpSession { _cleanup: UdpSessionCleanup { session_close: Some(close), shutdown: None, - tasks: vec![send_task], + tasks: vec![send_task, shutdown_task], layer_guard: None, }, } @@ -471,7 +550,7 @@ impl UdpSessionSocket for UdpSession { return Err(udp_session_closed_error()); } let (completion, sent) = oneshot::channel(); - let outbound = UdpSessionOutbound { + let outbound = UdpSessionOutbound::Datagram { payload: BytesMut::from(data), completion, }; @@ -508,8 +587,8 @@ impl UdpSessionSocket for UdpSession { .ok_or_else(udp_session_closed_error)? .map_err(ring_socket_error_to_io)?, }; - let len = payload.payload.len().min(buf.len()); - buf[..len].copy_from_slice(&payload.payload[..len]); + let len = payload.payload().len().min(buf.len()); + buf[..len].copy_from_slice(&payload.payload()[..len]); Ok(( len, UdpSessionRecvMeta { @@ -618,56 +697,86 @@ async fn forward_udp_session_to_socket( peer_addr: SocketAddr, codec: UdpSessionCodec, mut outgoing: RingSocketReceiver, - mut shutdown: watch::Receiver, close: UdpSessionClose, ) where S: VirtualUdpSocket, { loop { - tokio::select! { - biased; - _ = shutdown.changed() => { + let Some(outbound) = outgoing.next().await else { + break; + }; + let outbound = match outbound { + Ok(outbound) => outbound, + Err(err) => { + tracing::debug!(?err, ?peer_addr, "udp session outgoing ring closed"); close.close(); break; } - outbound = outgoing.next() => { - let Some(outbound) = outbound else { - break; - }; - let outbound = match outbound { - Ok(outbound) => outbound, + }; + let (datagram, completion) = match outbound { + UdpSessionOutbound::Datagram { + payload, + completion, + } => { + let payload_len = payload.len(); + let datagram = match codec.encode(&payload) { + Ok(datagram) => datagram.freeze(), Err(err) => { - tracing::debug!(?err, ?peer_addr, "udp session outgoing ring closed"); + tracing::debug!( + ?err, + ?peer_addr, + ?codec, + "udp session datagram encode error" + ); + let _ = completion.send(Err(err)); close.close(); break; } }; - let payload_len = outbound.payload.len(); - let datagram = match codec.encode(&outbound.payload) { + (datagram, Some((completion, payload_len))) + } + UdpSessionOutbound::TunnelPacket(packet) => { + let datagram = match codec.encode_tunnel_packet(packet) { Ok(datagram) => datagram, Err(err) => { - tracing::debug!(?err, ?peer_addr, ?codec, "udp session datagram encode error"); - let _ = outbound.completion.send(Err(err)); + tracing::debug!(?err, ?peer_addr, ?codec, "udp tunnel packet encode error"); close.close(); break; } }; - match socket.send_to(&datagram, peer_addr).await { - Ok(_) => { - let _ = outbound.completion.send(Ok(payload_len)); - } - Err(err) => { - tracing::debug!(?err, ?peer_addr, "udp session send error"); - let _ = outbound.completion.send(Err(err)); - close.close(); - break; - } + (datagram, None) + } + }; + match socket.send_to(&datagram, peer_addr).await { + Ok(_) => { + if let Some((completion, payload_len)) = completion { + let _ = completion.send(Ok(payload_len)); } } + Err(err) => { + tracing::debug!(?err, ?peer_addr, "udp session send error"); + if let Some((completion, _)) = completion { + let _ = completion.send(Err(err)); + } + close.close(); + break; + } } } } +async fn close_udp_session_on_shutdown( + mut shutdown: watch::Receiver, + close: UdpSessionClose, + send_task: tokio::task::AbortHandle, +) { + if !*shutdown.borrow() { + let _ = shutdown.changed().await; + } + close.close(); + send_task.abort(); +} + pub(super) fn dispatch_payload_to_session( incoming: &Arc>>, payload: impl Into, diff --git a/easytier-core/src/socket/udp/tests.rs b/easytier-core/src/socket/udp/tests.rs index 694b30a2..301932d5 100644 --- a/easytier-core/src/socket/udp/tests.rs +++ b/easytier-core/src/socket/udp/tests.rs @@ -945,6 +945,30 @@ async fn udp_layer_routes_quic_like_easytier_packet_to_existing_quic_session() { assert_eq!(layer.active_classified_session_count(), 1); } +#[tokio::test] +async fn udp_layer_drops_oversized_datagram_without_stopping_portable_socket() { + let local_addr = SocketAddr::from(([127, 0, 0, 1], 12000)); + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 12001)); + let socket = Arc::new(AutoSackVirtualUdpSocket::new(local_addr)); + let layer = UdpSessionLayer::new(socket.clone()); + let session = layer + .open_classified_session(UdpSessionProtocol::Quic, peer_addr) + .unwrap(); + socket.incoming.lock().unwrap().extend([ + (vec![0xAA; MAX_UDP_SESSION_DATAGRAM_SIZE + 1], peer_addr), + (b"after-oversized".to_vec(), peer_addr), + ]); + socket.incoming_notify.notify_one(); + + let mut buf = [0; 32]; + let len = tokio::time::timeout(Duration::from_secs(1), session.recv(&mut buf)) + .await + .unwrap() + .unwrap(); + + assert_eq!(&buf[..len], b"after-oversized"); +} + #[tokio::test] async fn udp_layer_keeps_easy_tier_syn_out_of_wireguard_session() { let local_addr = SocketAddr::from(([127, 0, 0, 1], 12000)); @@ -1021,13 +1045,37 @@ async fn easy_tier_mux_udp_session_rejects_oversized_payload_before_enqueue() { sessions, ); - let payload = vec![0; u16::MAX as usize + 1]; + let payload = vec![0; MAX_UDP_SESSION_DATAGRAM_SIZE - UDP_TUNNEL_HEADER_SIZE + 1]; let err = session.send(&payload).await.unwrap_err(); assert_eq!(err.kind(), io::ErrorKind::InvalidInput); assert!(socket.sent().is_empty()); } +#[test] +fn udp_session_codecs_enforce_datagram_boundary() { + let identity = UdpSessionCodec::Identity; + assert!( + identity + .validate_payload(&vec![0; MAX_UDP_SESSION_DATAGRAM_SIZE]) + .is_ok() + ); + let err = identity + .validate_payload(&vec![0; MAX_UDP_SESSION_DATAGRAM_SIZE + 1]) + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + + let easy_tier = UdpSessionCodec::EasyTierData { + conn_id: 0x1122_3344, + }; + let max_payload = MAX_UDP_SESSION_DATAGRAM_SIZE - UDP_TUNNEL_HEADER_SIZE; + assert!(easy_tier.validate_payload(&vec![0; max_payload]).is_ok()); + let err = easy_tier + .validate_payload(&vec![0; max_payload + 1]) + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); +} + #[tokio::test] async fn easy_tier_mux_udp_session_send_failure_closes_session() { let local_addr = SocketAddr::from(([127, 0, 0, 1], 12000)); @@ -1074,26 +1122,35 @@ async fn easy_tier_mux_udp_session_receives_only_peer_data_payloads() { sessions.clone(), ); - dispatch_data_packet( - &sessions, - unexpected_addr, - conn_id, - &new_data_packet(conn_id, b"wrong-peer").unwrap(), - Default::default(), + assert!( + dispatch_data_packet( + &sessions, + unexpected_addr, + conn_id, + new_data_packet(conn_id, b"wrong-peer").unwrap(), + Default::default(), + ) + .is_err() ); - dispatch_data_packet( - &sessions, - peer_addr, - conn_id + 1, - &new_data_packet(conn_id + 1, b"wrong-conn").unwrap(), - Default::default(), + assert!( + dispatch_data_packet( + &sessions, + peer_addr, + conn_id + 1, + new_data_packet(conn_id + 1, b"wrong-conn").unwrap(), + Default::default(), + ) + .is_err() ); - dispatch_data_packet( - &sessions, - peer_addr, - conn_id, - &new_data_packet(conn_id, b"payload").unwrap(), - Default::default(), + assert!( + dispatch_data_packet( + &sessions, + peer_addr, + conn_id, + new_data_packet(conn_id, b"payload").unwrap(), + Default::default(), + ) + .is_ok() ); let mut buf = [0; 16]; @@ -1163,6 +1220,26 @@ async fn dropping_udp_session_layer_closes_session_recv() { assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof); } +#[tokio::test] +async fn idle_udp_session_closes_when_shutdown_is_signaled() { + let local_addr = SocketAddr::from(([127, 0, 0, 1], 12000)); + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 12001)); + let key = UdpSessionKey::new(peer_addr, 0x1122_3344); + let socket = Arc::new(MockVirtualUdpSocket::new(local_addr, Vec::new())); + let sessions = Arc::new(DashMap::new()); + let (session, shutdown_tx) = create_test_easy_tier_mux_session(socket, key, sessions.clone()); + + shutdown_tx.send(true).unwrap(); + + let mut buf = [0; 16]; + let err = tokio::time::timeout(Duration::from_secs(1), session.recv(&mut buf)) + .await + .unwrap() + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof); + assert!(!sessions.contains_key(&key)); +} + #[tokio::test] async fn udp_session_recv_loop_error_closes_registered_sessions() { let local_addr = SocketAddr::from(([127, 0, 0, 1], 12000)); @@ -1591,12 +1668,15 @@ async fn sack_from_actual_remote_rekeys_pending_session_before_data_dispatch() { }, ); - dispatch_data_packet( - &sessions, - expected_addr, - conn_id, - &new_data_packet(conn_id, b"pre-sack").unwrap(), - Default::default(), + assert!( + dispatch_data_packet( + &sessions, + expected_addr, + conn_id, + new_data_packet(conn_id, b"pre-sack").unwrap(), + Default::default(), + ) + .is_err() ); dispatch_sack_packet( &sessions, @@ -1605,12 +1685,15 @@ async fn sack_from_actual_remote_rekeys_pending_session_before_data_dispatch() { conn_id, &new_sack_packet(conn_id, magic), ); - dispatch_data_packet( - &sessions, - actual_addr, - conn_id, - &new_data_packet(conn_id, b"payload").unwrap(), - Default::default(), + assert!( + dispatch_data_packet( + &sessions, + actual_addr, + conn_id, + new_data_packet(conn_id, b"payload").unwrap(), + Default::default(), + ) + .is_ok() ); assert!(sessions.contains_key(&actual_key)); @@ -1624,7 +1707,7 @@ async fn sack_from_actual_remote_rekeys_pending_session_before_data_dispatch() { .await .unwrap() .unwrap(); - assert_eq!(payload.payload, BytesMut::from(&b"payload"[..])); + assert_eq!(payload.payload(), b"payload"); } #[tokio::test] diff --git a/easytier-core/src/socket/udp/virtual_socket.rs b/easytier-core/src/socket/udp/virtual_socket.rs index 9ace385c..cb0a9d35 100644 --- a/easytier-core/src/socket/udp/virtual_socket.rs +++ b/easytier-core/src/socket/udp/virtual_socket.rs @@ -5,6 +5,7 @@ use std::{ }; use async_trait::async_trait; +use bytes::BytesMut; use serde::{Deserialize, Serialize}; use crate::socket::{IpVersion, SocketContext}; @@ -22,6 +23,23 @@ pub struct UdpSocketSendMeta { pub src_ifindex: Option, } +/// Largest UDP datagram that portable socket implementations must receive. +pub const MAX_UDP_DATAGRAM_SIZE: usize = u16::MAX as usize; + +/// Largest datagram accepted by the UDP session/multiplexer data plane. +/// +/// EasyTier, WireGuard, and QUIC datagrams are bounded by their transport MTU. +/// Keeping this capacity explicit avoids allocating the theoretical UDP maximum +/// for every packet on native hosts that can detect truncation. +pub const MAX_UDP_SESSION_DATAGRAM_SIZE: usize = 8 * 1024; + +#[derive(Debug)] +pub struct UdpSocketDatagram { + pub payload: BytesMut, + pub remote_addr: SocketAddr, + pub meta: UdpSocketRecvMeta, +} + #[async_trait] pub trait VirtualUdpSocket: Send + Sync + 'static { fn local_addr(&self) -> std::io::Result; @@ -51,6 +69,23 @@ pub trait VirtualUdpSocket: Send + Sync + 'static { let (len, addr) = self.recv_from(buf).await?; Ok((len, addr, UdpSocketRecvMeta::default())) } + + /// Receives one datagram into an owned buffer. + /// + /// Portable hosts can use this default implementation. Native hosts should + /// override it when their socket API can write directly into owned storage, + /// avoiding a second allocation and copy at the Host boundary. + async fn recv_datagram(&self) -> std::io::Result { + let mut payload = BytesMut::new(); + payload.resize(MAX_UDP_DATAGRAM_SIZE, 0); + let (len, remote_addr, meta) = self.recv_from_with_meta(&mut payload).await?; + payload.truncate(len); + Ok(UdpSocketDatagram { + payload, + remote_addr, + meta, + }) + } } #[async_trait] diff --git a/easytier-core/src/tunnel/encrypt/mod.rs b/easytier-core/src/tunnel/encrypt/mod.rs index e6e1348f..8d7da21c 100644 --- a/easytier-core/src/tunnel/encrypt/mod.rs +++ b/easytier-core/src/tunnel/encrypt/mod.rs @@ -2,9 +2,21 @@ use crate::{config::EncryptionAlgorithm, packet::ZCPacket}; use std::{collections::hash_map::DefaultHasher, hash::Hasher, sync::Arc}; #[cfg(feature = "aes-gcm")] +#[cfg_attr( + any(feature = "openssl-crypto", feature = "ring-crypto"), + allow(dead_code) +)] pub mod aes_gcm; #[cfg(feature = "chacha20")] +#[cfg_attr( + any(feature = "openssl-crypto", feature = "ring-crypto"), + allow(dead_code) +)] pub mod chacha20; +#[cfg(feature = "openssl-crypto")] +mod openssl; +#[cfg(all(feature = "ring-crypto", any(not(feature = "openssl-crypto"), test)))] +mod ring; pub mod xor; @@ -119,47 +131,123 @@ fn unavailable_encryptor(algorithm: &str) -> Arc { }) } -fn algorithm_is_available(algorithm: EncryptionAlgorithm) -> bool { +pub(crate) fn algorithm_is_available(algorithm: EncryptionAlgorithm) -> bool { match algorithm { EncryptionAlgorithm::Xor => true, - EncryptionAlgorithm::AesGcm | EncryptionAlgorithm::Aes256Gcm => cfg!(feature = "aes-gcm"), - EncryptionAlgorithm::ChaCha20 => cfg!(feature = "chacha20"), + EncryptionAlgorithm::AesGcm | EncryptionAlgorithm::Aes256Gcm => cfg!(any( + feature = "aes-gcm", + feature = "openssl-crypto", + feature = "ring-crypto" + )), + EncryptionAlgorithm::ChaCha20 => cfg!(any( + feature = "chacha20", + feature = "openssl-crypto", + feature = "ring-crypto" + )), } } +fn is_aead_algorithm(algorithm: EncryptionAlgorithm) -> bool { + matches!( + algorithm, + EncryptionAlgorithm::AesGcm + | EncryptionAlgorithm::Aes256Gcm + | EncryptionAlgorithm::ChaCha20 + ) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AeadBackend { + #[cfg(feature = "openssl-crypto")] + OpenSsl, + #[cfg(all(not(feature = "openssl-crypto"), feature = "ring-crypto"))] + Ring, + #[cfg(all( + not(feature = "openssl-crypto"), + not(feature = "ring-crypto"), + any(feature = "aes-gcm", feature = "chacha20") + ))] + RustCrypto, +} + +#[cfg(feature = "openssl-crypto")] +fn preferred_aead_backend(algorithm: EncryptionAlgorithm) -> Option { + is_aead_algorithm(algorithm).then_some(AeadBackend::OpenSsl) +} + +#[cfg(all(not(feature = "openssl-crypto"), feature = "ring-crypto"))] +fn preferred_aead_backend(algorithm: EncryptionAlgorithm) -> Option { + is_aead_algorithm(algorithm).then_some(AeadBackend::Ring) +} + +#[cfg(all( + not(feature = "openssl-crypto"), + not(feature = "ring-crypto"), + any(feature = "aes-gcm", feature = "chacha20") +))] +fn preferred_aead_backend(algorithm: EncryptionAlgorithm) -> Option { + (is_aead_algorithm(algorithm) && algorithm_is_available(algorithm)) + .then_some(AeadBackend::RustCrypto) +} + +#[cfg(not(any( + feature = "openssl-crypto", + feature = "ring-crypto", + feature = "aes-gcm", + feature = "chacha20" +)))] +fn preferred_aead_backend(_algorithm: EncryptionAlgorithm) -> Option { + None +} + +#[allow(unreachable_patterns)] fn create_aes_128(key: [u8; 16]) -> Arc { - #[cfg(feature = "aes-gcm")] - { - Arc::new(aes_gcm::AesGcmCipher::new_128(key)) - } - #[cfg(not(feature = "aes-gcm"))] - { - let _ = key; - unavailable_encryptor("aes-gcm") + match preferred_aead_backend(EncryptionAlgorithm::AesGcm) { + #[cfg(feature = "openssl-crypto")] + Some(AeadBackend::OpenSsl) => Arc::new(openssl::OpenSslCipher::new_aes128_gcm(key)), + #[cfg(all(not(feature = "openssl-crypto"), feature = "ring-crypto"))] + Some(AeadBackend::Ring) => Arc::new(ring::RingCipher::new_aes128_gcm(key)), + #[cfg(all( + not(feature = "openssl-crypto"), + not(feature = "ring-crypto"), + feature = "aes-gcm" + ))] + Some(AeadBackend::RustCrypto) => Arc::new(aes_gcm::AesGcmCipher::new_128(key)), + _ => unavailable_encryptor("aes-gcm"), } } +#[allow(unreachable_patterns)] fn create_aes_256(key: [u8; 32]) -> Arc { - #[cfg(feature = "aes-gcm")] - { - Arc::new(aes_gcm::AesGcmCipher::new_256(key)) - } - #[cfg(not(feature = "aes-gcm"))] - { - let _ = key; - unavailable_encryptor("aes-256-gcm") + match preferred_aead_backend(EncryptionAlgorithm::Aes256Gcm) { + #[cfg(feature = "openssl-crypto")] + Some(AeadBackend::OpenSsl) => Arc::new(openssl::OpenSslCipher::new_aes256_gcm(key)), + #[cfg(all(not(feature = "openssl-crypto"), feature = "ring-crypto"))] + Some(AeadBackend::Ring) => Arc::new(ring::RingCipher::new_aes256_gcm(key)), + #[cfg(all( + not(feature = "openssl-crypto"), + not(feature = "ring-crypto"), + feature = "aes-gcm" + ))] + Some(AeadBackend::RustCrypto) => Arc::new(aes_gcm::AesGcmCipher::new_256(key)), + _ => unavailable_encryptor("aes-256-gcm"), } } +#[allow(unreachable_patterns)] fn create_chacha20(key: [u8; 32]) -> Arc { - #[cfg(feature = "chacha20")] - { - Arc::new(chacha20::ChaCha20Cipher::new(key)) - } - #[cfg(not(feature = "chacha20"))] - { - let _ = key; - unavailable_encryptor("chacha20") + match preferred_aead_backend(EncryptionAlgorithm::ChaCha20) { + #[cfg(feature = "openssl-crypto")] + Some(AeadBackend::OpenSsl) => Arc::new(openssl::OpenSslCipher::new_chacha20(key)), + #[cfg(all(not(feature = "openssl-crypto"), feature = "ring-crypto"))] + Some(AeadBackend::Ring) => Arc::new(ring::RingCipher::new_chacha20(key)), + #[cfg(all( + not(feature = "openssl-crypto"), + not(feature = "ring-crypto"), + feature = "chacha20" + ))] + Some(AeadBackend::RustCrypto) => Arc::new(chacha20::ChaCha20Cipher::new(key)), + _ => unavailable_encryptor("chacha20"), } } @@ -203,6 +291,28 @@ pub fn create_encryptor( #[cfg(test)] mod tests { use super::*; + use crate::packet::{StandardAeadTail, ZCPacket}; + + fn assert_interoperable(left: &dyn Encryptor, right: &dyn Encryptor) { + let plaintext = b"cross-backend compatibility"; + let nonce = [9; StandardAeadTail::NONCE_SIZE]; + let mut left_packet = ZCPacket::new_with_payload(plaintext); + left_packet.fill_peer_manager_hdr(1, 2, 1); + let mut right_packet = ZCPacket::new_with_payload(plaintext); + right_packet.fill_peer_manager_hdr(1, 2, 1); + + left.encrypt_with_nonce(&mut left_packet, Some(&nonce)) + .unwrap(); + right + .encrypt_with_nonce(&mut right_packet, Some(&nonce)) + .unwrap(); + assert_eq!(left_packet.payload(), right_packet.payload()); + + left.decrypt(&mut right_packet).unwrap(); + right.decrypt(&mut left_packet).unwrap(); + assert_eq!(left_packet.payload(), plaintext); + assert_eq!(right_packet.payload(), plaintext); + } #[test] fn network_secret_key_derivation_is_stable() { @@ -229,7 +339,11 @@ mod tests { assert!(!effective_algorithm_uses_xor("aes-gcm")); } - #[cfg(not(feature = "aes-gcm"))] + #[cfg(not(any( + feature = "aes-gcm", + feature = "openssl-crypto", + feature = "ring-crypto" + )))] #[test] fn unavailable_aes_is_known_but_rejected() { assert_eq!( @@ -238,14 +352,22 @@ mod tests { ); } - #[cfg(feature = "aes-gcm")] + #[cfg(any( + feature = "aes-gcm", + feature = "openssl-crypto", + feature = "ring-crypto" + ))] #[test] fn compiled_aes_is_available() { validate_algorithm("aes-gcm").unwrap(); validate_algorithm("aes-256-gcm").unwrap(); } - #[cfg(not(feature = "chacha20"))] + #[cfg(not(any( + feature = "chacha20", + feature = "openssl-crypto", + feature = "ring-crypto" + )))] #[test] fn unavailable_chacha20_is_known_but_rejected() { assert_eq!( @@ -256,12 +378,102 @@ mod tests { ); } - #[cfg(feature = "chacha20")] + #[cfg(any( + feature = "chacha20", + feature = "openssl-crypto", + feature = "ring-crypto" + ))] #[test] fn compiled_chacha20_is_available() { validate_algorithm("chacha20").unwrap(); } + #[test] + fn accelerated_backends_take_precedence_over_rustcrypto() { + #[cfg(feature = "openssl-crypto")] + assert_eq!( + preferred_aead_backend(EncryptionAlgorithm::AesGcm), + Some(AeadBackend::OpenSsl) + ); + + #[cfg(all(not(feature = "openssl-crypto"), feature = "ring-crypto"))] + assert_eq!( + preferred_aead_backend(EncryptionAlgorithm::AesGcm), + Some(AeadBackend::Ring) + ); + + #[cfg(all( + not(feature = "openssl-crypto"), + not(feature = "ring-crypto"), + feature = "aes-gcm" + ))] + assert_eq!( + preferred_aead_backend(EncryptionAlgorithm::AesGcm), + Some(AeadBackend::RustCrypto) + ); + } + + #[cfg(all(feature = "ring-crypto", feature = "aes-gcm"))] + #[test] + fn ring_and_rustcrypto_aes_are_interoperable() { + assert_interoperable( + &ring::RingCipher::new_aes128_gcm([1; 16]), + &aes_gcm::AesGcmCipher::new_128([1; 16]), + ); + assert_interoperable( + &ring::RingCipher::new_aes256_gcm([2; 32]), + &aes_gcm::AesGcmCipher::new_256([2; 32]), + ); + } + + #[cfg(all(feature = "ring-crypto", feature = "chacha20"))] + #[test] + fn ring_and_rustcrypto_chacha20_are_interoperable() { + assert_interoperable( + &ring::RingCipher::new_chacha20([3; 32]), + &chacha20::ChaCha20Cipher::new([3; 32]), + ); + } + + #[cfg(all(feature = "openssl-crypto", feature = "aes-gcm"))] + #[test] + fn openssl_and_rustcrypto_aes_are_interoperable() { + assert_interoperable( + &openssl::OpenSslCipher::new_aes128_gcm([1; 16]), + &aes_gcm::AesGcmCipher::new_128([1; 16]), + ); + assert_interoperable( + &openssl::OpenSslCipher::new_aes256_gcm([2; 32]), + &aes_gcm::AesGcmCipher::new_256([2; 32]), + ); + } + + #[cfg(all(feature = "openssl-crypto", feature = "chacha20"))] + #[test] + fn openssl_and_rustcrypto_chacha20_are_interoperable() { + assert_interoperable( + &openssl::OpenSslCipher::new_chacha20([3; 32]), + &chacha20::ChaCha20Cipher::new([3; 32]), + ); + } + + #[cfg(all(feature = "openssl-crypto", feature = "ring-crypto"))] + #[test] + fn openssl_and_ring_algorithms_are_interoperable() { + assert_interoperable( + &openssl::OpenSslCipher::new_aes128_gcm([1; 16]), + &ring::RingCipher::new_aes128_gcm([1; 16]), + ); + assert_interoperable( + &openssl::OpenSslCipher::new_aes256_gcm([2; 32]), + &ring::RingCipher::new_aes256_gcm([2; 32]), + ); + assert_interoperable( + &openssl::OpenSslCipher::new_chacha20([3; 32]), + &ring::RingCipher::new_chacha20([3; 32]), + ); + } + #[test] fn invalid_algorithm_is_rejected() { assert_eq!( diff --git a/easytier-core/src/tunnel/encrypt/openssl.rs b/easytier-core/src/tunnel/encrypt/openssl.rs new file mode 100644 index 00000000..642b3927 --- /dev/null +++ b/easytier-core/src/tunnel/encrypt/openssl.rs @@ -0,0 +1,159 @@ +use openssl::symm::{Cipher, Crypter, Mode}; +use rand::RngCore as _; +use zerocopy::{AsBytes as _, FromBytes as _, FromZeroes as _}; + +use crate::packet::{StandardAeadTail, ZCPacket}; + +use super::{Encryptor, Error}; + +#[derive(Clone)] +pub struct OpenSslCipher { + cipher: OpenSslCipherKind, +} + +#[derive(Clone, Copy)] +enum OpenSslCipherKind { + Aes128Gcm([u8; 16]), + Aes256Gcm([u8; 32]), + ChaCha20Poly1305([u8; 32]), +} + +impl OpenSslCipher { + pub fn new_aes128_gcm(key: [u8; 16]) -> Self { + Self { + cipher: OpenSslCipherKind::Aes128Gcm(key), + } + } + + pub fn new_aes256_gcm(key: [u8; 32]) -> Self { + Self { + cipher: OpenSslCipherKind::Aes256Gcm(key), + } + } + + pub fn new_chacha20(key: [u8; 32]) -> Self { + Self { + cipher: OpenSslCipherKind::ChaCha20Poly1305(key), + } + } + + fn cipher_and_key(&self) -> (Cipher, &[u8]) { + match &self.cipher { + OpenSslCipherKind::Aes128Gcm(key) => (Cipher::aes_128_gcm(), key), + OpenSslCipherKind::Aes256Gcm(key) => (Cipher::aes_256_gcm(), key), + OpenSslCipherKind::ChaCha20Poly1305(key) => (Cipher::chacha20_poly1305(), key), + } + } +} + +impl Encryptor for OpenSslCipher { + fn decrypt(&self, packet: &mut ZCPacket) -> Result<(), Error> { + let header = packet.peer_manager_header().unwrap(); + if !header.is_encrypted() { + return Ok(()); + } + + let payload_len = packet.payload().len(); + if payload_len < StandardAeadTail::SIZE { + return Err(Error::PacketTooShort(payload_len)); + } + + let (cipher, key) = self.cipher_and_key(); + let tail = StandardAeadTail::ref_from_suffix(packet.payload()).unwrap(); + let mut decrypter = Crypter::new(cipher, Mode::Decrypt, key, Some(&tail.nonce)) + .map_err(|_| Error::DecryptionFailed)?; + decrypter + .set_tag(&tail.tag) + .map_err(|_| Error::DecryptionFailed)?; + + let text_len = payload_len - StandardAeadTail::SIZE; + let mut output = vec![0; text_len + cipher.block_size()]; + let mut written = decrypter + .update(&packet.payload()[..text_len], &mut output) + .map_err(|_| Error::DecryptionFailed)?; + written += decrypter + .finalize(&mut output[written..]) + .map_err(|_| Error::DecryptionFailed)?; + + packet.mut_payload()[..written].copy_from_slice(&output[..written]); + packet + .mut_peer_manager_header() + .unwrap() + .set_encrypted(false); + let old_len = packet.buf_len(); + packet + .mut_inner() + .truncate(old_len - (payload_len - written)); + Ok(()) + } + + fn encrypt(&self, packet: &mut ZCPacket) -> Result<(), Error> { + self.encrypt_with_nonce(packet, None) + } + + fn encrypt_with_nonce(&self, packet: &mut ZCPacket, nonce: Option<&[u8]>) -> Result<(), Error> { + let header = packet.peer_manager_header().unwrap(); + if header.is_encrypted() { + tracing::warn!(?packet, "packet is already encrypted"); + return Ok(()); + } + + let (cipher, key) = self.cipher_and_key(); + let mut tail = StandardAeadTail::new_zeroed(); + match nonce { + Some(nonce) => { + tail.nonce = nonce.try_into().map_err(|_| Error::EncryptionFailed)?; + } + None => rand::thread_rng().fill_bytes(&mut tail.nonce), + } + + let mut encrypter = Crypter::new(cipher, Mode::Encrypt, key, Some(&tail.nonce)) + .map_err(|_| Error::EncryptionFailed)?; + let payload_len = packet.payload().len(); + let mut output = vec![0; payload_len + cipher.block_size()]; + let mut written = encrypter + .update(packet.payload(), &mut output) + .map_err(|_| Error::EncryptionFailed)?; + written += encrypter + .finalize(&mut output[written..]) + .map_err(|_| Error::EncryptionFailed)?; + packet.mut_payload()[..written].copy_from_slice(&output[..written]); + encrypter + .get_tag(&mut tail.tag) + .map_err(|_| Error::EncryptionFailed)?; + + packet + .mut_peer_manager_header() + .unwrap() + .set_encrypted(true); + packet.mut_inner().extend_from_slice(tail.as_bytes()); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn round_trip(cipher: OpenSslCipher) { + let plaintext = b"openssl accelerated packet"; + let mut packet = ZCPacket::new_with_payload(plaintext); + packet.fill_peer_manager_hdr(1, 2, 1); + + cipher + .encrypt_with_nonce(&mut packet, Some(&[3; StandardAeadTail::NONCE_SIZE])) + .unwrap(); + assert!(packet.peer_manager_header().unwrap().is_encrypted()); + + cipher.decrypt(&mut packet).unwrap(); + assert_eq!(packet.payload(), plaintext); + assert!(!packet.peer_manager_header().unwrap().is_encrypted()); + } + + #[test] + fn openssl_algorithms_round_trip() { + round_trip(OpenSslCipher::new_aes128_gcm([1; 16])); + round_trip(OpenSslCipher::new_aes256_gcm([2; 32])); + round_trip(OpenSslCipher::new_chacha20([3; 32])); + } +} diff --git a/easytier-core/src/tunnel/encrypt/ring.rs b/easytier-core/src/tunnel/encrypt/ring.rs new file mode 100644 index 00000000..f89091d6 --- /dev/null +++ b/easytier-core/src/tunnel/encrypt/ring.rs @@ -0,0 +1,175 @@ +use rand::RngCore as _; +use ring::aead::{self, LessSafeKey, UnboundKey}; +use zerocopy::{AsBytes as _, FromBytes as _, FromZeroes as _}; + +use crate::packet::{StandardAeadTail, ZCPacket}; + +use super::{Encryptor, Error}; + +#[derive(Clone)] +pub struct RingCipher { + cipher: RingCipherKind, +} + +enum RingCipherKind { + Aes128Gcm(LessSafeKey, [u8; 16]), + Aes256Gcm(LessSafeKey, [u8; 32]), + ChaCha20Poly1305(LessSafeKey, [u8; 32]), +} + +impl RingCipherKind { + fn key(&self) -> &LessSafeKey { + match self { + Self::Aes128Gcm(cipher, _) + | Self::Aes256Gcm(cipher, _) + | Self::ChaCha20Poly1305(cipher, _) => cipher, + } + } +} + +impl Clone for RingCipherKind { + fn clone(&self) -> Self { + match self { + Self::Aes128Gcm(_, key) => Self::Aes128Gcm( + LessSafeKey::new(UnboundKey::new(&aead::AES_128_GCM, key).unwrap()), + *key, + ), + Self::Aes256Gcm(_, key) => Self::Aes256Gcm( + LessSafeKey::new(UnboundKey::new(&aead::AES_256_GCM, key).unwrap()), + *key, + ), + Self::ChaCha20Poly1305(_, key) => Self::ChaCha20Poly1305( + LessSafeKey::new(UnboundKey::new(&aead::CHACHA20_POLY1305, key).unwrap()), + *key, + ), + } + } +} + +impl RingCipher { + pub fn new_aes128_gcm(key: [u8; 16]) -> Self { + Self { + cipher: RingCipherKind::Aes128Gcm( + LessSafeKey::new(UnboundKey::new(&aead::AES_128_GCM, &key).unwrap()), + key, + ), + } + } + + pub fn new_aes256_gcm(key: [u8; 32]) -> Self { + Self { + cipher: RingCipherKind::Aes256Gcm( + LessSafeKey::new(UnboundKey::new(&aead::AES_256_GCM, &key).unwrap()), + key, + ), + } + } + + pub fn new_chacha20(key: [u8; 32]) -> Self { + Self { + cipher: RingCipherKind::ChaCha20Poly1305( + LessSafeKey::new(UnboundKey::new(&aead::CHACHA20_POLY1305, &key).unwrap()), + key, + ), + } + } +} + +impl Encryptor for RingCipher { + fn decrypt(&self, packet: &mut ZCPacket) -> Result<(), Error> { + let header = packet.peer_manager_header().unwrap(); + if !header.is_encrypted() { + return Ok(()); + } + + let payload_len = packet.payload().len(); + if payload_len < StandardAeadTail::SIZE { + return Err(Error::PacketTooShort(payload_len)); + } + + let text_and_tag_len = payload_len - StandardAeadTail::SIZE + StandardAeadTail::TAG_SIZE; + let tail = StandardAeadTail::ref_from_suffix(packet.payload()).unwrap(); + let nonce = aead::Nonce::assume_unique_for_key(tail.nonce); + + self.cipher + .key() + .open_in_place( + nonce, + aead::Aad::empty(), + &mut packet.mut_payload()[..text_and_tag_len], + ) + .map_err(|_| Error::DecryptionFailed)?; + + packet + .mut_peer_manager_header() + .unwrap() + .set_encrypted(false); + let old_len = packet.buf_len(); + packet + .mut_inner() + .truncate(old_len - StandardAeadTail::SIZE); + Ok(()) + } + + fn encrypt(&self, packet: &mut ZCPacket) -> Result<(), Error> { + self.encrypt_with_nonce(packet, None) + } + + fn encrypt_with_nonce(&self, packet: &mut ZCPacket, nonce: Option<&[u8]>) -> Result<(), Error> { + let header = packet.peer_manager_header().unwrap(); + if header.is_encrypted() { + tracing::warn!(?packet, "packet is already encrypted"); + return Ok(()); + } + + let mut tail = StandardAeadTail::new_zeroed(); + match nonce { + Some(nonce) => { + tail.nonce = nonce.try_into().map_err(|_| Error::EncryptionFailed)?; + } + None => rand::thread_rng().fill_bytes(&mut tail.nonce), + } + + let nonce = aead::Nonce::assume_unique_for_key(tail.nonce); + let tag = self + .cipher + .key() + .seal_in_place_separate_tag(nonce, aead::Aad::empty(), packet.mut_payload()) + .map_err(|_| Error::EncryptionFailed)?; + tail.tag.copy_from_slice(tag.as_ref()); + + packet + .mut_peer_manager_header() + .unwrap() + .set_encrypted(true); + packet.mut_inner().extend_from_slice(tail.as_bytes()); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn round_trip(cipher: RingCipher) { + let plaintext = b"ring accelerated packet"; + let mut packet = ZCPacket::new_with_payload(plaintext); + packet.fill_peer_manager_hdr(1, 2, 1); + + cipher + .encrypt_with_nonce(&mut packet, Some(&[3; StandardAeadTail::NONCE_SIZE])) + .unwrap(); + assert!(packet.peer_manager_header().unwrap().is_encrypted()); + + cipher.decrypt(&mut packet).unwrap(); + assert_eq!(packet.payload(), plaintext); + assert!(!packet.peer_manager_header().unwrap().is_encrypted()); + } + + #[test] + fn ring_algorithms_round_trip() { + round_trip(RingCipher::new_aes128_gcm([1; 16])); + round_trip(RingCipher::new_aes256_gcm([2; 32])); + round_trip(RingCipher::new_chacha20([3; 32])); + } +} diff --git a/easytier-core/src/tunnel/secure_datagram.rs b/easytier-core/src/tunnel/secure_datagram.rs index 0d57e3b7..db118a07 100644 --- a/easytier-core/src/tunnel/secure_datagram.rs +++ b/easytier-core/src/tunnel/secure_datagram.rs @@ -755,7 +755,18 @@ fn now_ms() -> u64 { #[cfg(test)] mod tests { use super::*; - #[cfg(all(feature = "aes-gcm", feature = "chacha20"))] + #[cfg(all( + any( + feature = "aes-gcm", + feature = "openssl-crypto", + feature = "ring-crypto" + ), + any( + feature = "chacha20", + feature = "openssl-crypto", + feature = "ring-crypto" + ) + ))] use crate::packet::PacketType; impl SecureDatagramSession { @@ -775,7 +786,18 @@ mod tests { } #[test] - #[cfg(all(feature = "aes-gcm", feature = "chacha20"))] + #[cfg(all( + any( + feature = "aes-gcm", + feature = "openssl-crypto", + feature = "ring-crypto" + ), + any( + feature = "chacha20", + feature = "openssl-crypto", + feature = "ring-crypto" + ) + ))] fn secure_datagram_supports_asymmetric_algorithms() { let root_key = SecureDatagramSession::new_root_key(); let generation = 1u32; @@ -837,7 +859,11 @@ mod tests { } #[test] - #[cfg(feature = "aes-gcm")] + #[cfg(any( + feature = "aes-gcm", + feature = "openssl-crypto", + feature = "ring-crypto" + ))] fn failed_decrypt_does_not_poison_replay_window() { use crate::packet::PacketType; diff --git a/easytier-core/src/tunnel/tcp.rs b/easytier-core/src/tunnel/tcp.rs index d5a9aaca..4dc8de65 100644 --- a/easytier-core/src/tunnel/tcp.rs +++ b/easytier-core/src/tunnel/tcp.rs @@ -34,7 +34,7 @@ where .unwrap() .take() .expect("TcpTunnel can only be split once"); - let (reader, writer) = tokio::io::split(socket); + let (reader, writer) = socket.into_split(); ( Box::pin(FramedReader::new(reader, self.max_packet_size)), Box::pin(FramedWriter::new(writer)), diff --git a/easytier-core/src/tunnel/udp.rs b/easytier-core/src/tunnel/udp.rs index 2cc98049..94d18e21 100644 --- a/easytier-core/src/tunnel/udp.rs +++ b/easytier-core/src/tunnel/udp.rs @@ -6,7 +6,7 @@ use std::{ use bytes::BytesMut; use futures::{Sink, Stream}; -use tokio::sync::{oneshot, watch}; +use tokio::sync::watch; use crate::{ packet::{UDP_TUNNEL_HEADER_SIZE, UdpPacketType, ZCPacket, ZCPacketType}, @@ -76,7 +76,10 @@ impl Stream for UdpTunnelStream { Poll::Ready(ret.map(|payload| { payload .map_err(ring_socket_error_to_tunnel) - .and_then(|datagram| zcpacket_from_udp_session_payload(&datagram.payload)) + .and_then(|datagram| match datagram.into_tunnel_packet() { + Ok(packet) => Ok(packet), + Err(payload) => zcpacket_from_udp_session_payload(&payload), + }) })) } } @@ -114,15 +117,10 @@ impl Sink for UdpTunnelSink { } let packet = item.convert_type(ZCPacketType::UDP); - let payload = BytesMut::from(packet.udp_payload()); this.codec - .validate_payload(&payload) + .validate_payload(packet.udp_payload()) .map_err(TunnelError::IOError)?; - let (completion, _sent) = oneshot::channel(); - let outbound = UdpSessionOutbound { - payload, - completion, - }; + let outbound = UdpSessionOutbound::TunnelPacket(packet); this.session_send_tx .force_send(outbound) .map_err(ring_send_error_to_tunnel) diff --git a/easytier-core/src/tunnel/web_security.rs b/easytier-core/src/tunnel/web_security.rs index b09d4790..f95ebdd8 100644 --- a/easytier-core/src/tunnel/web_security.rs +++ b/easytier-core/src/tunnel/web_security.rs @@ -158,7 +158,7 @@ fn decode_noise_payload(payload: &[u8]) -> Option<&[u8]> { } pub fn web_secure_tunnel_supported() -> bool { - cfg!(feature = "aes-gcm") + crate::tunnel::encrypt::algorithm_is_available(crate::config::EncryptionAlgorithm::AesGcm) } fn web_secure_cipher_algorithm() -> Result<&'static str, TunnelError> { diff --git a/easytier-core/src/wasi/runtime.rs b/easytier-core/src/wasi/runtime.rs index 08a1aff5..6bba04fc 100644 --- a/easytier-core/src/wasi/runtime.rs +++ b/easytier-core/src/wasi/runtime.rs @@ -34,7 +34,11 @@ pub(super) fn new_wasi_core_runtime( ) -> anyhow::Result { use std::sync::Arc; - use crate::host::{dns::HostDnsResolver, packet::HostPacketSink, socket::HostSocketRuntime}; + use crate::host::{ + dns::HostDnsResolver, + packet::{HostPacket, HostPacketSink}, + socket::HostSocketRuntime, + }; use crate::{ connectivity::connector_host::new_connector_host, instance::{CoreHostAdapters, CoreInstance}, @@ -402,7 +406,10 @@ mod abi { fn send_packet(&self, packet: Vec) { let packet_plane = self.core.core().packet_plane(); self.execution.lock().unwrap().runtime.spawn(async move { - if let Err(error) = packet_plane.send_ip_packet(packet).await { + if let Err(error) = packet_plane + .send_ip_packet(HostPacket::copy_from_payload(&packet)) + .await + { tracing::warn!(?error, "host packet ingress failed"); } }); diff --git a/easytier/Cargo.toml b/easytier/Cargo.toml index 3bf19ddd..512cc43b 100644 --- a/easytier/Cargo.toml +++ b/easytier/Cargo.toml @@ -172,9 +172,6 @@ network-interface = "2.0" # for wireguard boringtun = { package = "boringtun-easytier", version = "0.6.1", optional = true } -# for encryption -ring = { version = "0.17", optional = true } - # for cli tabled = "0.16" humansize = "2.1.3" @@ -336,12 +333,13 @@ full = [ "extended-services", "tcp-hole-punch", ] -wireguard = ["vpn-portal", "dep:boringtun", "dep:ring", "easytier-core/aes-gcm", "easytier-core/chacha20", "easytier-proto/wireguard"] +wireguard = ["vpn-portal", "dep:boringtun", "ring-crypto", "easytier-proto/wireguard"] quic = ["wrapped-transport", "easytier-core/proxy-packet", "dep:quinn", "dep:quinn-proto", "dep:seahash", "dep:rustls", "easytier-proto/quic"] kcp = ["wrapped-transport", "easytier-core/proxy-packet", "dep:kcp-sys"] mimalloc = ["dep:mimalloc"] aes-gcm = ["easytier-core/aes-gcm"] -openssl-crypto = ["easytier-core/aes-gcm", "easytier-core/chacha20"] +openssl-crypto = ["easytier-core/openssl-crypto"] +ring-crypto = ["easytier-core/ring-crypto"] tun = ["dep:tun", "linux-netlink"] linux-netlink = ["dep:netlink-sys"] proxy-cidr-monitor = ["easytier-core/proxy-cidr-monitor"] diff --git a/easytier/src/instance/composition.rs b/easytier/src/instance/composition.rs index f30cc1c8..60437bc5 100644 --- a/easytier/src/instance/composition.rs +++ b/easytier/src/instance/composition.rs @@ -4,6 +4,8 @@ use std::sync::Arc; use easytier_core::gateway::proxy::wrapped_transport::WrappedTransportEngines; #[cfg(feature = "wireguard")] use easytier_core::gateway::vpn_portal::VpnPortalHost; +#[cfg(test)] +use easytier_core::host::packet::{HostPacket, PacketSink}; #[cfg(feature = "management")] use easytier_core::{ connectivity::manual::ManualTunnelConnector, @@ -12,8 +14,7 @@ use easytier_core::{ }; use easytier_core::{ events::{CoreEvent, CoreEventSink}, - host::packet::PacketSink, - instance::{CoreHostAdapters, CoreInstance}, + instance::{CoreHostAdapters, CoreInstance, PacketEgressHost}, process_runtime::CoreProcessRuntime, }; @@ -45,10 +46,13 @@ pub(crate) fn compose_native_core_instance( process_runtime: Arc, ) -> anyhow::Result> { let global_ctx = Arc::new(GlobalCtx::new(config.clone())); - let (packet_sender, packet_receiver) = tokio::sync::mpsc::channel(128); - let mut adapters = - runtime_core_host_adapters(global_ctx.clone(), process_runtime, Arc::new(packet_sender)); - adapters.instance_runtime = NativeInstanceRuntimeHost::new(global_ctx.clone(), packet_receiver); + let runtime_host = NativeInstanceRuntimeHost::new(global_ctx.clone()); + let mut adapters = runtime_core_host_adapters_with_packet_egress( + global_ctx.clone(), + process_runtime, + runtime_host.clone(), + ); + adapters.instance_runtime = runtime_host; NativeCoreInstance::from_toml(config, adapters) } @@ -148,6 +152,7 @@ fn runtime_wrapped_transport_engines() -> WrappedTransportEngines { WrappedTransportEngines { kcp, quic } } +#[cfg(test)] pub(crate) fn runtime_core_host_adapters( global_ctx: ArcGlobalCtx, process_runtime: Arc, @@ -155,7 +160,26 @@ pub(crate) fn runtime_core_host_adapters( ) -> CoreHostAdapters { let host = native_instance_host(global_ctx.clone()); let runtime_dns = native_host_runtime(); - let mut adapters = CoreHostAdapters::new(host, runtime_dns, packet_sink, process_runtime); + let adapters = CoreHostAdapters::new(host, runtime_dns, packet_sink, process_runtime); + configure_runtime_core_host_adapters(global_ctx, adapters) +} + +pub(crate) fn runtime_core_host_adapters_with_packet_egress( + global_ctx: ArcGlobalCtx, + process_runtime: Arc, + packet_egress: Arc, +) -> CoreHostAdapters { + let host = native_instance_host(global_ctx.clone()); + let runtime_dns = native_host_runtime(); + let adapters = + CoreHostAdapters::new_with_packet_egress(host, runtime_dns, packet_egress, process_runtime); + configure_runtime_core_host_adapters(global_ctx, adapters) +} + +fn configure_runtime_core_host_adapters( + global_ctx: ArcGlobalCtx, + mut adapters: CoreHostAdapters, +) -> CoreHostAdapters { #[cfg(test)] adapters.replace_stun_provider(Arc::new(crate::common::stun::MockStunInfoCollector { udp_nat_type: crate::proto::common::NatType::Unknown, @@ -504,7 +528,7 @@ mod tests { loop { instance_a .packet_plane() - .send_ip_packet(ip_packet.clone()) + .send_ip_packet(HostPacket::copy_from_payload(&ip_packet)) .await .unwrap(); match tokio::time::timeout( diff --git a/easytier/src/instance/dns_server/tests.rs b/easytier/src/instance/dns_server/tests.rs index de097601..2f5a3d84 100644 --- a/easytier/src/instance/dns_server/tests.rs +++ b/easytier/src/instance/dns_server/tests.rs @@ -4,7 +4,11 @@ use std::sync::Arc; use std::time::Duration; use cidr::Ipv4Inet; -use easytier_core::{gateway::magic_dns::MagicDnsRoute, process_runtime::CoreProcessRuntime}; +use easytier_core::{ + gateway::magic_dns::MagicDnsRoute, + host::packet::{HostPacket, HostPacketChannelSink, HostPacketReceiver}, + process_runtime::CoreProcessRuntime, +}; use hickory_client::client::{Client, ClientHandle as _}; use hickory_proto::rr; use hickory_proto::runtime::TokioRuntimeProvider; @@ -33,21 +37,16 @@ pub async fn prepare_env( prepare_env_with_tld_dns_zone(dns_name, tun_ip, None).await } -async fn build_test_core( - ctx: ArcGlobalCtx, -) -> ( - Arc, - tokio::sync::mpsc::Receiver>, -) { - let (packet_sink, packet_receiver) = tokio::sync::mpsc::channel(128); +async fn build_test_core(ctx: ArcGlobalCtx) -> (Arc, HostPacketReceiver) { + let (packet_sink, packet_receiver) = tokio::sync::mpsc::channel::(128); let adapters = runtime_core_host_adapters( ctx.clone(), CoreProcessRuntime::new(), - Arc::new(packet_sink), + Arc::new(HostPacketChannelSink::new(packet_sink)), ); let core_instance = NativeCoreInstance::new(test_core_instance_config(&ctx), adapters).unwrap(); core_instance.start().await.unwrap(); - (core_instance, packet_receiver) + (core_instance, HostPacketReceiver::new(packet_receiver)) } pub async fn prepare_env_with_tld_dns_zone( diff --git a/easytier/src/instance/runtime_host.rs b/easytier/src/instance/runtime_host.rs index c1f38295..06cc668b 100644 --- a/easytier/src/instance/runtime_host.rs +++ b/easytier/src/instance/runtime_host.rs @@ -1,7 +1,9 @@ use std::sync::Arc; -use easytier_core::{gateway::dhcp::DhcpIpv4Host, instance::CorePacketPlane}; -use tokio::sync::{Mutex, mpsc}; +use easytier_core::{ + gateway::dhcp::DhcpIpv4Host, host::packet::HostPacketReceiver, instance::CorePacketPlane, +}; +use tokio::sync::Mutex; use tokio_util::sync::CancellationToken; use crate::common::global_ctx::ArcGlobalCtx; @@ -27,8 +29,6 @@ use event_journal::EventJournal; use magic_dns::MagicDnsRuntime; use tun_runtime::NativeTunRuntime; -pub(super) type HostPacketReceiver = mpsc::Receiver>; - pub(crate) struct NativeInstanceRuntimeHost { global_ctx: ArcGlobalCtx, operation: Arc>, @@ -38,12 +38,9 @@ pub(crate) struct NativeInstanceRuntimeHost { } impl NativeInstanceRuntimeHost { - pub(crate) fn new( - global_ctx: ArcGlobalCtx, - peer_packet_receiver: HostPacketReceiver, - ) -> Arc { + pub(crate) fn new(global_ctx: ArcGlobalCtx) -> Arc { let cancel = CancellationToken::new(); - let tun = NativeTunRuntime::new(global_ctx.clone(), cancel.clone(), peer_packet_receiver); + let tun = NativeTunRuntime::new(global_ctx.clone(), cancel.clone()); let event_journal = EventJournal::new(&global_ctx); Arc::new(Self { global_ctx, @@ -87,6 +84,10 @@ impl NativeInstanceRuntimeHost { fn attach_runtime_tun_fd(&self, fd: i32) -> anyhow::Result<()> { self.tun.attach_fd(fd) } + + fn install_packet_receiver(&self, receiver: HostPacketReceiver) -> anyhow::Result<()> { + self.tun.install_packet_receiver(receiver) + } } #[cfg(test)] @@ -100,8 +101,7 @@ mod tests { #[test] fn runtime_host_owns_event_subscription_context() { let global_ctx = Arc::new(GlobalCtx::new(TomlConfig::default())); - let (_packet_sender, packet_receiver) = mpsc::channel(1); - let runtime_host = NativeInstanceRuntimeHost::new(global_ctx.clone(), packet_receiver); + let runtime_host = NativeInstanceRuntimeHost::new(global_ctx.clone()); let mut events = runtime_host.subscribe_event(); global_ctx.issue_event(GlobalCtxEvent::CredentialChanged); diff --git a/easytier/src/instance/runtime_host/implementation.rs b/easytier/src/instance/runtime_host/implementation.rs index 1e4e6e75..d070330b 100644 --- a/easytier/src/instance/runtime_host/implementation.rs +++ b/easytier/src/instance/runtime_host/implementation.rs @@ -2,7 +2,8 @@ use std::sync::Arc; use easytier_core::{ gateway::dhcp::DhcpIpv4Host, - instance::{CorePacketPlane, InstanceRuntimeHost}, + host::packet::HostPacketReceiver, + instance::{CorePacketPlane, InstanceRuntimeHost, PacketEgressHost}, }; use super::NativeInstanceRuntimeHost; @@ -42,3 +43,16 @@ impl InstanceRuntimeHost for NativeInstanceRuntimeHost { self.attach_runtime_tun_fd(fd) } } + +#[async_trait::async_trait] +impl PacketEgressHost for NativeInstanceRuntimeHost { + async fn start(&self, receiver: HostPacketReceiver) -> anyhow::Result<()> { + self.install_packet_receiver(receiver) + } + + async fn stop(&self) {} + + fn request_stop(&self) { + self.request_runtime_shutdown(); + } +} diff --git a/easytier/src/instance/runtime_host/tun_common.rs b/easytier/src/instance/runtime_host/tun_common.rs index c4bfa91b..a86aead9 100644 --- a/easytier/src/instance/runtime_host/tun_common.rs +++ b/easytier/src/instance/runtime_host/tun_common.rs @@ -1,8 +1,12 @@ -use std::{any::Any, sync::Arc}; +use std::{ + any::Any, + sync::{Arc, OnceLock}, +}; +use easytier_core::host::packet::HostPacketReceiver; use tokio::{sync::Mutex, task::JoinSet}; -use super::{HostPacketReceiver, MagicDnsRuntime}; +use super::MagicDnsRuntime; use crate::instance::virtual_nic::NicCtx; struct NicCtxContainer { @@ -29,19 +33,28 @@ impl NicCtxContainer { #[derive(Clone)] pub(super) struct TunNicState { nic_ctx: Arc>>, - receiver: Arc>, + receiver: Arc>>>, } impl TunNicState { - pub(super) fn new(receiver: HostPacketReceiver) -> Self { + pub(super) fn empty() -> Self { Self { nic_ctx: Arc::new(Mutex::new(None)), - receiver: Arc::new(Mutex::new(receiver)), + receiver: Arc::new(OnceLock::new()), } } + pub(super) fn install_receiver(&self, receiver: HostPacketReceiver) -> anyhow::Result<()> { + self.receiver + .set(Arc::new(Mutex::new(receiver))) + .map_err(|_| anyhow::anyhow!("native packet receiver is already installed")) + } + pub(super) fn receiver(&self) -> Arc> { - self.receiver.clone() + self.receiver + .get() + .expect("packet receiver must be installed before preparing TUN") + .clone() } pub(super) async fn stop(&self) { @@ -54,7 +67,7 @@ impl TunNicState { pub(super) async fn drain(&self) { self.stop().await; - let receiver = self.receiver.clone(); + let receiver = self.receiver(); let mut tasks = JoinSet::new(); tasks.spawn(async move { let mut receiver = receiver.lock().await; diff --git a/easytier/src/instance/runtime_host/tun_desktop.rs b/easytier/src/instance/runtime_host/tun_desktop.rs index 96225c7a..c7343238 100644 --- a/easytier/src/instance/runtime_host/tun_desktop.rs +++ b/easytier/src/instance/runtime_host/tun_desktop.rs @@ -4,6 +4,7 @@ use anyhow::Context as _; use cidr::Ipv4Inet; use easytier_core::{ gateway::dhcp::{DhcpIpv4ApplyOutcome, DhcpIpv4ApplyPermit, DhcpIpv4Host}, + host::packet::HostPacketReceiver, instance::CorePacketPlane, }; use futures::FutureExt as _; @@ -13,7 +14,7 @@ use tokio::{ }; use tokio_util::sync::CancellationToken; -use super::{HostPacketReceiver, MagicDnsRuntime, tun_common::TunNicState}; +use super::{MagicDnsRuntime, tun_common::TunNicState}; use crate::{ common::{ config::ConfigLoader as _, @@ -31,19 +32,22 @@ pub(super) struct NativeTunRuntime { } impl NativeTunRuntime { - pub(super) fn new( - global_ctx: ArcGlobalCtx, - cancel: CancellationToken, - peer_packet_receiver: HostPacketReceiver, - ) -> Self { + pub(super) fn new(global_ctx: ArcGlobalCtx, cancel: CancellationToken) -> Self { Self { global_ctx, cancel, - nic: TunNicState::new(peer_packet_receiver), + nic: TunNicState::empty(), static_ip_task: Mutex::new(None), } } + pub(super) fn install_packet_receiver( + &self, + receiver: HostPacketReceiver, + ) -> anyhow::Result<()> { + self.nic.install_receiver(receiver) + } + fn report_static_ip_cancelled(output: &mut Option>>) { if let Some(output) = output.take() { let _ = output.send(Err(anyhow::anyhow!( diff --git a/easytier/src/instance/runtime_host/tun_disabled.rs b/easytier/src/instance/runtime_host/tun_disabled.rs index 6a34dd6a..6de9c9d8 100644 --- a/easytier/src/instance/runtime_host/tun_disabled.rs +++ b/easytier/src/instance/runtime_host/tun_disabled.rs @@ -3,12 +3,12 @@ use std::sync::Arc; use cidr::Ipv4Inet; use easytier_core::{ gateway::dhcp::{DhcpIpv4ApplyOutcome, DhcpIpv4ApplyPermit, DhcpIpv4Host}, + host::packet::HostPacketReceiver, instance::CorePacketPlane, }; use tokio::sync::Mutex; use tokio_util::sync::CancellationToken; -use super::HostPacketReceiver; use crate::common::global_ctx::{ArcGlobalCtx, GlobalCtxEvent}; pub(super) struct NativeTunRuntime { @@ -17,15 +17,18 @@ pub(super) struct NativeTunRuntime { } impl NativeTunRuntime { - pub(super) fn new( - global_ctx: ArcGlobalCtx, - cancel: CancellationToken, - peer_packet_receiver: HostPacketReceiver, - ) -> Self { - drop(peer_packet_receiver); + pub(super) fn new(global_ctx: ArcGlobalCtx, cancel: CancellationToken) -> Self { Self { global_ctx, cancel } } + pub(super) fn install_packet_receiver( + &self, + receiver: HostPacketReceiver, + ) -> anyhow::Result<()> { + drop(receiver); + Ok(()) + } + pub(super) async fn prepare(&self, _packet_plane: Arc) -> anyhow::Result<()> { Ok(()) } diff --git a/easytier/src/instance/runtime_host/tun_mobile.rs b/easytier/src/instance/runtime_host/tun_mobile.rs index 79216a55..4f459326 100644 --- a/easytier/src/instance/runtime_host/tun_mobile.rs +++ b/easytier/src/instance/runtime_host/tun_mobile.rs @@ -4,13 +4,14 @@ use anyhow::Context as _; use cidr::Ipv4Inet; use easytier_core::{ gateway::dhcp::{DhcpIpv4ApplyOutcome, DhcpIpv4ApplyPermit, DhcpIpv4Host}, + host::packet::HostPacketReceiver, instance::CorePacketPlane, }; use futures::FutureExt as _; use tokio::sync::{Mutex, Notify, mpsc}; use tokio_util::sync::CancellationToken; -use super::{HostPacketReceiver, MagicDnsRuntime, tun_common::TunNicState}; +use super::{MagicDnsRuntime, tun_common::TunNicState}; use crate::{ common::global_ctx::{ArcGlobalCtx, GlobalCtxEvent}, instance::virtual_nic::NicCtx, @@ -26,22 +27,25 @@ pub(super) struct NativeTunRuntime { } impl NativeTunRuntime { - pub(super) fn new( - global_ctx: ArcGlobalCtx, - cancel: CancellationToken, - peer_packet_receiver: HostPacketReceiver, - ) -> Self { + pub(super) fn new(global_ctx: ArcGlobalCtx, cancel: CancellationToken) -> Self { let (tun_fd, tun_fd_receiver) = mpsc::channel(16); Self { global_ctx, cancel, - nic: TunNicState::new(peer_packet_receiver), + nic: TunNicState::empty(), tun_fd, tun_fd_receiver: Mutex::new(Some(tun_fd_receiver)), task: Mutex::new(None), } } + pub(super) fn install_packet_receiver( + &self, + receiver: HostPacketReceiver, + ) -> anyhow::Result<()> { + self.nic.install_receiver(receiver) + } + async fn install_mobile_tun( nic_state: TunNicState, global_ctx: ArcGlobalCtx, diff --git a/easytier/src/instance/test_instance.rs b/easytier/src/instance/test_instance.rs index 6058cce7..0f03af81 100644 --- a/easytier/src/instance/test_instance.rs +++ b/easytier/src/instance/test_instance.rs @@ -10,7 +10,7 @@ use easytier_core::{ use crate::{ common::global_ctx::{ArcGlobalCtx, GlobalCtx}, instance::{ - composition::{NativeCoreInstance, runtime_core_host_adapters}, + composition::{NativeCoreInstance, runtime_core_host_adapters_with_packet_egress}, runtime_host::NativeInstanceRuntimeHost, }, socket::udp::RuntimeUdpSocket, @@ -50,15 +50,14 @@ impl TestInstance { ), ) -> Self { let global_ctx = Arc::new(GlobalCtx::new(config.clone())); - let (packet_sender, packet_receiver) = tokio::sync::mpsc::channel(128); - let mut adapters = runtime_core_host_adapters( + let runtime_host = NativeInstanceRuntimeHost::new(global_ctx.clone()); + let mut adapters = runtime_core_host_adapters_with_packet_egress( global_ctx.clone(), process_runtime, - Arc::new(packet_sender), + runtime_host.clone(), ); customize(&mut adapters); - adapters.instance_runtime = - NativeInstanceRuntimeHost::new(global_ctx.clone(), packet_receiver); + adapters.instance_runtime = runtime_host; let core = CoreInstance::from_toml(config, adapters) .expect("test CoreInstance composition should be valid"); Self { core, global_ctx } diff --git a/easytier/src/instance/virtual_nic.rs b/easytier/src/instance/virtual_nic.rs index 49845388..a182645b 100644 --- a/easytier/src/instance/virtual_nic.rs +++ b/easytier/src/instance/virtual_nic.rs @@ -14,6 +14,7 @@ use crate::common::{ }; use easytier_core::{ + host::packet::{HostPacket, HostPacketReceiver}, instance::CorePacketPlane, packet::{TAIL_RESERVED_SIZE, ZCPacket, ZCPacketType}, tunnel::{ @@ -42,8 +43,6 @@ use zerocopy::{NativeEndian, NetworkEndian}; #[cfg(target_os = "windows")] use crate::common::ifcfg::RegistryManager; -type HostPacketReceiver = tokio::sync::mpsc::Receiver>; - pin_project! { pub struct TunStream { #[pin] @@ -866,15 +865,17 @@ impl NicCtx { } async fn do_forward_nic_to_peers(ret: ZCPacket, packet_plane: &CorePacketPlane) { - let payload = ret.payload(); - if payload.is_empty() { + if ret.payload().is_empty() { return; } tracing::trace!( ?ret, "[USER_PACKET] recv new packet from tun device and forward to peers." ); - if let Err(error) = packet_plane.send_ip_packet(payload.to_vec()).await { + if let Err(error) = packet_plane + .send_ip_packet(HostPacket::from_tun_packet(ret)) + .await + { tracing::trace!(?error, "[USER_PACKET] send_msg failed"); } } @@ -912,7 +913,7 @@ impl NicCtx { "[USER_PACKET] forward packet from peers to nic. packet: {:?}", packet ); - let ret = sink.send(ZCPacket::new_with_payload(&packet)).await; + let ret = sink.send(packet.into_tun_packet()).await; if ret.is_err() { tracing::error!(?ret, "do_forward_tunnel_to_nic sink error"); } diff --git a/easytier/src/instance/windows_udp_broadcast/runtime.rs b/easytier/src/instance/windows_udp_broadcast/runtime.rs index 883ddcb4..59c32bc3 100644 --- a/easytier/src/instance/windows_udp_broadcast/runtime.rs +++ b/easytier/src/instance/windows_udp_broadcast/runtime.rs @@ -10,7 +10,10 @@ use easytier_core::gateway::udp_broadcast::{ use { crate::common::global_ctx::{ArcGlobalCtx, GlobalCtxEvent}, anyhow::Context, - easytier_core::{gateway::udp_broadcast::UdpBroadcastRelayStats, instance::CorePacketPlane}, + easytier_core::{ + gateway::udp_broadcast::UdpBroadcastRelayStats, host::packet::HostPacket, + instance::CorePacketPlane, + }, network_interface::{Addr, NetworkInterface, NetworkInterfaceConfig}, socket2::{Domain, Protocol, SockAddr, Socket, Type}, std::{ @@ -188,7 +191,7 @@ async fn forward_normalized_packet( stats: &UdpBroadcastRelayStats, ) { let ret = packet_plane - .send_local_ip_packet(normalized.packet.clone()) + .send_local_ip_packet(HostPacket::copy_from_payload(&normalized.packet)) .await; let summary = UdpPacketSummary::parse(&normalized.packet); diff --git a/easytier/src/socket/tcp.rs b/easytier/src/socket/tcp.rs index fa27d230..db9709d9 100644 --- a/easytier/src/socket/tcp.rs +++ b/easytier/src/socket/tcp.rs @@ -9,7 +9,7 @@ use std::{ use easytier_core::{ socket::tcp::{ TcpBindOptions, TcpConnectOptions, TcpListenOptions, TcpListenPurpose, TcpSocketPurpose, - VirtualTcpListener, VirtualTcpSocket, + VirtualTcpListener, VirtualTcpSocket, VirtualTcpSplit, }, tunnel::TunnelError, }; @@ -123,6 +123,25 @@ impl AsyncWrite for RuntimeTcpSocket { } impl VirtualTcpSocket for RuntimeTcpSocket { + fn into_split(self) -> VirtualTcpSplit { + match self.inner { + RuntimeTcpSocketInner::Tcp(stream) => { + let (reader, writer) = stream.into_split(); + (Box::new(reader), Box::new(writer)) + } + #[cfg(unix)] + RuntimeTcpSocketInner::Unix(stream) => { + let (reader, writer) = stream.into_split(); + (Box::new(reader), Box::new(writer)) + } + #[cfg(feature = "faketcp")] + RuntimeTcpSocketInner::FakeTcp(socket) => { + let (reader, writer) = tokio::io::split(socket); + (Box::new(reader), Box::new(writer)) + } + } + } + fn local_addr(&self) -> io::Result { match &self.inner { RuntimeTcpSocketInner::Tcp(stream) => stream.local_addr(), @@ -349,8 +368,34 @@ pub(crate) fn prepare_proxy_tcp_socket(stream: &TcpStream) -> io::Result<()> { #[cfg(test)] mod tests { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use super::*; + #[tokio::test] + async fn runtime_tcp_owned_split_preserves_full_duplex_and_shutdown() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (client, accepted) = tokio::join!(TcpStream::connect(address), listener.accept()); + let client = client.unwrap(); + let (mut server, _) = accepted.unwrap(); + let (mut reader, mut writer) = RuntimeTcpSocket::new(client).into_split(); + + writer.write_all(b"client-to-server").await.unwrap(); + let mut from_client = [0; 16]; + server.read_exact(&mut from_client).await.unwrap(); + assert_eq!(&from_client, b"client-to-server"); + + server.write_all(b"server-to-client").await.unwrap(); + let mut from_server = [0; 16]; + reader.read_exact(&mut from_server).await.unwrap(); + assert_eq!(&from_server, b"server-to-client"); + + writer.shutdown().await.unwrap(); + let mut after_shutdown = [0; 1]; + assert_eq!(server.read(&mut after_shutdown).await.unwrap(), 0); + } + #[test] fn tcp_connect_binds_when_socket_option_requires_pre_connect_setup() { assert!(must_bind_before_connect( diff --git a/easytier/src/socket/udp.rs b/easytier/src/socket/udp.rs index b66bb037..87a45545 100644 --- a/easytier/src/socket/udp.rs +++ b/easytier/src/socket/udp.rs @@ -12,8 +12,8 @@ use easytier_core::socket::{ use easytier_core::socket::{ SocketContext, udp::{ - UdpBindOptions, UdpSocketPurpose, UdpSocketRecvMeta, UdpSocketSendMeta, VirtualUdpSocket, - VirtualUdpSocketFactory, + MAX_UDP_SESSION_DATAGRAM_SIZE, UdpBindOptions, UdpSocketDatagram, UdpSocketPurpose, + UdpSocketRecvMeta, UdpSocketSendMeta, VirtualUdpSocket, VirtualUdpSocketFactory, }, }; use tokio::net::UdpSocket; @@ -108,6 +108,17 @@ impl VirtualUdpSocket for RuntimeUdpSocket { let (len, addr, dst_ip) = udp_src::recv_from_with_dst_ip(&self.socket, buf).await?; Ok((len, addr, UdpSocketRecvMeta { dst_ip })) } + + #[cfg(unix)] + async fn recv_datagram(&self) -> std::io::Result { + let (payload, remote_addr, dst_ip) = + udp_src::recv_datagram_with_dst_ip(&self.socket, MAX_UDP_SESSION_DATAGRAM_SIZE).await?; + Ok(UdpSocketDatagram { + payload, + remote_addr, + meta: UdpSocketRecvMeta { dst_ip }, + }) + } } #[derive(Debug, Clone, Copy, Default)] @@ -212,6 +223,57 @@ mod tests { assert_eq!(meta.dst_ip, Some(std::net::IpAddr::V4(Ipv4Addr::LOCALHOST))); } + #[cfg(unix)] + #[tokio::test] + async fn runtime_udp_socket_receives_owned_datagram_with_destination_ip() { + let socket = Arc::new(UdpSocket::bind("0.0.0.0:0").await.unwrap()); + let runtime_socket = RuntimeUdpSocket::new(socket.clone()); + let client = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + client + .send_to( + b"owned-pktinfo", + SocketAddr::from(([127, 0, 0, 1], socket.local_addr().unwrap().port())), + ) + .await + .unwrap(); + + let datagram = runtime_socket.recv_datagram().await.unwrap(); + + assert_eq!(datagram.payload, b"owned-pktinfo".as_slice()); + assert_eq!( + datagram.meta.dst_ip, + Some(std::net::IpAddr::V4(Ipv4Addr::LOCALHOST)) + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn runtime_udp_socket_drops_truncated_datagrams() { + let socket = Arc::new(UdpSocket::bind("127.0.0.1:0").await.unwrap()); + let runtime_socket = RuntimeUdpSocket::new(socket.clone()); + let client = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let server_addr = socket.local_addr().unwrap(); + + client + .send_to(&vec![0xAA; MAX_UDP_SESSION_DATAGRAM_SIZE + 1], server_addr) + .await + .unwrap(); + client + .send_to(b"after-oversized", server_addr) + .await + .unwrap(); + + let datagram = tokio::time::timeout( + std::time::Duration::from_secs(1), + runtime_socket.recv_datagram(), + ) + .await + .unwrap() + .unwrap(); + + assert_eq!(datagram.payload, b"after-oversized".as_slice()); + } + #[tokio::test] async fn runtime_v4_hole_punch_control_packet_is_forwarded() { let local_addr = SocketAddr::from(([0, 0, 0, 0], 0)); diff --git a/easytier/src/socket/udp_src/unix.rs b/easytier/src/socket/udp_src/unix.rs index 60962662..3aa6a6c6 100644 --- a/easytier/src/socket/udp_src/unix.rs +++ b/easytier/src/socket/udp_src/unix.rs @@ -3,6 +3,7 @@ use std::{ net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}, }; +use bytes::BytesMut; use tokio::net::UdpSocket; pub(crate) fn enable_recv_pktinfo(socket: &UdpSocket) -> io::Result<()> { @@ -67,6 +68,36 @@ pub(crate) async fn recv_from_with_dst_ip( .await } +pub(crate) async fn recv_datagram_with_dst_ip( + socket: &UdpSocket, + capacity: usize, +) -> io::Result<(BytesMut, SocketAddr, Option)> { + let mut payload = BytesMut::with_capacity(capacity); + let (len, remote_addr, dst_ip) = socket + .async_io(tokio::io::Interest::READABLE, || { + loop { + let ret = unsafe { + recv_from_with_dst_ip_raw(socket, payload.as_mut_ptr(), payload.capacity()) + }; + match ret { + Err(err) if err.kind() == io::ErrorKind::Interrupted => continue, + Ok((_len, _remote_addr, _dst_ip, true)) => { + tracing::debug!(capacity, "dropping oversized udp session datagram"); + } + Ok((len, remote_addr, dst_ip, false)) => { + break Ok((len, remote_addr, dst_ip)); + } + Err(err) => break Err(err), + } + } + }) + .await?; + unsafe { + payload.set_len(len); + } + Ok((payload, remote_addr, dst_ip)) +} + #[cfg(not(any(unix, windows)))] pub(crate) async fn recv_from_with_dst_ip( socket: &UdpSocket, @@ -80,6 +111,15 @@ fn recv_from_with_dst_ip_once( socket: &UdpSocket, buf: &mut [u8], ) -> io::Result<(usize, SocketAddr, Option)> { + unsafe { recv_from_with_dst_ip_raw(socket, buf.as_mut_ptr(), buf.len()) } + .map(|(len, remote_addr, dst_ip, _truncated)| (len, remote_addr, dst_ip)) +} + +unsafe fn recv_from_with_dst_ip_raw( + socket: &UdpSocket, + buf_ptr: *mut u8, + buf_len: usize, +) -> io::Result<(usize, SocketAddr, Option, bool)> { use std::{mem, os::fd::AsRawFd}; use nix::libc; @@ -129,8 +169,8 @@ fn recv_from_with_dst_ip_once( } let mut iov = libc::iovec { - iov_base: buf.as_mut_ptr() as *mut libc::c_void, - iov_len: buf.len(), + iov_base: buf_ptr as *mut libc::c_void, + iov_len: buf_len, }; let mut name = unsafe { mem::zeroed::() }; let mut control = ControlBuffer([0u8; 256]); @@ -184,7 +224,8 @@ fn recv_from_with_dst_ip_once( } } - Ok((len as usize, remote_addr, dst_ip)) + let truncated = msg.msg_flags & libc::MSG_TRUNC != 0; + Ok((len as usize, remote_addr, dst_ip, truncated)) } #[cfg(any(target_os = "linux", target_os = "android"))] diff --git a/script/benchmark-two-node.sh b/script/benchmark-two-node.sh new file mode 100755 index 00000000..311eda24 --- /dev/null +++ b/script/benchmark-two-node.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +usage() { + cat <<'EOF' +Usage: benchmark-two-node.sh BINARY {udp|tcp} [OUTPUT_DIR] [REPEATS] [SECONDS] + +Runs two EasyTier cores in separate network namespaces and measures a +single iperf3 TCP flow in both directions. The underlay transport between +the EasyTier peers is selected by the second argument. + +Environment: + CORE_A_CPU CPU for the listener core (default: 8) + CORE_B_CPU CPU for the connector core (default: 10) + IPERF_SERVER_CPU CPU for the iperf3 server (default: 12) + IPERF_CLIENT_CPU CPU for the iperf3 client (default: 14) + IPERF_OMIT_SECONDS Warm-up omitted by iperf3 (default: 2) +EOF +} + +if [[ $# -lt 2 || $# -gt 5 ]]; then + usage >&2 + exit 2 +fi + +if [[ ${EUID} -ne 0 ]]; then + echo "benchmark must run as root" >&2 + exit 1 +fi + +binary=$(realpath "$1") +protocol=$2 +output_dir=${3:-} +repeats=${4:-3} +duration=${5:-10} + +case "$protocol" in + udp | tcp) ;; + *) + echo "unsupported EasyTier transport: $protocol" >&2 + exit 2 + ;; +esac + +if [[ ! -x "$binary" ]]; then + echo "EasyTier binary is not executable: $binary" >&2 + exit 1 +fi + +for command in ip iperf3 jq taskset; do + if ! command -v "$command" >/dev/null; then + echo "required command is missing: $command" >&2 + exit 1 + fi +done + +if [[ -z "$output_dir" ]]; then + output_dir=$(mktemp -d "/tmp/easytier-two-node-${protocol}.XXXXXX") +elif [[ -e "$output_dir" ]]; then + echo "output directory already exists: $output_dir" >&2 + exit 1 +else + mkdir -p "$output_dir" +fi +output_dir=$(realpath "$output_dir") + +core_a_cpu=${CORE_A_CPU:-8} +core_b_cpu=${CORE_B_CPU:-10} +iperf_server_cpu=${IPERF_SERVER_CPU:-12} +iperf_client_cpu=${IPERF_CLIENT_CPU:-14} +omit_seconds=${IPERF_OMIT_SECONDS:-2} + +namespace_a="etpa$$" +namespace_b="etpb$$" +veth_a="etva$$" +veth_b="etvb$$" +core_a_pid= +core_b_pid= +iperf_server_pid= + +cleanup() { + trap - EXIT INT TERM + for pid in "$iperf_server_pid" "$core_b_pid" "$core_a_pid"; do + if [[ -n "$pid" ]]; then + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + fi + done + ip netns delete "$namespace_b" 2>/dev/null || true + ip netns delete "$namespace_a" 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +ip netns add "$namespace_a" +ip netns add "$namespace_b" +ip link add "$veth_a" type veth peer name "$veth_b" +ip link set "$veth_a" netns "$namespace_a" +ip link set "$veth_b" netns "$namespace_b" +ip -n "$namespace_a" link set lo up +ip -n "$namespace_b" link set lo up +ip -n "$namespace_a" addr add 198.18.0.1/24 dev "$veth_a" +ip -n "$namespace_b" addr add 198.18.0.2/24 dev "$veth_b" +ip -n "$namespace_a" link set "$veth_a" up +ip -n "$namespace_b" link set "$veth_b" up + +common_args=( + --network-name codex-perf + --network-secret codex-perf + --disable-p2p true + --disable-tcp-hole-punching true + --disable-udp-hole-punching true + --disable-upnp true + --disable-ipv6 true + --multi-thread false + --mtu 1360 + --console-log-level warn +) + +ip netns exec "$namespace_a" taskset -c "$core_a_cpu" "$binary" \ + "${common_args[@]}" \ + --hostname perf-a \ + --instance-name perf-a \ + --ipv4 10.250.0.1 \ + --listeners "${protocol}://0.0.0.0:11010" \ + >"$output_dir/core-a.log" 2>&1 & +core_a_pid=$! + +sleep 1 + +ip netns exec "$namespace_b" taskset -c "$core_b_cpu" "$binary" \ + "${common_args[@]}" \ + --hostname perf-b \ + --instance-name perf-b \ + --ipv4 10.250.0.2 \ + --no-listener \ + --peers "${protocol}://198.18.0.1:11010" \ + >"$output_dir/core-b.log" 2>&1 & +core_b_pid=$! + +connected=false +for _ in $(seq 1 100); do + if ip netns exec "$namespace_b" ping -c 1 -W 1 10.250.0.1 \ + >"$output_dir/ping.log" 2>&1; then + connected=true + break + fi + if ! kill -0 "$core_a_pid" 2>/dev/null || + ! kill -0 "$core_b_pid" 2>/dev/null; then + echo "an EasyTier core exited before connectivity was established" >&2 + exit 1 + fi + sleep 0.2 +done + +if [[ "$connected" != true ]]; then + echo "EasyTier peers did not become reachable" >&2 + exit 1 +fi + +ip netns exec "$namespace_a" taskset -c "$iperf_server_cpu" iperf3 -s \ + >"$output_dir/iperf-server.log" 2>&1 & +iperf_server_pid=$! +sleep 0.5 + +ip netns exec "$namespace_b" taskset -c "$iperf_client_cpu" \ + iperf3 -c 10.250.0.1 -t 2 -O 1 --json \ + >"$output_dir/warmup.json" + +for direction in forward reverse; do + reverse_arg=() + if [[ "$direction" == reverse ]]; then + reverse_arg=(-R) + fi + for iteration in $(seq 1 "$repeats"); do + ip netns exec "$namespace_b" taskset -c "$iperf_client_cpu" \ + iperf3 -c 10.250.0.1 \ + -t "$duration" \ + -O "$omit_seconds" \ + --json \ + "${reverse_arg[@]}" \ + >"$output_dir/${direction}-${iteration}.json" + done +done + +median_bps() { + jq -s ' + map(.end.sum_received.bits_per_second) | sort | + if length % 2 == 1 then + .[length / 2 | floor] + else + (.[length / 2 - 1] + .[length / 2]) / 2 + end + ' "$@" +} + +forward_bps=$(median_bps "$output_dir"/forward-*.json) +reverse_bps=$(median_bps "$output_dir"/reverse-*.json) +jq -n \ + --arg binary "$binary" \ + --arg protocol "$protocol" \ + --argjson repeats "$repeats" \ + --argjson duration_seconds "$duration" \ + --argjson forward_bps "$forward_bps" \ + --argjson reverse_bps "$reverse_bps" \ + '{ + binary: $binary, + peer_transport: $protocol, + repeats: $repeats, + duration_seconds: $duration_seconds, + forward_bps: $forward_bps, + reverse_bps: $reverse_bps, + directional_median_bps: (($forward_bps + $reverse_bps) / 2) + }' | tee "$output_dir/summary.json" + +echo "results: $output_dir"