Merge branch 'main' into feat/dns-policy

# Conflicts:
#	easytier/src/common/log.rs
#	easytier/src/connector/dns_connector.rs
#	easytier/src/dns/system/windows.rs
#	easytier/src/gateway/socks5.rs
#	easytier/src/instance/dns_server/server.rs
#	easytier/src/instance/dns_server/server_instance.rs
#	easytier/src/instance/instance.rs
#	easytier/src/peers/peer_manager.rs
#	easytier/src/peers/peer_ospf_route.rs
#	easytier/src/proto/utils.rs
#	easytier/src/tunnel/common.rs
#	easytier/src/utils/panic.rs
This commit is contained in:
Luna Yao
2026-04-17 17:16:55 +02:00
184 changed files with 3768 additions and 8198 deletions
+7 -18
View File
@@ -110,11 +110,7 @@ pub mod instance {
ret += stats.rx_bytes;
}
if ret == 0 {
None
} else {
Some(ret)
}
if ret == 0 { None } else { Some(ret) }
}
pub fn get_tx_bytes(&self) -> Option<u64> {
@@ -127,11 +123,7 @@ pub mod instance {
ret += stats.tx_bytes;
}
if ret == 0 {
None
} else {
Some(ret)
}
if ret == 0 { None } else { Some(ret) }
}
pub fn get_loss_rate(&self) -> Option<f64> {
@@ -167,11 +159,7 @@ pub mod instance {
}
}
if ret.is_empty() {
None
} else {
Some(ret)
}
if ret.is_empty() { None } else { Some(ret) }
}
pub fn get_udp_nat_type(&self) -> String {
@@ -366,9 +354,10 @@ mod tests {
..Default::default()
};
assert!(pair
.get_loss_rate()
.is_some_and(|loss_rate| (loss_rate - 0.4).abs() < 1e-6));
assert!(
pair.get_loss_rate()
.is_some_and(|loss_rate| (loss_rate - 0.4).abs() < 1e-6)
);
}
#[test]
+12 -13
View File
@@ -4,10 +4,10 @@ use std::{
};
use anyhow::Context;
use base64::{prelude::BASE64_STANDARD, Engine as _};
use base64::{Engine as _, prelude::BASE64_STANDARD};
use strum::VariantArray;
use crate::tunnel::{packet_def::CompressorAlgo, IpScheme};
use crate::tunnel::{IpScheme, packet_def::CompressorAlgo};
include!(concat!(env!("OUT_DIR"), "/common.rs"));
@@ -292,18 +292,17 @@ impl fmt::Display for Url {
fn split_tunnel_scheme(raw_scheme: &str) -> Option<(&str, &'static str, bool)> {
for scheme in IpScheme::VARIANTS {
let scheme: &'static str = scheme.into();
if let Some(base) = raw_scheme.strip_suffix('6') {
if let Some(prefix) = base.strip_suffix(scheme) {
if prefix.is_empty() || prefix.ends_with('-') {
return Some((prefix, scheme, true));
}
}
if let Some(base) = raw_scheme.strip_suffix('6')
&& let Some(prefix) = base.strip_suffix(scheme)
&& (prefix.is_empty() || prefix.ends_with('-'))
{
return Some((prefix, scheme, true));
}
if let Some(prefix) = raw_scheme.strip_suffix(scheme) {
if prefix.is_empty() || prefix.ends_with('-') {
return Some((prefix, scheme, false));
}
if let Some(prefix) = raw_scheme.strip_suffix(scheme)
&& (prefix.is_empty() || prefix.ends_with('-'))
{
return Some((prefix, scheme, false));
}
}
@@ -536,7 +535,7 @@ impl SecureModeConfig {
#[cfg(test)]
mod tests {
use super::{normalize_tunnel_url, TunnelInfo, Url};
use super::{TunnelInfo, Url, normalize_tunnel_url};
fn assert_ipv6_tunnel_normalization(scheme: &str, port: u16) {
let expected = format!("{scheme}6://[2001:db8::1]:{port}");
+2 -2
View File
@@ -1,4 +1,4 @@
use std::sync::{atomic::AtomicBool, Arc, Mutex};
use std::sync::{Arc, Mutex, atomic::AtomicBool};
use futures::{SinkExt as _, StreamExt};
use tokio::{task::JoinSet, time::timeout};
@@ -6,7 +6,7 @@ use tokio::{task::JoinSet, time::timeout};
use crate::{
defer,
proto::rpc_types::error::Error,
tunnel::{packet_def::PacketType, ring::create_ring_tunnel_pair, Tunnel},
tunnel::{Tunnel, packet_def::PacketType, ring::create_ring_tunnel_pair},
};
use super::{client::Client, server::Server, service_registry::ServiceRegistry};
+2 -2
View File
@@ -12,15 +12,15 @@ use tokio_stream::StreamExt;
use crate::common::shrink_dashmap;
use crate::common::{
stats_manager::{LabelSet, LabelType, MetricName, StatsManager},
PeerId,
stats_manager::{LabelSet, LabelType, MetricName, StatsManager},
};
use crate::defer;
use crate::proto::common::{
CompressionAlgoPb, RpcCompressionInfo, RpcDescriptor, RpcPacket, RpcRequest, RpcResponse,
};
use crate::proto::rpc_impl::packet::{
build_rpc_packet, compress_packet, decompress_packet, BuildRpcPacketArgs,
BuildRpcPacketArgs, build_rpc_packet, compress_packet, decompress_packet,
};
use crate::proto::rpc_types::controller::Controller;
use crate::proto::rpc_types::descriptor::MethodDescriptor;
+1 -1
View File
@@ -1,4 +1,4 @@
use crate::tunnel::{mpsc::MpscTunnel, Tunnel};
use crate::tunnel::{Tunnel, mpsc::MpscTunnel};
pub type RpcController = super::rpc_types::controller::BaseController;
+224 -44
View File
@@ -1,17 +1,20 @@
use prost::Message as _;
use prost::{Message as _, length_delimiter_len};
use crate::{
common::{compressor::DefaultCompressor, PeerId},
common::{PeerId, compressor::DefaultCompressor},
proto::{
common::{CompressionAlgoPb, RpcCompressionInfo, RpcDescriptor, RpcPacket},
rpc_types::error::Error,
},
tunnel::packet_def::{CompressorAlgo, PacketType, ZCPacket},
tunnel::packet_def::{CompressorAlgo, PacketType, TAIL_RESERVED_SIZE, ZCPacket, ZCPacketType},
};
use super::RpcTransactId;
const RPC_PACKET_CONTENT_MTU: usize = 1300;
// Budget the final UDP payload size on the wire for peer RPC over `udp://`.
// This includes EasyTier's UDP tunnel header, peer header, and reserved tail
// space for encryption/compression metadata, but excludes the outer IP header.
const RPC_PACKET_UDP_PAYLOAD_BUDGET: usize = 1300;
pub async fn compress_packet(
accepted_compression_algo: CompressionAlgoPb,
@@ -150,44 +153,166 @@ pub struct BuildRpcPacketArgs<'a> {
pub compression_info: RpcCompressionInfo,
}
// Fixed transport overhead for peer RPC carried by EasyTier's UDP tunnel:
//
// UDP payload budget
// +-------------------------------------------------------------------------+
// | EasyTier UDP tunnel hdr | PeerManager hdr | RpcPacket bytes | tail room |
// +-------------------------------------------------------------------------+
// |<------ ZCPacketType::UDP payload_offset ------>|<-- TAIL_RESERVED_SIZE -->|
//
// `udp_rpc_tunnel_overhead()` is everything except `RpcPacket bytes`.
fn udp_rpc_tunnel_overhead() -> usize {
ZCPacketType::UDP.get_packet_offsets().payload_offset + TAIL_RESERVED_SIZE
}
// Maximum encoded RpcPacket size we can admit before adding it to a UDP tunnel.
// This budget excludes the outer UDP/IP headers because the caller only controls
// the EasyTier payload carried inside the UDP datagram.
fn max_rpc_packet_encoded_len_for_udp() -> usize {
RPC_PACKET_UDP_PAYLOAD_BUDGET.saturating_sub(udp_rpc_tunnel_overhead())
}
// Build one logical RpcPacket piece. This is reused both for the actual output
// packets and for sizing templates that estimate worst-case protobuf overhead.
fn build_rpc_piece(
args: &BuildRpcPacketArgs<'_>,
total_pieces: u32,
piece_idx: u32,
body: &[u8],
) -> RpcPacket {
RpcPacket {
from_peer: args.from_peer,
to_peer: args.to_peer,
descriptor: if piece_idx == 0
|| args.compression_info.algo == CompressionAlgoPb::None as i32
{
// old version must have descriptor on every piece
Some(args.rpc_desc.clone())
} else {
None
},
is_request: args.is_req,
total_pieces,
piece_idx,
transaction_id: args.transaction_id,
body: body.to_vec(),
trace_id: args.trace_id,
compression_info: if piece_idx == 0 {
Some(args.compression_info)
} else {
None
},
}
}
fn pick_piece_len_for_budget(
base_encoded_len_without_body: usize,
remaining: usize,
max_encoded_len: usize,
) -> usize {
if remaining == 0 {
return 0;
}
// Minimum non-empty body field encoding cost:
// body tag (1 byte) + body length (1 byte) + body data (1 byte)
if base_encoded_len_without_body + 3 > max_encoded_len {
tracing::warn!(
base_encoded_len_without_body,
max_encoded_len,
"rpc metadata exceeds udp payload budget; falling back to a minimal piece"
);
return 1;
}
// `budget` is what remains for the protobuf `body` field after all fixed
// RpcPacket metadata has been accounted for.
let budget = max_encoded_len - base_encoded_len_without_body;
// Reserve the bytes field wrapper conservatively, then use the rest for
// the body itself.
//
// Encoded RpcPacket layout relevant to `body`:
//
// +------------------------------- max_encoded_len -------------------------------+
// | fixed RpcPacket fields | body tag (1B) | body len varint (worst-case) | body |
// +--------------------------------------------------------------------------- --+
// ^ ^
// | `- reserve by using the varint width of `budget`
// `- base_encoded_len_without_body
//
// This is intentionally conservative. A few bytes may be left unused, but
// every piece stays within the UDP payload budget without iterative sizing.
let reserved_for_body_header = 1 + length_delimiter_len(budget);
remaining
.min(budget.saturating_sub(reserved_for_body_header))
.max(1)
}
// Pre-split the raw RPC content using conservative worst-case protobuf sizing.
// We compute separate base sizes for the first piece and later pieces because
// only the first piece carries `compression_info`, and old compatibility rules
// may also force `descriptor` to appear on every piece.
//
// Split flow:
//
// raw RPC content
// +--------------------------------------------------------------+
// | args.content |
// +--------------------------------------------------------------+
// | first piece uses first_piece_base_len
// | later pieces use other_piece_base_len
// v
// +-----------+-----------+-----------+----- ...
// | offset,len| offset,len| offset,len|
// +-----------+-----------+-----------+----- ...
//
// The result is only a slicing plan. Actual RpcPacket objects are built later
// with the real `total_pieces`.
fn split_rpc_content_for_udp_budget(args: &BuildRpcPacketArgs<'_>) -> Vec<(usize, usize)> {
if args.content.is_empty() {
return vec![(0, 0)];
}
let max_encoded_len = max_rpc_packet_encoded_len_for_udp().max(1);
// Use the worst-case varint width for piece counters so the budget remains
// valid without iterating on `total_pieces`/`piece_idx`.
let first_piece_base_len = build_rpc_piece(args, u32::MAX, 0, &[]).encoded_len();
let other_piece_base_len = build_rpc_piece(args, u32::MAX, u32::MAX, &[]).encoded_len();
let mut pieces = Vec::new();
let mut offset = 0usize;
while offset < args.content.len() {
// First and subsequent pieces have different metadata shapes, so they
// use different fixed-size templates.
let base_len = if pieces.is_empty() {
first_piece_base_len
} else {
other_piece_base_len
};
let piece_len =
pick_piece_len_for_budget(base_len, args.content.len() - offset, max_encoded_len);
pieces.push((offset, piece_len));
offset += piece_len;
}
pieces
}
// Build the final transport packets after the payload has been split. We do the
// actual `total_pieces` assignment only here so the wire packet stays accurate,
// while the earlier sizing step remains simple and conservatively safe.
pub fn build_rpc_packet(args: BuildRpcPacketArgs<'_>) -> Vec<ZCPacket> {
let mut ret = Vec::new();
let content_mtu = RPC_PACKET_CONTENT_MTU;
let total_pieces = args.content.len().div_ceil(content_mtu);
let mut cur_offset = 0;
while cur_offset < args.content.len() || args.content.is_empty() {
let mut cur_len = content_mtu;
if cur_offset + cur_len > args.content.len() {
cur_len = args.content.len() - cur_offset;
}
let mut cur_content = Vec::new();
cur_content.extend_from_slice(&args.content[cur_offset..cur_offset + cur_len]);
let cur_packet = RpcPacket {
from_peer: args.from_peer,
to_peer: args.to_peer,
descriptor: if cur_offset == 0
|| args.compression_info.algo == CompressionAlgoPb::None as i32
{
// old version must have descriptor on every piece
Some(args.rpc_desc.clone())
} else {
None
},
is_request: args.is_req,
total_pieces: total_pieces as u32,
piece_idx: (cur_offset / RPC_PACKET_CONTENT_MTU) as u32,
transaction_id: args.transaction_id,
body: cur_content,
trace_id: args.trace_id,
compression_info: if cur_offset == 0 {
Some(args.compression_info)
} else {
None
},
};
cur_offset += cur_len;
let pieces = split_rpc_content_for_udp_budget(&args);
let total_pieces = pieces.len() as u32;
for (piece_idx, (offset, len)) in pieces.into_iter().enumerate() {
let cur_packet = build_rpc_piece(
&args,
total_pieces,
piece_idx as u32,
&args.content[offset..offset + len],
);
let packet_type = if args.is_req {
PacketType::RpcReq
@@ -200,11 +325,66 @@ pub fn build_rpc_packet(args: BuildRpcPacketArgs<'_>) -> Vec<ZCPacket> {
let mut zc_packet = ZCPacket::new_with_payload(&buf);
zc_packet.fill_peer_manager_hdr(args.from_peer, args.to_peer, packet_type as u8);
ret.push(zc_packet);
if args.content.is_empty() {
break;
}
}
ret
}
#[cfg(test)]
mod tests {
use super::*;
fn build_test_args<'a>(
content: &'a [u8],
compression_algo: CompressionAlgoPb,
) -> BuildRpcPacketArgs<'a> {
BuildRpcPacketArgs {
from_peer: 11,
to_peer: 22,
rpc_desc: RpcDescriptor {
domain_name: "very-long-domain-name-for-rpc-packet-budget-check".repeat(2),
proto_name: "extremely.verbose.proto.name.for.rpc.packet.tests".repeat(2),
service_name: "LargeMetadataServiceForRpcPacketBudget".repeat(2),
method_index: 7,
},
transaction_id: 33,
is_req: true,
content,
trace_id: 44,
compression_info: RpcCompressionInfo {
algo: compression_algo.into(),
accepted_algo: CompressionAlgoPb::Zstd.into(),
},
}
}
fn udp_packet_size_after_tail(packet: &ZCPacket) -> usize {
ZCPacketType::UDP.get_packet_offsets().payload_offset
+ packet.payload_len()
+ TAIL_RESERVED_SIZE
}
#[test]
fn build_rpc_packet_respects_udp_budget_with_large_metadata() {
let content = vec![0x5a; 4096];
let packets = build_rpc_packet(build_test_args(&content, CompressionAlgoPb::None));
assert!(packets.len() > 1);
for packet in packets {
assert!(
udp_packet_size_after_tail(&packet) <= RPC_PACKET_UDP_PAYLOAD_BUDGET,
"packet size {} exceeded budget {}",
udp_packet_size_after_tail(&packet),
RPC_PACKET_UDP_PAYLOAD_BUDGET
);
}
}
#[test]
fn build_rpc_packet_respects_udp_budget_for_empty_payload() {
let packets = build_rpc_packet(build_test_args(&[], CompressionAlgoPb::Zstd));
assert_eq!(1, packets.len());
assert!(udp_packet_size_after_tail(&packets[0]) <= RPC_PACKET_UDP_PAYLOAD_BUDGET);
}
}
+4 -5
View File
@@ -11,9 +11,8 @@ use tokio_stream::StreamExt;
use crate::{
common::{
join_joinset_background,
PeerId, join_joinset_background,
stats_manager::{LabelSet, LabelType, MetricName, StatsManager},
PeerId,
},
proto::{
common::{
@@ -24,16 +23,16 @@ use crate::{
rpc_types::{controller::Controller, error::Result},
},
tunnel::{
Tunnel, ZCPacketStream,
mpsc::{MpscTunnel, MpscTunnelSender},
ring::create_ring_tunnel_pair,
Tunnel, ZCPacketStream,
},
};
use super::{
packet::{build_rpc_packet, compress_packet, decompress_packet, PacketMerger},
service_registry::ServiceRegistry,
RpcController, Transport,
packet::{PacketMerger, build_rpc_packet, compress_packet, decompress_packet},
service_registry::ServiceRegistry,
};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+2 -2
View File
@@ -1,5 +1,5 @@
use std::{
sync::{atomic::AtomicU32, Arc, Mutex},
sync::{Arc, Mutex, atomic::AtomicU32},
time::Duration,
};
@@ -204,8 +204,8 @@ mod tests {
use crate::{
proto::rpc_impl::standalone::StandAloneServer,
tunnel::{
tcp::{TcpTunnelConnector, TcpTunnelListener},
TunnelConnector as _,
tcp::{TcpTunnelConnector, TcpTunnelListener},
},
};
+1 -1
View File
@@ -191,8 +191,8 @@ impl TestContext {
}
fn random_string(len: usize) -> String {
use rand::distributions::Alphanumeric;
use rand::Rng;
use rand::distributions::Alphanumeric;
let mut rng = rand::thread_rng();
let s: Vec<u8> = std::iter::repeat(())
.map(|()| rng.sample(Alphanumeric))
+54 -8
View File
@@ -1,6 +1,27 @@
use delegate::delegate;
use derivative::Derivative;
use derive_more::{Deref, DerefMut, From, IntoIterator};
use prost::Message;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
/// Generates a stable digest strictly within the lifecycle of the current process.
///
/// ⚠️ WARNING:
/// - This digest is ONLY guaranteed to be deterministic within a **single process and the exact same binary build**.
pub trait TransientDigest: Message {
fn digest(&self) -> [u8; 32]
where
Self: Sized,
{
let buf = self.encode_to_vec();
let mut hasher = Sha256::new();
hasher.update(buf);
hasher.finalize().into()
}
}
impl<S: Message> TransientDigest for S {}
pub trait MessageModel<Message: prost::Message>:
Into<Message> + for<'m> TryFrom<&'m Message>
@@ -39,20 +60,45 @@ impl<Model> RepeatedMessageModel<Model> {
}
}
impl<'m, Message, Model> TryFrom<&'m Vec<Message>> for RepeatedMessageModel<Model>
impl<Model> FromIterator<Model> for RepeatedMessageModel<Model> {
fn from_iter<I: IntoIterator<Item = Model>>(iter: I) -> Self {
Self(iter.into_iter().collect())
}
}
impl<Model> Extend<Model> for RepeatedMessageModel<Model> {
delegate! {
to self.0 {
fn extend<T: IntoIterator<Item = Model>>(&mut self, iter: T);
}
}
}
impl<Model> AsRef<[Model]> for RepeatedMessageModel<Model> {
delegate! {
to self.0 {
fn as_ref(&self) -> &[Model];
}
}
}
impl<Model> AsMut<[Model]> for RepeatedMessageModel<Model> {
delegate! {
to self.0 {
fn as_mut(&mut self) -> &mut [Model];
}
}
}
impl<'m, Message, Model> TryFrom<&'m [Message]> for RepeatedMessageModel<Model>
where
Message: prost::Message,
Model: MessageModel<Message>,
{
type Error = <Model as TryFrom<&'m Message>>::Error;
fn try_from(value: &'m Vec<Message>) -> Result<Self, Self::Error> {
Ok(Self(
value
.iter()
.map(TryInto::try_into)
.collect::<Result<_, _>>()?,
))
fn try_from(value: &'m [Message]) -> Result<Self, Self::Error> {
value.iter().map(TryInto::try_into).collect()
}
}