perf(easytier-web): improve easytier-web webhook performance (#2383)

* feat(web): reconcile managed config revisions
* feat(web): cache managed runtime configs per session
* test: cover managed web config delivery
This commit is contained in:
KKRainbow
2026-06-28 13:14:40 +08:00
committed by GitHub
parent 7205517160
commit 9cb3833216
21 changed files with 6037 additions and 995 deletions
+3
View File
@@ -40,6 +40,9 @@ cli:
geoip_db:
en: "The path to the GeoIP2 database file, used to lookup the location of the client, default is the embedded file (only country information) , recommend https://github.com/P3TERX/GeoLite.mmdb"
zh-CN: "GeoIP2 数据库文件路径,用于查找客户端的位置,默认为嵌入文件(仅国家信息),推荐 https://github.com/P3TERX/GeoLite.mmdb"
heartbeat_min_response_ms:
en: "Minimum response time for config-server heartbeat RPCs in milliseconds, default is 0"
zh-CN: "配置服务心跳 RPC 的最短响应时间,单位毫秒,默认为 0"
disable_registration:
en: "Disable user registration"
zh-CN: "禁用用户注册"
File diff suppressed because it is too large Load Diff
+687 -5
View File
@@ -1,3 +1,5 @@
mod managed_config;
mod runtime_reconcile;
pub mod session;
pub mod storage;
@@ -5,6 +7,7 @@ use std::sync::{
Arc,
atomic::{AtomicU32, Ordering},
};
use std::time::Duration;
use dashmap::DashMap;
use easytier::{
@@ -30,6 +33,10 @@ use crate::db::{Db, UserIdInDb, entity::user_running_network_configs};
#[include = "geoip2-cn.mmdb"]
struct GeoipDb;
pub fn is_managed_config_revision_conflict(error: &anyhow::Error) -> bool {
managed_config::is_revision_conflict(error)
}
fn load_geoip_db(geoip_db: Option<String>) -> Option<maxminddb::Reader<Vec<u8>>> {
if let Some(path) = geoip_db {
match maxminddb::Reader::open_readfile(&path) {
@@ -63,12 +70,14 @@ pub struct ClientManager {
webhook_config: SharedWebhookConfig,
geoip_db: Arc<Option<maxminddb::Reader<Vec<u8>>>>,
heartbeat_min_response_delay: Duration,
}
impl ClientManager {
pub fn new(
db: Db,
geoip_db: Option<String>,
heartbeat_min_response_delay: Duration,
feature_flags: Arc<FeatureFlags>,
webhook_config: SharedWebhookConfig,
) -> Self {
@@ -92,6 +101,7 @@ impl ClientManager {
webhook_config,
geoip_db: Arc::new(load_geoip_db(geoip_db)),
heartbeat_min_response_delay,
}
}
@@ -105,6 +115,7 @@ impl ClientManager {
let storage = self.storage.weak_ref();
let listeners_cnt = self.listeners_cnt.clone();
let geoip_db = self.geoip_db.clone();
let heartbeat_min_response_delay = self.heartbeat_min_response_delay;
let feature_flags = self.feature_flags.clone();
let webhook_config = self.webhook_config.clone();
self.tasks.spawn(async move {
@@ -129,6 +140,7 @@ impl ClientManager {
storage.clone(),
client_url.clone(),
location,
heartbeat_min_response_delay,
feature_flags.clone(),
webhook_config.clone(),
);
@@ -149,6 +161,10 @@ impl ClientManager {
self.storage.list_clients()
}
pub async fn list_all_sessions(&self) -> Vec<StorageToken> {
self.storage.list_all_clients()
}
pub fn get_session_by_machine_id(
&self,
user_id: UserIdInDb,
@@ -169,7 +185,7 @@ impl ClientManager {
) -> bool {
let Some(client_url) = self
.storage
.get_client_url_by_machine_id(user_id, machine_id)
.get_client_url_by_machine_id_with_auth(user_id, machine_id, false)
else {
return false;
};
@@ -189,14 +205,30 @@ impl ClientManager {
user_id: UserIdInDb,
machine_id: uuid::Uuid,
desired_configs: Vec<ManagedNetworkConfig>,
config_revision: Option<String>,
expected_config_revision: Option<String>,
) -> anyhow::Result<()> {
session::SessionRpcService::reconcile_web_source_configs(
let expected_config_revision = match expected_config_revision.as_deref().map(str::trim) {
None => managed_config::ExpectedConfigRevision::Any,
Some("") => managed_config::ExpectedConfigRevision::Exact(None),
Some(revision) => managed_config::ExpectedConfigRevision::Exact(Some(revision)),
};
managed_config::reconcile_web_source_configs(
&self.storage,
user_id,
machine_id,
desired_configs,
config_revision.as_deref(),
expected_config_revision,
)
.await?;
if let Some(config_revision) = config_revision
&& let Some(session) = self.get_session_by_machine_id(user_id, &machine_id)
{
session
.notify_config_revision_changed(user_id, machine_id, config_revision)
.await;
}
Ok(())
}
@@ -331,19 +363,449 @@ impl
#[cfg(test)]
mod tests {
use std::{sync::Arc, time::Duration};
use std::{
collections::VecDeque,
sync::{
Arc,
atomic::{AtomicBool, AtomicUsize, Ordering},
},
time::Duration,
};
use axum::{Json, Router, extract::State, routing::post};
use easytier::{
common::MachineIdOptions,
instance_manager::NetworkInstanceManager,
proto::{
api::manage::{NetworkConfig, NetworkingMethod, PortForwardConfig},
common::CompressionAlgoPb,
},
rpc_service::remote_client::Storage as RemoteStorage,
tunnel::{
common::tests::wait_for_condition,
udp::{UdpTunnelConnector, UdpTunnelListener},
},
web_client::WebClient,
web_client::{WebClient, run_web_client},
};
use serde_json::json;
use sqlx::Executor;
use tokio::net::UdpSocket;
use crate::{FeatureFlags, client_manager::ClientManager, db::Db};
use crate::{
FeatureFlags, client_manager::ClientManager, db::Db, webhook::ManagedNetworkConfig,
};
const MANAGED_CONFIG_TOKEN: &str = "managed-config-token";
#[derive(Debug, Clone)]
struct TestWebhookState {
validate_responses: Arc<tokio::sync::Mutex<VecDeque<bool>>>,
validate_count: Arc<AtomicUsize>,
block_second_validate: Arc<AtomicBool>,
allow_second_validate: Arc<AtomicBool>,
}
impl TestWebhookState {
fn new(validate_responses: impl IntoIterator<Item = bool>) -> Self {
Self {
validate_responses: Arc::new(tokio::sync::Mutex::new(
validate_responses.into_iter().collect(),
)),
validate_count: Arc::new(AtomicUsize::new(0)),
block_second_validate: Arc::new(AtomicBool::new(false)),
allow_second_validate: Arc::new(AtomicBool::new(true)),
}
}
fn with_blocked_second_validate(
validate_responses: impl IntoIterator<Item = bool>,
) -> Self {
let state = Self::new(validate_responses);
state.block_second_validate.store(true, Ordering::Release);
state.allow_second_validate.store(false, Ordering::Release);
state
}
fn allow_second_validate(&self) {
self.allow_second_validate.store(true, Ordering::Release);
}
fn validate_count(&self) -> usize {
self.validate_count.load(Ordering::Acquire)
}
}
async fn validate_token_handler(
State(state): State<TestWebhookState>,
) -> Json<serde_json::Value> {
let count = state.validate_count.fetch_add(1, Ordering::AcqRel) + 1;
if count == 2 && state.block_second_validate.load(Ordering::Acquire) {
while !state.allow_second_validate.load(Ordering::Acquire) {
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
let valid = state
.validate_responses
.lock()
.await
.pop_front()
.unwrap_or(true);
if !valid {
return Json(json!({ "valid": false }));
}
Json(json!({
"valid": true,
"binding_version": count,
"config_revision": format!("validated-rev-{count}")
}))
}
async fn webhook_ack_handler() -> Json<serde_json::Value> {
Json(json!({}))
}
async fn test_webhook_config() -> (
crate::webhook::SharedWebhookConfig,
tokio::task::JoinHandle<()>,
TestWebhookState,
) {
let state = TestWebhookState::new([true]);
test_webhook_config_with_state(state).await
}
async fn test_webhook_config_with_state(
state: TestWebhookState,
) -> (
crate::webhook::SharedWebhookConfig,
tokio::task::JoinHandle<()>,
TestWebhookState,
) {
let app = Router::new()
.route("/validate-token", post(validate_token_handler))
.route("/webhook/node-connected", post(webhook_ack_handler))
.route("/webhook/node-disconnected", post(webhook_ack_handler))
.with_state(state.clone());
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
(
Arc::new(crate::webhook::WebhookConfig::new(
Some(format!("http://{addr}")),
None,
None,
None,
None,
)),
server,
state,
)
}
async fn add_random_udp_listener(mgr: &mut ClientManager) -> std::net::SocketAddr {
let socket = Arc::new(UdpSocket::bind("127.0.0.1:0").await.unwrap());
let addr = socket.local_addr().unwrap();
let listener =
UdpTunnelListener::new_with_socket(format!("udp://{addr}").parse().unwrap(), socket);
mgr.add_listener(listener).await.unwrap();
addr
}
async fn wait_for_validated_user(mgr: &ClientManager, machine_id: uuid::Uuid) -> i32 {
tokio::time::timeout(Duration::from_secs(12), async {
loop {
if let Some(token) = mgr.list_sessions().await.into_iter().find(|token| {
token.token == MANAGED_CONFIG_TOKEN && token.machine_id == machine_id
}) {
break token.user_id;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
})
.await
.unwrap()
}
async fn wait_for_validate_count(state: &TestWebhookState, target: usize) {
tokio::time::timeout(Duration::from_secs(12), async {
loop {
if state.validate_count() >= target {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.unwrap();
}
async fn wait_for_session_urls(mgr: &ClientManager) -> Vec<url::Url> {
tokio::time::timeout(Duration::from_secs(12), async {
loop {
let urls = mgr
.client_sessions
.iter()
.map(|entry| entry.key().clone())
.collect::<Vec<_>>();
if !urls.is_empty() {
break urls;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
})
.await
.unwrap()
}
fn managed_config(
instance_id: uuid::Uuid,
network_config: serde_json::Value,
) -> ManagedNetworkConfig {
ManagedNetworkConfig {
instance_id: instance_id.to_string(),
network_config,
}
}
async fn wait_for_runtime_config(
manager: &NetworkInstanceManager,
inst_id: uuid::Uuid,
predicate: impl Fn(&NetworkConfig) -> bool,
) -> NetworkConfig {
tokio::time::timeout(Duration::from_secs(12), async {
loop {
if let Some(config) = manager
.get_instance_config(&inst_id)
.and_then(|config| NetworkConfig::new_from_config(&config).ok())
.filter(|config| predicate(config))
{
break config;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
})
.await
.unwrap()
}
async fn start_web_client_for_test(
config_server_addr: std::net::SocketAddr,
machine_id: uuid::Uuid,
manager: Arc<NetworkInstanceManager>,
) -> WebClient {
run_web_client(
&format!("udp://{config_server_addr}/{MANAGED_CONFIG_TOKEN}"),
MachineIdOptions {
explicit_machine_id: Some(machine_id.to_string()),
state_dir: None,
},
Some("managed-config-core".to_string()),
false,
manager,
None,
)
.await
.unwrap()
}
async fn clear_managed_config_db(
mgr: &ClientManager,
user_id: i32,
machine_id: uuid::Uuid,
instance_id: uuid::Uuid,
) {
mgr.db()
.delete_web_network_configs((user_id, machine_id), &[instance_id])
.await
.unwrap();
sqlx::query("DELETE FROM managed_config_revisions WHERE user_id = ? AND device_id = ?")
.bind(user_id)
.bind(machine_id.to_string())
.execute(&mgr.db().inner())
.await
.unwrap();
}
fn assert_updated_runtime_config(updated: &NetworkConfig, instance_id: uuid::Uuid) {
assert_eq!(
updated.instance_id.as_deref(),
Some(instance_id.to_string().as_str())
);
assert_eq!(updated.dhcp, Some(false));
assert_eq!(updated.virtual_ipv4.as_deref(), Some("10.88.0.7"));
assert_eq!(updated.network_length, Some(24));
assert_eq!(updated.hostname.as_deref(), Some("managed-updated-host"));
assert_eq!(updated.network_name.as_deref(), Some("managed-updated"));
assert_eq!(updated.network_secret.as_deref(), Some("secret-updated"));
assert_eq!(
updated.networking_method,
Some(NetworkingMethod::Manual as i32)
);
assert_eq!(updated.peer_urls, vec!["tcp://127.0.0.1:11010".to_string()]);
assert_eq!(
updated.proxy_cidrs,
vec![
"10.44.0.0/24".to_string(),
"10.45.0.0/24->10.46.0.0/24".to_string()
]
);
assert_eq!(updated.no_tun, Some(true));
assert_eq!(updated.disable_ipv6, Some(true));
assert_eq!(updated.enable_kcp_proxy, Some(true));
assert_eq!(updated.disable_kcp_input, Some(true));
assert_eq!(updated.enable_quic_proxy, Some(true));
assert_eq!(updated.disable_quic_input, Some(true));
assert_eq!(updated.disable_p2p, Some(true));
assert_eq!(updated.p2p_only, Some(true));
assert_eq!(updated.lazy_p2p, Some(true));
assert_eq!(updated.relay_all_peer_rpc, Some(true));
assert_eq!(updated.need_p2p, Some(true));
assert_eq!(updated.multi_thread, Some(false));
assert_eq!(updated.proxy_forward_by_system, Some(true));
assert_eq!(updated.disable_encryption, Some(true));
assert_eq!(updated.enable_relay_network_whitelist, Some(true));
assert_eq!(
updated.relay_network_whitelist,
vec!["10.44.0.0/24".to_string(), "10.45.0.0/24".to_string()]
);
assert_eq!(updated.enable_manual_routes, Some(true));
assert_eq!(
updated.routes,
vec!["10.60.0.0/16".to_string(), "10.61.0.0/16".to_string()]
);
assert_eq!(updated.port_forwards[0].bind_ip, "127.0.0.1");
assert_eq!(updated.port_forwards[0].bind_port, 0);
assert_eq!(updated.port_forwards[0].dst_ip, "10.88.0.8");
assert_eq!(updated.port_forwards[0].dst_port, 80);
assert_eq!(updated.port_forwards[0].proto, "tcp");
assert_eq!(updated.disable_udp_hole_punching, Some(true));
assert_eq!(updated.disable_tcp_hole_punching, Some(true));
assert_eq!(updated.disable_sym_hole_punching, Some(true));
assert_eq!(updated.disable_upnp, Some(true));
assert_eq!(updated.disable_relay_data, Some(true));
assert_eq!(updated.enable_magic_dns, Some(true));
assert_eq!(updated.enable_private_mode, Some(true));
assert_eq!(updated.mtu, Some(1360));
assert_eq!(
updated.data_compress_algo,
Some(CompressionAlgoPb::Zstd as i32)
);
assert_eq!(updated.encryption_algorithm.as_deref(), Some("xor"));
assert_eq!(updated.instance_recv_bps_limit, Some(123456));
assert_eq!(updated.enable_udp_broadcast_relay, Some(true));
assert_eq!(updated.socket_mark, Some(0));
}
fn initial_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value {
json!({
"instance_id": inst_id.to_string(),
"dhcp": true,
"network_name": "managed-initial",
"network_secret": "secret-initial",
"networking_method": "Standalone",
"no_tun": true,
"disable_ipv6": true,
"enable_kcp_proxy": false,
"disable_kcp_input": false,
"relay_all_peer_rpc": false,
"multi_thread": false,
"disable_relay_data": false,
"mtu": 1380
})
}
fn updated_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value {
serde_json::to_value(NetworkConfig {
instance_id: Some(inst_id.to_string()),
dhcp: Some(false),
virtual_ipv4: Some("10.88.0.7".to_string()),
network_length: Some(24),
hostname: Some("managed-updated-host".to_string()),
network_name: Some("managed-updated".to_string()),
network_secret: Some("secret-updated".to_string()),
networking_method: Some(NetworkingMethod::Manual as i32),
peer_urls: vec!["tcp://127.0.0.1:11010".to_string()],
proxy_cidrs: vec![
"10.44.0.0/24".to_string(),
"10.45.0.0/24->10.46.0.0/24".to_string(),
],
no_tun: Some(true),
disable_ipv6: Some(true),
enable_kcp_proxy: Some(true),
disable_kcp_input: Some(true),
enable_quic_proxy: Some(true),
disable_quic_input: Some(true),
disable_p2p: Some(true),
p2p_only: Some(true),
lazy_p2p: Some(true),
relay_all_peer_rpc: Some(true),
need_p2p: Some(true),
multi_thread: Some(false),
proxy_forward_by_system: Some(true),
disable_encryption: Some(true),
enable_relay_network_whitelist: Some(true),
relay_network_whitelist: vec!["10.44.0.0/24".to_string(), "10.45.0.0/24".to_string()],
enable_manual_routes: Some(true),
routes: vec!["10.60.0.0/16".to_string(), "10.61.0.0/16".to_string()],
port_forwards: vec![PortForwardConfig {
bind_ip: "127.0.0.1".to_string(),
bind_port: 0,
dst_ip: "10.88.0.8".to_string(),
dst_port: 80,
proto: "tcp".to_string(),
}],
disable_udp_hole_punching: Some(true),
disable_tcp_hole_punching: Some(true),
disable_sym_hole_punching: Some(true),
disable_upnp: Some(true),
disable_relay_data: Some(true),
enable_magic_dns: Some(true),
enable_private_mode: Some(true),
mtu: Some(1360),
data_compress_algo: Some(CompressionAlgoPb::Zstd as i32),
encryption_algorithm: Some("xor".to_string()),
instance_recv_bps_limit: Some(123456),
enable_udp_broadcast_relay: Some(true),
socket_mark: Some(0),
..Default::default()
})
.unwrap()
}
fn redelivered_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value {
serde_json::to_value(NetworkConfig {
instance_id: Some(inst_id.to_string()),
dhcp: Some(false),
virtual_ipv4: Some("10.88.0.7".to_string()),
network_length: Some(24),
hostname: Some("managed-redelivered-host".to_string()),
network_name: Some("managed-redelivered".to_string()),
network_secret: Some("secret-updated".to_string()),
networking_method: Some(NetworkingMethod::Manual as i32),
peer_urls: vec!["tcp://127.0.0.1:11010".to_string()],
proxy_cidrs: vec![
"10.44.0.0/24".to_string(),
"10.45.0.0/24->10.46.0.0/24".to_string(),
],
no_tun: Some(true),
disable_ipv6: Some(true),
enable_kcp_proxy: Some(true),
disable_kcp_input: Some(true),
relay_all_peer_rpc: Some(true),
need_p2p: Some(true),
multi_thread: Some(false),
enable_private_mode: Some(true),
mtu: Some(1360),
data_compress_algo: Some(CompressionAlgoPb::Zstd as i32),
encryption_algorithm: Some("xor".to_string()),
instance_recv_bps_limit: Some(654321),
..Default::default()
})
.unwrap()
}
#[tokio::test]
async fn test_client() {
@@ -351,6 +813,7 @@ mod tests {
let mut mgr = ClientManager::new(
Db::memory_db().await,
None,
Duration::ZERO,
Arc::new(FeatureFlags::default()),
Arc::new(crate::webhook::WebhookConfig::new(
None, None, None, None, None,
@@ -410,4 +873,223 @@ mod tests {
println!("{:?}", req);
println!("{:?}", mgr);
}
#[tokio::test]
async fn managed_web_config_revision_updates_running_core_config() {
let (webhook_config, webhook_server, _) = test_webhook_config().await;
let mut mgr = ClientManager::new(
Db::memory_db().await,
None,
Duration::ZERO,
Arc::new(FeatureFlags::default()),
webhook_config,
);
let config_server_addr = add_random_udp_listener(&mut mgr).await;
let machine_id = uuid::Uuid::new_v4();
let instance_id = uuid::Uuid::new_v4();
let core_manager = Arc::new(NetworkInstanceManager::new());
let client =
start_web_client_for_test(config_server_addr, machine_id, core_manager.clone()).await;
let user_id = wait_for_validated_user(&mgr, machine_id).await;
mgr.reconcile_managed_network_configs(
user_id,
machine_id,
vec![managed_config(
instance_id,
initial_managed_network_config(instance_id),
)],
Some("rev-initial".to_string()),
None,
)
.await
.unwrap();
wait_for_runtime_config(&core_manager, instance_id, |config| {
config.network_name.as_deref() == Some("managed-initial")
})
.await;
// Online revision update: web-owned running config is fully overwritten
// when non-hot-patch flags such as enable_kcp_proxy change.
mgr.reconcile_managed_network_configs(
user_id,
machine_id,
vec![managed_config(
instance_id,
updated_managed_network_config(instance_id),
)],
Some("rev-updated".to_string()),
Some("rev-initial".to_string()),
)
.await
.unwrap();
let updated = wait_for_runtime_config(&core_manager, instance_id, |config| {
config.network_name.as_deref() == Some("managed-updated")
&& config.enable_kcp_proxy == Some(true)
&& config.port_forwards.len() == 1
})
.await;
assert_updated_runtime_config(&updated, instance_id);
assert_eq!(
core_manager.get_instance_network_config_source(&instance_id),
Some(easytier::common::config::ConfigSource::Web)
);
assert_eq!(
mgr.db()
.get_managed_config_revision((user_id, machine_id))
.await
.unwrap()
.as_deref(),
Some("rev-updated")
);
// Web DB loss path: clear web-owned config and revision, then simulate
// the webhook re-posting the authoritative desired config. The already
// connected session should receive the distinguishable re-delivered
// revision without restarting.
clear_managed_config_db(&mgr, user_id, machine_id, instance_id).await;
assert!(
mgr.db()
.get_network_config((user_id, machine_id), &instance_id.to_string())
.await
.unwrap()
.is_none()
);
assert!(
mgr.db()
.get_managed_config_revision((user_id, machine_id))
.await
.unwrap()
.is_none()
);
mgr.reconcile_managed_network_configs(
user_id,
machine_id,
vec![managed_config(
instance_id,
redelivered_managed_network_config(instance_id),
)],
Some("rev-webhook-redelivery".to_string()),
None,
)
.await
.unwrap();
let redelivered = wait_for_runtime_config(&core_manager, instance_id, |config| {
config.network_name.as_deref() == Some("managed-redelivered")
&& config.instance_recv_bps_limit == Some(654321)
})
.await;
assert_eq!(
redelivered.instance_id.as_deref(),
Some(instance_id.to_string().as_str())
);
assert_eq!(
redelivered.hostname.as_deref(),
Some("managed-redelivered-host")
);
assert_eq!(
redelivered.network_name.as_deref(),
Some("managed-redelivered")
);
assert_eq!(redelivered.enable_kcp_proxy, Some(true));
assert_eq!(redelivered.instance_recv_bps_limit, Some(654321));
assert_eq!(
core_manager.get_instance_network_config_source(&instance_id),
Some(easytier::common::config::ConfigSource::Web)
);
assert_eq!(
mgr.db()
.get_managed_config_revision((user_id, machine_id))
.await
.unwrap()
.as_deref(),
Some("rev-webhook-redelivery")
);
// Reconnect path: a fresh core manager has no local runtime state, so
// the new session must replay the managed config persisted in web DB.
drop(client);
let reconnected_core_manager = Arc::new(NetworkInstanceManager::new());
let _reconnected_client = start_web_client_for_test(
config_server_addr,
machine_id,
reconnected_core_manager.clone(),
)
.await;
wait_for_validated_user(&mgr, machine_id).await;
let replayed = wait_for_runtime_config(&reconnected_core_manager, instance_id, |config| {
config.network_name.as_deref() == Some("managed-redelivered")
&& config.instance_recv_bps_limit == Some(654321)
})
.await;
assert_eq!(
replayed.network_name.as_deref(),
Some("managed-redelivered")
);
assert_eq!(replayed.enable_kcp_proxy, Some(true));
assert_eq!(replayed.instance_recv_bps_limit, Some(654321));
webhook_server.abort();
}
#[tokio::test]
async fn webhook_reject_disconnects_and_revalidates_after_reconnect() {
let webhook_state = TestWebhookState::with_blocked_second_validate([false, true]);
let (webhook_config, webhook_server, webhook_state) =
test_webhook_config_with_state(webhook_state).await;
let mut mgr = ClientManager::new(
Db::memory_db().await,
None,
Duration::ZERO,
Arc::new(FeatureFlags::default()),
webhook_config,
);
let config_server_addr = add_random_udp_listener(&mut mgr).await;
let machine_id = uuid::Uuid::new_v4();
let core_manager = Arc::new(NetworkInstanceManager::new());
let client =
start_web_client_for_test(config_server_addr, machine_id, core_manager.clone()).await;
let first_session_urls = wait_for_session_urls(&mgr).await;
wait_for_validate_count(&webhook_state, 1).await;
wait_for_validate_count(&webhook_state, 2).await;
assert!(
mgr.list_sessions().await.is_empty(),
"invalid validate-token response must not authorize the session"
);
webhook_state.allow_second_validate();
let user_id = wait_for_validated_user(&mgr, machine_id).await;
tokio::time::timeout(Duration::from_secs(12), async {
loop {
let reconnected = mgr
.client_sessions
.iter()
.any(|entry| !first_session_urls.iter().any(|url| url == entry.key()));
if reconnected {
break;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
})
.await
.unwrap();
assert!(
client.is_connected(),
"web client should reconnect after invalid session heartbeat failure"
);
assert!(webhook_state.validate_count() >= 2);
assert!(
mgr.get_session_by_machine_id(user_id, &machine_id)
.is_some()
);
webhook_server.abort();
}
}
@@ -0,0 +1,812 @@
use anyhow::Context as _;
use easytier::{
common::config::{
ConfigLoader, EncryptionAlgorithm, PortForwardConfig as RuntimePortForwardConfig,
},
proto::{
acl::Acl,
api::{
config::{
AclPatch, ConfigPatchAction, InstanceConfigPatch, PatchConfigRequest,
PortForwardPatch, ProxyNetworkPatch,
},
instance::{InstanceIdentifier, instance_identifier},
manage::{
ConfigSource as RpcConfigSource, GetNetworkInstanceConfigRequest, NetworkConfig,
RunNetworkInstanceRequest,
},
},
common::{CompressionAlgoPb, Ipv4Inet as RpcIpv4Inet},
rpc_types::controller::BaseController,
},
};
use super::session::{SessionConfigClient, SessionRpcClient};
pub(super) enum RuntimeReconcileAction {
None,
Run {
config: Box<NetworkConfig>,
overwrite: bool,
},
Patch(Box<InstanceConfigPatch>),
}
#[derive(Clone, PartialEq)]
struct RuntimeProxyNetwork {
cidr: String,
mapped_cidr: Option<String>,
}
fn instance_identifier(inst_id: &str) -> anyhow::Result<InstanceIdentifier> {
let inst_id = uuid::Uuid::parse_str(inst_id)
.with_context(|| format!("invalid runtime instance id: {inst_id}"))?;
Ok(InstanceIdentifier {
selector: Some(instance_identifier::Selector::Id(inst_id.into())),
})
}
fn hot_patch_base(config: &NetworkConfig) -> anyhow::Result<NetworkConfig> {
let data_compress_algo = normalized_data_compress_algo(config.data_compress_algo);
let encryption_algorithm = normalized_encryption_algorithm(config.encryption_algorithm.clone());
let mut config = NetworkConfig::new_from_config(config.gen_config()?)?;
let is_credential_mode = config.network_secret.is_none()
&& config
.secure_mode
.as_ref()
.and_then(|mode| mode.local_private_key.as_deref())
.is_some_and(|key| !key.is_empty());
config.acl = None;
config.port_forwards.clear();
config.proxy_cidrs.clear();
config.disable_relay_data = None;
if config.dhcp.unwrap_or_default() {
config.virtual_ipv4 = None;
config.network_length = None;
}
if let Some(secure_mode) = config.secure_mode.as_mut() {
if !is_credential_mode {
secure_mode.local_private_key = None;
}
secure_mode.local_public_key = None;
}
config.data_compress_algo = data_compress_algo;
config.encryption_algorithm = encryption_algorithm;
Ok(config)
}
fn normalized_data_compress_algo(algo: Option<i32>) -> Option<i32> {
let default = CompressionAlgoPb::None as i32;
let effective = algo.map(|algo| if algo < default { default } else { algo });
effective.filter(|algo| *algo != default)
}
fn normalized_encryption_algorithm(algo: Option<String>) -> Option<String> {
let default = EncryptionAlgorithm::default().to_string();
algo.filter(|algo| algo != &default)
}
fn diff_port_forwards(
current: &[RuntimePortForwardConfig],
desired: &[RuntimePortForwardConfig],
) -> Vec<PortForwardPatch> {
let mut patches = Vec::new();
for cfg in unique_port_forwards(current, desired) {
let current_count = current.iter().filter(|item| *item == &cfg).count();
let desired_count = desired.iter().filter(|item| *item == &cfg).count();
if current_count == desired_count {
continue;
}
if current_count > 0 {
patches.push(PortForwardPatch {
action: ConfigPatchAction::Remove as i32,
cfg: Some(cfg.clone().into()),
});
}
patches.extend((0..desired_count).map(|_| PortForwardPatch {
action: ConfigPatchAction::Add as i32,
cfg: Some(cfg.clone().into()),
}));
}
patches
}
fn unique_port_forwards(
current: &[RuntimePortForwardConfig],
desired: &[RuntimePortForwardConfig],
) -> Vec<RuntimePortForwardConfig> {
let mut unique = Vec::new();
for cfg in current.iter().chain(desired.iter()) {
if !unique.contains(cfg) {
unique.push(cfg.clone());
}
}
unique
}
fn parse_rpc_ipv4_inet(value: &str) -> anyhow::Result<RpcIpv4Inet> {
value
.parse::<RpcIpv4Inet>()
.with_context(|| format!("failed to parse runtime ipv4 cidr: {value}"))
}
fn diff_proxy_networks(
current: &[RuntimeProxyNetwork],
desired: &[RuntimeProxyNetwork],
) -> anyhow::Result<Vec<ProxyNetworkPatch>> {
if current == desired {
return Ok(Vec::new());
}
let mut patches = vec![ProxyNetworkPatch {
action: ConfigPatchAction::Clear as i32,
cidr: Some(clear_proxy_network_cidr(current, desired)?),
..Default::default()
}];
for proxy_network in desired {
patches.push(ProxyNetworkPatch {
action: ConfigPatchAction::Add as i32,
cidr: Some(parse_rpc_ipv4_inet(&proxy_network.cidr)?),
mapped_cidr: proxy_network
.mapped_cidr
.as_deref()
.map(parse_rpc_ipv4_inet)
.transpose()?,
});
}
Ok(patches)
}
fn clear_proxy_network_cidr(
current: &[RuntimeProxyNetwork],
desired: &[RuntimeProxyNetwork],
) -> anyhow::Result<RpcIpv4Inet> {
let cidr = desired
.first()
.or_else(|| current.first())
.map(|proxy_network| proxy_network.cidr.as_str())
.unwrap_or("0.0.0.0/0");
parse_rpc_ipv4_inet(cidr)
}
fn normalized_acl(acl: &Option<Acl>) -> Option<Acl> {
let acl = acl.clone().unwrap_or_default();
(acl != Acl::default()).then_some(acl)
}
fn normalized_port_forwards(
config: &NetworkConfig,
) -> anyhow::Result<Vec<RuntimePortForwardConfig>> {
Ok(config
.gen_config()?
.get_port_forwards()
.into_iter()
.map(|cfg| {
RuntimePortForwardConfig::from(easytier::proto::common::PortForwardConfigPb::from(cfg))
})
.collect())
}
fn normalized_proxy_networks(config: &NetworkConfig) -> anyhow::Result<Vec<RuntimeProxyNetwork>> {
Ok(config
.gen_config()?
.get_proxy_cidrs()
.into_iter()
.map(|proxy_network| RuntimeProxyNetwork {
cidr: proxy_network.cidr.to_string(),
mapped_cidr: proxy_network.mapped_cidr.map(|cidr| cidr.to_string()),
})
.collect())
}
fn normalized_disable_relay_data(config: &NetworkConfig) -> anyhow::Result<bool> {
Ok(config.gen_config()?.get_flags().disable_relay_data)
}
fn web_source_runtime_patch(
current: &NetworkConfig,
desired: &NetworkConfig,
) -> anyhow::Result<Option<InstanceConfigPatch>> {
if let Some(desired_hostname) = desired
.hostname
.as_deref()
.filter(|hostname| !hostname.is_empty())
&& current.hostname.as_deref() != Some(desired_hostname)
{
return Ok(None);
}
let mut current_base = hot_patch_base(current)?;
let mut desired_base = hot_patch_base(desired)?;
current_base.hostname = None;
desired_base.hostname = None;
if current_base != desired_base {
return Ok(None);
}
let mut patch = InstanceConfigPatch::default();
let current_acl = normalized_acl(&current.acl);
let desired_acl = normalized_acl(&desired.acl);
if current_acl != desired_acl {
patch.acl = Some(AclPatch {
acl: Some(desired_acl.unwrap_or_default()),
..Default::default()
});
}
let current_port_forwards = normalized_port_forwards(current)?;
let desired_port_forwards = normalized_port_forwards(desired)?;
if current_port_forwards != desired_port_forwards {
patch.port_forwards = diff_port_forwards(&current_port_forwards, &desired_port_forwards);
}
let current_proxy_networks = normalized_proxy_networks(current)?;
let desired_proxy_networks = normalized_proxy_networks(desired)?;
if current_proxy_networks != desired_proxy_networks {
if current_proxy_networks.is_empty() {
return Ok(None);
}
patch.proxy_networks =
diff_proxy_networks(&current_proxy_networks, &desired_proxy_networks)?;
}
let current_disable_relay_data = normalized_disable_relay_data(current)?;
let desired_disable_relay_data = normalized_disable_relay_data(desired)?;
if current_disable_relay_data != desired_disable_relay_data {
patch.disable_relay_data = Some(desired_disable_relay_data);
}
Ok(Some(patch))
}
fn ensure_runtime_config_converged(
current: &NetworkConfig,
desired: &NetworkConfig,
) -> anyhow::Result<()> {
let patch = web_source_runtime_patch(current, desired)?;
match patch {
Some(patch) if patch == InstanceConfigPatch::default() => Ok(()),
Some(patch) => anyhow::bail!("runtime config still needs patch after reconcile: {patch:?}"),
None => anyhow::bail!("runtime config still needs full overwrite after reconcile"),
}
}
async fn run_web_source_instance(
rpc_client: &mut SessionRpcClient,
inst_id: &str,
config: NetworkConfig,
overwrite: bool,
) -> anyhow::Result<()> {
rpc_client
.run_network_instance(
BaseController::default(),
RunNetworkInstanceRequest {
inst_id: Some(inst_id.to_string().into()),
config: Some(config),
overwrite,
source: RpcConfigSource::Web as i32,
},
)
.await?;
Ok(())
}
pub(super) async fn get_runtime_config(
rpc_client: &mut SessionRpcClient,
inst_id: &str,
) -> anyhow::Result<NetworkConfig> {
rpc_client
.get_network_instance_config(
BaseController::default(),
GetNetworkInstanceConfigRequest {
inst_id: Some(inst_id.to_string().into()),
},
)
.await?
.config
.ok_or_else(|| anyhow::anyhow!("runtime returned empty config for {inst_id}"))
}
pub(super) async fn prepare_web_source_runtime_reconcile(
rpc_client: &mut SessionRpcClient,
inst_id: &str,
desired_config: NetworkConfig,
is_running: bool,
) -> anyhow::Result<RuntimeReconcileAction> {
if !is_running {
return Ok(RuntimeReconcileAction::Run {
config: Box::new(desired_config),
overwrite: false,
});
}
let current_config = get_runtime_config(rpc_client, inst_id).await?;
prepare_web_source_runtime_reconcile_from_current(&current_config, desired_config)
}
pub(super) fn prepare_web_source_runtime_reconcile_from_current(
current_config: &NetworkConfig,
desired_config: NetworkConfig,
) -> anyhow::Result<RuntimeReconcileAction> {
let Some(patch) = web_source_runtime_patch(current_config, &desired_config)? else {
return Ok(RuntimeReconcileAction::Run {
config: Box::new(desired_config),
overwrite: true,
});
};
if patch == InstanceConfigPatch::default() {
return Ok(RuntimeReconcileAction::None);
}
Ok(RuntimeReconcileAction::Patch(Box::new(patch)))
}
pub(super) async fn apply_web_source_runtime_reconcile(
rpc_client: &mut SessionRpcClient,
config_client: &mut SessionConfigClient,
inst_id: &str,
desired_config: NetworkConfig,
action: RuntimeReconcileAction,
) -> anyhow::Result<NetworkConfig> {
match action {
RuntimeReconcileAction::None => Ok(desired_config),
RuntimeReconcileAction::Run { config, overwrite } => {
run_web_source_instance(rpc_client, inst_id, *config, overwrite).await?;
Ok(desired_config)
}
RuntimeReconcileAction::Patch(patch) => {
config_client
.patch_config(
BaseController::default(),
PatchConfigRequest {
instance: Some(instance_identifier(inst_id)?),
patch: Some(*patch),
},
)
.await?;
let current_config = get_runtime_config(rpc_client, inst_id).await?;
ensure_runtime_config_converged(&current_config, &desired_config)?;
Ok(current_config)
}
}
}
#[cfg(test)]
mod tests {
use easytier::proto::{
api::{
config::ConfigPatchAction,
manage::{NetworkingMethod, PortForwardConfig},
},
common::{CompressionAlgoPb, SocketType},
};
use super::*;
fn config_with_port_forwards(port_forwards: Vec<PortForwardConfig>) -> NetworkConfig {
NetworkConfig {
instance_id: Some("11111111-1111-1111-1111-111111111111".to_string()),
dhcp: Some(true),
network_name: Some("managed".to_string()),
network_secret: Some("secret".to_string()),
networking_method: Some(NetworkingMethod::Manual as i32),
port_forwards,
..Default::default()
}
}
fn port_forward(bind_port: u32, dst_port: u32) -> PortForwardConfig {
PortForwardConfig {
bind_ip: "127.0.0.1".to_string(),
bind_port,
dst_ip: "10.144.0.1".to_string(),
dst_port,
proto: "tcp".to_string(),
}
}
fn patch_port(patch: &PortForwardPatch) -> (i32, u32, u32, i32) {
let cfg = patch.cfg.as_ref().expect("port forward patch cfg");
(
patch.action,
cfg.bind_addr.as_ref().expect("bind addr").port,
cfg.dst_addr.as_ref().expect("dst addr").port,
cfg.socket_type,
)
}
fn patch_proxy_network(patch: &ProxyNetworkPatch) -> (i32, String, Option<String>) {
(
patch.action,
patch.cidr.map(|cidr| cidr.to_string()).unwrap_or_default(),
patch.mapped_cidr.map(|cidr| cidr.to_string()),
)
}
#[test]
fn runtime_patch_ignores_runtime_defaults_and_adds_port_forward() {
let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
current.virtual_ipv4 = Some("10.144.0.2".to_string());
current.network_length = Some(16);
current.bind_device = Some(true);
current.dev_name = Some(String::new());
current.disable_ipv6 = Some(false);
current.mtu = Some(1380);
current.multi_thread = Some(true);
let desired =
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
let patch = web_source_runtime_patch(&current, &desired)
.expect("build patch")
.expect("hot patch");
assert_eq!(patch.port_forwards.len(), 1);
assert_eq!(
patch_port(&patch.port_forwards[0]),
(
ConfigPatchAction::Add as i32,
23007,
3389,
SocketType::Tcp as i32
)
);
}
#[test]
fn runtime_patch_removes_deleted_port_forward_without_clear() {
let current =
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
let desired = config_with_port_forwards(vec![port_forward(23000, 5174)]);
let patch = web_source_runtime_patch(&current, &desired)
.expect("build patch")
.expect("hot patch");
assert_eq!(patch.port_forwards.len(), 1);
assert_eq!(
patch_port(&patch.port_forwards[0]),
(
ConfigPatchAction::Remove as i32,
23007,
3389,
SocketType::Tcp as i32
)
);
}
#[test]
fn runtime_patch_reconciles_duplicate_port_forward_count() {
let current =
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23000, 5174)]);
let desired = config_with_port_forwards(vec![port_forward(23000, 5174)]);
let patch = web_source_runtime_patch(&current, &desired)
.expect("build patch")
.expect("hot patch");
assert_eq!(patch.port_forwards.len(), 2);
assert_eq!(
patch_port(&patch.port_forwards[0]),
(
ConfigPatchAction::Remove as i32,
23000,
5174,
SocketType::Tcp as i32
)
);
assert_eq!(
patch_port(&patch.port_forwards[1]),
(
ConfigPatchAction::Add as i32,
23000,
5174,
SocketType::Tcp as i32
)
);
}
#[test]
fn runtime_convergence_rejects_stale_extra_port_forward() {
let current = config_with_port_forwards(vec![
port_forward(23000, 5174),
port_forward(23007, 3389),
port_forward(23100, 8080),
]);
let desired =
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
let err = ensure_runtime_config_converged(&current, &desired)
.expect_err("extra runtime port forward should not converge");
assert!(
err.to_string()
.contains("runtime config still needs patch after reconcile"),
"unexpected error: {err:?}"
);
}
#[test]
fn runtime_patch_canonicalizes_port_forward_protocol() {
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
let mut desired_port_forward = port_forward(23000, 5174);
desired_port_forward.proto = "TCP".to_string();
let desired = config_with_port_forwards(vec![desired_port_forward]);
let patch = web_source_runtime_patch(&current, &desired)
.expect("build patch")
.expect("hot patch");
assert_eq!(patch, InstanceConfigPatch::default());
ensure_runtime_config_converged(&current, &desired).expect("runtime converged");
}
#[test]
fn runtime_patch_rejects_non_hot_config_change() {
let current = config_with_port_forwards(Vec::new());
let mut desired = current.clone();
desired.network_secret = Some("new-secret".to_string());
let patch = web_source_runtime_patch(&current, &desired).expect("build patch");
assert!(patch.is_none());
}
#[test]
fn runtime_patch_rejects_routes_change() {
let mut current = config_with_port_forwards(Vec::new());
current.enable_manual_routes = Some(true);
current.routes = vec!["10.1.0.0/16".to_string(), "10.2.0.0/16".to_string()];
let mut desired = config_with_port_forwards(Vec::new());
desired.enable_manual_routes = Some(true);
desired.routes = vec!["10.2.0.0/16".to_string(), "10.3.0.0/16".to_string()];
let patch = web_source_runtime_patch(&current, &desired).expect("build patch");
assert!(patch.is_none());
}
#[test]
fn runtime_patch_replaces_proxy_networks() {
let mut current = config_with_port_forwards(Vec::new());
current.proxy_cidrs = vec![
"10.1.0.0/16".to_string(),
"10.2.0.0/16->10.20.0.0/16".to_string(),
];
let mut desired = config_with_port_forwards(Vec::new());
desired.proxy_cidrs = vec![
"10.2.0.0/16->10.21.0.0/16".to_string(),
"10.3.0.0/16".to_string(),
];
let patch = web_source_runtime_patch(&current, &desired)
.expect("build patch")
.expect("hot patch");
assert_eq!(patch.proxy_networks.len(), 3);
assert_eq!(
patch_proxy_network(&patch.proxy_networks[0]),
(
ConfigPatchAction::Clear as i32,
"10.2.0.0/16".to_string(),
None
)
);
assert_eq!(
patch_proxy_network(&patch.proxy_networks[1]),
(
ConfigPatchAction::Add as i32,
"10.2.0.0/16".to_string(),
Some("10.21.0.0/16".to_string())
)
);
assert_eq!(
patch_proxy_network(&patch.proxy_networks[2]),
(
ConfigPatchAction::Add as i32,
"10.3.0.0/16".to_string(),
None
)
);
}
#[test]
fn runtime_patch_replaces_proxy_networks_with_same_source_cidr() {
let mut current = config_with_port_forwards(Vec::new());
current.proxy_cidrs = vec![
"10.1.2.0/24".to_string(),
"10.1.2.0/24->10.1.3.0/24".to_string(),
];
let mut desired = config_with_port_forwards(Vec::new());
desired.proxy_cidrs = vec!["10.1.2.0/24->10.1.3.0/24".to_string()];
let patch = web_source_runtime_patch(&current, &desired)
.expect("build patch")
.expect("hot patch");
assert_eq!(patch.proxy_networks.len(), 2);
assert_eq!(
patch_proxy_network(&patch.proxy_networks[0]),
(
ConfigPatchAction::Clear as i32,
"10.1.2.0/24".to_string(),
None
)
);
assert_eq!(
patch_proxy_network(&patch.proxy_networks[1]),
(
ConfigPatchAction::Add as i32,
"10.1.2.0/24".to_string(),
Some("10.1.3.0/24".to_string())
)
);
}
#[test]
fn runtime_patch_rejects_proxy_network_empty_to_nonempty() {
let current = config_with_port_forwards(Vec::new());
let mut desired = config_with_port_forwards(Vec::new());
desired.proxy_cidrs = vec!["10.1.2.0/24".to_string()];
let patch = web_source_runtime_patch(&current, &desired).expect("build patch");
assert!(patch.is_none());
}
#[test]
fn runtime_patch_clears_proxy_networks_with_legacy_compatible_cidr() {
let mut current = config_with_port_forwards(Vec::new());
current.proxy_cidrs = vec!["10.1.2.0/24".to_string()];
let desired = config_with_port_forwards(Vec::new());
let patch = web_source_runtime_patch(&current, &desired)
.expect("build patch")
.expect("hot patch");
assert_eq!(patch.proxy_networks.len(), 1);
assert_eq!(
patch_proxy_network(&patch.proxy_networks[0]),
(
ConfigPatchAction::Clear as i32,
"10.1.2.0/24".to_string(),
None
)
);
}
#[test]
fn runtime_patch_updates_disable_relay_data() {
let current = config_with_port_forwards(Vec::new());
let mut desired = current.clone();
desired.disable_relay_data = Some(true);
let patch = web_source_runtime_patch(&current, &desired)
.expect("build patch")
.expect("hot patch");
assert_eq!(patch.disable_relay_data, Some(true));
}
#[test]
fn runtime_patch_still_rejects_unsupported_flag_change() {
let current = config_with_port_forwards(Vec::new());
let mut desired = current.clone();
desired.no_tun = Some(true);
let patch = web_source_runtime_patch(&current, &desired).expect("build patch");
assert!(patch.is_none());
}
#[test]
fn runtime_patch_rejects_encryption_algorithm_change() {
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
let mut desired = current.clone();
desired.encryption_algorithm = Some("managed-test-algo".to_string());
let patch = web_source_runtime_patch(&current, &desired).expect("build patch");
assert!(patch.is_none());
}
#[test]
fn runtime_patch_rejects_data_compress_algo_change() {
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
let mut desired = current.clone();
desired.data_compress_algo = Some(CompressionAlgoPb::Zstd as i32);
let patch = web_source_runtime_patch(&current, &desired).expect("build patch");
assert!(patch.is_none());
}
#[test]
fn runtime_patch_rejects_credential_private_key_change() {
let mut current = config_with_port_forwards(Vec::new());
current.network_secret = None;
current.secure_mode = Some(easytier::proto::common::SecureModeConfig {
enabled: true,
local_private_key: Some("mUuD5fsIm/ftvgS4WBAYFMNLqWX3qT9rnm4PrnOqb9s=".to_string()),
local_public_key: None,
});
let mut desired = current.clone();
desired.secure_mode = Some(easytier::proto::common::SecureModeConfig {
enabled: true,
local_private_key: Some("aEpz80FuYbaY4QLJizAIuIcK4TYsoSA9jHHCXCOQJoc=".to_string()),
local_public_key: None,
});
let patch = web_source_runtime_patch(&current, &desired).expect("build patch");
assert!(patch.is_none());
}
#[test]
fn runtime_patch_ignores_generated_secure_key_when_network_secret_exists() {
let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
current.secure_mode = Some(easytier::proto::common::SecureModeConfig {
enabled: true,
local_private_key: Some("mUuD5fsIm/ftvgS4WBAYFMNLqWX3qT9rnm4PrnOqb9s=".to_string()),
local_public_key: Some("4x6L5dZjB8hsPO4f96Hyhi4xFealBu6i3BxRVBYR1Fc=".to_string()),
});
let mut desired =
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
desired.secure_mode = Some(easytier::proto::common::SecureModeConfig {
enabled: true,
local_private_key: None,
local_public_key: None,
});
let patch = web_source_runtime_patch(&current, &desired)
.expect("build patch")
.expect("hot patch");
assert_eq!(patch.port_forwards.len(), 1);
assert_eq!(
patch_port(&patch.port_forwards[0]),
(
ConfigPatchAction::Add as i32,
23007,
3389,
SocketType::Tcp as i32
)
);
}
#[test]
fn runtime_patch_ignores_runtime_hostname_when_desired_omits_hostname() {
let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
current.hostname = Some("runtime-host".to_string());
let desired =
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
let patch = web_source_runtime_patch(&current, &desired)
.expect("build patch")
.expect("hot patch");
assert_eq!(patch.port_forwards.len(), 1);
assert_eq!(
patch_port(&patch.port_forwards[0]),
(
ConfigPatchAction::Add as i32,
23007,
3389,
SocketType::Tcp as i32
)
);
}
#[test]
fn runtime_patch_rejects_explicit_desired_hostname_change() {
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
let mut desired =
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
desired.hostname =
Some(easytier::common::config::TomlConfigLoader::default().get_hostname());
let patch = web_source_runtime_patch(&current, &desired).expect("build patch");
assert!(patch.is_none());
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,913 @@
use std::collections::{HashMap, HashSet};
use easytier::{
proto::{
api::manage::{
DeleteNetworkInstanceRequest, ListNetworkInstanceMetaRequest,
ListNetworkInstanceRequest, NetworkConfig, NetworkMeta, RunNetworkInstanceRequest,
},
rpc_types::controller::BaseController,
web::HeartbeatRequest,
},
rpc_service::remote_client::{ListNetworkProps, Storage as _},
};
use tokio::sync::{RwLock, broadcast};
use super::{SessionConfigClient, SessionData, SessionRpcClient, SessionRpcService};
use crate::client_manager::{
managed_config::{self, PersistedConfigSource},
runtime_reconcile,
storage::{StorageInner, WeakRefStorage},
};
async fn recv_latest_heartbeat(
heartbeat_waiter: &mut broadcast::Receiver<HeartbeatRequest>,
) -> Option<HeartbeatRequest> {
let mut req = loop {
match heartbeat_waiter.recv().await {
Ok(req) => break req,
Err(broadcast::error::RecvError::Lagged(skipped)) => {
tracing::warn!(
skipped,
"heartbeat reconcile worker lagged, waiting for latest request"
);
}
Err(broadcast::error::RecvError::Closed) => {
tracing::error!("Failed to receive heartbeat request: channel closed");
return None;
}
}
};
// Drop any heartbeat backlog accumulated while the previous reconcile
// round was doing DB/RPC IO. The newest heartbeat has the freshest
// runtime instance list, which is all this task needs.
loop {
match heartbeat_waiter.try_recv() {
Ok(next_req) => req = next_req,
Err(broadcast::error::TryRecvError::Empty) => break,
Err(broadcast::error::TryRecvError::Lagged(_)) => continue,
Err(broadcast::error::TryRecvError::Closed) => return None,
}
}
Some(req)
}
pub(super) async fn reconcile_network_configs_on_heartbeat(
session_data: std::sync::Weak<RwLock<SessionData>>,
mut heartbeat_waiter: broadcast::Receiver<HeartbeatRequest>,
storage: WeakRefStorage,
mut rpc_client: SessionRpcClient,
mut config_client: SessionConfigClient,
) {
let mut cache = ReconcileCache::default();
loop {
let Some(req) = recv_latest_heartbeat(&mut heartbeat_waiter).await else {
return;
};
let Some(storage) = storage.upgrade() else {
tracing::error!("Failed to get storage");
return;
};
let mut round =
match prepare_reconcile_round(&session_data, &storage, &mut rpc_client, req).await {
RoundStatus::Ready(round) => round,
RoundStatus::Skip => continue,
RoundStatus::Stop => return,
};
let running_metas =
match sync_running_sources_for_round(&mut rpc_client, &storage, &mut round).await {
RoundStatus::Ready(running_metas) => running_metas,
RoundStatus::Skip => continue,
RoundStatus::Stop => return,
};
let desired_web_inst_ids =
managed_config::desired_web_source_instance_ids(&round.local_configs);
cache.runtime_configs.retain_desired(&desired_web_inst_ids);
let mut outcome = match cleanup_stale_web_source_instances(
&session_data,
&storage,
&mut rpc_client,
&round,
running_metas.as_deref(),
&desired_web_inst_ids,
&mut cache,
)
.await
{
RoundStatus::Ready(outcome) => outcome,
RoundStatus::Skip => continue,
RoundStatus::Stop => return,
};
outcome.merge(
reconcile_desired_runtime_configs(
&session_data,
&mut rpc_client,
&mut config_client,
&round,
&mut cache,
)
.await,
);
if !outcome.has_failed {
cache.last_desired_web_inst_ids = Some(desired_web_inst_ids);
}
match mark_config_revision_applied_if_current(&session_data, &storage, &round, &outcome)
.await
{
RoundStatus::Ready(()) | RoundStatus::Skip => {}
RoundStatus::Stop => return,
}
}
}
enum RoundStatus<T> {
Ready(T),
Skip,
Stop,
}
enum ConfigActionResult {
Success,
Failed,
StopRound,
}
#[derive(Default)]
struct ReconcileCache {
cleaned_web_source_instances: bool,
last_desired_web_inst_ids: Option<HashSet<String>>,
runtime_configs: SessionRuntimeConfigCache,
}
#[derive(Default)]
struct SessionRuntimeConfigCache {
entries: HashMap<String, NetworkConfig>,
}
impl SessionRuntimeConfigCache {
fn plan(
&self,
inst_id: &str,
desired_config: NetworkConfig,
) -> anyhow::Result<Option<runtime_reconcile::RuntimeReconcileAction>> {
let Some(observed_config) = self.entries.get(inst_id) else {
return Ok(None);
};
runtime_reconcile::prepare_web_source_runtime_reconcile_from_current(
observed_config,
desired_config,
)
.map(Some)
}
fn remember(&mut self, inst_id: &str, observed_config: NetworkConfig) {
self.entries.insert(inst_id.to_string(), observed_config);
}
fn forget(&mut self, inst_id: &str) {
self.entries.remove(inst_id);
}
fn forget_many<'a>(&mut self, inst_ids: impl IntoIterator<Item = &'a String>) {
for inst_id in inst_ids {
self.entries.remove(inst_id);
}
}
fn retain_desired(&mut self, desired_web_inst_ids: &HashSet<String>) {
self.entries
.retain(|inst_id, _| desired_web_inst_ids.contains(inst_id));
}
}
#[derive(Default)]
struct ReconcileOutcome {
has_failed: bool,
managed_revision_failed: bool,
}
impl ReconcileOutcome {
fn record_failure(&mut self, managed_revision_failed: bool) {
self.has_failed = true;
self.managed_revision_failed |= managed_revision_failed;
}
fn merge(&mut self, other: Self) {
self.has_failed |= other.has_failed;
self.managed_revision_failed |= other.managed_revision_failed;
}
}
struct ReconcileRound {
req: HeartbeatRequest,
machine_id: uuid::Uuid,
user_id: i32,
running_inst_ids: HashSet<String>,
local_configs: Vec<crate::db::entity::user_running_network_configs::Model>,
target_config_revision: Option<String>,
should_apply_runtime_revision: bool,
}
async fn prepare_reconcile_round(
session_data: &std::sync::Weak<RwLock<SessionData>>,
storage: &StorageInner,
rpc_client: &mut SessionRpcClient,
req: HeartbeatRequest,
) -> RoundStatus<ReconcileRound> {
let Some(machine_id) = req.machine_id.map(uuid::Uuid::from) else {
tracing::warn!(?req, "Machine id is not set, ignore");
return RoundStatus::Skip;
};
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &req).await {
tracing::debug!(?machine_id, "skip stale heartbeat reconcile request");
return RoundStatus::Skip;
}
let user_id = match storage
.db
.get_user_id_by_token(req.user_token.clone())
.await
{
Ok(Some(user_id)) => user_id,
Ok(None) => {
tracing::info!("User not found by token: {:?}", req.user_token);
return RoundStatus::Stop;
}
Err(e) => {
tracing::error!("Failed to get user id by token, error: {:?}", e);
return RoundStatus::Stop;
}
};
let applied_config_revision = {
let Some(data) = session_data.upgrade() else {
return RoundStatus::Stop;
};
data.read().await.applied_config_revision.clone()
};
let target_config_revision = match storage
.db
.get_managed_config_revision((user_id, machine_id))
.await
{
Ok(revision) => revision,
Err(e) => {
tracing::error!("Failed to read managed config revision, error: {:?}", e);
return RoundStatus::Stop;
}
};
let should_apply_runtime_revision =
target_config_revision.is_some() && target_config_revision != applied_config_revision;
let running_inst_ids = match running_instance_ids_for_round(
rpc_client,
&req,
user_id,
machine_id,
should_apply_runtime_revision,
)
.await
{
RoundStatus::Ready(ids) => ids,
RoundStatus::Skip => return RoundStatus::Skip,
RoundStatus::Stop => return RoundStatus::Stop,
};
let local_configs = match storage
.db
.list_network_configs((user_id, machine_id), ListNetworkProps::EnabledOnly)
.await
{
Ok(configs) => configs,
Err(e) => {
tracing::error!("Failed to list network configs, error: {:?}", e);
return RoundStatus::Stop;
}
};
RoundStatus::Ready(ReconcileRound {
req,
machine_id,
user_id,
running_inst_ids,
local_configs,
target_config_revision,
should_apply_runtime_revision,
})
}
async fn running_instance_ids_for_round(
rpc_client: &mut SessionRpcClient,
req: &HeartbeatRequest,
user_id: i32,
machine_id: uuid::Uuid,
should_apply_runtime_revision: bool,
) -> RoundStatus<HashSet<String>> {
if !should_apply_runtime_revision {
return RoundStatus::Ready(
req.running_network_instances
.iter()
.map(|x| x.to_string())
.collect(),
);
}
match rpc_client
.list_network_instance(BaseController::default(), ListNetworkInstanceRequest {})
.await
{
Ok(resp) => RoundStatus::Ready(resp.inst_ids.iter().map(|x| x.to_string()).collect()),
Err(error) => {
tracing::warn!(
?user_id,
?machine_id,
?error,
"Failed to refresh running instances for managed config revision"
);
RoundStatus::Skip
}
}
}
async fn sync_running_sources_for_round(
rpc_client: &mut SessionRpcClient,
storage: &StorageInner,
round: &mut ReconcileRound,
) -> RoundStatus<Option<Vec<NetworkMeta>>> {
if !round.req.support_config_source {
return RoundStatus::Ready(None);
}
let ret = if round.running_inst_ids.is_empty() {
Ok(Vec::new())
} else {
rpc_client
.list_network_instance_meta(
BaseController::default(),
ListNetworkInstanceMetaRequest {
inst_ids: managed_config::parse_instance_ids(
round.running_inst_ids.iter().cloned(),
),
},
)
.await
.map(|resp| resp.metas)
};
match ret {
Ok(metas) => {
if let Err(e) = managed_config::sync_running_config_sources(
&storage.db,
round.user_id,
round.machine_id,
&round.local_configs,
&metas,
)
.await
{
tracing::warn!(
user_id = ?round.user_id,
machine_id = ?round.machine_id,
%e,
"Failed to sync running network config sources"
);
} else if !metas.is_empty() {
round.local_configs = match storage
.db
.list_network_configs(
(round.user_id, round.machine_id),
ListNetworkProps::EnabledOnly,
)
.await
{
Ok(configs) => configs,
Err(e) => {
tracing::error!(
"Failed to reload network configs after source sync, error: {:?}",
e
);
return RoundStatus::Stop;
}
};
}
RoundStatus::Ready(Some(metas))
}
Err(e) => {
tracing::warn!(
user_id = ?round.user_id,
%e,
"Failed to list running network instance metadata"
);
RoundStatus::Ready(None)
}
}
}
async fn cleanup_stale_web_source_instances(
session_data: &std::sync::Weak<RwLock<SessionData>>,
storage: &StorageInner,
rpc_client: &mut SessionRpcClient,
round: &ReconcileRound,
running_metas: Option<&[NetworkMeta]>,
desired_web_inst_ids: &HashSet<String>,
cache: &mut ReconcileCache,
) -> RoundStatus<ReconcileOutcome> {
let desired_changed = cache
.last_desired_web_inst_ids
.as_ref()
.is_none_or(|last| last != desired_web_inst_ids);
if cache.cleaned_web_source_instances && !desired_changed {
return RoundStatus::Ready(ReconcileOutcome::default());
}
let db_web_inst_ids = match storage
.db
.list_network_configs((round.user_id, round.machine_id), ListNetworkProps::All)
.await
{
Ok(configs) => managed_config::desired_web_source_instance_ids(&configs),
Err(e) => {
tracing::error!("Failed to list all network configs, error: {:?}", e);
return RoundStatus::Stop;
}
};
let running_web_inst_ids = managed_config::running_web_source_instance_ids(
&round.running_inst_ids,
&db_web_inst_ids,
running_metas,
);
let should_delete_inst_ids = running_web_inst_ids
.difference(desired_web_inst_ids)
.cloned()
.collect::<HashSet<_>>();
let should_delete_ids =
managed_config::parse_instance_ids(should_delete_inst_ids.iter().cloned());
let mut outcome = ReconcileOutcome::default();
if !should_delete_ids.is_empty() {
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
tracing::debug!(
machine_id = ?round.machine_id,
"skip stale cleanup because webhook session is no longer current"
);
return RoundStatus::Skip;
}
let ret = rpc_client
.delete_network_instance(
BaseController::default(),
DeleteNetworkInstanceRequest {
inst_ids: should_delete_ids,
},
)
.await;
tracing::info!(
user_id = ?round.user_id,
"Clean stale web-source network instances on heartbeat: {:?}, user_token: {:?}",
ret,
round.req.user_token
);
if ret.is_err() {
outcome.record_failure(true);
} else {
cache.runtime_configs.forget_many(&should_delete_inst_ids);
}
}
if !outcome.has_failed {
cache.cleaned_web_source_instances = true;
cache.last_desired_web_inst_ids = Some(desired_web_inst_ids.clone());
}
RoundStatus::Ready(outcome)
}
async fn reconcile_desired_runtime_configs(
session_data: &std::sync::Weak<RwLock<SessionData>>,
rpc_client: &mut SessionRpcClient,
config_client: &mut SessionConfigClient,
round: &ReconcileRound,
cache: &mut ReconcileCache,
) -> ReconcileOutcome {
let mut outcome = ReconcileOutcome::default();
// After stale web-owned instances are removed, start every enabled
// config that the latest heartbeat did not report as running. When
// a managed config revision is pending, also reconcile running
// web-owned configs before reporting that revision as applied.
for config in &round.local_configs {
let source = PersistedConfigSource::from_db(&config.source);
let is_running = round.running_inst_ids.contains(&config.network_instance_id);
let should_reconcile_running_web_config = is_running
&& round.should_apply_runtime_revision
&& source == PersistedConfigSource::Web;
if is_running && !should_reconcile_running_web_config {
continue;
}
let desired_config = match serde_json::from_str::<NetworkConfig>(&config.network_config) {
Ok(cfg) => cfg,
Err(e) => {
tracing::error!(
user_id = ?round.user_id,
machine_id = ?round.machine_id,
instance_id = %config.network_instance_id,
"Failed to deserialize network config, skipping: {:?}",
e
);
if source == PersistedConfigSource::Web {
cache.runtime_configs.forget(&config.network_instance_id);
}
outcome.record_failure(source == PersistedConfigSource::Web);
continue;
}
};
let action_result = if should_reconcile_running_web_config {
reconcile_running_web_config(
session_data,
rpc_client,
config_client,
round,
config,
desired_config,
&mut cache.runtime_configs,
)
.await
} else {
if source == PersistedConfigSource::Web {
cache.runtime_configs.forget(&config.network_instance_id);
}
let action_result = run_missing_network_config(
session_data,
rpc_client,
round,
config,
desired_config.clone(),
)
.await;
if matches!(action_result, ConfigActionResult::Success)
&& source == PersistedConfigSource::Web
{
if let Err(e) = remember_web_runtime_config_after_run(
rpc_client,
&config.network_instance_id,
&desired_config,
&mut cache.runtime_configs,
)
.await
{
tracing::error!(
user_id = ?round.user_id,
machine_id = ?round.machine_id,
instance_id = %config.network_instance_id,
"Failed to cache runtime config after run: {:?}",
e
);
ConfigActionResult::Failed
} else {
action_result
}
} else {
action_result
}
};
match action_result {
ConfigActionResult::Success => {}
ConfigActionResult::Failed => {
if source == PersistedConfigSource::Web {
cache.runtime_configs.forget(&config.network_instance_id);
}
outcome.record_failure(source == PersistedConfigSource::Web)
}
ConfigActionResult::StopRound => {
if source == PersistedConfigSource::Web {
cache.runtime_configs.forget(&config.network_instance_id);
}
outcome.record_failure(source == PersistedConfigSource::Web);
break;
}
}
}
outcome
}
async fn reconcile_running_web_config(
session_data: &std::sync::Weak<RwLock<SessionData>>,
rpc_client: &mut SessionRpcClient,
config_client: &mut SessionConfigClient,
round: &ReconcileRound,
config: &crate::db::entity::user_running_network_configs::Model,
desired_config: NetworkConfig,
runtime_config_cache: &mut SessionRuntimeConfigCache,
) -> ConfigActionResult {
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
tracing::debug!(
machine_id = ?round.machine_id,
instance_id = %config.network_instance_id,
"skip runtime reconcile because webhook session is no longer current"
);
return ConfigActionResult::StopRound;
}
let ret = async {
let action =
match runtime_config_cache.plan(&config.network_instance_id, desired_config.clone())? {
Some(action) => action,
None => {
runtime_reconcile::prepare_web_source_runtime_reconcile(
&mut *rpc_client,
&config.network_instance_id,
desired_config.clone(),
true,
)
.await?
}
};
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
anyhow::bail!("webhook session is no longer current before runtime reconcile apply");
}
let observed_config = runtime_reconcile::apply_web_source_runtime_reconcile(
&mut *rpc_client,
&mut *config_client,
&config.network_instance_id,
desired_config.clone(),
action,
)
.await?;
runtime_config_cache.remember(&config.network_instance_id, observed_config);
Ok::<(), anyhow::Error>(())
}
.await;
tracing::info!(
user_id = ?round.user_id,
instance_id = %config.network_instance_id,
"Reconcile running web-source network instance: {:?}, user_token: {:?}",
ret,
round.req.user_token
);
if ret.is_ok() {
ConfigActionResult::Success
} else {
runtime_config_cache.forget(&config.network_instance_id);
ConfigActionResult::Failed
}
}
async fn run_missing_network_config(
session_data: &std::sync::Weak<RwLock<SessionData>>,
rpc_client: &mut SessionRpcClient,
round: &ReconcileRound,
config: &crate::db::entity::user_running_network_configs::Model,
desired_config: NetworkConfig,
) -> ConfigActionResult {
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
tracing::debug!(
machine_id = ?round.machine_id,
instance_id = %config.network_instance_id,
"skip run network instance because webhook session is no longer current"
);
return ConfigActionResult::StopRound;
}
let ret = rpc_client
.run_network_instance(
BaseController::default(),
RunNetworkInstanceRequest {
inst_id: Some(config.network_instance_id.clone().into()),
config: Some(desired_config),
overwrite: false,
source: PersistedConfigSource::from_db(&config.source).auto_run_rpc_source() as i32,
},
)
.await;
tracing::info!(
user_id = ?round.user_id,
"Run network instance: {:?}, user_token: {:?}",
ret,
round.req.user_token
);
if ret.is_ok() {
ConfigActionResult::Success
} else {
ConfigActionResult::Failed
}
}
async fn remember_web_runtime_config_after_run(
rpc_client: &mut SessionRpcClient,
inst_id: &str,
desired_config: &NetworkConfig,
runtime_config_cache: &mut SessionRuntimeConfigCache,
) -> anyhow::Result<()> {
let observed_config = runtime_reconcile::get_runtime_config(rpc_client, inst_id).await?;
remember_if_runtime_matches_desired(
inst_id,
desired_config,
observed_config,
runtime_config_cache,
)
}
fn remember_if_runtime_matches_desired(
inst_id: &str,
desired_config: &NetworkConfig,
observed_config: NetworkConfig,
runtime_config_cache: &mut SessionRuntimeConfigCache,
) -> anyhow::Result<()> {
let action = runtime_reconcile::prepare_web_source_runtime_reconcile_from_current(
&observed_config,
desired_config.clone(),
)?;
if !matches!(action, runtime_reconcile::RuntimeReconcileAction::None) {
anyhow::bail!("runtime config still differs after managed run");
}
runtime_config_cache.remember(inst_id, observed_config);
Ok(())
}
async fn mark_config_revision_applied_if_current(
session_data: &std::sync::Weak<RwLock<SessionData>>,
storage: &StorageInner,
round: &ReconcileRound,
outcome: &ReconcileOutcome,
) -> RoundStatus<()> {
if outcome.managed_revision_failed || !round.should_apply_runtime_revision {
return RoundStatus::Ready(());
}
let current_target_config_revision = match storage
.db
.get_managed_config_revision((round.user_id, round.machine_id))
.await
{
Ok(revision) => revision,
Err(e) => {
tracing::error!("Failed to verify managed config revision, error: {:?}", e);
return RoundStatus::Stop;
}
};
if current_target_config_revision != round.target_config_revision {
return RoundStatus::Ready(());
}
let Some(data) = session_data.upgrade() else {
return RoundStatus::Stop;
};
let mut data = data.write().await;
if !SessionRpcService::runtime_heartbeat_is_current_locked(&data, &round.req) {
return RoundStatus::Ready(());
}
data.applied_config_revision = round.target_config_revision.clone();
RoundStatus::Ready(())
}
#[cfg(test)]
mod tests {
use easytier::proto::api::manage::{NetworkingMethod, PortForwardConfig};
use super::*;
fn config_with_port_forwards(port_forwards: Vec<PortForwardConfig>) -> NetworkConfig {
NetworkConfig {
instance_id: Some("11111111-1111-1111-1111-111111111111".to_string()),
dhcp: Some(true),
network_name: Some("managed".to_string()),
network_secret: Some("secret".to_string()),
networking_method: Some(NetworkingMethod::Manual as i32),
port_forwards,
..Default::default()
}
}
fn port_forward(bind_port: u32, dst_port: u32) -> PortForwardConfig {
PortForwardConfig {
bind_ip: "127.0.0.1".to_string(),
bind_port,
dst_ip: "10.144.0.1".to_string(),
dst_port,
proto: "tcp".to_string(),
}
}
#[test]
fn session_runtime_config_cache_misses_unknown_instance() {
let cache = SessionRuntimeConfigCache::default();
let action = cache
.plan("missing", config_with_port_forwards(Vec::new()))
.expect("prepare action");
assert!(action.is_none());
}
#[test]
fn session_runtime_config_cache_skips_matching_observed_config() {
let mut cache = SessionRuntimeConfigCache::default();
let config = config_with_port_forwards(vec![port_forward(23000, 5174)]);
cache.remember("managed", config.clone());
let action = cache
.plan("managed", config)
.expect("prepare action")
.expect("cached action");
assert!(matches!(
action,
runtime_reconcile::RuntimeReconcileAction::None
));
}
#[test]
fn session_runtime_config_cache_plans_patch_from_observed_config() {
let mut cache = SessionRuntimeConfigCache::default();
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
let desired =
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
cache.remember("managed", current);
let action = cache
.plan("managed", desired)
.expect("prepare action")
.expect("cached action");
let runtime_reconcile::RuntimeReconcileAction::Patch(patch) = action else {
panic!("expected cached runtime config to produce hot patch");
};
assert_eq!(patch.port_forwards.len(), 1);
}
#[test]
fn session_runtime_config_cache_retain_desired_removes_stale_entries() {
let mut cache = SessionRuntimeConfigCache::default();
let config = config_with_port_forwards(Vec::new());
cache.remember("keep", config.clone());
cache.remember("drop", config);
cache.retain_desired(&HashSet::from(["keep".to_string()]));
assert!(cache.entries.contains_key("keep"));
assert!(!cache.entries.contains_key("drop"));
}
#[test]
fn session_runtime_config_cache_forget_removes_observed_config() {
let mut cache = SessionRuntimeConfigCache::default();
let config = config_with_port_forwards(Vec::new());
cache.remember("managed", config.clone());
cache.forget("managed");
let action = cache
.plan("managed", config)
.expect("prepare action after remove");
assert!(action.is_none());
}
#[test]
fn missing_run_remembers_observed_config_when_it_matches_desired() {
let mut cache = SessionRuntimeConfigCache::default();
let config = config_with_port_forwards(vec![port_forward(23000, 5174)]);
remember_if_runtime_matches_desired("managed", &config, config.clone(), &mut cache)
.expect("remember observed config after run");
let action = cache
.plan("managed", config)
.expect("prepare action after run")
.expect("cached action");
assert!(matches!(
action,
runtime_reconcile::RuntimeReconcileAction::None
));
}
#[test]
fn missing_run_does_not_remember_observed_config_that_still_differs() {
let mut cache = SessionRuntimeConfigCache::default();
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
let desired =
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
let err = remember_if_runtime_matches_desired("managed", &desired, current, &mut cache)
.expect_err("expected stale run result not to be cached");
assert!(
err.to_string()
.contains("runtime config still differs after managed run")
);
let action = cache
.plan("managed", desired)
.expect("prepare action after stale run result");
assert!(action.is_none());
}
}
@@ -0,0 +1,402 @@
use std::{sync::Arc, time::Duration};
use anyhow::Context as _;
use easytier::proto::web::HeartbeatRequest;
use tokio::sync::RwLock;
use super::{
SessionAuthState, SessionData, SessionRpcService, WebhookConnectNotification,
WebhookDisconnectNotification, send_webhook_connection_transition,
};
use crate::{
client_manager::storage::{Storage, StorageToken},
webhook::SharedWebhookConfig,
};
pub(super) const VALIDATION_RETRY_MS: u64 = 60_000;
pub(super) struct WebhookHeartbeatValidation {
pub(super) config_revision: String,
pub(super) binding_version: u64,
}
pub(super) struct WebhookValidationInput {
pub(super) storage: Storage,
pub(super) webhook_config: SharedWebhookConfig,
pub(super) client_url: url::Url,
pub(super) applied_config_revision: Option<String>,
pub(super) req: HeartbeatRequest,
pub(super) machine_id: uuid::Uuid,
}
fn deterministic_machine_delay(machine_id: uuid::Uuid, max_delay_ms: u64) -> Duration {
let delay_ms = (machine_id.as_u128() % u128::from(max_delay_ms + 1)) as u64;
Duration::from_millis(delay_ms)
}
pub(super) fn retry_delay(machine_id: uuid::Uuid) -> Duration {
Duration::from_millis(VALIDATION_RETRY_MS)
+ deterministic_machine_delay(machine_id, VALIDATION_RETRY_MS)
}
async fn request_heartbeat_validation(
webhook_config: &crate::webhook::WebhookConfig,
client_url: &url::Url,
persisted_config_revision: Option<&str>,
applied_config_revision: Option<&str>,
req: &HeartbeatRequest,
machine_id: uuid::Uuid,
) -> anyhow::Result<Option<WebhookHeartbeatValidation>> {
let webhook_req = crate::webhook::ValidateTokenRequest {
token: req.user_token.clone(),
machine_id: machine_id.to_string(),
public_ip: client_url.host_str().map(str::to_string),
hostname: req.hostname.clone(),
version: req.easytier_version.clone(),
os_type: req.device_os.as_ref().map(|info| info.os_type.clone()),
os_version: req.device_os.as_ref().map(|info| info.version.clone()),
os_distribution: req.device_os.as_ref().map(|info| info.distribution.clone()),
web_instance_id: webhook_config.web_instance_id.clone(),
web_instance_api_base_url: webhook_config.web_instance_api_base_url.clone(),
persisted_config_revision: persisted_config_revision.map(str::to_string),
applied_config_revision: applied_config_revision.map(str::to_string),
};
let resp = webhook_config
.validate_token(&webhook_req)
.await
.map_err(|e| anyhow::anyhow!("Webhook token validation failed: {:?}", e))?;
if !resp.valid {
return Ok(None);
}
Ok(Some(WebhookHeartbeatValidation {
config_revision: resp.config_revision,
binding_version: resp.binding_version,
}))
}
async fn resolve_user_id(storage: &Storage, token: &str) -> anyhow::Result<i32> {
let user_id = match storage
.db()
.get_user_id_by_token(token)
.await
.map_err(|e| anyhow::anyhow!("DB error: {:?}", e))?
{
Some(id) => id,
None => storage
.auto_create_user(token)
.await
.with_context(|| format!("Failed to auto-create webhook user: {:?}", token))?,
};
Ok(user_id)
}
async fn persisted_config_revision_for_token(
storage: &Storage,
token: &str,
machine_id: uuid::Uuid,
) -> anyhow::Result<Option<String>> {
let Some(user_id) = storage
.db()
.get_user_id_by_token(token)
.await
.map_err(|e| anyhow::anyhow!("DB error: {:?}", e))?
else {
return Ok(None);
};
storage
.db()
.get_managed_config_revision((user_id, machine_id))
.await
.map_err(|e| anyhow::anyhow!("DB error: {:?}", e))
}
async fn wait_for_input(
session_data: std::sync::Weak<RwLock<SessionData>>,
) -> Option<WebhookValidationInput> {
loop {
let notify = {
let session_data = session_data.upgrade()?;
let mut data = session_data.write().await;
if matches!(data.auth_state, SessionAuthState::Invalid) {
data.webhook_validation_dirty = false;
tracing::info!(
client_url = %data.client_url,
"webhook validation stopped for invalid session; reconnect is required before revalidation"
);
return None;
}
if data.webhook_validation_dirty {
data.webhook_validation_dirty = false;
let req = data.req.clone()?;
let machine_id = req.machine_id.map(Into::into)?;
let storage = Storage::try_from(data.storage.clone()).ok()?;
return Some(WebhookValidationInput {
storage,
webhook_config: data.webhook_config.clone(),
client_url: data.client_url.clone(),
applied_config_revision: data.applied_config_revision.clone(),
req,
machine_id,
});
}
data.webhook_validation_notify.clone()
};
notify.notified().await;
}
}
pub(super) async fn run_worker(session_data: std::sync::Weak<RwLock<SessionData>>) {
while let Some(input) = wait_for_input(session_data.clone()).await {
let machine_id = input.machine_id;
if let Err(error) = run_round(session_data.clone(), input).await {
tracing::warn!(
?machine_id,
%error,
"webhook validation failed, will retry later"
);
tokio::time::sleep(retry_delay(machine_id)).await;
mark_dirty_if_current(&session_data, machine_id).await;
}
}
}
pub(super) async fn run_round(
session_data: std::sync::Weak<RwLock<SessionData>>,
input: WebhookValidationInput,
) -> anyhow::Result<()> {
let persisted_config_revision = persisted_config_revision_for_token(
&input.storage,
&input.req.user_token,
input.machine_id,
)
.await?;
let validation = request_heartbeat_validation(
&input.webhook_config,
&input.client_url,
persisted_config_revision.as_deref(),
input.applied_config_revision.as_deref(),
&input.req,
input.machine_id,
)
.await?;
let Some(validation) = validation else {
apply_rejected(&session_data, &input).await;
return Ok(());
};
let user_id = resolve_user_id(&input.storage, &input.req.user_token).await?;
apply_success(&session_data, input, validation, user_id).await;
Ok(())
}
async fn mark_dirty_if_current(
session_data: &std::sync::Weak<RwLock<SessionData>>,
machine_id: uuid::Uuid,
) {
let Some(session_data) = session_data.upgrade() else {
return;
};
let notify = {
let mut data = session_data.write().await;
let Some(req) = data.req.as_ref() else {
return;
};
if req.machine_id.map(uuid::Uuid::from) != Some(machine_id) {
return;
}
if matches!(data.auth_state, SessionAuthState::Invalid) {
data.webhook_validation_dirty = false;
tracing::debug!(
%machine_id,
"skip webhook validation retry for invalid session"
);
return;
}
SessionRpcService::mark_webhook_validation_dirty_locked(&mut data)
};
notify.notify_one();
}
pub(super) async fn apply_rejected(
session_data: &std::sync::Weak<RwLock<SessionData>>,
input: &WebhookValidationInput,
) {
let Some(session_data) = session_data.upgrade() else {
return;
};
let (storage_token, disconnect_notification) = {
let mut data = session_data.write().await;
if !data.req.as_ref().is_some_and(|req| {
SessionRpcService::heartbeat_matches_identity(
req,
&input.req.user_token,
input.machine_id,
)
}) {
return;
}
tracing::info!(
machine_id = %input.machine_id,
client_url = %data.client_url,
"webhook token rejected; marking session invalid and requiring client reconnect"
);
data.auth_state = SessionAuthState::Invalid;
data.webhook_validation_dirty = false;
data.binding_version = None;
data.applied_config_revision = None;
let storage_token = data.storage_token.clone();
let disconnect_notification = storage_token.as_ref().and_then(|storage_token| {
data.webhook_connected_binding_version
.take()
.map(|binding_version| WebhookDisconnectNotification {
webhook: data.webhook_config.clone(),
storage_token: storage_token.clone(),
binding_version,
})
});
(storage_token, disconnect_notification)
};
if let Some(storage_token) = storage_token {
let report_time = SessionRpcService::heartbeat_report_timestamp(&input.req);
input
.storage
.update_client(storage_token, report_time, false);
}
if disconnect_notification.is_some() {
wait_webhook_connection_transition(
Arc::downgrade(&session_data),
disconnect_notification,
None,
)
.await;
}
}
pub(super) async fn apply_success(
session_data: &std::sync::Weak<RwLock<SessionData>>,
input: WebhookValidationInput,
validation: WebhookHeartbeatValidation,
user_id: i32,
) {
let WebhookHeartbeatValidation {
config_revision: _,
binding_version,
} = validation;
let Some(session_data) = session_data.upgrade() else {
return;
};
let (storage_token, notifier, disconnect_notification, connect_notification, runtime_req) = {
let mut data = session_data.write().await;
let Some(runtime_req) = data.req.clone() else {
return;
};
if !SessionRpcService::heartbeat_matches_identity(
&runtime_req,
&input.req.user_token,
input.machine_id,
) {
return;
}
if matches!(data.auth_state, SessionAuthState::Invalid) {
tracing::info!(
machine_id = %input.machine_id,
client_url = %data.client_url,
"ignore webhook validation success for invalid session; reconnect is required before revalidation"
);
return;
}
let previous_connected_binding_version = data.webhook_connected_binding_version;
let client_url = data.client_url.clone();
let storage_token = data.storage_token.get_or_insert_with(|| StorageToken {
token: runtime_req.user_token.clone(),
client_url,
machine_id: input.machine_id,
user_id,
});
let storage_token = storage_token.clone();
data.auth_state = SessionAuthState::Authorized;
data.binding_version = Some(binding_version);
let should_notify_connected = previous_connected_binding_version != Some(binding_version);
let disconnect_notification = previous_connected_binding_version
.filter(|previous_binding_version| *previous_binding_version != binding_version)
.map(|previous_binding_version| {
data.webhook_connected_binding_version = None;
WebhookDisconnectNotification {
webhook: data.webhook_config.clone(),
storage_token: storage_token.clone(),
binding_version: previous_binding_version,
}
});
let connect_notification = should_notify_connected.then(|| WebhookConnectNotification {
webhook: data.webhook_config.clone(),
storage_token: storage_token.clone(),
binding_version,
req: crate::webhook::NodeConnectedRequest {
machine_id: input.machine_id.to_string(),
token: runtime_req.user_token.clone(),
user_id: Some(user_id),
hostname: runtime_req.hostname.clone(),
version: runtime_req.easytier_version.clone(),
os_type: runtime_req
.device_os
.as_ref()
.map(|info| info.os_type.clone()),
os_version: runtime_req
.device_os
.as_ref()
.map(|info| info.version.clone()),
os_distribution: runtime_req
.device_os
.as_ref()
.map(|info| info.distribution.clone()),
web_instance_id: data.webhook_config.web_instance_id.clone(),
binding_version: Some(binding_version),
},
});
(
storage_token,
data.notifier.clone(),
disconnect_notification,
connect_notification,
runtime_req,
)
};
if disconnect_notification.is_some() || connect_notification.is_some() {
wait_webhook_connection_transition(
Arc::downgrade(&session_data),
disconnect_notification,
connect_notification,
)
.await;
}
let report_time = SessionRpcService::heartbeat_report_timestamp(&runtime_req);
input
.storage
.update_client(storage_token, report_time, true);
let _ = notifier.send(runtime_req);
}
async fn wait_webhook_connection_transition(
session_data: std::sync::Weak<RwLock<SessionData>>,
disconnect: Option<WebhookDisconnectNotification>,
connect: Option<WebhookConnectNotification>,
) {
let transition = tokio::spawn(send_webhook_connection_transition(
session_data,
disconnect,
connect,
));
if let Err(error) = transition.await {
tracing::warn!(%error, "webhook connection transition task failed");
}
}
+123 -9
View File
@@ -17,6 +17,7 @@ pub struct StorageToken {
struct ClientInfo {
storage_token: StorageToken,
report_time: i64,
authorized: bool,
}
#[derive(Debug)]
@@ -55,7 +56,19 @@ impl Storage {
fn update_client_info_map(map: &DashMap<uuid::Uuid, ClientInfo>, client_info: &ClientInfo) {
map.entry(client_info.storage_token.machine_id)
.and_modify(|e| {
if e.report_time < client_info.report_time {
let same_client = e.storage_token.client_url
== client_info.storage_token.client_url
&& e.storage_token.user_id == client_info.storage_token.user_id;
let should_replace = if (same_client && e.authorized != client_info.authorized)
|| (!e.authorized && client_info.authorized)
{
true
} else if e.authorized && !client_info.authorized && !same_client {
false
} else {
e.report_time < client_info.report_time
};
if should_replace {
assert_eq!(
e.storage_token.machine_id,
client_info.storage_token.machine_id
@@ -66,12 +79,13 @@ impl Storage {
.or_insert(client_info.clone());
}
pub fn update_client(&self, stoken: StorageToken, report_time: i64) {
pub fn update_client(&self, stoken: StorageToken, report_time: i64, authorized: bool) {
let inner = self.0.user_clients_map.entry(stoken.user_id).or_default();
let client_info = ClientInfo {
storage_token: stoken.clone(),
report_time,
authorized,
};
Self::update_client_info_map(&inner, &client_info);
}
@@ -93,11 +107,21 @@ impl Storage {
&self,
user_id: UserIdInDb,
machine_id: &uuid::Uuid,
) -> Option<url::Url> {
self.get_client_url_by_machine_id_with_auth(user_id, machine_id, true)
}
pub fn get_client_url_by_machine_id_with_auth(
&self,
user_id: UserIdInDb,
machine_id: &uuid::Uuid,
require_authorized: bool,
) -> Option<url::Url> {
self.0.user_clients_map.get(&user_id).and_then(|info_map| {
info_map
.get(machine_id)
.map(|info| info.storage_token.client_url.clone())
info_map.get(machine_id).and_then(|info| {
(!require_authorized || info.authorized)
.then(|| info.storage_token.client_url.clone())
})
})
}
@@ -108,6 +132,7 @@ impl Storage {
.map(|info_map| {
info_map
.iter()
.filter(|info| info.value().authorized)
.map(|info| info.value().storage_token.client_url.clone())
.collect()
})
@@ -115,6 +140,14 @@ impl Storage {
}
pub fn list_clients(&self) -> Vec<StorageToken> {
self.list_clients_with_auth(true)
}
pub fn list_all_clients(&self) -> Vec<StorageToken> {
self.list_clients_with_auth(false)
}
fn list_clients_with_auth(&self, require_authorized: bool) -> Vec<StorageToken> {
self.0
.user_clients_map
.iter()
@@ -122,6 +155,7 @@ impl Storage {
user_clients
.value()
.iter()
.filter(|info| !require_authorized || info.value().authorized)
.map(|info| info.value().storage_token.clone())
.collect::<Vec<_>>()
})
@@ -164,8 +198,8 @@ mod tests {
let user1_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
let user2_token = make_storage_token(2, machine_id, "tcp://127.0.0.1:1002");
storage.update_client(user1_token.clone(), 10);
storage.update_client(user2_token.clone(), 20);
storage.update_client(user1_token.clone(), 10, true);
storage.update_client(user2_token.clone(), 20, true);
assert_eq!(
storage.get_client_url_by_machine_id(1, &machine_id),
@@ -195,8 +229,8 @@ mod tests {
let user1_token = make_storage_token(1, uuid::Uuid::new_v4(), "tcp://127.0.0.1:1001");
let user2_token = make_storage_token(2, uuid::Uuid::new_v4(), "tcp://127.0.0.1:1002");
storage.update_client(user1_token.clone(), 10);
storage.update_client(user2_token.clone(), 20);
storage.update_client(user1_token.clone(), 10, true);
storage.update_client(user2_token.clone(), 20, true);
let tokens = storage.list_clients();
assert_eq!(tokens.len(), 2);
@@ -209,4 +243,84 @@ mod tests {
assert_eq!(tokens.len(), 1);
assert_eq!(tokens[0].token, user2_token.token);
}
#[tokio::test]
async fn pending_client_is_listed_but_not_authorized_for_machine_lookup() {
let storage = Storage::new(Db::memory_db().await);
let machine_id = uuid::Uuid::new_v4();
let token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
storage.update_client(token.clone(), 10, false);
assert_eq!(storage.list_clients().len(), 0);
assert_eq!(storage.list_all_clients().len(), 1);
assert_eq!(storage.list_user_clients(1), Vec::<url::Url>::new());
assert_eq!(storage.get_client_url_by_machine_id(1, &machine_id), None);
assert_eq!(
storage.get_client_url_by_machine_id_with_auth(1, &machine_id, false),
Some(token.client_url.clone())
);
storage.update_client(token.clone(), 11, true);
assert_eq!(
storage.get_client_url_by_machine_id(1, &machine_id),
Some(token.client_url.clone())
);
storage.update_client(token.clone(), 11, false);
assert_eq!(storage.get_client_url_by_machine_id(1, &machine_id), None);
assert_eq!(storage.list_clients().len(), 0);
assert_eq!(storage.list_all_clients().len(), 1);
}
#[tokio::test]
async fn stale_client_authorization_update_does_not_replace_newer_client() {
let storage = Storage::new(Db::memory_db().await);
let machine_id = uuid::Uuid::new_v4();
let old_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
let new_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002");
storage.update_client(old_token.clone(), 10, true);
storage.update_client(new_token.clone(), 20, true);
storage.update_client(old_token, 10, false);
assert_eq!(
storage.get_client_url_by_machine_id(1, &machine_id),
Some(new_token.client_url)
);
}
#[tokio::test]
async fn pending_client_does_not_replace_authorized_route() {
let storage = Storage::new(Db::memory_db().await);
let machine_id = uuid::Uuid::new_v4();
let authorized_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
let pending_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002");
storage.update_client(authorized_token.clone(), 10, true);
storage.update_client(pending_token, i64::MAX, false);
assert_eq!(
storage.get_client_url_by_machine_id(1, &machine_id),
Some(authorized_token.client_url)
);
}
#[tokio::test]
async fn authorized_client_replaces_pending_route_regardless_of_report_time() {
let storage = Storage::new(Db::memory_db().await);
let machine_id = uuid::Uuid::new_v4();
let pending_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
let authorized_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002");
storage.update_client(pending_token, i64::MAX, false);
storage.update_client(authorized_token.clone(), 10, true);
assert_eq!(
storage.get_client_url_by_machine_id(1, &machine_id),
Some(authorized_token.client_url)
);
}
}
@@ -0,0 +1,38 @@
//! `SeaORM` Entity, hand-written to match the generated entity style.
use sea_orm::entity::prelude::*;
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
#[sea_orm(table_name = "managed_config_revisions")]
pub struct Model {
#[sea_orm(primary_key)]
pub id: i32,
pub user_id: i32,
#[sea_orm(column_type = "Text")]
pub device_id: String,
#[sea_orm(column_type = "Text")]
pub config_revision: String,
pub create_time: DateTimeWithTimeZone,
pub update_time: DateTimeWithTimeZone,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {
#[sea_orm(
belongs_to = "super::users::Entity",
from = "Column::UserId",
to = "super::users::Column::Id",
on_update = "Cascade",
on_delete = "Cascade"
)]
Users,
}
impl Related<super::users::Entity> for Entity {
fn to() -> RelationDef {
Relation::Users.def()
}
}
impl ActiveModelBehavior for ActiveModel {}
+1
View File
@@ -4,6 +4,7 @@ pub mod prelude;
pub mod groups;
pub mod groups_permissions;
pub mod managed_config_revisions;
pub mod permissions;
pub mod tower_sessions;
pub mod user_running_network_configs;
+1
View File
@@ -2,6 +2,7 @@
pub use super::groups::Entity as Groups;
pub use super::groups_permissions::Entity as GroupsPermissions;
pub use super::managed_config_revisions::Entity as ManagedConfigRevisions;
pub use super::permissions::Entity as Permissions;
pub use super::tower_sessions::Entity as TowerSessions;
pub use super::user_running_network_configs::Entity as UserRunningNetworkConfigs;
+146
View File
@@ -141,6 +141,110 @@ impl Db {
) -> Result<Option<UserIdInDb>, DbErr> {
self.get_user_id(token).await
}
pub async fn get_managed_config_revision(
&self,
(user_id, device_id): (UserIdInDb, Uuid),
) -> Result<Option<String>, DbErr> {
use entity::managed_config_revisions as mcr;
let revision = mcr::Entity::find()
.filter(mcr::Column::UserId.eq(user_id))
.filter(mcr::Column::DeviceId.eq(device_id.to_string()))
.one(self.orm_db())
.await?;
Ok(revision.map(|row| row.config_revision))
}
pub async fn set_managed_config_revision(
&self,
(user_id, device_id): (UserIdInDb, Uuid),
config_revision: &str,
) -> Result<(), DbErr> {
use entity::managed_config_revisions as mcr;
let now = chrono::Local::now().fixed_offset();
let on_conflict = OnConflict::columns([mcr::Column::UserId, mcr::Column::DeviceId])
.update_columns([mcr::Column::ConfigRevision, mcr::Column::UpdateTime])
.to_owned();
let insert_m = mcr::ActiveModel {
user_id: Set(user_id),
device_id: Set(device_id.to_string()),
config_revision: Set(config_revision.to_string()),
create_time: Set(now),
update_time: Set(now),
..Default::default()
};
mcr::Entity::insert(insert_m)
.on_conflict(on_conflict)
.do_nothing()
.exec(self.orm_db())
.await?;
Ok(())
}
pub async fn insert_or_update_web_network_config(
&self,
(user_id, device_id): (UserIdInDb, Uuid),
network_inst_id: Uuid,
network_config: NetworkConfig,
) -> Result<bool, DbErr> {
let now = chrono::Local::now().fixed_offset();
let network_config =
serde_json::to_string(&network_config).map_err(|e| DbErr::Json(e.to_string()))?;
let source = ConfigSource::Web.as_str();
let result = sqlx::query(
r#"
INSERT INTO user_running_network_configs (
user_id, device_id, network_instance_id, network_config,
source, disabled, create_time, update_time
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(user_id, device_id, network_instance_id) DO UPDATE SET
network_config = excluded.network_config,
source = excluded.source,
disabled = excluded.disabled,
update_time = excluded.update_time
WHERE user_running_network_configs.source = ?
"#,
)
.bind(user_id)
.bind(device_id.to_string())
.bind(network_inst_id.to_string())
.bind(network_config)
.bind(source)
.bind(false)
.bind(now)
.bind(now)
.bind(source)
.execute(&self.db)
.await
.map_err(|e| DbErr::Custom(e.to_string()))?;
Ok(result.rows_affected() > 0)
}
pub async fn delete_web_network_configs(
&self,
(user_id, device_id): (UserIdInDb, Uuid),
network_inst_ids: &[Uuid],
) -> Result<(), DbErr> {
use entity::user_running_network_configs as urnc;
urnc::Entity::delete_many()
.filter(urnc::Column::UserId.eq(user_id))
.filter(urnc::Column::DeviceId.eq(device_id.to_string()))
.filter(urnc::Column::Source.eq(ConfigSource::Web.as_str()))
.filter(
urnc::Column::NetworkInstanceId
.is_in(network_inst_ids.iter().map(|id| id.to_string())),
)
.exec(self.orm_db())
.await?;
Ok(())
}
}
#[async_trait]
@@ -468,4 +572,46 @@ mod tests {
assert_eq!(device1_configs.len(), 1);
assert_eq!(device2_configs.len(), 1);
}
#[tokio::test]
async fn test_web_network_config_does_not_replace_user_owned_config() {
let db = Db::memory_db().await;
let user_id = db.auto_create_user("user-web-race").await.unwrap().id;
let device_id = uuid::Uuid::new_v4();
let inst_id = uuid::Uuid::new_v4();
db.insert_or_update_user_network_config(
(user_id, device_id),
inst_id,
NetworkConfig {
network_name: Some("user-owned".to_string()),
..Default::default()
},
ConfigSource::User,
)
.await
.unwrap();
let updated = db
.insert_or_update_web_network_config(
(user_id, device_id),
inst_id,
NetworkConfig {
network_name: Some("web-owned".to_string()),
..Default::default()
},
)
.await
.unwrap();
assert!(!updated);
let saved = db
.get_network_config((user_id, device_id), &inst_id.to_string())
.await
.unwrap()
.unwrap();
assert_eq!(saved.get_network_config_source(), ConfigSource::User);
let saved_config = saved.get_network_config().unwrap();
assert_eq!(saved_config.network_name.as_deref(), Some("user-owned"));
}
}
+10 -1
View File
@@ -3,8 +3,8 @@
#[macro_use]
extern crate rust_i18n;
use std::net::IpAddr;
use std::sync::Arc;
use std::{net::IpAddr, time::Duration};
use clap::Parser;
use easytier::tunnel::websocket::WsTunnelListener;
@@ -113,6 +113,14 @@ struct Cli {
)]
geoip_db: Option<String>,
#[arg(
long,
env = "ET_HEARTBEAT_MIN_RESPONSE_MS",
default_value = "0",
help = t!("cli.heartbeat_min_response_ms").to_string(),
)]
heartbeat_min_response_ms: u64,
#[cfg(feature = "embed")]
#[arg(
long,
@@ -312,6 +320,7 @@ async fn main() {
let mut mgr = client_manager::ClientManager::new(
db.clone(),
cli.geoip_db,
Duration::from_millis(cli.heartbeat_min_response_ms),
feature_flags.clone(),
webhook_config.clone(),
);
@@ -0,0 +1,46 @@
use sea_orm_migration::prelude::*;
pub struct Migration;
impl MigrationName for Migration {
fn name(&self) -> &str {
"m20260619_000005_managed_config_revisions"
}
}
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.get_connection()
.execute_unprepared(
r#"
CREATE TABLE managed_config_revisions (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
user_id INTEGER NOT NULL,
device_id TEXT NOT NULL,
config_revision TEXT NOT NULL,
create_time TEXT NOT NULL,
update_time TEXT NOT NULL,
CONSTRAINT fk_managed_config_revisions_user_id_to_users_id
FOREIGN KEY (user_id) REFERENCES users(id)
ON DELETE CASCADE
ON UPDATE CASCADE
);
CREATE UNIQUE INDEX idx_managed_config_revisions_scope
ON managed_config_revisions(user_id, device_id);
"#,
)
.await?;
Ok(())
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.get_connection()
.execute_unprepared("DROP TABLE managed_config_revisions;")
.await?;
Ok(())
}
}
+2
View File
@@ -4,6 +4,7 @@ mod m20241029_000001_init;
mod m20260403_000002_scope_network_config_unique;
mod m20260421_000003_add_network_config_source;
mod m20260514_000004_rename_web_config_source;
mod m20260619_000005_managed_config_revisions;
pub struct Migrator;
@@ -15,6 +16,7 @@ impl MigratorTrait for Migrator {
Box::new(m20260403_000002_scope_network_config_unique::Migration),
Box::new(m20260421_000003_add_network_config_source::Migration),
Box::new(m20260514_000004_rename_web_config_source::Migration),
Box::new(m20260619_000005_managed_config_revisions::Migration),
]
}
}
+1 -1
View File
@@ -307,7 +307,7 @@ impl RestfulServer {
async fn handle_list_all_sessions_internal(
State(client_mgr): AppState,
) -> Result<Json<ListSessionJsonResp>, HttpHandleError> {
let ret = client_mgr.list_sessions().await;
let ret = client_mgr.list_all_sessions().await;
Ok(ListSessionJsonResp(ret).into())
}
+15 -5
View File
@@ -93,6 +93,8 @@ struct ManagedNetworkConfigJson {
#[derive(Debug, serde::Deserialize, serde::Serialize)]
struct ReconcileManagedNetworkConfigsJsonReq {
managed_network_configs: Vec<ManagedNetworkConfigJson>,
config_revision: Option<String>,
expected_config_revision: Option<String>,
}
#[derive(Debug, serde::Deserialize, serde::Serialize)]
@@ -357,13 +359,21 @@ impl NetworkApi {
})
.collect();
client_mgr
.reconcile_managed_network_configs(user_id, machine_id, desired)
.reconcile_managed_network_configs(
user_id,
machine_id,
desired,
payload.config_revision,
payload.expected_config_revision,
)
.await
.map_err(|err| {
(
StatusCode::INTERNAL_SERVER_ERROR,
other_error(err.to_string()).into(),
)
let status = if crate::client_manager::is_managed_config_revision_conflict(&err) {
StatusCode::CONFLICT
} else {
StatusCode::INTERNAL_SERVER_ERROR
};
(status, other_error(err.to_string()).into())
})?;
Ok(Void::default().into())
}
+589 -16
View File
@@ -1,6 +1,248 @@
use std::sync::Arc;
use std::{
cmp::Ordering,
collections::VecDeque,
fmt,
sync::{Arc, Mutex},
time::{Duration, Instant},
};
use serde::{Deserialize, Serialize};
use tokio::sync::oneshot;
const VALIDATE_TOKEN_INITIAL_CONCURRENCY: usize = 8;
const VALIDATE_TOKEN_MIN_CONCURRENCY: usize = 2;
const VALIDATE_TOKEN_MAX_CONCURRENCY: usize = 64;
const VALIDATE_TOKEN_ADJUST_WINDOW: Duration = Duration::from_secs(1);
const VALIDATE_TOKEN_SLOW_THRESHOLD: Duration = Duration::from_secs(2);
const WEBHOOK_HTTP_TIMEOUT: Duration = Duration::from_secs(10);
struct AdaptiveValidateLimiter {
state: Mutex<AdaptiveValidateLimiterState>,
}
impl fmt::Debug for AdaptiveValidateLimiter {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("AdaptiveValidateLimiter")
.field("state", &self.lock_state())
.finish_non_exhaustive()
}
}
struct AdaptiveValidateLimiterState {
limit: usize,
in_flight: usize,
waiters: VecDeque<oneshot::Sender<AdaptiveValidateGrant>>,
window_started_at: Instant,
samples: usize,
slow_samples: usize,
failures: usize,
had_queue: bool,
}
impl fmt::Debug for AdaptiveValidateLimiterState {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("AdaptiveValidateLimiterState")
.field("limit", &self.limit)
.field("in_flight", &self.in_flight)
.field("waiters", &self.waiters.len())
.field("window_started_at", &self.window_started_at)
.field("samples", &self.samples)
.field("slow_samples", &self.slow_samples)
.field("failures", &self.failures)
.field("had_queue", &self.had_queue)
.finish()
}
}
struct AdaptiveValidatePermit {
limiter: Arc<AdaptiveValidateLimiter>,
started_at: Instant,
completed: bool,
}
struct AdaptiveValidateGrant {
limiter: Arc<AdaptiveValidateLimiter>,
active: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LimitAdjustment {
Unchanged,
Increased,
Decreased,
}
impl AdaptiveValidateLimiter {
fn new() -> Arc<Self> {
Arc::new(Self {
state: Mutex::new(AdaptiveValidateLimiterState::new(Instant::now())),
})
}
async fn acquire(self: &Arc<Self>) -> AdaptiveValidatePermit {
loop {
let receiver = {
let mut state = self.lock_state();
state.complete_window_if_due(Instant::now());
if state.waiters.is_empty() && state.in_flight < state.limit {
state.in_flight += 1;
return AdaptiveValidatePermit::new(self.clone());
}
let (sender, receiver) = oneshot::channel();
state.had_queue = true;
state.waiters.push_back(sender);
self.grant_waiters(&mut state);
receiver
};
if let Ok(grant) = receiver.await {
return grant.into_permit();
}
}
}
fn grant_waiters(self: &Arc<Self>, state: &mut AdaptiveValidateLimiterState) {
while state.in_flight < state.limit {
let Some(waiter) = state.waiters.pop_front() else {
break;
};
state.in_flight += 1;
if let Err(mut grant) = waiter.send(AdaptiveValidateGrant::new(self.clone())) {
grant.disarm();
state.in_flight -= 1;
}
}
}
fn record_sample(self: &Arc<Self>, elapsed: Duration, success: bool) {
let mut state = self.lock_state();
let adjustment = state.record_sample(Instant::now(), elapsed, success);
if adjustment == LimitAdjustment::Increased {
self.grant_waiters(&mut state);
}
}
fn release_slot(self: &Arc<Self>) {
let mut state = self.lock_state();
state.in_flight = state.in_flight.saturating_sub(1);
self.grant_waiters(&mut state);
}
fn lock_state(&self) -> std::sync::MutexGuard<'_, AdaptiveValidateLimiterState> {
self.state
.lock()
.expect("adaptive validate limiter state should not be poisoned")
}
}
impl AdaptiveValidateLimiterState {
fn new(now: Instant) -> Self {
Self {
limit: VALIDATE_TOKEN_INITIAL_CONCURRENCY,
in_flight: 0,
waiters: VecDeque::new(),
window_started_at: now,
samples: 0,
slow_samples: 0,
failures: 0,
had_queue: false,
}
}
fn record_sample(&mut self, now: Instant, elapsed: Duration, success: bool) -> LimitAdjustment {
self.samples += 1;
if elapsed > VALIDATE_TOKEN_SLOW_THRESHOLD {
self.slow_samples += 1;
}
if !success {
self.failures += 1;
}
self.complete_window_if_due(now)
}
fn complete_window_if_due(&mut self, now: Instant) -> LimitAdjustment {
if now.duration_since(self.window_started_at) < VALIDATE_TOKEN_ADJUST_WINDOW {
return LimitAdjustment::Unchanged;
}
let old_limit = self.limit;
if self.samples > 0 {
if self.failures > 0 || self.is_p95_slow() {
self.limit = (self.limit / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY);
} else if self.had_queue {
self.limit = (self.limit + 1).min(VALIDATE_TOKEN_MAX_CONCURRENCY);
}
}
self.window_started_at = now;
self.samples = 0;
self.slow_samples = 0;
self.failures = 0;
self.had_queue = false;
match self.limit.cmp(&old_limit) {
Ordering::Greater => LimitAdjustment::Increased,
Ordering::Less => LimitAdjustment::Decreased,
Ordering::Equal => LimitAdjustment::Unchanged,
}
}
fn is_p95_slow(&self) -> bool {
self.slow_samples > 0 && self.slow_samples * 20 >= self.samples
}
}
impl AdaptiveValidatePermit {
fn new(limiter: Arc<AdaptiveValidateLimiter>) -> Self {
Self {
limiter,
started_at: Instant::now(),
completed: false,
}
}
fn complete(mut self, success: bool) {
self.limiter
.record_sample(self.started_at.elapsed(), success);
self.completed = true;
}
}
impl AdaptiveValidateGrant {
fn new(limiter: Arc<AdaptiveValidateLimiter>) -> Self {
Self {
limiter,
active: true,
}
}
fn into_permit(mut self) -> AdaptiveValidatePermit {
self.active = false;
AdaptiveValidatePermit::new(self.limiter.clone())
}
fn disarm(&mut self) {
self.active = false;
}
}
impl Drop for AdaptiveValidateGrant {
fn drop(&mut self) {
if self.active {
self.limiter.release_slot();
}
}
}
impl Drop for AdaptiveValidatePermit {
fn drop(&mut self) {
if !self.completed {
self.limiter.record_sample(self.started_at.elapsed(), false);
}
self.limiter.release_slot();
}
}
/// Webhook configuration for external integrations.
#[derive(Debug, Clone)]
@@ -11,6 +253,7 @@ pub struct WebhookConfig {
pub web_instance_id: Option<String>,
pub web_instance_api_base_url: Option<String>,
validate_limiter: Arc<AdaptiveValidateLimiter>,
client: reqwest::Client,
}
@@ -28,7 +271,11 @@ impl WebhookConfig {
internal_auth_token,
web_instance_id,
web_instance_api_base_url,
client: reqwest::Client::new(),
validate_limiter: AdaptiveValidateLimiter::new(),
client: reqwest::Client::builder()
.timeout(WEBHOOK_HTTP_TIMEOUT)
.build()
.expect("webhook HTTP client should be valid"),
}
}
@@ -58,6 +305,8 @@ pub struct ValidateTokenRequest {
pub web_instance_id: Option<String>,
pub web_instance_api_base_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub persisted_config_revision: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub applied_config_revision: Option<String>,
}
@@ -69,7 +318,6 @@ pub struct ValidateTokenResponse {
#[serde(default)]
pub binding_version: u64,
#[serde(default)]
pub managed_network_configs: Option<Vec<ManagedNetworkConfig>>,
pub config_revision: String,
}
@@ -125,21 +373,40 @@ impl WebhookConfig {
pub async fn validate_token(
&self,
req: &ValidateTokenRequest,
) -> anyhow::Result<ValidateTokenResponse> {
self.validate_token_with_http_timeout(req, WEBHOOK_HTTP_TIMEOUT)
.await
}
async fn validate_token_with_http_timeout(
&self,
req: &ValidateTokenRequest,
http_timeout: Duration,
) -> anyhow::Result<ValidateTokenResponse> {
let url = self.webhook_endpoint("validate-token")?;
let resp = self
.client
.post(&url)
.header("X-Internal-Auth", self.webhook_auth_secret())
.json(req)
.send()
.await?;
let permit = self.validate_limiter.acquire().await;
let ret = match tokio::time::timeout(http_timeout, async {
let resp = self
.client
.post(&url)
.header("X-Internal-Auth", self.webhook_auth_secret())
.json(req)
.send()
.await?;
if !resp.status().is_success() {
anyhow::bail!("webhook validate-token returned status {}", resp.status());
}
if !resp.status().is_success() {
anyhow::bail!("webhook validate-token returned status {}", resp.status());
}
Ok(resp.json().await?)
Ok(resp.json().await?)
})
.await
{
Ok(ret) => ret,
Err(_) => Err(anyhow::anyhow!("webhook validate-token timed out")),
};
permit.complete(ret.is_ok());
ret
}
/// Notify the webhook receiver that a node has connected.
@@ -191,13 +458,319 @@ pub type SharedWebhookConfig = Arc<WebhookConfig>;
#[cfg(test)]
mod tests {
use super::*;
use axum::{Json, Router, routing::post};
use serde_json::json;
#[test]
fn validate_token_response_allows_missing_managed_configs() {
fn adaptive_validate_limiter_increases_under_queue_pressure() {
let now = Instant::now();
let mut state = AdaptiveValidateLimiterState::new(now);
state.had_queue = true;
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
state.record_sample(now, Duration::from_millis(50), true);
}
assert_eq!(
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
LimitAdjustment::Increased
);
assert_eq!(state.limit, VALIDATE_TOKEN_INITIAL_CONCURRENCY + 1);
}
#[test]
fn adaptive_validate_limiter_does_not_increase_without_queue_pressure() {
let now = Instant::now();
let mut state = AdaptiveValidateLimiterState::new(now);
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
state.record_sample(now, Duration::from_millis(50), true);
}
assert_eq!(
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
LimitAdjustment::Unchanged
);
assert_eq!(state.limit, VALIDATE_TOKEN_INITIAL_CONCURRENCY);
}
#[test]
fn adaptive_validate_limiter_reduces_on_failure() {
let now = Instant::now();
let mut state = AdaptiveValidateLimiterState::new(now);
state.record_sample(now, Duration::from_millis(50), false);
assert_eq!(
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
LimitAdjustment::Decreased
);
assert_eq!(
state.limit,
(VALIDATE_TOKEN_INITIAL_CONCURRENCY / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY)
);
}
#[test]
fn adaptive_validate_limiter_reduces_on_slow_latency() {
let now = Instant::now();
let mut state = AdaptiveValidateLimiterState::new(now);
state.record_sample(
now,
VALIDATE_TOKEN_SLOW_THRESHOLD + Duration::from_millis(1),
true,
);
assert_eq!(
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
LimitAdjustment::Decreased
);
assert_eq!(
state.limit,
(VALIDATE_TOKEN_INITIAL_CONCURRENCY / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY)
);
}
#[tokio::test]
async fn adaptive_validate_limiter_waiter_acquires_after_release() {
let limiter = AdaptiveValidateLimiter::new();
let mut permits = Vec::new();
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
permits.push(limiter.acquire().await);
}
let waiter_limiter = limiter.clone();
let waiter = tokio::spawn(async move {
let permit = waiter_limiter.acquire().await;
permit.complete(true);
});
tokio::time::sleep(Duration::from_millis(10)).await;
assert!(!waiter.is_finished());
permits.pop().unwrap().complete(true);
tokio::time::timeout(Duration::from_secs(1), waiter)
.await
.unwrap()
.unwrap();
for permit in permits {
permit.complete(true);
}
}
#[tokio::test]
async fn adaptive_validate_limiter_releases_when_permit_is_dropped() {
let limiter = AdaptiveValidateLimiter::new();
let mut permits = Vec::new();
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
permits.push(limiter.acquire().await);
}
let waiter_limiter = limiter.clone();
let waiter = tokio::spawn(async move {
let permit = waiter_limiter.acquire().await;
permit.complete(true);
});
tokio::time::sleep(Duration::from_millis(10)).await;
assert!(!waiter.is_finished());
drop(permits.pop().unwrap());
tokio::time::timeout(Duration::from_secs(1), waiter)
.await
.unwrap()
.unwrap();
for permit in permits {
permit.complete(true);
}
}
#[tokio::test]
async fn adaptive_validate_limiter_skips_canceled_waiters() {
let limiter = AdaptiveValidateLimiter::new();
let mut permits = Vec::new();
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
permits.push(limiter.acquire().await);
}
let waiter_limiter = limiter.clone();
let waiter = tokio::spawn(async move {
let permit = waiter_limiter.acquire().await;
permit.complete(true);
});
tokio::time::sleep(Duration::from_millis(10)).await;
assert!(!waiter.is_finished());
waiter.abort();
assert!(waiter.await.unwrap_err().is_cancelled());
permits.pop().unwrap().complete(true);
tokio::time::sleep(Duration::from_millis(10)).await;
let state = limiter.lock_state();
assert_eq!(state.samples, 1);
assert_eq!(state.failures, 0);
drop(state);
for permit in permits {
permit.complete(true);
}
}
#[test]
fn adaptive_validate_limiter_releases_dropped_grant_without_failure_sample() {
let limiter = AdaptiveValidateLimiter::new();
{
let mut state = limiter.lock_state();
state.in_flight = 1;
}
drop(AdaptiveValidateGrant::new(limiter.clone()));
let state = limiter.lock_state();
assert_eq!(state.in_flight, 0);
assert_eq!(state.samples, 0);
assert_eq!(state.failures, 0);
}
#[tokio::test]
async fn adaptive_validate_limiter_wakes_multiple_waiters_in_order() {
let limiter = AdaptiveValidateLimiter::new();
let mut permits = Vec::new();
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
permits.push(limiter.acquire().await);
}
let (first_acquired_tx, first_acquired_rx) = oneshot::channel();
let (first_release_tx, first_release_rx) = oneshot::channel();
let first = {
let limiter = limiter.clone();
tokio::spawn(async move {
let permit = limiter.acquire().await;
first_acquired_tx.send(()).unwrap();
first_release_rx.await.unwrap();
permit.complete(true);
})
};
let (second_acquired_tx, mut second_acquired_rx) = oneshot::channel();
let second = {
let limiter = limiter.clone();
tokio::spawn(async move {
let permit = limiter.acquire().await;
second_acquired_tx.send(()).unwrap();
permit.complete(true);
})
};
tokio::time::sleep(Duration::from_millis(10)).await;
assert!(!first.is_finished());
assert!(!second.is_finished());
permits.pop().unwrap().complete(true);
tokio::time::timeout(Duration::from_secs(1), first_acquired_rx)
.await
.unwrap()
.unwrap();
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut second_acquired_rx)
.await
.is_err()
);
first_release_tx.send(()).unwrap();
tokio::time::timeout(Duration::from_secs(1), first)
.await
.unwrap()
.unwrap();
tokio::time::timeout(Duration::from_secs(1), &mut second_acquired_rx)
.await
.unwrap()
.unwrap();
tokio::time::timeout(Duration::from_secs(1), second)
.await
.unwrap()
.unwrap();
for permit in permits {
permit.complete(true);
}
}
#[tokio::test]
async fn validate_token_http_timeout_starts_after_limiter_permit() {
let app = Router::new().route(
"/validate-token",
post(|| async {
Json(json!({
"valid": true,
"config_revision": "rev-1"
}))
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
let webhook = WebhookConfig::new(Some(format!("http://{addr}")), None, None, None, None);
let mut permits = Vec::new();
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
permits.push(webhook.validate_limiter.acquire().await);
}
let validate_webhook = webhook.clone();
let validate = tokio::spawn(async move {
let req = ValidateTokenRequest {
token: "token".to_string(),
machine_id: uuid::Uuid::new_v4().to_string(),
public_ip: None,
hostname: String::new(),
version: String::new(),
os_type: None,
os_version: None,
os_distribution: None,
web_instance_id: None,
web_instance_api_base_url: None,
persisted_config_revision: None,
applied_config_revision: None,
};
validate_webhook
.validate_token_with_http_timeout(&req, Duration::from_millis(20))
.await
});
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(!validate.is_finished());
permits.pop().unwrap().complete(true);
let resp = tokio::time::timeout(Duration::from_secs(1), validate)
.await
.unwrap()
.unwrap()
.unwrap();
assert!(resp.valid);
for permit in permits {
permit.complete(true);
}
server.abort();
}
#[test]
fn validate_token_response_deserializes_config_revision() {
let resp: ValidateTokenResponse =
serde_json::from_str(r#"{"valid":true,"config_revision":"rev-1"}"#).unwrap();
assert!(resp.valid);
assert_eq!(resp.config_revision, "rev-1");
assert!(resp.managed_network_configs.is_none());
}
#[test]
fn validate_token_response_allows_missing_config_revision() {
let resp: ValidateTokenResponse = serde_json::from_str(r#"{"valid":true}"#).unwrap();
assert!(resp.valid);
assert!(resp.config_revision.is_empty());
}
}
+10 -6
View File
@@ -483,18 +483,22 @@ impl InstanceConfigPatcher {
}
let global_ctx = weak_upgrade(&self.global_ctx)?;
for proxy_network_patch in proxy_networks {
let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else {
tracing::warn!("Proxy network cidr is None, skipping.");
continue;
};
let mapped_cidr: Option<cidr::Ipv4Cidr> =
proxy_network_patch.mapped_cidr.map(|s| s.into());
match ConfigPatchAction::try_from(proxy_network_patch.action) {
Ok(ConfigPatchAction::Add) => {
let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else {
tracing::warn!("Proxy network cidr is None, skipping add.");
continue;
};
let mapped_cidr: Option<cidr::Ipv4Cidr> =
proxy_network_patch.mapped_cidr.map(|s| s.into());
tracing::info!("Proxy network added: {}", cidr);
global_ctx.config.add_proxy_cidr(cidr, mapped_cidr)?;
}
Ok(ConfigPatchAction::Remove) => {
let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else {
tracing::warn!("Proxy network cidr is None, skipping remove.");
continue;
};
tracing::info!("Proxy network removed: {}", cidr);
global_ctx.config.remove_proxy_cidr(cidr);
}
+40 -1
View File
@@ -1116,6 +1116,7 @@ impl NetworkConfig {
.get_credential_file()
.map(|path| path.to_string_lossy().into_owned());
let flags = config.get_flags();
let default_flags = default_config.get_flags();
result.latency_first = Some(flags.latency_first);
result.dev_name = Some(flags.dev_name.clone());
result.use_smoltcp = Some(flags.use_smoltcp);
@@ -1144,6 +1145,11 @@ impl NetworkConfig {
result.disable_sym_hole_punching = Some(flags.disable_sym_hole_punching);
result.enable_magic_dns = Some(flags.accept_dns);
result.mtu = Some(flags.mtu as i32);
result.data_compress_algo = (flags.data_compress_algo != default_flags.data_compress_algo)
.then_some(flags.data_compress_algo);
result.encryption_algorithm = (flags.encryption_algorithm
!= default_flags.encryption_algorithm)
.then_some(flags.encryption_algorithm.clone());
result.instance_recv_bps_limit =
(flags.instance_recv_bps_limit != u64::MAX).then_some(flags.instance_recv_bps_limit);
result.enable_private_mode = Some(flags.private_mode);
@@ -1173,7 +1179,7 @@ impl NetworkConfig {
mod tests {
use crate::{
common::config::{ConfigLoader, process_secure_mode_cfg},
proto::common::SecureModeConfig,
proto::common::{CompressionAlgoPb, SecureModeConfig},
};
use base64::prelude::{BASE64_STANDARD, Engine as _};
use rand::Rng;
@@ -1534,4 +1540,37 @@ mod tests {
Ok(())
}
#[test]
fn test_network_config_conversion_preserves_runtime_algorithm_flags()
-> Result<(), anyhow::Error> {
let config = gen_default_config();
let mut flags = config.get_flags();
flags.data_compress_algo = CompressionAlgoPb::Zstd.into();
flags.encryption_algorithm = "managed-test-algo".to_string();
config.set_flags(flags.clone());
let network_config = super::NetworkConfig::new_from_config(&config)?;
assert_eq!(
network_config.data_compress_algo,
Some(CompressionAlgoPb::Zstd as i32)
);
assert_eq!(
network_config.encryption_algorithm.as_deref(),
Some("managed-test-algo")
);
let generated_config = network_config.gen_config()?;
assert_eq!(
generated_config.get_flags().data_compress_algo,
flags.data_compress_algo
);
assert_eq!(
generated_config.get_flags().encryption_algorithm,
flags.encryption_algorithm
);
Ok(())
}
}
+19
View File
@@ -3653,6 +3653,25 @@ pub async fn config_patch_test() {
true
},
);
let patch = InstanceConfigPatch {
proxy_networks: vec![ProxyNetworkPatch {
action: ConfigPatchAction::Clear as i32,
..Default::default()
}],
..Default::default()
};
insts[1]
.get_config_patcher()
.apply_patch(patch)
.await
.unwrap();
assert!(
insts[1]
.get_global_ctx()
.config
.get_proxy_cidrs()
.is_empty()
);
// 测试1.1:修改公网 IPv6 provider 相关配置
let public_prefix = "2001:db8:100::/64";