mirror of
https://github.com/EasyTier/EasyTier.git
synced 2026-09-03 17:45:44 +00:00
perf(easytier-web): improve easytier-web webhook performance (#2383)
* feat(web): reconcile managed config revisions * feat(web): cache managed runtime configs per session * test: cover managed web config delivery
This commit is contained in:
@@ -40,6 +40,9 @@ cli:
|
||||
geoip_db:
|
||||
en: "The path to the GeoIP2 database file, used to lookup the location of the client, default is the embedded file (only country information) , recommend https://github.com/P3TERX/GeoLite.mmdb"
|
||||
zh-CN: "GeoIP2 数据库文件路径,用于查找客户端的位置,默认为嵌入文件(仅国家信息),推荐 https://github.com/P3TERX/GeoLite.mmdb"
|
||||
heartbeat_min_response_ms:
|
||||
en: "Minimum response time for config-server heartbeat RPCs in milliseconds, default is 0"
|
||||
zh-CN: "配置服务心跳 RPC 的最短响应时间,单位毫秒,默认为 0"
|
||||
disable_registration:
|
||||
en: "Disable user registration"
|
||||
zh-CN: "禁用用户注册"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,3 +1,5 @@
|
||||
mod managed_config;
|
||||
mod runtime_reconcile;
|
||||
pub mod session;
|
||||
pub mod storage;
|
||||
|
||||
@@ -5,6 +7,7 @@ use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicU32, Ordering},
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
use dashmap::DashMap;
|
||||
use easytier::{
|
||||
@@ -30,6 +33,10 @@ use crate::db::{Db, UserIdInDb, entity::user_running_network_configs};
|
||||
#[include = "geoip2-cn.mmdb"]
|
||||
struct GeoipDb;
|
||||
|
||||
pub fn is_managed_config_revision_conflict(error: &anyhow::Error) -> bool {
|
||||
managed_config::is_revision_conflict(error)
|
||||
}
|
||||
|
||||
fn load_geoip_db(geoip_db: Option<String>) -> Option<maxminddb::Reader<Vec<u8>>> {
|
||||
if let Some(path) = geoip_db {
|
||||
match maxminddb::Reader::open_readfile(&path) {
|
||||
@@ -63,12 +70,14 @@ pub struct ClientManager {
|
||||
webhook_config: SharedWebhookConfig,
|
||||
|
||||
geoip_db: Arc<Option<maxminddb::Reader<Vec<u8>>>>,
|
||||
heartbeat_min_response_delay: Duration,
|
||||
}
|
||||
|
||||
impl ClientManager {
|
||||
pub fn new(
|
||||
db: Db,
|
||||
geoip_db: Option<String>,
|
||||
heartbeat_min_response_delay: Duration,
|
||||
feature_flags: Arc<FeatureFlags>,
|
||||
webhook_config: SharedWebhookConfig,
|
||||
) -> Self {
|
||||
@@ -92,6 +101,7 @@ impl ClientManager {
|
||||
webhook_config,
|
||||
|
||||
geoip_db: Arc::new(load_geoip_db(geoip_db)),
|
||||
heartbeat_min_response_delay,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,6 +115,7 @@ impl ClientManager {
|
||||
let storage = self.storage.weak_ref();
|
||||
let listeners_cnt = self.listeners_cnt.clone();
|
||||
let geoip_db = self.geoip_db.clone();
|
||||
let heartbeat_min_response_delay = self.heartbeat_min_response_delay;
|
||||
let feature_flags = self.feature_flags.clone();
|
||||
let webhook_config = self.webhook_config.clone();
|
||||
self.tasks.spawn(async move {
|
||||
@@ -129,6 +140,7 @@ impl ClientManager {
|
||||
storage.clone(),
|
||||
client_url.clone(),
|
||||
location,
|
||||
heartbeat_min_response_delay,
|
||||
feature_flags.clone(),
|
||||
webhook_config.clone(),
|
||||
);
|
||||
@@ -149,6 +161,10 @@ impl ClientManager {
|
||||
self.storage.list_clients()
|
||||
}
|
||||
|
||||
pub async fn list_all_sessions(&self) -> Vec<StorageToken> {
|
||||
self.storage.list_all_clients()
|
||||
}
|
||||
|
||||
pub fn get_session_by_machine_id(
|
||||
&self,
|
||||
user_id: UserIdInDb,
|
||||
@@ -169,7 +185,7 @@ impl ClientManager {
|
||||
) -> bool {
|
||||
let Some(client_url) = self
|
||||
.storage
|
||||
.get_client_url_by_machine_id(user_id, machine_id)
|
||||
.get_client_url_by_machine_id_with_auth(user_id, machine_id, false)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
@@ -189,14 +205,30 @@ impl ClientManager {
|
||||
user_id: UserIdInDb,
|
||||
machine_id: uuid::Uuid,
|
||||
desired_configs: Vec<ManagedNetworkConfig>,
|
||||
config_revision: Option<String>,
|
||||
expected_config_revision: Option<String>,
|
||||
) -> anyhow::Result<()> {
|
||||
session::SessionRpcService::reconcile_web_source_configs(
|
||||
let expected_config_revision = match expected_config_revision.as_deref().map(str::trim) {
|
||||
None => managed_config::ExpectedConfigRevision::Any,
|
||||
Some("") => managed_config::ExpectedConfigRevision::Exact(None),
|
||||
Some(revision) => managed_config::ExpectedConfigRevision::Exact(Some(revision)),
|
||||
};
|
||||
managed_config::reconcile_web_source_configs(
|
||||
&self.storage,
|
||||
user_id,
|
||||
machine_id,
|
||||
desired_configs,
|
||||
config_revision.as_deref(),
|
||||
expected_config_revision,
|
||||
)
|
||||
.await?;
|
||||
if let Some(config_revision) = config_revision
|
||||
&& let Some(session) = self.get_session_by_machine_id(user_id, &machine_id)
|
||||
{
|
||||
session
|
||||
.notify_config_revision_changed(user_id, machine_id, config_revision)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -331,19 +363,449 @@ impl
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use axum::{Json, Router, extract::State, routing::post};
|
||||
use easytier::{
|
||||
common::MachineIdOptions,
|
||||
instance_manager::NetworkInstanceManager,
|
||||
proto::{
|
||||
api::manage::{NetworkConfig, NetworkingMethod, PortForwardConfig},
|
||||
common::CompressionAlgoPb,
|
||||
},
|
||||
rpc_service::remote_client::Storage as RemoteStorage,
|
||||
tunnel::{
|
||||
common::tests::wait_for_condition,
|
||||
udp::{UdpTunnelConnector, UdpTunnelListener},
|
||||
},
|
||||
web_client::WebClient,
|
||||
web_client::{WebClient, run_web_client},
|
||||
};
|
||||
use serde_json::json;
|
||||
use sqlx::Executor;
|
||||
use tokio::net::UdpSocket;
|
||||
|
||||
use crate::{FeatureFlags, client_manager::ClientManager, db::Db};
|
||||
use crate::{
|
||||
FeatureFlags, client_manager::ClientManager, db::Db, webhook::ManagedNetworkConfig,
|
||||
};
|
||||
|
||||
const MANAGED_CONFIG_TOKEN: &str = "managed-config-token";
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct TestWebhookState {
|
||||
validate_responses: Arc<tokio::sync::Mutex<VecDeque<bool>>>,
|
||||
validate_count: Arc<AtomicUsize>,
|
||||
block_second_validate: Arc<AtomicBool>,
|
||||
allow_second_validate: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl TestWebhookState {
|
||||
fn new(validate_responses: impl IntoIterator<Item = bool>) -> Self {
|
||||
Self {
|
||||
validate_responses: Arc::new(tokio::sync::Mutex::new(
|
||||
validate_responses.into_iter().collect(),
|
||||
)),
|
||||
validate_count: Arc::new(AtomicUsize::new(0)),
|
||||
block_second_validate: Arc::new(AtomicBool::new(false)),
|
||||
allow_second_validate: Arc::new(AtomicBool::new(true)),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_blocked_second_validate(
|
||||
validate_responses: impl IntoIterator<Item = bool>,
|
||||
) -> Self {
|
||||
let state = Self::new(validate_responses);
|
||||
state.block_second_validate.store(true, Ordering::Release);
|
||||
state.allow_second_validate.store(false, Ordering::Release);
|
||||
state
|
||||
}
|
||||
|
||||
fn allow_second_validate(&self) {
|
||||
self.allow_second_validate.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
fn validate_count(&self) -> usize {
|
||||
self.validate_count.load(Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
async fn validate_token_handler(
|
||||
State(state): State<TestWebhookState>,
|
||||
) -> Json<serde_json::Value> {
|
||||
let count = state.validate_count.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
if count == 2 && state.block_second_validate.load(Ordering::Acquire) {
|
||||
while !state.allow_second_validate.load(Ordering::Acquire) {
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
}
|
||||
let valid = state
|
||||
.validate_responses
|
||||
.lock()
|
||||
.await
|
||||
.pop_front()
|
||||
.unwrap_or(true);
|
||||
if !valid {
|
||||
return Json(json!({ "valid": false }));
|
||||
}
|
||||
|
||||
Json(json!({
|
||||
"valid": true,
|
||||
"binding_version": count,
|
||||
"config_revision": format!("validated-rev-{count}")
|
||||
}))
|
||||
}
|
||||
|
||||
async fn webhook_ack_handler() -> Json<serde_json::Value> {
|
||||
Json(json!({}))
|
||||
}
|
||||
|
||||
async fn test_webhook_config() -> (
|
||||
crate::webhook::SharedWebhookConfig,
|
||||
tokio::task::JoinHandle<()>,
|
||||
TestWebhookState,
|
||||
) {
|
||||
let state = TestWebhookState::new([true]);
|
||||
test_webhook_config_with_state(state).await
|
||||
}
|
||||
|
||||
async fn test_webhook_config_with_state(
|
||||
state: TestWebhookState,
|
||||
) -> (
|
||||
crate::webhook::SharedWebhookConfig,
|
||||
tokio::task::JoinHandle<()>,
|
||||
TestWebhookState,
|
||||
) {
|
||||
let app = Router::new()
|
||||
.route("/validate-token", post(validate_token_handler))
|
||||
.route("/webhook/node-connected", post(webhook_ack_handler))
|
||||
.route("/webhook/node-disconnected", post(webhook_ack_handler))
|
||||
.with_state(state.clone());
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
(
|
||||
Arc::new(crate::webhook::WebhookConfig::new(
|
||||
Some(format!("http://{addr}")),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)),
|
||||
server,
|
||||
state,
|
||||
)
|
||||
}
|
||||
|
||||
async fn add_random_udp_listener(mgr: &mut ClientManager) -> std::net::SocketAddr {
|
||||
let socket = Arc::new(UdpSocket::bind("127.0.0.1:0").await.unwrap());
|
||||
let addr = socket.local_addr().unwrap();
|
||||
let listener =
|
||||
UdpTunnelListener::new_with_socket(format!("udp://{addr}").parse().unwrap(), socket);
|
||||
mgr.add_listener(listener).await.unwrap();
|
||||
addr
|
||||
}
|
||||
|
||||
async fn wait_for_validated_user(mgr: &ClientManager, machine_id: uuid::Uuid) -> i32 {
|
||||
tokio::time::timeout(Duration::from_secs(12), async {
|
||||
loop {
|
||||
if let Some(token) = mgr.list_sessions().await.into_iter().find(|token| {
|
||||
token.token == MANAGED_CONFIG_TOKEN && token.machine_id == machine_id
|
||||
}) {
|
||||
break token.user_id;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn wait_for_validate_count(state: &TestWebhookState, target: usize) {
|
||||
tokio::time::timeout(Duration::from_secs(12), async {
|
||||
loop {
|
||||
if state.validate_count() >= target {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn wait_for_session_urls(mgr: &ClientManager) -> Vec<url::Url> {
|
||||
tokio::time::timeout(Duration::from_secs(12), async {
|
||||
loop {
|
||||
let urls = mgr
|
||||
.client_sessions
|
||||
.iter()
|
||||
.map(|entry| entry.key().clone())
|
||||
.collect::<Vec<_>>();
|
||||
if !urls.is_empty() {
|
||||
break urls;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn managed_config(
|
||||
instance_id: uuid::Uuid,
|
||||
network_config: serde_json::Value,
|
||||
) -> ManagedNetworkConfig {
|
||||
ManagedNetworkConfig {
|
||||
instance_id: instance_id.to_string(),
|
||||
network_config,
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_runtime_config(
|
||||
manager: &NetworkInstanceManager,
|
||||
inst_id: uuid::Uuid,
|
||||
predicate: impl Fn(&NetworkConfig) -> bool,
|
||||
) -> NetworkConfig {
|
||||
tokio::time::timeout(Duration::from_secs(12), async {
|
||||
loop {
|
||||
if let Some(config) = manager
|
||||
.get_instance_config(&inst_id)
|
||||
.and_then(|config| NetworkConfig::new_from_config(&config).ok())
|
||||
.filter(|config| predicate(config))
|
||||
{
|
||||
break config;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn start_web_client_for_test(
|
||||
config_server_addr: std::net::SocketAddr,
|
||||
machine_id: uuid::Uuid,
|
||||
manager: Arc<NetworkInstanceManager>,
|
||||
) -> WebClient {
|
||||
run_web_client(
|
||||
&format!("udp://{config_server_addr}/{MANAGED_CONFIG_TOKEN}"),
|
||||
MachineIdOptions {
|
||||
explicit_machine_id: Some(machine_id.to_string()),
|
||||
state_dir: None,
|
||||
},
|
||||
Some("managed-config-core".to_string()),
|
||||
false,
|
||||
manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn clear_managed_config_db(
|
||||
mgr: &ClientManager,
|
||||
user_id: i32,
|
||||
machine_id: uuid::Uuid,
|
||||
instance_id: uuid::Uuid,
|
||||
) {
|
||||
mgr.db()
|
||||
.delete_web_network_configs((user_id, machine_id), &[instance_id])
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("DELETE FROM managed_config_revisions WHERE user_id = ? AND device_id = ?")
|
||||
.bind(user_id)
|
||||
.bind(machine_id.to_string())
|
||||
.execute(&mgr.db().inner())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn assert_updated_runtime_config(updated: &NetworkConfig, instance_id: uuid::Uuid) {
|
||||
assert_eq!(
|
||||
updated.instance_id.as_deref(),
|
||||
Some(instance_id.to_string().as_str())
|
||||
);
|
||||
assert_eq!(updated.dhcp, Some(false));
|
||||
assert_eq!(updated.virtual_ipv4.as_deref(), Some("10.88.0.7"));
|
||||
assert_eq!(updated.network_length, Some(24));
|
||||
assert_eq!(updated.hostname.as_deref(), Some("managed-updated-host"));
|
||||
assert_eq!(updated.network_name.as_deref(), Some("managed-updated"));
|
||||
assert_eq!(updated.network_secret.as_deref(), Some("secret-updated"));
|
||||
assert_eq!(
|
||||
updated.networking_method,
|
||||
Some(NetworkingMethod::Manual as i32)
|
||||
);
|
||||
assert_eq!(updated.peer_urls, vec!["tcp://127.0.0.1:11010".to_string()]);
|
||||
assert_eq!(
|
||||
updated.proxy_cidrs,
|
||||
vec![
|
||||
"10.44.0.0/24".to_string(),
|
||||
"10.45.0.0/24->10.46.0.0/24".to_string()
|
||||
]
|
||||
);
|
||||
assert_eq!(updated.no_tun, Some(true));
|
||||
assert_eq!(updated.disable_ipv6, Some(true));
|
||||
assert_eq!(updated.enable_kcp_proxy, Some(true));
|
||||
assert_eq!(updated.disable_kcp_input, Some(true));
|
||||
assert_eq!(updated.enable_quic_proxy, Some(true));
|
||||
assert_eq!(updated.disable_quic_input, Some(true));
|
||||
assert_eq!(updated.disable_p2p, Some(true));
|
||||
assert_eq!(updated.p2p_only, Some(true));
|
||||
assert_eq!(updated.lazy_p2p, Some(true));
|
||||
assert_eq!(updated.relay_all_peer_rpc, Some(true));
|
||||
assert_eq!(updated.need_p2p, Some(true));
|
||||
assert_eq!(updated.multi_thread, Some(false));
|
||||
assert_eq!(updated.proxy_forward_by_system, Some(true));
|
||||
assert_eq!(updated.disable_encryption, Some(true));
|
||||
assert_eq!(updated.enable_relay_network_whitelist, Some(true));
|
||||
assert_eq!(
|
||||
updated.relay_network_whitelist,
|
||||
vec!["10.44.0.0/24".to_string(), "10.45.0.0/24".to_string()]
|
||||
);
|
||||
assert_eq!(updated.enable_manual_routes, Some(true));
|
||||
assert_eq!(
|
||||
updated.routes,
|
||||
vec!["10.60.0.0/16".to_string(), "10.61.0.0/16".to_string()]
|
||||
);
|
||||
assert_eq!(updated.port_forwards[0].bind_ip, "127.0.0.1");
|
||||
assert_eq!(updated.port_forwards[0].bind_port, 0);
|
||||
assert_eq!(updated.port_forwards[0].dst_ip, "10.88.0.8");
|
||||
assert_eq!(updated.port_forwards[0].dst_port, 80);
|
||||
assert_eq!(updated.port_forwards[0].proto, "tcp");
|
||||
assert_eq!(updated.disable_udp_hole_punching, Some(true));
|
||||
assert_eq!(updated.disable_tcp_hole_punching, Some(true));
|
||||
assert_eq!(updated.disable_sym_hole_punching, Some(true));
|
||||
assert_eq!(updated.disable_upnp, Some(true));
|
||||
assert_eq!(updated.disable_relay_data, Some(true));
|
||||
assert_eq!(updated.enable_magic_dns, Some(true));
|
||||
assert_eq!(updated.enable_private_mode, Some(true));
|
||||
assert_eq!(updated.mtu, Some(1360));
|
||||
assert_eq!(
|
||||
updated.data_compress_algo,
|
||||
Some(CompressionAlgoPb::Zstd as i32)
|
||||
);
|
||||
assert_eq!(updated.encryption_algorithm.as_deref(), Some("xor"));
|
||||
assert_eq!(updated.instance_recv_bps_limit, Some(123456));
|
||||
assert_eq!(updated.enable_udp_broadcast_relay, Some(true));
|
||||
assert_eq!(updated.socket_mark, Some(0));
|
||||
}
|
||||
|
||||
fn initial_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value {
|
||||
json!({
|
||||
"instance_id": inst_id.to_string(),
|
||||
"dhcp": true,
|
||||
"network_name": "managed-initial",
|
||||
"network_secret": "secret-initial",
|
||||
"networking_method": "Standalone",
|
||||
"no_tun": true,
|
||||
"disable_ipv6": true,
|
||||
"enable_kcp_proxy": false,
|
||||
"disable_kcp_input": false,
|
||||
"relay_all_peer_rpc": false,
|
||||
"multi_thread": false,
|
||||
"disable_relay_data": false,
|
||||
"mtu": 1380
|
||||
})
|
||||
}
|
||||
|
||||
fn updated_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value {
|
||||
serde_json::to_value(NetworkConfig {
|
||||
instance_id: Some(inst_id.to_string()),
|
||||
dhcp: Some(false),
|
||||
virtual_ipv4: Some("10.88.0.7".to_string()),
|
||||
network_length: Some(24),
|
||||
hostname: Some("managed-updated-host".to_string()),
|
||||
network_name: Some("managed-updated".to_string()),
|
||||
network_secret: Some("secret-updated".to_string()),
|
||||
networking_method: Some(NetworkingMethod::Manual as i32),
|
||||
peer_urls: vec!["tcp://127.0.0.1:11010".to_string()],
|
||||
proxy_cidrs: vec![
|
||||
"10.44.0.0/24".to_string(),
|
||||
"10.45.0.0/24->10.46.0.0/24".to_string(),
|
||||
],
|
||||
no_tun: Some(true),
|
||||
disable_ipv6: Some(true),
|
||||
enable_kcp_proxy: Some(true),
|
||||
disable_kcp_input: Some(true),
|
||||
enable_quic_proxy: Some(true),
|
||||
disable_quic_input: Some(true),
|
||||
disable_p2p: Some(true),
|
||||
p2p_only: Some(true),
|
||||
lazy_p2p: Some(true),
|
||||
relay_all_peer_rpc: Some(true),
|
||||
need_p2p: Some(true),
|
||||
multi_thread: Some(false),
|
||||
proxy_forward_by_system: Some(true),
|
||||
disable_encryption: Some(true),
|
||||
enable_relay_network_whitelist: Some(true),
|
||||
relay_network_whitelist: vec!["10.44.0.0/24".to_string(), "10.45.0.0/24".to_string()],
|
||||
enable_manual_routes: Some(true),
|
||||
routes: vec!["10.60.0.0/16".to_string(), "10.61.0.0/16".to_string()],
|
||||
port_forwards: vec![PortForwardConfig {
|
||||
bind_ip: "127.0.0.1".to_string(),
|
||||
bind_port: 0,
|
||||
dst_ip: "10.88.0.8".to_string(),
|
||||
dst_port: 80,
|
||||
proto: "tcp".to_string(),
|
||||
}],
|
||||
disable_udp_hole_punching: Some(true),
|
||||
disable_tcp_hole_punching: Some(true),
|
||||
disable_sym_hole_punching: Some(true),
|
||||
disable_upnp: Some(true),
|
||||
disable_relay_data: Some(true),
|
||||
enable_magic_dns: Some(true),
|
||||
enable_private_mode: Some(true),
|
||||
mtu: Some(1360),
|
||||
data_compress_algo: Some(CompressionAlgoPb::Zstd as i32),
|
||||
encryption_algorithm: Some("xor".to_string()),
|
||||
instance_recv_bps_limit: Some(123456),
|
||||
enable_udp_broadcast_relay: Some(true),
|
||||
socket_mark: Some(0),
|
||||
..Default::default()
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn redelivered_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value {
|
||||
serde_json::to_value(NetworkConfig {
|
||||
instance_id: Some(inst_id.to_string()),
|
||||
dhcp: Some(false),
|
||||
virtual_ipv4: Some("10.88.0.7".to_string()),
|
||||
network_length: Some(24),
|
||||
hostname: Some("managed-redelivered-host".to_string()),
|
||||
network_name: Some("managed-redelivered".to_string()),
|
||||
network_secret: Some("secret-updated".to_string()),
|
||||
networking_method: Some(NetworkingMethod::Manual as i32),
|
||||
peer_urls: vec!["tcp://127.0.0.1:11010".to_string()],
|
||||
proxy_cidrs: vec![
|
||||
"10.44.0.0/24".to_string(),
|
||||
"10.45.0.0/24->10.46.0.0/24".to_string(),
|
||||
],
|
||||
no_tun: Some(true),
|
||||
disable_ipv6: Some(true),
|
||||
enable_kcp_proxy: Some(true),
|
||||
disable_kcp_input: Some(true),
|
||||
relay_all_peer_rpc: Some(true),
|
||||
need_p2p: Some(true),
|
||||
multi_thread: Some(false),
|
||||
enable_private_mode: Some(true),
|
||||
mtu: Some(1360),
|
||||
data_compress_algo: Some(CompressionAlgoPb::Zstd as i32),
|
||||
encryption_algorithm: Some("xor".to_string()),
|
||||
instance_recv_bps_limit: Some(654321),
|
||||
..Default::default()
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_client() {
|
||||
@@ -351,6 +813,7 @@ mod tests {
|
||||
let mut mgr = ClientManager::new(
|
||||
Db::memory_db().await,
|
||||
None,
|
||||
Duration::ZERO,
|
||||
Arc::new(FeatureFlags::default()),
|
||||
Arc::new(crate::webhook::WebhookConfig::new(
|
||||
None, None, None, None, None,
|
||||
@@ -410,4 +873,223 @@ mod tests {
|
||||
println!("{:?}", req);
|
||||
println!("{:?}", mgr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn managed_web_config_revision_updates_running_core_config() {
|
||||
let (webhook_config, webhook_server, _) = test_webhook_config().await;
|
||||
let mut mgr = ClientManager::new(
|
||||
Db::memory_db().await,
|
||||
None,
|
||||
Duration::ZERO,
|
||||
Arc::new(FeatureFlags::default()),
|
||||
webhook_config,
|
||||
);
|
||||
let config_server_addr = add_random_udp_listener(&mut mgr).await;
|
||||
|
||||
let machine_id = uuid::Uuid::new_v4();
|
||||
let instance_id = uuid::Uuid::new_v4();
|
||||
let core_manager = Arc::new(NetworkInstanceManager::new());
|
||||
let client =
|
||||
start_web_client_for_test(config_server_addr, machine_id, core_manager.clone()).await;
|
||||
|
||||
let user_id = wait_for_validated_user(&mgr, machine_id).await;
|
||||
mgr.reconcile_managed_network_configs(
|
||||
user_id,
|
||||
machine_id,
|
||||
vec![managed_config(
|
||||
instance_id,
|
||||
initial_managed_network_config(instance_id),
|
||||
)],
|
||||
Some("rev-initial".to_string()),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
wait_for_runtime_config(&core_manager, instance_id, |config| {
|
||||
config.network_name.as_deref() == Some("managed-initial")
|
||||
})
|
||||
.await;
|
||||
|
||||
// Online revision update: web-owned running config is fully overwritten
|
||||
// when non-hot-patch flags such as enable_kcp_proxy change.
|
||||
mgr.reconcile_managed_network_configs(
|
||||
user_id,
|
||||
machine_id,
|
||||
vec![managed_config(
|
||||
instance_id,
|
||||
updated_managed_network_config(instance_id),
|
||||
)],
|
||||
Some("rev-updated".to_string()),
|
||||
Some("rev-initial".to_string()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let updated = wait_for_runtime_config(&core_manager, instance_id, |config| {
|
||||
config.network_name.as_deref() == Some("managed-updated")
|
||||
&& config.enable_kcp_proxy == Some(true)
|
||||
&& config.port_forwards.len() == 1
|
||||
})
|
||||
.await;
|
||||
assert_updated_runtime_config(&updated, instance_id);
|
||||
|
||||
assert_eq!(
|
||||
core_manager.get_instance_network_config_source(&instance_id),
|
||||
Some(easytier::common::config::ConfigSource::Web)
|
||||
);
|
||||
assert_eq!(
|
||||
mgr.db()
|
||||
.get_managed_config_revision((user_id, machine_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some("rev-updated")
|
||||
);
|
||||
|
||||
// Web DB loss path: clear web-owned config and revision, then simulate
|
||||
// the webhook re-posting the authoritative desired config. The already
|
||||
// connected session should receive the distinguishable re-delivered
|
||||
// revision without restarting.
|
||||
clear_managed_config_db(&mgr, user_id, machine_id, instance_id).await;
|
||||
assert!(
|
||||
mgr.db()
|
||||
.get_network_config((user_id, machine_id), &instance_id.to_string())
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
mgr.db()
|
||||
.get_managed_config_revision((user_id, machine_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
|
||||
mgr.reconcile_managed_network_configs(
|
||||
user_id,
|
||||
machine_id,
|
||||
vec![managed_config(
|
||||
instance_id,
|
||||
redelivered_managed_network_config(instance_id),
|
||||
)],
|
||||
Some("rev-webhook-redelivery".to_string()),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let redelivered = wait_for_runtime_config(&core_manager, instance_id, |config| {
|
||||
config.network_name.as_deref() == Some("managed-redelivered")
|
||||
&& config.instance_recv_bps_limit == Some(654321)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
redelivered.instance_id.as_deref(),
|
||||
Some(instance_id.to_string().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
redelivered.hostname.as_deref(),
|
||||
Some("managed-redelivered-host")
|
||||
);
|
||||
assert_eq!(
|
||||
redelivered.network_name.as_deref(),
|
||||
Some("managed-redelivered")
|
||||
);
|
||||
assert_eq!(redelivered.enable_kcp_proxy, Some(true));
|
||||
assert_eq!(redelivered.instance_recv_bps_limit, Some(654321));
|
||||
assert_eq!(
|
||||
core_manager.get_instance_network_config_source(&instance_id),
|
||||
Some(easytier::common::config::ConfigSource::Web)
|
||||
);
|
||||
assert_eq!(
|
||||
mgr.db()
|
||||
.get_managed_config_revision((user_id, machine_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some("rev-webhook-redelivery")
|
||||
);
|
||||
|
||||
// Reconnect path: a fresh core manager has no local runtime state, so
|
||||
// the new session must replay the managed config persisted in web DB.
|
||||
drop(client);
|
||||
let reconnected_core_manager = Arc::new(NetworkInstanceManager::new());
|
||||
let _reconnected_client = start_web_client_for_test(
|
||||
config_server_addr,
|
||||
machine_id,
|
||||
reconnected_core_manager.clone(),
|
||||
)
|
||||
.await;
|
||||
wait_for_validated_user(&mgr, machine_id).await;
|
||||
let replayed = wait_for_runtime_config(&reconnected_core_manager, instance_id, |config| {
|
||||
config.network_name.as_deref() == Some("managed-redelivered")
|
||||
&& config.instance_recv_bps_limit == Some(654321)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
replayed.network_name.as_deref(),
|
||||
Some("managed-redelivered")
|
||||
);
|
||||
assert_eq!(replayed.enable_kcp_proxy, Some(true));
|
||||
assert_eq!(replayed.instance_recv_bps_limit, Some(654321));
|
||||
|
||||
webhook_server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webhook_reject_disconnects_and_revalidates_after_reconnect() {
|
||||
let webhook_state = TestWebhookState::with_blocked_second_validate([false, true]);
|
||||
let (webhook_config, webhook_server, webhook_state) =
|
||||
test_webhook_config_with_state(webhook_state).await;
|
||||
let mut mgr = ClientManager::new(
|
||||
Db::memory_db().await,
|
||||
None,
|
||||
Duration::ZERO,
|
||||
Arc::new(FeatureFlags::default()),
|
||||
webhook_config,
|
||||
);
|
||||
let config_server_addr = add_random_udp_listener(&mut mgr).await;
|
||||
let machine_id = uuid::Uuid::new_v4();
|
||||
let core_manager = Arc::new(NetworkInstanceManager::new());
|
||||
let client =
|
||||
start_web_client_for_test(config_server_addr, machine_id, core_manager.clone()).await;
|
||||
|
||||
let first_session_urls = wait_for_session_urls(&mgr).await;
|
||||
wait_for_validate_count(&webhook_state, 1).await;
|
||||
wait_for_validate_count(&webhook_state, 2).await;
|
||||
assert!(
|
||||
mgr.list_sessions().await.is_empty(),
|
||||
"invalid validate-token response must not authorize the session"
|
||||
);
|
||||
|
||||
webhook_state.allow_second_validate();
|
||||
let user_id = wait_for_validated_user(&mgr, machine_id).await;
|
||||
tokio::time::timeout(Duration::from_secs(12), async {
|
||||
loop {
|
||||
let reconnected = mgr
|
||||
.client_sessions
|
||||
.iter()
|
||||
.any(|entry| !first_session_urls.iter().any(|url| url == entry.key()));
|
||||
if reconnected {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
client.is_connected(),
|
||||
"web client should reconnect after invalid session heartbeat failure"
|
||||
);
|
||||
assert!(webhook_state.validate_count() >= 2);
|
||||
assert!(
|
||||
mgr.get_session_by_machine_id(user_id, &machine_id)
|
||||
.is_some()
|
||||
);
|
||||
|
||||
webhook_server.abort();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,812 @@
|
||||
use anyhow::Context as _;
|
||||
use easytier::{
|
||||
common::config::{
|
||||
ConfigLoader, EncryptionAlgorithm, PortForwardConfig as RuntimePortForwardConfig,
|
||||
},
|
||||
proto::{
|
||||
acl::Acl,
|
||||
api::{
|
||||
config::{
|
||||
AclPatch, ConfigPatchAction, InstanceConfigPatch, PatchConfigRequest,
|
||||
PortForwardPatch, ProxyNetworkPatch,
|
||||
},
|
||||
instance::{InstanceIdentifier, instance_identifier},
|
||||
manage::{
|
||||
ConfigSource as RpcConfigSource, GetNetworkInstanceConfigRequest, NetworkConfig,
|
||||
RunNetworkInstanceRequest,
|
||||
},
|
||||
},
|
||||
common::{CompressionAlgoPb, Ipv4Inet as RpcIpv4Inet},
|
||||
rpc_types::controller::BaseController,
|
||||
},
|
||||
};
|
||||
|
||||
use super::session::{SessionConfigClient, SessionRpcClient};
|
||||
|
||||
pub(super) enum RuntimeReconcileAction {
|
||||
None,
|
||||
Run {
|
||||
config: Box<NetworkConfig>,
|
||||
overwrite: bool,
|
||||
},
|
||||
Patch(Box<InstanceConfigPatch>),
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq)]
|
||||
struct RuntimeProxyNetwork {
|
||||
cidr: String,
|
||||
mapped_cidr: Option<String>,
|
||||
}
|
||||
|
||||
fn instance_identifier(inst_id: &str) -> anyhow::Result<InstanceIdentifier> {
|
||||
let inst_id = uuid::Uuid::parse_str(inst_id)
|
||||
.with_context(|| format!("invalid runtime instance id: {inst_id}"))?;
|
||||
Ok(InstanceIdentifier {
|
||||
selector: Some(instance_identifier::Selector::Id(inst_id.into())),
|
||||
})
|
||||
}
|
||||
|
||||
fn hot_patch_base(config: &NetworkConfig) -> anyhow::Result<NetworkConfig> {
|
||||
let data_compress_algo = normalized_data_compress_algo(config.data_compress_algo);
|
||||
let encryption_algorithm = normalized_encryption_algorithm(config.encryption_algorithm.clone());
|
||||
let mut config = NetworkConfig::new_from_config(config.gen_config()?)?;
|
||||
let is_credential_mode = config.network_secret.is_none()
|
||||
&& config
|
||||
.secure_mode
|
||||
.as_ref()
|
||||
.and_then(|mode| mode.local_private_key.as_deref())
|
||||
.is_some_and(|key| !key.is_empty());
|
||||
config.acl = None;
|
||||
config.port_forwards.clear();
|
||||
config.proxy_cidrs.clear();
|
||||
config.disable_relay_data = None;
|
||||
if config.dhcp.unwrap_or_default() {
|
||||
config.virtual_ipv4 = None;
|
||||
config.network_length = None;
|
||||
}
|
||||
if let Some(secure_mode) = config.secure_mode.as_mut() {
|
||||
if !is_credential_mode {
|
||||
secure_mode.local_private_key = None;
|
||||
}
|
||||
secure_mode.local_public_key = None;
|
||||
}
|
||||
config.data_compress_algo = data_compress_algo;
|
||||
config.encryption_algorithm = encryption_algorithm;
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
fn normalized_data_compress_algo(algo: Option<i32>) -> Option<i32> {
|
||||
let default = CompressionAlgoPb::None as i32;
|
||||
let effective = algo.map(|algo| if algo < default { default } else { algo });
|
||||
effective.filter(|algo| *algo != default)
|
||||
}
|
||||
|
||||
fn normalized_encryption_algorithm(algo: Option<String>) -> Option<String> {
|
||||
let default = EncryptionAlgorithm::default().to_string();
|
||||
algo.filter(|algo| algo != &default)
|
||||
}
|
||||
|
||||
fn diff_port_forwards(
|
||||
current: &[RuntimePortForwardConfig],
|
||||
desired: &[RuntimePortForwardConfig],
|
||||
) -> Vec<PortForwardPatch> {
|
||||
let mut patches = Vec::new();
|
||||
for cfg in unique_port_forwards(current, desired) {
|
||||
let current_count = current.iter().filter(|item| *item == &cfg).count();
|
||||
let desired_count = desired.iter().filter(|item| *item == &cfg).count();
|
||||
if current_count == desired_count {
|
||||
continue;
|
||||
}
|
||||
if current_count > 0 {
|
||||
patches.push(PortForwardPatch {
|
||||
action: ConfigPatchAction::Remove as i32,
|
||||
cfg: Some(cfg.clone().into()),
|
||||
});
|
||||
}
|
||||
patches.extend((0..desired_count).map(|_| PortForwardPatch {
|
||||
action: ConfigPatchAction::Add as i32,
|
||||
cfg: Some(cfg.clone().into()),
|
||||
}));
|
||||
}
|
||||
patches
|
||||
}
|
||||
|
||||
fn unique_port_forwards(
|
||||
current: &[RuntimePortForwardConfig],
|
||||
desired: &[RuntimePortForwardConfig],
|
||||
) -> Vec<RuntimePortForwardConfig> {
|
||||
let mut unique = Vec::new();
|
||||
for cfg in current.iter().chain(desired.iter()) {
|
||||
if !unique.contains(cfg) {
|
||||
unique.push(cfg.clone());
|
||||
}
|
||||
}
|
||||
unique
|
||||
}
|
||||
|
||||
fn parse_rpc_ipv4_inet(value: &str) -> anyhow::Result<RpcIpv4Inet> {
|
||||
value
|
||||
.parse::<RpcIpv4Inet>()
|
||||
.with_context(|| format!("failed to parse runtime ipv4 cidr: {value}"))
|
||||
}
|
||||
|
||||
fn diff_proxy_networks(
|
||||
current: &[RuntimeProxyNetwork],
|
||||
desired: &[RuntimeProxyNetwork],
|
||||
) -> anyhow::Result<Vec<ProxyNetworkPatch>> {
|
||||
if current == desired {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut patches = vec![ProxyNetworkPatch {
|
||||
action: ConfigPatchAction::Clear as i32,
|
||||
cidr: Some(clear_proxy_network_cidr(current, desired)?),
|
||||
..Default::default()
|
||||
}];
|
||||
for proxy_network in desired {
|
||||
patches.push(ProxyNetworkPatch {
|
||||
action: ConfigPatchAction::Add as i32,
|
||||
cidr: Some(parse_rpc_ipv4_inet(&proxy_network.cidr)?),
|
||||
mapped_cidr: proxy_network
|
||||
.mapped_cidr
|
||||
.as_deref()
|
||||
.map(parse_rpc_ipv4_inet)
|
||||
.transpose()?,
|
||||
});
|
||||
}
|
||||
Ok(patches)
|
||||
}
|
||||
|
||||
fn clear_proxy_network_cidr(
|
||||
current: &[RuntimeProxyNetwork],
|
||||
desired: &[RuntimeProxyNetwork],
|
||||
) -> anyhow::Result<RpcIpv4Inet> {
|
||||
let cidr = desired
|
||||
.first()
|
||||
.or_else(|| current.first())
|
||||
.map(|proxy_network| proxy_network.cidr.as_str())
|
||||
.unwrap_or("0.0.0.0/0");
|
||||
parse_rpc_ipv4_inet(cidr)
|
||||
}
|
||||
|
||||
fn normalized_acl(acl: &Option<Acl>) -> Option<Acl> {
|
||||
let acl = acl.clone().unwrap_or_default();
|
||||
(acl != Acl::default()).then_some(acl)
|
||||
}
|
||||
|
||||
fn normalized_port_forwards(
|
||||
config: &NetworkConfig,
|
||||
) -> anyhow::Result<Vec<RuntimePortForwardConfig>> {
|
||||
Ok(config
|
||||
.gen_config()?
|
||||
.get_port_forwards()
|
||||
.into_iter()
|
||||
.map(|cfg| {
|
||||
RuntimePortForwardConfig::from(easytier::proto::common::PortForwardConfigPb::from(cfg))
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn normalized_proxy_networks(config: &NetworkConfig) -> anyhow::Result<Vec<RuntimeProxyNetwork>> {
|
||||
Ok(config
|
||||
.gen_config()?
|
||||
.get_proxy_cidrs()
|
||||
.into_iter()
|
||||
.map(|proxy_network| RuntimeProxyNetwork {
|
||||
cidr: proxy_network.cidr.to_string(),
|
||||
mapped_cidr: proxy_network.mapped_cidr.map(|cidr| cidr.to_string()),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn normalized_disable_relay_data(config: &NetworkConfig) -> anyhow::Result<bool> {
|
||||
Ok(config.gen_config()?.get_flags().disable_relay_data)
|
||||
}
|
||||
|
||||
fn web_source_runtime_patch(
|
||||
current: &NetworkConfig,
|
||||
desired: &NetworkConfig,
|
||||
) -> anyhow::Result<Option<InstanceConfigPatch>> {
|
||||
if let Some(desired_hostname) = desired
|
||||
.hostname
|
||||
.as_deref()
|
||||
.filter(|hostname| !hostname.is_empty())
|
||||
&& current.hostname.as_deref() != Some(desired_hostname)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let mut current_base = hot_patch_base(current)?;
|
||||
let mut desired_base = hot_patch_base(desired)?;
|
||||
current_base.hostname = None;
|
||||
desired_base.hostname = None;
|
||||
if current_base != desired_base {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let mut patch = InstanceConfigPatch::default();
|
||||
let current_acl = normalized_acl(¤t.acl);
|
||||
let desired_acl = normalized_acl(&desired.acl);
|
||||
if current_acl != desired_acl {
|
||||
patch.acl = Some(AclPatch {
|
||||
acl: Some(desired_acl.unwrap_or_default()),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
|
||||
let current_port_forwards = normalized_port_forwards(current)?;
|
||||
let desired_port_forwards = normalized_port_forwards(desired)?;
|
||||
if current_port_forwards != desired_port_forwards {
|
||||
patch.port_forwards = diff_port_forwards(¤t_port_forwards, &desired_port_forwards);
|
||||
}
|
||||
|
||||
let current_proxy_networks = normalized_proxy_networks(current)?;
|
||||
let desired_proxy_networks = normalized_proxy_networks(desired)?;
|
||||
if current_proxy_networks != desired_proxy_networks {
|
||||
if current_proxy_networks.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
patch.proxy_networks =
|
||||
diff_proxy_networks(¤t_proxy_networks, &desired_proxy_networks)?;
|
||||
}
|
||||
|
||||
let current_disable_relay_data = normalized_disable_relay_data(current)?;
|
||||
let desired_disable_relay_data = normalized_disable_relay_data(desired)?;
|
||||
if current_disable_relay_data != desired_disable_relay_data {
|
||||
patch.disable_relay_data = Some(desired_disable_relay_data);
|
||||
}
|
||||
|
||||
Ok(Some(patch))
|
||||
}
|
||||
|
||||
fn ensure_runtime_config_converged(
|
||||
current: &NetworkConfig,
|
||||
desired: &NetworkConfig,
|
||||
) -> anyhow::Result<()> {
|
||||
let patch = web_source_runtime_patch(current, desired)?;
|
||||
match patch {
|
||||
Some(patch) if patch == InstanceConfigPatch::default() => Ok(()),
|
||||
Some(patch) => anyhow::bail!("runtime config still needs patch after reconcile: {patch:?}"),
|
||||
None => anyhow::bail!("runtime config still needs full overwrite after reconcile"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_web_source_instance(
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
inst_id: &str,
|
||||
config: NetworkConfig,
|
||||
overwrite: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
rpc_client
|
||||
.run_network_instance(
|
||||
BaseController::default(),
|
||||
RunNetworkInstanceRequest {
|
||||
inst_id: Some(inst_id.to_string().into()),
|
||||
config: Some(config),
|
||||
overwrite,
|
||||
source: RpcConfigSource::Web as i32,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn get_runtime_config(
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
inst_id: &str,
|
||||
) -> anyhow::Result<NetworkConfig> {
|
||||
rpc_client
|
||||
.get_network_instance_config(
|
||||
BaseController::default(),
|
||||
GetNetworkInstanceConfigRequest {
|
||||
inst_id: Some(inst_id.to_string().into()),
|
||||
},
|
||||
)
|
||||
.await?
|
||||
.config
|
||||
.ok_or_else(|| anyhow::anyhow!("runtime returned empty config for {inst_id}"))
|
||||
}
|
||||
|
||||
pub(super) async fn prepare_web_source_runtime_reconcile(
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
inst_id: &str,
|
||||
desired_config: NetworkConfig,
|
||||
is_running: bool,
|
||||
) -> anyhow::Result<RuntimeReconcileAction> {
|
||||
if !is_running {
|
||||
return Ok(RuntimeReconcileAction::Run {
|
||||
config: Box::new(desired_config),
|
||||
overwrite: false,
|
||||
});
|
||||
}
|
||||
|
||||
let current_config = get_runtime_config(rpc_client, inst_id).await?;
|
||||
|
||||
prepare_web_source_runtime_reconcile_from_current(¤t_config, desired_config)
|
||||
}
|
||||
|
||||
pub(super) fn prepare_web_source_runtime_reconcile_from_current(
|
||||
current_config: &NetworkConfig,
|
||||
desired_config: NetworkConfig,
|
||||
) -> anyhow::Result<RuntimeReconcileAction> {
|
||||
let Some(patch) = web_source_runtime_patch(current_config, &desired_config)? else {
|
||||
return Ok(RuntimeReconcileAction::Run {
|
||||
config: Box::new(desired_config),
|
||||
overwrite: true,
|
||||
});
|
||||
};
|
||||
if patch == InstanceConfigPatch::default() {
|
||||
return Ok(RuntimeReconcileAction::None);
|
||||
}
|
||||
|
||||
Ok(RuntimeReconcileAction::Patch(Box::new(patch)))
|
||||
}
|
||||
|
||||
pub(super) async fn apply_web_source_runtime_reconcile(
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
config_client: &mut SessionConfigClient,
|
||||
inst_id: &str,
|
||||
desired_config: NetworkConfig,
|
||||
action: RuntimeReconcileAction,
|
||||
) -> anyhow::Result<NetworkConfig> {
|
||||
match action {
|
||||
RuntimeReconcileAction::None => Ok(desired_config),
|
||||
RuntimeReconcileAction::Run { config, overwrite } => {
|
||||
run_web_source_instance(rpc_client, inst_id, *config, overwrite).await?;
|
||||
Ok(desired_config)
|
||||
}
|
||||
RuntimeReconcileAction::Patch(patch) => {
|
||||
config_client
|
||||
.patch_config(
|
||||
BaseController::default(),
|
||||
PatchConfigRequest {
|
||||
instance: Some(instance_identifier(inst_id)?),
|
||||
patch: Some(*patch),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let current_config = get_runtime_config(rpc_client, inst_id).await?;
|
||||
ensure_runtime_config_converged(¤t_config, &desired_config)?;
|
||||
Ok(current_config)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use easytier::proto::{
|
||||
api::{
|
||||
config::ConfigPatchAction,
|
||||
manage::{NetworkingMethod, PortForwardConfig},
|
||||
},
|
||||
common::{CompressionAlgoPb, SocketType},
|
||||
};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn config_with_port_forwards(port_forwards: Vec<PortForwardConfig>) -> NetworkConfig {
|
||||
NetworkConfig {
|
||||
instance_id: Some("11111111-1111-1111-1111-111111111111".to_string()),
|
||||
dhcp: Some(true),
|
||||
network_name: Some("managed".to_string()),
|
||||
network_secret: Some("secret".to_string()),
|
||||
networking_method: Some(NetworkingMethod::Manual as i32),
|
||||
port_forwards,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn port_forward(bind_port: u32, dst_port: u32) -> PortForwardConfig {
|
||||
PortForwardConfig {
|
||||
bind_ip: "127.0.0.1".to_string(),
|
||||
bind_port,
|
||||
dst_ip: "10.144.0.1".to_string(),
|
||||
dst_port,
|
||||
proto: "tcp".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn patch_port(patch: &PortForwardPatch) -> (i32, u32, u32, i32) {
|
||||
let cfg = patch.cfg.as_ref().expect("port forward patch cfg");
|
||||
(
|
||||
patch.action,
|
||||
cfg.bind_addr.as_ref().expect("bind addr").port,
|
||||
cfg.dst_addr.as_ref().expect("dst addr").port,
|
||||
cfg.socket_type,
|
||||
)
|
||||
}
|
||||
|
||||
fn patch_proxy_network(patch: &ProxyNetworkPatch) -> (i32, String, Option<String>) {
|
||||
(
|
||||
patch.action,
|
||||
patch.cidr.map(|cidr| cidr.to_string()).unwrap_or_default(),
|
||||
patch.mapped_cidr.map(|cidr| cidr.to_string()),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_ignores_runtime_defaults_and_adds_port_forward() {
|
||||
let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
current.virtual_ipv4 = Some("10.144.0.2".to_string());
|
||||
current.network_length = Some(16);
|
||||
current.bind_device = Some(true);
|
||||
current.dev_name = Some(String::new());
|
||||
current.disable_ipv6 = Some(false);
|
||||
current.mtu = Some(1380);
|
||||
current.multi_thread = Some(true);
|
||||
|
||||
let desired =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.port_forwards.len(), 1);
|
||||
assert_eq!(
|
||||
patch_port(&patch.port_forwards[0]),
|
||||
(
|
||||
ConfigPatchAction::Add as i32,
|
||||
23007,
|
||||
3389,
|
||||
SocketType::Tcp as i32
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_removes_deleted_port_forward_without_clear() {
|
||||
let current =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
let desired = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.port_forwards.len(), 1);
|
||||
assert_eq!(
|
||||
patch_port(&patch.port_forwards[0]),
|
||||
(
|
||||
ConfigPatchAction::Remove as i32,
|
||||
23007,
|
||||
3389,
|
||||
SocketType::Tcp as i32
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_reconciles_duplicate_port_forward_count() {
|
||||
let current =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23000, 5174)]);
|
||||
let desired = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.port_forwards.len(), 2);
|
||||
assert_eq!(
|
||||
patch_port(&patch.port_forwards[0]),
|
||||
(
|
||||
ConfigPatchAction::Remove as i32,
|
||||
23000,
|
||||
5174,
|
||||
SocketType::Tcp as i32
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
patch_port(&patch.port_forwards[1]),
|
||||
(
|
||||
ConfigPatchAction::Add as i32,
|
||||
23000,
|
||||
5174,
|
||||
SocketType::Tcp as i32
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_convergence_rejects_stale_extra_port_forward() {
|
||||
let current = config_with_port_forwards(vec![
|
||||
port_forward(23000, 5174),
|
||||
port_forward(23007, 3389),
|
||||
port_forward(23100, 8080),
|
||||
]);
|
||||
let desired =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
|
||||
let err = ensure_runtime_config_converged(¤t, &desired)
|
||||
.expect_err("extra runtime port forward should not converge");
|
||||
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("runtime config still needs patch after reconcile"),
|
||||
"unexpected error: {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_canonicalizes_port_forward_protocol() {
|
||||
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
let mut desired_port_forward = port_forward(23000, 5174);
|
||||
desired_port_forward.proto = "TCP".to_string();
|
||||
let desired = config_with_port_forwards(vec![desired_port_forward]);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch, InstanceConfigPatch::default());
|
||||
ensure_runtime_config_converged(¤t, &desired).expect("runtime converged");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_rejects_non_hot_config_change() {
|
||||
let current = config_with_port_forwards(Vec::new());
|
||||
let mut desired = current.clone();
|
||||
desired.network_secret = Some("new-secret".to_string());
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_rejects_routes_change() {
|
||||
let mut current = config_with_port_forwards(Vec::new());
|
||||
current.enable_manual_routes = Some(true);
|
||||
current.routes = vec!["10.1.0.0/16".to_string(), "10.2.0.0/16".to_string()];
|
||||
let mut desired = config_with_port_forwards(Vec::new());
|
||||
desired.enable_manual_routes = Some(true);
|
||||
desired.routes = vec!["10.2.0.0/16".to_string(), "10.3.0.0/16".to_string()];
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_replaces_proxy_networks() {
|
||||
let mut current = config_with_port_forwards(Vec::new());
|
||||
current.proxy_cidrs = vec![
|
||||
"10.1.0.0/16".to_string(),
|
||||
"10.2.0.0/16->10.20.0.0/16".to_string(),
|
||||
];
|
||||
let mut desired = config_with_port_forwards(Vec::new());
|
||||
desired.proxy_cidrs = vec![
|
||||
"10.2.0.0/16->10.21.0.0/16".to_string(),
|
||||
"10.3.0.0/16".to_string(),
|
||||
];
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.proxy_networks.len(), 3);
|
||||
assert_eq!(
|
||||
patch_proxy_network(&patch.proxy_networks[0]),
|
||||
(
|
||||
ConfigPatchAction::Clear as i32,
|
||||
"10.2.0.0/16".to_string(),
|
||||
None
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
patch_proxy_network(&patch.proxy_networks[1]),
|
||||
(
|
||||
ConfigPatchAction::Add as i32,
|
||||
"10.2.0.0/16".to_string(),
|
||||
Some("10.21.0.0/16".to_string())
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
patch_proxy_network(&patch.proxy_networks[2]),
|
||||
(
|
||||
ConfigPatchAction::Add as i32,
|
||||
"10.3.0.0/16".to_string(),
|
||||
None
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_replaces_proxy_networks_with_same_source_cidr() {
|
||||
let mut current = config_with_port_forwards(Vec::new());
|
||||
current.proxy_cidrs = vec![
|
||||
"10.1.2.0/24".to_string(),
|
||||
"10.1.2.0/24->10.1.3.0/24".to_string(),
|
||||
];
|
||||
let mut desired = config_with_port_forwards(Vec::new());
|
||||
desired.proxy_cidrs = vec!["10.1.2.0/24->10.1.3.0/24".to_string()];
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.proxy_networks.len(), 2);
|
||||
assert_eq!(
|
||||
patch_proxy_network(&patch.proxy_networks[0]),
|
||||
(
|
||||
ConfigPatchAction::Clear as i32,
|
||||
"10.1.2.0/24".to_string(),
|
||||
None
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
patch_proxy_network(&patch.proxy_networks[1]),
|
||||
(
|
||||
ConfigPatchAction::Add as i32,
|
||||
"10.1.2.0/24".to_string(),
|
||||
Some("10.1.3.0/24".to_string())
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_rejects_proxy_network_empty_to_nonempty() {
|
||||
let current = config_with_port_forwards(Vec::new());
|
||||
let mut desired = config_with_port_forwards(Vec::new());
|
||||
desired.proxy_cidrs = vec!["10.1.2.0/24".to_string()];
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_clears_proxy_networks_with_legacy_compatible_cidr() {
|
||||
let mut current = config_with_port_forwards(Vec::new());
|
||||
current.proxy_cidrs = vec!["10.1.2.0/24".to_string()];
|
||||
let desired = config_with_port_forwards(Vec::new());
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.proxy_networks.len(), 1);
|
||||
assert_eq!(
|
||||
patch_proxy_network(&patch.proxy_networks[0]),
|
||||
(
|
||||
ConfigPatchAction::Clear as i32,
|
||||
"10.1.2.0/24".to_string(),
|
||||
None
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_updates_disable_relay_data() {
|
||||
let current = config_with_port_forwards(Vec::new());
|
||||
let mut desired = current.clone();
|
||||
desired.disable_relay_data = Some(true);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.disable_relay_data, Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_still_rejects_unsupported_flag_change() {
|
||||
let current = config_with_port_forwards(Vec::new());
|
||||
let mut desired = current.clone();
|
||||
desired.no_tun = Some(true);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_rejects_encryption_algorithm_change() {
|
||||
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
let mut desired = current.clone();
|
||||
desired.encryption_algorithm = Some("managed-test-algo".to_string());
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_rejects_data_compress_algo_change() {
|
||||
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
let mut desired = current.clone();
|
||||
desired.data_compress_algo = Some(CompressionAlgoPb::Zstd as i32);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_rejects_credential_private_key_change() {
|
||||
let mut current = config_with_port_forwards(Vec::new());
|
||||
current.network_secret = None;
|
||||
current.secure_mode = Some(easytier::proto::common::SecureModeConfig {
|
||||
enabled: true,
|
||||
local_private_key: Some("mUuD5fsIm/ftvgS4WBAYFMNLqWX3qT9rnm4PrnOqb9s=".to_string()),
|
||||
local_public_key: None,
|
||||
});
|
||||
let mut desired = current.clone();
|
||||
desired.secure_mode = Some(easytier::proto::common::SecureModeConfig {
|
||||
enabled: true,
|
||||
local_private_key: Some("aEpz80FuYbaY4QLJizAIuIcK4TYsoSA9jHHCXCOQJoc=".to_string()),
|
||||
local_public_key: None,
|
||||
});
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_ignores_generated_secure_key_when_network_secret_exists() {
|
||||
let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
current.secure_mode = Some(easytier::proto::common::SecureModeConfig {
|
||||
enabled: true,
|
||||
local_private_key: Some("mUuD5fsIm/ftvgS4WBAYFMNLqWX3qT9rnm4PrnOqb9s=".to_string()),
|
||||
local_public_key: Some("4x6L5dZjB8hsPO4f96Hyhi4xFealBu6i3BxRVBYR1Fc=".to_string()),
|
||||
});
|
||||
let mut desired =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
desired.secure_mode = Some(easytier::proto::common::SecureModeConfig {
|
||||
enabled: true,
|
||||
local_private_key: None,
|
||||
local_public_key: None,
|
||||
});
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.port_forwards.len(), 1);
|
||||
assert_eq!(
|
||||
patch_port(&patch.port_forwards[0]),
|
||||
(
|
||||
ConfigPatchAction::Add as i32,
|
||||
23007,
|
||||
3389,
|
||||
SocketType::Tcp as i32
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_ignores_runtime_hostname_when_desired_omits_hostname() {
|
||||
let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
current.hostname = Some("runtime-host".to_string());
|
||||
let desired =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.port_forwards.len(), 1);
|
||||
assert_eq!(
|
||||
patch_port(&patch.port_forwards[0]),
|
||||
(
|
||||
ConfigPatchAction::Add as i32,
|
||||
23007,
|
||||
3389,
|
||||
SocketType::Tcp as i32
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_rejects_explicit_desired_hostname_change() {
|
||||
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
let mut desired =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
desired.hostname =
|
||||
Some(easytier::common::config::TomlConfigLoader::default().get_hostname());
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,913 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use easytier::{
|
||||
proto::{
|
||||
api::manage::{
|
||||
DeleteNetworkInstanceRequest, ListNetworkInstanceMetaRequest,
|
||||
ListNetworkInstanceRequest, NetworkConfig, NetworkMeta, RunNetworkInstanceRequest,
|
||||
},
|
||||
rpc_types::controller::BaseController,
|
||||
web::HeartbeatRequest,
|
||||
},
|
||||
rpc_service::remote_client::{ListNetworkProps, Storage as _},
|
||||
};
|
||||
use tokio::sync::{RwLock, broadcast};
|
||||
|
||||
use super::{SessionConfigClient, SessionData, SessionRpcClient, SessionRpcService};
|
||||
use crate::client_manager::{
|
||||
managed_config::{self, PersistedConfigSource},
|
||||
runtime_reconcile,
|
||||
storage::{StorageInner, WeakRefStorage},
|
||||
};
|
||||
|
||||
async fn recv_latest_heartbeat(
|
||||
heartbeat_waiter: &mut broadcast::Receiver<HeartbeatRequest>,
|
||||
) -> Option<HeartbeatRequest> {
|
||||
let mut req = loop {
|
||||
match heartbeat_waiter.recv().await {
|
||||
Ok(req) => break req,
|
||||
Err(broadcast::error::RecvError::Lagged(skipped)) => {
|
||||
tracing::warn!(
|
||||
skipped,
|
||||
"heartbeat reconcile worker lagged, waiting for latest request"
|
||||
);
|
||||
}
|
||||
Err(broadcast::error::RecvError::Closed) => {
|
||||
tracing::error!("Failed to receive heartbeat request: channel closed");
|
||||
return None;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Drop any heartbeat backlog accumulated while the previous reconcile
|
||||
// round was doing DB/RPC IO. The newest heartbeat has the freshest
|
||||
// runtime instance list, which is all this task needs.
|
||||
loop {
|
||||
match heartbeat_waiter.try_recv() {
|
||||
Ok(next_req) => req = next_req,
|
||||
Err(broadcast::error::TryRecvError::Empty) => break,
|
||||
Err(broadcast::error::TryRecvError::Lagged(_)) => continue,
|
||||
Err(broadcast::error::TryRecvError::Closed) => return None,
|
||||
}
|
||||
}
|
||||
|
||||
Some(req)
|
||||
}
|
||||
|
||||
pub(super) async fn reconcile_network_configs_on_heartbeat(
|
||||
session_data: std::sync::Weak<RwLock<SessionData>>,
|
||||
mut heartbeat_waiter: broadcast::Receiver<HeartbeatRequest>,
|
||||
storage: WeakRefStorage,
|
||||
mut rpc_client: SessionRpcClient,
|
||||
mut config_client: SessionConfigClient,
|
||||
) {
|
||||
let mut cache = ReconcileCache::default();
|
||||
loop {
|
||||
let Some(req) = recv_latest_heartbeat(&mut heartbeat_waiter).await else {
|
||||
return;
|
||||
};
|
||||
let Some(storage) = storage.upgrade() else {
|
||||
tracing::error!("Failed to get storage");
|
||||
return;
|
||||
};
|
||||
|
||||
let mut round =
|
||||
match prepare_reconcile_round(&session_data, &storage, &mut rpc_client, req).await {
|
||||
RoundStatus::Ready(round) => round,
|
||||
RoundStatus::Skip => continue,
|
||||
RoundStatus::Stop => return,
|
||||
};
|
||||
let running_metas =
|
||||
match sync_running_sources_for_round(&mut rpc_client, &storage, &mut round).await {
|
||||
RoundStatus::Ready(running_metas) => running_metas,
|
||||
RoundStatus::Skip => continue,
|
||||
RoundStatus::Stop => return,
|
||||
};
|
||||
|
||||
let desired_web_inst_ids =
|
||||
managed_config::desired_web_source_instance_ids(&round.local_configs);
|
||||
cache.runtime_configs.retain_desired(&desired_web_inst_ids);
|
||||
let mut outcome = match cleanup_stale_web_source_instances(
|
||||
&session_data,
|
||||
&storage,
|
||||
&mut rpc_client,
|
||||
&round,
|
||||
running_metas.as_deref(),
|
||||
&desired_web_inst_ids,
|
||||
&mut cache,
|
||||
)
|
||||
.await
|
||||
{
|
||||
RoundStatus::Ready(outcome) => outcome,
|
||||
RoundStatus::Skip => continue,
|
||||
RoundStatus::Stop => return,
|
||||
};
|
||||
|
||||
outcome.merge(
|
||||
reconcile_desired_runtime_configs(
|
||||
&session_data,
|
||||
&mut rpc_client,
|
||||
&mut config_client,
|
||||
&round,
|
||||
&mut cache,
|
||||
)
|
||||
.await,
|
||||
);
|
||||
|
||||
if !outcome.has_failed {
|
||||
cache.last_desired_web_inst_ids = Some(desired_web_inst_ids);
|
||||
}
|
||||
match mark_config_revision_applied_if_current(&session_data, &storage, &round, &outcome)
|
||||
.await
|
||||
{
|
||||
RoundStatus::Ready(()) | RoundStatus::Skip => {}
|
||||
RoundStatus::Stop => return,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum RoundStatus<T> {
|
||||
Ready(T),
|
||||
Skip,
|
||||
Stop,
|
||||
}
|
||||
|
||||
enum ConfigActionResult {
|
||||
Success,
|
||||
Failed,
|
||||
StopRound,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ReconcileCache {
|
||||
cleaned_web_source_instances: bool,
|
||||
last_desired_web_inst_ids: Option<HashSet<String>>,
|
||||
runtime_configs: SessionRuntimeConfigCache,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct SessionRuntimeConfigCache {
|
||||
entries: HashMap<String, NetworkConfig>,
|
||||
}
|
||||
|
||||
impl SessionRuntimeConfigCache {
|
||||
fn plan(
|
||||
&self,
|
||||
inst_id: &str,
|
||||
desired_config: NetworkConfig,
|
||||
) -> anyhow::Result<Option<runtime_reconcile::RuntimeReconcileAction>> {
|
||||
let Some(observed_config) = self.entries.get(inst_id) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
runtime_reconcile::prepare_web_source_runtime_reconcile_from_current(
|
||||
observed_config,
|
||||
desired_config,
|
||||
)
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
fn remember(&mut self, inst_id: &str, observed_config: NetworkConfig) {
|
||||
self.entries.insert(inst_id.to_string(), observed_config);
|
||||
}
|
||||
|
||||
fn forget(&mut self, inst_id: &str) {
|
||||
self.entries.remove(inst_id);
|
||||
}
|
||||
|
||||
fn forget_many<'a>(&mut self, inst_ids: impl IntoIterator<Item = &'a String>) {
|
||||
for inst_id in inst_ids {
|
||||
self.entries.remove(inst_id);
|
||||
}
|
||||
}
|
||||
|
||||
fn retain_desired(&mut self, desired_web_inst_ids: &HashSet<String>) {
|
||||
self.entries
|
||||
.retain(|inst_id, _| desired_web_inst_ids.contains(inst_id));
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ReconcileOutcome {
|
||||
has_failed: bool,
|
||||
managed_revision_failed: bool,
|
||||
}
|
||||
|
||||
impl ReconcileOutcome {
|
||||
fn record_failure(&mut self, managed_revision_failed: bool) {
|
||||
self.has_failed = true;
|
||||
self.managed_revision_failed |= managed_revision_failed;
|
||||
}
|
||||
|
||||
fn merge(&mut self, other: Self) {
|
||||
self.has_failed |= other.has_failed;
|
||||
self.managed_revision_failed |= other.managed_revision_failed;
|
||||
}
|
||||
}
|
||||
|
||||
struct ReconcileRound {
|
||||
req: HeartbeatRequest,
|
||||
machine_id: uuid::Uuid,
|
||||
user_id: i32,
|
||||
running_inst_ids: HashSet<String>,
|
||||
local_configs: Vec<crate::db::entity::user_running_network_configs::Model>,
|
||||
target_config_revision: Option<String>,
|
||||
should_apply_runtime_revision: bool,
|
||||
}
|
||||
|
||||
async fn prepare_reconcile_round(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
storage: &StorageInner,
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
req: HeartbeatRequest,
|
||||
) -> RoundStatus<ReconcileRound> {
|
||||
let Some(machine_id) = req.machine_id.map(uuid::Uuid::from) else {
|
||||
tracing::warn!(?req, "Machine id is not set, ignore");
|
||||
return RoundStatus::Skip;
|
||||
};
|
||||
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &req).await {
|
||||
tracing::debug!(?machine_id, "skip stale heartbeat reconcile request");
|
||||
return RoundStatus::Skip;
|
||||
}
|
||||
|
||||
let user_id = match storage
|
||||
.db
|
||||
.get_user_id_by_token(req.user_token.clone())
|
||||
.await
|
||||
{
|
||||
Ok(Some(user_id)) => user_id,
|
||||
Ok(None) => {
|
||||
tracing::info!("User not found by token: {:?}", req.user_token);
|
||||
return RoundStatus::Stop;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to get user id by token, error: {:?}", e);
|
||||
return RoundStatus::Stop;
|
||||
}
|
||||
};
|
||||
|
||||
let applied_config_revision = {
|
||||
let Some(data) = session_data.upgrade() else {
|
||||
return RoundStatus::Stop;
|
||||
};
|
||||
data.read().await.applied_config_revision.clone()
|
||||
};
|
||||
let target_config_revision = match storage
|
||||
.db
|
||||
.get_managed_config_revision((user_id, machine_id))
|
||||
.await
|
||||
{
|
||||
Ok(revision) => revision,
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to read managed config revision, error: {:?}", e);
|
||||
return RoundStatus::Stop;
|
||||
}
|
||||
};
|
||||
let should_apply_runtime_revision =
|
||||
target_config_revision.is_some() && target_config_revision != applied_config_revision;
|
||||
let running_inst_ids = match running_instance_ids_for_round(
|
||||
rpc_client,
|
||||
&req,
|
||||
user_id,
|
||||
machine_id,
|
||||
should_apply_runtime_revision,
|
||||
)
|
||||
.await
|
||||
{
|
||||
RoundStatus::Ready(ids) => ids,
|
||||
RoundStatus::Skip => return RoundStatus::Skip,
|
||||
RoundStatus::Stop => return RoundStatus::Stop,
|
||||
};
|
||||
|
||||
let local_configs = match storage
|
||||
.db
|
||||
.list_network_configs((user_id, machine_id), ListNetworkProps::EnabledOnly)
|
||||
.await
|
||||
{
|
||||
Ok(configs) => configs,
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to list network configs, error: {:?}", e);
|
||||
return RoundStatus::Stop;
|
||||
}
|
||||
};
|
||||
|
||||
RoundStatus::Ready(ReconcileRound {
|
||||
req,
|
||||
machine_id,
|
||||
user_id,
|
||||
running_inst_ids,
|
||||
local_configs,
|
||||
target_config_revision,
|
||||
should_apply_runtime_revision,
|
||||
})
|
||||
}
|
||||
|
||||
async fn running_instance_ids_for_round(
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
req: &HeartbeatRequest,
|
||||
user_id: i32,
|
||||
machine_id: uuid::Uuid,
|
||||
should_apply_runtime_revision: bool,
|
||||
) -> RoundStatus<HashSet<String>> {
|
||||
if !should_apply_runtime_revision {
|
||||
return RoundStatus::Ready(
|
||||
req.running_network_instances
|
||||
.iter()
|
||||
.map(|x| x.to_string())
|
||||
.collect(),
|
||||
);
|
||||
}
|
||||
|
||||
match rpc_client
|
||||
.list_network_instance(BaseController::default(), ListNetworkInstanceRequest {})
|
||||
.await
|
||||
{
|
||||
Ok(resp) => RoundStatus::Ready(resp.inst_ids.iter().map(|x| x.to_string()).collect()),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
?user_id,
|
||||
?machine_id,
|
||||
?error,
|
||||
"Failed to refresh running instances for managed config revision"
|
||||
);
|
||||
RoundStatus::Skip
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn sync_running_sources_for_round(
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
storage: &StorageInner,
|
||||
round: &mut ReconcileRound,
|
||||
) -> RoundStatus<Option<Vec<NetworkMeta>>> {
|
||||
if !round.req.support_config_source {
|
||||
return RoundStatus::Ready(None);
|
||||
}
|
||||
|
||||
let ret = if round.running_inst_ids.is_empty() {
|
||||
Ok(Vec::new())
|
||||
} else {
|
||||
rpc_client
|
||||
.list_network_instance_meta(
|
||||
BaseController::default(),
|
||||
ListNetworkInstanceMetaRequest {
|
||||
inst_ids: managed_config::parse_instance_ids(
|
||||
round.running_inst_ids.iter().cloned(),
|
||||
),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map(|resp| resp.metas)
|
||||
};
|
||||
|
||||
match ret {
|
||||
Ok(metas) => {
|
||||
if let Err(e) = managed_config::sync_running_config_sources(
|
||||
&storage.db,
|
||||
round.user_id,
|
||||
round.machine_id,
|
||||
&round.local_configs,
|
||||
&metas,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
user_id = ?round.user_id,
|
||||
machine_id = ?round.machine_id,
|
||||
%e,
|
||||
"Failed to sync running network config sources"
|
||||
);
|
||||
} else if !metas.is_empty() {
|
||||
round.local_configs = match storage
|
||||
.db
|
||||
.list_network_configs(
|
||||
(round.user_id, round.machine_id),
|
||||
ListNetworkProps::EnabledOnly,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(configs) => configs,
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
"Failed to reload network configs after source sync, error: {:?}",
|
||||
e
|
||||
);
|
||||
return RoundStatus::Stop;
|
||||
}
|
||||
};
|
||||
}
|
||||
RoundStatus::Ready(Some(metas))
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
user_id = ?round.user_id,
|
||||
%e,
|
||||
"Failed to list running network instance metadata"
|
||||
);
|
||||
RoundStatus::Ready(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_stale_web_source_instances(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
storage: &StorageInner,
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
round: &ReconcileRound,
|
||||
running_metas: Option<&[NetworkMeta]>,
|
||||
desired_web_inst_ids: &HashSet<String>,
|
||||
cache: &mut ReconcileCache,
|
||||
) -> RoundStatus<ReconcileOutcome> {
|
||||
let desired_changed = cache
|
||||
.last_desired_web_inst_ids
|
||||
.as_ref()
|
||||
.is_none_or(|last| last != desired_web_inst_ids);
|
||||
if cache.cleaned_web_source_instances && !desired_changed {
|
||||
return RoundStatus::Ready(ReconcileOutcome::default());
|
||||
}
|
||||
|
||||
let db_web_inst_ids = match storage
|
||||
.db
|
||||
.list_network_configs((round.user_id, round.machine_id), ListNetworkProps::All)
|
||||
.await
|
||||
{
|
||||
Ok(configs) => managed_config::desired_web_source_instance_ids(&configs),
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to list all network configs, error: {:?}", e);
|
||||
return RoundStatus::Stop;
|
||||
}
|
||||
};
|
||||
|
||||
let running_web_inst_ids = managed_config::running_web_source_instance_ids(
|
||||
&round.running_inst_ids,
|
||||
&db_web_inst_ids,
|
||||
running_metas,
|
||||
);
|
||||
let should_delete_inst_ids = running_web_inst_ids
|
||||
.difference(desired_web_inst_ids)
|
||||
.cloned()
|
||||
.collect::<HashSet<_>>();
|
||||
let should_delete_ids =
|
||||
managed_config::parse_instance_ids(should_delete_inst_ids.iter().cloned());
|
||||
|
||||
let mut outcome = ReconcileOutcome::default();
|
||||
if !should_delete_ids.is_empty() {
|
||||
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
|
||||
tracing::debug!(
|
||||
machine_id = ?round.machine_id,
|
||||
"skip stale cleanup because webhook session is no longer current"
|
||||
);
|
||||
return RoundStatus::Skip;
|
||||
}
|
||||
let ret = rpc_client
|
||||
.delete_network_instance(
|
||||
BaseController::default(),
|
||||
DeleteNetworkInstanceRequest {
|
||||
inst_ids: should_delete_ids,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
tracing::info!(
|
||||
user_id = ?round.user_id,
|
||||
"Clean stale web-source network instances on heartbeat: {:?}, user_token: {:?}",
|
||||
ret,
|
||||
round.req.user_token
|
||||
);
|
||||
if ret.is_err() {
|
||||
outcome.record_failure(true);
|
||||
} else {
|
||||
cache.runtime_configs.forget_many(&should_delete_inst_ids);
|
||||
}
|
||||
}
|
||||
|
||||
if !outcome.has_failed {
|
||||
cache.cleaned_web_source_instances = true;
|
||||
cache.last_desired_web_inst_ids = Some(desired_web_inst_ids.clone());
|
||||
}
|
||||
|
||||
RoundStatus::Ready(outcome)
|
||||
}
|
||||
|
||||
async fn reconcile_desired_runtime_configs(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
config_client: &mut SessionConfigClient,
|
||||
round: &ReconcileRound,
|
||||
cache: &mut ReconcileCache,
|
||||
) -> ReconcileOutcome {
|
||||
let mut outcome = ReconcileOutcome::default();
|
||||
|
||||
// After stale web-owned instances are removed, start every enabled
|
||||
// config that the latest heartbeat did not report as running. When
|
||||
// a managed config revision is pending, also reconcile running
|
||||
// web-owned configs before reporting that revision as applied.
|
||||
for config in &round.local_configs {
|
||||
let source = PersistedConfigSource::from_db(&config.source);
|
||||
let is_running = round.running_inst_ids.contains(&config.network_instance_id);
|
||||
let should_reconcile_running_web_config = is_running
|
||||
&& round.should_apply_runtime_revision
|
||||
&& source == PersistedConfigSource::Web;
|
||||
if is_running && !should_reconcile_running_web_config {
|
||||
continue;
|
||||
}
|
||||
|
||||
let desired_config = match serde_json::from_str::<NetworkConfig>(&config.network_config) {
|
||||
Ok(cfg) => cfg,
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
user_id = ?round.user_id,
|
||||
machine_id = ?round.machine_id,
|
||||
instance_id = %config.network_instance_id,
|
||||
"Failed to deserialize network config, skipping: {:?}",
|
||||
e
|
||||
);
|
||||
if source == PersistedConfigSource::Web {
|
||||
cache.runtime_configs.forget(&config.network_instance_id);
|
||||
}
|
||||
outcome.record_failure(source == PersistedConfigSource::Web);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let action_result = if should_reconcile_running_web_config {
|
||||
reconcile_running_web_config(
|
||||
session_data,
|
||||
rpc_client,
|
||||
config_client,
|
||||
round,
|
||||
config,
|
||||
desired_config,
|
||||
&mut cache.runtime_configs,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
if source == PersistedConfigSource::Web {
|
||||
cache.runtime_configs.forget(&config.network_instance_id);
|
||||
}
|
||||
let action_result = run_missing_network_config(
|
||||
session_data,
|
||||
rpc_client,
|
||||
round,
|
||||
config,
|
||||
desired_config.clone(),
|
||||
)
|
||||
.await;
|
||||
if matches!(action_result, ConfigActionResult::Success)
|
||||
&& source == PersistedConfigSource::Web
|
||||
{
|
||||
if let Err(e) = remember_web_runtime_config_after_run(
|
||||
rpc_client,
|
||||
&config.network_instance_id,
|
||||
&desired_config,
|
||||
&mut cache.runtime_configs,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::error!(
|
||||
user_id = ?round.user_id,
|
||||
machine_id = ?round.machine_id,
|
||||
instance_id = %config.network_instance_id,
|
||||
"Failed to cache runtime config after run: {:?}",
|
||||
e
|
||||
);
|
||||
ConfigActionResult::Failed
|
||||
} else {
|
||||
action_result
|
||||
}
|
||||
} else {
|
||||
action_result
|
||||
}
|
||||
};
|
||||
|
||||
match action_result {
|
||||
ConfigActionResult::Success => {}
|
||||
ConfigActionResult::Failed => {
|
||||
if source == PersistedConfigSource::Web {
|
||||
cache.runtime_configs.forget(&config.network_instance_id);
|
||||
}
|
||||
outcome.record_failure(source == PersistedConfigSource::Web)
|
||||
}
|
||||
ConfigActionResult::StopRound => {
|
||||
if source == PersistedConfigSource::Web {
|
||||
cache.runtime_configs.forget(&config.network_instance_id);
|
||||
}
|
||||
outcome.record_failure(source == PersistedConfigSource::Web);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
outcome
|
||||
}
|
||||
|
||||
async fn reconcile_running_web_config(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
config_client: &mut SessionConfigClient,
|
||||
round: &ReconcileRound,
|
||||
config: &crate::db::entity::user_running_network_configs::Model,
|
||||
desired_config: NetworkConfig,
|
||||
runtime_config_cache: &mut SessionRuntimeConfigCache,
|
||||
) -> ConfigActionResult {
|
||||
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
|
||||
tracing::debug!(
|
||||
machine_id = ?round.machine_id,
|
||||
instance_id = %config.network_instance_id,
|
||||
"skip runtime reconcile because webhook session is no longer current"
|
||||
);
|
||||
return ConfigActionResult::StopRound;
|
||||
}
|
||||
|
||||
let ret = async {
|
||||
let action =
|
||||
match runtime_config_cache.plan(&config.network_instance_id, desired_config.clone())? {
|
||||
Some(action) => action,
|
||||
None => {
|
||||
runtime_reconcile::prepare_web_source_runtime_reconcile(
|
||||
&mut *rpc_client,
|
||||
&config.network_instance_id,
|
||||
desired_config.clone(),
|
||||
true,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
|
||||
anyhow::bail!("webhook session is no longer current before runtime reconcile apply");
|
||||
}
|
||||
let observed_config = runtime_reconcile::apply_web_source_runtime_reconcile(
|
||||
&mut *rpc_client,
|
||||
&mut *config_client,
|
||||
&config.network_instance_id,
|
||||
desired_config.clone(),
|
||||
action,
|
||||
)
|
||||
.await?;
|
||||
runtime_config_cache.remember(&config.network_instance_id, observed_config);
|
||||
Ok::<(), anyhow::Error>(())
|
||||
}
|
||||
.await;
|
||||
tracing::info!(
|
||||
user_id = ?round.user_id,
|
||||
instance_id = %config.network_instance_id,
|
||||
"Reconcile running web-source network instance: {:?}, user_token: {:?}",
|
||||
ret,
|
||||
round.req.user_token
|
||||
);
|
||||
|
||||
if ret.is_ok() {
|
||||
ConfigActionResult::Success
|
||||
} else {
|
||||
runtime_config_cache.forget(&config.network_instance_id);
|
||||
ConfigActionResult::Failed
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_missing_network_config(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
round: &ReconcileRound,
|
||||
config: &crate::db::entity::user_running_network_configs::Model,
|
||||
desired_config: NetworkConfig,
|
||||
) -> ConfigActionResult {
|
||||
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
|
||||
tracing::debug!(
|
||||
machine_id = ?round.machine_id,
|
||||
instance_id = %config.network_instance_id,
|
||||
"skip run network instance because webhook session is no longer current"
|
||||
);
|
||||
return ConfigActionResult::StopRound;
|
||||
}
|
||||
|
||||
let ret = rpc_client
|
||||
.run_network_instance(
|
||||
BaseController::default(),
|
||||
RunNetworkInstanceRequest {
|
||||
inst_id: Some(config.network_instance_id.clone().into()),
|
||||
config: Some(desired_config),
|
||||
overwrite: false,
|
||||
source: PersistedConfigSource::from_db(&config.source).auto_run_rpc_source() as i32,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
tracing::info!(
|
||||
user_id = ?round.user_id,
|
||||
"Run network instance: {:?}, user_token: {:?}",
|
||||
ret,
|
||||
round.req.user_token
|
||||
);
|
||||
|
||||
if ret.is_ok() {
|
||||
ConfigActionResult::Success
|
||||
} else {
|
||||
ConfigActionResult::Failed
|
||||
}
|
||||
}
|
||||
|
||||
async fn remember_web_runtime_config_after_run(
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
inst_id: &str,
|
||||
desired_config: &NetworkConfig,
|
||||
runtime_config_cache: &mut SessionRuntimeConfigCache,
|
||||
) -> anyhow::Result<()> {
|
||||
let observed_config = runtime_reconcile::get_runtime_config(rpc_client, inst_id).await?;
|
||||
remember_if_runtime_matches_desired(
|
||||
inst_id,
|
||||
desired_config,
|
||||
observed_config,
|
||||
runtime_config_cache,
|
||||
)
|
||||
}
|
||||
|
||||
fn remember_if_runtime_matches_desired(
|
||||
inst_id: &str,
|
||||
desired_config: &NetworkConfig,
|
||||
observed_config: NetworkConfig,
|
||||
runtime_config_cache: &mut SessionRuntimeConfigCache,
|
||||
) -> anyhow::Result<()> {
|
||||
let action = runtime_reconcile::prepare_web_source_runtime_reconcile_from_current(
|
||||
&observed_config,
|
||||
desired_config.clone(),
|
||||
)?;
|
||||
if !matches!(action, runtime_reconcile::RuntimeReconcileAction::None) {
|
||||
anyhow::bail!("runtime config still differs after managed run");
|
||||
}
|
||||
runtime_config_cache.remember(inst_id, observed_config);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn mark_config_revision_applied_if_current(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
storage: &StorageInner,
|
||||
round: &ReconcileRound,
|
||||
outcome: &ReconcileOutcome,
|
||||
) -> RoundStatus<()> {
|
||||
if outcome.managed_revision_failed || !round.should_apply_runtime_revision {
|
||||
return RoundStatus::Ready(());
|
||||
}
|
||||
|
||||
let current_target_config_revision = match storage
|
||||
.db
|
||||
.get_managed_config_revision((round.user_id, round.machine_id))
|
||||
.await
|
||||
{
|
||||
Ok(revision) => revision,
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to verify managed config revision, error: {:?}", e);
|
||||
return RoundStatus::Stop;
|
||||
}
|
||||
};
|
||||
if current_target_config_revision != round.target_config_revision {
|
||||
return RoundStatus::Ready(());
|
||||
}
|
||||
let Some(data) = session_data.upgrade() else {
|
||||
return RoundStatus::Stop;
|
||||
};
|
||||
let mut data = data.write().await;
|
||||
if !SessionRpcService::runtime_heartbeat_is_current_locked(&data, &round.req) {
|
||||
return RoundStatus::Ready(());
|
||||
}
|
||||
data.applied_config_revision = round.target_config_revision.clone();
|
||||
|
||||
RoundStatus::Ready(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use easytier::proto::api::manage::{NetworkingMethod, PortForwardConfig};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn config_with_port_forwards(port_forwards: Vec<PortForwardConfig>) -> NetworkConfig {
|
||||
NetworkConfig {
|
||||
instance_id: Some("11111111-1111-1111-1111-111111111111".to_string()),
|
||||
dhcp: Some(true),
|
||||
network_name: Some("managed".to_string()),
|
||||
network_secret: Some("secret".to_string()),
|
||||
networking_method: Some(NetworkingMethod::Manual as i32),
|
||||
port_forwards,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn port_forward(bind_port: u32, dst_port: u32) -> PortForwardConfig {
|
||||
PortForwardConfig {
|
||||
bind_ip: "127.0.0.1".to_string(),
|
||||
bind_port,
|
||||
dst_ip: "10.144.0.1".to_string(),
|
||||
dst_port,
|
||||
proto: "tcp".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_runtime_config_cache_misses_unknown_instance() {
|
||||
let cache = SessionRuntimeConfigCache::default();
|
||||
let action = cache
|
||||
.plan("missing", config_with_port_forwards(Vec::new()))
|
||||
.expect("prepare action");
|
||||
|
||||
assert!(action.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_runtime_config_cache_skips_matching_observed_config() {
|
||||
let mut cache = SessionRuntimeConfigCache::default();
|
||||
let config = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
|
||||
cache.remember("managed", config.clone());
|
||||
let action = cache
|
||||
.plan("managed", config)
|
||||
.expect("prepare action")
|
||||
.expect("cached action");
|
||||
|
||||
assert!(matches!(
|
||||
action,
|
||||
runtime_reconcile::RuntimeReconcileAction::None
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_runtime_config_cache_plans_patch_from_observed_config() {
|
||||
let mut cache = SessionRuntimeConfigCache::default();
|
||||
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
let desired =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
|
||||
cache.remember("managed", current);
|
||||
let action = cache
|
||||
.plan("managed", desired)
|
||||
.expect("prepare action")
|
||||
.expect("cached action");
|
||||
|
||||
let runtime_reconcile::RuntimeReconcileAction::Patch(patch) = action else {
|
||||
panic!("expected cached runtime config to produce hot patch");
|
||||
};
|
||||
assert_eq!(patch.port_forwards.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_runtime_config_cache_retain_desired_removes_stale_entries() {
|
||||
let mut cache = SessionRuntimeConfigCache::default();
|
||||
let config = config_with_port_forwards(Vec::new());
|
||||
cache.remember("keep", config.clone());
|
||||
cache.remember("drop", config);
|
||||
|
||||
cache.retain_desired(&HashSet::from(["keep".to_string()]));
|
||||
|
||||
assert!(cache.entries.contains_key("keep"));
|
||||
assert!(!cache.entries.contains_key("drop"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_runtime_config_cache_forget_removes_observed_config() {
|
||||
let mut cache = SessionRuntimeConfigCache::default();
|
||||
let config = config_with_port_forwards(Vec::new());
|
||||
cache.remember("managed", config.clone());
|
||||
|
||||
cache.forget("managed");
|
||||
|
||||
let action = cache
|
||||
.plan("managed", config)
|
||||
.expect("prepare action after remove");
|
||||
assert!(action.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_run_remembers_observed_config_when_it_matches_desired() {
|
||||
let mut cache = SessionRuntimeConfigCache::default();
|
||||
let config = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
|
||||
remember_if_runtime_matches_desired("managed", &config, config.clone(), &mut cache)
|
||||
.expect("remember observed config after run");
|
||||
let action = cache
|
||||
.plan("managed", config)
|
||||
.expect("prepare action after run")
|
||||
.expect("cached action");
|
||||
|
||||
assert!(matches!(
|
||||
action,
|
||||
runtime_reconcile::RuntimeReconcileAction::None
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_run_does_not_remember_observed_config_that_still_differs() {
|
||||
let mut cache = SessionRuntimeConfigCache::default();
|
||||
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
let desired =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
|
||||
let err = remember_if_runtime_matches_desired("managed", &desired, current, &mut cache)
|
||||
.expect_err("expected stale run result not to be cached");
|
||||
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("runtime config still differs after managed run")
|
||||
);
|
||||
let action = cache
|
||||
.plan("managed", desired)
|
||||
.expect("prepare action after stale run result");
|
||||
assert!(action.is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use anyhow::Context as _;
|
||||
use easytier::proto::web::HeartbeatRequest;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use super::{
|
||||
SessionAuthState, SessionData, SessionRpcService, WebhookConnectNotification,
|
||||
WebhookDisconnectNotification, send_webhook_connection_transition,
|
||||
};
|
||||
use crate::{
|
||||
client_manager::storage::{Storage, StorageToken},
|
||||
webhook::SharedWebhookConfig,
|
||||
};
|
||||
|
||||
pub(super) const VALIDATION_RETRY_MS: u64 = 60_000;
|
||||
|
||||
pub(super) struct WebhookHeartbeatValidation {
|
||||
pub(super) config_revision: String,
|
||||
pub(super) binding_version: u64,
|
||||
}
|
||||
|
||||
pub(super) struct WebhookValidationInput {
|
||||
pub(super) storage: Storage,
|
||||
pub(super) webhook_config: SharedWebhookConfig,
|
||||
pub(super) client_url: url::Url,
|
||||
pub(super) applied_config_revision: Option<String>,
|
||||
pub(super) req: HeartbeatRequest,
|
||||
pub(super) machine_id: uuid::Uuid,
|
||||
}
|
||||
|
||||
fn deterministic_machine_delay(machine_id: uuid::Uuid, max_delay_ms: u64) -> Duration {
|
||||
let delay_ms = (machine_id.as_u128() % u128::from(max_delay_ms + 1)) as u64;
|
||||
Duration::from_millis(delay_ms)
|
||||
}
|
||||
|
||||
pub(super) fn retry_delay(machine_id: uuid::Uuid) -> Duration {
|
||||
Duration::from_millis(VALIDATION_RETRY_MS)
|
||||
+ deterministic_machine_delay(machine_id, VALIDATION_RETRY_MS)
|
||||
}
|
||||
|
||||
async fn request_heartbeat_validation(
|
||||
webhook_config: &crate::webhook::WebhookConfig,
|
||||
client_url: &url::Url,
|
||||
persisted_config_revision: Option<&str>,
|
||||
applied_config_revision: Option<&str>,
|
||||
req: &HeartbeatRequest,
|
||||
machine_id: uuid::Uuid,
|
||||
) -> anyhow::Result<Option<WebhookHeartbeatValidation>> {
|
||||
let webhook_req = crate::webhook::ValidateTokenRequest {
|
||||
token: req.user_token.clone(),
|
||||
machine_id: machine_id.to_string(),
|
||||
public_ip: client_url.host_str().map(str::to_string),
|
||||
hostname: req.hostname.clone(),
|
||||
version: req.easytier_version.clone(),
|
||||
os_type: req.device_os.as_ref().map(|info| info.os_type.clone()),
|
||||
os_version: req.device_os.as_ref().map(|info| info.version.clone()),
|
||||
os_distribution: req.device_os.as_ref().map(|info| info.distribution.clone()),
|
||||
web_instance_id: webhook_config.web_instance_id.clone(),
|
||||
web_instance_api_base_url: webhook_config.web_instance_api_base_url.clone(),
|
||||
persisted_config_revision: persisted_config_revision.map(str::to_string),
|
||||
applied_config_revision: applied_config_revision.map(str::to_string),
|
||||
};
|
||||
let resp = webhook_config
|
||||
.validate_token(&webhook_req)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Webhook token validation failed: {:?}", e))?;
|
||||
|
||||
if !resp.valid {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
Ok(Some(WebhookHeartbeatValidation {
|
||||
config_revision: resp.config_revision,
|
||||
binding_version: resp.binding_version,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn resolve_user_id(storage: &Storage, token: &str) -> anyhow::Result<i32> {
|
||||
let user_id = match storage
|
||||
.db()
|
||||
.get_user_id_by_token(token)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("DB error: {:?}", e))?
|
||||
{
|
||||
Some(id) => id,
|
||||
None => storage
|
||||
.auto_create_user(token)
|
||||
.await
|
||||
.with_context(|| format!("Failed to auto-create webhook user: {:?}", token))?,
|
||||
};
|
||||
|
||||
Ok(user_id)
|
||||
}
|
||||
|
||||
async fn persisted_config_revision_for_token(
|
||||
storage: &Storage,
|
||||
token: &str,
|
||||
machine_id: uuid::Uuid,
|
||||
) -> anyhow::Result<Option<String>> {
|
||||
let Some(user_id) = storage
|
||||
.db()
|
||||
.get_user_id_by_token(token)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("DB error: {:?}", e))?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
storage
|
||||
.db()
|
||||
.get_managed_config_revision((user_id, machine_id))
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("DB error: {:?}", e))
|
||||
}
|
||||
|
||||
async fn wait_for_input(
|
||||
session_data: std::sync::Weak<RwLock<SessionData>>,
|
||||
) -> Option<WebhookValidationInput> {
|
||||
loop {
|
||||
let notify = {
|
||||
let session_data = session_data.upgrade()?;
|
||||
let mut data = session_data.write().await;
|
||||
if matches!(data.auth_state, SessionAuthState::Invalid) {
|
||||
data.webhook_validation_dirty = false;
|
||||
tracing::info!(
|
||||
client_url = %data.client_url,
|
||||
"webhook validation stopped for invalid session; reconnect is required before revalidation"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
if data.webhook_validation_dirty {
|
||||
data.webhook_validation_dirty = false;
|
||||
let req = data.req.clone()?;
|
||||
let machine_id = req.machine_id.map(Into::into)?;
|
||||
let storage = Storage::try_from(data.storage.clone()).ok()?;
|
||||
return Some(WebhookValidationInput {
|
||||
storage,
|
||||
webhook_config: data.webhook_config.clone(),
|
||||
client_url: data.client_url.clone(),
|
||||
applied_config_revision: data.applied_config_revision.clone(),
|
||||
req,
|
||||
machine_id,
|
||||
});
|
||||
}
|
||||
data.webhook_validation_notify.clone()
|
||||
};
|
||||
notify.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn run_worker(session_data: std::sync::Weak<RwLock<SessionData>>) {
|
||||
while let Some(input) = wait_for_input(session_data.clone()).await {
|
||||
let machine_id = input.machine_id;
|
||||
if let Err(error) = run_round(session_data.clone(), input).await {
|
||||
tracing::warn!(
|
||||
?machine_id,
|
||||
%error,
|
||||
"webhook validation failed, will retry later"
|
||||
);
|
||||
tokio::time::sleep(retry_delay(machine_id)).await;
|
||||
mark_dirty_if_current(&session_data, machine_id).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn run_round(
|
||||
session_data: std::sync::Weak<RwLock<SessionData>>,
|
||||
input: WebhookValidationInput,
|
||||
) -> anyhow::Result<()> {
|
||||
let persisted_config_revision = persisted_config_revision_for_token(
|
||||
&input.storage,
|
||||
&input.req.user_token,
|
||||
input.machine_id,
|
||||
)
|
||||
.await?;
|
||||
let validation = request_heartbeat_validation(
|
||||
&input.webhook_config,
|
||||
&input.client_url,
|
||||
persisted_config_revision.as_deref(),
|
||||
input.applied_config_revision.as_deref(),
|
||||
&input.req,
|
||||
input.machine_id,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let Some(validation) = validation else {
|
||||
apply_rejected(&session_data, &input).await;
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let user_id = resolve_user_id(&input.storage, &input.req.user_token).await?;
|
||||
apply_success(&session_data, input, validation, user_id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn mark_dirty_if_current(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
machine_id: uuid::Uuid,
|
||||
) {
|
||||
let Some(session_data) = session_data.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let notify = {
|
||||
let mut data = session_data.write().await;
|
||||
let Some(req) = data.req.as_ref() else {
|
||||
return;
|
||||
};
|
||||
if req.machine_id.map(uuid::Uuid::from) != Some(machine_id) {
|
||||
return;
|
||||
}
|
||||
if matches!(data.auth_state, SessionAuthState::Invalid) {
|
||||
data.webhook_validation_dirty = false;
|
||||
tracing::debug!(
|
||||
%machine_id,
|
||||
"skip webhook validation retry for invalid session"
|
||||
);
|
||||
return;
|
||||
}
|
||||
SessionRpcService::mark_webhook_validation_dirty_locked(&mut data)
|
||||
};
|
||||
notify.notify_one();
|
||||
}
|
||||
|
||||
pub(super) async fn apply_rejected(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
input: &WebhookValidationInput,
|
||||
) {
|
||||
let Some(session_data) = session_data.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let (storage_token, disconnect_notification) = {
|
||||
let mut data = session_data.write().await;
|
||||
if !data.req.as_ref().is_some_and(|req| {
|
||||
SessionRpcService::heartbeat_matches_identity(
|
||||
req,
|
||||
&input.req.user_token,
|
||||
input.machine_id,
|
||||
)
|
||||
}) {
|
||||
return;
|
||||
}
|
||||
tracing::info!(
|
||||
machine_id = %input.machine_id,
|
||||
client_url = %data.client_url,
|
||||
"webhook token rejected; marking session invalid and requiring client reconnect"
|
||||
);
|
||||
data.auth_state = SessionAuthState::Invalid;
|
||||
data.webhook_validation_dirty = false;
|
||||
data.binding_version = None;
|
||||
data.applied_config_revision = None;
|
||||
let storage_token = data.storage_token.clone();
|
||||
let disconnect_notification = storage_token.as_ref().and_then(|storage_token| {
|
||||
data.webhook_connected_binding_version
|
||||
.take()
|
||||
.map(|binding_version| WebhookDisconnectNotification {
|
||||
webhook: data.webhook_config.clone(),
|
||||
storage_token: storage_token.clone(),
|
||||
binding_version,
|
||||
})
|
||||
});
|
||||
(storage_token, disconnect_notification)
|
||||
};
|
||||
if let Some(storage_token) = storage_token {
|
||||
let report_time = SessionRpcService::heartbeat_report_timestamp(&input.req);
|
||||
input
|
||||
.storage
|
||||
.update_client(storage_token, report_time, false);
|
||||
}
|
||||
if disconnect_notification.is_some() {
|
||||
wait_webhook_connection_transition(
|
||||
Arc::downgrade(&session_data),
|
||||
disconnect_notification,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn apply_success(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
input: WebhookValidationInput,
|
||||
validation: WebhookHeartbeatValidation,
|
||||
user_id: i32,
|
||||
) {
|
||||
let WebhookHeartbeatValidation {
|
||||
config_revision: _,
|
||||
binding_version,
|
||||
} = validation;
|
||||
|
||||
let Some(session_data) = session_data.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let (storage_token, notifier, disconnect_notification, connect_notification, runtime_req) = {
|
||||
let mut data = session_data.write().await;
|
||||
let Some(runtime_req) = data.req.clone() else {
|
||||
return;
|
||||
};
|
||||
if !SessionRpcService::heartbeat_matches_identity(
|
||||
&runtime_req,
|
||||
&input.req.user_token,
|
||||
input.machine_id,
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if matches!(data.auth_state, SessionAuthState::Invalid) {
|
||||
tracing::info!(
|
||||
machine_id = %input.machine_id,
|
||||
client_url = %data.client_url,
|
||||
"ignore webhook validation success for invalid session; reconnect is required before revalidation"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let previous_connected_binding_version = data.webhook_connected_binding_version;
|
||||
let client_url = data.client_url.clone();
|
||||
let storage_token = data.storage_token.get_or_insert_with(|| StorageToken {
|
||||
token: runtime_req.user_token.clone(),
|
||||
client_url,
|
||||
machine_id: input.machine_id,
|
||||
user_id,
|
||||
});
|
||||
let storage_token = storage_token.clone();
|
||||
data.auth_state = SessionAuthState::Authorized;
|
||||
data.binding_version = Some(binding_version);
|
||||
let should_notify_connected = previous_connected_binding_version != Some(binding_version);
|
||||
let disconnect_notification = previous_connected_binding_version
|
||||
.filter(|previous_binding_version| *previous_binding_version != binding_version)
|
||||
.map(|previous_binding_version| {
|
||||
data.webhook_connected_binding_version = None;
|
||||
WebhookDisconnectNotification {
|
||||
webhook: data.webhook_config.clone(),
|
||||
storage_token: storage_token.clone(),
|
||||
binding_version: previous_binding_version,
|
||||
}
|
||||
});
|
||||
|
||||
let connect_notification = should_notify_connected.then(|| WebhookConnectNotification {
|
||||
webhook: data.webhook_config.clone(),
|
||||
storage_token: storage_token.clone(),
|
||||
binding_version,
|
||||
req: crate::webhook::NodeConnectedRequest {
|
||||
machine_id: input.machine_id.to_string(),
|
||||
token: runtime_req.user_token.clone(),
|
||||
user_id: Some(user_id),
|
||||
hostname: runtime_req.hostname.clone(),
|
||||
version: runtime_req.easytier_version.clone(),
|
||||
os_type: runtime_req
|
||||
.device_os
|
||||
.as_ref()
|
||||
.map(|info| info.os_type.clone()),
|
||||
os_version: runtime_req
|
||||
.device_os
|
||||
.as_ref()
|
||||
.map(|info| info.version.clone()),
|
||||
os_distribution: runtime_req
|
||||
.device_os
|
||||
.as_ref()
|
||||
.map(|info| info.distribution.clone()),
|
||||
web_instance_id: data.webhook_config.web_instance_id.clone(),
|
||||
binding_version: Some(binding_version),
|
||||
},
|
||||
});
|
||||
|
||||
(
|
||||
storage_token,
|
||||
data.notifier.clone(),
|
||||
disconnect_notification,
|
||||
connect_notification,
|
||||
runtime_req,
|
||||
)
|
||||
};
|
||||
|
||||
if disconnect_notification.is_some() || connect_notification.is_some() {
|
||||
wait_webhook_connection_transition(
|
||||
Arc::downgrade(&session_data),
|
||||
disconnect_notification,
|
||||
connect_notification,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let report_time = SessionRpcService::heartbeat_report_timestamp(&runtime_req);
|
||||
input
|
||||
.storage
|
||||
.update_client(storage_token, report_time, true);
|
||||
let _ = notifier.send(runtime_req);
|
||||
}
|
||||
|
||||
async fn wait_webhook_connection_transition(
|
||||
session_data: std::sync::Weak<RwLock<SessionData>>,
|
||||
disconnect: Option<WebhookDisconnectNotification>,
|
||||
connect: Option<WebhookConnectNotification>,
|
||||
) {
|
||||
let transition = tokio::spawn(send_webhook_connection_transition(
|
||||
session_data,
|
||||
disconnect,
|
||||
connect,
|
||||
));
|
||||
if let Err(error) = transition.await {
|
||||
tracing::warn!(%error, "webhook connection transition task failed");
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ pub struct StorageToken {
|
||||
struct ClientInfo {
|
||||
storage_token: StorageToken,
|
||||
report_time: i64,
|
||||
authorized: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -55,7 +56,19 @@ impl Storage {
|
||||
fn update_client_info_map(map: &DashMap<uuid::Uuid, ClientInfo>, client_info: &ClientInfo) {
|
||||
map.entry(client_info.storage_token.machine_id)
|
||||
.and_modify(|e| {
|
||||
if e.report_time < client_info.report_time {
|
||||
let same_client = e.storage_token.client_url
|
||||
== client_info.storage_token.client_url
|
||||
&& e.storage_token.user_id == client_info.storage_token.user_id;
|
||||
let should_replace = if (same_client && e.authorized != client_info.authorized)
|
||||
|| (!e.authorized && client_info.authorized)
|
||||
{
|
||||
true
|
||||
} else if e.authorized && !client_info.authorized && !same_client {
|
||||
false
|
||||
} else {
|
||||
e.report_time < client_info.report_time
|
||||
};
|
||||
if should_replace {
|
||||
assert_eq!(
|
||||
e.storage_token.machine_id,
|
||||
client_info.storage_token.machine_id
|
||||
@@ -66,12 +79,13 @@ impl Storage {
|
||||
.or_insert(client_info.clone());
|
||||
}
|
||||
|
||||
pub fn update_client(&self, stoken: StorageToken, report_time: i64) {
|
||||
pub fn update_client(&self, stoken: StorageToken, report_time: i64, authorized: bool) {
|
||||
let inner = self.0.user_clients_map.entry(stoken.user_id).or_default();
|
||||
|
||||
let client_info = ClientInfo {
|
||||
storage_token: stoken.clone(),
|
||||
report_time,
|
||||
authorized,
|
||||
};
|
||||
Self::update_client_info_map(&inner, &client_info);
|
||||
}
|
||||
@@ -93,11 +107,21 @@ impl Storage {
|
||||
&self,
|
||||
user_id: UserIdInDb,
|
||||
machine_id: &uuid::Uuid,
|
||||
) -> Option<url::Url> {
|
||||
self.get_client_url_by_machine_id_with_auth(user_id, machine_id, true)
|
||||
}
|
||||
|
||||
pub fn get_client_url_by_machine_id_with_auth(
|
||||
&self,
|
||||
user_id: UserIdInDb,
|
||||
machine_id: &uuid::Uuid,
|
||||
require_authorized: bool,
|
||||
) -> Option<url::Url> {
|
||||
self.0.user_clients_map.get(&user_id).and_then(|info_map| {
|
||||
info_map
|
||||
.get(machine_id)
|
||||
.map(|info| info.storage_token.client_url.clone())
|
||||
info_map.get(machine_id).and_then(|info| {
|
||||
(!require_authorized || info.authorized)
|
||||
.then(|| info.storage_token.client_url.clone())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
@@ -108,6 +132,7 @@ impl Storage {
|
||||
.map(|info_map| {
|
||||
info_map
|
||||
.iter()
|
||||
.filter(|info| info.value().authorized)
|
||||
.map(|info| info.value().storage_token.client_url.clone())
|
||||
.collect()
|
||||
})
|
||||
@@ -115,6 +140,14 @@ impl Storage {
|
||||
}
|
||||
|
||||
pub fn list_clients(&self) -> Vec<StorageToken> {
|
||||
self.list_clients_with_auth(true)
|
||||
}
|
||||
|
||||
pub fn list_all_clients(&self) -> Vec<StorageToken> {
|
||||
self.list_clients_with_auth(false)
|
||||
}
|
||||
|
||||
fn list_clients_with_auth(&self, require_authorized: bool) -> Vec<StorageToken> {
|
||||
self.0
|
||||
.user_clients_map
|
||||
.iter()
|
||||
@@ -122,6 +155,7 @@ impl Storage {
|
||||
user_clients
|
||||
.value()
|
||||
.iter()
|
||||
.filter(|info| !require_authorized || info.value().authorized)
|
||||
.map(|info| info.value().storage_token.clone())
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
@@ -164,8 +198,8 @@ mod tests {
|
||||
let user1_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||
let user2_token = make_storage_token(2, machine_id, "tcp://127.0.0.1:1002");
|
||||
|
||||
storage.update_client(user1_token.clone(), 10);
|
||||
storage.update_client(user2_token.clone(), 20);
|
||||
storage.update_client(user1_token.clone(), 10, true);
|
||||
storage.update_client(user2_token.clone(), 20, true);
|
||||
|
||||
assert_eq!(
|
||||
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||
@@ -195,8 +229,8 @@ mod tests {
|
||||
let user1_token = make_storage_token(1, uuid::Uuid::new_v4(), "tcp://127.0.0.1:1001");
|
||||
let user2_token = make_storage_token(2, uuid::Uuid::new_v4(), "tcp://127.0.0.1:1002");
|
||||
|
||||
storage.update_client(user1_token.clone(), 10);
|
||||
storage.update_client(user2_token.clone(), 20);
|
||||
storage.update_client(user1_token.clone(), 10, true);
|
||||
storage.update_client(user2_token.clone(), 20, true);
|
||||
|
||||
let tokens = storage.list_clients();
|
||||
assert_eq!(tokens.len(), 2);
|
||||
@@ -209,4 +243,84 @@ mod tests {
|
||||
assert_eq!(tokens.len(), 1);
|
||||
assert_eq!(tokens[0].token, user2_token.token);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pending_client_is_listed_but_not_authorized_for_machine_lookup() {
|
||||
let storage = Storage::new(Db::memory_db().await);
|
||||
let machine_id = uuid::Uuid::new_v4();
|
||||
let token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||
|
||||
storage.update_client(token.clone(), 10, false);
|
||||
|
||||
assert_eq!(storage.list_clients().len(), 0);
|
||||
assert_eq!(storage.list_all_clients().len(), 1);
|
||||
assert_eq!(storage.list_user_clients(1), Vec::<url::Url>::new());
|
||||
assert_eq!(storage.get_client_url_by_machine_id(1, &machine_id), None);
|
||||
assert_eq!(
|
||||
storage.get_client_url_by_machine_id_with_auth(1, &machine_id, false),
|
||||
Some(token.client_url.clone())
|
||||
);
|
||||
|
||||
storage.update_client(token.clone(), 11, true);
|
||||
|
||||
assert_eq!(
|
||||
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||
Some(token.client_url.clone())
|
||||
);
|
||||
|
||||
storage.update_client(token.clone(), 11, false);
|
||||
|
||||
assert_eq!(storage.get_client_url_by_machine_id(1, &machine_id), None);
|
||||
assert_eq!(storage.list_clients().len(), 0);
|
||||
assert_eq!(storage.list_all_clients().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_client_authorization_update_does_not_replace_newer_client() {
|
||||
let storage = Storage::new(Db::memory_db().await);
|
||||
let machine_id = uuid::Uuid::new_v4();
|
||||
let old_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||
let new_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002");
|
||||
|
||||
storage.update_client(old_token.clone(), 10, true);
|
||||
storage.update_client(new_token.clone(), 20, true);
|
||||
storage.update_client(old_token, 10, false);
|
||||
|
||||
assert_eq!(
|
||||
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||
Some(new_token.client_url)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pending_client_does_not_replace_authorized_route() {
|
||||
let storage = Storage::new(Db::memory_db().await);
|
||||
let machine_id = uuid::Uuid::new_v4();
|
||||
let authorized_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||
let pending_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002");
|
||||
|
||||
storage.update_client(authorized_token.clone(), 10, true);
|
||||
storage.update_client(pending_token, i64::MAX, false);
|
||||
|
||||
assert_eq!(
|
||||
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||
Some(authorized_token.client_url)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authorized_client_replaces_pending_route_regardless_of_report_time() {
|
||||
let storage = Storage::new(Db::memory_db().await);
|
||||
let machine_id = uuid::Uuid::new_v4();
|
||||
let pending_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||
let authorized_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002");
|
||||
|
||||
storage.update_client(pending_token, i64::MAX, false);
|
||||
storage.update_client(authorized_token.clone(), 10, true);
|
||||
|
||||
assert_eq!(
|
||||
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||
Some(authorized_token.client_url)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
//! `SeaORM` Entity, hand-written to match the generated entity style.
|
||||
|
||||
use sea_orm::entity::prelude::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
|
||||
#[sea_orm(table_name = "managed_config_revisions")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key)]
|
||||
pub id: i32,
|
||||
pub user_id: i32,
|
||||
#[sea_orm(column_type = "Text")]
|
||||
pub device_id: String,
|
||||
#[sea_orm(column_type = "Text")]
|
||||
pub config_revision: String,
|
||||
pub create_time: DateTimeWithTimeZone,
|
||||
pub update_time: DateTimeWithTimeZone,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {
|
||||
#[sea_orm(
|
||||
belongs_to = "super::users::Entity",
|
||||
from = "Column::UserId",
|
||||
to = "super::users::Column::Id",
|
||||
on_update = "Cascade",
|
||||
on_delete = "Cascade"
|
||||
)]
|
||||
Users,
|
||||
}
|
||||
|
||||
impl Related<super::users::Entity> for Entity {
|
||||
fn to() -> RelationDef {
|
||||
Relation::Users.def()
|
||||
}
|
||||
}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
@@ -4,6 +4,7 @@ pub mod prelude;
|
||||
|
||||
pub mod groups;
|
||||
pub mod groups_permissions;
|
||||
pub mod managed_config_revisions;
|
||||
pub mod permissions;
|
||||
pub mod tower_sessions;
|
||||
pub mod user_running_network_configs;
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
pub use super::groups::Entity as Groups;
|
||||
pub use super::groups_permissions::Entity as GroupsPermissions;
|
||||
pub use super::managed_config_revisions::Entity as ManagedConfigRevisions;
|
||||
pub use super::permissions::Entity as Permissions;
|
||||
pub use super::tower_sessions::Entity as TowerSessions;
|
||||
pub use super::user_running_network_configs::Entity as UserRunningNetworkConfigs;
|
||||
|
||||
@@ -141,6 +141,110 @@ impl Db {
|
||||
) -> Result<Option<UserIdInDb>, DbErr> {
|
||||
self.get_user_id(token).await
|
||||
}
|
||||
|
||||
pub async fn get_managed_config_revision(
|
||||
&self,
|
||||
(user_id, device_id): (UserIdInDb, Uuid),
|
||||
) -> Result<Option<String>, DbErr> {
|
||||
use entity::managed_config_revisions as mcr;
|
||||
|
||||
let revision = mcr::Entity::find()
|
||||
.filter(mcr::Column::UserId.eq(user_id))
|
||||
.filter(mcr::Column::DeviceId.eq(device_id.to_string()))
|
||||
.one(self.orm_db())
|
||||
.await?;
|
||||
|
||||
Ok(revision.map(|row| row.config_revision))
|
||||
}
|
||||
|
||||
pub async fn set_managed_config_revision(
|
||||
&self,
|
||||
(user_id, device_id): (UserIdInDb, Uuid),
|
||||
config_revision: &str,
|
||||
) -> Result<(), DbErr> {
|
||||
use entity::managed_config_revisions as mcr;
|
||||
|
||||
let now = chrono::Local::now().fixed_offset();
|
||||
let on_conflict = OnConflict::columns([mcr::Column::UserId, mcr::Column::DeviceId])
|
||||
.update_columns([mcr::Column::ConfigRevision, mcr::Column::UpdateTime])
|
||||
.to_owned();
|
||||
let insert_m = mcr::ActiveModel {
|
||||
user_id: Set(user_id),
|
||||
device_id: Set(device_id.to_string()),
|
||||
config_revision: Set(config_revision.to_string()),
|
||||
create_time: Set(now),
|
||||
update_time: Set(now),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
mcr::Entity::insert(insert_m)
|
||||
.on_conflict(on_conflict)
|
||||
.do_nothing()
|
||||
.exec(self.orm_db())
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn insert_or_update_web_network_config(
|
||||
&self,
|
||||
(user_id, device_id): (UserIdInDb, Uuid),
|
||||
network_inst_id: Uuid,
|
||||
network_config: NetworkConfig,
|
||||
) -> Result<bool, DbErr> {
|
||||
let now = chrono::Local::now().fixed_offset();
|
||||
let network_config =
|
||||
serde_json::to_string(&network_config).map_err(|e| DbErr::Json(e.to_string()))?;
|
||||
let source = ConfigSource::Web.as_str();
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO user_running_network_configs (
|
||||
user_id, device_id, network_instance_id, network_config,
|
||||
source, disabled, create_time, update_time
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(user_id, device_id, network_instance_id) DO UPDATE SET
|
||||
network_config = excluded.network_config,
|
||||
source = excluded.source,
|
||||
disabled = excluded.disabled,
|
||||
update_time = excluded.update_time
|
||||
WHERE user_running_network_configs.source = ?
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(device_id.to_string())
|
||||
.bind(network_inst_id.to_string())
|
||||
.bind(network_config)
|
||||
.bind(source)
|
||||
.bind(false)
|
||||
.bind(now)
|
||||
.bind(now)
|
||||
.bind(source)
|
||||
.execute(&self.db)
|
||||
.await
|
||||
.map_err(|e| DbErr::Custom(e.to_string()))?;
|
||||
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
|
||||
pub async fn delete_web_network_configs(
|
||||
&self,
|
||||
(user_id, device_id): (UserIdInDb, Uuid),
|
||||
network_inst_ids: &[Uuid],
|
||||
) -> Result<(), DbErr> {
|
||||
use entity::user_running_network_configs as urnc;
|
||||
|
||||
urnc::Entity::delete_many()
|
||||
.filter(urnc::Column::UserId.eq(user_id))
|
||||
.filter(urnc::Column::DeviceId.eq(device_id.to_string()))
|
||||
.filter(urnc::Column::Source.eq(ConfigSource::Web.as_str()))
|
||||
.filter(
|
||||
urnc::Column::NetworkInstanceId
|
||||
.is_in(network_inst_ids.iter().map(|id| id.to_string())),
|
||||
)
|
||||
.exec(self.orm_db())
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -468,4 +572,46 @@ mod tests {
|
||||
assert_eq!(device1_configs.len(), 1);
|
||||
assert_eq!(device2_configs.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_web_network_config_does_not_replace_user_owned_config() {
|
||||
let db = Db::memory_db().await;
|
||||
let user_id = db.auto_create_user("user-web-race").await.unwrap().id;
|
||||
let device_id = uuid::Uuid::new_v4();
|
||||
let inst_id = uuid::Uuid::new_v4();
|
||||
|
||||
db.insert_or_update_user_network_config(
|
||||
(user_id, device_id),
|
||||
inst_id,
|
||||
NetworkConfig {
|
||||
network_name: Some("user-owned".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
ConfigSource::User,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let updated = db
|
||||
.insert_or_update_web_network_config(
|
||||
(user_id, device_id),
|
||||
inst_id,
|
||||
NetworkConfig {
|
||||
network_name: Some("web-owned".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!updated);
|
||||
let saved = db
|
||||
.get_network_config((user_id, device_id), &inst_id.to_string())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(saved.get_network_config_source(), ConfigSource::User);
|
||||
let saved_config = saved.get_network_config().unwrap();
|
||||
assert_eq!(saved_config.network_name.as_deref(), Some("user-owned"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
#[macro_use]
|
||||
extern crate rust_i18n;
|
||||
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::{net::IpAddr, time::Duration};
|
||||
|
||||
use clap::Parser;
|
||||
use easytier::tunnel::websocket::WsTunnelListener;
|
||||
@@ -113,6 +113,14 @@ struct Cli {
|
||||
)]
|
||||
geoip_db: Option<String>,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "ET_HEARTBEAT_MIN_RESPONSE_MS",
|
||||
default_value = "0",
|
||||
help = t!("cli.heartbeat_min_response_ms").to_string(),
|
||||
)]
|
||||
heartbeat_min_response_ms: u64,
|
||||
|
||||
#[cfg(feature = "embed")]
|
||||
#[arg(
|
||||
long,
|
||||
@@ -312,6 +320,7 @@ async fn main() {
|
||||
let mut mgr = client_manager::ClientManager::new(
|
||||
db.clone(),
|
||||
cli.geoip_db,
|
||||
Duration::from_millis(cli.heartbeat_min_response_ms),
|
||||
feature_flags.clone(),
|
||||
webhook_config.clone(),
|
||||
);
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
use sea_orm_migration::prelude::*;
|
||||
|
||||
pub struct Migration;
|
||||
|
||||
impl MigrationName for Migration {
|
||||
fn name(&self) -> &str {
|
||||
"m20260619_000005_managed_config_revisions"
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MigrationTrait for Migration {
|
||||
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||
manager
|
||||
.get_connection()
|
||||
.execute_unprepared(
|
||||
r#"
|
||||
CREATE TABLE managed_config_revisions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
user_id INTEGER NOT NULL,
|
||||
device_id TEXT NOT NULL,
|
||||
config_revision TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL,
|
||||
update_time TEXT NOT NULL,
|
||||
CONSTRAINT fk_managed_config_revisions_user_id_to_users_id
|
||||
FOREIGN KEY (user_id) REFERENCES users(id)
|
||||
ON DELETE CASCADE
|
||||
ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX idx_managed_config_revisions_scope
|
||||
ON managed_config_revisions(user_id, device_id);
|
||||
"#,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||
manager
|
||||
.get_connection()
|
||||
.execute_unprepared("DROP TABLE managed_config_revisions;")
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ mod m20241029_000001_init;
|
||||
mod m20260403_000002_scope_network_config_unique;
|
||||
mod m20260421_000003_add_network_config_source;
|
||||
mod m20260514_000004_rename_web_config_source;
|
||||
mod m20260619_000005_managed_config_revisions;
|
||||
|
||||
pub struct Migrator;
|
||||
|
||||
@@ -15,6 +16,7 @@ impl MigratorTrait for Migrator {
|
||||
Box::new(m20260403_000002_scope_network_config_unique::Migration),
|
||||
Box::new(m20260421_000003_add_network_config_source::Migration),
|
||||
Box::new(m20260514_000004_rename_web_config_source::Migration),
|
||||
Box::new(m20260619_000005_managed_config_revisions::Migration),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -307,7 +307,7 @@ impl RestfulServer {
|
||||
async fn handle_list_all_sessions_internal(
|
||||
State(client_mgr): AppState,
|
||||
) -> Result<Json<ListSessionJsonResp>, HttpHandleError> {
|
||||
let ret = client_mgr.list_sessions().await;
|
||||
let ret = client_mgr.list_all_sessions().await;
|
||||
Ok(ListSessionJsonResp(ret).into())
|
||||
}
|
||||
|
||||
|
||||
@@ -93,6 +93,8 @@ struct ManagedNetworkConfigJson {
|
||||
#[derive(Debug, serde::Deserialize, serde::Serialize)]
|
||||
struct ReconcileManagedNetworkConfigsJsonReq {
|
||||
managed_network_configs: Vec<ManagedNetworkConfigJson>,
|
||||
config_revision: Option<String>,
|
||||
expected_config_revision: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, serde::Serialize)]
|
||||
@@ -357,13 +359,21 @@ impl NetworkApi {
|
||||
})
|
||||
.collect();
|
||||
client_mgr
|
||||
.reconcile_managed_network_configs(user_id, machine_id, desired)
|
||||
.reconcile_managed_network_configs(
|
||||
user_id,
|
||||
machine_id,
|
||||
desired,
|
||||
payload.config_revision,
|
||||
payload.expected_config_revision,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
other_error(err.to_string()).into(),
|
||||
)
|
||||
let status = if crate::client_manager::is_managed_config_revision_conflict(&err) {
|
||||
StatusCode::CONFLICT
|
||||
} else {
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
};
|
||||
(status, other_error(err.to_string()).into())
|
||||
})?;
|
||||
Ok(Void::default().into())
|
||||
}
|
||||
|
||||
+589
-16
@@ -1,6 +1,248 @@
|
||||
use std::sync::Arc;
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
collections::VecDeque,
|
||||
fmt,
|
||||
sync::{Arc, Mutex},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
const VALIDATE_TOKEN_INITIAL_CONCURRENCY: usize = 8;
|
||||
const VALIDATE_TOKEN_MIN_CONCURRENCY: usize = 2;
|
||||
const VALIDATE_TOKEN_MAX_CONCURRENCY: usize = 64;
|
||||
const VALIDATE_TOKEN_ADJUST_WINDOW: Duration = Duration::from_secs(1);
|
||||
const VALIDATE_TOKEN_SLOW_THRESHOLD: Duration = Duration::from_secs(2);
|
||||
const WEBHOOK_HTTP_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
struct AdaptiveValidateLimiter {
|
||||
state: Mutex<AdaptiveValidateLimiterState>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for AdaptiveValidateLimiter {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("AdaptiveValidateLimiter")
|
||||
.field("state", &self.lock_state())
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
struct AdaptiveValidateLimiterState {
|
||||
limit: usize,
|
||||
in_flight: usize,
|
||||
waiters: VecDeque<oneshot::Sender<AdaptiveValidateGrant>>,
|
||||
window_started_at: Instant,
|
||||
samples: usize,
|
||||
slow_samples: usize,
|
||||
failures: usize,
|
||||
had_queue: bool,
|
||||
}
|
||||
|
||||
impl fmt::Debug for AdaptiveValidateLimiterState {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("AdaptiveValidateLimiterState")
|
||||
.field("limit", &self.limit)
|
||||
.field("in_flight", &self.in_flight)
|
||||
.field("waiters", &self.waiters.len())
|
||||
.field("window_started_at", &self.window_started_at)
|
||||
.field("samples", &self.samples)
|
||||
.field("slow_samples", &self.slow_samples)
|
||||
.field("failures", &self.failures)
|
||||
.field("had_queue", &self.had_queue)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
struct AdaptiveValidatePermit {
|
||||
limiter: Arc<AdaptiveValidateLimiter>,
|
||||
started_at: Instant,
|
||||
completed: bool,
|
||||
}
|
||||
|
||||
struct AdaptiveValidateGrant {
|
||||
limiter: Arc<AdaptiveValidateLimiter>,
|
||||
active: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum LimitAdjustment {
|
||||
Unchanged,
|
||||
Increased,
|
||||
Decreased,
|
||||
}
|
||||
|
||||
impl AdaptiveValidateLimiter {
|
||||
fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
state: Mutex::new(AdaptiveValidateLimiterState::new(Instant::now())),
|
||||
})
|
||||
}
|
||||
|
||||
async fn acquire(self: &Arc<Self>) -> AdaptiveValidatePermit {
|
||||
loop {
|
||||
let receiver = {
|
||||
let mut state = self.lock_state();
|
||||
state.complete_window_if_due(Instant::now());
|
||||
if state.waiters.is_empty() && state.in_flight < state.limit {
|
||||
state.in_flight += 1;
|
||||
return AdaptiveValidatePermit::new(self.clone());
|
||||
}
|
||||
|
||||
let (sender, receiver) = oneshot::channel();
|
||||
state.had_queue = true;
|
||||
state.waiters.push_back(sender);
|
||||
self.grant_waiters(&mut state);
|
||||
receiver
|
||||
};
|
||||
|
||||
if let Ok(grant) = receiver.await {
|
||||
return grant.into_permit();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn grant_waiters(self: &Arc<Self>, state: &mut AdaptiveValidateLimiterState) {
|
||||
while state.in_flight < state.limit {
|
||||
let Some(waiter) = state.waiters.pop_front() else {
|
||||
break;
|
||||
};
|
||||
state.in_flight += 1;
|
||||
if let Err(mut grant) = waiter.send(AdaptiveValidateGrant::new(self.clone())) {
|
||||
grant.disarm();
|
||||
state.in_flight -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_sample(self: &Arc<Self>, elapsed: Duration, success: bool) {
|
||||
let mut state = self.lock_state();
|
||||
let adjustment = state.record_sample(Instant::now(), elapsed, success);
|
||||
if adjustment == LimitAdjustment::Increased {
|
||||
self.grant_waiters(&mut state);
|
||||
}
|
||||
}
|
||||
|
||||
fn release_slot(self: &Arc<Self>) {
|
||||
let mut state = self.lock_state();
|
||||
state.in_flight = state.in_flight.saturating_sub(1);
|
||||
self.grant_waiters(&mut state);
|
||||
}
|
||||
|
||||
fn lock_state(&self) -> std::sync::MutexGuard<'_, AdaptiveValidateLimiterState> {
|
||||
self.state
|
||||
.lock()
|
||||
.expect("adaptive validate limiter state should not be poisoned")
|
||||
}
|
||||
}
|
||||
|
||||
impl AdaptiveValidateLimiterState {
|
||||
fn new(now: Instant) -> Self {
|
||||
Self {
|
||||
limit: VALIDATE_TOKEN_INITIAL_CONCURRENCY,
|
||||
in_flight: 0,
|
||||
waiters: VecDeque::new(),
|
||||
window_started_at: now,
|
||||
samples: 0,
|
||||
slow_samples: 0,
|
||||
failures: 0,
|
||||
had_queue: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn record_sample(&mut self, now: Instant, elapsed: Duration, success: bool) -> LimitAdjustment {
|
||||
self.samples += 1;
|
||||
if elapsed > VALIDATE_TOKEN_SLOW_THRESHOLD {
|
||||
self.slow_samples += 1;
|
||||
}
|
||||
if !success {
|
||||
self.failures += 1;
|
||||
}
|
||||
self.complete_window_if_due(now)
|
||||
}
|
||||
|
||||
fn complete_window_if_due(&mut self, now: Instant) -> LimitAdjustment {
|
||||
if now.duration_since(self.window_started_at) < VALIDATE_TOKEN_ADJUST_WINDOW {
|
||||
return LimitAdjustment::Unchanged;
|
||||
}
|
||||
|
||||
let old_limit = self.limit;
|
||||
if self.samples > 0 {
|
||||
if self.failures > 0 || self.is_p95_slow() {
|
||||
self.limit = (self.limit / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY);
|
||||
} else if self.had_queue {
|
||||
self.limit = (self.limit + 1).min(VALIDATE_TOKEN_MAX_CONCURRENCY);
|
||||
}
|
||||
}
|
||||
|
||||
self.window_started_at = now;
|
||||
self.samples = 0;
|
||||
self.slow_samples = 0;
|
||||
self.failures = 0;
|
||||
self.had_queue = false;
|
||||
|
||||
match self.limit.cmp(&old_limit) {
|
||||
Ordering::Greater => LimitAdjustment::Increased,
|
||||
Ordering::Less => LimitAdjustment::Decreased,
|
||||
Ordering::Equal => LimitAdjustment::Unchanged,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_p95_slow(&self) -> bool {
|
||||
self.slow_samples > 0 && self.slow_samples * 20 >= self.samples
|
||||
}
|
||||
}
|
||||
|
||||
impl AdaptiveValidatePermit {
|
||||
fn new(limiter: Arc<AdaptiveValidateLimiter>) -> Self {
|
||||
Self {
|
||||
limiter,
|
||||
started_at: Instant::now(),
|
||||
completed: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn complete(mut self, success: bool) {
|
||||
self.limiter
|
||||
.record_sample(self.started_at.elapsed(), success);
|
||||
self.completed = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl AdaptiveValidateGrant {
|
||||
fn new(limiter: Arc<AdaptiveValidateLimiter>) -> Self {
|
||||
Self {
|
||||
limiter,
|
||||
active: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn into_permit(mut self) -> AdaptiveValidatePermit {
|
||||
self.active = false;
|
||||
AdaptiveValidatePermit::new(self.limiter.clone())
|
||||
}
|
||||
|
||||
fn disarm(&mut self) {
|
||||
self.active = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AdaptiveValidateGrant {
|
||||
fn drop(&mut self) {
|
||||
if self.active {
|
||||
self.limiter.release_slot();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AdaptiveValidatePermit {
|
||||
fn drop(&mut self) {
|
||||
if !self.completed {
|
||||
self.limiter.record_sample(self.started_at.elapsed(), false);
|
||||
}
|
||||
self.limiter.release_slot();
|
||||
}
|
||||
}
|
||||
|
||||
/// Webhook configuration for external integrations.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -11,6 +253,7 @@ pub struct WebhookConfig {
|
||||
pub web_instance_id: Option<String>,
|
||||
pub web_instance_api_base_url: Option<String>,
|
||||
|
||||
validate_limiter: Arc<AdaptiveValidateLimiter>,
|
||||
client: reqwest::Client,
|
||||
}
|
||||
|
||||
@@ -28,7 +271,11 @@ impl WebhookConfig {
|
||||
internal_auth_token,
|
||||
web_instance_id,
|
||||
web_instance_api_base_url,
|
||||
client: reqwest::Client::new(),
|
||||
validate_limiter: AdaptiveValidateLimiter::new(),
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(WEBHOOK_HTTP_TIMEOUT)
|
||||
.build()
|
||||
.expect("webhook HTTP client should be valid"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,6 +305,8 @@ pub struct ValidateTokenRequest {
|
||||
pub web_instance_id: Option<String>,
|
||||
pub web_instance_api_base_url: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub persisted_config_revision: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub applied_config_revision: Option<String>,
|
||||
}
|
||||
|
||||
@@ -69,7 +318,6 @@ pub struct ValidateTokenResponse {
|
||||
#[serde(default)]
|
||||
pub binding_version: u64,
|
||||
#[serde(default)]
|
||||
pub managed_network_configs: Option<Vec<ManagedNetworkConfig>>,
|
||||
pub config_revision: String,
|
||||
}
|
||||
|
||||
@@ -125,21 +373,40 @@ impl WebhookConfig {
|
||||
pub async fn validate_token(
|
||||
&self,
|
||||
req: &ValidateTokenRequest,
|
||||
) -> anyhow::Result<ValidateTokenResponse> {
|
||||
self.validate_token_with_http_timeout(req, WEBHOOK_HTTP_TIMEOUT)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn validate_token_with_http_timeout(
|
||||
&self,
|
||||
req: &ValidateTokenRequest,
|
||||
http_timeout: Duration,
|
||||
) -> anyhow::Result<ValidateTokenResponse> {
|
||||
let url = self.webhook_endpoint("validate-token")?;
|
||||
let resp = self
|
||||
.client
|
||||
.post(&url)
|
||||
.header("X-Internal-Auth", self.webhook_auth_secret())
|
||||
.json(req)
|
||||
.send()
|
||||
.await?;
|
||||
let permit = self.validate_limiter.acquire().await;
|
||||
let ret = match tokio::time::timeout(http_timeout, async {
|
||||
let resp = self
|
||||
.client
|
||||
.post(&url)
|
||||
.header("X-Internal-Auth", self.webhook_auth_secret())
|
||||
.json(req)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
anyhow::bail!("webhook validate-token returned status {}", resp.status());
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
anyhow::bail!("webhook validate-token returned status {}", resp.status());
|
||||
}
|
||||
|
||||
Ok(resp.json().await?)
|
||||
Ok(resp.json().await?)
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(ret) => ret,
|
||||
Err(_) => Err(anyhow::anyhow!("webhook validate-token timed out")),
|
||||
};
|
||||
permit.complete(ret.is_ok());
|
||||
ret
|
||||
}
|
||||
|
||||
/// Notify the webhook receiver that a node has connected.
|
||||
@@ -191,13 +458,319 @@ pub type SharedWebhookConfig = Arc<WebhookConfig>;
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::{Json, Router, routing::post};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn validate_token_response_allows_missing_managed_configs() {
|
||||
fn adaptive_validate_limiter_increases_under_queue_pressure() {
|
||||
let now = Instant::now();
|
||||
let mut state = AdaptiveValidateLimiterState::new(now);
|
||||
state.had_queue = true;
|
||||
|
||||
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||
state.record_sample(now, Duration::from_millis(50), true);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
|
||||
LimitAdjustment::Increased
|
||||
);
|
||||
assert_eq!(state.limit, VALIDATE_TOKEN_INITIAL_CONCURRENCY + 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adaptive_validate_limiter_does_not_increase_without_queue_pressure() {
|
||||
let now = Instant::now();
|
||||
let mut state = AdaptiveValidateLimiterState::new(now);
|
||||
|
||||
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||
state.record_sample(now, Duration::from_millis(50), true);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
|
||||
LimitAdjustment::Unchanged
|
||||
);
|
||||
assert_eq!(state.limit, VALIDATE_TOKEN_INITIAL_CONCURRENCY);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adaptive_validate_limiter_reduces_on_failure() {
|
||||
let now = Instant::now();
|
||||
let mut state = AdaptiveValidateLimiterState::new(now);
|
||||
|
||||
state.record_sample(now, Duration::from_millis(50), false);
|
||||
|
||||
assert_eq!(
|
||||
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
|
||||
LimitAdjustment::Decreased
|
||||
);
|
||||
assert_eq!(
|
||||
state.limit,
|
||||
(VALIDATE_TOKEN_INITIAL_CONCURRENCY / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adaptive_validate_limiter_reduces_on_slow_latency() {
|
||||
let now = Instant::now();
|
||||
let mut state = AdaptiveValidateLimiterState::new(now);
|
||||
|
||||
state.record_sample(
|
||||
now,
|
||||
VALIDATE_TOKEN_SLOW_THRESHOLD + Duration::from_millis(1),
|
||||
true,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
|
||||
LimitAdjustment::Decreased
|
||||
);
|
||||
assert_eq!(
|
||||
state.limit,
|
||||
(VALIDATE_TOKEN_INITIAL_CONCURRENCY / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_validate_limiter_waiter_acquires_after_release() {
|
||||
let limiter = AdaptiveValidateLimiter::new();
|
||||
let mut permits = Vec::new();
|
||||
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||
permits.push(limiter.acquire().await);
|
||||
}
|
||||
|
||||
let waiter_limiter = limiter.clone();
|
||||
let waiter = tokio::spawn(async move {
|
||||
let permit = waiter_limiter.acquire().await;
|
||||
permit.complete(true);
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
assert!(!waiter.is_finished());
|
||||
|
||||
permits.pop().unwrap().complete(true);
|
||||
tokio::time::timeout(Duration::from_secs(1), waiter)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
for permit in permits {
|
||||
permit.complete(true);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_validate_limiter_releases_when_permit_is_dropped() {
|
||||
let limiter = AdaptiveValidateLimiter::new();
|
||||
let mut permits = Vec::new();
|
||||
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||
permits.push(limiter.acquire().await);
|
||||
}
|
||||
|
||||
let waiter_limiter = limiter.clone();
|
||||
let waiter = tokio::spawn(async move {
|
||||
let permit = waiter_limiter.acquire().await;
|
||||
permit.complete(true);
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
assert!(!waiter.is_finished());
|
||||
|
||||
drop(permits.pop().unwrap());
|
||||
tokio::time::timeout(Duration::from_secs(1), waiter)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
for permit in permits {
|
||||
permit.complete(true);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_validate_limiter_skips_canceled_waiters() {
|
||||
let limiter = AdaptiveValidateLimiter::new();
|
||||
let mut permits = Vec::new();
|
||||
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||
permits.push(limiter.acquire().await);
|
||||
}
|
||||
|
||||
let waiter_limiter = limiter.clone();
|
||||
let waiter = tokio::spawn(async move {
|
||||
let permit = waiter_limiter.acquire().await;
|
||||
permit.complete(true);
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
assert!(!waiter.is_finished());
|
||||
waiter.abort();
|
||||
assert!(waiter.await.unwrap_err().is_cancelled());
|
||||
|
||||
permits.pop().unwrap().complete(true);
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
|
||||
let state = limiter.lock_state();
|
||||
assert_eq!(state.samples, 1);
|
||||
assert_eq!(state.failures, 0);
|
||||
drop(state);
|
||||
|
||||
for permit in permits {
|
||||
permit.complete(true);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adaptive_validate_limiter_releases_dropped_grant_without_failure_sample() {
|
||||
let limiter = AdaptiveValidateLimiter::new();
|
||||
{
|
||||
let mut state = limiter.lock_state();
|
||||
state.in_flight = 1;
|
||||
}
|
||||
|
||||
drop(AdaptiveValidateGrant::new(limiter.clone()));
|
||||
|
||||
let state = limiter.lock_state();
|
||||
assert_eq!(state.in_flight, 0);
|
||||
assert_eq!(state.samples, 0);
|
||||
assert_eq!(state.failures, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_validate_limiter_wakes_multiple_waiters_in_order() {
|
||||
let limiter = AdaptiveValidateLimiter::new();
|
||||
let mut permits = Vec::new();
|
||||
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||
permits.push(limiter.acquire().await);
|
||||
}
|
||||
|
||||
let (first_acquired_tx, first_acquired_rx) = oneshot::channel();
|
||||
let (first_release_tx, first_release_rx) = oneshot::channel();
|
||||
let first = {
|
||||
let limiter = limiter.clone();
|
||||
tokio::spawn(async move {
|
||||
let permit = limiter.acquire().await;
|
||||
first_acquired_tx.send(()).unwrap();
|
||||
first_release_rx.await.unwrap();
|
||||
permit.complete(true);
|
||||
})
|
||||
};
|
||||
let (second_acquired_tx, mut second_acquired_rx) = oneshot::channel();
|
||||
let second = {
|
||||
let limiter = limiter.clone();
|
||||
tokio::spawn(async move {
|
||||
let permit = limiter.acquire().await;
|
||||
second_acquired_tx.send(()).unwrap();
|
||||
permit.complete(true);
|
||||
})
|
||||
};
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
assert!(!first.is_finished());
|
||||
assert!(!second.is_finished());
|
||||
|
||||
permits.pop().unwrap().complete(true);
|
||||
tokio::time::timeout(Duration::from_secs(1), first_acquired_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut second_acquired_rx)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
||||
first_release_tx.send(()).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(1), first)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(1), &mut second_acquired_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(1), second)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
for permit in permits {
|
||||
permit.complete(true);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_token_http_timeout_starts_after_limiter_permit() {
|
||||
let app = Router::new().route(
|
||||
"/validate-token",
|
||||
post(|| async {
|
||||
Json(json!({
|
||||
"valid": true,
|
||||
"config_revision": "rev-1"
|
||||
}))
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
let webhook = WebhookConfig::new(Some(format!("http://{addr}")), None, None, None, None);
|
||||
let mut permits = Vec::new();
|
||||
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||
permits.push(webhook.validate_limiter.acquire().await);
|
||||
}
|
||||
|
||||
let validate_webhook = webhook.clone();
|
||||
let validate = tokio::spawn(async move {
|
||||
let req = ValidateTokenRequest {
|
||||
token: "token".to_string(),
|
||||
machine_id: uuid::Uuid::new_v4().to_string(),
|
||||
public_ip: None,
|
||||
hostname: String::new(),
|
||||
version: String::new(),
|
||||
os_type: None,
|
||||
os_version: None,
|
||||
os_distribution: None,
|
||||
web_instance_id: None,
|
||||
web_instance_api_base_url: None,
|
||||
persisted_config_revision: None,
|
||||
applied_config_revision: None,
|
||||
};
|
||||
validate_webhook
|
||||
.validate_token_with_http_timeout(&req, Duration::from_millis(20))
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
assert!(!validate.is_finished());
|
||||
|
||||
permits.pop().unwrap().complete(true);
|
||||
let resp = tokio::time::timeout(Duration::from_secs(1), validate)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(resp.valid);
|
||||
|
||||
for permit in permits {
|
||||
permit.complete(true);
|
||||
}
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_token_response_deserializes_config_revision() {
|
||||
let resp: ValidateTokenResponse =
|
||||
serde_json::from_str(r#"{"valid":true,"config_revision":"rev-1"}"#).unwrap();
|
||||
assert!(resp.valid);
|
||||
assert_eq!(resp.config_revision, "rev-1");
|
||||
assert!(resp.managed_network_configs.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_token_response_allows_missing_config_revision() {
|
||||
let resp: ValidateTokenResponse = serde_json::from_str(r#"{"valid":true}"#).unwrap();
|
||||
assert!(resp.valid);
|
||||
assert!(resp.config_revision.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -483,18 +483,22 @@ impl InstanceConfigPatcher {
|
||||
}
|
||||
let global_ctx = weak_upgrade(&self.global_ctx)?;
|
||||
for proxy_network_patch in proxy_networks {
|
||||
let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else {
|
||||
tracing::warn!("Proxy network cidr is None, skipping.");
|
||||
continue;
|
||||
};
|
||||
let mapped_cidr: Option<cidr::Ipv4Cidr> =
|
||||
proxy_network_patch.mapped_cidr.map(|s| s.into());
|
||||
match ConfigPatchAction::try_from(proxy_network_patch.action) {
|
||||
Ok(ConfigPatchAction::Add) => {
|
||||
let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else {
|
||||
tracing::warn!("Proxy network cidr is None, skipping add.");
|
||||
continue;
|
||||
};
|
||||
let mapped_cidr: Option<cidr::Ipv4Cidr> =
|
||||
proxy_network_patch.mapped_cidr.map(|s| s.into());
|
||||
tracing::info!("Proxy network added: {}", cidr);
|
||||
global_ctx.config.add_proxy_cidr(cidr, mapped_cidr)?;
|
||||
}
|
||||
Ok(ConfigPatchAction::Remove) => {
|
||||
let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else {
|
||||
tracing::warn!("Proxy network cidr is None, skipping remove.");
|
||||
continue;
|
||||
};
|
||||
tracing::info!("Proxy network removed: {}", cidr);
|
||||
global_ctx.config.remove_proxy_cidr(cidr);
|
||||
}
|
||||
|
||||
@@ -1116,6 +1116,7 @@ impl NetworkConfig {
|
||||
.get_credential_file()
|
||||
.map(|path| path.to_string_lossy().into_owned());
|
||||
let flags = config.get_flags();
|
||||
let default_flags = default_config.get_flags();
|
||||
result.latency_first = Some(flags.latency_first);
|
||||
result.dev_name = Some(flags.dev_name.clone());
|
||||
result.use_smoltcp = Some(flags.use_smoltcp);
|
||||
@@ -1144,6 +1145,11 @@ impl NetworkConfig {
|
||||
result.disable_sym_hole_punching = Some(flags.disable_sym_hole_punching);
|
||||
result.enable_magic_dns = Some(flags.accept_dns);
|
||||
result.mtu = Some(flags.mtu as i32);
|
||||
result.data_compress_algo = (flags.data_compress_algo != default_flags.data_compress_algo)
|
||||
.then_some(flags.data_compress_algo);
|
||||
result.encryption_algorithm = (flags.encryption_algorithm
|
||||
!= default_flags.encryption_algorithm)
|
||||
.then_some(flags.encryption_algorithm.clone());
|
||||
result.instance_recv_bps_limit =
|
||||
(flags.instance_recv_bps_limit != u64::MAX).then_some(flags.instance_recv_bps_limit);
|
||||
result.enable_private_mode = Some(flags.private_mode);
|
||||
@@ -1173,7 +1179,7 @@ impl NetworkConfig {
|
||||
mod tests {
|
||||
use crate::{
|
||||
common::config::{ConfigLoader, process_secure_mode_cfg},
|
||||
proto::common::SecureModeConfig,
|
||||
proto::common::{CompressionAlgoPb, SecureModeConfig},
|
||||
};
|
||||
use base64::prelude::{BASE64_STANDARD, Engine as _};
|
||||
use rand::Rng;
|
||||
@@ -1534,4 +1540,37 @@ mod tests {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_network_config_conversion_preserves_runtime_algorithm_flags()
|
||||
-> Result<(), anyhow::Error> {
|
||||
let config = gen_default_config();
|
||||
let mut flags = config.get_flags();
|
||||
flags.data_compress_algo = CompressionAlgoPb::Zstd.into();
|
||||
flags.encryption_algorithm = "managed-test-algo".to_string();
|
||||
config.set_flags(flags.clone());
|
||||
|
||||
let network_config = super::NetworkConfig::new_from_config(&config)?;
|
||||
|
||||
assert_eq!(
|
||||
network_config.data_compress_algo,
|
||||
Some(CompressionAlgoPb::Zstd as i32)
|
||||
);
|
||||
assert_eq!(
|
||||
network_config.encryption_algorithm.as_deref(),
|
||||
Some("managed-test-algo")
|
||||
);
|
||||
|
||||
let generated_config = network_config.gen_config()?;
|
||||
assert_eq!(
|
||||
generated_config.get_flags().data_compress_algo,
|
||||
flags.data_compress_algo
|
||||
);
|
||||
assert_eq!(
|
||||
generated_config.get_flags().encryption_algorithm,
|
||||
flags.encryption_algorithm
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3653,6 +3653,25 @@ pub async fn config_patch_test() {
|
||||
true
|
||||
},
|
||||
);
|
||||
let patch = InstanceConfigPatch {
|
||||
proxy_networks: vec![ProxyNetworkPatch {
|
||||
action: ConfigPatchAction::Clear as i32,
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
insts[1]
|
||||
.get_config_patcher()
|
||||
.apply_patch(patch)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
insts[1]
|
||||
.get_global_ctx()
|
||||
.config
|
||||
.get_proxy_cidrs()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
// 测试1.1:修改公网 IPv6 provider 相关配置
|
||||
let public_prefix = "2001:db8:100::/64";
|
||||
|
||||
Reference in New Issue
Block a user