mirror of
https://github.com/EasyTier/EasyTier.git
synced 2026-09-04 01:55:41 +00:00
perf(easytier-web): improve easytier-web webhook performance (#2383)
* feat(web): reconcile managed config revisions * feat(web): cache managed runtime configs per session * test: cover managed web config delivery
This commit is contained in:
@@ -40,6 +40,9 @@ cli:
|
|||||||
geoip_db:
|
geoip_db:
|
||||||
en: "The path to the GeoIP2 database file, used to lookup the location of the client, default is the embedded file (only country information) , recommend https://github.com/P3TERX/GeoLite.mmdb"
|
en: "The path to the GeoIP2 database file, used to lookup the location of the client, default is the embedded file (only country information) , recommend https://github.com/P3TERX/GeoLite.mmdb"
|
||||||
zh-CN: "GeoIP2 数据库文件路径,用于查找客户端的位置,默认为嵌入文件(仅国家信息),推荐 https://github.com/P3TERX/GeoLite.mmdb"
|
zh-CN: "GeoIP2 数据库文件路径,用于查找客户端的位置,默认为嵌入文件(仅国家信息),推荐 https://github.com/P3TERX/GeoLite.mmdb"
|
||||||
|
heartbeat_min_response_ms:
|
||||||
|
en: "Minimum response time for config-server heartbeat RPCs in milliseconds, default is 0"
|
||||||
|
zh-CN: "配置服务心跳 RPC 的最短响应时间,单位毫秒,默认为 0"
|
||||||
disable_registration:
|
disable_registration:
|
||||||
en: "Disable user registration"
|
en: "Disable user registration"
|
||||||
zh-CN: "禁用用户注册"
|
zh-CN: "禁用用户注册"
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,3 +1,5 @@
|
|||||||
|
mod managed_config;
|
||||||
|
mod runtime_reconcile;
|
||||||
pub mod session;
|
pub mod session;
|
||||||
pub mod storage;
|
pub mod storage;
|
||||||
|
|
||||||
@@ -5,6 +7,7 @@ use std::sync::{
|
|||||||
Arc,
|
Arc,
|
||||||
atomic::{AtomicU32, Ordering},
|
atomic::{AtomicU32, Ordering},
|
||||||
};
|
};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use dashmap::DashMap;
|
use dashmap::DashMap;
|
||||||
use easytier::{
|
use easytier::{
|
||||||
@@ -30,6 +33,10 @@ use crate::db::{Db, UserIdInDb, entity::user_running_network_configs};
|
|||||||
#[include = "geoip2-cn.mmdb"]
|
#[include = "geoip2-cn.mmdb"]
|
||||||
struct GeoipDb;
|
struct GeoipDb;
|
||||||
|
|
||||||
|
pub fn is_managed_config_revision_conflict(error: &anyhow::Error) -> bool {
|
||||||
|
managed_config::is_revision_conflict(error)
|
||||||
|
}
|
||||||
|
|
||||||
fn load_geoip_db(geoip_db: Option<String>) -> Option<maxminddb::Reader<Vec<u8>>> {
|
fn load_geoip_db(geoip_db: Option<String>) -> Option<maxminddb::Reader<Vec<u8>>> {
|
||||||
if let Some(path) = geoip_db {
|
if let Some(path) = geoip_db {
|
||||||
match maxminddb::Reader::open_readfile(&path) {
|
match maxminddb::Reader::open_readfile(&path) {
|
||||||
@@ -63,12 +70,14 @@ pub struct ClientManager {
|
|||||||
webhook_config: SharedWebhookConfig,
|
webhook_config: SharedWebhookConfig,
|
||||||
|
|
||||||
geoip_db: Arc<Option<maxminddb::Reader<Vec<u8>>>>,
|
geoip_db: Arc<Option<maxminddb::Reader<Vec<u8>>>>,
|
||||||
|
heartbeat_min_response_delay: Duration,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ClientManager {
|
impl ClientManager {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
db: Db,
|
db: Db,
|
||||||
geoip_db: Option<String>,
|
geoip_db: Option<String>,
|
||||||
|
heartbeat_min_response_delay: Duration,
|
||||||
feature_flags: Arc<FeatureFlags>,
|
feature_flags: Arc<FeatureFlags>,
|
||||||
webhook_config: SharedWebhookConfig,
|
webhook_config: SharedWebhookConfig,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
@@ -92,6 +101,7 @@ impl ClientManager {
|
|||||||
webhook_config,
|
webhook_config,
|
||||||
|
|
||||||
geoip_db: Arc::new(load_geoip_db(geoip_db)),
|
geoip_db: Arc::new(load_geoip_db(geoip_db)),
|
||||||
|
heartbeat_min_response_delay,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -105,6 +115,7 @@ impl ClientManager {
|
|||||||
let storage = self.storage.weak_ref();
|
let storage = self.storage.weak_ref();
|
||||||
let listeners_cnt = self.listeners_cnt.clone();
|
let listeners_cnt = self.listeners_cnt.clone();
|
||||||
let geoip_db = self.geoip_db.clone();
|
let geoip_db = self.geoip_db.clone();
|
||||||
|
let heartbeat_min_response_delay = self.heartbeat_min_response_delay;
|
||||||
let feature_flags = self.feature_flags.clone();
|
let feature_flags = self.feature_flags.clone();
|
||||||
let webhook_config = self.webhook_config.clone();
|
let webhook_config = self.webhook_config.clone();
|
||||||
self.tasks.spawn(async move {
|
self.tasks.spawn(async move {
|
||||||
@@ -129,6 +140,7 @@ impl ClientManager {
|
|||||||
storage.clone(),
|
storage.clone(),
|
||||||
client_url.clone(),
|
client_url.clone(),
|
||||||
location,
|
location,
|
||||||
|
heartbeat_min_response_delay,
|
||||||
feature_flags.clone(),
|
feature_flags.clone(),
|
||||||
webhook_config.clone(),
|
webhook_config.clone(),
|
||||||
);
|
);
|
||||||
@@ -149,6 +161,10 @@ impl ClientManager {
|
|||||||
self.storage.list_clients()
|
self.storage.list_clients()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn list_all_sessions(&self) -> Vec<StorageToken> {
|
||||||
|
self.storage.list_all_clients()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn get_session_by_machine_id(
|
pub fn get_session_by_machine_id(
|
||||||
&self,
|
&self,
|
||||||
user_id: UserIdInDb,
|
user_id: UserIdInDb,
|
||||||
@@ -169,7 +185,7 @@ impl ClientManager {
|
|||||||
) -> bool {
|
) -> bool {
|
||||||
let Some(client_url) = self
|
let Some(client_url) = self
|
||||||
.storage
|
.storage
|
||||||
.get_client_url_by_machine_id(user_id, machine_id)
|
.get_client_url_by_machine_id_with_auth(user_id, machine_id, false)
|
||||||
else {
|
else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
@@ -189,14 +205,30 @@ impl ClientManager {
|
|||||||
user_id: UserIdInDb,
|
user_id: UserIdInDb,
|
||||||
machine_id: uuid::Uuid,
|
machine_id: uuid::Uuid,
|
||||||
desired_configs: Vec<ManagedNetworkConfig>,
|
desired_configs: Vec<ManagedNetworkConfig>,
|
||||||
|
config_revision: Option<String>,
|
||||||
|
expected_config_revision: Option<String>,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
session::SessionRpcService::reconcile_web_source_configs(
|
let expected_config_revision = match expected_config_revision.as_deref().map(str::trim) {
|
||||||
|
None => managed_config::ExpectedConfigRevision::Any,
|
||||||
|
Some("") => managed_config::ExpectedConfigRevision::Exact(None),
|
||||||
|
Some(revision) => managed_config::ExpectedConfigRevision::Exact(Some(revision)),
|
||||||
|
};
|
||||||
|
managed_config::reconcile_web_source_configs(
|
||||||
&self.storage,
|
&self.storage,
|
||||||
user_id,
|
user_id,
|
||||||
machine_id,
|
machine_id,
|
||||||
desired_configs,
|
desired_configs,
|
||||||
|
config_revision.as_deref(),
|
||||||
|
expected_config_revision,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
if let Some(config_revision) = config_revision
|
||||||
|
&& let Some(session) = self.get_session_by_machine_id(user_id, &machine_id)
|
||||||
|
{
|
||||||
|
session
|
||||||
|
.notify_config_revision_changed(user_id, machine_id, config_revision)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -331,19 +363,449 @@ impl
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use std::{sync::Arc, time::Duration};
|
use std::{
|
||||||
|
collections::VecDeque,
|
||||||
|
sync::{
|
||||||
|
Arc,
|
||||||
|
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||||
|
},
|
||||||
|
time::Duration,
|
||||||
|
};
|
||||||
|
|
||||||
|
use axum::{Json, Router, extract::State, routing::post};
|
||||||
use easytier::{
|
use easytier::{
|
||||||
|
common::MachineIdOptions,
|
||||||
instance_manager::NetworkInstanceManager,
|
instance_manager::NetworkInstanceManager,
|
||||||
|
proto::{
|
||||||
|
api::manage::{NetworkConfig, NetworkingMethod, PortForwardConfig},
|
||||||
|
common::CompressionAlgoPb,
|
||||||
|
},
|
||||||
|
rpc_service::remote_client::Storage as RemoteStorage,
|
||||||
tunnel::{
|
tunnel::{
|
||||||
common::tests::wait_for_condition,
|
common::tests::wait_for_condition,
|
||||||
udp::{UdpTunnelConnector, UdpTunnelListener},
|
udp::{UdpTunnelConnector, UdpTunnelListener},
|
||||||
},
|
},
|
||||||
web_client::WebClient,
|
web_client::{WebClient, run_web_client},
|
||||||
};
|
};
|
||||||
|
use serde_json::json;
|
||||||
use sqlx::Executor;
|
use sqlx::Executor;
|
||||||
|
use tokio::net::UdpSocket;
|
||||||
|
|
||||||
use crate::{FeatureFlags, client_manager::ClientManager, db::Db};
|
use crate::{
|
||||||
|
FeatureFlags, client_manager::ClientManager, db::Db, webhook::ManagedNetworkConfig,
|
||||||
|
};
|
||||||
|
|
||||||
|
const MANAGED_CONFIG_TOKEN: &str = "managed-config-token";
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct TestWebhookState {
|
||||||
|
validate_responses: Arc<tokio::sync::Mutex<VecDeque<bool>>>,
|
||||||
|
validate_count: Arc<AtomicUsize>,
|
||||||
|
block_second_validate: Arc<AtomicBool>,
|
||||||
|
allow_second_validate: Arc<AtomicBool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TestWebhookState {
|
||||||
|
fn new(validate_responses: impl IntoIterator<Item = bool>) -> Self {
|
||||||
|
Self {
|
||||||
|
validate_responses: Arc::new(tokio::sync::Mutex::new(
|
||||||
|
validate_responses.into_iter().collect(),
|
||||||
|
)),
|
||||||
|
validate_count: Arc::new(AtomicUsize::new(0)),
|
||||||
|
block_second_validate: Arc::new(AtomicBool::new(false)),
|
||||||
|
allow_second_validate: Arc::new(AtomicBool::new(true)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn with_blocked_second_validate(
|
||||||
|
validate_responses: impl IntoIterator<Item = bool>,
|
||||||
|
) -> Self {
|
||||||
|
let state = Self::new(validate_responses);
|
||||||
|
state.block_second_validate.store(true, Ordering::Release);
|
||||||
|
state.allow_second_validate.store(false, Ordering::Release);
|
||||||
|
state
|
||||||
|
}
|
||||||
|
|
||||||
|
fn allow_second_validate(&self) {
|
||||||
|
self.allow_second_validate.store(true, Ordering::Release);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_count(&self) -> usize {
|
||||||
|
self.validate_count.load(Ordering::Acquire)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn validate_token_handler(
|
||||||
|
State(state): State<TestWebhookState>,
|
||||||
|
) -> Json<serde_json::Value> {
|
||||||
|
let count = state.validate_count.fetch_add(1, Ordering::AcqRel) + 1;
|
||||||
|
if count == 2 && state.block_second_validate.load(Ordering::Acquire) {
|
||||||
|
while !state.allow_second_validate.load(Ordering::Acquire) {
|
||||||
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let valid = state
|
||||||
|
.validate_responses
|
||||||
|
.lock()
|
||||||
|
.await
|
||||||
|
.pop_front()
|
||||||
|
.unwrap_or(true);
|
||||||
|
if !valid {
|
||||||
|
return Json(json!({ "valid": false }));
|
||||||
|
}
|
||||||
|
|
||||||
|
Json(json!({
|
||||||
|
"valid": true,
|
||||||
|
"binding_version": count,
|
||||||
|
"config_revision": format!("validated-rev-{count}")
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn webhook_ack_handler() -> Json<serde_json::Value> {
|
||||||
|
Json(json!({}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn test_webhook_config() -> (
|
||||||
|
crate::webhook::SharedWebhookConfig,
|
||||||
|
tokio::task::JoinHandle<()>,
|
||||||
|
TestWebhookState,
|
||||||
|
) {
|
||||||
|
let state = TestWebhookState::new([true]);
|
||||||
|
test_webhook_config_with_state(state).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn test_webhook_config_with_state(
|
||||||
|
state: TestWebhookState,
|
||||||
|
) -> (
|
||||||
|
crate::webhook::SharedWebhookConfig,
|
||||||
|
tokio::task::JoinHandle<()>,
|
||||||
|
TestWebhookState,
|
||||||
|
) {
|
||||||
|
let app = Router::new()
|
||||||
|
.route("/validate-token", post(validate_token_handler))
|
||||||
|
.route("/webhook/node-connected", post(webhook_ack_handler))
|
||||||
|
.route("/webhook/node-disconnected", post(webhook_ack_handler))
|
||||||
|
.with_state(state.clone());
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let addr = listener.local_addr().unwrap();
|
||||||
|
let server = tokio::spawn(async move {
|
||||||
|
axum::serve(listener, app).await.unwrap();
|
||||||
|
});
|
||||||
|
|
||||||
|
(
|
||||||
|
Arc::new(crate::webhook::WebhookConfig::new(
|
||||||
|
Some(format!("http://{addr}")),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)),
|
||||||
|
server,
|
||||||
|
state,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn add_random_udp_listener(mgr: &mut ClientManager) -> std::net::SocketAddr {
|
||||||
|
let socket = Arc::new(UdpSocket::bind("127.0.0.1:0").await.unwrap());
|
||||||
|
let addr = socket.local_addr().unwrap();
|
||||||
|
let listener =
|
||||||
|
UdpTunnelListener::new_with_socket(format!("udp://{addr}").parse().unwrap(), socket);
|
||||||
|
mgr.add_listener(listener).await.unwrap();
|
||||||
|
addr
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_for_validated_user(mgr: &ClientManager, machine_id: uuid::Uuid) -> i32 {
|
||||||
|
tokio::time::timeout(Duration::from_secs(12), async {
|
||||||
|
loop {
|
||||||
|
if let Some(token) = mgr.list_sessions().await.into_iter().find(|token| {
|
||||||
|
token.token == MANAGED_CONFIG_TOKEN && token.machine_id == machine_id
|
||||||
|
}) {
|
||||||
|
break token.user_id;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_for_validate_count(state: &TestWebhookState, target: usize) {
|
||||||
|
tokio::time::timeout(Duration::from_secs(12), async {
|
||||||
|
loop {
|
||||||
|
if state.validate_count() >= target {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_for_session_urls(mgr: &ClientManager) -> Vec<url::Url> {
|
||||||
|
tokio::time::timeout(Duration::from_secs(12), async {
|
||||||
|
loop {
|
||||||
|
let urls = mgr
|
||||||
|
.client_sessions
|
||||||
|
.iter()
|
||||||
|
.map(|entry| entry.key().clone())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if !urls.is_empty() {
|
||||||
|
break urls;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn managed_config(
|
||||||
|
instance_id: uuid::Uuid,
|
||||||
|
network_config: serde_json::Value,
|
||||||
|
) -> ManagedNetworkConfig {
|
||||||
|
ManagedNetworkConfig {
|
||||||
|
instance_id: instance_id.to_string(),
|
||||||
|
network_config,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_for_runtime_config(
|
||||||
|
manager: &NetworkInstanceManager,
|
||||||
|
inst_id: uuid::Uuid,
|
||||||
|
predicate: impl Fn(&NetworkConfig) -> bool,
|
||||||
|
) -> NetworkConfig {
|
||||||
|
tokio::time::timeout(Duration::from_secs(12), async {
|
||||||
|
loop {
|
||||||
|
if let Some(config) = manager
|
||||||
|
.get_instance_config(&inst_id)
|
||||||
|
.and_then(|config| NetworkConfig::new_from_config(&config).ok())
|
||||||
|
.filter(|config| predicate(config))
|
||||||
|
{
|
||||||
|
break config;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn start_web_client_for_test(
|
||||||
|
config_server_addr: std::net::SocketAddr,
|
||||||
|
machine_id: uuid::Uuid,
|
||||||
|
manager: Arc<NetworkInstanceManager>,
|
||||||
|
) -> WebClient {
|
||||||
|
run_web_client(
|
||||||
|
&format!("udp://{config_server_addr}/{MANAGED_CONFIG_TOKEN}"),
|
||||||
|
MachineIdOptions {
|
||||||
|
explicit_machine_id: Some(machine_id.to_string()),
|
||||||
|
state_dir: None,
|
||||||
|
},
|
||||||
|
Some("managed-config-core".to_string()),
|
||||||
|
false,
|
||||||
|
manager,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn clear_managed_config_db(
|
||||||
|
mgr: &ClientManager,
|
||||||
|
user_id: i32,
|
||||||
|
machine_id: uuid::Uuid,
|
||||||
|
instance_id: uuid::Uuid,
|
||||||
|
) {
|
||||||
|
mgr.db()
|
||||||
|
.delete_web_network_configs((user_id, machine_id), &[instance_id])
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
sqlx::query("DELETE FROM managed_config_revisions WHERE user_id = ? AND device_id = ?")
|
||||||
|
.bind(user_id)
|
||||||
|
.bind(machine_id.to_string())
|
||||||
|
.execute(&mgr.db().inner())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_updated_runtime_config(updated: &NetworkConfig, instance_id: uuid::Uuid) {
|
||||||
|
assert_eq!(
|
||||||
|
updated.instance_id.as_deref(),
|
||||||
|
Some(instance_id.to_string().as_str())
|
||||||
|
);
|
||||||
|
assert_eq!(updated.dhcp, Some(false));
|
||||||
|
assert_eq!(updated.virtual_ipv4.as_deref(), Some("10.88.0.7"));
|
||||||
|
assert_eq!(updated.network_length, Some(24));
|
||||||
|
assert_eq!(updated.hostname.as_deref(), Some("managed-updated-host"));
|
||||||
|
assert_eq!(updated.network_name.as_deref(), Some("managed-updated"));
|
||||||
|
assert_eq!(updated.network_secret.as_deref(), Some("secret-updated"));
|
||||||
|
assert_eq!(
|
||||||
|
updated.networking_method,
|
||||||
|
Some(NetworkingMethod::Manual as i32)
|
||||||
|
);
|
||||||
|
assert_eq!(updated.peer_urls, vec!["tcp://127.0.0.1:11010".to_string()]);
|
||||||
|
assert_eq!(
|
||||||
|
updated.proxy_cidrs,
|
||||||
|
vec![
|
||||||
|
"10.44.0.0/24".to_string(),
|
||||||
|
"10.45.0.0/24->10.46.0.0/24".to_string()
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(updated.no_tun, Some(true));
|
||||||
|
assert_eq!(updated.disable_ipv6, Some(true));
|
||||||
|
assert_eq!(updated.enable_kcp_proxy, Some(true));
|
||||||
|
assert_eq!(updated.disable_kcp_input, Some(true));
|
||||||
|
assert_eq!(updated.enable_quic_proxy, Some(true));
|
||||||
|
assert_eq!(updated.disable_quic_input, Some(true));
|
||||||
|
assert_eq!(updated.disable_p2p, Some(true));
|
||||||
|
assert_eq!(updated.p2p_only, Some(true));
|
||||||
|
assert_eq!(updated.lazy_p2p, Some(true));
|
||||||
|
assert_eq!(updated.relay_all_peer_rpc, Some(true));
|
||||||
|
assert_eq!(updated.need_p2p, Some(true));
|
||||||
|
assert_eq!(updated.multi_thread, Some(false));
|
||||||
|
assert_eq!(updated.proxy_forward_by_system, Some(true));
|
||||||
|
assert_eq!(updated.disable_encryption, Some(true));
|
||||||
|
assert_eq!(updated.enable_relay_network_whitelist, Some(true));
|
||||||
|
assert_eq!(
|
||||||
|
updated.relay_network_whitelist,
|
||||||
|
vec!["10.44.0.0/24".to_string(), "10.45.0.0/24".to_string()]
|
||||||
|
);
|
||||||
|
assert_eq!(updated.enable_manual_routes, Some(true));
|
||||||
|
assert_eq!(
|
||||||
|
updated.routes,
|
||||||
|
vec!["10.60.0.0/16".to_string(), "10.61.0.0/16".to_string()]
|
||||||
|
);
|
||||||
|
assert_eq!(updated.port_forwards[0].bind_ip, "127.0.0.1");
|
||||||
|
assert_eq!(updated.port_forwards[0].bind_port, 0);
|
||||||
|
assert_eq!(updated.port_forwards[0].dst_ip, "10.88.0.8");
|
||||||
|
assert_eq!(updated.port_forwards[0].dst_port, 80);
|
||||||
|
assert_eq!(updated.port_forwards[0].proto, "tcp");
|
||||||
|
assert_eq!(updated.disable_udp_hole_punching, Some(true));
|
||||||
|
assert_eq!(updated.disable_tcp_hole_punching, Some(true));
|
||||||
|
assert_eq!(updated.disable_sym_hole_punching, Some(true));
|
||||||
|
assert_eq!(updated.disable_upnp, Some(true));
|
||||||
|
assert_eq!(updated.disable_relay_data, Some(true));
|
||||||
|
assert_eq!(updated.enable_magic_dns, Some(true));
|
||||||
|
assert_eq!(updated.enable_private_mode, Some(true));
|
||||||
|
assert_eq!(updated.mtu, Some(1360));
|
||||||
|
assert_eq!(
|
||||||
|
updated.data_compress_algo,
|
||||||
|
Some(CompressionAlgoPb::Zstd as i32)
|
||||||
|
);
|
||||||
|
assert_eq!(updated.encryption_algorithm.as_deref(), Some("xor"));
|
||||||
|
assert_eq!(updated.instance_recv_bps_limit, Some(123456));
|
||||||
|
assert_eq!(updated.enable_udp_broadcast_relay, Some(true));
|
||||||
|
assert_eq!(updated.socket_mark, Some(0));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn initial_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value {
|
||||||
|
json!({
|
||||||
|
"instance_id": inst_id.to_string(),
|
||||||
|
"dhcp": true,
|
||||||
|
"network_name": "managed-initial",
|
||||||
|
"network_secret": "secret-initial",
|
||||||
|
"networking_method": "Standalone",
|
||||||
|
"no_tun": true,
|
||||||
|
"disable_ipv6": true,
|
||||||
|
"enable_kcp_proxy": false,
|
||||||
|
"disable_kcp_input": false,
|
||||||
|
"relay_all_peer_rpc": false,
|
||||||
|
"multi_thread": false,
|
||||||
|
"disable_relay_data": false,
|
||||||
|
"mtu": 1380
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn updated_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value {
|
||||||
|
serde_json::to_value(NetworkConfig {
|
||||||
|
instance_id: Some(inst_id.to_string()),
|
||||||
|
dhcp: Some(false),
|
||||||
|
virtual_ipv4: Some("10.88.0.7".to_string()),
|
||||||
|
network_length: Some(24),
|
||||||
|
hostname: Some("managed-updated-host".to_string()),
|
||||||
|
network_name: Some("managed-updated".to_string()),
|
||||||
|
network_secret: Some("secret-updated".to_string()),
|
||||||
|
networking_method: Some(NetworkingMethod::Manual as i32),
|
||||||
|
peer_urls: vec!["tcp://127.0.0.1:11010".to_string()],
|
||||||
|
proxy_cidrs: vec![
|
||||||
|
"10.44.0.0/24".to_string(),
|
||||||
|
"10.45.0.0/24->10.46.0.0/24".to_string(),
|
||||||
|
],
|
||||||
|
no_tun: Some(true),
|
||||||
|
disable_ipv6: Some(true),
|
||||||
|
enable_kcp_proxy: Some(true),
|
||||||
|
disable_kcp_input: Some(true),
|
||||||
|
enable_quic_proxy: Some(true),
|
||||||
|
disable_quic_input: Some(true),
|
||||||
|
disable_p2p: Some(true),
|
||||||
|
p2p_only: Some(true),
|
||||||
|
lazy_p2p: Some(true),
|
||||||
|
relay_all_peer_rpc: Some(true),
|
||||||
|
need_p2p: Some(true),
|
||||||
|
multi_thread: Some(false),
|
||||||
|
proxy_forward_by_system: Some(true),
|
||||||
|
disable_encryption: Some(true),
|
||||||
|
enable_relay_network_whitelist: Some(true),
|
||||||
|
relay_network_whitelist: vec!["10.44.0.0/24".to_string(), "10.45.0.0/24".to_string()],
|
||||||
|
enable_manual_routes: Some(true),
|
||||||
|
routes: vec!["10.60.0.0/16".to_string(), "10.61.0.0/16".to_string()],
|
||||||
|
port_forwards: vec![PortForwardConfig {
|
||||||
|
bind_ip: "127.0.0.1".to_string(),
|
||||||
|
bind_port: 0,
|
||||||
|
dst_ip: "10.88.0.8".to_string(),
|
||||||
|
dst_port: 80,
|
||||||
|
proto: "tcp".to_string(),
|
||||||
|
}],
|
||||||
|
disable_udp_hole_punching: Some(true),
|
||||||
|
disable_tcp_hole_punching: Some(true),
|
||||||
|
disable_sym_hole_punching: Some(true),
|
||||||
|
disable_upnp: Some(true),
|
||||||
|
disable_relay_data: Some(true),
|
||||||
|
enable_magic_dns: Some(true),
|
||||||
|
enable_private_mode: Some(true),
|
||||||
|
mtu: Some(1360),
|
||||||
|
data_compress_algo: Some(CompressionAlgoPb::Zstd as i32),
|
||||||
|
encryption_algorithm: Some("xor".to_string()),
|
||||||
|
instance_recv_bps_limit: Some(123456),
|
||||||
|
enable_udp_broadcast_relay: Some(true),
|
||||||
|
socket_mark: Some(0),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn redelivered_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value {
|
||||||
|
serde_json::to_value(NetworkConfig {
|
||||||
|
instance_id: Some(inst_id.to_string()),
|
||||||
|
dhcp: Some(false),
|
||||||
|
virtual_ipv4: Some("10.88.0.7".to_string()),
|
||||||
|
network_length: Some(24),
|
||||||
|
hostname: Some("managed-redelivered-host".to_string()),
|
||||||
|
network_name: Some("managed-redelivered".to_string()),
|
||||||
|
network_secret: Some("secret-updated".to_string()),
|
||||||
|
networking_method: Some(NetworkingMethod::Manual as i32),
|
||||||
|
peer_urls: vec!["tcp://127.0.0.1:11010".to_string()],
|
||||||
|
proxy_cidrs: vec![
|
||||||
|
"10.44.0.0/24".to_string(),
|
||||||
|
"10.45.0.0/24->10.46.0.0/24".to_string(),
|
||||||
|
],
|
||||||
|
no_tun: Some(true),
|
||||||
|
disable_ipv6: Some(true),
|
||||||
|
enable_kcp_proxy: Some(true),
|
||||||
|
disable_kcp_input: Some(true),
|
||||||
|
relay_all_peer_rpc: Some(true),
|
||||||
|
need_p2p: Some(true),
|
||||||
|
multi_thread: Some(false),
|
||||||
|
enable_private_mode: Some(true),
|
||||||
|
mtu: Some(1360),
|
||||||
|
data_compress_algo: Some(CompressionAlgoPb::Zstd as i32),
|
||||||
|
encryption_algorithm: Some("xor".to_string()),
|
||||||
|
instance_recv_bps_limit: Some(654321),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_client() {
|
async fn test_client() {
|
||||||
@@ -351,6 +813,7 @@ mod tests {
|
|||||||
let mut mgr = ClientManager::new(
|
let mut mgr = ClientManager::new(
|
||||||
Db::memory_db().await,
|
Db::memory_db().await,
|
||||||
None,
|
None,
|
||||||
|
Duration::ZERO,
|
||||||
Arc::new(FeatureFlags::default()),
|
Arc::new(FeatureFlags::default()),
|
||||||
Arc::new(crate::webhook::WebhookConfig::new(
|
Arc::new(crate::webhook::WebhookConfig::new(
|
||||||
None, None, None, None, None,
|
None, None, None, None, None,
|
||||||
@@ -410,4 +873,223 @@ mod tests {
|
|||||||
println!("{:?}", req);
|
println!("{:?}", req);
|
||||||
println!("{:?}", mgr);
|
println!("{:?}", mgr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn managed_web_config_revision_updates_running_core_config() {
|
||||||
|
let (webhook_config, webhook_server, _) = test_webhook_config().await;
|
||||||
|
let mut mgr = ClientManager::new(
|
||||||
|
Db::memory_db().await,
|
||||||
|
None,
|
||||||
|
Duration::ZERO,
|
||||||
|
Arc::new(FeatureFlags::default()),
|
||||||
|
webhook_config,
|
||||||
|
);
|
||||||
|
let config_server_addr = add_random_udp_listener(&mut mgr).await;
|
||||||
|
|
||||||
|
let machine_id = uuid::Uuid::new_v4();
|
||||||
|
let instance_id = uuid::Uuid::new_v4();
|
||||||
|
let core_manager = Arc::new(NetworkInstanceManager::new());
|
||||||
|
let client =
|
||||||
|
start_web_client_for_test(config_server_addr, machine_id, core_manager.clone()).await;
|
||||||
|
|
||||||
|
let user_id = wait_for_validated_user(&mgr, machine_id).await;
|
||||||
|
mgr.reconcile_managed_network_configs(
|
||||||
|
user_id,
|
||||||
|
machine_id,
|
||||||
|
vec![managed_config(
|
||||||
|
instance_id,
|
||||||
|
initial_managed_network_config(instance_id),
|
||||||
|
)],
|
||||||
|
Some("rev-initial".to_string()),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
wait_for_runtime_config(&core_manager, instance_id, |config| {
|
||||||
|
config.network_name.as_deref() == Some("managed-initial")
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
|
// Online revision update: web-owned running config is fully overwritten
|
||||||
|
// when non-hot-patch flags such as enable_kcp_proxy change.
|
||||||
|
mgr.reconcile_managed_network_configs(
|
||||||
|
user_id,
|
||||||
|
machine_id,
|
||||||
|
vec![managed_config(
|
||||||
|
instance_id,
|
||||||
|
updated_managed_network_config(instance_id),
|
||||||
|
)],
|
||||||
|
Some("rev-updated".to_string()),
|
||||||
|
Some("rev-initial".to_string()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let updated = wait_for_runtime_config(&core_manager, instance_id, |config| {
|
||||||
|
config.network_name.as_deref() == Some("managed-updated")
|
||||||
|
&& config.enable_kcp_proxy == Some(true)
|
||||||
|
&& config.port_forwards.len() == 1
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
assert_updated_runtime_config(&updated, instance_id);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
core_manager.get_instance_network_config_source(&instance_id),
|
||||||
|
Some(easytier::common::config::ConfigSource::Web)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
mgr.db()
|
||||||
|
.get_managed_config_revision((user_id, machine_id))
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some("rev-updated")
|
||||||
|
);
|
||||||
|
|
||||||
|
// Web DB loss path: clear web-owned config and revision, then simulate
|
||||||
|
// the webhook re-posting the authoritative desired config. The already
|
||||||
|
// connected session should receive the distinguishable re-delivered
|
||||||
|
// revision without restarting.
|
||||||
|
clear_managed_config_db(&mgr, user_id, machine_id, instance_id).await;
|
||||||
|
assert!(
|
||||||
|
mgr.db()
|
||||||
|
.get_network_config((user_id, machine_id), &instance_id.to_string())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
mgr.db()
|
||||||
|
.get_managed_config_revision((user_id, machine_id))
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
|
||||||
|
mgr.reconcile_managed_network_configs(
|
||||||
|
user_id,
|
||||||
|
machine_id,
|
||||||
|
vec![managed_config(
|
||||||
|
instance_id,
|
||||||
|
redelivered_managed_network_config(instance_id),
|
||||||
|
)],
|
||||||
|
Some("rev-webhook-redelivery".to_string()),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let redelivered = wait_for_runtime_config(&core_manager, instance_id, |config| {
|
||||||
|
config.network_name.as_deref() == Some("managed-redelivered")
|
||||||
|
&& config.instance_recv_bps_limit == Some(654321)
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
redelivered.instance_id.as_deref(),
|
||||||
|
Some(instance_id.to_string().as_str())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
redelivered.hostname.as_deref(),
|
||||||
|
Some("managed-redelivered-host")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
redelivered.network_name.as_deref(),
|
||||||
|
Some("managed-redelivered")
|
||||||
|
);
|
||||||
|
assert_eq!(redelivered.enable_kcp_proxy, Some(true));
|
||||||
|
assert_eq!(redelivered.instance_recv_bps_limit, Some(654321));
|
||||||
|
assert_eq!(
|
||||||
|
core_manager.get_instance_network_config_source(&instance_id),
|
||||||
|
Some(easytier::common::config::ConfigSource::Web)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
mgr.db()
|
||||||
|
.get_managed_config_revision((user_id, machine_id))
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some("rev-webhook-redelivery")
|
||||||
|
);
|
||||||
|
|
||||||
|
// Reconnect path: a fresh core manager has no local runtime state, so
|
||||||
|
// the new session must replay the managed config persisted in web DB.
|
||||||
|
drop(client);
|
||||||
|
let reconnected_core_manager = Arc::new(NetworkInstanceManager::new());
|
||||||
|
let _reconnected_client = start_web_client_for_test(
|
||||||
|
config_server_addr,
|
||||||
|
machine_id,
|
||||||
|
reconnected_core_manager.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
wait_for_validated_user(&mgr, machine_id).await;
|
||||||
|
let replayed = wait_for_runtime_config(&reconnected_core_manager, instance_id, |config| {
|
||||||
|
config.network_name.as_deref() == Some("managed-redelivered")
|
||||||
|
&& config.instance_recv_bps_limit == Some(654321)
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
replayed.network_name.as_deref(),
|
||||||
|
Some("managed-redelivered")
|
||||||
|
);
|
||||||
|
assert_eq!(replayed.enable_kcp_proxy, Some(true));
|
||||||
|
assert_eq!(replayed.instance_recv_bps_limit, Some(654321));
|
||||||
|
|
||||||
|
webhook_server.abort();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn webhook_reject_disconnects_and_revalidates_after_reconnect() {
|
||||||
|
let webhook_state = TestWebhookState::with_blocked_second_validate([false, true]);
|
||||||
|
let (webhook_config, webhook_server, webhook_state) =
|
||||||
|
test_webhook_config_with_state(webhook_state).await;
|
||||||
|
let mut mgr = ClientManager::new(
|
||||||
|
Db::memory_db().await,
|
||||||
|
None,
|
||||||
|
Duration::ZERO,
|
||||||
|
Arc::new(FeatureFlags::default()),
|
||||||
|
webhook_config,
|
||||||
|
);
|
||||||
|
let config_server_addr = add_random_udp_listener(&mut mgr).await;
|
||||||
|
let machine_id = uuid::Uuid::new_v4();
|
||||||
|
let core_manager = Arc::new(NetworkInstanceManager::new());
|
||||||
|
let client =
|
||||||
|
start_web_client_for_test(config_server_addr, machine_id, core_manager.clone()).await;
|
||||||
|
|
||||||
|
let first_session_urls = wait_for_session_urls(&mgr).await;
|
||||||
|
wait_for_validate_count(&webhook_state, 1).await;
|
||||||
|
wait_for_validate_count(&webhook_state, 2).await;
|
||||||
|
assert!(
|
||||||
|
mgr.list_sessions().await.is_empty(),
|
||||||
|
"invalid validate-token response must not authorize the session"
|
||||||
|
);
|
||||||
|
|
||||||
|
webhook_state.allow_second_validate();
|
||||||
|
let user_id = wait_for_validated_user(&mgr, machine_id).await;
|
||||||
|
tokio::time::timeout(Duration::from_secs(12), async {
|
||||||
|
loop {
|
||||||
|
let reconnected = mgr
|
||||||
|
.client_sessions
|
||||||
|
.iter()
|
||||||
|
.any(|entry| !first_session_urls.iter().any(|url| url == entry.key()));
|
||||||
|
if reconnected {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
client.is_connected(),
|
||||||
|
"web client should reconnect after invalid session heartbeat failure"
|
||||||
|
);
|
||||||
|
assert!(webhook_state.validate_count() >= 2);
|
||||||
|
assert!(
|
||||||
|
mgr.get_session_by_machine_id(user_id, &machine_id)
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
|
||||||
|
webhook_server.abort();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,812 @@
|
|||||||
|
use anyhow::Context as _;
|
||||||
|
use easytier::{
|
||||||
|
common::config::{
|
||||||
|
ConfigLoader, EncryptionAlgorithm, PortForwardConfig as RuntimePortForwardConfig,
|
||||||
|
},
|
||||||
|
proto::{
|
||||||
|
acl::Acl,
|
||||||
|
api::{
|
||||||
|
config::{
|
||||||
|
AclPatch, ConfigPatchAction, InstanceConfigPatch, PatchConfigRequest,
|
||||||
|
PortForwardPatch, ProxyNetworkPatch,
|
||||||
|
},
|
||||||
|
instance::{InstanceIdentifier, instance_identifier},
|
||||||
|
manage::{
|
||||||
|
ConfigSource as RpcConfigSource, GetNetworkInstanceConfigRequest, NetworkConfig,
|
||||||
|
RunNetworkInstanceRequest,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
common::{CompressionAlgoPb, Ipv4Inet as RpcIpv4Inet},
|
||||||
|
rpc_types::controller::BaseController,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
use super::session::{SessionConfigClient, SessionRpcClient};
|
||||||
|
|
||||||
|
pub(super) enum RuntimeReconcileAction {
|
||||||
|
None,
|
||||||
|
Run {
|
||||||
|
config: Box<NetworkConfig>,
|
||||||
|
overwrite: bool,
|
||||||
|
},
|
||||||
|
Patch(Box<InstanceConfigPatch>),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, PartialEq)]
|
||||||
|
struct RuntimeProxyNetwork {
|
||||||
|
cidr: String,
|
||||||
|
mapped_cidr: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn instance_identifier(inst_id: &str) -> anyhow::Result<InstanceIdentifier> {
|
||||||
|
let inst_id = uuid::Uuid::parse_str(inst_id)
|
||||||
|
.with_context(|| format!("invalid runtime instance id: {inst_id}"))?;
|
||||||
|
Ok(InstanceIdentifier {
|
||||||
|
selector: Some(instance_identifier::Selector::Id(inst_id.into())),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hot_patch_base(config: &NetworkConfig) -> anyhow::Result<NetworkConfig> {
|
||||||
|
let data_compress_algo = normalized_data_compress_algo(config.data_compress_algo);
|
||||||
|
let encryption_algorithm = normalized_encryption_algorithm(config.encryption_algorithm.clone());
|
||||||
|
let mut config = NetworkConfig::new_from_config(config.gen_config()?)?;
|
||||||
|
let is_credential_mode = config.network_secret.is_none()
|
||||||
|
&& config
|
||||||
|
.secure_mode
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|mode| mode.local_private_key.as_deref())
|
||||||
|
.is_some_and(|key| !key.is_empty());
|
||||||
|
config.acl = None;
|
||||||
|
config.port_forwards.clear();
|
||||||
|
config.proxy_cidrs.clear();
|
||||||
|
config.disable_relay_data = None;
|
||||||
|
if config.dhcp.unwrap_or_default() {
|
||||||
|
config.virtual_ipv4 = None;
|
||||||
|
config.network_length = None;
|
||||||
|
}
|
||||||
|
if let Some(secure_mode) = config.secure_mode.as_mut() {
|
||||||
|
if !is_credential_mode {
|
||||||
|
secure_mode.local_private_key = None;
|
||||||
|
}
|
||||||
|
secure_mode.local_public_key = None;
|
||||||
|
}
|
||||||
|
config.data_compress_algo = data_compress_algo;
|
||||||
|
config.encryption_algorithm = encryption_algorithm;
|
||||||
|
Ok(config)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalized_data_compress_algo(algo: Option<i32>) -> Option<i32> {
|
||||||
|
let default = CompressionAlgoPb::None as i32;
|
||||||
|
let effective = algo.map(|algo| if algo < default { default } else { algo });
|
||||||
|
effective.filter(|algo| *algo != default)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalized_encryption_algorithm(algo: Option<String>) -> Option<String> {
|
||||||
|
let default = EncryptionAlgorithm::default().to_string();
|
||||||
|
algo.filter(|algo| algo != &default)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn diff_port_forwards(
|
||||||
|
current: &[RuntimePortForwardConfig],
|
||||||
|
desired: &[RuntimePortForwardConfig],
|
||||||
|
) -> Vec<PortForwardPatch> {
|
||||||
|
let mut patches = Vec::new();
|
||||||
|
for cfg in unique_port_forwards(current, desired) {
|
||||||
|
let current_count = current.iter().filter(|item| *item == &cfg).count();
|
||||||
|
let desired_count = desired.iter().filter(|item| *item == &cfg).count();
|
||||||
|
if current_count == desired_count {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if current_count > 0 {
|
||||||
|
patches.push(PortForwardPatch {
|
||||||
|
action: ConfigPatchAction::Remove as i32,
|
||||||
|
cfg: Some(cfg.clone().into()),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
patches.extend((0..desired_count).map(|_| PortForwardPatch {
|
||||||
|
action: ConfigPatchAction::Add as i32,
|
||||||
|
cfg: Some(cfg.clone().into()),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
patches
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unique_port_forwards(
|
||||||
|
current: &[RuntimePortForwardConfig],
|
||||||
|
desired: &[RuntimePortForwardConfig],
|
||||||
|
) -> Vec<RuntimePortForwardConfig> {
|
||||||
|
let mut unique = Vec::new();
|
||||||
|
for cfg in current.iter().chain(desired.iter()) {
|
||||||
|
if !unique.contains(cfg) {
|
||||||
|
unique.push(cfg.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
unique
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_rpc_ipv4_inet(value: &str) -> anyhow::Result<RpcIpv4Inet> {
|
||||||
|
value
|
||||||
|
.parse::<RpcIpv4Inet>()
|
||||||
|
.with_context(|| format!("failed to parse runtime ipv4 cidr: {value}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn diff_proxy_networks(
|
||||||
|
current: &[RuntimeProxyNetwork],
|
||||||
|
desired: &[RuntimeProxyNetwork],
|
||||||
|
) -> anyhow::Result<Vec<ProxyNetworkPatch>> {
|
||||||
|
if current == desired {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut patches = vec![ProxyNetworkPatch {
|
||||||
|
action: ConfigPatchAction::Clear as i32,
|
||||||
|
cidr: Some(clear_proxy_network_cidr(current, desired)?),
|
||||||
|
..Default::default()
|
||||||
|
}];
|
||||||
|
for proxy_network in desired {
|
||||||
|
patches.push(ProxyNetworkPatch {
|
||||||
|
action: ConfigPatchAction::Add as i32,
|
||||||
|
cidr: Some(parse_rpc_ipv4_inet(&proxy_network.cidr)?),
|
||||||
|
mapped_cidr: proxy_network
|
||||||
|
.mapped_cidr
|
||||||
|
.as_deref()
|
||||||
|
.map(parse_rpc_ipv4_inet)
|
||||||
|
.transpose()?,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(patches)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn clear_proxy_network_cidr(
|
||||||
|
current: &[RuntimeProxyNetwork],
|
||||||
|
desired: &[RuntimeProxyNetwork],
|
||||||
|
) -> anyhow::Result<RpcIpv4Inet> {
|
||||||
|
let cidr = desired
|
||||||
|
.first()
|
||||||
|
.or_else(|| current.first())
|
||||||
|
.map(|proxy_network| proxy_network.cidr.as_str())
|
||||||
|
.unwrap_or("0.0.0.0/0");
|
||||||
|
parse_rpc_ipv4_inet(cidr)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalized_acl(acl: &Option<Acl>) -> Option<Acl> {
|
||||||
|
let acl = acl.clone().unwrap_or_default();
|
||||||
|
(acl != Acl::default()).then_some(acl)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalized_port_forwards(
|
||||||
|
config: &NetworkConfig,
|
||||||
|
) -> anyhow::Result<Vec<RuntimePortForwardConfig>> {
|
||||||
|
Ok(config
|
||||||
|
.gen_config()?
|
||||||
|
.get_port_forwards()
|
||||||
|
.into_iter()
|
||||||
|
.map(|cfg| {
|
||||||
|
RuntimePortForwardConfig::from(easytier::proto::common::PortForwardConfigPb::from(cfg))
|
||||||
|
})
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalized_proxy_networks(config: &NetworkConfig) -> anyhow::Result<Vec<RuntimeProxyNetwork>> {
|
||||||
|
Ok(config
|
||||||
|
.gen_config()?
|
||||||
|
.get_proxy_cidrs()
|
||||||
|
.into_iter()
|
||||||
|
.map(|proxy_network| RuntimeProxyNetwork {
|
||||||
|
cidr: proxy_network.cidr.to_string(),
|
||||||
|
mapped_cidr: proxy_network.mapped_cidr.map(|cidr| cidr.to_string()),
|
||||||
|
})
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalized_disable_relay_data(config: &NetworkConfig) -> anyhow::Result<bool> {
|
||||||
|
Ok(config.gen_config()?.get_flags().disable_relay_data)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn web_source_runtime_patch(
|
||||||
|
current: &NetworkConfig,
|
||||||
|
desired: &NetworkConfig,
|
||||||
|
) -> anyhow::Result<Option<InstanceConfigPatch>> {
|
||||||
|
if let Some(desired_hostname) = desired
|
||||||
|
.hostname
|
||||||
|
.as_deref()
|
||||||
|
.filter(|hostname| !hostname.is_empty())
|
||||||
|
&& current.hostname.as_deref() != Some(desired_hostname)
|
||||||
|
{
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let mut current_base = hot_patch_base(current)?;
|
||||||
|
let mut desired_base = hot_patch_base(desired)?;
|
||||||
|
current_base.hostname = None;
|
||||||
|
desired_base.hostname = None;
|
||||||
|
if current_base != desired_base {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut patch = InstanceConfigPatch::default();
|
||||||
|
let current_acl = normalized_acl(¤t.acl);
|
||||||
|
let desired_acl = normalized_acl(&desired.acl);
|
||||||
|
if current_acl != desired_acl {
|
||||||
|
patch.acl = Some(AclPatch {
|
||||||
|
acl: Some(desired_acl.unwrap_or_default()),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let current_port_forwards = normalized_port_forwards(current)?;
|
||||||
|
let desired_port_forwards = normalized_port_forwards(desired)?;
|
||||||
|
if current_port_forwards != desired_port_forwards {
|
||||||
|
patch.port_forwards = diff_port_forwards(¤t_port_forwards, &desired_port_forwards);
|
||||||
|
}
|
||||||
|
|
||||||
|
let current_proxy_networks = normalized_proxy_networks(current)?;
|
||||||
|
let desired_proxy_networks = normalized_proxy_networks(desired)?;
|
||||||
|
if current_proxy_networks != desired_proxy_networks {
|
||||||
|
if current_proxy_networks.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
patch.proxy_networks =
|
||||||
|
diff_proxy_networks(¤t_proxy_networks, &desired_proxy_networks)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let current_disable_relay_data = normalized_disable_relay_data(current)?;
|
||||||
|
let desired_disable_relay_data = normalized_disable_relay_data(desired)?;
|
||||||
|
if current_disable_relay_data != desired_disable_relay_data {
|
||||||
|
patch.disable_relay_data = Some(desired_disable_relay_data);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Some(patch))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ensure_runtime_config_converged(
|
||||||
|
current: &NetworkConfig,
|
||||||
|
desired: &NetworkConfig,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let patch = web_source_runtime_patch(current, desired)?;
|
||||||
|
match patch {
|
||||||
|
Some(patch) if patch == InstanceConfigPatch::default() => Ok(()),
|
||||||
|
Some(patch) => anyhow::bail!("runtime config still needs patch after reconcile: {patch:?}"),
|
||||||
|
None => anyhow::bail!("runtime config still needs full overwrite after reconcile"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_web_source_instance(
|
||||||
|
rpc_client: &mut SessionRpcClient,
|
||||||
|
inst_id: &str,
|
||||||
|
config: NetworkConfig,
|
||||||
|
overwrite: bool,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
rpc_client
|
||||||
|
.run_network_instance(
|
||||||
|
BaseController::default(),
|
||||||
|
RunNetworkInstanceRequest {
|
||||||
|
inst_id: Some(inst_id.to_string().into()),
|
||||||
|
config: Some(config),
|
||||||
|
overwrite,
|
||||||
|
source: RpcConfigSource::Web as i32,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn get_runtime_config(
|
||||||
|
rpc_client: &mut SessionRpcClient,
|
||||||
|
inst_id: &str,
|
||||||
|
) -> anyhow::Result<NetworkConfig> {
|
||||||
|
rpc_client
|
||||||
|
.get_network_instance_config(
|
||||||
|
BaseController::default(),
|
||||||
|
GetNetworkInstanceConfigRequest {
|
||||||
|
inst_id: Some(inst_id.to_string().into()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
.config
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("runtime returned empty config for {inst_id}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn prepare_web_source_runtime_reconcile(
|
||||||
|
rpc_client: &mut SessionRpcClient,
|
||||||
|
inst_id: &str,
|
||||||
|
desired_config: NetworkConfig,
|
||||||
|
is_running: bool,
|
||||||
|
) -> anyhow::Result<RuntimeReconcileAction> {
|
||||||
|
if !is_running {
|
||||||
|
return Ok(RuntimeReconcileAction::Run {
|
||||||
|
config: Box::new(desired_config),
|
||||||
|
overwrite: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let current_config = get_runtime_config(rpc_client, inst_id).await?;
|
||||||
|
|
||||||
|
prepare_web_source_runtime_reconcile_from_current(¤t_config, desired_config)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn prepare_web_source_runtime_reconcile_from_current(
|
||||||
|
current_config: &NetworkConfig,
|
||||||
|
desired_config: NetworkConfig,
|
||||||
|
) -> anyhow::Result<RuntimeReconcileAction> {
|
||||||
|
let Some(patch) = web_source_runtime_patch(current_config, &desired_config)? else {
|
||||||
|
return Ok(RuntimeReconcileAction::Run {
|
||||||
|
config: Box::new(desired_config),
|
||||||
|
overwrite: true,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
if patch == InstanceConfigPatch::default() {
|
||||||
|
return Ok(RuntimeReconcileAction::None);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(RuntimeReconcileAction::Patch(Box::new(patch)))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn apply_web_source_runtime_reconcile(
|
||||||
|
rpc_client: &mut SessionRpcClient,
|
||||||
|
config_client: &mut SessionConfigClient,
|
||||||
|
inst_id: &str,
|
||||||
|
desired_config: NetworkConfig,
|
||||||
|
action: RuntimeReconcileAction,
|
||||||
|
) -> anyhow::Result<NetworkConfig> {
|
||||||
|
match action {
|
||||||
|
RuntimeReconcileAction::None => Ok(desired_config),
|
||||||
|
RuntimeReconcileAction::Run { config, overwrite } => {
|
||||||
|
run_web_source_instance(rpc_client, inst_id, *config, overwrite).await?;
|
||||||
|
Ok(desired_config)
|
||||||
|
}
|
||||||
|
RuntimeReconcileAction::Patch(patch) => {
|
||||||
|
config_client
|
||||||
|
.patch_config(
|
||||||
|
BaseController::default(),
|
||||||
|
PatchConfigRequest {
|
||||||
|
instance: Some(instance_identifier(inst_id)?),
|
||||||
|
patch: Some(*patch),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let current_config = get_runtime_config(rpc_client, inst_id).await?;
|
||||||
|
ensure_runtime_config_converged(¤t_config, &desired_config)?;
|
||||||
|
Ok(current_config)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use easytier::proto::{
|
||||||
|
api::{
|
||||||
|
config::ConfigPatchAction,
|
||||||
|
manage::{NetworkingMethod, PortForwardConfig},
|
||||||
|
},
|
||||||
|
common::{CompressionAlgoPb, SocketType},
|
||||||
|
};
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn config_with_port_forwards(port_forwards: Vec<PortForwardConfig>) -> NetworkConfig {
|
||||||
|
NetworkConfig {
|
||||||
|
instance_id: Some("11111111-1111-1111-1111-111111111111".to_string()),
|
||||||
|
dhcp: Some(true),
|
||||||
|
network_name: Some("managed".to_string()),
|
||||||
|
network_secret: Some("secret".to_string()),
|
||||||
|
networking_method: Some(NetworkingMethod::Manual as i32),
|
||||||
|
port_forwards,
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn port_forward(bind_port: u32, dst_port: u32) -> PortForwardConfig {
|
||||||
|
PortForwardConfig {
|
||||||
|
bind_ip: "127.0.0.1".to_string(),
|
||||||
|
bind_port,
|
||||||
|
dst_ip: "10.144.0.1".to_string(),
|
||||||
|
dst_port,
|
||||||
|
proto: "tcp".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn patch_port(patch: &PortForwardPatch) -> (i32, u32, u32, i32) {
|
||||||
|
let cfg = patch.cfg.as_ref().expect("port forward patch cfg");
|
||||||
|
(
|
||||||
|
patch.action,
|
||||||
|
cfg.bind_addr.as_ref().expect("bind addr").port,
|
||||||
|
cfg.dst_addr.as_ref().expect("dst addr").port,
|
||||||
|
cfg.socket_type,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn patch_proxy_network(patch: &ProxyNetworkPatch) -> (i32, String, Option<String>) {
|
||||||
|
(
|
||||||
|
patch.action,
|
||||||
|
patch.cidr.map(|cidr| cidr.to_string()).unwrap_or_default(),
|
||||||
|
patch.mapped_cidr.map(|cidr| cidr.to_string()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_ignores_runtime_defaults_and_adds_port_forward() {
|
||||||
|
let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||||
|
current.virtual_ipv4 = Some("10.144.0.2".to_string());
|
||||||
|
current.network_length = Some(16);
|
||||||
|
current.bind_device = Some(true);
|
||||||
|
current.dev_name = Some(String::new());
|
||||||
|
current.disable_ipv6 = Some(false);
|
||||||
|
current.mtu = Some(1380);
|
||||||
|
current.multi_thread = Some(true);
|
||||||
|
|
||||||
|
let desired =
|
||||||
|
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired)
|
||||||
|
.expect("build patch")
|
||||||
|
.expect("hot patch");
|
||||||
|
|
||||||
|
assert_eq!(patch.port_forwards.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
patch_port(&patch.port_forwards[0]),
|
||||||
|
(
|
||||||
|
ConfigPatchAction::Add as i32,
|
||||||
|
23007,
|
||||||
|
3389,
|
||||||
|
SocketType::Tcp as i32
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_removes_deleted_port_forward_without_clear() {
|
||||||
|
let current =
|
||||||
|
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||||
|
let desired = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired)
|
||||||
|
.expect("build patch")
|
||||||
|
.expect("hot patch");
|
||||||
|
|
||||||
|
assert_eq!(patch.port_forwards.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
patch_port(&patch.port_forwards[0]),
|
||||||
|
(
|
||||||
|
ConfigPatchAction::Remove as i32,
|
||||||
|
23007,
|
||||||
|
3389,
|
||||||
|
SocketType::Tcp as i32
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_reconciles_duplicate_port_forward_count() {
|
||||||
|
let current =
|
||||||
|
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23000, 5174)]);
|
||||||
|
let desired = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired)
|
||||||
|
.expect("build patch")
|
||||||
|
.expect("hot patch");
|
||||||
|
|
||||||
|
assert_eq!(patch.port_forwards.len(), 2);
|
||||||
|
assert_eq!(
|
||||||
|
patch_port(&patch.port_forwards[0]),
|
||||||
|
(
|
||||||
|
ConfigPatchAction::Remove as i32,
|
||||||
|
23000,
|
||||||
|
5174,
|
||||||
|
SocketType::Tcp as i32
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
patch_port(&patch.port_forwards[1]),
|
||||||
|
(
|
||||||
|
ConfigPatchAction::Add as i32,
|
||||||
|
23000,
|
||||||
|
5174,
|
||||||
|
SocketType::Tcp as i32
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_convergence_rejects_stale_extra_port_forward() {
|
||||||
|
let current = config_with_port_forwards(vec![
|
||||||
|
port_forward(23000, 5174),
|
||||||
|
port_forward(23007, 3389),
|
||||||
|
port_forward(23100, 8080),
|
||||||
|
]);
|
||||||
|
let desired =
|
||||||
|
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||||
|
|
||||||
|
let err = ensure_runtime_config_converged(¤t, &desired)
|
||||||
|
.expect_err("extra runtime port forward should not converge");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
err.to_string()
|
||||||
|
.contains("runtime config still needs patch after reconcile"),
|
||||||
|
"unexpected error: {err:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_canonicalizes_port_forward_protocol() {
|
||||||
|
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||||
|
let mut desired_port_forward = port_forward(23000, 5174);
|
||||||
|
desired_port_forward.proto = "TCP".to_string();
|
||||||
|
let desired = config_with_port_forwards(vec![desired_port_forward]);
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired)
|
||||||
|
.expect("build patch")
|
||||||
|
.expect("hot patch");
|
||||||
|
|
||||||
|
assert_eq!(patch, InstanceConfigPatch::default());
|
||||||
|
ensure_runtime_config_converged(¤t, &desired).expect("runtime converged");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_rejects_non_hot_config_change() {
|
||||||
|
let current = config_with_port_forwards(Vec::new());
|
||||||
|
let mut desired = current.clone();
|
||||||
|
desired.network_secret = Some("new-secret".to_string());
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||||
|
|
||||||
|
assert!(patch.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_rejects_routes_change() {
|
||||||
|
let mut current = config_with_port_forwards(Vec::new());
|
||||||
|
current.enable_manual_routes = Some(true);
|
||||||
|
current.routes = vec!["10.1.0.0/16".to_string(), "10.2.0.0/16".to_string()];
|
||||||
|
let mut desired = config_with_port_forwards(Vec::new());
|
||||||
|
desired.enable_manual_routes = Some(true);
|
||||||
|
desired.routes = vec!["10.2.0.0/16".to_string(), "10.3.0.0/16".to_string()];
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||||
|
|
||||||
|
assert!(patch.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_replaces_proxy_networks() {
|
||||||
|
let mut current = config_with_port_forwards(Vec::new());
|
||||||
|
current.proxy_cidrs = vec![
|
||||||
|
"10.1.0.0/16".to_string(),
|
||||||
|
"10.2.0.0/16->10.20.0.0/16".to_string(),
|
||||||
|
];
|
||||||
|
let mut desired = config_with_port_forwards(Vec::new());
|
||||||
|
desired.proxy_cidrs = vec![
|
||||||
|
"10.2.0.0/16->10.21.0.0/16".to_string(),
|
||||||
|
"10.3.0.0/16".to_string(),
|
||||||
|
];
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired)
|
||||||
|
.expect("build patch")
|
||||||
|
.expect("hot patch");
|
||||||
|
|
||||||
|
assert_eq!(patch.proxy_networks.len(), 3);
|
||||||
|
assert_eq!(
|
||||||
|
patch_proxy_network(&patch.proxy_networks[0]),
|
||||||
|
(
|
||||||
|
ConfigPatchAction::Clear as i32,
|
||||||
|
"10.2.0.0/16".to_string(),
|
||||||
|
None
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
patch_proxy_network(&patch.proxy_networks[1]),
|
||||||
|
(
|
||||||
|
ConfigPatchAction::Add as i32,
|
||||||
|
"10.2.0.0/16".to_string(),
|
||||||
|
Some("10.21.0.0/16".to_string())
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
patch_proxy_network(&patch.proxy_networks[2]),
|
||||||
|
(
|
||||||
|
ConfigPatchAction::Add as i32,
|
||||||
|
"10.3.0.0/16".to_string(),
|
||||||
|
None
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_replaces_proxy_networks_with_same_source_cidr() {
|
||||||
|
let mut current = config_with_port_forwards(Vec::new());
|
||||||
|
current.proxy_cidrs = vec![
|
||||||
|
"10.1.2.0/24".to_string(),
|
||||||
|
"10.1.2.0/24->10.1.3.0/24".to_string(),
|
||||||
|
];
|
||||||
|
let mut desired = config_with_port_forwards(Vec::new());
|
||||||
|
desired.proxy_cidrs = vec!["10.1.2.0/24->10.1.3.0/24".to_string()];
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired)
|
||||||
|
.expect("build patch")
|
||||||
|
.expect("hot patch");
|
||||||
|
|
||||||
|
assert_eq!(patch.proxy_networks.len(), 2);
|
||||||
|
assert_eq!(
|
||||||
|
patch_proxy_network(&patch.proxy_networks[0]),
|
||||||
|
(
|
||||||
|
ConfigPatchAction::Clear as i32,
|
||||||
|
"10.1.2.0/24".to_string(),
|
||||||
|
None
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
patch_proxy_network(&patch.proxy_networks[1]),
|
||||||
|
(
|
||||||
|
ConfigPatchAction::Add as i32,
|
||||||
|
"10.1.2.0/24".to_string(),
|
||||||
|
Some("10.1.3.0/24".to_string())
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_rejects_proxy_network_empty_to_nonempty() {
|
||||||
|
let current = config_with_port_forwards(Vec::new());
|
||||||
|
let mut desired = config_with_port_forwards(Vec::new());
|
||||||
|
desired.proxy_cidrs = vec!["10.1.2.0/24".to_string()];
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||||
|
|
||||||
|
assert!(patch.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_clears_proxy_networks_with_legacy_compatible_cidr() {
|
||||||
|
let mut current = config_with_port_forwards(Vec::new());
|
||||||
|
current.proxy_cidrs = vec!["10.1.2.0/24".to_string()];
|
||||||
|
let desired = config_with_port_forwards(Vec::new());
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired)
|
||||||
|
.expect("build patch")
|
||||||
|
.expect("hot patch");
|
||||||
|
|
||||||
|
assert_eq!(patch.proxy_networks.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
patch_proxy_network(&patch.proxy_networks[0]),
|
||||||
|
(
|
||||||
|
ConfigPatchAction::Clear as i32,
|
||||||
|
"10.1.2.0/24".to_string(),
|
||||||
|
None
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_updates_disable_relay_data() {
|
||||||
|
let current = config_with_port_forwards(Vec::new());
|
||||||
|
let mut desired = current.clone();
|
||||||
|
desired.disable_relay_data = Some(true);
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired)
|
||||||
|
.expect("build patch")
|
||||||
|
.expect("hot patch");
|
||||||
|
|
||||||
|
assert_eq!(patch.disable_relay_data, Some(true));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_still_rejects_unsupported_flag_change() {
|
||||||
|
let current = config_with_port_forwards(Vec::new());
|
||||||
|
let mut desired = current.clone();
|
||||||
|
desired.no_tun = Some(true);
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||||
|
|
||||||
|
assert!(patch.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_rejects_encryption_algorithm_change() {
|
||||||
|
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||||
|
let mut desired = current.clone();
|
||||||
|
desired.encryption_algorithm = Some("managed-test-algo".to_string());
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||||
|
|
||||||
|
assert!(patch.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_rejects_data_compress_algo_change() {
|
||||||
|
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||||
|
let mut desired = current.clone();
|
||||||
|
desired.data_compress_algo = Some(CompressionAlgoPb::Zstd as i32);
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||||
|
|
||||||
|
assert!(patch.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_rejects_credential_private_key_change() {
|
||||||
|
let mut current = config_with_port_forwards(Vec::new());
|
||||||
|
current.network_secret = None;
|
||||||
|
current.secure_mode = Some(easytier::proto::common::SecureModeConfig {
|
||||||
|
enabled: true,
|
||||||
|
local_private_key: Some("mUuD5fsIm/ftvgS4WBAYFMNLqWX3qT9rnm4PrnOqb9s=".to_string()),
|
||||||
|
local_public_key: None,
|
||||||
|
});
|
||||||
|
let mut desired = current.clone();
|
||||||
|
desired.secure_mode = Some(easytier::proto::common::SecureModeConfig {
|
||||||
|
enabled: true,
|
||||||
|
local_private_key: Some("aEpz80FuYbaY4QLJizAIuIcK4TYsoSA9jHHCXCOQJoc=".to_string()),
|
||||||
|
local_public_key: None,
|
||||||
|
});
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||||
|
|
||||||
|
assert!(patch.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_ignores_generated_secure_key_when_network_secret_exists() {
|
||||||
|
let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||||
|
current.secure_mode = Some(easytier::proto::common::SecureModeConfig {
|
||||||
|
enabled: true,
|
||||||
|
local_private_key: Some("mUuD5fsIm/ftvgS4WBAYFMNLqWX3qT9rnm4PrnOqb9s=".to_string()),
|
||||||
|
local_public_key: Some("4x6L5dZjB8hsPO4f96Hyhi4xFealBu6i3BxRVBYR1Fc=".to_string()),
|
||||||
|
});
|
||||||
|
let mut desired =
|
||||||
|
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||||
|
desired.secure_mode = Some(easytier::proto::common::SecureModeConfig {
|
||||||
|
enabled: true,
|
||||||
|
local_private_key: None,
|
||||||
|
local_public_key: None,
|
||||||
|
});
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired)
|
||||||
|
.expect("build patch")
|
||||||
|
.expect("hot patch");
|
||||||
|
|
||||||
|
assert_eq!(patch.port_forwards.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
patch_port(&patch.port_forwards[0]),
|
||||||
|
(
|
||||||
|
ConfigPatchAction::Add as i32,
|
||||||
|
23007,
|
||||||
|
3389,
|
||||||
|
SocketType::Tcp as i32
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_ignores_runtime_hostname_when_desired_omits_hostname() {
|
||||||
|
let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||||
|
current.hostname = Some("runtime-host".to_string());
|
||||||
|
let desired =
|
||||||
|
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired)
|
||||||
|
.expect("build patch")
|
||||||
|
.expect("hot patch");
|
||||||
|
|
||||||
|
assert_eq!(patch.port_forwards.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
patch_port(&patch.port_forwards[0]),
|
||||||
|
(
|
||||||
|
ConfigPatchAction::Add as i32,
|
||||||
|
23007,
|
||||||
|
3389,
|
||||||
|
SocketType::Tcp as i32
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_patch_rejects_explicit_desired_hostname_change() {
|
||||||
|
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||||
|
let mut desired =
|
||||||
|
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||||
|
desired.hostname =
|
||||||
|
Some(easytier::common::config::TomlConfigLoader::default().get_hostname());
|
||||||
|
|
||||||
|
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||||
|
|
||||||
|
assert!(patch.is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,913 @@
|
|||||||
|
use std::collections::{HashMap, HashSet};
|
||||||
|
|
||||||
|
use easytier::{
|
||||||
|
proto::{
|
||||||
|
api::manage::{
|
||||||
|
DeleteNetworkInstanceRequest, ListNetworkInstanceMetaRequest,
|
||||||
|
ListNetworkInstanceRequest, NetworkConfig, NetworkMeta, RunNetworkInstanceRequest,
|
||||||
|
},
|
||||||
|
rpc_types::controller::BaseController,
|
||||||
|
web::HeartbeatRequest,
|
||||||
|
},
|
||||||
|
rpc_service::remote_client::{ListNetworkProps, Storage as _},
|
||||||
|
};
|
||||||
|
use tokio::sync::{RwLock, broadcast};
|
||||||
|
|
||||||
|
use super::{SessionConfigClient, SessionData, SessionRpcClient, SessionRpcService};
|
||||||
|
use crate::client_manager::{
|
||||||
|
managed_config::{self, PersistedConfigSource},
|
||||||
|
runtime_reconcile,
|
||||||
|
storage::{StorageInner, WeakRefStorage},
|
||||||
|
};
|
||||||
|
|
||||||
|
async fn recv_latest_heartbeat(
|
||||||
|
heartbeat_waiter: &mut broadcast::Receiver<HeartbeatRequest>,
|
||||||
|
) -> Option<HeartbeatRequest> {
|
||||||
|
let mut req = loop {
|
||||||
|
match heartbeat_waiter.recv().await {
|
||||||
|
Ok(req) => break req,
|
||||||
|
Err(broadcast::error::RecvError::Lagged(skipped)) => {
|
||||||
|
tracing::warn!(
|
||||||
|
skipped,
|
||||||
|
"heartbeat reconcile worker lagged, waiting for latest request"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(broadcast::error::RecvError::Closed) => {
|
||||||
|
tracing::error!("Failed to receive heartbeat request: channel closed");
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Drop any heartbeat backlog accumulated while the previous reconcile
|
||||||
|
// round was doing DB/RPC IO. The newest heartbeat has the freshest
|
||||||
|
// runtime instance list, which is all this task needs.
|
||||||
|
loop {
|
||||||
|
match heartbeat_waiter.try_recv() {
|
||||||
|
Ok(next_req) => req = next_req,
|
||||||
|
Err(broadcast::error::TryRecvError::Empty) => break,
|
||||||
|
Err(broadcast::error::TryRecvError::Lagged(_)) => continue,
|
||||||
|
Err(broadcast::error::TryRecvError::Closed) => return None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Some(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn reconcile_network_configs_on_heartbeat(
|
||||||
|
session_data: std::sync::Weak<RwLock<SessionData>>,
|
||||||
|
mut heartbeat_waiter: broadcast::Receiver<HeartbeatRequest>,
|
||||||
|
storage: WeakRefStorage,
|
||||||
|
mut rpc_client: SessionRpcClient,
|
||||||
|
mut config_client: SessionConfigClient,
|
||||||
|
) {
|
||||||
|
let mut cache = ReconcileCache::default();
|
||||||
|
loop {
|
||||||
|
let Some(req) = recv_latest_heartbeat(&mut heartbeat_waiter).await else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let Some(storage) = storage.upgrade() else {
|
||||||
|
tracing::error!("Failed to get storage");
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut round =
|
||||||
|
match prepare_reconcile_round(&session_data, &storage, &mut rpc_client, req).await {
|
||||||
|
RoundStatus::Ready(round) => round,
|
||||||
|
RoundStatus::Skip => continue,
|
||||||
|
RoundStatus::Stop => return,
|
||||||
|
};
|
||||||
|
let running_metas =
|
||||||
|
match sync_running_sources_for_round(&mut rpc_client, &storage, &mut round).await {
|
||||||
|
RoundStatus::Ready(running_metas) => running_metas,
|
||||||
|
RoundStatus::Skip => continue,
|
||||||
|
RoundStatus::Stop => return,
|
||||||
|
};
|
||||||
|
|
||||||
|
let desired_web_inst_ids =
|
||||||
|
managed_config::desired_web_source_instance_ids(&round.local_configs);
|
||||||
|
cache.runtime_configs.retain_desired(&desired_web_inst_ids);
|
||||||
|
let mut outcome = match cleanup_stale_web_source_instances(
|
||||||
|
&session_data,
|
||||||
|
&storage,
|
||||||
|
&mut rpc_client,
|
||||||
|
&round,
|
||||||
|
running_metas.as_deref(),
|
||||||
|
&desired_web_inst_ids,
|
||||||
|
&mut cache,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
RoundStatus::Ready(outcome) => outcome,
|
||||||
|
RoundStatus::Skip => continue,
|
||||||
|
RoundStatus::Stop => return,
|
||||||
|
};
|
||||||
|
|
||||||
|
outcome.merge(
|
||||||
|
reconcile_desired_runtime_configs(
|
||||||
|
&session_data,
|
||||||
|
&mut rpc_client,
|
||||||
|
&mut config_client,
|
||||||
|
&round,
|
||||||
|
&mut cache,
|
||||||
|
)
|
||||||
|
.await,
|
||||||
|
);
|
||||||
|
|
||||||
|
if !outcome.has_failed {
|
||||||
|
cache.last_desired_web_inst_ids = Some(desired_web_inst_ids);
|
||||||
|
}
|
||||||
|
match mark_config_revision_applied_if_current(&session_data, &storage, &round, &outcome)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
RoundStatus::Ready(()) | RoundStatus::Skip => {}
|
||||||
|
RoundStatus::Stop => return,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enum RoundStatus<T> {
|
||||||
|
Ready(T),
|
||||||
|
Skip,
|
||||||
|
Stop,
|
||||||
|
}
|
||||||
|
|
||||||
|
enum ConfigActionResult {
|
||||||
|
Success,
|
||||||
|
Failed,
|
||||||
|
StopRound,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct ReconcileCache {
|
||||||
|
cleaned_web_source_instances: bool,
|
||||||
|
last_desired_web_inst_ids: Option<HashSet<String>>,
|
||||||
|
runtime_configs: SessionRuntimeConfigCache,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct SessionRuntimeConfigCache {
|
||||||
|
entries: HashMap<String, NetworkConfig>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SessionRuntimeConfigCache {
|
||||||
|
fn plan(
|
||||||
|
&self,
|
||||||
|
inst_id: &str,
|
||||||
|
desired_config: NetworkConfig,
|
||||||
|
) -> anyhow::Result<Option<runtime_reconcile::RuntimeReconcileAction>> {
|
||||||
|
let Some(observed_config) = self.entries.get(inst_id) else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
|
||||||
|
runtime_reconcile::prepare_web_source_runtime_reconcile_from_current(
|
||||||
|
observed_config,
|
||||||
|
desired_config,
|
||||||
|
)
|
||||||
|
.map(Some)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remember(&mut self, inst_id: &str, observed_config: NetworkConfig) {
|
||||||
|
self.entries.insert(inst_id.to_string(), observed_config);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn forget(&mut self, inst_id: &str) {
|
||||||
|
self.entries.remove(inst_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn forget_many<'a>(&mut self, inst_ids: impl IntoIterator<Item = &'a String>) {
|
||||||
|
for inst_id in inst_ids {
|
||||||
|
self.entries.remove(inst_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn retain_desired(&mut self, desired_web_inst_ids: &HashSet<String>) {
|
||||||
|
self.entries
|
||||||
|
.retain(|inst_id, _| desired_web_inst_ids.contains(inst_id));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct ReconcileOutcome {
|
||||||
|
has_failed: bool,
|
||||||
|
managed_revision_failed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ReconcileOutcome {
|
||||||
|
fn record_failure(&mut self, managed_revision_failed: bool) {
|
||||||
|
self.has_failed = true;
|
||||||
|
self.managed_revision_failed |= managed_revision_failed;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn merge(&mut self, other: Self) {
|
||||||
|
self.has_failed |= other.has_failed;
|
||||||
|
self.managed_revision_failed |= other.managed_revision_failed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ReconcileRound {
|
||||||
|
req: HeartbeatRequest,
|
||||||
|
machine_id: uuid::Uuid,
|
||||||
|
user_id: i32,
|
||||||
|
running_inst_ids: HashSet<String>,
|
||||||
|
local_configs: Vec<crate::db::entity::user_running_network_configs::Model>,
|
||||||
|
target_config_revision: Option<String>,
|
||||||
|
should_apply_runtime_revision: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn prepare_reconcile_round(
|
||||||
|
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||||
|
storage: &StorageInner,
|
||||||
|
rpc_client: &mut SessionRpcClient,
|
||||||
|
req: HeartbeatRequest,
|
||||||
|
) -> RoundStatus<ReconcileRound> {
|
||||||
|
let Some(machine_id) = req.machine_id.map(uuid::Uuid::from) else {
|
||||||
|
tracing::warn!(?req, "Machine id is not set, ignore");
|
||||||
|
return RoundStatus::Skip;
|
||||||
|
};
|
||||||
|
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &req).await {
|
||||||
|
tracing::debug!(?machine_id, "skip stale heartbeat reconcile request");
|
||||||
|
return RoundStatus::Skip;
|
||||||
|
}
|
||||||
|
|
||||||
|
let user_id = match storage
|
||||||
|
.db
|
||||||
|
.get_user_id_by_token(req.user_token.clone())
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(Some(user_id)) => user_id,
|
||||||
|
Ok(None) => {
|
||||||
|
tracing::info!("User not found by token: {:?}", req.user_token);
|
||||||
|
return RoundStatus::Stop;
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!("Failed to get user id by token, error: {:?}", e);
|
||||||
|
return RoundStatus::Stop;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let applied_config_revision = {
|
||||||
|
let Some(data) = session_data.upgrade() else {
|
||||||
|
return RoundStatus::Stop;
|
||||||
|
};
|
||||||
|
data.read().await.applied_config_revision.clone()
|
||||||
|
};
|
||||||
|
let target_config_revision = match storage
|
||||||
|
.db
|
||||||
|
.get_managed_config_revision((user_id, machine_id))
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(revision) => revision,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!("Failed to read managed config revision, error: {:?}", e);
|
||||||
|
return RoundStatus::Stop;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let should_apply_runtime_revision =
|
||||||
|
target_config_revision.is_some() && target_config_revision != applied_config_revision;
|
||||||
|
let running_inst_ids = match running_instance_ids_for_round(
|
||||||
|
rpc_client,
|
||||||
|
&req,
|
||||||
|
user_id,
|
||||||
|
machine_id,
|
||||||
|
should_apply_runtime_revision,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
RoundStatus::Ready(ids) => ids,
|
||||||
|
RoundStatus::Skip => return RoundStatus::Skip,
|
||||||
|
RoundStatus::Stop => return RoundStatus::Stop,
|
||||||
|
};
|
||||||
|
|
||||||
|
let local_configs = match storage
|
||||||
|
.db
|
||||||
|
.list_network_configs((user_id, machine_id), ListNetworkProps::EnabledOnly)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(configs) => configs,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!("Failed to list network configs, error: {:?}", e);
|
||||||
|
return RoundStatus::Stop;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
RoundStatus::Ready(ReconcileRound {
|
||||||
|
req,
|
||||||
|
machine_id,
|
||||||
|
user_id,
|
||||||
|
running_inst_ids,
|
||||||
|
local_configs,
|
||||||
|
target_config_revision,
|
||||||
|
should_apply_runtime_revision,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn running_instance_ids_for_round(
|
||||||
|
rpc_client: &mut SessionRpcClient,
|
||||||
|
req: &HeartbeatRequest,
|
||||||
|
user_id: i32,
|
||||||
|
machine_id: uuid::Uuid,
|
||||||
|
should_apply_runtime_revision: bool,
|
||||||
|
) -> RoundStatus<HashSet<String>> {
|
||||||
|
if !should_apply_runtime_revision {
|
||||||
|
return RoundStatus::Ready(
|
||||||
|
req.running_network_instances
|
||||||
|
.iter()
|
||||||
|
.map(|x| x.to_string())
|
||||||
|
.collect(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
match rpc_client
|
||||||
|
.list_network_instance(BaseController::default(), ListNetworkInstanceRequest {})
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(resp) => RoundStatus::Ready(resp.inst_ids.iter().map(|x| x.to_string()).collect()),
|
||||||
|
Err(error) => {
|
||||||
|
tracing::warn!(
|
||||||
|
?user_id,
|
||||||
|
?machine_id,
|
||||||
|
?error,
|
||||||
|
"Failed to refresh running instances for managed config revision"
|
||||||
|
);
|
||||||
|
RoundStatus::Skip
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn sync_running_sources_for_round(
|
||||||
|
rpc_client: &mut SessionRpcClient,
|
||||||
|
storage: &StorageInner,
|
||||||
|
round: &mut ReconcileRound,
|
||||||
|
) -> RoundStatus<Option<Vec<NetworkMeta>>> {
|
||||||
|
if !round.req.support_config_source {
|
||||||
|
return RoundStatus::Ready(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
let ret = if round.running_inst_ids.is_empty() {
|
||||||
|
Ok(Vec::new())
|
||||||
|
} else {
|
||||||
|
rpc_client
|
||||||
|
.list_network_instance_meta(
|
||||||
|
BaseController::default(),
|
||||||
|
ListNetworkInstanceMetaRequest {
|
||||||
|
inst_ids: managed_config::parse_instance_ids(
|
||||||
|
round.running_inst_ids.iter().cloned(),
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map(|resp| resp.metas)
|
||||||
|
};
|
||||||
|
|
||||||
|
match ret {
|
||||||
|
Ok(metas) => {
|
||||||
|
if let Err(e) = managed_config::sync_running_config_sources(
|
||||||
|
&storage.db,
|
||||||
|
round.user_id,
|
||||||
|
round.machine_id,
|
||||||
|
&round.local_configs,
|
||||||
|
&metas,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
tracing::warn!(
|
||||||
|
user_id = ?round.user_id,
|
||||||
|
machine_id = ?round.machine_id,
|
||||||
|
%e,
|
||||||
|
"Failed to sync running network config sources"
|
||||||
|
);
|
||||||
|
} else if !metas.is_empty() {
|
||||||
|
round.local_configs = match storage
|
||||||
|
.db
|
||||||
|
.list_network_configs(
|
||||||
|
(round.user_id, round.machine_id),
|
||||||
|
ListNetworkProps::EnabledOnly,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(configs) => configs,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!(
|
||||||
|
"Failed to reload network configs after source sync, error: {:?}",
|
||||||
|
e
|
||||||
|
);
|
||||||
|
return RoundStatus::Stop;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
RoundStatus::Ready(Some(metas))
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(
|
||||||
|
user_id = ?round.user_id,
|
||||||
|
%e,
|
||||||
|
"Failed to list running network instance metadata"
|
||||||
|
);
|
||||||
|
RoundStatus::Ready(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cleanup_stale_web_source_instances(
|
||||||
|
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||||
|
storage: &StorageInner,
|
||||||
|
rpc_client: &mut SessionRpcClient,
|
||||||
|
round: &ReconcileRound,
|
||||||
|
running_metas: Option<&[NetworkMeta]>,
|
||||||
|
desired_web_inst_ids: &HashSet<String>,
|
||||||
|
cache: &mut ReconcileCache,
|
||||||
|
) -> RoundStatus<ReconcileOutcome> {
|
||||||
|
let desired_changed = cache
|
||||||
|
.last_desired_web_inst_ids
|
||||||
|
.as_ref()
|
||||||
|
.is_none_or(|last| last != desired_web_inst_ids);
|
||||||
|
if cache.cleaned_web_source_instances && !desired_changed {
|
||||||
|
return RoundStatus::Ready(ReconcileOutcome::default());
|
||||||
|
}
|
||||||
|
|
||||||
|
let db_web_inst_ids = match storage
|
||||||
|
.db
|
||||||
|
.list_network_configs((round.user_id, round.machine_id), ListNetworkProps::All)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(configs) => managed_config::desired_web_source_instance_ids(&configs),
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!("Failed to list all network configs, error: {:?}", e);
|
||||||
|
return RoundStatus::Stop;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let running_web_inst_ids = managed_config::running_web_source_instance_ids(
|
||||||
|
&round.running_inst_ids,
|
||||||
|
&db_web_inst_ids,
|
||||||
|
running_metas,
|
||||||
|
);
|
||||||
|
let should_delete_inst_ids = running_web_inst_ids
|
||||||
|
.difference(desired_web_inst_ids)
|
||||||
|
.cloned()
|
||||||
|
.collect::<HashSet<_>>();
|
||||||
|
let should_delete_ids =
|
||||||
|
managed_config::parse_instance_ids(should_delete_inst_ids.iter().cloned());
|
||||||
|
|
||||||
|
let mut outcome = ReconcileOutcome::default();
|
||||||
|
if !should_delete_ids.is_empty() {
|
||||||
|
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
|
||||||
|
tracing::debug!(
|
||||||
|
machine_id = ?round.machine_id,
|
||||||
|
"skip stale cleanup because webhook session is no longer current"
|
||||||
|
);
|
||||||
|
return RoundStatus::Skip;
|
||||||
|
}
|
||||||
|
let ret = rpc_client
|
||||||
|
.delete_network_instance(
|
||||||
|
BaseController::default(),
|
||||||
|
DeleteNetworkInstanceRequest {
|
||||||
|
inst_ids: should_delete_ids,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
tracing::info!(
|
||||||
|
user_id = ?round.user_id,
|
||||||
|
"Clean stale web-source network instances on heartbeat: {:?}, user_token: {:?}",
|
||||||
|
ret,
|
||||||
|
round.req.user_token
|
||||||
|
);
|
||||||
|
if ret.is_err() {
|
||||||
|
outcome.record_failure(true);
|
||||||
|
} else {
|
||||||
|
cache.runtime_configs.forget_many(&should_delete_inst_ids);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !outcome.has_failed {
|
||||||
|
cache.cleaned_web_source_instances = true;
|
||||||
|
cache.last_desired_web_inst_ids = Some(desired_web_inst_ids.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
RoundStatus::Ready(outcome)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn reconcile_desired_runtime_configs(
|
||||||
|
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||||
|
rpc_client: &mut SessionRpcClient,
|
||||||
|
config_client: &mut SessionConfigClient,
|
||||||
|
round: &ReconcileRound,
|
||||||
|
cache: &mut ReconcileCache,
|
||||||
|
) -> ReconcileOutcome {
|
||||||
|
let mut outcome = ReconcileOutcome::default();
|
||||||
|
|
||||||
|
// After stale web-owned instances are removed, start every enabled
|
||||||
|
// config that the latest heartbeat did not report as running. When
|
||||||
|
// a managed config revision is pending, also reconcile running
|
||||||
|
// web-owned configs before reporting that revision as applied.
|
||||||
|
for config in &round.local_configs {
|
||||||
|
let source = PersistedConfigSource::from_db(&config.source);
|
||||||
|
let is_running = round.running_inst_ids.contains(&config.network_instance_id);
|
||||||
|
let should_reconcile_running_web_config = is_running
|
||||||
|
&& round.should_apply_runtime_revision
|
||||||
|
&& source == PersistedConfigSource::Web;
|
||||||
|
if is_running && !should_reconcile_running_web_config {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let desired_config = match serde_json::from_str::<NetworkConfig>(&config.network_config) {
|
||||||
|
Ok(cfg) => cfg,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!(
|
||||||
|
user_id = ?round.user_id,
|
||||||
|
machine_id = ?round.machine_id,
|
||||||
|
instance_id = %config.network_instance_id,
|
||||||
|
"Failed to deserialize network config, skipping: {:?}",
|
||||||
|
e
|
||||||
|
);
|
||||||
|
if source == PersistedConfigSource::Web {
|
||||||
|
cache.runtime_configs.forget(&config.network_instance_id);
|
||||||
|
}
|
||||||
|
outcome.record_failure(source == PersistedConfigSource::Web);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let action_result = if should_reconcile_running_web_config {
|
||||||
|
reconcile_running_web_config(
|
||||||
|
session_data,
|
||||||
|
rpc_client,
|
||||||
|
config_client,
|
||||||
|
round,
|
||||||
|
config,
|
||||||
|
desired_config,
|
||||||
|
&mut cache.runtime_configs,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
} else {
|
||||||
|
if source == PersistedConfigSource::Web {
|
||||||
|
cache.runtime_configs.forget(&config.network_instance_id);
|
||||||
|
}
|
||||||
|
let action_result = run_missing_network_config(
|
||||||
|
session_data,
|
||||||
|
rpc_client,
|
||||||
|
round,
|
||||||
|
config,
|
||||||
|
desired_config.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
if matches!(action_result, ConfigActionResult::Success)
|
||||||
|
&& source == PersistedConfigSource::Web
|
||||||
|
{
|
||||||
|
if let Err(e) = remember_web_runtime_config_after_run(
|
||||||
|
rpc_client,
|
||||||
|
&config.network_instance_id,
|
||||||
|
&desired_config,
|
||||||
|
&mut cache.runtime_configs,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
tracing::error!(
|
||||||
|
user_id = ?round.user_id,
|
||||||
|
machine_id = ?round.machine_id,
|
||||||
|
instance_id = %config.network_instance_id,
|
||||||
|
"Failed to cache runtime config after run: {:?}",
|
||||||
|
e
|
||||||
|
);
|
||||||
|
ConfigActionResult::Failed
|
||||||
|
} else {
|
||||||
|
action_result
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
action_result
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
match action_result {
|
||||||
|
ConfigActionResult::Success => {}
|
||||||
|
ConfigActionResult::Failed => {
|
||||||
|
if source == PersistedConfigSource::Web {
|
||||||
|
cache.runtime_configs.forget(&config.network_instance_id);
|
||||||
|
}
|
||||||
|
outcome.record_failure(source == PersistedConfigSource::Web)
|
||||||
|
}
|
||||||
|
ConfigActionResult::StopRound => {
|
||||||
|
if source == PersistedConfigSource::Web {
|
||||||
|
cache.runtime_configs.forget(&config.network_instance_id);
|
||||||
|
}
|
||||||
|
outcome.record_failure(source == PersistedConfigSource::Web);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
outcome
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn reconcile_running_web_config(
|
||||||
|
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||||
|
rpc_client: &mut SessionRpcClient,
|
||||||
|
config_client: &mut SessionConfigClient,
|
||||||
|
round: &ReconcileRound,
|
||||||
|
config: &crate::db::entity::user_running_network_configs::Model,
|
||||||
|
desired_config: NetworkConfig,
|
||||||
|
runtime_config_cache: &mut SessionRuntimeConfigCache,
|
||||||
|
) -> ConfigActionResult {
|
||||||
|
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
|
||||||
|
tracing::debug!(
|
||||||
|
machine_id = ?round.machine_id,
|
||||||
|
instance_id = %config.network_instance_id,
|
||||||
|
"skip runtime reconcile because webhook session is no longer current"
|
||||||
|
);
|
||||||
|
return ConfigActionResult::StopRound;
|
||||||
|
}
|
||||||
|
|
||||||
|
let ret = async {
|
||||||
|
let action =
|
||||||
|
match runtime_config_cache.plan(&config.network_instance_id, desired_config.clone())? {
|
||||||
|
Some(action) => action,
|
||||||
|
None => {
|
||||||
|
runtime_reconcile::prepare_web_source_runtime_reconcile(
|
||||||
|
&mut *rpc_client,
|
||||||
|
&config.network_instance_id,
|
||||||
|
desired_config.clone(),
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
|
||||||
|
anyhow::bail!("webhook session is no longer current before runtime reconcile apply");
|
||||||
|
}
|
||||||
|
let observed_config = runtime_reconcile::apply_web_source_runtime_reconcile(
|
||||||
|
&mut *rpc_client,
|
||||||
|
&mut *config_client,
|
||||||
|
&config.network_instance_id,
|
||||||
|
desired_config.clone(),
|
||||||
|
action,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
runtime_config_cache.remember(&config.network_instance_id, observed_config);
|
||||||
|
Ok::<(), anyhow::Error>(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
tracing::info!(
|
||||||
|
user_id = ?round.user_id,
|
||||||
|
instance_id = %config.network_instance_id,
|
||||||
|
"Reconcile running web-source network instance: {:?}, user_token: {:?}",
|
||||||
|
ret,
|
||||||
|
round.req.user_token
|
||||||
|
);
|
||||||
|
|
||||||
|
if ret.is_ok() {
|
||||||
|
ConfigActionResult::Success
|
||||||
|
} else {
|
||||||
|
runtime_config_cache.forget(&config.network_instance_id);
|
||||||
|
ConfigActionResult::Failed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_missing_network_config(
|
||||||
|
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||||
|
rpc_client: &mut SessionRpcClient,
|
||||||
|
round: &ReconcileRound,
|
||||||
|
config: &crate::db::entity::user_running_network_configs::Model,
|
||||||
|
desired_config: NetworkConfig,
|
||||||
|
) -> ConfigActionResult {
|
||||||
|
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
|
||||||
|
tracing::debug!(
|
||||||
|
machine_id = ?round.machine_id,
|
||||||
|
instance_id = %config.network_instance_id,
|
||||||
|
"skip run network instance because webhook session is no longer current"
|
||||||
|
);
|
||||||
|
return ConfigActionResult::StopRound;
|
||||||
|
}
|
||||||
|
|
||||||
|
let ret = rpc_client
|
||||||
|
.run_network_instance(
|
||||||
|
BaseController::default(),
|
||||||
|
RunNetworkInstanceRequest {
|
||||||
|
inst_id: Some(config.network_instance_id.clone().into()),
|
||||||
|
config: Some(desired_config),
|
||||||
|
overwrite: false,
|
||||||
|
source: PersistedConfigSource::from_db(&config.source).auto_run_rpc_source() as i32,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
tracing::info!(
|
||||||
|
user_id = ?round.user_id,
|
||||||
|
"Run network instance: {:?}, user_token: {:?}",
|
||||||
|
ret,
|
||||||
|
round.req.user_token
|
||||||
|
);
|
||||||
|
|
||||||
|
if ret.is_ok() {
|
||||||
|
ConfigActionResult::Success
|
||||||
|
} else {
|
||||||
|
ConfigActionResult::Failed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn remember_web_runtime_config_after_run(
|
||||||
|
rpc_client: &mut SessionRpcClient,
|
||||||
|
inst_id: &str,
|
||||||
|
desired_config: &NetworkConfig,
|
||||||
|
runtime_config_cache: &mut SessionRuntimeConfigCache,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let observed_config = runtime_reconcile::get_runtime_config(rpc_client, inst_id).await?;
|
||||||
|
remember_if_runtime_matches_desired(
|
||||||
|
inst_id,
|
||||||
|
desired_config,
|
||||||
|
observed_config,
|
||||||
|
runtime_config_cache,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remember_if_runtime_matches_desired(
|
||||||
|
inst_id: &str,
|
||||||
|
desired_config: &NetworkConfig,
|
||||||
|
observed_config: NetworkConfig,
|
||||||
|
runtime_config_cache: &mut SessionRuntimeConfigCache,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let action = runtime_reconcile::prepare_web_source_runtime_reconcile_from_current(
|
||||||
|
&observed_config,
|
||||||
|
desired_config.clone(),
|
||||||
|
)?;
|
||||||
|
if !matches!(action, runtime_reconcile::RuntimeReconcileAction::None) {
|
||||||
|
anyhow::bail!("runtime config still differs after managed run");
|
||||||
|
}
|
||||||
|
runtime_config_cache.remember(inst_id, observed_config);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn mark_config_revision_applied_if_current(
|
||||||
|
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||||
|
storage: &StorageInner,
|
||||||
|
round: &ReconcileRound,
|
||||||
|
outcome: &ReconcileOutcome,
|
||||||
|
) -> RoundStatus<()> {
|
||||||
|
if outcome.managed_revision_failed || !round.should_apply_runtime_revision {
|
||||||
|
return RoundStatus::Ready(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let current_target_config_revision = match storage
|
||||||
|
.db
|
||||||
|
.get_managed_config_revision((round.user_id, round.machine_id))
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(revision) => revision,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!("Failed to verify managed config revision, error: {:?}", e);
|
||||||
|
return RoundStatus::Stop;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if current_target_config_revision != round.target_config_revision {
|
||||||
|
return RoundStatus::Ready(());
|
||||||
|
}
|
||||||
|
let Some(data) = session_data.upgrade() else {
|
||||||
|
return RoundStatus::Stop;
|
||||||
|
};
|
||||||
|
let mut data = data.write().await;
|
||||||
|
if !SessionRpcService::runtime_heartbeat_is_current_locked(&data, &round.req) {
|
||||||
|
return RoundStatus::Ready(());
|
||||||
|
}
|
||||||
|
data.applied_config_revision = round.target_config_revision.clone();
|
||||||
|
|
||||||
|
RoundStatus::Ready(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use easytier::proto::api::manage::{NetworkingMethod, PortForwardConfig};
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn config_with_port_forwards(port_forwards: Vec<PortForwardConfig>) -> NetworkConfig {
|
||||||
|
NetworkConfig {
|
||||||
|
instance_id: Some("11111111-1111-1111-1111-111111111111".to_string()),
|
||||||
|
dhcp: Some(true),
|
||||||
|
network_name: Some("managed".to_string()),
|
||||||
|
network_secret: Some("secret".to_string()),
|
||||||
|
networking_method: Some(NetworkingMethod::Manual as i32),
|
||||||
|
port_forwards,
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn port_forward(bind_port: u32, dst_port: u32) -> PortForwardConfig {
|
||||||
|
PortForwardConfig {
|
||||||
|
bind_ip: "127.0.0.1".to_string(),
|
||||||
|
bind_port,
|
||||||
|
dst_ip: "10.144.0.1".to_string(),
|
||||||
|
dst_port,
|
||||||
|
proto: "tcp".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn session_runtime_config_cache_misses_unknown_instance() {
|
||||||
|
let cache = SessionRuntimeConfigCache::default();
|
||||||
|
let action = cache
|
||||||
|
.plan("missing", config_with_port_forwards(Vec::new()))
|
||||||
|
.expect("prepare action");
|
||||||
|
|
||||||
|
assert!(action.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn session_runtime_config_cache_skips_matching_observed_config() {
|
||||||
|
let mut cache = SessionRuntimeConfigCache::default();
|
||||||
|
let config = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||||
|
|
||||||
|
cache.remember("managed", config.clone());
|
||||||
|
let action = cache
|
||||||
|
.plan("managed", config)
|
||||||
|
.expect("prepare action")
|
||||||
|
.expect("cached action");
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
action,
|
||||||
|
runtime_reconcile::RuntimeReconcileAction::None
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn session_runtime_config_cache_plans_patch_from_observed_config() {
|
||||||
|
let mut cache = SessionRuntimeConfigCache::default();
|
||||||
|
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||||
|
let desired =
|
||||||
|
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||||
|
|
||||||
|
cache.remember("managed", current);
|
||||||
|
let action = cache
|
||||||
|
.plan("managed", desired)
|
||||||
|
.expect("prepare action")
|
||||||
|
.expect("cached action");
|
||||||
|
|
||||||
|
let runtime_reconcile::RuntimeReconcileAction::Patch(patch) = action else {
|
||||||
|
panic!("expected cached runtime config to produce hot patch");
|
||||||
|
};
|
||||||
|
assert_eq!(patch.port_forwards.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn session_runtime_config_cache_retain_desired_removes_stale_entries() {
|
||||||
|
let mut cache = SessionRuntimeConfigCache::default();
|
||||||
|
let config = config_with_port_forwards(Vec::new());
|
||||||
|
cache.remember("keep", config.clone());
|
||||||
|
cache.remember("drop", config);
|
||||||
|
|
||||||
|
cache.retain_desired(&HashSet::from(["keep".to_string()]));
|
||||||
|
|
||||||
|
assert!(cache.entries.contains_key("keep"));
|
||||||
|
assert!(!cache.entries.contains_key("drop"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn session_runtime_config_cache_forget_removes_observed_config() {
|
||||||
|
let mut cache = SessionRuntimeConfigCache::default();
|
||||||
|
let config = config_with_port_forwards(Vec::new());
|
||||||
|
cache.remember("managed", config.clone());
|
||||||
|
|
||||||
|
cache.forget("managed");
|
||||||
|
|
||||||
|
let action = cache
|
||||||
|
.plan("managed", config)
|
||||||
|
.expect("prepare action after remove");
|
||||||
|
assert!(action.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn missing_run_remembers_observed_config_when_it_matches_desired() {
|
||||||
|
let mut cache = SessionRuntimeConfigCache::default();
|
||||||
|
let config = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||||
|
|
||||||
|
remember_if_runtime_matches_desired("managed", &config, config.clone(), &mut cache)
|
||||||
|
.expect("remember observed config after run");
|
||||||
|
let action = cache
|
||||||
|
.plan("managed", config)
|
||||||
|
.expect("prepare action after run")
|
||||||
|
.expect("cached action");
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
action,
|
||||||
|
runtime_reconcile::RuntimeReconcileAction::None
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn missing_run_does_not_remember_observed_config_that_still_differs() {
|
||||||
|
let mut cache = SessionRuntimeConfigCache::default();
|
||||||
|
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||||
|
let desired =
|
||||||
|
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||||
|
|
||||||
|
let err = remember_if_runtime_matches_desired("managed", &desired, current, &mut cache)
|
||||||
|
.expect_err("expected stale run result not to be cached");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
err.to_string()
|
||||||
|
.contains("runtime config still differs after managed run")
|
||||||
|
);
|
||||||
|
let action = cache
|
||||||
|
.plan("managed", desired)
|
||||||
|
.expect("prepare action after stale run result");
|
||||||
|
assert!(action.is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,402 @@
|
|||||||
|
use std::{sync::Arc, time::Duration};
|
||||||
|
|
||||||
|
use anyhow::Context as _;
|
||||||
|
use easytier::proto::web::HeartbeatRequest;
|
||||||
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
SessionAuthState, SessionData, SessionRpcService, WebhookConnectNotification,
|
||||||
|
WebhookDisconnectNotification, send_webhook_connection_transition,
|
||||||
|
};
|
||||||
|
use crate::{
|
||||||
|
client_manager::storage::{Storage, StorageToken},
|
||||||
|
webhook::SharedWebhookConfig,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub(super) const VALIDATION_RETRY_MS: u64 = 60_000;
|
||||||
|
|
||||||
|
pub(super) struct WebhookHeartbeatValidation {
|
||||||
|
pub(super) config_revision: String,
|
||||||
|
pub(super) binding_version: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) struct WebhookValidationInput {
|
||||||
|
pub(super) storage: Storage,
|
||||||
|
pub(super) webhook_config: SharedWebhookConfig,
|
||||||
|
pub(super) client_url: url::Url,
|
||||||
|
pub(super) applied_config_revision: Option<String>,
|
||||||
|
pub(super) req: HeartbeatRequest,
|
||||||
|
pub(super) machine_id: uuid::Uuid,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn deterministic_machine_delay(machine_id: uuid::Uuid, max_delay_ms: u64) -> Duration {
|
||||||
|
let delay_ms = (machine_id.as_u128() % u128::from(max_delay_ms + 1)) as u64;
|
||||||
|
Duration::from_millis(delay_ms)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn retry_delay(machine_id: uuid::Uuid) -> Duration {
|
||||||
|
Duration::from_millis(VALIDATION_RETRY_MS)
|
||||||
|
+ deterministic_machine_delay(machine_id, VALIDATION_RETRY_MS)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn request_heartbeat_validation(
|
||||||
|
webhook_config: &crate::webhook::WebhookConfig,
|
||||||
|
client_url: &url::Url,
|
||||||
|
persisted_config_revision: Option<&str>,
|
||||||
|
applied_config_revision: Option<&str>,
|
||||||
|
req: &HeartbeatRequest,
|
||||||
|
machine_id: uuid::Uuid,
|
||||||
|
) -> anyhow::Result<Option<WebhookHeartbeatValidation>> {
|
||||||
|
let webhook_req = crate::webhook::ValidateTokenRequest {
|
||||||
|
token: req.user_token.clone(),
|
||||||
|
machine_id: machine_id.to_string(),
|
||||||
|
public_ip: client_url.host_str().map(str::to_string),
|
||||||
|
hostname: req.hostname.clone(),
|
||||||
|
version: req.easytier_version.clone(),
|
||||||
|
os_type: req.device_os.as_ref().map(|info| info.os_type.clone()),
|
||||||
|
os_version: req.device_os.as_ref().map(|info| info.version.clone()),
|
||||||
|
os_distribution: req.device_os.as_ref().map(|info| info.distribution.clone()),
|
||||||
|
web_instance_id: webhook_config.web_instance_id.clone(),
|
||||||
|
web_instance_api_base_url: webhook_config.web_instance_api_base_url.clone(),
|
||||||
|
persisted_config_revision: persisted_config_revision.map(str::to_string),
|
||||||
|
applied_config_revision: applied_config_revision.map(str::to_string),
|
||||||
|
};
|
||||||
|
let resp = webhook_config
|
||||||
|
.validate_token(&webhook_req)
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("Webhook token validation failed: {:?}", e))?;
|
||||||
|
|
||||||
|
if !resp.valid {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Some(WebhookHeartbeatValidation {
|
||||||
|
config_revision: resp.config_revision,
|
||||||
|
binding_version: resp.binding_version,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn resolve_user_id(storage: &Storage, token: &str) -> anyhow::Result<i32> {
|
||||||
|
let user_id = match storage
|
||||||
|
.db()
|
||||||
|
.get_user_id_by_token(token)
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("DB error: {:?}", e))?
|
||||||
|
{
|
||||||
|
Some(id) => id,
|
||||||
|
None => storage
|
||||||
|
.auto_create_user(token)
|
||||||
|
.await
|
||||||
|
.with_context(|| format!("Failed to auto-create webhook user: {:?}", token))?,
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(user_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn persisted_config_revision_for_token(
|
||||||
|
storage: &Storage,
|
||||||
|
token: &str,
|
||||||
|
machine_id: uuid::Uuid,
|
||||||
|
) -> anyhow::Result<Option<String>> {
|
||||||
|
let Some(user_id) = storage
|
||||||
|
.db()
|
||||||
|
.get_user_id_by_token(token)
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("DB error: {:?}", e))?
|
||||||
|
else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
storage
|
||||||
|
.db()
|
||||||
|
.get_managed_config_revision((user_id, machine_id))
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("DB error: {:?}", e))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_for_input(
|
||||||
|
session_data: std::sync::Weak<RwLock<SessionData>>,
|
||||||
|
) -> Option<WebhookValidationInput> {
|
||||||
|
loop {
|
||||||
|
let notify = {
|
||||||
|
let session_data = session_data.upgrade()?;
|
||||||
|
let mut data = session_data.write().await;
|
||||||
|
if matches!(data.auth_state, SessionAuthState::Invalid) {
|
||||||
|
data.webhook_validation_dirty = false;
|
||||||
|
tracing::info!(
|
||||||
|
client_url = %data.client_url,
|
||||||
|
"webhook validation stopped for invalid session; reconnect is required before revalidation"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if data.webhook_validation_dirty {
|
||||||
|
data.webhook_validation_dirty = false;
|
||||||
|
let req = data.req.clone()?;
|
||||||
|
let machine_id = req.machine_id.map(Into::into)?;
|
||||||
|
let storage = Storage::try_from(data.storage.clone()).ok()?;
|
||||||
|
return Some(WebhookValidationInput {
|
||||||
|
storage,
|
||||||
|
webhook_config: data.webhook_config.clone(),
|
||||||
|
client_url: data.client_url.clone(),
|
||||||
|
applied_config_revision: data.applied_config_revision.clone(),
|
||||||
|
req,
|
||||||
|
machine_id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
data.webhook_validation_notify.clone()
|
||||||
|
};
|
||||||
|
notify.notified().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn run_worker(session_data: std::sync::Weak<RwLock<SessionData>>) {
|
||||||
|
while let Some(input) = wait_for_input(session_data.clone()).await {
|
||||||
|
let machine_id = input.machine_id;
|
||||||
|
if let Err(error) = run_round(session_data.clone(), input).await {
|
||||||
|
tracing::warn!(
|
||||||
|
?machine_id,
|
||||||
|
%error,
|
||||||
|
"webhook validation failed, will retry later"
|
||||||
|
);
|
||||||
|
tokio::time::sleep(retry_delay(machine_id)).await;
|
||||||
|
mark_dirty_if_current(&session_data, machine_id).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn run_round(
|
||||||
|
session_data: std::sync::Weak<RwLock<SessionData>>,
|
||||||
|
input: WebhookValidationInput,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let persisted_config_revision = persisted_config_revision_for_token(
|
||||||
|
&input.storage,
|
||||||
|
&input.req.user_token,
|
||||||
|
input.machine_id,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let validation = request_heartbeat_validation(
|
||||||
|
&input.webhook_config,
|
||||||
|
&input.client_url,
|
||||||
|
persisted_config_revision.as_deref(),
|
||||||
|
input.applied_config_revision.as_deref(),
|
||||||
|
&input.req,
|
||||||
|
input.machine_id,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let Some(validation) = validation else {
|
||||||
|
apply_rejected(&session_data, &input).await;
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
|
||||||
|
let user_id = resolve_user_id(&input.storage, &input.req.user_token).await?;
|
||||||
|
apply_success(&session_data, input, validation, user_id).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn mark_dirty_if_current(
|
||||||
|
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||||
|
machine_id: uuid::Uuid,
|
||||||
|
) {
|
||||||
|
let Some(session_data) = session_data.upgrade() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let notify = {
|
||||||
|
let mut data = session_data.write().await;
|
||||||
|
let Some(req) = data.req.as_ref() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if req.machine_id.map(uuid::Uuid::from) != Some(machine_id) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if matches!(data.auth_state, SessionAuthState::Invalid) {
|
||||||
|
data.webhook_validation_dirty = false;
|
||||||
|
tracing::debug!(
|
||||||
|
%machine_id,
|
||||||
|
"skip webhook validation retry for invalid session"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
SessionRpcService::mark_webhook_validation_dirty_locked(&mut data)
|
||||||
|
};
|
||||||
|
notify.notify_one();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn apply_rejected(
|
||||||
|
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||||
|
input: &WebhookValidationInput,
|
||||||
|
) {
|
||||||
|
let Some(session_data) = session_data.upgrade() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let (storage_token, disconnect_notification) = {
|
||||||
|
let mut data = session_data.write().await;
|
||||||
|
if !data.req.as_ref().is_some_and(|req| {
|
||||||
|
SessionRpcService::heartbeat_matches_identity(
|
||||||
|
req,
|
||||||
|
&input.req.user_token,
|
||||||
|
input.machine_id,
|
||||||
|
)
|
||||||
|
}) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
tracing::info!(
|
||||||
|
machine_id = %input.machine_id,
|
||||||
|
client_url = %data.client_url,
|
||||||
|
"webhook token rejected; marking session invalid and requiring client reconnect"
|
||||||
|
);
|
||||||
|
data.auth_state = SessionAuthState::Invalid;
|
||||||
|
data.webhook_validation_dirty = false;
|
||||||
|
data.binding_version = None;
|
||||||
|
data.applied_config_revision = None;
|
||||||
|
let storage_token = data.storage_token.clone();
|
||||||
|
let disconnect_notification = storage_token.as_ref().and_then(|storage_token| {
|
||||||
|
data.webhook_connected_binding_version
|
||||||
|
.take()
|
||||||
|
.map(|binding_version| WebhookDisconnectNotification {
|
||||||
|
webhook: data.webhook_config.clone(),
|
||||||
|
storage_token: storage_token.clone(),
|
||||||
|
binding_version,
|
||||||
|
})
|
||||||
|
});
|
||||||
|
(storage_token, disconnect_notification)
|
||||||
|
};
|
||||||
|
if let Some(storage_token) = storage_token {
|
||||||
|
let report_time = SessionRpcService::heartbeat_report_timestamp(&input.req);
|
||||||
|
input
|
||||||
|
.storage
|
||||||
|
.update_client(storage_token, report_time, false);
|
||||||
|
}
|
||||||
|
if disconnect_notification.is_some() {
|
||||||
|
wait_webhook_connection_transition(
|
||||||
|
Arc::downgrade(&session_data),
|
||||||
|
disconnect_notification,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn apply_success(
|
||||||
|
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||||
|
input: WebhookValidationInput,
|
||||||
|
validation: WebhookHeartbeatValidation,
|
||||||
|
user_id: i32,
|
||||||
|
) {
|
||||||
|
let WebhookHeartbeatValidation {
|
||||||
|
config_revision: _,
|
||||||
|
binding_version,
|
||||||
|
} = validation;
|
||||||
|
|
||||||
|
let Some(session_data) = session_data.upgrade() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let (storage_token, notifier, disconnect_notification, connect_notification, runtime_req) = {
|
||||||
|
let mut data = session_data.write().await;
|
||||||
|
let Some(runtime_req) = data.req.clone() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if !SessionRpcService::heartbeat_matches_identity(
|
||||||
|
&runtime_req,
|
||||||
|
&input.req.user_token,
|
||||||
|
input.machine_id,
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if matches!(data.auth_state, SessionAuthState::Invalid) {
|
||||||
|
tracing::info!(
|
||||||
|
machine_id = %input.machine_id,
|
||||||
|
client_url = %data.client_url,
|
||||||
|
"ignore webhook validation success for invalid session; reconnect is required before revalidation"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let previous_connected_binding_version = data.webhook_connected_binding_version;
|
||||||
|
let client_url = data.client_url.clone();
|
||||||
|
let storage_token = data.storage_token.get_or_insert_with(|| StorageToken {
|
||||||
|
token: runtime_req.user_token.clone(),
|
||||||
|
client_url,
|
||||||
|
machine_id: input.machine_id,
|
||||||
|
user_id,
|
||||||
|
});
|
||||||
|
let storage_token = storage_token.clone();
|
||||||
|
data.auth_state = SessionAuthState::Authorized;
|
||||||
|
data.binding_version = Some(binding_version);
|
||||||
|
let should_notify_connected = previous_connected_binding_version != Some(binding_version);
|
||||||
|
let disconnect_notification = previous_connected_binding_version
|
||||||
|
.filter(|previous_binding_version| *previous_binding_version != binding_version)
|
||||||
|
.map(|previous_binding_version| {
|
||||||
|
data.webhook_connected_binding_version = None;
|
||||||
|
WebhookDisconnectNotification {
|
||||||
|
webhook: data.webhook_config.clone(),
|
||||||
|
storage_token: storage_token.clone(),
|
||||||
|
binding_version: previous_binding_version,
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let connect_notification = should_notify_connected.then(|| WebhookConnectNotification {
|
||||||
|
webhook: data.webhook_config.clone(),
|
||||||
|
storage_token: storage_token.clone(),
|
||||||
|
binding_version,
|
||||||
|
req: crate::webhook::NodeConnectedRequest {
|
||||||
|
machine_id: input.machine_id.to_string(),
|
||||||
|
token: runtime_req.user_token.clone(),
|
||||||
|
user_id: Some(user_id),
|
||||||
|
hostname: runtime_req.hostname.clone(),
|
||||||
|
version: runtime_req.easytier_version.clone(),
|
||||||
|
os_type: runtime_req
|
||||||
|
.device_os
|
||||||
|
.as_ref()
|
||||||
|
.map(|info| info.os_type.clone()),
|
||||||
|
os_version: runtime_req
|
||||||
|
.device_os
|
||||||
|
.as_ref()
|
||||||
|
.map(|info| info.version.clone()),
|
||||||
|
os_distribution: runtime_req
|
||||||
|
.device_os
|
||||||
|
.as_ref()
|
||||||
|
.map(|info| info.distribution.clone()),
|
||||||
|
web_instance_id: data.webhook_config.web_instance_id.clone(),
|
||||||
|
binding_version: Some(binding_version),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
(
|
||||||
|
storage_token,
|
||||||
|
data.notifier.clone(),
|
||||||
|
disconnect_notification,
|
||||||
|
connect_notification,
|
||||||
|
runtime_req,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
|
if disconnect_notification.is_some() || connect_notification.is_some() {
|
||||||
|
wait_webhook_connection_transition(
|
||||||
|
Arc::downgrade(&session_data),
|
||||||
|
disconnect_notification,
|
||||||
|
connect_notification,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
let report_time = SessionRpcService::heartbeat_report_timestamp(&runtime_req);
|
||||||
|
input
|
||||||
|
.storage
|
||||||
|
.update_client(storage_token, report_time, true);
|
||||||
|
let _ = notifier.send(runtime_req);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_webhook_connection_transition(
|
||||||
|
session_data: std::sync::Weak<RwLock<SessionData>>,
|
||||||
|
disconnect: Option<WebhookDisconnectNotification>,
|
||||||
|
connect: Option<WebhookConnectNotification>,
|
||||||
|
) {
|
||||||
|
let transition = tokio::spawn(send_webhook_connection_transition(
|
||||||
|
session_data,
|
||||||
|
disconnect,
|
||||||
|
connect,
|
||||||
|
));
|
||||||
|
if let Err(error) = transition.await {
|
||||||
|
tracing::warn!(%error, "webhook connection transition task failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,6 +17,7 @@ pub struct StorageToken {
|
|||||||
struct ClientInfo {
|
struct ClientInfo {
|
||||||
storage_token: StorageToken,
|
storage_token: StorageToken,
|
||||||
report_time: i64,
|
report_time: i64,
|
||||||
|
authorized: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@@ -55,7 +56,19 @@ impl Storage {
|
|||||||
fn update_client_info_map(map: &DashMap<uuid::Uuid, ClientInfo>, client_info: &ClientInfo) {
|
fn update_client_info_map(map: &DashMap<uuid::Uuid, ClientInfo>, client_info: &ClientInfo) {
|
||||||
map.entry(client_info.storage_token.machine_id)
|
map.entry(client_info.storage_token.machine_id)
|
||||||
.and_modify(|e| {
|
.and_modify(|e| {
|
||||||
if e.report_time < client_info.report_time {
|
let same_client = e.storage_token.client_url
|
||||||
|
== client_info.storage_token.client_url
|
||||||
|
&& e.storage_token.user_id == client_info.storage_token.user_id;
|
||||||
|
let should_replace = if (same_client && e.authorized != client_info.authorized)
|
||||||
|
|| (!e.authorized && client_info.authorized)
|
||||||
|
{
|
||||||
|
true
|
||||||
|
} else if e.authorized && !client_info.authorized && !same_client {
|
||||||
|
false
|
||||||
|
} else {
|
||||||
|
e.report_time < client_info.report_time
|
||||||
|
};
|
||||||
|
if should_replace {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
e.storage_token.machine_id,
|
e.storage_token.machine_id,
|
||||||
client_info.storage_token.machine_id
|
client_info.storage_token.machine_id
|
||||||
@@ -66,12 +79,13 @@ impl Storage {
|
|||||||
.or_insert(client_info.clone());
|
.or_insert(client_info.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn update_client(&self, stoken: StorageToken, report_time: i64) {
|
pub fn update_client(&self, stoken: StorageToken, report_time: i64, authorized: bool) {
|
||||||
let inner = self.0.user_clients_map.entry(stoken.user_id).or_default();
|
let inner = self.0.user_clients_map.entry(stoken.user_id).or_default();
|
||||||
|
|
||||||
let client_info = ClientInfo {
|
let client_info = ClientInfo {
|
||||||
storage_token: stoken.clone(),
|
storage_token: stoken.clone(),
|
||||||
report_time,
|
report_time,
|
||||||
|
authorized,
|
||||||
};
|
};
|
||||||
Self::update_client_info_map(&inner, &client_info);
|
Self::update_client_info_map(&inner, &client_info);
|
||||||
}
|
}
|
||||||
@@ -93,11 +107,21 @@ impl Storage {
|
|||||||
&self,
|
&self,
|
||||||
user_id: UserIdInDb,
|
user_id: UserIdInDb,
|
||||||
machine_id: &uuid::Uuid,
|
machine_id: &uuid::Uuid,
|
||||||
|
) -> Option<url::Url> {
|
||||||
|
self.get_client_url_by_machine_id_with_auth(user_id, machine_id, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_client_url_by_machine_id_with_auth(
|
||||||
|
&self,
|
||||||
|
user_id: UserIdInDb,
|
||||||
|
machine_id: &uuid::Uuid,
|
||||||
|
require_authorized: bool,
|
||||||
) -> Option<url::Url> {
|
) -> Option<url::Url> {
|
||||||
self.0.user_clients_map.get(&user_id).and_then(|info_map| {
|
self.0.user_clients_map.get(&user_id).and_then(|info_map| {
|
||||||
info_map
|
info_map.get(machine_id).and_then(|info| {
|
||||||
.get(machine_id)
|
(!require_authorized || info.authorized)
|
||||||
.map(|info| info.storage_token.client_url.clone())
|
.then(|| info.storage_token.client_url.clone())
|
||||||
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -108,6 +132,7 @@ impl Storage {
|
|||||||
.map(|info_map| {
|
.map(|info_map| {
|
||||||
info_map
|
info_map
|
||||||
.iter()
|
.iter()
|
||||||
|
.filter(|info| info.value().authorized)
|
||||||
.map(|info| info.value().storage_token.client_url.clone())
|
.map(|info| info.value().storage_token.client_url.clone())
|
||||||
.collect()
|
.collect()
|
||||||
})
|
})
|
||||||
@@ -115,6 +140,14 @@ impl Storage {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn list_clients(&self) -> Vec<StorageToken> {
|
pub fn list_clients(&self) -> Vec<StorageToken> {
|
||||||
|
self.list_clients_with_auth(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_all_clients(&self) -> Vec<StorageToken> {
|
||||||
|
self.list_clients_with_auth(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn list_clients_with_auth(&self, require_authorized: bool) -> Vec<StorageToken> {
|
||||||
self.0
|
self.0
|
||||||
.user_clients_map
|
.user_clients_map
|
||||||
.iter()
|
.iter()
|
||||||
@@ -122,6 +155,7 @@ impl Storage {
|
|||||||
user_clients
|
user_clients
|
||||||
.value()
|
.value()
|
||||||
.iter()
|
.iter()
|
||||||
|
.filter(|info| !require_authorized || info.value().authorized)
|
||||||
.map(|info| info.value().storage_token.clone())
|
.map(|info| info.value().storage_token.clone())
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
})
|
})
|
||||||
@@ -164,8 +198,8 @@ mod tests {
|
|||||||
let user1_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
let user1_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||||
let user2_token = make_storage_token(2, machine_id, "tcp://127.0.0.1:1002");
|
let user2_token = make_storage_token(2, machine_id, "tcp://127.0.0.1:1002");
|
||||||
|
|
||||||
storage.update_client(user1_token.clone(), 10);
|
storage.update_client(user1_token.clone(), 10, true);
|
||||||
storage.update_client(user2_token.clone(), 20);
|
storage.update_client(user2_token.clone(), 20, true);
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
storage.get_client_url_by_machine_id(1, &machine_id),
|
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||||
@@ -195,8 +229,8 @@ mod tests {
|
|||||||
let user1_token = make_storage_token(1, uuid::Uuid::new_v4(), "tcp://127.0.0.1:1001");
|
let user1_token = make_storage_token(1, uuid::Uuid::new_v4(), "tcp://127.0.0.1:1001");
|
||||||
let user2_token = make_storage_token(2, uuid::Uuid::new_v4(), "tcp://127.0.0.1:1002");
|
let user2_token = make_storage_token(2, uuid::Uuid::new_v4(), "tcp://127.0.0.1:1002");
|
||||||
|
|
||||||
storage.update_client(user1_token.clone(), 10);
|
storage.update_client(user1_token.clone(), 10, true);
|
||||||
storage.update_client(user2_token.clone(), 20);
|
storage.update_client(user2_token.clone(), 20, true);
|
||||||
|
|
||||||
let tokens = storage.list_clients();
|
let tokens = storage.list_clients();
|
||||||
assert_eq!(tokens.len(), 2);
|
assert_eq!(tokens.len(), 2);
|
||||||
@@ -209,4 +243,84 @@ mod tests {
|
|||||||
assert_eq!(tokens.len(), 1);
|
assert_eq!(tokens.len(), 1);
|
||||||
assert_eq!(tokens[0].token, user2_token.token);
|
assert_eq!(tokens[0].token, user2_token.token);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn pending_client_is_listed_but_not_authorized_for_machine_lookup() {
|
||||||
|
let storage = Storage::new(Db::memory_db().await);
|
||||||
|
let machine_id = uuid::Uuid::new_v4();
|
||||||
|
let token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||||
|
|
||||||
|
storage.update_client(token.clone(), 10, false);
|
||||||
|
|
||||||
|
assert_eq!(storage.list_clients().len(), 0);
|
||||||
|
assert_eq!(storage.list_all_clients().len(), 1);
|
||||||
|
assert_eq!(storage.list_user_clients(1), Vec::<url::Url>::new());
|
||||||
|
assert_eq!(storage.get_client_url_by_machine_id(1, &machine_id), None);
|
||||||
|
assert_eq!(
|
||||||
|
storage.get_client_url_by_machine_id_with_auth(1, &machine_id, false),
|
||||||
|
Some(token.client_url.clone())
|
||||||
|
);
|
||||||
|
|
||||||
|
storage.update_client(token.clone(), 11, true);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||||
|
Some(token.client_url.clone())
|
||||||
|
);
|
||||||
|
|
||||||
|
storage.update_client(token.clone(), 11, false);
|
||||||
|
|
||||||
|
assert_eq!(storage.get_client_url_by_machine_id(1, &machine_id), None);
|
||||||
|
assert_eq!(storage.list_clients().len(), 0);
|
||||||
|
assert_eq!(storage.list_all_clients().len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stale_client_authorization_update_does_not_replace_newer_client() {
|
||||||
|
let storage = Storage::new(Db::memory_db().await);
|
||||||
|
let machine_id = uuid::Uuid::new_v4();
|
||||||
|
let old_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||||
|
let new_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002");
|
||||||
|
|
||||||
|
storage.update_client(old_token.clone(), 10, true);
|
||||||
|
storage.update_client(new_token.clone(), 20, true);
|
||||||
|
storage.update_client(old_token, 10, false);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||||
|
Some(new_token.client_url)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn pending_client_does_not_replace_authorized_route() {
|
||||||
|
let storage = Storage::new(Db::memory_db().await);
|
||||||
|
let machine_id = uuid::Uuid::new_v4();
|
||||||
|
let authorized_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||||
|
let pending_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002");
|
||||||
|
|
||||||
|
storage.update_client(authorized_token.clone(), 10, true);
|
||||||
|
storage.update_client(pending_token, i64::MAX, false);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||||
|
Some(authorized_token.client_url)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn authorized_client_replaces_pending_route_regardless_of_report_time() {
|
||||||
|
let storage = Storage::new(Db::memory_db().await);
|
||||||
|
let machine_id = uuid::Uuid::new_v4();
|
||||||
|
let pending_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||||
|
let authorized_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002");
|
||||||
|
|
||||||
|
storage.update_client(pending_token, i64::MAX, false);
|
||||||
|
storage.update_client(authorized_token.clone(), 10, true);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||||
|
Some(authorized_token.client_url)
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
//! `SeaORM` Entity, hand-written to match the generated entity style.
|
||||||
|
|
||||||
|
use sea_orm::entity::prelude::*;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
|
||||||
|
#[sea_orm(table_name = "managed_config_revisions")]
|
||||||
|
pub struct Model {
|
||||||
|
#[sea_orm(primary_key)]
|
||||||
|
pub id: i32,
|
||||||
|
pub user_id: i32,
|
||||||
|
#[sea_orm(column_type = "Text")]
|
||||||
|
pub device_id: String,
|
||||||
|
#[sea_orm(column_type = "Text")]
|
||||||
|
pub config_revision: String,
|
||||||
|
pub create_time: DateTimeWithTimeZone,
|
||||||
|
pub update_time: DateTimeWithTimeZone,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||||
|
pub enum Relation {
|
||||||
|
#[sea_orm(
|
||||||
|
belongs_to = "super::users::Entity",
|
||||||
|
from = "Column::UserId",
|
||||||
|
to = "super::users::Column::Id",
|
||||||
|
on_update = "Cascade",
|
||||||
|
on_delete = "Cascade"
|
||||||
|
)]
|
||||||
|
Users,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Related<super::users::Entity> for Entity {
|
||||||
|
fn to() -> RelationDef {
|
||||||
|
Relation::Users.def()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ActiveModelBehavior for ActiveModel {}
|
||||||
@@ -4,6 +4,7 @@ pub mod prelude;
|
|||||||
|
|
||||||
pub mod groups;
|
pub mod groups;
|
||||||
pub mod groups_permissions;
|
pub mod groups_permissions;
|
||||||
|
pub mod managed_config_revisions;
|
||||||
pub mod permissions;
|
pub mod permissions;
|
||||||
pub mod tower_sessions;
|
pub mod tower_sessions;
|
||||||
pub mod user_running_network_configs;
|
pub mod user_running_network_configs;
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
pub use super::groups::Entity as Groups;
|
pub use super::groups::Entity as Groups;
|
||||||
pub use super::groups_permissions::Entity as GroupsPermissions;
|
pub use super::groups_permissions::Entity as GroupsPermissions;
|
||||||
|
pub use super::managed_config_revisions::Entity as ManagedConfigRevisions;
|
||||||
pub use super::permissions::Entity as Permissions;
|
pub use super::permissions::Entity as Permissions;
|
||||||
pub use super::tower_sessions::Entity as TowerSessions;
|
pub use super::tower_sessions::Entity as TowerSessions;
|
||||||
pub use super::user_running_network_configs::Entity as UserRunningNetworkConfigs;
|
pub use super::user_running_network_configs::Entity as UserRunningNetworkConfigs;
|
||||||
|
|||||||
@@ -141,6 +141,110 @@ impl Db {
|
|||||||
) -> Result<Option<UserIdInDb>, DbErr> {
|
) -> Result<Option<UserIdInDb>, DbErr> {
|
||||||
self.get_user_id(token).await
|
self.get_user_id(token).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn get_managed_config_revision(
|
||||||
|
&self,
|
||||||
|
(user_id, device_id): (UserIdInDb, Uuid),
|
||||||
|
) -> Result<Option<String>, DbErr> {
|
||||||
|
use entity::managed_config_revisions as mcr;
|
||||||
|
|
||||||
|
let revision = mcr::Entity::find()
|
||||||
|
.filter(mcr::Column::UserId.eq(user_id))
|
||||||
|
.filter(mcr::Column::DeviceId.eq(device_id.to_string()))
|
||||||
|
.one(self.orm_db())
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(revision.map(|row| row.config_revision))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn set_managed_config_revision(
|
||||||
|
&self,
|
||||||
|
(user_id, device_id): (UserIdInDb, Uuid),
|
||||||
|
config_revision: &str,
|
||||||
|
) -> Result<(), DbErr> {
|
||||||
|
use entity::managed_config_revisions as mcr;
|
||||||
|
|
||||||
|
let now = chrono::Local::now().fixed_offset();
|
||||||
|
let on_conflict = OnConflict::columns([mcr::Column::UserId, mcr::Column::DeviceId])
|
||||||
|
.update_columns([mcr::Column::ConfigRevision, mcr::Column::UpdateTime])
|
||||||
|
.to_owned();
|
||||||
|
let insert_m = mcr::ActiveModel {
|
||||||
|
user_id: Set(user_id),
|
||||||
|
device_id: Set(device_id.to_string()),
|
||||||
|
config_revision: Set(config_revision.to_string()),
|
||||||
|
create_time: Set(now),
|
||||||
|
update_time: Set(now),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
mcr::Entity::insert(insert_m)
|
||||||
|
.on_conflict(on_conflict)
|
||||||
|
.do_nothing()
|
||||||
|
.exec(self.orm_db())
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn insert_or_update_web_network_config(
|
||||||
|
&self,
|
||||||
|
(user_id, device_id): (UserIdInDb, Uuid),
|
||||||
|
network_inst_id: Uuid,
|
||||||
|
network_config: NetworkConfig,
|
||||||
|
) -> Result<bool, DbErr> {
|
||||||
|
let now = chrono::Local::now().fixed_offset();
|
||||||
|
let network_config =
|
||||||
|
serde_json::to_string(&network_config).map_err(|e| DbErr::Json(e.to_string()))?;
|
||||||
|
let source = ConfigSource::Web.as_str();
|
||||||
|
let result = sqlx::query(
|
||||||
|
r#"
|
||||||
|
INSERT INTO user_running_network_configs (
|
||||||
|
user_id, device_id, network_instance_id, network_config,
|
||||||
|
source, disabled, create_time, update_time
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(user_id, device_id, network_instance_id) DO UPDATE SET
|
||||||
|
network_config = excluded.network_config,
|
||||||
|
source = excluded.source,
|
||||||
|
disabled = excluded.disabled,
|
||||||
|
update_time = excluded.update_time
|
||||||
|
WHERE user_running_network_configs.source = ?
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(user_id)
|
||||||
|
.bind(device_id.to_string())
|
||||||
|
.bind(network_inst_id.to_string())
|
||||||
|
.bind(network_config)
|
||||||
|
.bind(source)
|
||||||
|
.bind(false)
|
||||||
|
.bind(now)
|
||||||
|
.bind(now)
|
||||||
|
.bind(source)
|
||||||
|
.execute(&self.db)
|
||||||
|
.await
|
||||||
|
.map_err(|e| DbErr::Custom(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(result.rows_affected() > 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn delete_web_network_configs(
|
||||||
|
&self,
|
||||||
|
(user_id, device_id): (UserIdInDb, Uuid),
|
||||||
|
network_inst_ids: &[Uuid],
|
||||||
|
) -> Result<(), DbErr> {
|
||||||
|
use entity::user_running_network_configs as urnc;
|
||||||
|
|
||||||
|
urnc::Entity::delete_many()
|
||||||
|
.filter(urnc::Column::UserId.eq(user_id))
|
||||||
|
.filter(urnc::Column::DeviceId.eq(device_id.to_string()))
|
||||||
|
.filter(urnc::Column::Source.eq(ConfigSource::Web.as_str()))
|
||||||
|
.filter(
|
||||||
|
urnc::Column::NetworkInstanceId
|
||||||
|
.is_in(network_inst_ids.iter().map(|id| id.to_string())),
|
||||||
|
)
|
||||||
|
.exec(self.orm_db())
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
@@ -468,4 +572,46 @@ mod tests {
|
|||||||
assert_eq!(device1_configs.len(), 1);
|
assert_eq!(device1_configs.len(), 1);
|
||||||
assert_eq!(device2_configs.len(), 1);
|
assert_eq!(device2_configs.len(), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_web_network_config_does_not_replace_user_owned_config() {
|
||||||
|
let db = Db::memory_db().await;
|
||||||
|
let user_id = db.auto_create_user("user-web-race").await.unwrap().id;
|
||||||
|
let device_id = uuid::Uuid::new_v4();
|
||||||
|
let inst_id = uuid::Uuid::new_v4();
|
||||||
|
|
||||||
|
db.insert_or_update_user_network_config(
|
||||||
|
(user_id, device_id),
|
||||||
|
inst_id,
|
||||||
|
NetworkConfig {
|
||||||
|
network_name: Some("user-owned".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
ConfigSource::User,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let updated = db
|
||||||
|
.insert_or_update_web_network_config(
|
||||||
|
(user_id, device_id),
|
||||||
|
inst_id,
|
||||||
|
NetworkConfig {
|
||||||
|
network_name: Some("web-owned".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(!updated);
|
||||||
|
let saved = db
|
||||||
|
.get_network_config((user_id, device_id), &inst_id.to_string())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(saved.get_network_config_source(), ConfigSource::User);
|
||||||
|
let saved_config = saved.get_network_config().unwrap();
|
||||||
|
assert_eq!(saved_config.network_name.as_deref(), Some("user-owned"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,8 +3,8 @@
|
|||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate rust_i18n;
|
extern crate rust_i18n;
|
||||||
|
|
||||||
use std::net::IpAddr;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::{net::IpAddr, time::Duration};
|
||||||
|
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use easytier::tunnel::websocket::WsTunnelListener;
|
use easytier::tunnel::websocket::WsTunnelListener;
|
||||||
@@ -113,6 +113,14 @@ struct Cli {
|
|||||||
)]
|
)]
|
||||||
geoip_db: Option<String>,
|
geoip_db: Option<String>,
|
||||||
|
|
||||||
|
#[arg(
|
||||||
|
long,
|
||||||
|
env = "ET_HEARTBEAT_MIN_RESPONSE_MS",
|
||||||
|
default_value = "0",
|
||||||
|
help = t!("cli.heartbeat_min_response_ms").to_string(),
|
||||||
|
)]
|
||||||
|
heartbeat_min_response_ms: u64,
|
||||||
|
|
||||||
#[cfg(feature = "embed")]
|
#[cfg(feature = "embed")]
|
||||||
#[arg(
|
#[arg(
|
||||||
long,
|
long,
|
||||||
@@ -312,6 +320,7 @@ async fn main() {
|
|||||||
let mut mgr = client_manager::ClientManager::new(
|
let mut mgr = client_manager::ClientManager::new(
|
||||||
db.clone(),
|
db.clone(),
|
||||||
cli.geoip_db,
|
cli.geoip_db,
|
||||||
|
Duration::from_millis(cli.heartbeat_min_response_ms),
|
||||||
feature_flags.clone(),
|
feature_flags.clone(),
|
||||||
webhook_config.clone(),
|
webhook_config.clone(),
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
use sea_orm_migration::prelude::*;
|
||||||
|
|
||||||
|
pub struct Migration;
|
||||||
|
|
||||||
|
impl MigrationName for Migration {
|
||||||
|
fn name(&self) -> &str {
|
||||||
|
"m20260619_000005_managed_config_revisions"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl MigrationTrait for Migration {
|
||||||
|
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||||
|
manager
|
||||||
|
.get_connection()
|
||||||
|
.execute_unprepared(
|
||||||
|
r#"
|
||||||
|
CREATE TABLE managed_config_revisions (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
user_id INTEGER NOT NULL,
|
||||||
|
device_id TEXT NOT NULL,
|
||||||
|
config_revision TEXT NOT NULL,
|
||||||
|
create_time TEXT NOT NULL,
|
||||||
|
update_time TEXT NOT NULL,
|
||||||
|
CONSTRAINT fk_managed_config_revisions_user_id_to_users_id
|
||||||
|
FOREIGN KEY (user_id) REFERENCES users(id)
|
||||||
|
ON DELETE CASCADE
|
||||||
|
ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX idx_managed_config_revisions_scope
|
||||||
|
ON managed_config_revisions(user_id, device_id);
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||||
|
manager
|
||||||
|
.get_connection()
|
||||||
|
.execute_unprepared("DROP TABLE managed_config_revisions;")
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ mod m20241029_000001_init;
|
|||||||
mod m20260403_000002_scope_network_config_unique;
|
mod m20260403_000002_scope_network_config_unique;
|
||||||
mod m20260421_000003_add_network_config_source;
|
mod m20260421_000003_add_network_config_source;
|
||||||
mod m20260514_000004_rename_web_config_source;
|
mod m20260514_000004_rename_web_config_source;
|
||||||
|
mod m20260619_000005_managed_config_revisions;
|
||||||
|
|
||||||
pub struct Migrator;
|
pub struct Migrator;
|
||||||
|
|
||||||
@@ -15,6 +16,7 @@ impl MigratorTrait for Migrator {
|
|||||||
Box::new(m20260403_000002_scope_network_config_unique::Migration),
|
Box::new(m20260403_000002_scope_network_config_unique::Migration),
|
||||||
Box::new(m20260421_000003_add_network_config_source::Migration),
|
Box::new(m20260421_000003_add_network_config_source::Migration),
|
||||||
Box::new(m20260514_000004_rename_web_config_source::Migration),
|
Box::new(m20260514_000004_rename_web_config_source::Migration),
|
||||||
|
Box::new(m20260619_000005_managed_config_revisions::Migration),
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -307,7 +307,7 @@ impl RestfulServer {
|
|||||||
async fn handle_list_all_sessions_internal(
|
async fn handle_list_all_sessions_internal(
|
||||||
State(client_mgr): AppState,
|
State(client_mgr): AppState,
|
||||||
) -> Result<Json<ListSessionJsonResp>, HttpHandleError> {
|
) -> Result<Json<ListSessionJsonResp>, HttpHandleError> {
|
||||||
let ret = client_mgr.list_sessions().await;
|
let ret = client_mgr.list_all_sessions().await;
|
||||||
Ok(ListSessionJsonResp(ret).into())
|
Ok(ListSessionJsonResp(ret).into())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -93,6 +93,8 @@ struct ManagedNetworkConfigJson {
|
|||||||
#[derive(Debug, serde::Deserialize, serde::Serialize)]
|
#[derive(Debug, serde::Deserialize, serde::Serialize)]
|
||||||
struct ReconcileManagedNetworkConfigsJsonReq {
|
struct ReconcileManagedNetworkConfigsJsonReq {
|
||||||
managed_network_configs: Vec<ManagedNetworkConfigJson>,
|
managed_network_configs: Vec<ManagedNetworkConfigJson>,
|
||||||
|
config_revision: Option<String>,
|
||||||
|
expected_config_revision: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, serde::Deserialize, serde::Serialize)]
|
#[derive(Debug, serde::Deserialize, serde::Serialize)]
|
||||||
@@ -357,13 +359,21 @@ impl NetworkApi {
|
|||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
client_mgr
|
client_mgr
|
||||||
.reconcile_managed_network_configs(user_id, machine_id, desired)
|
.reconcile_managed_network_configs(
|
||||||
|
user_id,
|
||||||
|
machine_id,
|
||||||
|
desired,
|
||||||
|
payload.config_revision,
|
||||||
|
payload.expected_config_revision,
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|err| {
|
.map_err(|err| {
|
||||||
(
|
let status = if crate::client_manager::is_managed_config_revision_conflict(&err) {
|
||||||
StatusCode::INTERNAL_SERVER_ERROR,
|
StatusCode::CONFLICT
|
||||||
other_error(err.to_string()).into(),
|
} else {
|
||||||
)
|
StatusCode::INTERNAL_SERVER_ERROR
|
||||||
|
};
|
||||||
|
(status, other_error(err.to_string()).into())
|
||||||
})?;
|
})?;
|
||||||
Ok(Void::default().into())
|
Ok(Void::default().into())
|
||||||
}
|
}
|
||||||
|
|||||||
+589
-16
@@ -1,6 +1,248 @@
|
|||||||
use std::sync::Arc;
|
use std::{
|
||||||
|
cmp::Ordering,
|
||||||
|
collections::VecDeque,
|
||||||
|
fmt,
|
||||||
|
sync::{Arc, Mutex},
|
||||||
|
time::{Duration, Instant},
|
||||||
|
};
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
use tokio::sync::oneshot;
|
||||||
|
|
||||||
|
const VALIDATE_TOKEN_INITIAL_CONCURRENCY: usize = 8;
|
||||||
|
const VALIDATE_TOKEN_MIN_CONCURRENCY: usize = 2;
|
||||||
|
const VALIDATE_TOKEN_MAX_CONCURRENCY: usize = 64;
|
||||||
|
const VALIDATE_TOKEN_ADJUST_WINDOW: Duration = Duration::from_secs(1);
|
||||||
|
const VALIDATE_TOKEN_SLOW_THRESHOLD: Duration = Duration::from_secs(2);
|
||||||
|
const WEBHOOK_HTTP_TIMEOUT: Duration = Duration::from_secs(10);
|
||||||
|
|
||||||
|
struct AdaptiveValidateLimiter {
|
||||||
|
state: Mutex<AdaptiveValidateLimiterState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Debug for AdaptiveValidateLimiter {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
f.debug_struct("AdaptiveValidateLimiter")
|
||||||
|
.field("state", &self.lock_state())
|
||||||
|
.finish_non_exhaustive()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct AdaptiveValidateLimiterState {
|
||||||
|
limit: usize,
|
||||||
|
in_flight: usize,
|
||||||
|
waiters: VecDeque<oneshot::Sender<AdaptiveValidateGrant>>,
|
||||||
|
window_started_at: Instant,
|
||||||
|
samples: usize,
|
||||||
|
slow_samples: usize,
|
||||||
|
failures: usize,
|
||||||
|
had_queue: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Debug for AdaptiveValidateLimiterState {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
f.debug_struct("AdaptiveValidateLimiterState")
|
||||||
|
.field("limit", &self.limit)
|
||||||
|
.field("in_flight", &self.in_flight)
|
||||||
|
.field("waiters", &self.waiters.len())
|
||||||
|
.field("window_started_at", &self.window_started_at)
|
||||||
|
.field("samples", &self.samples)
|
||||||
|
.field("slow_samples", &self.slow_samples)
|
||||||
|
.field("failures", &self.failures)
|
||||||
|
.field("had_queue", &self.had_queue)
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct AdaptiveValidatePermit {
|
||||||
|
limiter: Arc<AdaptiveValidateLimiter>,
|
||||||
|
started_at: Instant,
|
||||||
|
completed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct AdaptiveValidateGrant {
|
||||||
|
limiter: Arc<AdaptiveValidateLimiter>,
|
||||||
|
active: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
enum LimitAdjustment {
|
||||||
|
Unchanged,
|
||||||
|
Increased,
|
||||||
|
Decreased,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AdaptiveValidateLimiter {
|
||||||
|
fn new() -> Arc<Self> {
|
||||||
|
Arc::new(Self {
|
||||||
|
state: Mutex::new(AdaptiveValidateLimiterState::new(Instant::now())),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn acquire(self: &Arc<Self>) -> AdaptiveValidatePermit {
|
||||||
|
loop {
|
||||||
|
let receiver = {
|
||||||
|
let mut state = self.lock_state();
|
||||||
|
state.complete_window_if_due(Instant::now());
|
||||||
|
if state.waiters.is_empty() && state.in_flight < state.limit {
|
||||||
|
state.in_flight += 1;
|
||||||
|
return AdaptiveValidatePermit::new(self.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
let (sender, receiver) = oneshot::channel();
|
||||||
|
state.had_queue = true;
|
||||||
|
state.waiters.push_back(sender);
|
||||||
|
self.grant_waiters(&mut state);
|
||||||
|
receiver
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Ok(grant) = receiver.await {
|
||||||
|
return grant.into_permit();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn grant_waiters(self: &Arc<Self>, state: &mut AdaptiveValidateLimiterState) {
|
||||||
|
while state.in_flight < state.limit {
|
||||||
|
let Some(waiter) = state.waiters.pop_front() else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
state.in_flight += 1;
|
||||||
|
if let Err(mut grant) = waiter.send(AdaptiveValidateGrant::new(self.clone())) {
|
||||||
|
grant.disarm();
|
||||||
|
state.in_flight -= 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_sample(self: &Arc<Self>, elapsed: Duration, success: bool) {
|
||||||
|
let mut state = self.lock_state();
|
||||||
|
let adjustment = state.record_sample(Instant::now(), elapsed, success);
|
||||||
|
if adjustment == LimitAdjustment::Increased {
|
||||||
|
self.grant_waiters(&mut state);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn release_slot(self: &Arc<Self>) {
|
||||||
|
let mut state = self.lock_state();
|
||||||
|
state.in_flight = state.in_flight.saturating_sub(1);
|
||||||
|
self.grant_waiters(&mut state);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lock_state(&self) -> std::sync::MutexGuard<'_, AdaptiveValidateLimiterState> {
|
||||||
|
self.state
|
||||||
|
.lock()
|
||||||
|
.expect("adaptive validate limiter state should not be poisoned")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AdaptiveValidateLimiterState {
|
||||||
|
fn new(now: Instant) -> Self {
|
||||||
|
Self {
|
||||||
|
limit: VALIDATE_TOKEN_INITIAL_CONCURRENCY,
|
||||||
|
in_flight: 0,
|
||||||
|
waiters: VecDeque::new(),
|
||||||
|
window_started_at: now,
|
||||||
|
samples: 0,
|
||||||
|
slow_samples: 0,
|
||||||
|
failures: 0,
|
||||||
|
had_queue: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_sample(&mut self, now: Instant, elapsed: Duration, success: bool) -> LimitAdjustment {
|
||||||
|
self.samples += 1;
|
||||||
|
if elapsed > VALIDATE_TOKEN_SLOW_THRESHOLD {
|
||||||
|
self.slow_samples += 1;
|
||||||
|
}
|
||||||
|
if !success {
|
||||||
|
self.failures += 1;
|
||||||
|
}
|
||||||
|
self.complete_window_if_due(now)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn complete_window_if_due(&mut self, now: Instant) -> LimitAdjustment {
|
||||||
|
if now.duration_since(self.window_started_at) < VALIDATE_TOKEN_ADJUST_WINDOW {
|
||||||
|
return LimitAdjustment::Unchanged;
|
||||||
|
}
|
||||||
|
|
||||||
|
let old_limit = self.limit;
|
||||||
|
if self.samples > 0 {
|
||||||
|
if self.failures > 0 || self.is_p95_slow() {
|
||||||
|
self.limit = (self.limit / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY);
|
||||||
|
} else if self.had_queue {
|
||||||
|
self.limit = (self.limit + 1).min(VALIDATE_TOKEN_MAX_CONCURRENCY);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
self.window_started_at = now;
|
||||||
|
self.samples = 0;
|
||||||
|
self.slow_samples = 0;
|
||||||
|
self.failures = 0;
|
||||||
|
self.had_queue = false;
|
||||||
|
|
||||||
|
match self.limit.cmp(&old_limit) {
|
||||||
|
Ordering::Greater => LimitAdjustment::Increased,
|
||||||
|
Ordering::Less => LimitAdjustment::Decreased,
|
||||||
|
Ordering::Equal => LimitAdjustment::Unchanged,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_p95_slow(&self) -> bool {
|
||||||
|
self.slow_samples > 0 && self.slow_samples * 20 >= self.samples
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AdaptiveValidatePermit {
|
||||||
|
fn new(limiter: Arc<AdaptiveValidateLimiter>) -> Self {
|
||||||
|
Self {
|
||||||
|
limiter,
|
||||||
|
started_at: Instant::now(),
|
||||||
|
completed: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn complete(mut self, success: bool) {
|
||||||
|
self.limiter
|
||||||
|
.record_sample(self.started_at.elapsed(), success);
|
||||||
|
self.completed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AdaptiveValidateGrant {
|
||||||
|
fn new(limiter: Arc<AdaptiveValidateLimiter>) -> Self {
|
||||||
|
Self {
|
||||||
|
limiter,
|
||||||
|
active: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn into_permit(mut self) -> AdaptiveValidatePermit {
|
||||||
|
self.active = false;
|
||||||
|
AdaptiveValidatePermit::new(self.limiter.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn disarm(&mut self) {
|
||||||
|
self.active = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for AdaptiveValidateGrant {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if self.active {
|
||||||
|
self.limiter.release_slot();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for AdaptiveValidatePermit {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if !self.completed {
|
||||||
|
self.limiter.record_sample(self.started_at.elapsed(), false);
|
||||||
|
}
|
||||||
|
self.limiter.release_slot();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Webhook configuration for external integrations.
|
/// Webhook configuration for external integrations.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
@@ -11,6 +253,7 @@ pub struct WebhookConfig {
|
|||||||
pub web_instance_id: Option<String>,
|
pub web_instance_id: Option<String>,
|
||||||
pub web_instance_api_base_url: Option<String>,
|
pub web_instance_api_base_url: Option<String>,
|
||||||
|
|
||||||
|
validate_limiter: Arc<AdaptiveValidateLimiter>,
|
||||||
client: reqwest::Client,
|
client: reqwest::Client,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -28,7 +271,11 @@ impl WebhookConfig {
|
|||||||
internal_auth_token,
|
internal_auth_token,
|
||||||
web_instance_id,
|
web_instance_id,
|
||||||
web_instance_api_base_url,
|
web_instance_api_base_url,
|
||||||
client: reqwest::Client::new(),
|
validate_limiter: AdaptiveValidateLimiter::new(),
|
||||||
|
client: reqwest::Client::builder()
|
||||||
|
.timeout(WEBHOOK_HTTP_TIMEOUT)
|
||||||
|
.build()
|
||||||
|
.expect("webhook HTTP client should be valid"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,6 +305,8 @@ pub struct ValidateTokenRequest {
|
|||||||
pub web_instance_id: Option<String>,
|
pub web_instance_id: Option<String>,
|
||||||
pub web_instance_api_base_url: Option<String>,
|
pub web_instance_api_base_url: Option<String>,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub persisted_config_revision: Option<String>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub applied_config_revision: Option<String>,
|
pub applied_config_revision: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,7 +318,6 @@ pub struct ValidateTokenResponse {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub binding_version: u64,
|
pub binding_version: u64,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub managed_network_configs: Option<Vec<ManagedNetworkConfig>>,
|
|
||||||
pub config_revision: String,
|
pub config_revision: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -125,21 +373,40 @@ impl WebhookConfig {
|
|||||||
pub async fn validate_token(
|
pub async fn validate_token(
|
||||||
&self,
|
&self,
|
||||||
req: &ValidateTokenRequest,
|
req: &ValidateTokenRequest,
|
||||||
|
) -> anyhow::Result<ValidateTokenResponse> {
|
||||||
|
self.validate_token_with_http_timeout(req, WEBHOOK_HTTP_TIMEOUT)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn validate_token_with_http_timeout(
|
||||||
|
&self,
|
||||||
|
req: &ValidateTokenRequest,
|
||||||
|
http_timeout: Duration,
|
||||||
) -> anyhow::Result<ValidateTokenResponse> {
|
) -> anyhow::Result<ValidateTokenResponse> {
|
||||||
let url = self.webhook_endpoint("validate-token")?;
|
let url = self.webhook_endpoint("validate-token")?;
|
||||||
let resp = self
|
let permit = self.validate_limiter.acquire().await;
|
||||||
.client
|
let ret = match tokio::time::timeout(http_timeout, async {
|
||||||
.post(&url)
|
let resp = self
|
||||||
.header("X-Internal-Auth", self.webhook_auth_secret())
|
.client
|
||||||
.json(req)
|
.post(&url)
|
||||||
.send()
|
.header("X-Internal-Auth", self.webhook_auth_secret())
|
||||||
.await?;
|
.json(req)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
|
||||||
if !resp.status().is_success() {
|
if !resp.status().is_success() {
|
||||||
anyhow::bail!("webhook validate-token returned status {}", resp.status());
|
anyhow::bail!("webhook validate-token returned status {}", resp.status());
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(resp.json().await?)
|
Ok(resp.json().await?)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(ret) => ret,
|
||||||
|
Err(_) => Err(anyhow::anyhow!("webhook validate-token timed out")),
|
||||||
|
};
|
||||||
|
permit.complete(ret.is_ok());
|
||||||
|
ret
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Notify the webhook receiver that a node has connected.
|
/// Notify the webhook receiver that a node has connected.
|
||||||
@@ -191,13 +458,319 @@ pub type SharedWebhookConfig = Arc<WebhookConfig>;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use axum::{Json, Router, routing::post};
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn validate_token_response_allows_missing_managed_configs() {
|
fn adaptive_validate_limiter_increases_under_queue_pressure() {
|
||||||
|
let now = Instant::now();
|
||||||
|
let mut state = AdaptiveValidateLimiterState::new(now);
|
||||||
|
state.had_queue = true;
|
||||||
|
|
||||||
|
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||||
|
state.record_sample(now, Duration::from_millis(50), true);
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
|
||||||
|
LimitAdjustment::Increased
|
||||||
|
);
|
||||||
|
assert_eq!(state.limit, VALIDATE_TOKEN_INITIAL_CONCURRENCY + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn adaptive_validate_limiter_does_not_increase_without_queue_pressure() {
|
||||||
|
let now = Instant::now();
|
||||||
|
let mut state = AdaptiveValidateLimiterState::new(now);
|
||||||
|
|
||||||
|
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||||
|
state.record_sample(now, Duration::from_millis(50), true);
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
|
||||||
|
LimitAdjustment::Unchanged
|
||||||
|
);
|
||||||
|
assert_eq!(state.limit, VALIDATE_TOKEN_INITIAL_CONCURRENCY);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn adaptive_validate_limiter_reduces_on_failure() {
|
||||||
|
let now = Instant::now();
|
||||||
|
let mut state = AdaptiveValidateLimiterState::new(now);
|
||||||
|
|
||||||
|
state.record_sample(now, Duration::from_millis(50), false);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
|
||||||
|
LimitAdjustment::Decreased
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
state.limit,
|
||||||
|
(VALIDATE_TOKEN_INITIAL_CONCURRENCY / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn adaptive_validate_limiter_reduces_on_slow_latency() {
|
||||||
|
let now = Instant::now();
|
||||||
|
let mut state = AdaptiveValidateLimiterState::new(now);
|
||||||
|
|
||||||
|
state.record_sample(
|
||||||
|
now,
|
||||||
|
VALIDATE_TOKEN_SLOW_THRESHOLD + Duration::from_millis(1),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
|
||||||
|
LimitAdjustment::Decreased
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
state.limit,
|
||||||
|
(VALIDATE_TOKEN_INITIAL_CONCURRENCY / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn adaptive_validate_limiter_waiter_acquires_after_release() {
|
||||||
|
let limiter = AdaptiveValidateLimiter::new();
|
||||||
|
let mut permits = Vec::new();
|
||||||
|
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||||
|
permits.push(limiter.acquire().await);
|
||||||
|
}
|
||||||
|
|
||||||
|
let waiter_limiter = limiter.clone();
|
||||||
|
let waiter = tokio::spawn(async move {
|
||||||
|
let permit = waiter_limiter.acquire().await;
|
||||||
|
permit.complete(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||||
|
assert!(!waiter.is_finished());
|
||||||
|
|
||||||
|
permits.pop().unwrap().complete(true);
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), waiter)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
for permit in permits {
|
||||||
|
permit.complete(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn adaptive_validate_limiter_releases_when_permit_is_dropped() {
|
||||||
|
let limiter = AdaptiveValidateLimiter::new();
|
||||||
|
let mut permits = Vec::new();
|
||||||
|
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||||
|
permits.push(limiter.acquire().await);
|
||||||
|
}
|
||||||
|
|
||||||
|
let waiter_limiter = limiter.clone();
|
||||||
|
let waiter = tokio::spawn(async move {
|
||||||
|
let permit = waiter_limiter.acquire().await;
|
||||||
|
permit.complete(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||||
|
assert!(!waiter.is_finished());
|
||||||
|
|
||||||
|
drop(permits.pop().unwrap());
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), waiter)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
for permit in permits {
|
||||||
|
permit.complete(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn adaptive_validate_limiter_skips_canceled_waiters() {
|
||||||
|
let limiter = AdaptiveValidateLimiter::new();
|
||||||
|
let mut permits = Vec::new();
|
||||||
|
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||||
|
permits.push(limiter.acquire().await);
|
||||||
|
}
|
||||||
|
|
||||||
|
let waiter_limiter = limiter.clone();
|
||||||
|
let waiter = tokio::spawn(async move {
|
||||||
|
let permit = waiter_limiter.acquire().await;
|
||||||
|
permit.complete(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||||
|
assert!(!waiter.is_finished());
|
||||||
|
waiter.abort();
|
||||||
|
assert!(waiter.await.unwrap_err().is_cancelled());
|
||||||
|
|
||||||
|
permits.pop().unwrap().complete(true);
|
||||||
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||||
|
|
||||||
|
let state = limiter.lock_state();
|
||||||
|
assert_eq!(state.samples, 1);
|
||||||
|
assert_eq!(state.failures, 0);
|
||||||
|
drop(state);
|
||||||
|
|
||||||
|
for permit in permits {
|
||||||
|
permit.complete(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn adaptive_validate_limiter_releases_dropped_grant_without_failure_sample() {
|
||||||
|
let limiter = AdaptiveValidateLimiter::new();
|
||||||
|
{
|
||||||
|
let mut state = limiter.lock_state();
|
||||||
|
state.in_flight = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
drop(AdaptiveValidateGrant::new(limiter.clone()));
|
||||||
|
|
||||||
|
let state = limiter.lock_state();
|
||||||
|
assert_eq!(state.in_flight, 0);
|
||||||
|
assert_eq!(state.samples, 0);
|
||||||
|
assert_eq!(state.failures, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn adaptive_validate_limiter_wakes_multiple_waiters_in_order() {
|
||||||
|
let limiter = AdaptiveValidateLimiter::new();
|
||||||
|
let mut permits = Vec::new();
|
||||||
|
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||||
|
permits.push(limiter.acquire().await);
|
||||||
|
}
|
||||||
|
|
||||||
|
let (first_acquired_tx, first_acquired_rx) = oneshot::channel();
|
||||||
|
let (first_release_tx, first_release_rx) = oneshot::channel();
|
||||||
|
let first = {
|
||||||
|
let limiter = limiter.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let permit = limiter.acquire().await;
|
||||||
|
first_acquired_tx.send(()).unwrap();
|
||||||
|
first_release_rx.await.unwrap();
|
||||||
|
permit.complete(true);
|
||||||
|
})
|
||||||
|
};
|
||||||
|
let (second_acquired_tx, mut second_acquired_rx) = oneshot::channel();
|
||||||
|
let second = {
|
||||||
|
let limiter = limiter.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let permit = limiter.acquire().await;
|
||||||
|
second_acquired_tx.send(()).unwrap();
|
||||||
|
permit.complete(true);
|
||||||
|
})
|
||||||
|
};
|
||||||
|
|
||||||
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||||
|
assert!(!first.is_finished());
|
||||||
|
assert!(!second.is_finished());
|
||||||
|
|
||||||
|
permits.pop().unwrap().complete(true);
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), first_acquired_rx)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(50), &mut second_acquired_rx)
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
|
||||||
|
first_release_tx.send(()).unwrap();
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), first)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), &mut second_acquired_rx)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), second)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
for permit in permits {
|
||||||
|
permit.complete(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn validate_token_http_timeout_starts_after_limiter_permit() {
|
||||||
|
let app = Router::new().route(
|
||||||
|
"/validate-token",
|
||||||
|
post(|| async {
|
||||||
|
Json(json!({
|
||||||
|
"valid": true,
|
||||||
|
"config_revision": "rev-1"
|
||||||
|
}))
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let addr = listener.local_addr().unwrap();
|
||||||
|
let server = tokio::spawn(async move {
|
||||||
|
axum::serve(listener, app).await.unwrap();
|
||||||
|
});
|
||||||
|
let webhook = WebhookConfig::new(Some(format!("http://{addr}")), None, None, None, None);
|
||||||
|
let mut permits = Vec::new();
|
||||||
|
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||||
|
permits.push(webhook.validate_limiter.acquire().await);
|
||||||
|
}
|
||||||
|
|
||||||
|
let validate_webhook = webhook.clone();
|
||||||
|
let validate = tokio::spawn(async move {
|
||||||
|
let req = ValidateTokenRequest {
|
||||||
|
token: "token".to_string(),
|
||||||
|
machine_id: uuid::Uuid::new_v4().to_string(),
|
||||||
|
public_ip: None,
|
||||||
|
hostname: String::new(),
|
||||||
|
version: String::new(),
|
||||||
|
os_type: None,
|
||||||
|
os_version: None,
|
||||||
|
os_distribution: None,
|
||||||
|
web_instance_id: None,
|
||||||
|
web_instance_api_base_url: None,
|
||||||
|
persisted_config_revision: None,
|
||||||
|
applied_config_revision: None,
|
||||||
|
};
|
||||||
|
validate_webhook
|
||||||
|
.validate_token_with_http_timeout(&req, Duration::from_millis(20))
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
|
||||||
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
|
assert!(!validate.is_finished());
|
||||||
|
|
||||||
|
permits.pop().unwrap().complete(true);
|
||||||
|
let resp = tokio::time::timeout(Duration::from_secs(1), validate)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(resp.valid);
|
||||||
|
|
||||||
|
for permit in permits {
|
||||||
|
permit.complete(true);
|
||||||
|
}
|
||||||
|
server.abort();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validate_token_response_deserializes_config_revision() {
|
||||||
let resp: ValidateTokenResponse =
|
let resp: ValidateTokenResponse =
|
||||||
serde_json::from_str(r#"{"valid":true,"config_revision":"rev-1"}"#).unwrap();
|
serde_json::from_str(r#"{"valid":true,"config_revision":"rev-1"}"#).unwrap();
|
||||||
assert!(resp.valid);
|
assert!(resp.valid);
|
||||||
assert_eq!(resp.config_revision, "rev-1");
|
assert_eq!(resp.config_revision, "rev-1");
|
||||||
assert!(resp.managed_network_configs.is_none());
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validate_token_response_allows_missing_config_revision() {
|
||||||
|
let resp: ValidateTokenResponse = serde_json::from_str(r#"{"valid":true}"#).unwrap();
|
||||||
|
assert!(resp.valid);
|
||||||
|
assert!(resp.config_revision.is_empty());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -483,18 +483,22 @@ impl InstanceConfigPatcher {
|
|||||||
}
|
}
|
||||||
let global_ctx = weak_upgrade(&self.global_ctx)?;
|
let global_ctx = weak_upgrade(&self.global_ctx)?;
|
||||||
for proxy_network_patch in proxy_networks {
|
for proxy_network_patch in proxy_networks {
|
||||||
let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else {
|
|
||||||
tracing::warn!("Proxy network cidr is None, skipping.");
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let mapped_cidr: Option<cidr::Ipv4Cidr> =
|
|
||||||
proxy_network_patch.mapped_cidr.map(|s| s.into());
|
|
||||||
match ConfigPatchAction::try_from(proxy_network_patch.action) {
|
match ConfigPatchAction::try_from(proxy_network_patch.action) {
|
||||||
Ok(ConfigPatchAction::Add) => {
|
Ok(ConfigPatchAction::Add) => {
|
||||||
|
let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else {
|
||||||
|
tracing::warn!("Proxy network cidr is None, skipping add.");
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let mapped_cidr: Option<cidr::Ipv4Cidr> =
|
||||||
|
proxy_network_patch.mapped_cidr.map(|s| s.into());
|
||||||
tracing::info!("Proxy network added: {}", cidr);
|
tracing::info!("Proxy network added: {}", cidr);
|
||||||
global_ctx.config.add_proxy_cidr(cidr, mapped_cidr)?;
|
global_ctx.config.add_proxy_cidr(cidr, mapped_cidr)?;
|
||||||
}
|
}
|
||||||
Ok(ConfigPatchAction::Remove) => {
|
Ok(ConfigPatchAction::Remove) => {
|
||||||
|
let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else {
|
||||||
|
tracing::warn!("Proxy network cidr is None, skipping remove.");
|
||||||
|
continue;
|
||||||
|
};
|
||||||
tracing::info!("Proxy network removed: {}", cidr);
|
tracing::info!("Proxy network removed: {}", cidr);
|
||||||
global_ctx.config.remove_proxy_cidr(cidr);
|
global_ctx.config.remove_proxy_cidr(cidr);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1116,6 +1116,7 @@ impl NetworkConfig {
|
|||||||
.get_credential_file()
|
.get_credential_file()
|
||||||
.map(|path| path.to_string_lossy().into_owned());
|
.map(|path| path.to_string_lossy().into_owned());
|
||||||
let flags = config.get_flags();
|
let flags = config.get_flags();
|
||||||
|
let default_flags = default_config.get_flags();
|
||||||
result.latency_first = Some(flags.latency_first);
|
result.latency_first = Some(flags.latency_first);
|
||||||
result.dev_name = Some(flags.dev_name.clone());
|
result.dev_name = Some(flags.dev_name.clone());
|
||||||
result.use_smoltcp = Some(flags.use_smoltcp);
|
result.use_smoltcp = Some(flags.use_smoltcp);
|
||||||
@@ -1144,6 +1145,11 @@ impl NetworkConfig {
|
|||||||
result.disable_sym_hole_punching = Some(flags.disable_sym_hole_punching);
|
result.disable_sym_hole_punching = Some(flags.disable_sym_hole_punching);
|
||||||
result.enable_magic_dns = Some(flags.accept_dns);
|
result.enable_magic_dns = Some(flags.accept_dns);
|
||||||
result.mtu = Some(flags.mtu as i32);
|
result.mtu = Some(flags.mtu as i32);
|
||||||
|
result.data_compress_algo = (flags.data_compress_algo != default_flags.data_compress_algo)
|
||||||
|
.then_some(flags.data_compress_algo);
|
||||||
|
result.encryption_algorithm = (flags.encryption_algorithm
|
||||||
|
!= default_flags.encryption_algorithm)
|
||||||
|
.then_some(flags.encryption_algorithm.clone());
|
||||||
result.instance_recv_bps_limit =
|
result.instance_recv_bps_limit =
|
||||||
(flags.instance_recv_bps_limit != u64::MAX).then_some(flags.instance_recv_bps_limit);
|
(flags.instance_recv_bps_limit != u64::MAX).then_some(flags.instance_recv_bps_limit);
|
||||||
result.enable_private_mode = Some(flags.private_mode);
|
result.enable_private_mode = Some(flags.private_mode);
|
||||||
@@ -1173,7 +1179,7 @@ impl NetworkConfig {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use crate::{
|
use crate::{
|
||||||
common::config::{ConfigLoader, process_secure_mode_cfg},
|
common::config::{ConfigLoader, process_secure_mode_cfg},
|
||||||
proto::common::SecureModeConfig,
|
proto::common::{CompressionAlgoPb, SecureModeConfig},
|
||||||
};
|
};
|
||||||
use base64::prelude::{BASE64_STANDARD, Engine as _};
|
use base64::prelude::{BASE64_STANDARD, Engine as _};
|
||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
@@ -1534,4 +1540,37 @@ mod tests {
|
|||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_network_config_conversion_preserves_runtime_algorithm_flags()
|
||||||
|
-> Result<(), anyhow::Error> {
|
||||||
|
let config = gen_default_config();
|
||||||
|
let mut flags = config.get_flags();
|
||||||
|
flags.data_compress_algo = CompressionAlgoPb::Zstd.into();
|
||||||
|
flags.encryption_algorithm = "managed-test-algo".to_string();
|
||||||
|
config.set_flags(flags.clone());
|
||||||
|
|
||||||
|
let network_config = super::NetworkConfig::new_from_config(&config)?;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
network_config.data_compress_algo,
|
||||||
|
Some(CompressionAlgoPb::Zstd as i32)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
network_config.encryption_algorithm.as_deref(),
|
||||||
|
Some("managed-test-algo")
|
||||||
|
);
|
||||||
|
|
||||||
|
let generated_config = network_config.gen_config()?;
|
||||||
|
assert_eq!(
|
||||||
|
generated_config.get_flags().data_compress_algo,
|
||||||
|
flags.data_compress_algo
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
generated_config.get_flags().encryption_algorithm,
|
||||||
|
flags.encryption_algorithm
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3653,6 +3653,25 @@ pub async fn config_patch_test() {
|
|||||||
true
|
true
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
let patch = InstanceConfigPatch {
|
||||||
|
proxy_networks: vec![ProxyNetworkPatch {
|
||||||
|
action: ConfigPatchAction::Clear as i32,
|
||||||
|
..Default::default()
|
||||||
|
}],
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
insts[1]
|
||||||
|
.get_config_patcher()
|
||||||
|
.apply_patch(patch)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
insts[1]
|
||||||
|
.get_global_ctx()
|
||||||
|
.config
|
||||||
|
.get_proxy_cidrs()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
|
||||||
// 测试1.1:修改公网 IPv6 provider 相关配置
|
// 测试1.1:修改公网 IPv6 provider 相关配置
|
||||||
let public_prefix = "2001:db8:100::/64";
|
let public_prefix = "2001:db8:100::/64";
|
||||||
|
|||||||
Reference in New Issue
Block a user