From 9cb38332164cc0804734c0d0c2ea806f3b49e163 Mon Sep 17 00:00:00 2001 From: KKRainbow <443152178@qq.com> Date: Sun, 28 Jun 2026 13:14:40 +0800 Subject: [PATCH] perf(easytier-web): improve easytier-web webhook performance (#2383) * feat(web): reconcile managed config revisions * feat(web): cache managed runtime configs per session * test: cover managed web config delivery --- easytier-web/locales/app.yml | 3 + .../src/client_manager/managed_config.rs | 1024 ++++++++ easytier-web/src/client_manager/mod.rs | 692 +++++- .../src/client_manager/runtime_reconcile.rs | 812 +++++++ easytier-web/src/client_manager/session.rs | 2106 +++++++++-------- .../session/runtime_revision.rs | 913 +++++++ .../session/webhook_validation.rs | 402 ++++ easytier-web/src/client_manager/storage.rs | 132 +- .../src/db/entity/managed_config_revisions.rs | 38 + easytier-web/src/db/entity/mod.rs | 1 + easytier-web/src/db/entity/prelude.rs | 1 + easytier-web/src/db/mod.rs | 146 ++ easytier-web/src/main.rs | 11 +- ...0260619_000005_managed_config_revisions.rs | 46 + easytier-web/src/migrator/mod.rs | 2 + easytier-web/src/restful/mod.rs | 2 +- easytier-web/src/restful/network.rs | 20 +- easytier-web/src/webhook.rs | 605 ++++- easytier/src/instance/instance.rs | 16 +- easytier/src/launcher.rs | 41 +- easytier/src/tests/three_node.rs | 19 + 21 files changed, 6037 insertions(+), 995 deletions(-) create mode 100644 easytier-web/src/client_manager/managed_config.rs create mode 100644 easytier-web/src/client_manager/runtime_reconcile.rs create mode 100644 easytier-web/src/client_manager/session/runtime_revision.rs create mode 100644 easytier-web/src/client_manager/session/webhook_validation.rs create mode 100644 easytier-web/src/db/entity/managed_config_revisions.rs create mode 100644 easytier-web/src/migrator/m20260619_000005_managed_config_revisions.rs diff --git a/easytier-web/locales/app.yml b/easytier-web/locales/app.yml index bfe3239a..8ebf5455 100644 --- a/easytier-web/locales/app.yml +++ b/easytier-web/locales/app.yml @@ -40,6 +40,9 @@ cli: geoip_db: en: "The path to the GeoIP2 database file, used to lookup the location of the client, default is the embedded file (only country information) , recommend https://github.com/P3TERX/GeoLite.mmdb" zh-CN: "GeoIP2 数据库文件路径，用于查找客户端的位置，默认为嵌入文件（仅国家信息），推荐 https://github.com/P3TERX/GeoLite.mmdb" + heartbeat_min_response_ms: + en: "Minimum response time for config-server heartbeat RPCs in milliseconds, default is 0" + zh-CN: "配置服务心跳 RPC 的最短响应时间，单位毫秒，默认为 0" disable_registration: en: "Disable user registration" zh-CN: "禁用用户注册" diff --git a/easytier-web/src/client_manager/managed_config.rs b/easytier-web/src/client_manager/managed_config.rs new file mode 100644 index 00000000..3320e6b7 --- /dev/null +++ b/easytier-web/src/client_manager/managed_config.rs @@ -0,0 +1,1024 @@ +use std::{ + collections::{HashMap, HashSet}, + sync::{Arc, Weak}, +}; + +use anyhow::Context as _; +use dashmap::{DashMap, mapref::entry::Entry}; +use easytier::{ + common::config::ConfigSource, + proto::{ + api::manage::{ConfigSource as RpcConfigSource, NetworkConfig, NetworkMeta}, + common::Uuid as RpcUuid, + }, + rpc_service::remote_client::{ListNetworkProps, PersistentConfig as _, Storage as _}, +}; + +use super::storage::Storage; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum PersistedConfigSource { + User, + Web, +} + +pub(super) enum ExpectedConfigRevision<'a> { + Any, + Exact(Option<&'a str>), +} + +#[derive(Debug, thiserror::Error)] +pub(super) enum ManagedConfigError { + #[error( + "managed config revision changed while reconciling: expected {expected:?}, current {current:?}" + )] + RevisionConflict { + expected: Option, + current: Option, + }, +} + +impl PersistedConfigSource { + pub(super) fn from_db(source: &str) -> Self { + match source { + "web" => Self::Web, + "user" => Self::User, + _ => Self::User, + } + } + + fn should_update_from_runtime(self, runtime_source: ConfigSource) -> bool { + match (self, runtime_source) { + // Older clients report missing source as `user`, which is not authoritative enough + // to downgrade an existing web-owned row. + (Self::Web, ConfigSource::User) => false, + _ => self.as_runtime_source() != runtime_source, + } + } + + fn as_runtime_source(self) -> ConfigSource { + match self { + Self::User => ConfigSource::User, + Self::Web => ConfigSource::Web, + } + } + + pub(super) fn auto_run_rpc_source(self) -> RpcConfigSource { + match self { + Self::User => RpcConfigSource::User, + Self::Web => RpcConfigSource::Web, + } + } +} + +type ManagedConfigReconcileKey = (i32, uuid::Uuid); +type ManagedConfigReconcileLock = tokio::sync::Mutex<()>; +type ManagedConfigReconcileLockRef = Arc; +type ManagedConfigReconcileLockWeak = Weak; + +static MANAGED_CONFIG_RECONCILE_LOCKS: std::sync::LazyLock< + DashMap, +> = std::sync::LazyLock::new(DashMap::new); + +fn managed_config_reconcile_lock(key: ManagedConfigReconcileKey) -> ManagedConfigReconcileLockRef { + match MANAGED_CONFIG_RECONCILE_LOCKS.entry(key) { + Entry::Occupied(mut entry) => match entry.get().upgrade() { + Some(lock) => lock, + None => { + let lock = Arc::new(tokio::sync::Mutex::new(())); + entry.insert(Arc::downgrade(&lock)); + lock + } + }, + Entry::Vacant(entry) => { + let lock = Arc::new(tokio::sync::Mutex::new(())); + entry.insert(Arc::downgrade(&lock)); + lock + } + } +} + +fn remove_unused_managed_config_reconcile_lock( + key: ManagedConfigReconcileKey, + lock: &ManagedConfigReconcileLockRef, +) { + let expected_lock = Arc::downgrade(lock); + MANAGED_CONFIG_RECONCILE_LOCKS.remove_if(&key, |_, current_lock| { + current_lock.ptr_eq(&expected_lock) && current_lock.strong_count() == 1 + }); +} + +pub(super) fn is_revision_conflict(error: &anyhow::Error) -> bool { + error.downcast_ref::().is_some() +} + +fn snake_to_lower_camel(key: &str) -> Option { + if !key.contains('_') { + return None; + } + + let mut result = String::with_capacity(key.len()); + let mut uppercase_next = false; + for ch in key.chars() { + if ch == '_' { + uppercase_next = true; + continue; + } + if uppercase_next { + result.push(ch.to_ascii_uppercase()); + uppercase_next = false; + } else { + result.push(ch); + } + } + + (!result.is_empty()).then_some(result) +} + +fn normalize_lower_camel_keys(value: &mut serde_json::Value) -> anyhow::Result<()> { + match value { + serde_json::Value::Object(map) => { + let old_map = std::mem::take(map); + for (key, mut value) in old_map { + normalize_lower_camel_keys(&mut value)?; + let normalized_key = snake_to_lower_camel(&key).unwrap_or(key); + if map.insert(normalized_key.clone(), value).is_some() { + anyhow::bail!( + "duplicate network_config field after key normalization: {normalized_key}" + ); + } + } + } + serde_json::Value::Array(items) => { + for item in items { + normalize_lower_camel_keys(item)?; + } + } + _ => {} + } + Ok(()) +} + +fn normalize_network_config( + mut network_config: serde_json::Value, + inst_id: uuid::Uuid, +) -> anyhow::Result { + let config_obj = network_config + .as_object_mut() + .ok_or_else(|| anyhow::anyhow!("webhook network_config must be a JSON object"))?; + config_obj.remove("instance_id"); + config_obj.remove("instanceId"); + normalize_lower_camel_keys(&mut network_config)?; + let config_obj = network_config + .as_object_mut() + .ok_or_else(|| anyhow::anyhow!("webhook network_config must be a JSON object"))?; + config_obj.insert( + "instanceId".to_string(), + serde_json::Value::String(inst_id.to_string()), + ); + network_config + .get("networkName") + .and_then(|v| v.as_str()) + .filter(|v| !v.is_empty()) + .ok_or_else(|| anyhow::anyhow!("webhook response missing network_name"))?; + + Ok(serde_json::from_value::(network_config)?) +} + +struct ExistingConfigSources { + sources: HashMap, + web_ids: HashSet, +} + +struct NormalizedWebConfigs { + desired_ids: HashSet, + configs: HashMap, +} + +async fn ensure_expected_config_revision( + storage: &Storage, + user_id: i32, + machine_id: uuid::Uuid, + expected_config_revision: ExpectedConfigRevision<'_>, +) -> anyhow::Result<()> { + let ExpectedConfigRevision::Exact(expected) = expected_config_revision else { + return Ok(()); + }; + + let current = storage + .db() + .get_managed_config_revision((user_id, machine_id)) + .await + .map_err(|e| anyhow::anyhow!("failed to get managed config revision: {:?}", e))?; + if current.as_deref() != expected { + return Err(ManagedConfigError::RevisionConflict { + expected: expected.map(str::to_string), + current, + } + .into()); + } + + Ok(()) +} + +async fn load_existing_config_sources( + storage: &Storage, + user_id: i32, + machine_id: uuid::Uuid, +) -> anyhow::Result { + let existing_configs = storage + .db() + .list_network_configs((user_id, machine_id), ListNetworkProps::All) + .await + .map_err(|e| anyhow::anyhow!("failed to list existing network configs: {:?}", e))?; + let sources = existing_configs + .iter() + .filter_map(|cfg| { + uuid::Uuid::parse_str(&cfg.network_instance_id) + .ok() + .map(|inst_id| (inst_id, PersistedConfigSource::from_db(&cfg.source))) + }) + .collect::>(); + let web_ids = sources + .iter() + .filter_map(|(inst_id, source)| (*source == PersistedConfigSource::Web).then_some(*inst_id)) + .collect::>(); + + Ok(ExistingConfigSources { sources, web_ids }) +} + +fn normalize_desired_web_configs( + user_id: i32, + machine_id: uuid::Uuid, + desired_configs: Vec, + config_revision: Option<&str>, + existing_sources: &HashMap, +) -> anyhow::Result { + let mut desired_ids = HashSet::with_capacity(desired_configs.len()); + let mut configs = HashMap::with_capacity(desired_configs.len()); + + for desired in desired_configs { + let inst_id = uuid::Uuid::parse_str(&desired.instance_id).with_context(|| { + format!( + "invalid desired web config instance id: {}", + desired.instance_id + ) + })?; + if let Some(PersistedConfigSource::User) = existing_sources.get(&inst_id) { + if config_revision.is_some() { + anyhow::bail!( + "cannot persist managed config revision because instance {} is user-owned", + inst_id + ); + } + tracing::warn!( + ?user_id, + ?machine_id, + instance_id = %inst_id, + "skip web config because a user-owned config already exists" + ); + continue; + } + let config = normalize_network_config(desired.network_config, inst_id)?; + desired_ids.insert(inst_id); + configs.insert(inst_id, config); + } + + Ok(NormalizedWebConfigs { + desired_ids, + configs, + }) +} + +async fn upsert_web_configs( + storage: &Storage, + user_id: i32, + machine_id: uuid::Uuid, + configs: HashMap, +) -> anyhow::Result<()> { + for (inst_id, config) in configs { + let updated = storage + .db() + .insert_or_update_web_network_config((user_id, machine_id), inst_id, config) + .await + .map_err(|e| { + anyhow::anyhow!("failed to persist web network config {}: {:?}", inst_id, e) + })?; + if !updated { + anyhow::bail!( + "cannot persist managed config revision because instance {} is user-owned", + inst_id + ); + } + } + + Ok(()) +} + +async fn delete_stale_web_configs( + storage: &Storage, + user_id: i32, + machine_id: uuid::Uuid, + existing_web_ids: &HashSet, + desired_ids: &HashSet, +) -> anyhow::Result<()> { + let stale_ids = existing_web_ids + .difference(desired_ids) + .copied() + .collect::>(); + if stale_ids.is_empty() { + return Ok(()); + } + + storage + .db() + .delete_web_network_configs((user_id, machine_id), &stale_ids) + .await + .map_err(|e| anyhow::anyhow!("failed to delete stale network configs: {:?}", e))?; + + Ok(()) +} + +async fn persist_config_revision( + storage: &Storage, + user_id: i32, + machine_id: uuid::Uuid, + config_revision: Option<&str>, +) -> anyhow::Result<()> { + let Some(config_revision) = config_revision else { + return Ok(()); + }; + + storage + .db() + .set_managed_config_revision((user_id, machine_id), config_revision) + .await + .map_err(|e| anyhow::anyhow!("failed to persist managed config revision: {:?}", e))?; + + Ok(()) +} + +pub(super) async fn reconcile_web_source_configs( + storage: &Storage, + user_id: i32, + machine_id: uuid::Uuid, + desired_configs: Vec, + config_revision: Option<&str>, + expected_config_revision: ExpectedConfigRevision<'_>, +) -> anyhow::Result<()> { + let key = (user_id, machine_id); + let reconcile_lock = managed_config_reconcile_lock(key); + let result = async { + let _guard = reconcile_lock.lock().await; + + ensure_expected_config_revision(storage, user_id, machine_id, expected_config_revision) + .await?; + let existing = load_existing_config_sources(storage, user_id, machine_id).await?; + let normalized = normalize_desired_web_configs( + user_id, + machine_id, + desired_configs, + config_revision, + &existing.sources, + )?; + upsert_web_configs(storage, user_id, machine_id, normalized.configs).await?; + delete_stale_web_configs( + storage, + user_id, + machine_id, + &existing.web_ids, + &normalized.desired_ids, + ) + .await?; + persist_config_revision(storage, user_id, machine_id, config_revision).await?; + + Ok(()) + } + .await; + remove_unused_managed_config_reconcile_lock(key, &reconcile_lock); + result +} + +fn collect_web_source_instance_ids(metas: &[NetworkMeta]) -> HashSet { + metas + .iter() + .filter_map(|meta| { + (RpcConfigSource::try_from(meta.source).ok() == Some(RpcConfigSource::Web)) + .then(|| { + meta.inst_id + .as_ref() + .map(|inst_id| Into::::into(*inst_id).to_string()) + }) + .flatten() + }) + .collect() +} + +pub(super) fn desired_web_source_instance_ids( + local_configs: &[crate::db::entity::user_running_network_configs::Model], +) -> HashSet { + local_configs + .iter() + .filter(|cfg| cfg.get_runtime_network_config_source() == ConfigSource::Web) + .map(|cfg| cfg.network_instance_id.clone()) + .collect() +} + +pub(super) fn running_web_source_instance_ids( + running_inst_ids: &HashSet, + db_web_inst_ids: &HashSet, + running_metas: Option<&[NetworkMeta]>, +) -> HashSet { + match running_metas { + Some(metas) => collect_web_source_instance_ids(metas), + None => running_inst_ids + .intersection(db_web_inst_ids) + .cloned() + .collect(), + } +} + +pub(super) fn parse_instance_ids(instance_ids: impl Iterator) -> Vec { + instance_ids + .filter_map(|inst_id| uuid::Uuid::parse_str(&inst_id).ok()) + .map(Into::into) + .collect() +} + +pub(super) async fn sync_running_config_sources( + db: &crate::db::Db, + user_id: i32, + machine_id: uuid::Uuid, + local_configs: &[crate::db::entity::user_running_network_configs::Model], + metas: &[NetworkMeta], +) -> anyhow::Result<()> { + let local_configs_by_id = local_configs + .iter() + .map(|cfg| (cfg.network_instance_id.clone(), cfg)) + .collect::>(); + + for meta in metas { + let Some(inst_id) = meta.inst_id.as_ref().map(|inst_id| { + let inst_id: uuid::Uuid = (*inst_id).into(); + inst_id + }) else { + continue; + }; + let inst_id_str = inst_id.to_string(); + let Some(local_cfg) = local_configs_by_id.get(&inst_id_str) else { + continue; + }; + + let Some(running_source) = ConfigSource::from_rpc(meta.source) else { + continue; + }; + let local_source = PersistedConfigSource::from_db(&local_cfg.source); + if !local_source.should_update_from_runtime(running_source) { + continue; + } + + db.insert_or_update_user_network_config( + (user_id, machine_id), + inst_id, + local_cfg.get_network_config().map_err(|e| { + anyhow::anyhow!("failed to decode local network config {}: {:?}", inst_id, e) + })?, + running_source, + ) + .await + .map_err(|e| { + anyhow::anyhow!( + "failed to sync running network config source {}: {:?}", + inst_id, + e + ) + })?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use easytier::{ + common::config::{ConfigLoader as _, ConfigSource}, + proto::api::manage::{ConfigSource as RpcConfigSource, NetworkConfig, NetworkMeta}, + rpc_service::remote_client::{ListNetworkProps, PersistentConfig as _, Storage as _}, + }; + use serde_json::json; + + use super::*; + + #[tokio::test] + async fn reconcile_web_source_configs_upserts_and_deletes_exact_set() { + let storage = Storage::new(crate::db::Db::memory_db().await); + let user_id = storage.db().auto_create_user("web-user").await.unwrap().id; + let machine_id = uuid::Uuid::new_v4(); + let keep_id = uuid::Uuid::new_v4(); + let stale_id = uuid::Uuid::new_v4(); + let new_id = uuid::Uuid::new_v4(); + + storage + .db() + .insert_or_update_user_network_config( + (user_id, machine_id), + keep_id, + NetworkConfig { + network_name: Some("old-name".to_string()), + ..Default::default() + }, + ConfigSource::Web, + ) + .await + .unwrap(); + storage + .db() + .insert_or_update_user_network_config( + (user_id, machine_id), + stale_id, + NetworkConfig { + network_name: Some("stale".to_string()), + ..Default::default() + }, + ConfigSource::Web, + ) + .await + .unwrap(); + + reconcile_web_source_configs( + &storage, + user_id, + machine_id, + vec![ + crate::webhook::ManagedNetworkConfig { + instance_id: keep_id.to_string(), + network_config: json!({ + "instance_id": keep_id.to_string(), + "network_name": "updated-name" + }), + }, + crate::webhook::ManagedNetworkConfig { + instance_id: new_id.to_string(), + network_config: json!({ + "instance_id": new_id.to_string(), + "network_name": "new-name" + }), + }, + ], + None, + ExpectedConfigRevision::Any, + ) + .await + .unwrap(); + + let configs = storage + .db() + .list_network_configs((user_id, machine_id), ListNetworkProps::All) + .await + .unwrap(); + let config_ids = configs + .iter() + .map(|cfg| cfg.network_instance_id.clone()) + .collect::>(); + + assert_eq!(configs.len(), 2); + assert!(config_ids.contains(&keep_id.to_string())); + assert!(config_ids.contains(&new_id.to_string())); + assert!(!config_ids.contains(&stale_id.to_string())); + + let updated_keep = storage + .db() + .get_network_config((user_id, machine_id), &keep_id.to_string()) + .await + .unwrap() + .unwrap(); + let updated_keep_config: NetworkConfig = + serde_json::from_str(&updated_keep.network_config).unwrap(); + assert_eq!( + updated_keep_config.network_name.as_deref(), + Some("updated-name") + ); + assert_eq!(updated_keep.get_network_config_source(), ConfigSource::Web); + } + + #[tokio::test] + async fn reconcile_web_source_configs_keep_user_owned_configs() { + let storage = Storage::new(crate::db::Db::memory_db().await); + let user_id = storage + .db() + .auto_create_user("web-user-keep-user") + .await + .unwrap() + .id; + let machine_id = uuid::Uuid::new_v4(); + let user_owned_id = uuid::Uuid::new_v4(); + let web_owned_id = uuid::Uuid::new_v4(); + + storage + .db() + .insert_or_update_user_network_config( + (user_id, machine_id), + user_owned_id, + NetworkConfig { + network_name: Some("user-owned".to_string()), + ..Default::default() + }, + ConfigSource::User, + ) + .await + .unwrap(); + storage + .db() + .insert_or_update_user_network_config( + (user_id, machine_id), + web_owned_id, + NetworkConfig { + network_name: Some("web-owned".to_string()), + ..Default::default() + }, + ConfigSource::Web, + ) + .await + .unwrap(); + + reconcile_web_source_configs( + &storage, + user_id, + machine_id, + vec![crate::webhook::ManagedNetworkConfig { + instance_id: user_owned_id.to_string(), + network_config: json!({ + "instance_id": user_owned_id.to_string(), + "network_name": "web-tries-to-take-over" + }), + }], + None, + ExpectedConfigRevision::Any, + ) + .await + .unwrap(); + + let user_owned = storage + .db() + .get_network_config((user_id, machine_id), &user_owned_id.to_string()) + .await + .unwrap() + .unwrap(); + assert_eq!(user_owned.get_network_config_source(), ConfigSource::User); + let user_owned_cfg: NetworkConfig = + serde_json::from_str(&user_owned.network_config).unwrap(); + assert_eq!(user_owned_cfg.network_name.as_deref(), Some("user-owned")); + + let web_owned = storage + .db() + .get_network_config((user_id, machine_id), &web_owned_id.to_string()) + .await + .unwrap(); + assert!(web_owned.is_none()); + } + + #[tokio::test] + async fn reconcile_web_source_configs_rejects_revision_for_user_owned_config() { + let storage = Storage::new(crate::db::Db::memory_db().await); + let user_id = storage + .db() + .auto_create_user("web-user-reject-user-owned") + .await + .unwrap() + .id; + let machine_id = uuid::Uuid::new_v4(); + let user_owned_id = uuid::Uuid::new_v4(); + + storage + .db() + .insert_or_update_user_network_config( + (user_id, machine_id), + user_owned_id, + NetworkConfig { + network_name: Some("user-owned".to_string()), + ..Default::default() + }, + ConfigSource::User, + ) + .await + .unwrap(); + + let err = reconcile_web_source_configs( + &storage, + user_id, + machine_id, + vec![crate::webhook::ManagedNetworkConfig { + instance_id: user_owned_id.to_string(), + network_config: json!({ + "instance_id": user_owned_id.to_string(), + "network_name": "web-tries-to-take-over" + }), + }], + Some("rev-user-owned"), + ExpectedConfigRevision::Any, + ) + .await + .unwrap_err(); + + assert!( + err.to_string().contains("user-owned"), + "unexpected error: {err:?}" + ); + assert_eq!( + storage + .db() + .get_managed_config_revision((user_id, machine_id)) + .await + .unwrap(), + None + ); + } + + #[tokio::test] + async fn reconcile_web_source_configs_persists_config_revision() { + let storage = Storage::new(crate::db::Db::memory_db().await); + let user_id = storage + .db() + .auto_create_user("web-user-revision") + .await + .unwrap() + .id; + let machine_id = uuid::Uuid::new_v4(); + + reconcile_web_source_configs( + &storage, + user_id, + machine_id, + Vec::new(), + Some("rev-1"), + ExpectedConfigRevision::Any, + ) + .await + .unwrap(); + + assert_eq!( + storage + .db() + .get_managed_config_revision((user_id, machine_id)) + .await + .unwrap() + .as_deref(), + Some("rev-1") + ); + } + + #[tokio::test] + async fn reconcile_web_source_configs_rejects_changed_expected_revision() { + let storage = Storage::new(crate::db::Db::memory_db().await); + let user_id = storage + .db() + .auto_create_user("web-user-expected-revision") + .await + .unwrap() + .id; + let machine_id = uuid::Uuid::new_v4(); + let inst_id = uuid::Uuid::new_v4(); + storage + .db() + .set_managed_config_revision((user_id, machine_id), "rev-new") + .await + .unwrap(); + + let err = reconcile_web_source_configs( + &storage, + user_id, + machine_id, + vec![crate::webhook::ManagedNetworkConfig { + instance_id: inst_id.to_string(), + network_config: json!({ + "instance_id": inst_id.to_string(), + "network_name": "stale-config" + }), + }], + Some("rev-old"), + ExpectedConfigRevision::Exact(Some("rev-old")), + ) + .await + .unwrap_err(); + + assert!(is_revision_conflict(&err)); + let conflict = err + .downcast_ref::() + .expect("expected typed revision conflict"); + match conflict { + ManagedConfigError::RevisionConflict { expected, current } => { + assert_eq!(expected.as_deref(), Some("rev-old")); + assert_eq!(current.as_deref(), Some("rev-new")); + } + } + assert_eq!( + storage + .db() + .get_managed_config_revision((user_id, machine_id)) + .await + .unwrap() + .as_deref(), + Some("rev-new") + ); + assert!( + storage + .db() + .get_network_config((user_id, machine_id), &inst_id.to_string()) + .await + .unwrap() + .is_none() + ); + } + + #[test] + fn managed_config_reconcile_lock_reuses_live_entry_and_replaces_stale_entry() { + let key = (i32::MIN, uuid::Uuid::new_v4()); + MANAGED_CONFIG_RECONCILE_LOCKS.remove(&key); + + let first = managed_config_reconcile_lock(key); + let second = managed_config_reconcile_lock(key); + assert!(Arc::ptr_eq(&first, &second)); + + let stale = Arc::downgrade(&first); + drop(first); + drop(second); + assert!(stale.upgrade().is_none()); + + let third = managed_config_reconcile_lock(key); + let stored = MANAGED_CONFIG_RECONCILE_LOCKS + .get(&key) + .and_then(|lock| lock.upgrade()) + .expect("expected refreshed reconcile lock"); + assert!(Arc::ptr_eq(&third, &stored)); + drop(stored); + + remove_unused_managed_config_reconcile_lock(key, &third); + assert!(!MANAGED_CONFIG_RECONCILE_LOCKS.contains_key(&key)); + } + + #[test] + fn managed_config_reconcile_lock_cleanup_keeps_live_waiters() { + let key = (i32::MIN + 1, uuid::Uuid::new_v4()); + MANAGED_CONFIG_RECONCILE_LOCKS.remove(&key); + + let current = managed_config_reconcile_lock(key); + let waiter = managed_config_reconcile_lock(key); + + remove_unused_managed_config_reconcile_lock(key, ¤t); + assert!(MANAGED_CONFIG_RECONCILE_LOCKS.contains_key(&key)); + + drop(waiter); + remove_unused_managed_config_reconcile_lock(key, ¤t); + assert!(!MANAGED_CONFIG_RECONCILE_LOCKS.contains_key(&key)); + } + + #[test] + fn normalize_network_config_accepts_console_snake_case_fields() { + let inst_id = uuid::Uuid::new_v4(); + + let config = normalize_network_config( + json!({ + "instance_id": uuid::Uuid::new_v4().to_string(), + "instanceId": uuid::Uuid::new_v4().to_string(), + "dhcp": true, + "network_name": "managed", + "network_secret": "secret", + "networking_method": "Manual", + "peer_urls": ["http://console.example/peer"], + "listener_urls": ["udp://0.0.0.0:0"], + "no_tun": true, + "relay_all_peer_rpc": true, + "disable_udp_hole_punching": true, + "enable_private_mode": true, + "port_forwards": [{ + "bind_ip": "127.0.0.1", + "bind_port": 23000, + "dst_ip": "10.144.0.1", + "dst_port": 5174, + "proto": "tcp" + }], + "disable_sym_hole_punching": true, + "disable_tcp_hole_punching": true, + "secure_mode": { + "enabled": true + }, + "credential_file": "/tmp/e2e.credentials.json", + "need_p2p": true, + "disable_relay_data": false + }), + inst_id, + ) + .unwrap(); + + assert_eq!( + config.instance_id.as_deref(), + Some(inst_id.to_string().as_str()) + ); + assert_eq!(config.no_tun, Some(true)); + assert_eq!(config.relay_all_peer_rpc, Some(true)); + assert_eq!(config.disable_udp_hole_punching, Some(true)); + assert_eq!(config.enable_private_mode, Some(true)); + assert_eq!(config.disable_sym_hole_punching, Some(true)); + assert_eq!(config.disable_tcp_hole_punching, Some(true)); + assert_eq!(config.need_p2p, Some(true)); + assert_eq!(config.disable_relay_data, Some(false)); + assert_eq!(config.port_forwards.len(), 1); + + let runtime_config = config.gen_config().unwrap(); + let flags = runtime_config.get_flags(); + assert!(flags.no_tun); + assert!(flags.private_mode); + assert!(flags.need_p2p); + assert!(flags.relay_all_peer_rpc); + assert!(flags.disable_tcp_hole_punching); + assert!(flags.disable_udp_hole_punching); + assert!(flags.disable_sym_hole_punching); + assert_eq!(runtime_config.get_port_forwards().len(), 1); + } + + #[test] + fn normalize_network_config_rejects_conflicting_key_spellings() { + let err = normalize_network_config( + json!({ + "network_name": "snake", + "networkName": "camel" + }), + uuid::Uuid::new_v4(), + ) + .unwrap_err(); + + assert!( + err.to_string() + .contains("duplicate network_config field after key normalization"), + "unexpected error: {err:?}" + ); + } + + #[tokio::test] + async fn sync_running_config_sources_updates_enabled_config_source_from_runtime() { + let storage = Storage::new(crate::db::Db::memory_db().await); + let user_id = storage + .db() + .auto_create_user("web-user-sync-source") + .await + .unwrap() + .id; + let machine_id = uuid::Uuid::new_v4(); + let inst_id = uuid::Uuid::new_v4(); + + storage + .db() + .insert_or_update_user_network_config( + (user_id, machine_id), + inst_id, + NetworkConfig { + network_name: Some("web-owned".to_string()), + ..Default::default() + }, + ConfigSource::Web, + ) + .await + .unwrap(); + + let local_configs = storage + .db() + .list_network_configs((user_id, machine_id), ListNetworkProps::EnabledOnly) + .await + .unwrap(); + sync_running_config_sources( + storage.db(), + user_id, + machine_id, + &local_configs, + &[NetworkMeta { + inst_id: Some(inst_id.into()), + source: RpcConfigSource::User as i32, + ..Default::default() + }], + ) + .await + .unwrap(); + + let updated = storage + .db() + .get_network_config((user_id, machine_id), &inst_id.to_string()) + .await + .unwrap() + .unwrap(); + assert_eq!(updated.get_network_config_source(), ConfigSource::Web); + } + + #[test] + fn persisted_sources_map_to_rpc_sources() { + assert_eq!( + PersistedConfigSource::Web.auto_run_rpc_source(), + RpcConfigSource::Web + ); + assert_eq!( + PersistedConfigSource::User.auto_run_rpc_source(), + RpcConfigSource::User + ); + } +} diff --git a/easytier-web/src/client_manager/mod.rs b/easytier-web/src/client_manager/mod.rs index 1befe3e5..a08e95ac 100644 --- a/easytier-web/src/client_manager/mod.rs +++ b/easytier-web/src/client_manager/mod.rs @@ -1,3 +1,5 @@ +mod managed_config; +mod runtime_reconcile; pub mod session; pub mod storage; @@ -5,6 +7,7 @@ use std::sync::{ Arc, atomic::{AtomicU32, Ordering}, }; +use std::time::Duration; use dashmap::DashMap; use easytier::{ @@ -30,6 +33,10 @@ use crate::db::{Db, UserIdInDb, entity::user_running_network_configs}; #[include = "geoip2-cn.mmdb"] struct GeoipDb; +pub fn is_managed_config_revision_conflict(error: &anyhow::Error) -> bool { + managed_config::is_revision_conflict(error) +} + fn load_geoip_db(geoip_db: Option) -> Option>> { if let Some(path) = geoip_db { match maxminddb::Reader::open_readfile(&path) { @@ -63,12 +70,14 @@ pub struct ClientManager { webhook_config: SharedWebhookConfig, geoip_db: Arc>>>, + heartbeat_min_response_delay: Duration, } impl ClientManager { pub fn new( db: Db, geoip_db: Option, + heartbeat_min_response_delay: Duration, feature_flags: Arc, webhook_config: SharedWebhookConfig, ) -> Self { @@ -92,6 +101,7 @@ impl ClientManager { webhook_config, geoip_db: Arc::new(load_geoip_db(geoip_db)), + heartbeat_min_response_delay, } } @@ -105,6 +115,7 @@ impl ClientManager { let storage = self.storage.weak_ref(); let listeners_cnt = self.listeners_cnt.clone(); let geoip_db = self.geoip_db.clone(); + let heartbeat_min_response_delay = self.heartbeat_min_response_delay; let feature_flags = self.feature_flags.clone(); let webhook_config = self.webhook_config.clone(); self.tasks.spawn(async move { @@ -129,6 +140,7 @@ impl ClientManager { storage.clone(), client_url.clone(), location, + heartbeat_min_response_delay, feature_flags.clone(), webhook_config.clone(), ); @@ -149,6 +161,10 @@ impl ClientManager { self.storage.list_clients() } + pub async fn list_all_sessions(&self) -> Vec { + self.storage.list_all_clients() + } + pub fn get_session_by_machine_id( &self, user_id: UserIdInDb, @@ -169,7 +185,7 @@ impl ClientManager { ) -> bool { let Some(client_url) = self .storage - .get_client_url_by_machine_id(user_id, machine_id) + .get_client_url_by_machine_id_with_auth(user_id, machine_id, false) else { return false; }; @@ -189,14 +205,30 @@ impl ClientManager { user_id: UserIdInDb, machine_id: uuid::Uuid, desired_configs: Vec, + config_revision: Option, + expected_config_revision: Option, ) -> anyhow::Result<()> { - session::SessionRpcService::reconcile_web_source_configs( + let expected_config_revision = match expected_config_revision.as_deref().map(str::trim) { + None => managed_config::ExpectedConfigRevision::Any, + Some("") => managed_config::ExpectedConfigRevision::Exact(None), + Some(revision) => managed_config::ExpectedConfigRevision::Exact(Some(revision)), + }; + managed_config::reconcile_web_source_configs( &self.storage, user_id, machine_id, desired_configs, + config_revision.as_deref(), + expected_config_revision, ) .await?; + if let Some(config_revision) = config_revision + && let Some(session) = self.get_session_by_machine_id(user_id, &machine_id) + { + session + .notify_config_revision_changed(user_id, machine_id, config_revision) + .await; + } Ok(()) } @@ -331,19 +363,449 @@ impl #[cfg(test)] mod tests { - use std::{sync::Arc, time::Duration}; + use std::{ + collections::VecDeque, + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::Duration, + }; + use axum::{Json, Router, extract::State, routing::post}; use easytier::{ + common::MachineIdOptions, instance_manager::NetworkInstanceManager, + proto::{ + api::manage::{NetworkConfig, NetworkingMethod, PortForwardConfig}, + common::CompressionAlgoPb, + }, + rpc_service::remote_client::Storage as RemoteStorage, tunnel::{ common::tests::wait_for_condition, udp::{UdpTunnelConnector, UdpTunnelListener}, }, - web_client::WebClient, + web_client::{WebClient, run_web_client}, }; + use serde_json::json; use sqlx::Executor; + use tokio::net::UdpSocket; - use crate::{FeatureFlags, client_manager::ClientManager, db::Db}; + use crate::{ + FeatureFlags, client_manager::ClientManager, db::Db, webhook::ManagedNetworkConfig, + }; + + const MANAGED_CONFIG_TOKEN: &str = "managed-config-token"; + + #[derive(Debug, Clone)] + struct TestWebhookState { + validate_responses: Arc>>, + validate_count: Arc, + block_second_validate: Arc, + allow_second_validate: Arc, + } + + impl TestWebhookState { + fn new(validate_responses: impl IntoIterator) -> Self { + Self { + validate_responses: Arc::new(tokio::sync::Mutex::new( + validate_responses.into_iter().collect(), + )), + validate_count: Arc::new(AtomicUsize::new(0)), + block_second_validate: Arc::new(AtomicBool::new(false)), + allow_second_validate: Arc::new(AtomicBool::new(true)), + } + } + + fn with_blocked_second_validate( + validate_responses: impl IntoIterator, + ) -> Self { + let state = Self::new(validate_responses); + state.block_second_validate.store(true, Ordering::Release); + state.allow_second_validate.store(false, Ordering::Release); + state + } + + fn allow_second_validate(&self) { + self.allow_second_validate.store(true, Ordering::Release); + } + + fn validate_count(&self) -> usize { + self.validate_count.load(Ordering::Acquire) + } + } + + async fn validate_token_handler( + State(state): State, + ) -> Json { + let count = state.validate_count.fetch_add(1, Ordering::AcqRel) + 1; + if count == 2 && state.block_second_validate.load(Ordering::Acquire) { + while !state.allow_second_validate.load(Ordering::Acquire) { + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + let valid = state + .validate_responses + .lock() + .await + .pop_front() + .unwrap_or(true); + if !valid { + return Json(json!({ "valid": false })); + } + + Json(json!({ + "valid": true, + "binding_version": count, + "config_revision": format!("validated-rev-{count}") + })) + } + + async fn webhook_ack_handler() -> Json { + Json(json!({})) + } + + async fn test_webhook_config() -> ( + crate::webhook::SharedWebhookConfig, + tokio::task::JoinHandle<()>, + TestWebhookState, + ) { + let state = TestWebhookState::new([true]); + test_webhook_config_with_state(state).await + } + + async fn test_webhook_config_with_state( + state: TestWebhookState, + ) -> ( + crate::webhook::SharedWebhookConfig, + tokio::task::JoinHandle<()>, + TestWebhookState, + ) { + let app = Router::new() + .route("/validate-token", post(validate_token_handler)) + .route("/webhook/node-connected", post(webhook_ack_handler)) + .route("/webhook/node-disconnected", post(webhook_ack_handler)) + .with_state(state.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + ( + Arc::new(crate::webhook::WebhookConfig::new( + Some(format!("http://{addr}")), + None, + None, + None, + None, + )), + server, + state, + ) + } + + async fn add_random_udp_listener(mgr: &mut ClientManager) -> std::net::SocketAddr { + let socket = Arc::new(UdpSocket::bind("127.0.0.1:0").await.unwrap()); + let addr = socket.local_addr().unwrap(); + let listener = + UdpTunnelListener::new_with_socket(format!("udp://{addr}").parse().unwrap(), socket); + mgr.add_listener(listener).await.unwrap(); + addr + } + + async fn wait_for_validated_user(mgr: &ClientManager, machine_id: uuid::Uuid) -> i32 { + tokio::time::timeout(Duration::from_secs(12), async { + loop { + if let Some(token) = mgr.list_sessions().await.into_iter().find(|token| { + token.token == MANAGED_CONFIG_TOKEN && token.machine_id == machine_id + }) { + break token.user_id; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .unwrap() + } + + async fn wait_for_validate_count(state: &TestWebhookState, target: usize) { + tokio::time::timeout(Duration::from_secs(12), async { + loop { + if state.validate_count() >= target { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .unwrap(); + } + + async fn wait_for_session_urls(mgr: &ClientManager) -> Vec { + tokio::time::timeout(Duration::from_secs(12), async { + loop { + let urls = mgr + .client_sessions + .iter() + .map(|entry| entry.key().clone()) + .collect::>(); + if !urls.is_empty() { + break urls; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .unwrap() + } + + fn managed_config( + instance_id: uuid::Uuid, + network_config: serde_json::Value, + ) -> ManagedNetworkConfig { + ManagedNetworkConfig { + instance_id: instance_id.to_string(), + network_config, + } + } + + async fn wait_for_runtime_config( + manager: &NetworkInstanceManager, + inst_id: uuid::Uuid, + predicate: impl Fn(&NetworkConfig) -> bool, + ) -> NetworkConfig { + tokio::time::timeout(Duration::from_secs(12), async { + loop { + if let Some(config) = manager + .get_instance_config(&inst_id) + .and_then(|config| NetworkConfig::new_from_config(&config).ok()) + .filter(|config| predicate(config)) + { + break config; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .unwrap() + } + + async fn start_web_client_for_test( + config_server_addr: std::net::SocketAddr, + machine_id: uuid::Uuid, + manager: Arc, + ) -> WebClient { + run_web_client( + &format!("udp://{config_server_addr}/{MANAGED_CONFIG_TOKEN}"), + MachineIdOptions { + explicit_machine_id: Some(machine_id.to_string()), + state_dir: None, + }, + Some("managed-config-core".to_string()), + false, + manager, + None, + ) + .await + .unwrap() + } + + async fn clear_managed_config_db( + mgr: &ClientManager, + user_id: i32, + machine_id: uuid::Uuid, + instance_id: uuid::Uuid, + ) { + mgr.db() + .delete_web_network_configs((user_id, machine_id), &[instance_id]) + .await + .unwrap(); + sqlx::query("DELETE FROM managed_config_revisions WHERE user_id = ? AND device_id = ?") + .bind(user_id) + .bind(machine_id.to_string()) + .execute(&mgr.db().inner()) + .await + .unwrap(); + } + + fn assert_updated_runtime_config(updated: &NetworkConfig, instance_id: uuid::Uuid) { + assert_eq!( + updated.instance_id.as_deref(), + Some(instance_id.to_string().as_str()) + ); + assert_eq!(updated.dhcp, Some(false)); + assert_eq!(updated.virtual_ipv4.as_deref(), Some("10.88.0.7")); + assert_eq!(updated.network_length, Some(24)); + assert_eq!(updated.hostname.as_deref(), Some("managed-updated-host")); + assert_eq!(updated.network_name.as_deref(), Some("managed-updated")); + assert_eq!(updated.network_secret.as_deref(), Some("secret-updated")); + assert_eq!( + updated.networking_method, + Some(NetworkingMethod::Manual as i32) + ); + assert_eq!(updated.peer_urls, vec!["tcp://127.0.0.1:11010".to_string()]); + assert_eq!( + updated.proxy_cidrs, + vec![ + "10.44.0.0/24".to_string(), + "10.45.0.0/24->10.46.0.0/24".to_string() + ] + ); + assert_eq!(updated.no_tun, Some(true)); + assert_eq!(updated.disable_ipv6, Some(true)); + assert_eq!(updated.enable_kcp_proxy, Some(true)); + assert_eq!(updated.disable_kcp_input, Some(true)); + assert_eq!(updated.enable_quic_proxy, Some(true)); + assert_eq!(updated.disable_quic_input, Some(true)); + assert_eq!(updated.disable_p2p, Some(true)); + assert_eq!(updated.p2p_only, Some(true)); + assert_eq!(updated.lazy_p2p, Some(true)); + assert_eq!(updated.relay_all_peer_rpc, Some(true)); + assert_eq!(updated.need_p2p, Some(true)); + assert_eq!(updated.multi_thread, Some(false)); + assert_eq!(updated.proxy_forward_by_system, Some(true)); + assert_eq!(updated.disable_encryption, Some(true)); + assert_eq!(updated.enable_relay_network_whitelist, Some(true)); + assert_eq!( + updated.relay_network_whitelist, + vec!["10.44.0.0/24".to_string(), "10.45.0.0/24".to_string()] + ); + assert_eq!(updated.enable_manual_routes, Some(true)); + assert_eq!( + updated.routes, + vec!["10.60.0.0/16".to_string(), "10.61.0.0/16".to_string()] + ); + assert_eq!(updated.port_forwards[0].bind_ip, "127.0.0.1"); + assert_eq!(updated.port_forwards[0].bind_port, 0); + assert_eq!(updated.port_forwards[0].dst_ip, "10.88.0.8"); + assert_eq!(updated.port_forwards[0].dst_port, 80); + assert_eq!(updated.port_forwards[0].proto, "tcp"); + assert_eq!(updated.disable_udp_hole_punching, Some(true)); + assert_eq!(updated.disable_tcp_hole_punching, Some(true)); + assert_eq!(updated.disable_sym_hole_punching, Some(true)); + assert_eq!(updated.disable_upnp, Some(true)); + assert_eq!(updated.disable_relay_data, Some(true)); + assert_eq!(updated.enable_magic_dns, Some(true)); + assert_eq!(updated.enable_private_mode, Some(true)); + assert_eq!(updated.mtu, Some(1360)); + assert_eq!( + updated.data_compress_algo, + Some(CompressionAlgoPb::Zstd as i32) + ); + assert_eq!(updated.encryption_algorithm.as_deref(), Some("xor")); + assert_eq!(updated.instance_recv_bps_limit, Some(123456)); + assert_eq!(updated.enable_udp_broadcast_relay, Some(true)); + assert_eq!(updated.socket_mark, Some(0)); + } + + fn initial_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value { + json!({ + "instance_id": inst_id.to_string(), + "dhcp": true, + "network_name": "managed-initial", + "network_secret": "secret-initial", + "networking_method": "Standalone", + "no_tun": true, + "disable_ipv6": true, + "enable_kcp_proxy": false, + "disable_kcp_input": false, + "relay_all_peer_rpc": false, + "multi_thread": false, + "disable_relay_data": false, + "mtu": 1380 + }) + } + + fn updated_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value { + serde_json::to_value(NetworkConfig { + instance_id: Some(inst_id.to_string()), + dhcp: Some(false), + virtual_ipv4: Some("10.88.0.7".to_string()), + network_length: Some(24), + hostname: Some("managed-updated-host".to_string()), + network_name: Some("managed-updated".to_string()), + network_secret: Some("secret-updated".to_string()), + networking_method: Some(NetworkingMethod::Manual as i32), + peer_urls: vec!["tcp://127.0.0.1:11010".to_string()], + proxy_cidrs: vec![ + "10.44.0.0/24".to_string(), + "10.45.0.0/24->10.46.0.0/24".to_string(), + ], + no_tun: Some(true), + disable_ipv6: Some(true), + enable_kcp_proxy: Some(true), + disable_kcp_input: Some(true), + enable_quic_proxy: Some(true), + disable_quic_input: Some(true), + disable_p2p: Some(true), + p2p_only: Some(true), + lazy_p2p: Some(true), + relay_all_peer_rpc: Some(true), + need_p2p: Some(true), + multi_thread: Some(false), + proxy_forward_by_system: Some(true), + disable_encryption: Some(true), + enable_relay_network_whitelist: Some(true), + relay_network_whitelist: vec!["10.44.0.0/24".to_string(), "10.45.0.0/24".to_string()], + enable_manual_routes: Some(true), + routes: vec!["10.60.0.0/16".to_string(), "10.61.0.0/16".to_string()], + port_forwards: vec![PortForwardConfig { + bind_ip: "127.0.0.1".to_string(), + bind_port: 0, + dst_ip: "10.88.0.8".to_string(), + dst_port: 80, + proto: "tcp".to_string(), + }], + disable_udp_hole_punching: Some(true), + disable_tcp_hole_punching: Some(true), + disable_sym_hole_punching: Some(true), + disable_upnp: Some(true), + disable_relay_data: Some(true), + enable_magic_dns: Some(true), + enable_private_mode: Some(true), + mtu: Some(1360), + data_compress_algo: Some(CompressionAlgoPb::Zstd as i32), + encryption_algorithm: Some("xor".to_string()), + instance_recv_bps_limit: Some(123456), + enable_udp_broadcast_relay: Some(true), + socket_mark: Some(0), + ..Default::default() + }) + .unwrap() + } + + fn redelivered_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value { + serde_json::to_value(NetworkConfig { + instance_id: Some(inst_id.to_string()), + dhcp: Some(false), + virtual_ipv4: Some("10.88.0.7".to_string()), + network_length: Some(24), + hostname: Some("managed-redelivered-host".to_string()), + network_name: Some("managed-redelivered".to_string()), + network_secret: Some("secret-updated".to_string()), + networking_method: Some(NetworkingMethod::Manual as i32), + peer_urls: vec!["tcp://127.0.0.1:11010".to_string()], + proxy_cidrs: vec![ + "10.44.0.0/24".to_string(), + "10.45.0.0/24->10.46.0.0/24".to_string(), + ], + no_tun: Some(true), + disable_ipv6: Some(true), + enable_kcp_proxy: Some(true), + disable_kcp_input: Some(true), + relay_all_peer_rpc: Some(true), + need_p2p: Some(true), + multi_thread: Some(false), + enable_private_mode: Some(true), + mtu: Some(1360), + data_compress_algo: Some(CompressionAlgoPb::Zstd as i32), + encryption_algorithm: Some("xor".to_string()), + instance_recv_bps_limit: Some(654321), + ..Default::default() + }) + .unwrap() + } #[tokio::test] async fn test_client() { @@ -351,6 +813,7 @@ mod tests { let mut mgr = ClientManager::new( Db::memory_db().await, None, + Duration::ZERO, Arc::new(FeatureFlags::default()), Arc::new(crate::webhook::WebhookConfig::new( None, None, None, None, None, @@ -410,4 +873,223 @@ mod tests { println!("{:?}", req); println!("{:?}", mgr); } + + #[tokio::test] + async fn managed_web_config_revision_updates_running_core_config() { + let (webhook_config, webhook_server, _) = test_webhook_config().await; + let mut mgr = ClientManager::new( + Db::memory_db().await, + None, + Duration::ZERO, + Arc::new(FeatureFlags::default()), + webhook_config, + ); + let config_server_addr = add_random_udp_listener(&mut mgr).await; + + let machine_id = uuid::Uuid::new_v4(); + let instance_id = uuid::Uuid::new_v4(); + let core_manager = Arc::new(NetworkInstanceManager::new()); + let client = + start_web_client_for_test(config_server_addr, machine_id, core_manager.clone()).await; + + let user_id = wait_for_validated_user(&mgr, machine_id).await; + mgr.reconcile_managed_network_configs( + user_id, + machine_id, + vec![managed_config( + instance_id, + initial_managed_network_config(instance_id), + )], + Some("rev-initial".to_string()), + None, + ) + .await + .unwrap(); + + wait_for_runtime_config(&core_manager, instance_id, |config| { + config.network_name.as_deref() == Some("managed-initial") + }) + .await; + + // Online revision update: web-owned running config is fully overwritten + // when non-hot-patch flags such as enable_kcp_proxy change. + mgr.reconcile_managed_network_configs( + user_id, + machine_id, + vec![managed_config( + instance_id, + updated_managed_network_config(instance_id), + )], + Some("rev-updated".to_string()), + Some("rev-initial".to_string()), + ) + .await + .unwrap(); + + let updated = wait_for_runtime_config(&core_manager, instance_id, |config| { + config.network_name.as_deref() == Some("managed-updated") + && config.enable_kcp_proxy == Some(true) + && config.port_forwards.len() == 1 + }) + .await; + assert_updated_runtime_config(&updated, instance_id); + + assert_eq!( + core_manager.get_instance_network_config_source(&instance_id), + Some(easytier::common::config::ConfigSource::Web) + ); + assert_eq!( + mgr.db() + .get_managed_config_revision((user_id, machine_id)) + .await + .unwrap() + .as_deref(), + Some("rev-updated") + ); + + // Web DB loss path: clear web-owned config and revision, then simulate + // the webhook re-posting the authoritative desired config. The already + // connected session should receive the distinguishable re-delivered + // revision without restarting. + clear_managed_config_db(&mgr, user_id, machine_id, instance_id).await; + assert!( + mgr.db() + .get_network_config((user_id, machine_id), &instance_id.to_string()) + .await + .unwrap() + .is_none() + ); + assert!( + mgr.db() + .get_managed_config_revision((user_id, machine_id)) + .await + .unwrap() + .is_none() + ); + + mgr.reconcile_managed_network_configs( + user_id, + machine_id, + vec![managed_config( + instance_id, + redelivered_managed_network_config(instance_id), + )], + Some("rev-webhook-redelivery".to_string()), + None, + ) + .await + .unwrap(); + let redelivered = wait_for_runtime_config(&core_manager, instance_id, |config| { + config.network_name.as_deref() == Some("managed-redelivered") + && config.instance_recv_bps_limit == Some(654321) + }) + .await; + assert_eq!( + redelivered.instance_id.as_deref(), + Some(instance_id.to_string().as_str()) + ); + assert_eq!( + redelivered.hostname.as_deref(), + Some("managed-redelivered-host") + ); + assert_eq!( + redelivered.network_name.as_deref(), + Some("managed-redelivered") + ); + assert_eq!(redelivered.enable_kcp_proxy, Some(true)); + assert_eq!(redelivered.instance_recv_bps_limit, Some(654321)); + assert_eq!( + core_manager.get_instance_network_config_source(&instance_id), + Some(easytier::common::config::ConfigSource::Web) + ); + assert_eq!( + mgr.db() + .get_managed_config_revision((user_id, machine_id)) + .await + .unwrap() + .as_deref(), + Some("rev-webhook-redelivery") + ); + + // Reconnect path: a fresh core manager has no local runtime state, so + // the new session must replay the managed config persisted in web DB. + drop(client); + let reconnected_core_manager = Arc::new(NetworkInstanceManager::new()); + let _reconnected_client = start_web_client_for_test( + config_server_addr, + machine_id, + reconnected_core_manager.clone(), + ) + .await; + wait_for_validated_user(&mgr, machine_id).await; + let replayed = wait_for_runtime_config(&reconnected_core_manager, instance_id, |config| { + config.network_name.as_deref() == Some("managed-redelivered") + && config.instance_recv_bps_limit == Some(654321) + }) + .await; + assert_eq!( + replayed.network_name.as_deref(), + Some("managed-redelivered") + ); + assert_eq!(replayed.enable_kcp_proxy, Some(true)); + assert_eq!(replayed.instance_recv_bps_limit, Some(654321)); + + webhook_server.abort(); + } + + #[tokio::test] + async fn webhook_reject_disconnects_and_revalidates_after_reconnect() { + let webhook_state = TestWebhookState::with_blocked_second_validate([false, true]); + let (webhook_config, webhook_server, webhook_state) = + test_webhook_config_with_state(webhook_state).await; + let mut mgr = ClientManager::new( + Db::memory_db().await, + None, + Duration::ZERO, + Arc::new(FeatureFlags::default()), + webhook_config, + ); + let config_server_addr = add_random_udp_listener(&mut mgr).await; + let machine_id = uuid::Uuid::new_v4(); + let core_manager = Arc::new(NetworkInstanceManager::new()); + let client = + start_web_client_for_test(config_server_addr, machine_id, core_manager.clone()).await; + + let first_session_urls = wait_for_session_urls(&mgr).await; + wait_for_validate_count(&webhook_state, 1).await; + wait_for_validate_count(&webhook_state, 2).await; + assert!( + mgr.list_sessions().await.is_empty(), + "invalid validate-token response must not authorize the session" + ); + + webhook_state.allow_second_validate(); + let user_id = wait_for_validated_user(&mgr, machine_id).await; + tokio::time::timeout(Duration::from_secs(12), async { + loop { + let reconnected = mgr + .client_sessions + .iter() + .any(|entry| !first_session_urls.iter().any(|url| url == entry.key())); + if reconnected { + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .unwrap(); + + assert!( + client.is_connected(), + "web client should reconnect after invalid session heartbeat failure" + ); + assert!(webhook_state.validate_count() >= 2); + assert!( + mgr.get_session_by_machine_id(user_id, &machine_id) + .is_some() + ); + + webhook_server.abort(); + } } diff --git a/easytier-web/src/client_manager/runtime_reconcile.rs b/easytier-web/src/client_manager/runtime_reconcile.rs new file mode 100644 index 00000000..42a063f2 --- /dev/null +++ b/easytier-web/src/client_manager/runtime_reconcile.rs @@ -0,0 +1,812 @@ +use anyhow::Context as _; +use easytier::{ + common::config::{ + ConfigLoader, EncryptionAlgorithm, PortForwardConfig as RuntimePortForwardConfig, + }, + proto::{ + acl::Acl, + api::{ + config::{ + AclPatch, ConfigPatchAction, InstanceConfigPatch, PatchConfigRequest, + PortForwardPatch, ProxyNetworkPatch, + }, + instance::{InstanceIdentifier, instance_identifier}, + manage::{ + ConfigSource as RpcConfigSource, GetNetworkInstanceConfigRequest, NetworkConfig, + RunNetworkInstanceRequest, + }, + }, + common::{CompressionAlgoPb, Ipv4Inet as RpcIpv4Inet}, + rpc_types::controller::BaseController, + }, +}; + +use super::session::{SessionConfigClient, SessionRpcClient}; + +pub(super) enum RuntimeReconcileAction { + None, + Run { + config: Box, + overwrite: bool, + }, + Patch(Box), +} + +#[derive(Clone, PartialEq)] +struct RuntimeProxyNetwork { + cidr: String, + mapped_cidr: Option, +} + +fn instance_identifier(inst_id: &str) -> anyhow::Result { + let inst_id = uuid::Uuid::parse_str(inst_id) + .with_context(|| format!("invalid runtime instance id: {inst_id}"))?; + Ok(InstanceIdentifier { + selector: Some(instance_identifier::Selector::Id(inst_id.into())), + }) +} + +fn hot_patch_base(config: &NetworkConfig) -> anyhow::Result { + let data_compress_algo = normalized_data_compress_algo(config.data_compress_algo); + let encryption_algorithm = normalized_encryption_algorithm(config.encryption_algorithm.clone()); + let mut config = NetworkConfig::new_from_config(config.gen_config()?)?; + let is_credential_mode = config.network_secret.is_none() + && config + .secure_mode + .as_ref() + .and_then(|mode| mode.local_private_key.as_deref()) + .is_some_and(|key| !key.is_empty()); + config.acl = None; + config.port_forwards.clear(); + config.proxy_cidrs.clear(); + config.disable_relay_data = None; + if config.dhcp.unwrap_or_default() { + config.virtual_ipv4 = None; + config.network_length = None; + } + if let Some(secure_mode) = config.secure_mode.as_mut() { + if !is_credential_mode { + secure_mode.local_private_key = None; + } + secure_mode.local_public_key = None; + } + config.data_compress_algo = data_compress_algo; + config.encryption_algorithm = encryption_algorithm; + Ok(config) +} + +fn normalized_data_compress_algo(algo: Option) -> Option { + let default = CompressionAlgoPb::None as i32; + let effective = algo.map(|algo| if algo < default { default } else { algo }); + effective.filter(|algo| *algo != default) +} + +fn normalized_encryption_algorithm(algo: Option) -> Option { + let default = EncryptionAlgorithm::default().to_string(); + algo.filter(|algo| algo != &default) +} + +fn diff_port_forwards( + current: &[RuntimePortForwardConfig], + desired: &[RuntimePortForwardConfig], +) -> Vec { + let mut patches = Vec::new(); + for cfg in unique_port_forwards(current, desired) { + let current_count = current.iter().filter(|item| *item == &cfg).count(); + let desired_count = desired.iter().filter(|item| *item == &cfg).count(); + if current_count == desired_count { + continue; + } + if current_count > 0 { + patches.push(PortForwardPatch { + action: ConfigPatchAction::Remove as i32, + cfg: Some(cfg.clone().into()), + }); + } + patches.extend((0..desired_count).map(|_| PortForwardPatch { + action: ConfigPatchAction::Add as i32, + cfg: Some(cfg.clone().into()), + })); + } + patches +} + +fn unique_port_forwards( + current: &[RuntimePortForwardConfig], + desired: &[RuntimePortForwardConfig], +) -> Vec { + let mut unique = Vec::new(); + for cfg in current.iter().chain(desired.iter()) { + if !unique.contains(cfg) { + unique.push(cfg.clone()); + } + } + unique +} + +fn parse_rpc_ipv4_inet(value: &str) -> anyhow::Result { + value + .parse::() + .with_context(|| format!("failed to parse runtime ipv4 cidr: {value}")) +} + +fn diff_proxy_networks( + current: &[RuntimeProxyNetwork], + desired: &[RuntimeProxyNetwork], +) -> anyhow::Result> { + if current == desired { + return Ok(Vec::new()); + } + + let mut patches = vec![ProxyNetworkPatch { + action: ConfigPatchAction::Clear as i32, + cidr: Some(clear_proxy_network_cidr(current, desired)?), + ..Default::default() + }]; + for proxy_network in desired { + patches.push(ProxyNetworkPatch { + action: ConfigPatchAction::Add as i32, + cidr: Some(parse_rpc_ipv4_inet(&proxy_network.cidr)?), + mapped_cidr: proxy_network + .mapped_cidr + .as_deref() + .map(parse_rpc_ipv4_inet) + .transpose()?, + }); + } + Ok(patches) +} + +fn clear_proxy_network_cidr( + current: &[RuntimeProxyNetwork], + desired: &[RuntimeProxyNetwork], +) -> anyhow::Result { + let cidr = desired + .first() + .or_else(|| current.first()) + .map(|proxy_network| proxy_network.cidr.as_str()) + .unwrap_or("0.0.0.0/0"); + parse_rpc_ipv4_inet(cidr) +} + +fn normalized_acl(acl: &Option) -> Option { + let acl = acl.clone().unwrap_or_default(); + (acl != Acl::default()).then_some(acl) +} + +fn normalized_port_forwards( + config: &NetworkConfig, +) -> anyhow::Result> { + Ok(config + .gen_config()? + .get_port_forwards() + .into_iter() + .map(|cfg| { + RuntimePortForwardConfig::from(easytier::proto::common::PortForwardConfigPb::from(cfg)) + }) + .collect()) +} + +fn normalized_proxy_networks(config: &NetworkConfig) -> anyhow::Result> { + Ok(config + .gen_config()? + .get_proxy_cidrs() + .into_iter() + .map(|proxy_network| RuntimeProxyNetwork { + cidr: proxy_network.cidr.to_string(), + mapped_cidr: proxy_network.mapped_cidr.map(|cidr| cidr.to_string()), + }) + .collect()) +} + +fn normalized_disable_relay_data(config: &NetworkConfig) -> anyhow::Result { + Ok(config.gen_config()?.get_flags().disable_relay_data) +} + +fn web_source_runtime_patch( + current: &NetworkConfig, + desired: &NetworkConfig, +) -> anyhow::Result> { + if let Some(desired_hostname) = desired + .hostname + .as_deref() + .filter(|hostname| !hostname.is_empty()) + && current.hostname.as_deref() != Some(desired_hostname) + { + return Ok(None); + } + let mut current_base = hot_patch_base(current)?; + let mut desired_base = hot_patch_base(desired)?; + current_base.hostname = None; + desired_base.hostname = None; + if current_base != desired_base { + return Ok(None); + } + + let mut patch = InstanceConfigPatch::default(); + let current_acl = normalized_acl(¤t.acl); + let desired_acl = normalized_acl(&desired.acl); + if current_acl != desired_acl { + patch.acl = Some(AclPatch { + acl: Some(desired_acl.unwrap_or_default()), + ..Default::default() + }); + } + + let current_port_forwards = normalized_port_forwards(current)?; + let desired_port_forwards = normalized_port_forwards(desired)?; + if current_port_forwards != desired_port_forwards { + patch.port_forwards = diff_port_forwards(¤t_port_forwards, &desired_port_forwards); + } + + let current_proxy_networks = normalized_proxy_networks(current)?; + let desired_proxy_networks = normalized_proxy_networks(desired)?; + if current_proxy_networks != desired_proxy_networks { + if current_proxy_networks.is_empty() { + return Ok(None); + } + patch.proxy_networks = + diff_proxy_networks(¤t_proxy_networks, &desired_proxy_networks)?; + } + + let current_disable_relay_data = normalized_disable_relay_data(current)?; + let desired_disable_relay_data = normalized_disable_relay_data(desired)?; + if current_disable_relay_data != desired_disable_relay_data { + patch.disable_relay_data = Some(desired_disable_relay_data); + } + + Ok(Some(patch)) +} + +fn ensure_runtime_config_converged( + current: &NetworkConfig, + desired: &NetworkConfig, +) -> anyhow::Result<()> { + let patch = web_source_runtime_patch(current, desired)?; + match patch { + Some(patch) if patch == InstanceConfigPatch::default() => Ok(()), + Some(patch) => anyhow::bail!("runtime config still needs patch after reconcile: {patch:?}"), + None => anyhow::bail!("runtime config still needs full overwrite after reconcile"), + } +} + +async fn run_web_source_instance( + rpc_client: &mut SessionRpcClient, + inst_id: &str, + config: NetworkConfig, + overwrite: bool, +) -> anyhow::Result<()> { + rpc_client + .run_network_instance( + BaseController::default(), + RunNetworkInstanceRequest { + inst_id: Some(inst_id.to_string().into()), + config: Some(config), + overwrite, + source: RpcConfigSource::Web as i32, + }, + ) + .await?; + Ok(()) +} + +pub(super) async fn get_runtime_config( + rpc_client: &mut SessionRpcClient, + inst_id: &str, +) -> anyhow::Result { + rpc_client + .get_network_instance_config( + BaseController::default(), + GetNetworkInstanceConfigRequest { + inst_id: Some(inst_id.to_string().into()), + }, + ) + .await? + .config + .ok_or_else(|| anyhow::anyhow!("runtime returned empty config for {inst_id}")) +} + +pub(super) async fn prepare_web_source_runtime_reconcile( + rpc_client: &mut SessionRpcClient, + inst_id: &str, + desired_config: NetworkConfig, + is_running: bool, +) -> anyhow::Result { + if !is_running { + return Ok(RuntimeReconcileAction::Run { + config: Box::new(desired_config), + overwrite: false, + }); + } + + let current_config = get_runtime_config(rpc_client, inst_id).await?; + + prepare_web_source_runtime_reconcile_from_current(¤t_config, desired_config) +} + +pub(super) fn prepare_web_source_runtime_reconcile_from_current( + current_config: &NetworkConfig, + desired_config: NetworkConfig, +) -> anyhow::Result { + let Some(patch) = web_source_runtime_patch(current_config, &desired_config)? else { + return Ok(RuntimeReconcileAction::Run { + config: Box::new(desired_config), + overwrite: true, + }); + }; + if patch == InstanceConfigPatch::default() { + return Ok(RuntimeReconcileAction::None); + } + + Ok(RuntimeReconcileAction::Patch(Box::new(patch))) +} + +pub(super) async fn apply_web_source_runtime_reconcile( + rpc_client: &mut SessionRpcClient, + config_client: &mut SessionConfigClient, + inst_id: &str, + desired_config: NetworkConfig, + action: RuntimeReconcileAction, +) -> anyhow::Result { + match action { + RuntimeReconcileAction::None => Ok(desired_config), + RuntimeReconcileAction::Run { config, overwrite } => { + run_web_source_instance(rpc_client, inst_id, *config, overwrite).await?; + Ok(desired_config) + } + RuntimeReconcileAction::Patch(patch) => { + config_client + .patch_config( + BaseController::default(), + PatchConfigRequest { + instance: Some(instance_identifier(inst_id)?), + patch: Some(*patch), + }, + ) + .await?; + let current_config = get_runtime_config(rpc_client, inst_id).await?; + ensure_runtime_config_converged(¤t_config, &desired_config)?; + Ok(current_config) + } + } +} + +#[cfg(test)] +mod tests { + use easytier::proto::{ + api::{ + config::ConfigPatchAction, + manage::{NetworkingMethod, PortForwardConfig}, + }, + common::{CompressionAlgoPb, SocketType}, + }; + + use super::*; + + fn config_with_port_forwards(port_forwards: Vec) -> NetworkConfig { + NetworkConfig { + instance_id: Some("11111111-1111-1111-1111-111111111111".to_string()), + dhcp: Some(true), + network_name: Some("managed".to_string()), + network_secret: Some("secret".to_string()), + networking_method: Some(NetworkingMethod::Manual as i32), + port_forwards, + ..Default::default() + } + } + + fn port_forward(bind_port: u32, dst_port: u32) -> PortForwardConfig { + PortForwardConfig { + bind_ip: "127.0.0.1".to_string(), + bind_port, + dst_ip: "10.144.0.1".to_string(), + dst_port, + proto: "tcp".to_string(), + } + } + + fn patch_port(patch: &PortForwardPatch) -> (i32, u32, u32, i32) { + let cfg = patch.cfg.as_ref().expect("port forward patch cfg"); + ( + patch.action, + cfg.bind_addr.as_ref().expect("bind addr").port, + cfg.dst_addr.as_ref().expect("dst addr").port, + cfg.socket_type, + ) + } + + fn patch_proxy_network(patch: &ProxyNetworkPatch) -> (i32, String, Option) { + ( + patch.action, + patch.cidr.map(|cidr| cidr.to_string()).unwrap_or_default(), + patch.mapped_cidr.map(|cidr| cidr.to_string()), + ) + } + + #[test] + fn runtime_patch_ignores_runtime_defaults_and_adds_port_forward() { + let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]); + current.virtual_ipv4 = Some("10.144.0.2".to_string()); + current.network_length = Some(16); + current.bind_device = Some(true); + current.dev_name = Some(String::new()); + current.disable_ipv6 = Some(false); + current.mtu = Some(1380); + current.multi_thread = Some(true); + + let desired = + config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]); + + let patch = web_source_runtime_patch(¤t, &desired) + .expect("build patch") + .expect("hot patch"); + + assert_eq!(patch.port_forwards.len(), 1); + assert_eq!( + patch_port(&patch.port_forwards[0]), + ( + ConfigPatchAction::Add as i32, + 23007, + 3389, + SocketType::Tcp as i32 + ) + ); + } + + #[test] + fn runtime_patch_removes_deleted_port_forward_without_clear() { + let current = + config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]); + let desired = config_with_port_forwards(vec![port_forward(23000, 5174)]); + + let patch = web_source_runtime_patch(¤t, &desired) + .expect("build patch") + .expect("hot patch"); + + assert_eq!(patch.port_forwards.len(), 1); + assert_eq!( + patch_port(&patch.port_forwards[0]), + ( + ConfigPatchAction::Remove as i32, + 23007, + 3389, + SocketType::Tcp as i32 + ) + ); + } + + #[test] + fn runtime_patch_reconciles_duplicate_port_forward_count() { + let current = + config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23000, 5174)]); + let desired = config_with_port_forwards(vec![port_forward(23000, 5174)]); + + let patch = web_source_runtime_patch(¤t, &desired) + .expect("build patch") + .expect("hot patch"); + + assert_eq!(patch.port_forwards.len(), 2); + assert_eq!( + patch_port(&patch.port_forwards[0]), + ( + ConfigPatchAction::Remove as i32, + 23000, + 5174, + SocketType::Tcp as i32 + ) + ); + assert_eq!( + patch_port(&patch.port_forwards[1]), + ( + ConfigPatchAction::Add as i32, + 23000, + 5174, + SocketType::Tcp as i32 + ) + ); + } + + #[test] + fn runtime_convergence_rejects_stale_extra_port_forward() { + let current = config_with_port_forwards(vec![ + port_forward(23000, 5174), + port_forward(23007, 3389), + port_forward(23100, 8080), + ]); + let desired = + config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]); + + let err = ensure_runtime_config_converged(¤t, &desired) + .expect_err("extra runtime port forward should not converge"); + + assert!( + err.to_string() + .contains("runtime config still needs patch after reconcile"), + "unexpected error: {err:?}" + ); + } + + #[test] + fn runtime_patch_canonicalizes_port_forward_protocol() { + let current = config_with_port_forwards(vec![port_forward(23000, 5174)]); + let mut desired_port_forward = port_forward(23000, 5174); + desired_port_forward.proto = "TCP".to_string(); + let desired = config_with_port_forwards(vec![desired_port_forward]); + + let patch = web_source_runtime_patch(¤t, &desired) + .expect("build patch") + .expect("hot patch"); + + assert_eq!(patch, InstanceConfigPatch::default()); + ensure_runtime_config_converged(¤t, &desired).expect("runtime converged"); + } + + #[test] + fn runtime_patch_rejects_non_hot_config_change() { + let current = config_with_port_forwards(Vec::new()); + let mut desired = current.clone(); + desired.network_secret = Some("new-secret".to_string()); + + let patch = web_source_runtime_patch(¤t, &desired).expect("build patch"); + + assert!(patch.is_none()); + } + + #[test] + fn runtime_patch_rejects_routes_change() { + let mut current = config_with_port_forwards(Vec::new()); + current.enable_manual_routes = Some(true); + current.routes = vec!["10.1.0.0/16".to_string(), "10.2.0.0/16".to_string()]; + let mut desired = config_with_port_forwards(Vec::new()); + desired.enable_manual_routes = Some(true); + desired.routes = vec!["10.2.0.0/16".to_string(), "10.3.0.0/16".to_string()]; + + let patch = web_source_runtime_patch(¤t, &desired).expect("build patch"); + + assert!(patch.is_none()); + } + + #[test] + fn runtime_patch_replaces_proxy_networks() { + let mut current = config_with_port_forwards(Vec::new()); + current.proxy_cidrs = vec![ + "10.1.0.0/16".to_string(), + "10.2.0.0/16->10.20.0.0/16".to_string(), + ]; + let mut desired = config_with_port_forwards(Vec::new()); + desired.proxy_cidrs = vec![ + "10.2.0.0/16->10.21.0.0/16".to_string(), + "10.3.0.0/16".to_string(), + ]; + + let patch = web_source_runtime_patch(¤t, &desired) + .expect("build patch") + .expect("hot patch"); + + assert_eq!(patch.proxy_networks.len(), 3); + assert_eq!( + patch_proxy_network(&patch.proxy_networks[0]), + ( + ConfigPatchAction::Clear as i32, + "10.2.0.0/16".to_string(), + None + ) + ); + assert_eq!( + patch_proxy_network(&patch.proxy_networks[1]), + ( + ConfigPatchAction::Add as i32, + "10.2.0.0/16".to_string(), + Some("10.21.0.0/16".to_string()) + ) + ); + assert_eq!( + patch_proxy_network(&patch.proxy_networks[2]), + ( + ConfigPatchAction::Add as i32, + "10.3.0.0/16".to_string(), + None + ) + ); + } + + #[test] + fn runtime_patch_replaces_proxy_networks_with_same_source_cidr() { + let mut current = config_with_port_forwards(Vec::new()); + current.proxy_cidrs = vec![ + "10.1.2.0/24".to_string(), + "10.1.2.0/24->10.1.3.0/24".to_string(), + ]; + let mut desired = config_with_port_forwards(Vec::new()); + desired.proxy_cidrs = vec!["10.1.2.0/24->10.1.3.0/24".to_string()]; + + let patch = web_source_runtime_patch(¤t, &desired) + .expect("build patch") + .expect("hot patch"); + + assert_eq!(patch.proxy_networks.len(), 2); + assert_eq!( + patch_proxy_network(&patch.proxy_networks[0]), + ( + ConfigPatchAction::Clear as i32, + "10.1.2.0/24".to_string(), + None + ) + ); + assert_eq!( + patch_proxy_network(&patch.proxy_networks[1]), + ( + ConfigPatchAction::Add as i32, + "10.1.2.0/24".to_string(), + Some("10.1.3.0/24".to_string()) + ) + ); + } + + #[test] + fn runtime_patch_rejects_proxy_network_empty_to_nonempty() { + let current = config_with_port_forwards(Vec::new()); + let mut desired = config_with_port_forwards(Vec::new()); + desired.proxy_cidrs = vec!["10.1.2.0/24".to_string()]; + + let patch = web_source_runtime_patch(¤t, &desired).expect("build patch"); + + assert!(patch.is_none()); + } + + #[test] + fn runtime_patch_clears_proxy_networks_with_legacy_compatible_cidr() { + let mut current = config_with_port_forwards(Vec::new()); + current.proxy_cidrs = vec!["10.1.2.0/24".to_string()]; + let desired = config_with_port_forwards(Vec::new()); + + let patch = web_source_runtime_patch(¤t, &desired) + .expect("build patch") + .expect("hot patch"); + + assert_eq!(patch.proxy_networks.len(), 1); + assert_eq!( + patch_proxy_network(&patch.proxy_networks[0]), + ( + ConfigPatchAction::Clear as i32, + "10.1.2.0/24".to_string(), + None + ) + ); + } + + #[test] + fn runtime_patch_updates_disable_relay_data() { + let current = config_with_port_forwards(Vec::new()); + let mut desired = current.clone(); + desired.disable_relay_data = Some(true); + + let patch = web_source_runtime_patch(¤t, &desired) + .expect("build patch") + .expect("hot patch"); + + assert_eq!(patch.disable_relay_data, Some(true)); + } + + #[test] + fn runtime_patch_still_rejects_unsupported_flag_change() { + let current = config_with_port_forwards(Vec::new()); + let mut desired = current.clone(); + desired.no_tun = Some(true); + + let patch = web_source_runtime_patch(¤t, &desired).expect("build patch"); + + assert!(patch.is_none()); + } + + #[test] + fn runtime_patch_rejects_encryption_algorithm_change() { + let current = config_with_port_forwards(vec![port_forward(23000, 5174)]); + let mut desired = current.clone(); + desired.encryption_algorithm = Some("managed-test-algo".to_string()); + + let patch = web_source_runtime_patch(¤t, &desired).expect("build patch"); + + assert!(patch.is_none()); + } + + #[test] + fn runtime_patch_rejects_data_compress_algo_change() { + let current = config_with_port_forwards(vec![port_forward(23000, 5174)]); + let mut desired = current.clone(); + desired.data_compress_algo = Some(CompressionAlgoPb::Zstd as i32); + + let patch = web_source_runtime_patch(¤t, &desired).expect("build patch"); + + assert!(patch.is_none()); + } + + #[test] + fn runtime_patch_rejects_credential_private_key_change() { + let mut current = config_with_port_forwards(Vec::new()); + current.network_secret = None; + current.secure_mode = Some(easytier::proto::common::SecureModeConfig { + enabled: true, + local_private_key: Some("mUuD5fsIm/ftvgS4WBAYFMNLqWX3qT9rnm4PrnOqb9s=".to_string()), + local_public_key: None, + }); + let mut desired = current.clone(); + desired.secure_mode = Some(easytier::proto::common::SecureModeConfig { + enabled: true, + local_private_key: Some("aEpz80FuYbaY4QLJizAIuIcK4TYsoSA9jHHCXCOQJoc=".to_string()), + local_public_key: None, + }); + + let patch = web_source_runtime_patch(¤t, &desired).expect("build patch"); + + assert!(patch.is_none()); + } + + #[test] + fn runtime_patch_ignores_generated_secure_key_when_network_secret_exists() { + let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]); + current.secure_mode = Some(easytier::proto::common::SecureModeConfig { + enabled: true, + local_private_key: Some("mUuD5fsIm/ftvgS4WBAYFMNLqWX3qT9rnm4PrnOqb9s=".to_string()), + local_public_key: Some("4x6L5dZjB8hsPO4f96Hyhi4xFealBu6i3BxRVBYR1Fc=".to_string()), + }); + let mut desired = + config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]); + desired.secure_mode = Some(easytier::proto::common::SecureModeConfig { + enabled: true, + local_private_key: None, + local_public_key: None, + }); + + let patch = web_source_runtime_patch(¤t, &desired) + .expect("build patch") + .expect("hot patch"); + + assert_eq!(patch.port_forwards.len(), 1); + assert_eq!( + patch_port(&patch.port_forwards[0]), + ( + ConfigPatchAction::Add as i32, + 23007, + 3389, + SocketType::Tcp as i32 + ) + ); + } + + #[test] + fn runtime_patch_ignores_runtime_hostname_when_desired_omits_hostname() { + let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]); + current.hostname = Some("runtime-host".to_string()); + let desired = + config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]); + + let patch = web_source_runtime_patch(¤t, &desired) + .expect("build patch") + .expect("hot patch"); + + assert_eq!(patch.port_forwards.len(), 1); + assert_eq!( + patch_port(&patch.port_forwards[0]), + ( + ConfigPatchAction::Add as i32, + 23007, + 3389, + SocketType::Tcp as i32 + ) + ); + } + + #[test] + fn runtime_patch_rejects_explicit_desired_hostname_change() { + let current = config_with_port_forwards(vec![port_forward(23000, 5174)]); + let mut desired = + config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]); + desired.hostname = + Some(easytier::common::config::TomlConfigLoader::default().get_hostname()); + + let patch = web_source_runtime_patch(¤t, &desired).expect("build patch"); + + assert!(patch.is_none()); + } +} diff --git a/easytier-web/src/client_manager/session.rs b/easytier-web/src/client_manager/session.rs index 9636f16e..88075fcd 100644 --- a/easytier-web/src/client_manager/session.rs +++ b/easytier-web/src/client_manager/session.rs @@ -1,72 +1,34 @@ use std::{ - collections::{HashMap, HashSet}, fmt::Debug, str::FromStr as _, sync::Arc, + time::{Duration, Instant}, }; use anyhow::Context; use easytier::{ - common::config::ConfigSource, proto::{ - api::manage::{ - ConfigSource as RpcConfigSource, DeleteNetworkInstanceRequest, - ListNetworkInstanceMetaRequest, NetworkConfig, NetworkMeta, RunNetworkInstanceRequest, - WebClientService, WebClientServiceClientFactory, + api::{ + config::{ConfigRpc, ConfigRpcClientFactory}, + manage::{WebClientService, WebClientServiceClientFactory}, }, - common::Uuid as RpcUuid, rpc_impl::bidirect::BidirectRpcManager, rpc_types::{self, controller::BaseController}, web::{HeartbeatRequest, HeartbeatResponse, WebServerService, WebServerServiceServer}, }, - rpc_service::remote_client::{ListNetworkProps, PersistentConfig as _, Storage as _}, tunnel::Tunnel, }; -use tokio::sync::{RwLock, broadcast}; +use tokio::sync::{Notify, RwLock, broadcast}; use tokio_util::task::AbortOnDropHandle; use super::storage::{Storage, StorageToken, WeakRefStorage}; use crate::FeatureFlags; use crate::webhook::SharedWebhookConfig; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum PersistedConfigSource { - User, - Web, -} +mod runtime_revision; +mod webhook_validation; -impl PersistedConfigSource { - fn from_db(source: &str) -> Self { - match source { - "web" => Self::Web, - "user" => Self::User, - _ => Self::User, - } - } - - fn should_update_from_runtime(self, runtime_source: ConfigSource) -> bool { - match (self, runtime_source) { - // Older clients report missing source as `user`, which is not authoritative enough - // to downgrade an existing web-owned row. - (Self::Web, ConfigSource::User) => false, - _ => self.as_runtime_source() != runtime_source, - } - } - - fn as_runtime_source(self) -> ConfigSource { - match self { - Self::User => ConfigSource::User, - Self::Web => ConfigSource::Web, - } - } - - fn auto_run_rpc_source(self) -> RpcConfigSource { - match self { - Self::User => RpcConfigSource::User, - Self::Web => RpcConfigSource::Web, - } - } -} +const WEBHOOK_VALIDATION_HEARTBEAT_INTERVAL: u32 = 10; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct Location { @@ -75,6 +37,19 @@ pub struct Location { pub region: Option, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SessionAuthState { + Init, + Authorized, + Invalid, +} + +impl SessionAuthState { + fn is_authorized(self) -> bool { + matches!(self, Self::Authorized) + } +} + #[derive(Debug)] pub struct SessionData { storage: WeakRefStorage, @@ -89,6 +64,11 @@ pub struct SessionData { req: Option, location: Option, heartbeat_count: std::sync::atomic::AtomicU32, + session_identity: Option, + auth_state: SessionAuthState, + webhook_connected_binding_version: Option, + webhook_validation_dirty: bool, + webhook_validation_notify: Arc, } impl SessionData { @@ -113,6 +93,11 @@ impl SessionData { req: None, location, heartbeat_count: std::sync::atomic::AtomicU32::new(0), + session_identity: None, + auth_state: SessionAuthState::Init, + webhook_connected_binding_version: None, + webhook_validation_dirty: false, + webhook_validation_notify: Arc::new(Notify::new()), } } @@ -129,6 +114,141 @@ impl SessionData { } } +async fn send_webhook_node_disconnected( + webhook: SharedWebhookConfig, + token: StorageToken, + binding_version: u64, +) { + let machine_id = token.machine_id.to_string(); + let user_id = Some(token.user_id); + let token_value = token.token.clone(); + let web_instance_id = webhook.web_instance_id.clone(); + webhook + .notify_node_disconnected(&crate::webhook::NodeDisconnectedRequest { + machine_id, + token: token_value, + user_id, + web_instance_id, + binding_version: Some(binding_version), + }) + .await; +} + +fn notify_webhook_node_disconnected( + webhook: SharedWebhookConfig, + token: StorageToken, + binding_version: u64, +) { + tokio::spawn(async move { + send_webhook_node_disconnected(webhook, token, binding_version).await; + }); +} + +struct WebhookDisconnectNotification { + webhook: SharedWebhookConfig, + storage_token: StorageToken, + binding_version: u64, +} + +struct WebhookConnectNotification { + webhook: SharedWebhookConfig, + storage_token: StorageToken, + binding_version: u64, + req: crate::webhook::NodeConnectedRequest, +} + +fn storage_tokens_match(left: &StorageToken, right: &StorageToken) -> bool { + left.token == right.token + && left.client_url == right.client_url + && left.machine_id == right.machine_id + && left.user_id == right.user_id +} + +fn connection_state_matches( + data: &SessionData, + storage_token: &StorageToken, + binding_version: u64, +) -> bool { + data.auth_state.is_authorized() + && data.binding_version == Some(binding_version) + && data + .storage_token + .as_ref() + .is_some_and(|current| storage_tokens_match(current, storage_token)) +} + +async fn connection_state_is_current( + session_data: &std::sync::Weak>, + storage_token: &StorageToken, + binding_version: u64, +) -> bool { + let Some(session_data) = session_data.upgrade() else { + return false; + }; + let data = session_data.read().await; + connection_state_matches(&data, storage_token, binding_version) +} + +async fn record_webhook_connected_binding_if_current( + session_data: &std::sync::Weak>, + storage_token: &StorageToken, + binding_version: u64, +) -> bool { + let Some(session_data) = session_data.upgrade() else { + return false; + }; + let mut data = session_data.write().await; + if !connection_state_matches(&data, storage_token, binding_version) { + return false; + } + data.webhook_connected_binding_version = Some(binding_version); + true +} + +async fn send_webhook_connection_transition( + session_data: std::sync::Weak>, + disconnect: Option, + connect: Option, +) { + if let Some(disconnect) = disconnect { + send_webhook_node_disconnected( + disconnect.webhook, + disconnect.storage_token, + disconnect.binding_version, + ) + .await; + } + + let Some(connect) = connect else { + return; + }; + if !connection_state_is_current( + &session_data, + &connect.storage_token, + connect.binding_version, + ) + .await + { + return; + } + + connect.webhook.notify_node_connected(&connect.req).await; + if !record_webhook_connected_binding_if_current( + &session_data, + &connect.storage_token, + connect.binding_version, + ) + .await + { + send_webhook_node_disconnected( + connect.webhook, + connect.storage_token, + connect.binding_version, + ) + .await; + } +} + impl Drop for SessionData { fn drop(&mut self) { if let Ok(storage) = Storage::try_from(self.storage.clone()) @@ -137,24 +257,14 @@ impl Drop for SessionData { storage.remove_client(token); // Notify the webhook receiver when a node disconnects. - if self.webhook_config.is_enabled() { - let webhook = self.webhook_config.clone(); - let machine_id = token.machine_id.to_string(); - let user_id = Some(token.user_id); - let token_value = token.token.clone(); - let web_instance_id = webhook.web_instance_id.clone(); - let binding_version = self.binding_version; - tokio::spawn(async move { - webhook - .notify_node_disconnected(&crate::webhook::NodeDisconnectedRequest { - machine_id, - token: token_value, - user_id, - web_instance_id, - binding_version, - }) - .await; - }); + if self.webhook_config.is_enabled() + && let Some(binding_version) = self.webhook_connected_binding_version + { + notify_webhook_node_disconnected( + self.webhook_config.clone(), + token.clone(), + binding_version, + ); } } } @@ -165,136 +275,180 @@ pub type SharedSessionData = Arc>; #[derive(Clone)] pub(super) struct SessionRpcService { data: SharedSessionData, + heartbeat_min_response_delay: Duration, +} + +fn heartbeat_response_delay(elapsed: Duration, min_response_delay: Duration) -> Option { + min_response_delay + .checked_sub(elapsed) + .filter(|delay| !delay.is_zero()) +} + +fn should_delay_heartbeat_response(is_paced_session: bool, is_first_heartbeat: bool) -> bool { + is_paced_session && !is_first_heartbeat +} + +fn should_delay_session_heartbeat_response(data: &SessionData) -> bool { + should_delay_heartbeat_response( + data.webhook_config.is_enabled() || data.auth_state.is_authorized(), + data.req.is_none(), + ) +} + +fn should_notify_webhook_validation(heartbeat_count: u32) -> bool { + heartbeat_count % WEBHOOK_VALIDATION_HEARTBEAT_INTERVAL == 1 +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct HeartbeatIdentity { + token: String, + machine_id: uuid::Uuid, +} + +impl HeartbeatIdentity { + fn new(token: String, machine_id: uuid::Uuid) -> Self { + Self { token, machine_id } + } } impl SessionRpcService { - fn normalize_network_config( - mut network_config: serde_json::Value, - inst_id: uuid::Uuid, - ) -> anyhow::Result { - let network_name = network_config - .get("network_name") - .and_then(|v| v.as_str()) - .filter(|v| !v.is_empty()) - .ok_or_else(|| anyhow::anyhow!("webhook response missing network_name"))? - .to_string(); - let config_obj = network_config - .as_object_mut() - .ok_or_else(|| anyhow::anyhow!("webhook network_config must be a JSON object"))?; - config_obj.insert( - "instance_id".to_string(), - serde_json::Value::String(inst_id.to_string()), - ); - config_obj - .entry("instance_name".to_string()) - .or_insert_with(|| serde_json::Value::String(network_name)); - - Ok(serde_json::from_value::(network_config)?) + fn heartbeat_report_timestamp(req: &HeartbeatRequest) -> i64 { + match chrono::DateTime::::from_str(&req.report_time) { + Ok(report_time) => report_time.timestamp(), + Err(error) => { + tracing::warn!( + report_time = %req.report_time, + %error, + "invalid heartbeat report time, using server time" + ); + chrono::Local::now().timestamp() + } + } } - pub(super) async fn reconcile_web_source_configs( - storage: &Storage, - user_id: i32, + fn store_latest_heartbeat_req( + data: &mut SessionData, + req: HeartbeatRequest, + ) -> HeartbeatRequest { + data.req = Some(req); + data.req + .clone() + .expect("heartbeat request should be initialized") + } + + fn storage_token_matches_heartbeat( + storage_token: &StorageToken, + req: &HeartbeatRequest, + ) -> bool { + req.user_token == storage_token.token + && req.machine_id.map(uuid::Uuid::from) == Some(storage_token.machine_id) + } + + async fn runtime_heartbeat_is_current( + session_data: &std::sync::Weak>, + req: &HeartbeatRequest, + ) -> bool { + let Some(session_data) = session_data.upgrade() else { + return false; + }; + let data = session_data.read().await; + Self::runtime_heartbeat_is_current_locked(&data, req) + } + + fn runtime_heartbeat_is_current_locked(data: &SessionData, req: &HeartbeatRequest) -> bool { + data.storage_token.as_ref().is_some_and(|storage_token| { + Self::storage_token_matches_heartbeat(storage_token, req) + && data.req.as_ref().is_some_and(|current_req| { + Self::storage_token_matches_heartbeat(storage_token, current_req) + }) + && data.auth_state.is_authorized() + }) + } + + fn heartbeat_matches_identity( + req: &HeartbeatRequest, + token: &str, + machine_id: uuid::Uuid, + ) -> bool { + req.user_token == token && req.machine_id.map(uuid::Uuid::from) == Some(machine_id) + } + + fn heartbeat_identity(req: &HeartbeatRequest, machine_id: uuid::Uuid) -> HeartbeatIdentity { + HeartbeatIdentity::new(req.user_token.clone(), machine_id) + } + + fn ensure_session_identity_locked( + data: &mut SessionData, + req: &HeartbeatRequest, machine_id: uuid::Uuid, - desired_configs: Vec, ) -> anyhow::Result<()> { - let existing_configs = storage - .db() - .list_network_configs((user_id, machine_id), ListNetworkProps::All) - .await - .map_err(|e| anyhow::anyhow!("failed to list existing network configs: {:?}", e))?; - let existing_sources = existing_configs - .iter() - .filter_map(|cfg| { - uuid::Uuid::parse_str(&cfg.network_instance_id) - .ok() - .map(|inst_id| (inst_id, PersistedConfigSource::from_db(&cfg.source))) - }) - .collect::>(); - let existing_web_ids = existing_sources - .iter() - .filter_map(|(inst_id, source)| { - (*source == PersistedConfigSource::Web).then_some(*inst_id) - }) - .collect::>(); - - let mut desired_ids = HashSet::with_capacity(desired_configs.len()); - let mut normalized = HashMap::with_capacity(desired_configs.len()); - for desired in desired_configs { - let inst_id = uuid::Uuid::parse_str(&desired.instance_id).with_context(|| { - format!( - "invalid desired web config instance id: {}", - desired.instance_id - ) - })?; - if let Some(PersistedConfigSource::User) = existing_sources.get(&inst_id) { - tracing::warn!( - ?user_id, - ?machine_id, - instance_id = %inst_id, - "skip web config because a user-owned config already exists" + let identity = Self::heartbeat_identity(req, machine_id); + match data.session_identity.as_ref() { + Some(existing) if existing != &identity => { + anyhow::bail!( + "Heartbeat identity does not match session token, machine_id: {:?}", + machine_id ); - continue; } - let config = Self::normalize_network_config(desired.network_config, inst_id)?; - desired_ids.insert(inst_id); - normalized.insert(inst_id, config); + Some(_) => {} + None => data.session_identity = Some(identity), } - - for (inst_id, config) in normalized { - storage - .db() - .insert_or_update_user_network_config( - (user_id, machine_id), - inst_id, - config, - ConfigSource::Web, - ) - .await - .map_err(|e| { - anyhow::anyhow!("failed to persist web network config {}: {:?}", inst_id, e) - })?; - } - - let stale_ids = existing_web_ids - .difference(&desired_ids) - .copied() - .collect::>(); - if !stale_ids.is_empty() { - storage - .db() - .delete_network_configs((user_id, machine_id), &stale_ids) - .await - .map_err(|e| anyhow::anyhow!("failed to delete stale network configs: {:?}", e))?; - } - Ok(()) } - fn managed_configs_for_revision( - applied_config_revision: Option<&str>, - resp: crate::webhook::ValidateTokenResponse, - ) -> anyhow::Result<(Vec, String)> { - let config_revision = resp.config_revision; - let managed_configs = match resp.managed_network_configs { - Some(configs) => configs, - None if applied_config_revision == Some(config_revision.as_str()) => Vec::new(), - None => { - anyhow::bail!( - "Webhook token validation response omitted managed configs for changed revision {:?}", - config_revision + fn mark_webhook_validation_dirty_locked(data: &mut SessionData) -> Arc { + data.webhook_validation_dirty = true; + data.webhook_validation_notify.clone() + } + + async fn handle_webhook_heartbeat( + &self, + storage: &Storage, + req: HeartbeatRequest, + machine_id: uuid::Uuid, + ) -> rpc_types::error::Result { + let (notify, runtime_notify) = { + let mut data = self.data.write().await; + Self::ensure_session_identity_locked(&mut data, &req, machine_id) + .map_err(rpc_types::error::Error::from)?; + if matches!(data.auth_state, SessionAuthState::Invalid) { + tracing::info!( + %machine_id, + "webhook session is invalid; failing heartbeat to require client reconnect" ); + return Err(anyhow::anyhow!("webhook session is invalid").into()); } + let runtime_req = Self::store_latest_heartbeat_req(&mut data, req); + let heartbeat_count = data + .heartbeat_count + .fetch_add(1, std::sync::atomic::Ordering::Relaxed) + + 1; + let notify = should_notify_webhook_validation(heartbeat_count) + .then(|| Self::mark_webhook_validation_dirty_locked(&mut data)); + let authorized = data.auth_state.is_authorized(); + if let Some(storage_token) = data.storage_token.clone() { + let report_time = Self::heartbeat_report_timestamp(&runtime_req); + storage.update_client(storage_token, report_time, authorized); + } + let runtime_notify = (authorized && data.storage_token.is_some()) + .then(|| (data.notifier.clone(), runtime_req)); + (notify, runtime_notify) }; - Ok((managed_configs, config_revision)) + if let Some((notifier, runtime_req)) = runtime_notify { + let _ = notifier.send(runtime_req); + } + if let Some(notify) = notify { + notify.notify_one(); + } + Ok(HeartbeatResponse {}) } async fn handle_heartbeat( &self, req: HeartbeatRequest, ) -> rpc_types::error::Result { - let (storage, feature_flags, webhook_config, client_url, applied_config_revision) = { + let (storage, feature_flags, webhook_config) = { let data = self.data.read().await; let Ok(storage) = Storage::try_from(data.storage.clone()) else { tracing::error!("Failed to get storage"); @@ -304,8 +458,6 @@ impl SessionRpcService { storage, data.feature_flags.clone(), data.webhook_config.clone(), - data.client_url.clone(), - data.applied_config_revision.clone(), ) }; @@ -314,193 +466,67 @@ impl SessionRpcService { req.machine_id ))?; - // First heartbeat must validate token through webhook; - // afterwards only every 10th heartbeat calls the webhook. - let (should_call_webhook, cached_storage_token) = { - let data = self.data.read().await; - let count = data - .heartbeat_count - .fetch_add(1, std::sync::atomic::Ordering::Relaxed) - + 1; - let is_first = data.req.is_none(); - let should_call = webhook_config.is_enabled() && (is_first || count % 10 == 1); - (should_call, data.storage_token.clone()) - }; - - let ( - user_id, - webhook_source_configs, - webhook_config_revision, - webhook_validated, - binding_version, - ) = if webhook_config.is_enabled() { - if should_call_webhook { - let webhook_req = crate::webhook::ValidateTokenRequest { - token: req.user_token.clone(), - machine_id: machine_id.to_string(), - public_ip: client_url.host_str().map(str::to_string), - hostname: req.hostname.clone(), - version: req.easytier_version.clone(), - os_type: req.device_os.as_ref().map(|info| info.os_type.clone()), - os_version: req.device_os.as_ref().map(|info| info.version.clone()), - os_distribution: req.device_os.as_ref().map(|info| info.distribution.clone()), - web_instance_id: webhook_config.web_instance_id.clone(), - web_instance_api_base_url: webhook_config.web_instance_api_base_url.clone(), - applied_config_revision: applied_config_revision.clone(), - }; - let resp = webhook_config - .validate_token(&webhook_req) - .await - .map_err(|e| anyhow::anyhow!("Webhook token validation failed: {:?}", e))?; - - if resp.valid { - let user_id = match storage - .db() - .get_user_id_by_token(req.user_token.clone()) - .await - .map_err(|e| anyhow::anyhow!("DB error: {:?}", e))? - { - Some(id) => id, - None => storage - .auto_create_user(&req.user_token) - .await - .with_context(|| { - format!("Failed to auto-create webhook user: {:?}", req.user_token) - })?, - }; - let binding_version = resp.binding_version; - let (webhook_source_configs, webhook_config_revision) = - Self::managed_configs_for_revision( - applied_config_revision.as_deref(), - resp, - ) - .map_err(rpc_types::error::Error::from)?; - ( - user_id, - webhook_source_configs, - webhook_config_revision, - true, - Some(binding_version), - ) - } else { - return Err(anyhow::anyhow!( - "Webhook rejected token for machine {:?}: {:?}", - machine_id, - req.user_token - ) - .into()); - } - } else { - let user_id = cached_storage_token - .as_ref() - .ok_or_else(|| { - anyhow::anyhow!("Storage token not found for non-first heartbeat") - })? - .user_id; - let binding_version = { - let data = self.data.read().await; - data.binding_version - }; - (user_id, Vec::new(), String::new(), false, binding_version) - } - } else { - let user_id = match storage - .db() - .get_user_id_by_token(req.user_token.clone()) - .await - .with_context(|| { - format!( - "Failed to get user id by token from db: {:?}", - req.user_token - ) - })? { - Some(id) => id, - None if feature_flags.allow_auto_create_user => storage - .auto_create_user(&req.user_token) - .await - .with_context(|| format!("Failed to auto-create user: {:?}", req.user_token))?, - None => { - return Err( - anyhow::anyhow!("User not found by token: {:?}", req.user_token).into(), - ); - } - }; - (user_id, Vec::new(), String::new(), false, None) - }; - - let should_reconcile = webhook_validated - && applied_config_revision.as_deref() != Some(webhook_config_revision.as_str()); - if should_reconcile { - Self::reconcile_web_source_configs( - &storage, - user_id, - machine_id, - webhook_source_configs, - ) - .await - .map_err(rpc_types::error::Error::from)?; + if webhook_config.is_enabled() { + return self + .handle_webhook_heartbeat(&storage, req, machine_id) + .await; } - let mut connect_notification = None; - let (storage_token, notifier) = { + { let mut data = self.data.write().await; + Self::ensure_session_identity_locked(&mut data, &req, machine_id) + .map_err(rpc_types::error::Error::from)?; + } - if should_reconcile { - data.applied_config_revision = Some(webhook_config_revision); + let user_id = match storage + .db() + .get_user_id_by_token(req.user_token.clone()) + .await + .with_context(|| { + format!( + "Failed to get user id by token from db: {:?}", + req.user_token + ) + })? { + Some(id) => id, + None if feature_flags.allow_auto_create_user => storage + .auto_create_user(&req.user_token) + .await + .with_context(|| format!("Failed to auto-create user: {:?}", req.user_token))?, + None => { + return Err( + anyhow::anyhow!("User not found by token: {:?}", req.user_token).into(), + ); } + }; - if data.req.replace(req.clone()).is_none() { + let (storage_token, notifier, runtime_req) = { + let mut data = self.data.write().await; + let is_new_storage_token = data.storage_token.is_none(); + let runtime_req = Self::store_latest_heartbeat_req(&mut data, req.clone()); + data.heartbeat_count + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if is_new_storage_token { assert!(data.storage_token.is_none()); data.storage_token = Some(StorageToken { - token: req.user_token.clone(), + token: runtime_req.user_token.clone(), client_url: data.client_url.clone(), machine_id, user_id, }); - data.binding_version = binding_version; - - if data.webhook_config.is_enabled() { - connect_notification = Some(( - data.webhook_config.clone(), - crate::webhook::NodeConnectedRequest { - machine_id: machine_id.to_string(), - token: req.user_token.clone(), - user_id: Some(user_id), - hostname: req.hostname.clone(), - version: req.easytier_version.clone(), - os_type: req.device_os.as_ref().map(|info| info.os_type.clone()), - os_version: req.device_os.as_ref().map(|info| info.version.clone()), - os_distribution: req - .device_os - .as_ref() - .map(|info| info.distribution.clone()), - web_instance_id: data.webhook_config.web_instance_id.clone(), - binding_version, - }, - )); - } } + data.auth_state = SessionAuthState::Authorized; let Some(storage_token) = data.storage_token.as_ref().cloned() else { tracing::error!("Heartbeat succeeded before session token was initialized"); return Ok(HeartbeatResponse {}); }; - (storage_token, data.notifier.clone()) + (storage_token, data.notifier.clone(), runtime_req) }; - if let Some((webhook, connect_req)) = connect_notification { - tokio::spawn(async move { - webhook.notify_node_connected(&connect_req).await; - }); - } - - let Ok(report_time) = chrono::DateTime::::from_str(&req.report_time) else { - tracing::error!("Failed to parse report time: {:?}", req.report_time); - return Ok(HeartbeatResponse {}); - }; - storage.update_client(storage_token, report_time.timestamp()); - - let _ = notifier.send(req); + let report_time = Self::heartbeat_report_timestamp(&runtime_req); + storage.update_client(storage_token, report_time, true); + let _ = notifier.send(runtime_req); Ok(HeartbeatResponse {}) } } @@ -514,11 +540,21 @@ impl WebServerService for SessionRpcService { _: BaseController, req: HeartbeatRequest, ) -> rpc_types::error::Result { + let started_at = Instant::now(); + let should_delay_response = { + let data = self.data.read().await; + should_delay_session_heartbeat_response(&data) + }; let ret = self.handle_heartbeat(req).await; if ret.is_err() { tracing::warn!("Failed to handle heartbeat: {:?}", ret); // sleep for a while to avoid client busy loop tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } else if should_delay_response + && let Some(delay) = + heartbeat_response_delay(started_at.elapsed(), self.heartbeat_min_response_delay) + { + tokio::time::sleep(delay).await; } ret } @@ -539,6 +575,7 @@ pub struct Session { data: SharedSessionData, + webhook_validation_task: Option>, config_reconcile_task: Option>, } @@ -548,13 +585,15 @@ impl Debug for Session { } } -type SessionRpcClient = Box + Send>; +pub(super) type SessionRpcClient = Box + Send>; +pub(super) type SessionConfigClient = Box + Send>; impl Session { pub fn new( storage: WeakRefStorage, client_url: url::Url, location: Option, + heartbeat_min_response_delay: Duration, feature_flags: Arc, webhook_config: SharedWebhookConfig, ) -> Self { @@ -566,13 +605,17 @@ impl Session { BidirectRpcManager::new().set_rx_timeout(Some(std::time::Duration::from_secs(30))); rpc_mgr.rpc_server().registry().register( - WebServerServiceServer::new(SessionRpcService { data: data.clone() }), + WebServerServiceServer::new(SessionRpcService { + data: data.clone(), + heartbeat_min_response_delay, + }), "", ); Session { rpc_mgr, data, + webhook_validation_task: None, config_reconcile_task: None, } } @@ -581,362 +624,24 @@ impl Session { self.rpc_mgr.run_with_tunnel(tunnel); let data = self.data.read().await; + if data.webhook_config.is_enabled() { + self.webhook_validation_task + .replace(AbortOnDropHandle::new(tokio::spawn( + webhook_validation::run_worker(Arc::downgrade(&self.data)), + ))); + } self.config_reconcile_task .replace(AbortOnDropHandle::new(tokio::spawn( - Self::reconcile_network_configs_on_heartbeat( + runtime_revision::reconcile_network_configs_on_heartbeat( + Arc::downgrade(&self.data), data.heartbeat_waiter(), data.storage.clone(), self.scoped_rpc_client(), + self.scoped_config_client(), ), ))); } - fn collect_web_source_instance_ids(metas: &[NetworkMeta]) -> HashSet { - metas - .iter() - .filter_map(|meta| { - (RpcConfigSource::try_from(meta.source).ok() == Some(RpcConfigSource::Web)) - .then(|| { - meta.inst_id - .as_ref() - .map(|inst_id| Into::::into(*inst_id).to_string()) - }) - .flatten() - }) - .collect() - } - - fn desired_web_source_instance_ids( - local_configs: &[crate::db::entity::user_running_network_configs::Model], - ) -> HashSet { - local_configs - .iter() - .filter(|cfg| cfg.get_runtime_network_config_source() == ConfigSource::Web) - .map(|cfg| cfg.network_instance_id.clone()) - .collect() - } - - fn running_web_source_instance_ids( - running_inst_ids: &HashSet, - db_web_inst_ids: &HashSet, - running_metas: Option<&[NetworkMeta]>, - ) -> HashSet { - match running_metas { - Some(metas) => Self::collect_web_source_instance_ids(metas), - None => running_inst_ids - .intersection(db_web_inst_ids) - .cloned() - .collect(), - } - } - - fn parse_instance_ids(instance_ids: impl Iterator) -> Vec { - instance_ids - .filter_map(|inst_id| uuid::Uuid::parse_str(&inst_id).ok()) - .map(Into::into) - .collect() - } - - async fn sync_running_config_sources( - db: &crate::db::Db, - user_id: i32, - machine_id: uuid::Uuid, - local_configs: &[crate::db::entity::user_running_network_configs::Model], - metas: &[NetworkMeta], - ) -> anyhow::Result<()> { - let local_configs_by_id = local_configs - .iter() - .map(|cfg| (cfg.network_instance_id.clone(), cfg)) - .collect::>(); - - for meta in metas { - let Some(inst_id) = meta.inst_id.as_ref().map(|inst_id| { - let inst_id: uuid::Uuid = (*inst_id).into(); - inst_id - }) else { - continue; - }; - let inst_id_str = inst_id.to_string(); - let Some(local_cfg) = local_configs_by_id.get(&inst_id_str) else { - continue; - }; - - let Some(running_source) = ConfigSource::from_rpc(meta.source) else { - continue; - }; - let local_source = PersistedConfigSource::from_db(&local_cfg.source); - if !local_source.should_update_from_runtime(running_source) { - continue; - } - - db.insert_or_update_user_network_config( - (user_id, machine_id), - inst_id, - local_cfg.get_network_config().map_err(|e| { - anyhow::anyhow!("failed to decode local network config {}: {:?}", inst_id, e) - })?, - running_source, - ) - .await - .map_err(|e| { - anyhow::anyhow!( - "failed to sync running network config source {}: {:?}", - inst_id, - e - ) - })?; - } - - Ok(()) - } - - async fn reconcile_network_configs_on_heartbeat( - mut heartbeat_waiter: broadcast::Receiver, - storage: WeakRefStorage, - rpc_client: SessionRpcClient, - ) { - // This is a per-session background task. It starts when the RPC session is - // created, then reconciles after each heartbeat reports the client's runtime - // instances. It is deliberately best-effort: a failed round is retried by a - // later heartbeat instead of blocking heartbeat handling itself. - let mut cleaned_web_source_instances = false; - // This is only an in-memory guard for RPC cleanup, not a second source of - // truth. The DB still owns desired state; the cache lets us avoid listing - // and deleting runtime instances on every heartbeat when desired web-owned - // configs have not changed. - let mut last_desired_web_inst_ids: Option> = None; - loop { - // Drop any heartbeat backlog accumulated while the previous reconcile - // round was doing DB/RPC IO. The newest heartbeat has the freshest - // runtime instance list, which is all this task needs. - heartbeat_waiter = heartbeat_waiter.resubscribe(); - let req = heartbeat_waiter.recv().await; - if req.is_err() { - tracing::error!( - "Failed to receive heartbeat request, error: {:?}", - req.err() - ); - return; - } - - let req = req.unwrap(); - let Some(machine_id) = req.machine_id else { - tracing::warn!(?req, "Machine id is not set, ignore"); - continue; - }; - - let running_inst_ids = req - .running_network_instances - .iter() - .map(|x| x.to_string()) - .collect::>(); - let Some(storage) = storage.upgrade() else { - tracing::error!("Failed to get storage"); - return; - }; - - let user_id = match storage - .db - .get_user_id_by_token(req.user_token.clone()) - .await - { - Ok(Some(user_id)) => user_id, - Ok(None) => { - tracing::info!("User not found by token: {:?}", req.user_token); - return; - } - Err(e) => { - tracing::error!("Failed to get user id by token, error: {:?}", e); - return; - } - }; - - let local_configs = match storage - .db - .list_network_configs((user_id, machine_id.into()), ListNetworkProps::EnabledOnly) - .await - { - Ok(configs) => configs, - Err(e) => { - tracing::error!("Failed to list network configs, error: {:?}", e); - return; - } - }; - - let mut local_configs = local_configs; - let running_metas = if req.support_config_source { - let ret = if running_inst_ids.is_empty() { - Ok(Vec::new()) - } else { - rpc_client - .list_network_instance_meta( - BaseController::default(), - ListNetworkInstanceMetaRequest { - inst_ids: Self::parse_instance_ids( - running_inst_ids.iter().cloned(), - ), - }, - ) - .await - .map(|resp| resp.metas) - }; - - match ret { - Ok(metas) => { - if let Err(e) = Self::sync_running_config_sources( - &storage.db, - user_id, - machine_id.into(), - &local_configs, - &metas, - ) - .await - { - tracing::warn!( - ?user_id, - ?machine_id, - %e, - "Failed to sync running network config sources" - ); - } else if !metas.is_empty() { - local_configs = match storage - .db - .list_network_configs( - (user_id, machine_id.into()), - ListNetworkProps::EnabledOnly, - ) - .await - { - Ok(configs) => configs, - Err(e) => { - tracing::error!( - "Failed to reload network configs after source sync, error: {:?}", - e - ); - return; - } - }; - } - Some(metas) - } - Err(e) => { - tracing::warn!( - ?user_id, - %e, - "Failed to list running network instance metadata" - ); - None - } - } - } else { - None - }; - - let should_be_alive_web_inst_ids = - Self::desired_web_source_instance_ids(&local_configs); - let desired_changed = last_desired_web_inst_ids - .as_ref() - .is_none_or(|last| last != &should_be_alive_web_inst_ids); - - let mut has_failed = false; - if !cleaned_web_source_instances || desired_changed { - let db_web_inst_ids = match storage - .db - .list_network_configs((user_id, machine_id.into()), ListNetworkProps::All) - .await - { - Ok(configs) => Self::desired_web_source_instance_ids(&configs), - Err(e) => { - tracing::error!("Failed to list all network configs, error: {:?}", e); - return; - } - }; - - let running_web_inst_ids = Self::running_web_source_instance_ids( - &running_inst_ids, - &db_web_inst_ids, - running_metas.as_deref(), - ); - - let should_delete_ids = Self::parse_instance_ids( - running_web_inst_ids - .difference(&should_be_alive_web_inst_ids) - .cloned(), - ); - - if !should_delete_ids.is_empty() { - let ret = rpc_client - .delete_network_instance( - BaseController::default(), - DeleteNetworkInstanceRequest { - inst_ids: should_delete_ids, - }, - ) - .await; - tracing::info!( - ?user_id, - "Clean stale web-source network instances on heartbeat: {:?}, user_token: {:?}", - ret, - req.user_token - ); - has_failed |= ret.is_err(); - } - - if !has_failed { - cleaned_web_source_instances = true; - last_desired_web_inst_ids = Some(should_be_alive_web_inst_ids.clone()); - } - } - - // After stale web-owned instances are removed, start every enabled - // config that the latest heartbeat did not report as running. - for c in local_configs { - if running_inst_ids.contains(&c.network_instance_id) { - continue; - } - let source = PersistedConfigSource::from_db(&c.source).auto_run_rpc_source(); - let network_config = match serde_json::from_str::(&c.network_config) - { - Ok(cfg) => cfg, - Err(e) => { - tracing::error!( - ?user_id, - ?machine_id, - instance_id = %c.network_instance_id, - "Failed to deserialize network config, skipping: {:?}", - e - ); - has_failed = true; - continue; - } - }; - let ret = rpc_client - .run_network_instance( - BaseController::default(), - RunNetworkInstanceRequest { - inst_id: Some(c.network_instance_id.clone().into()), - config: Some(network_config), - overwrite: false, - source: source as i32, - }, - ) - .await; - tracing::info!( - ?user_id, - "Run network instance: {:?}, user_token: {:?}", - ret, - req.user_token - ); - - has_failed |= ret.is_err(); - } - - if !has_failed { - last_desired_web_inst_ids = Some(should_be_alive_web_inst_ids); - } - } - } - pub fn is_running(&self) -> bool { self.rpc_mgr.is_running() } @@ -968,6 +673,38 @@ impl Session { self.scoped_client::>() } + pub fn scoped_config_client(&self) -> SessionConfigClient { + self.scoped_client::>() + } + + pub async fn notify_config_revision_changed( + &self, + user_id: i32, + machine_id: uuid::Uuid, + config_revision: String, + ) { + let notify = { + let data = self.data.read().await; + if !data.auth_state.is_authorized() { + return; + } + if !data + .storage_token + .as_ref() + .is_some_and(|token| token.user_id == user_id && token.machine_id == machine_id) + { + return; + } + if data.applied_config_revision.as_deref() == Some(config_revision.as_str()) { + return; + } + data.req.clone().map(|req| (data.notifier.clone(), req)) + }; + if let Some((notifier, req)) = notify { + let _ = notifier.send(req); + } + } + pub async fn get_token(&self) -> Option { self.data.read().await.storage_token.clone() } @@ -979,180 +716,737 @@ impl Session { #[cfg(test)] mod tests { - use easytier::{ - common::config::ConfigSource, - rpc_service::remote_client::{ListNetworkProps, PersistentConfig as _, Storage as _}, - }; + use axum::{Json, Router, extract::State, routing::post}; use serde_json::json; + use tokio::sync::{Mutex, Notify, oneshot}; use super::{super::storage::Storage, *}; - #[tokio::test] - async fn reconcile_web_source_configs_upserts_and_deletes_exact_set() { - let storage = Storage::new(crate::db::Db::memory_db().await); - let user_id = storage.db().auto_create_user("web-user").await.unwrap().id; - let machine_id = uuid::Uuid::new_v4(); - let keep_id = uuid::Uuid::new_v4(); - let stale_id = uuid::Uuid::new_v4(); - let new_id = uuid::Uuid::new_v4(); - - storage - .db() - .insert_or_update_user_network_config( - (user_id, machine_id), - keep_id, - NetworkConfig { - network_name: Some("old-name".to_string()), - ..Default::default() - }, - ConfigSource::Web, - ) - .await - .unwrap(); - storage - .db() - .insert_or_update_user_network_config( - (user_id, machine_id), - stale_id, - NetworkConfig { - network_name: Some("stale".to_string()), - ..Default::default() - }, - ConfigSource::Web, - ) - .await - .unwrap(); - - SessionRpcService::reconcile_web_source_configs( - &storage, - user_id, - machine_id, - vec![ - crate::webhook::ManagedNetworkConfig { - instance_id: keep_id.to_string(), - network_config: json!({ - "instance_id": keep_id.to_string(), - "network_name": "updated-name" - }), - }, - crate::webhook::ManagedNetworkConfig { - instance_id: new_id.to_string(), - network_config: json!({ - "instance_id": new_id.to_string(), - "network_name": "new-name" - }), - }, - ], - ) - .await - .unwrap(); - - let configs = storage - .db() - .list_network_configs((user_id, machine_id), ListNetworkProps::All) - .await - .unwrap(); - let config_ids = configs - .iter() - .map(|cfg| cfg.network_instance_id.clone()) - .collect::>(); - - assert_eq!(configs.len(), 2); - assert!(config_ids.contains(&keep_id.to_string())); - assert!(config_ids.contains(&new_id.to_string())); - assert!(!config_ids.contains(&stale_id.to_string())); - - let updated_keep = storage - .db() - .get_network_config((user_id, machine_id), &keep_id.to_string()) - .await - .unwrap() - .unwrap(); - let updated_keep_config: NetworkConfig = - serde_json::from_str(&updated_keep.network_config).unwrap(); + #[test] + fn heartbeat_response_delay_only_fills_remaining_time() { assert_eq!( - updated_keep_config.network_name.as_deref(), - Some("updated-name") + heartbeat_response_delay(Duration::from_millis(100), Duration::from_millis(3500)), + Some(Duration::from_millis(3400)) + ); + assert_eq!( + heartbeat_response_delay(Duration::from_millis(3500), Duration::from_millis(3500)), + None + ); + assert_eq!( + heartbeat_response_delay(Duration::from_millis(3600), Duration::from_millis(3500)), + None ); - assert_eq!(updated_keep.get_network_config_source(), ConfigSource::Web); - } - - #[tokio::test] - async fn reconcile_web_source_configs_keep_user_owned_configs() { - let storage = Storage::new(crate::db::Db::memory_db().await); - let user_id = storage - .db() - .auto_create_user("web-user-keep-user") - .await - .unwrap() - .id; - let machine_id = uuid::Uuid::new_v4(); - let user_owned_id = uuid::Uuid::new_v4(); - let web_owned_id = uuid::Uuid::new_v4(); - - storage - .db() - .insert_or_update_user_network_config( - (user_id, machine_id), - user_owned_id, - NetworkConfig { - network_name: Some("user-owned".to_string()), - ..Default::default() - }, - ConfigSource::User, - ) - .await - .unwrap(); - storage - .db() - .insert_or_update_user_network_config( - (user_id, machine_id), - web_owned_id, - NetworkConfig { - network_name: Some("web-owned".to_string()), - ..Default::default() - }, - ConfigSource::Web, - ) - .await - .unwrap(); - - SessionRpcService::reconcile_web_source_configs( - &storage, - user_id, - machine_id, - vec![crate::webhook::ManagedNetworkConfig { - instance_id: user_owned_id.to_string(), - network_config: json!({ - "instance_id": user_owned_id.to_string(), - "network_name": "web-tries-to-take-over" - }), - }], - ) - .await - .unwrap(); - - let user_owned = storage - .db() - .get_network_config((user_id, machine_id), &user_owned_id.to_string()) - .await - .unwrap() - .unwrap(); - assert_eq!(user_owned.get_network_config_source(), ConfigSource::User); - let user_owned_cfg: NetworkConfig = - serde_json::from_str(&user_owned.network_config).unwrap(); - assert_eq!(user_owned_cfg.network_name.as_deref(), Some("user-owned")); - - let web_owned = storage - .db() - .get_network_config((user_id, machine_id), &web_owned_id.to_string()) - .await - .unwrap(); - assert!(web_owned.is_none()); } #[test] - fn validate_token_request_includes_applied_config_revision() { + fn heartbeat_response_delay_skips_unpaced_and_first_heartbeat() { + assert!(!should_delay_heartbeat_response(false, true)); + assert!(!should_delay_heartbeat_response(false, false)); + assert!(!should_delay_heartbeat_response(true, true)); + assert!(should_delay_heartbeat_response(true, false)); + } + + #[tokio::test] + async fn webhook_heartbeat_response_pacing_does_not_require_authorized() { + let machine_id = uuid::Uuid::new_v4(); + let storage = Storage::new(crate::db::Db::memory_db().await); + let mut data = SessionData::new( + storage.weak_ref(), + url::Url::parse("http://127.0.0.1").unwrap(), + None, + Arc::new(FeatureFlags::default()), + Arc::new(crate::webhook::WebhookConfig::new( + Some("http://127.0.0.1:1".to_string()), + None, + None, + None, + None, + )), + ); + + assert!(!should_delay_session_heartbeat_response(&data)); + + data.req = Some(heartbeat_request("token", machine_id)); + assert!(should_delay_session_heartbeat_response(&data)); + + data.auth_state = SessionAuthState::Invalid; + assert!(should_delay_session_heartbeat_response(&data)); + } + + #[test] + fn webhook_validation_retry_delay_is_bounded() { + let retry_delay = webhook_validation::retry_delay(uuid::Uuid::new_v4()); + assert!(retry_delay >= Duration::from_millis(webhook_validation::VALIDATION_RETRY_MS)); + assert!( + retry_delay + <= Duration::from_millis( + webhook_validation::VALIDATION_RETRY_MS + + webhook_validation::VALIDATION_RETRY_MS + ) + ); + } + + fn heartbeat_request(token: &str, machine_id: uuid::Uuid) -> HeartbeatRequest { + HeartbeatRequest { + machine_id: Some(machine_id.into()), + user_token: token.to_string(), + ..Default::default() + } + } + + #[derive(Clone)] + struct ValidateWebhookTestState { + received: Arc>>>, + release: Arc, + } + + async fn valid_validate_token_handler( + State(state): State, + ) -> Json { + if let Some(sender) = state.received.lock().await.take() { + let _ = sender.send(()); + } + state.release.notified().await; + + Json(json!({ + "valid": true, + "binding_version": 1, + "config_revision": "rev-1" + })) + } + + async fn test_webhook_config( + state: ValidateWebhookTestState, + ) -> (SharedWebhookConfig, tokio::task::JoinHandle<()>) { + let app = Router::new() + .route("/validate-token", post(valid_validate_token_handler)) + .with_state(state); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + let webhook_config = Arc::new(crate::webhook::WebhookConfig::new( + Some(format!("http://{addr}")), + None, + None, + None, + None, + )); + + (webhook_config, server) + } + + #[test] + fn heartbeat_identity_requires_matching_token_and_machine_id() { + let machine_id = uuid::Uuid::new_v4(); + let other_machine_id = uuid::Uuid::new_v4(); + let req = heartbeat_request("token-a", machine_id); + + assert!(SessionRpcService::heartbeat_matches_identity( + &req, "token-a", machine_id + )); + assert!(!SessionRpcService::heartbeat_matches_identity( + &req, "token-b", machine_id + )); + assert!(!SessionRpcService::heartbeat_matches_identity( + &req, + "token-a", + other_machine_id + )); + } + + #[tokio::test] + async fn webhook_heartbeat_saves_latest_and_marks_validation_dirty() { + let machine_id = uuid::Uuid::new_v4(); + let storage = Storage::new(crate::db::Db::memory_db().await); + let data = Arc::new(RwLock::new(SessionData::new( + storage.weak_ref(), + url::Url::parse("http://127.0.0.1").unwrap(), + None, + Arc::new(FeatureFlags::default()), + Arc::new(crate::webhook::WebhookConfig::new( + Some("http://127.0.0.1:1".to_string()), + None, + None, + None, + None, + )), + ))); + let service = SessionRpcService { + data: data.clone(), + heartbeat_min_response_delay: Duration::ZERO, + }; + + service + .handle_heartbeat(heartbeat_request("token", machine_id)) + .await + .unwrap(); + + let data = data.read().await; + assert!(data.webhook_validation_dirty); + assert_eq!(data.auth_state, SessionAuthState::Init); + assert!(data.storage_token.is_none()); + assert!(SessionRpcService::heartbeat_matches_identity( + data.req.as_ref().unwrap(), + "token", + machine_id, + )); + drop(data); + assert!( + storage + .db() + .get_user_id_by_token("token") + .await + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn webhook_validation_round_sets_token_and_notifies_runtime() { + let machine_id = uuid::Uuid::new_v4(); + let req = heartbeat_request("token", machine_id); + let storage = Storage::new(crate::db::Db::memory_db().await); + let (received_tx, received_rx) = oneshot::channel(); + let release = Arc::new(Notify::new()); + let (webhook_config, server) = test_webhook_config(ValidateWebhookTestState { + received: Arc::new(Mutex::new(Some(received_tx))), + release: release.clone(), + }) + .await; + let mut session = SessionData::new( + storage.weak_ref(), + url::Url::parse("http://127.0.0.1").unwrap(), + None, + Arc::new(FeatureFlags::default()), + webhook_config.clone(), + ); + session.req = Some(req.clone()); + session.session_identity = Some(SessionRpcService::heartbeat_identity(&req, machine_id)); + let session_data = Arc::new(RwLock::new(session)); + let mut heartbeat_waiter = session_data.read().await.heartbeat_waiter(); + + let validation = tokio::spawn(webhook_validation::run_round( + Arc::downgrade(&session_data), + webhook_validation::WebhookValidationInput { + storage: storage.clone(), + webhook_config, + client_url: url::Url::parse("http://127.0.0.1").unwrap(), + applied_config_revision: None, + req, + machine_id, + }, + )); + received_rx.await.unwrap(); + release.notify_waiters(); + validation.await.unwrap().unwrap(); + server.abort(); + + let data = session_data.read().await; + assert_eq!(data.auth_state, SessionAuthState::Authorized); + assert!(data.storage_token.is_some()); + assert_eq!(data.binding_version, Some(1)); + assert_eq!(data.webhook_connected_binding_version, Some(1)); + drop(data); + assert_eq!(heartbeat_waiter.recv().await.unwrap().user_token, "token"); + assert!( + storage + .db() + .get_user_id_by_token("token") + .await + .unwrap() + .is_some() + ); + } + + #[tokio::test] + async fn authenticated_heartbeat_rejects_mismatched_identity() { + let machine_id = uuid::Uuid::new_v4(); + let other_machine_id = uuid::Uuid::new_v4(); + let req = heartbeat_request("token", machine_id); + let storage = Storage::new(crate::db::Db::memory_db().await); + let user_id = storage.db().auto_create_user("token").await.unwrap().id; + let (received_tx, received_rx) = oneshot::channel(); + let release = Arc::new(Notify::new()); + let (webhook_config, server) = test_webhook_config(ValidateWebhookTestState { + received: Arc::new(Mutex::new(Some(received_tx))), + release, + }) + .await; + let mut data = SessionData::new( + storage.weak_ref(), + url::Url::parse("http://127.0.0.1").unwrap(), + None, + Arc::new(FeatureFlags { + allow_auto_create_user: true, + ..Default::default() + }), + webhook_config, + ); + data.storage_token = Some(StorageToken { + token: "token".to_string(), + client_url: url::Url::parse("http://127.0.0.1").unwrap(), + machine_id, + user_id, + }); + data.session_identity = Some(SessionRpcService::heartbeat_identity(&req, machine_id)); + data.auth_state = SessionAuthState::Authorized; + data.req = Some(req); + let session_data = Arc::new(RwLock::new(data)); + let service = SessionRpcService { + data: session_data.clone(), + heartbeat_min_response_delay: Duration::ZERO, + }; + + let err = service + .handle_heartbeat(heartbeat_request("other-token", other_machine_id)) + .await + .expect_err("mismatched authenticated heartbeat must fail"); + assert!( + err.to_string() + .contains("Heartbeat identity does not match") + ); + assert!( + tokio::time::timeout(Duration::from_millis(50), received_rx) + .await + .is_err() + ); + server.abort(); + + let data = session_data.read().await; + assert!(SessionRpcService::storage_token_matches_heartbeat( + data.storage_token.as_ref().unwrap(), + data.req.as_ref().unwrap() + )); + assert_eq!( + data.heartbeat_count + .load(std::sync::atomic::Ordering::Relaxed), + 0 + ); + drop(data); + assert!( + storage + .db() + .get_user_id_by_token("other-token") + .await + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn non_webhook_mismatched_identity_does_not_auto_create_user() { + let machine_id = uuid::Uuid::new_v4(); + let other_machine_id = uuid::Uuid::new_v4(); + let req = heartbeat_request("token", machine_id); + let storage = Storage::new(crate::db::Db::memory_db().await); + let user_id = storage.db().auto_create_user("token").await.unwrap().id; + let mut data = SessionData::new( + storage.weak_ref(), + url::Url::parse("http://127.0.0.1").unwrap(), + None, + Arc::new(FeatureFlags { + allow_auto_create_user: true, + ..Default::default() + }), + Arc::new(crate::webhook::WebhookConfig::new( + None, None, None, None, None, + )), + ); + data.storage_token = Some(StorageToken { + token: "token".to_string(), + client_url: url::Url::parse("http://127.0.0.1").unwrap(), + machine_id, + user_id, + }); + data.session_identity = Some(SessionRpcService::heartbeat_identity(&req, machine_id)); + data.req = Some(req); + let session_data = Arc::new(RwLock::new(data)); + let service = SessionRpcService { + data: session_data, + heartbeat_min_response_delay: Duration::ZERO, + }; + + let err = service + .handle_heartbeat(heartbeat_request("other-token", other_machine_id)) + .await + .expect_err("mismatched heartbeat must fail before DB side effects"); + assert!( + err.to_string() + .contains("Heartbeat identity does not match") + ); + assert!( + storage + .db() + .get_user_id_by_token("other-token") + .await + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn webhook_reject_keeps_session_visible_but_invalid() { + let machine_id = uuid::Uuid::new_v4(); + let req = heartbeat_request("token", machine_id); + let storage = Storage::new(crate::db::Db::memory_db().await); + let user_id = storage.db().auto_create_user("token").await.unwrap().id; + let storage_token = StorageToken { + token: "token".to_string(), + client_url: url::Url::parse("http://127.0.0.1").unwrap(), + machine_id, + user_id, + }; + storage.update_client(storage_token.clone(), 1, true); + let mut data = SessionData::new( + storage.weak_ref(), + url::Url::parse("http://127.0.0.1").unwrap(), + None, + Arc::new(FeatureFlags::default()), + Arc::new(crate::webhook::WebhookConfig::new( + Some("http://127.0.0.1:1".to_string()), + None, + None, + None, + None, + )), + ); + data.storage_token = Some(storage_token); + data.session_identity = Some(SessionRpcService::heartbeat_identity(&req, machine_id)); + data.req = Some(req.clone()); + data.auth_state = SessionAuthState::Authorized; + data.webhook_validation_dirty = true; + data.webhook_connected_binding_version = Some(3); + let session_data = Arc::new(RwLock::new(data)); + + webhook_validation::apply_rejected( + &Arc::downgrade(&session_data), + &webhook_validation::WebhookValidationInput { + storage: storage.clone(), + webhook_config: Arc::new(crate::webhook::WebhookConfig::new( + None, None, None, None, None, + )), + client_url: url::Url::parse("http://127.0.0.1").unwrap(), + applied_config_revision: None, + req, + machine_id, + }, + ) + .await; + + let data = session_data.read().await; + assert!(data.storage_token.is_some()); + assert_eq!(data.auth_state, SessionAuthState::Invalid); + assert!(!data.webhook_validation_dirty); + assert_eq!(data.webhook_connected_binding_version, None); + drop(data); + assert_eq!( + storage.get_client_url_by_machine_id(user_id, &machine_id), + None + ); + assert_eq!( + storage.get_client_url_by_machine_id_with_auth(user_id, &machine_id, false), + Some(url::Url::parse("http://127.0.0.1").unwrap()) + ); + } + + #[tokio::test] + async fn webhook_reject_prevents_reauthorize_same_session() { + let machine_id = uuid::Uuid::new_v4(); + let req = heartbeat_request("token", machine_id); + let storage = Storage::new(crate::db::Db::memory_db().await); + let user_id = storage.db().auto_create_user("token").await.unwrap().id; + let client_url = url::Url::parse("http://127.0.0.1").unwrap(); + let storage_token = StorageToken { + token: "token".to_string(), + client_url: client_url.clone(), + machine_id, + user_id, + }; + storage.update_client(storage_token.clone(), 1, true); + let mut data = SessionData::new( + storage.weak_ref(), + client_url.clone(), + None, + Arc::new(FeatureFlags::default()), + Arc::new(crate::webhook::WebhookConfig::new( + Some("http://127.0.0.1:1".to_string()), + None, + None, + None, + None, + )), + ); + data.storage_token = Some(storage_token); + data.session_identity = Some(SessionRpcService::heartbeat_identity(&req, machine_id)); + data.req = Some(req.clone()); + data.auth_state = SessionAuthState::Authorized; + data.webhook_connected_binding_version = Some(6); + let session_data = Arc::new(RwLock::new(data)); + let weak_session = Arc::downgrade(&session_data); + + let input = webhook_validation::WebhookValidationInput { + storage: storage.clone(), + webhook_config: Arc::new(crate::webhook::WebhookConfig::new( + None, None, None, None, None, + )), + client_url: client_url.clone(), + applied_config_revision: None, + req: req.clone(), + machine_id, + }; + webhook_validation::apply_rejected(&weak_session, &input).await; + assert_eq!( + session_data.read().await.webhook_connected_binding_version, + None + ); + assert_eq!( + storage.get_client_url_by_machine_id(user_id, &machine_id), + None + ); + assert_eq!( + storage.get_client_url_by_machine_id_with_auth(user_id, &machine_id, false), + Some(client_url.clone()) + ); + + webhook_validation::apply_success( + &weak_session, + input, + webhook_validation::WebhookHeartbeatValidation { + config_revision: "rev-1".to_string(), + binding_version: 7, + }, + user_id, + ) + .await; + + let data = session_data.read().await; + assert!(data.storage_token.is_some()); + assert_eq!(data.auth_state, SessionAuthState::Invalid); + assert_eq!(data.binding_version, None); + assert_eq!(data.webhook_connected_binding_version, None); + drop(data); + assert_eq!( + storage.get_client_url_by_machine_id(user_id, &machine_id), + None + ); + assert_eq!( + storage.get_client_url_by_machine_id_with_auth(user_id, &machine_id, false), + Some(client_url) + ); + } + + #[tokio::test] + async fn invalid_webhook_session_does_not_revalidate_on_same_connection() { + let machine_id = uuid::Uuid::new_v4(); + let req = heartbeat_request("token", machine_id); + let storage = Storage::new(crate::db::Db::memory_db().await); + let user_id = storage.db().auto_create_user("token").await.unwrap().id; + let client_url = url::Url::parse("http://127.0.0.1").unwrap(); + let mut data = SessionData::new( + storage.weak_ref(), + client_url.clone(), + None, + Arc::new(FeatureFlags::default()), + Arc::new(crate::webhook::WebhookConfig::new( + Some("http://127.0.0.1:1".to_string()), + None, + None, + None, + None, + )), + ); + data.storage_token = Some(StorageToken { + token: "token".to_string(), + client_url, + machine_id, + user_id, + }); + data.session_identity = Some(SessionRpcService::heartbeat_identity(&req, machine_id)); + data.auth_state = SessionAuthState::Invalid; + data.webhook_validation_dirty = false; + data.heartbeat_count.store( + WEBHOOK_VALIDATION_HEARTBEAT_INTERVAL, + std::sync::atomic::Ordering::Relaxed, + ); + let session_data = Arc::new(RwLock::new(data)); + let service = SessionRpcService { + data: session_data.clone(), + heartbeat_min_response_delay: Duration::ZERO, + }; + + service + .handle_heartbeat(req) + .await + .expect_err("invalid webhook session must fail heartbeat"); + + let data = session_data.read().await; + assert!(!data.webhook_validation_dirty); + assert_eq!(data.auth_state, SessionAuthState::Invalid); + } + + #[tokio::test] + async fn invalid_webhook_heartbeat_returns_error() { + let machine_id = uuid::Uuid::new_v4(); + let req = heartbeat_request("token", machine_id); + let storage = Storage::new(crate::db::Db::memory_db().await); + let user_id = storage.db().auto_create_user("token").await.unwrap().id; + let client_url = url::Url::parse("http://127.0.0.1").unwrap(); + let storage_token = StorageToken { + token: "token".to_string(), + client_url: client_url.clone(), + machine_id, + user_id, + }; + storage.update_client(storage_token.clone(), 1, false); + let mut data = SessionData::new( + storage.weak_ref(), + client_url.clone(), + None, + Arc::new(FeatureFlags::default()), + Arc::new(crate::webhook::WebhookConfig::new( + Some("http://127.0.0.1:1".to_string()), + None, + None, + None, + None, + )), + ); + data.storage_token = Some(storage_token); + data.session_identity = Some(SessionRpcService::heartbeat_identity(&req, machine_id)); + data.auth_state = SessionAuthState::Invalid; + let session_data = Arc::new(RwLock::new(data)); + let service = SessionRpcService { + data: session_data, + heartbeat_min_response_delay: Duration::ZERO, + }; + + service + .handle_heartbeat(req) + .await + .expect_err("invalid webhook heartbeat must fail"); + + assert_eq!( + storage.get_client_url_by_machine_id(user_id, &machine_id), + None + ); + assert_eq!( + storage.get_client_url_by_machine_id_with_auth(user_id, &machine_id, false), + Some(client_url) + ); + } + + #[tokio::test] + async fn webhook_success_replaces_connected_binding_version() { + let machine_id = uuid::Uuid::new_v4(); + let req = heartbeat_request("token", machine_id); + let storage = Storage::new(crate::db::Db::memory_db().await); + let user_id = storage.db().auto_create_user("token").await.unwrap().id; + let client_url = url::Url::parse("http://127.0.0.1").unwrap(); + let storage_token = StorageToken { + token: "token".to_string(), + client_url: client_url.clone(), + machine_id, + user_id, + }; + let mut data = SessionData::new( + storage.weak_ref(), + client_url.clone(), + None, + Arc::new(FeatureFlags::default()), + Arc::new(crate::webhook::WebhookConfig::new( + None, None, None, None, None, + )), + ); + data.storage_token = Some(storage_token); + data.session_identity = Some(SessionRpcService::heartbeat_identity(&req, machine_id)); + data.req = Some(req.clone()); + data.auth_state = SessionAuthState::Authorized; + data.binding_version = Some(6); + data.webhook_connected_binding_version = Some(6); + let session_data = Arc::new(RwLock::new(data)); + + webhook_validation::apply_success( + &Arc::downgrade(&session_data), + webhook_validation::WebhookValidationInput { + storage, + webhook_config: Arc::new(crate::webhook::WebhookConfig::new( + None, None, None, None, None, + )), + client_url, + applied_config_revision: None, + req, + machine_id, + }, + webhook_validation::WebhookHeartbeatValidation { + config_revision: "rev-1".to_string(), + binding_version: 7, + }, + user_id, + ) + .await; + + let data = session_data.read().await; + assert_eq!(data.auth_state, SessionAuthState::Authorized); + assert_eq!(data.binding_version, Some(7)); + assert_eq!(data.webhook_connected_binding_version, Some(7)); + } + + #[tokio::test] + async fn runtime_heartbeat_rechecks_webhook_state_before_reconcile() { + let machine_id = uuid::Uuid::new_v4(); + let req = heartbeat_request("token", machine_id); + let storage = Storage::new(crate::db::Db::memory_db().await); + let user_id = storage.db().auto_create_user("token").await.unwrap().id; + let storage_token = StorageToken { + token: "token".to_string(), + client_url: url::Url::parse("http://127.0.0.1").unwrap(), + machine_id, + user_id, + }; + let mut data = SessionData::new( + storage.weak_ref(), + url::Url::parse("http://127.0.0.1").unwrap(), + None, + Arc::new(FeatureFlags::default()), + Arc::new(crate::webhook::WebhookConfig::new( + Some("http://127.0.0.1:1".to_string()), + None, + None, + None, + None, + )), + ); + data.storage_token = Some(storage_token); + data.session_identity = Some(SessionRpcService::heartbeat_identity(&req, machine_id)); + data.req = Some(req.clone()); + data.auth_state = SessionAuthState::Authorized; + let session_data = Arc::new(RwLock::new(data)); + let weak_session = Arc::downgrade(&session_data); + + assert!(SessionRpcService::runtime_heartbeat_is_current(&weak_session, &req).await); + + webhook_validation::apply_rejected( + &weak_session, + &webhook_validation::WebhookValidationInput { + storage, + webhook_config: Arc::new(crate::webhook::WebhookConfig::new( + None, None, None, None, None, + )), + client_url: url::Url::parse("http://127.0.0.1").unwrap(), + applied_config_revision: None, + req: req.clone(), + machine_id, + }, + ) + .await; + + assert!(!SessionRpcService::runtime_heartbeat_is_current(&weak_session, &req).await); + } + + #[test] + fn validate_token_request_includes_config_revisions() { let req = crate::webhook::ValidateTokenRequest { token: "token".to_string(), machine_id: "machine".to_string(), @@ -1164,10 +1458,17 @@ mod tests { os_distribution: None, web_instance_id: Some("web-1".to_string()), web_instance_api_base_url: Some("http://console".to_string()), + persisted_config_revision: Some("rev-0".to_string()), applied_config_revision: Some("rev-1".to_string()), }; let value = serde_json::to_value(req).unwrap(); + assert_eq!( + value + .get("persisted_config_revision") + .and_then(|v| v.as_str()), + Some("rev-0") + ); assert_eq!( value .get("applied_config_revision") @@ -1175,101 +1476,4 @@ mod tests { Some("rev-1") ); } - - #[test] - fn validate_token_response_without_configs_reuses_same_revision() { - let resp = crate::webhook::ValidateTokenResponse { - valid: true, - pre_approved: true, - binding_version: 1, - managed_network_configs: None, - config_revision: "rev-1".to_string(), - }; - - let (configs, revision) = - SessionRpcService::managed_configs_for_revision(Some("rev-1"), resp).unwrap(); - assert!(configs.is_empty()); - assert_eq!(revision, "rev-1"); - } - - #[test] - fn validate_token_response_without_configs_rejects_changed_revision() { - let resp = crate::webhook::ValidateTokenResponse { - valid: true, - pre_approved: true, - binding_version: 1, - managed_network_configs: None, - config_revision: "rev-2".to_string(), - }; - - let err = SessionRpcService::managed_configs_for_revision(Some("rev-1"), resp) - .expect_err("omitted configs with a changed revision must fail"); - assert!(err.to_string().contains("omitted managed configs")); - } - - #[tokio::test] - async fn sync_running_config_sources_updates_enabled_config_source_from_runtime() { - let storage = Storage::new(crate::db::Db::memory_db().await); - let user_id = storage - .db() - .auto_create_user("web-user-sync-source") - .await - .unwrap() - .id; - let machine_id = uuid::Uuid::new_v4(); - let inst_id = uuid::Uuid::new_v4(); - - storage - .db() - .insert_or_update_user_network_config( - (user_id, machine_id), - inst_id, - NetworkConfig { - network_name: Some("web-owned".to_string()), - ..Default::default() - }, - ConfigSource::Web, - ) - .await - .unwrap(); - - let local_configs = storage - .db() - .list_network_configs((user_id, machine_id), ListNetworkProps::EnabledOnly) - .await - .unwrap(); - Session::sync_running_config_sources( - storage.db(), - user_id, - machine_id, - &local_configs, - &[easytier::proto::api::manage::NetworkMeta { - inst_id: Some(inst_id.into()), - source: RpcConfigSource::User as i32, - ..Default::default() - }], - ) - .await - .unwrap(); - - let updated = storage - .db() - .get_network_config((user_id, machine_id), &inst_id.to_string()) - .await - .unwrap() - .unwrap(); - assert_eq!(updated.get_network_config_source(), ConfigSource::Web); - } - - #[test] - fn persisted_sources_map_to_rpc_sources() { - assert_eq!( - PersistedConfigSource::Web.auto_run_rpc_source(), - RpcConfigSource::Web - ); - assert_eq!( - PersistedConfigSource::User.auto_run_rpc_source(), - RpcConfigSource::User - ); - } } diff --git a/easytier-web/src/client_manager/session/runtime_revision.rs b/easytier-web/src/client_manager/session/runtime_revision.rs new file mode 100644 index 00000000..be91fb65 --- /dev/null +++ b/easytier-web/src/client_manager/session/runtime_revision.rs @@ -0,0 +1,913 @@ +use std::collections::{HashMap, HashSet}; + +use easytier::{ + proto::{ + api::manage::{ + DeleteNetworkInstanceRequest, ListNetworkInstanceMetaRequest, + ListNetworkInstanceRequest, NetworkConfig, NetworkMeta, RunNetworkInstanceRequest, + }, + rpc_types::controller::BaseController, + web::HeartbeatRequest, + }, + rpc_service::remote_client::{ListNetworkProps, Storage as _}, +}; +use tokio::sync::{RwLock, broadcast}; + +use super::{SessionConfigClient, SessionData, SessionRpcClient, SessionRpcService}; +use crate::client_manager::{ + managed_config::{self, PersistedConfigSource}, + runtime_reconcile, + storage::{StorageInner, WeakRefStorage}, +}; + +async fn recv_latest_heartbeat( + heartbeat_waiter: &mut broadcast::Receiver, +) -> Option { + let mut req = loop { + match heartbeat_waiter.recv().await { + Ok(req) => break req, + Err(broadcast::error::RecvError::Lagged(skipped)) => { + tracing::warn!( + skipped, + "heartbeat reconcile worker lagged, waiting for latest request" + ); + } + Err(broadcast::error::RecvError::Closed) => { + tracing::error!("Failed to receive heartbeat request: channel closed"); + return None; + } + } + }; + + // Drop any heartbeat backlog accumulated while the previous reconcile + // round was doing DB/RPC IO. The newest heartbeat has the freshest + // runtime instance list, which is all this task needs. + loop { + match heartbeat_waiter.try_recv() { + Ok(next_req) => req = next_req, + Err(broadcast::error::TryRecvError::Empty) => break, + Err(broadcast::error::TryRecvError::Lagged(_)) => continue, + Err(broadcast::error::TryRecvError::Closed) => return None, + } + } + + Some(req) +} + +pub(super) async fn reconcile_network_configs_on_heartbeat( + session_data: std::sync::Weak>, + mut heartbeat_waiter: broadcast::Receiver, + storage: WeakRefStorage, + mut rpc_client: SessionRpcClient, + mut config_client: SessionConfigClient, +) { + let mut cache = ReconcileCache::default(); + loop { + let Some(req) = recv_latest_heartbeat(&mut heartbeat_waiter).await else { + return; + }; + let Some(storage) = storage.upgrade() else { + tracing::error!("Failed to get storage"); + return; + }; + + let mut round = + match prepare_reconcile_round(&session_data, &storage, &mut rpc_client, req).await { + RoundStatus::Ready(round) => round, + RoundStatus::Skip => continue, + RoundStatus::Stop => return, + }; + let running_metas = + match sync_running_sources_for_round(&mut rpc_client, &storage, &mut round).await { + RoundStatus::Ready(running_metas) => running_metas, + RoundStatus::Skip => continue, + RoundStatus::Stop => return, + }; + + let desired_web_inst_ids = + managed_config::desired_web_source_instance_ids(&round.local_configs); + cache.runtime_configs.retain_desired(&desired_web_inst_ids); + let mut outcome = match cleanup_stale_web_source_instances( + &session_data, + &storage, + &mut rpc_client, + &round, + running_metas.as_deref(), + &desired_web_inst_ids, + &mut cache, + ) + .await + { + RoundStatus::Ready(outcome) => outcome, + RoundStatus::Skip => continue, + RoundStatus::Stop => return, + }; + + outcome.merge( + reconcile_desired_runtime_configs( + &session_data, + &mut rpc_client, + &mut config_client, + &round, + &mut cache, + ) + .await, + ); + + if !outcome.has_failed { + cache.last_desired_web_inst_ids = Some(desired_web_inst_ids); + } + match mark_config_revision_applied_if_current(&session_data, &storage, &round, &outcome) + .await + { + RoundStatus::Ready(()) | RoundStatus::Skip => {} + RoundStatus::Stop => return, + } + } +} + +enum RoundStatus { + Ready(T), + Skip, + Stop, +} + +enum ConfigActionResult { + Success, + Failed, + StopRound, +} + +#[derive(Default)] +struct ReconcileCache { + cleaned_web_source_instances: bool, + last_desired_web_inst_ids: Option>, + runtime_configs: SessionRuntimeConfigCache, +} + +#[derive(Default)] +struct SessionRuntimeConfigCache { + entries: HashMap, +} + +impl SessionRuntimeConfigCache { + fn plan( + &self, + inst_id: &str, + desired_config: NetworkConfig, + ) -> anyhow::Result> { + let Some(observed_config) = self.entries.get(inst_id) else { + return Ok(None); + }; + + runtime_reconcile::prepare_web_source_runtime_reconcile_from_current( + observed_config, + desired_config, + ) + .map(Some) + } + + fn remember(&mut self, inst_id: &str, observed_config: NetworkConfig) { + self.entries.insert(inst_id.to_string(), observed_config); + } + + fn forget(&mut self, inst_id: &str) { + self.entries.remove(inst_id); + } + + fn forget_many<'a>(&mut self, inst_ids: impl IntoIterator) { + for inst_id in inst_ids { + self.entries.remove(inst_id); + } + } + + fn retain_desired(&mut self, desired_web_inst_ids: &HashSet) { + self.entries + .retain(|inst_id, _| desired_web_inst_ids.contains(inst_id)); + } +} + +#[derive(Default)] +struct ReconcileOutcome { + has_failed: bool, + managed_revision_failed: bool, +} + +impl ReconcileOutcome { + fn record_failure(&mut self, managed_revision_failed: bool) { + self.has_failed = true; + self.managed_revision_failed |= managed_revision_failed; + } + + fn merge(&mut self, other: Self) { + self.has_failed |= other.has_failed; + self.managed_revision_failed |= other.managed_revision_failed; + } +} + +struct ReconcileRound { + req: HeartbeatRequest, + machine_id: uuid::Uuid, + user_id: i32, + running_inst_ids: HashSet, + local_configs: Vec, + target_config_revision: Option, + should_apply_runtime_revision: bool, +} + +async fn prepare_reconcile_round( + session_data: &std::sync::Weak>, + storage: &StorageInner, + rpc_client: &mut SessionRpcClient, + req: HeartbeatRequest, +) -> RoundStatus { + let Some(machine_id) = req.machine_id.map(uuid::Uuid::from) else { + tracing::warn!(?req, "Machine id is not set, ignore"); + return RoundStatus::Skip; + }; + if !SessionRpcService::runtime_heartbeat_is_current(session_data, &req).await { + tracing::debug!(?machine_id, "skip stale heartbeat reconcile request"); + return RoundStatus::Skip; + } + + let user_id = match storage + .db + .get_user_id_by_token(req.user_token.clone()) + .await + { + Ok(Some(user_id)) => user_id, + Ok(None) => { + tracing::info!("User not found by token: {:?}", req.user_token); + return RoundStatus::Stop; + } + Err(e) => { + tracing::error!("Failed to get user id by token, error: {:?}", e); + return RoundStatus::Stop; + } + }; + + let applied_config_revision = { + let Some(data) = session_data.upgrade() else { + return RoundStatus::Stop; + }; + data.read().await.applied_config_revision.clone() + }; + let target_config_revision = match storage + .db + .get_managed_config_revision((user_id, machine_id)) + .await + { + Ok(revision) => revision, + Err(e) => { + tracing::error!("Failed to read managed config revision, error: {:?}", e); + return RoundStatus::Stop; + } + }; + let should_apply_runtime_revision = + target_config_revision.is_some() && target_config_revision != applied_config_revision; + let running_inst_ids = match running_instance_ids_for_round( + rpc_client, + &req, + user_id, + machine_id, + should_apply_runtime_revision, + ) + .await + { + RoundStatus::Ready(ids) => ids, + RoundStatus::Skip => return RoundStatus::Skip, + RoundStatus::Stop => return RoundStatus::Stop, + }; + + let local_configs = match storage + .db + .list_network_configs((user_id, machine_id), ListNetworkProps::EnabledOnly) + .await + { + Ok(configs) => configs, + Err(e) => { + tracing::error!("Failed to list network configs, error: {:?}", e); + return RoundStatus::Stop; + } + }; + + RoundStatus::Ready(ReconcileRound { + req, + machine_id, + user_id, + running_inst_ids, + local_configs, + target_config_revision, + should_apply_runtime_revision, + }) +} + +async fn running_instance_ids_for_round( + rpc_client: &mut SessionRpcClient, + req: &HeartbeatRequest, + user_id: i32, + machine_id: uuid::Uuid, + should_apply_runtime_revision: bool, +) -> RoundStatus> { + if !should_apply_runtime_revision { + return RoundStatus::Ready( + req.running_network_instances + .iter() + .map(|x| x.to_string()) + .collect(), + ); + } + + match rpc_client + .list_network_instance(BaseController::default(), ListNetworkInstanceRequest {}) + .await + { + Ok(resp) => RoundStatus::Ready(resp.inst_ids.iter().map(|x| x.to_string()).collect()), + Err(error) => { + tracing::warn!( + ?user_id, + ?machine_id, + ?error, + "Failed to refresh running instances for managed config revision" + ); + RoundStatus::Skip + } + } +} + +async fn sync_running_sources_for_round( + rpc_client: &mut SessionRpcClient, + storage: &StorageInner, + round: &mut ReconcileRound, +) -> RoundStatus>> { + if !round.req.support_config_source { + return RoundStatus::Ready(None); + } + + let ret = if round.running_inst_ids.is_empty() { + Ok(Vec::new()) + } else { + rpc_client + .list_network_instance_meta( + BaseController::default(), + ListNetworkInstanceMetaRequest { + inst_ids: managed_config::parse_instance_ids( + round.running_inst_ids.iter().cloned(), + ), + }, + ) + .await + .map(|resp| resp.metas) + }; + + match ret { + Ok(metas) => { + if let Err(e) = managed_config::sync_running_config_sources( + &storage.db, + round.user_id, + round.machine_id, + &round.local_configs, + &metas, + ) + .await + { + tracing::warn!( + user_id = ?round.user_id, + machine_id = ?round.machine_id, + %e, + "Failed to sync running network config sources" + ); + } else if !metas.is_empty() { + round.local_configs = match storage + .db + .list_network_configs( + (round.user_id, round.machine_id), + ListNetworkProps::EnabledOnly, + ) + .await + { + Ok(configs) => configs, + Err(e) => { + tracing::error!( + "Failed to reload network configs after source sync, error: {:?}", + e + ); + return RoundStatus::Stop; + } + }; + } + RoundStatus::Ready(Some(metas)) + } + Err(e) => { + tracing::warn!( + user_id = ?round.user_id, + %e, + "Failed to list running network instance metadata" + ); + RoundStatus::Ready(None) + } + } +} + +async fn cleanup_stale_web_source_instances( + session_data: &std::sync::Weak>, + storage: &StorageInner, + rpc_client: &mut SessionRpcClient, + round: &ReconcileRound, + running_metas: Option<&[NetworkMeta]>, + desired_web_inst_ids: &HashSet, + cache: &mut ReconcileCache, +) -> RoundStatus { + let desired_changed = cache + .last_desired_web_inst_ids + .as_ref() + .is_none_or(|last| last != desired_web_inst_ids); + if cache.cleaned_web_source_instances && !desired_changed { + return RoundStatus::Ready(ReconcileOutcome::default()); + } + + let db_web_inst_ids = match storage + .db + .list_network_configs((round.user_id, round.machine_id), ListNetworkProps::All) + .await + { + Ok(configs) => managed_config::desired_web_source_instance_ids(&configs), + Err(e) => { + tracing::error!("Failed to list all network configs, error: {:?}", e); + return RoundStatus::Stop; + } + }; + + let running_web_inst_ids = managed_config::running_web_source_instance_ids( + &round.running_inst_ids, + &db_web_inst_ids, + running_metas, + ); + let should_delete_inst_ids = running_web_inst_ids + .difference(desired_web_inst_ids) + .cloned() + .collect::>(); + let should_delete_ids = + managed_config::parse_instance_ids(should_delete_inst_ids.iter().cloned()); + + let mut outcome = ReconcileOutcome::default(); + if !should_delete_ids.is_empty() { + if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await { + tracing::debug!( + machine_id = ?round.machine_id, + "skip stale cleanup because webhook session is no longer current" + ); + return RoundStatus::Skip; + } + let ret = rpc_client + .delete_network_instance( + BaseController::default(), + DeleteNetworkInstanceRequest { + inst_ids: should_delete_ids, + }, + ) + .await; + tracing::info!( + user_id = ?round.user_id, + "Clean stale web-source network instances on heartbeat: {:?}, user_token: {:?}", + ret, + round.req.user_token + ); + if ret.is_err() { + outcome.record_failure(true); + } else { + cache.runtime_configs.forget_many(&should_delete_inst_ids); + } + } + + if !outcome.has_failed { + cache.cleaned_web_source_instances = true; + cache.last_desired_web_inst_ids = Some(desired_web_inst_ids.clone()); + } + + RoundStatus::Ready(outcome) +} + +async fn reconcile_desired_runtime_configs( + session_data: &std::sync::Weak>, + rpc_client: &mut SessionRpcClient, + config_client: &mut SessionConfigClient, + round: &ReconcileRound, + cache: &mut ReconcileCache, +) -> ReconcileOutcome { + let mut outcome = ReconcileOutcome::default(); + + // After stale web-owned instances are removed, start every enabled + // config that the latest heartbeat did not report as running. When + // a managed config revision is pending, also reconcile running + // web-owned configs before reporting that revision as applied. + for config in &round.local_configs { + let source = PersistedConfigSource::from_db(&config.source); + let is_running = round.running_inst_ids.contains(&config.network_instance_id); + let should_reconcile_running_web_config = is_running + && round.should_apply_runtime_revision + && source == PersistedConfigSource::Web; + if is_running && !should_reconcile_running_web_config { + continue; + } + + let desired_config = match serde_json::from_str::(&config.network_config) { + Ok(cfg) => cfg, + Err(e) => { + tracing::error!( + user_id = ?round.user_id, + machine_id = ?round.machine_id, + instance_id = %config.network_instance_id, + "Failed to deserialize network config, skipping: {:?}", + e + ); + if source == PersistedConfigSource::Web { + cache.runtime_configs.forget(&config.network_instance_id); + } + outcome.record_failure(source == PersistedConfigSource::Web); + continue; + } + }; + + let action_result = if should_reconcile_running_web_config { + reconcile_running_web_config( + session_data, + rpc_client, + config_client, + round, + config, + desired_config, + &mut cache.runtime_configs, + ) + .await + } else { + if source == PersistedConfigSource::Web { + cache.runtime_configs.forget(&config.network_instance_id); + } + let action_result = run_missing_network_config( + session_data, + rpc_client, + round, + config, + desired_config.clone(), + ) + .await; + if matches!(action_result, ConfigActionResult::Success) + && source == PersistedConfigSource::Web + { + if let Err(e) = remember_web_runtime_config_after_run( + rpc_client, + &config.network_instance_id, + &desired_config, + &mut cache.runtime_configs, + ) + .await + { + tracing::error!( + user_id = ?round.user_id, + machine_id = ?round.machine_id, + instance_id = %config.network_instance_id, + "Failed to cache runtime config after run: {:?}", + e + ); + ConfigActionResult::Failed + } else { + action_result + } + } else { + action_result + } + }; + + match action_result { + ConfigActionResult::Success => {} + ConfigActionResult::Failed => { + if source == PersistedConfigSource::Web { + cache.runtime_configs.forget(&config.network_instance_id); + } + outcome.record_failure(source == PersistedConfigSource::Web) + } + ConfigActionResult::StopRound => { + if source == PersistedConfigSource::Web { + cache.runtime_configs.forget(&config.network_instance_id); + } + outcome.record_failure(source == PersistedConfigSource::Web); + break; + } + } + } + + outcome +} + +async fn reconcile_running_web_config( + session_data: &std::sync::Weak>, + rpc_client: &mut SessionRpcClient, + config_client: &mut SessionConfigClient, + round: &ReconcileRound, + config: &crate::db::entity::user_running_network_configs::Model, + desired_config: NetworkConfig, + runtime_config_cache: &mut SessionRuntimeConfigCache, +) -> ConfigActionResult { + if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await { + tracing::debug!( + machine_id = ?round.machine_id, + instance_id = %config.network_instance_id, + "skip runtime reconcile because webhook session is no longer current" + ); + return ConfigActionResult::StopRound; + } + + let ret = async { + let action = + match runtime_config_cache.plan(&config.network_instance_id, desired_config.clone())? { + Some(action) => action, + None => { + runtime_reconcile::prepare_web_source_runtime_reconcile( + &mut *rpc_client, + &config.network_instance_id, + desired_config.clone(), + true, + ) + .await? + } + }; + if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await { + anyhow::bail!("webhook session is no longer current before runtime reconcile apply"); + } + let observed_config = runtime_reconcile::apply_web_source_runtime_reconcile( + &mut *rpc_client, + &mut *config_client, + &config.network_instance_id, + desired_config.clone(), + action, + ) + .await?; + runtime_config_cache.remember(&config.network_instance_id, observed_config); + Ok::<(), anyhow::Error>(()) + } + .await; + tracing::info!( + user_id = ?round.user_id, + instance_id = %config.network_instance_id, + "Reconcile running web-source network instance: {:?}, user_token: {:?}", + ret, + round.req.user_token + ); + + if ret.is_ok() { + ConfigActionResult::Success + } else { + runtime_config_cache.forget(&config.network_instance_id); + ConfigActionResult::Failed + } +} + +async fn run_missing_network_config( + session_data: &std::sync::Weak>, + rpc_client: &mut SessionRpcClient, + round: &ReconcileRound, + config: &crate::db::entity::user_running_network_configs::Model, + desired_config: NetworkConfig, +) -> ConfigActionResult { + if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await { + tracing::debug!( + machine_id = ?round.machine_id, + instance_id = %config.network_instance_id, + "skip run network instance because webhook session is no longer current" + ); + return ConfigActionResult::StopRound; + } + + let ret = rpc_client + .run_network_instance( + BaseController::default(), + RunNetworkInstanceRequest { + inst_id: Some(config.network_instance_id.clone().into()), + config: Some(desired_config), + overwrite: false, + source: PersistedConfigSource::from_db(&config.source).auto_run_rpc_source() as i32, + }, + ) + .await; + tracing::info!( + user_id = ?round.user_id, + "Run network instance: {:?}, user_token: {:?}", + ret, + round.req.user_token + ); + + if ret.is_ok() { + ConfigActionResult::Success + } else { + ConfigActionResult::Failed + } +} + +async fn remember_web_runtime_config_after_run( + rpc_client: &mut SessionRpcClient, + inst_id: &str, + desired_config: &NetworkConfig, + runtime_config_cache: &mut SessionRuntimeConfigCache, +) -> anyhow::Result<()> { + let observed_config = runtime_reconcile::get_runtime_config(rpc_client, inst_id).await?; + remember_if_runtime_matches_desired( + inst_id, + desired_config, + observed_config, + runtime_config_cache, + ) +} + +fn remember_if_runtime_matches_desired( + inst_id: &str, + desired_config: &NetworkConfig, + observed_config: NetworkConfig, + runtime_config_cache: &mut SessionRuntimeConfigCache, +) -> anyhow::Result<()> { + let action = runtime_reconcile::prepare_web_source_runtime_reconcile_from_current( + &observed_config, + desired_config.clone(), + )?; + if !matches!(action, runtime_reconcile::RuntimeReconcileAction::None) { + anyhow::bail!("runtime config still differs after managed run"); + } + runtime_config_cache.remember(inst_id, observed_config); + Ok(()) +} + +async fn mark_config_revision_applied_if_current( + session_data: &std::sync::Weak>, + storage: &StorageInner, + round: &ReconcileRound, + outcome: &ReconcileOutcome, +) -> RoundStatus<()> { + if outcome.managed_revision_failed || !round.should_apply_runtime_revision { + return RoundStatus::Ready(()); + } + + let current_target_config_revision = match storage + .db + .get_managed_config_revision((round.user_id, round.machine_id)) + .await + { + Ok(revision) => revision, + Err(e) => { + tracing::error!("Failed to verify managed config revision, error: {:?}", e); + return RoundStatus::Stop; + } + }; + if current_target_config_revision != round.target_config_revision { + return RoundStatus::Ready(()); + } + let Some(data) = session_data.upgrade() else { + return RoundStatus::Stop; + }; + let mut data = data.write().await; + if !SessionRpcService::runtime_heartbeat_is_current_locked(&data, &round.req) { + return RoundStatus::Ready(()); + } + data.applied_config_revision = round.target_config_revision.clone(); + + RoundStatus::Ready(()) +} + +#[cfg(test)] +mod tests { + use easytier::proto::api::manage::{NetworkingMethod, PortForwardConfig}; + + use super::*; + + fn config_with_port_forwards(port_forwards: Vec) -> NetworkConfig { + NetworkConfig { + instance_id: Some("11111111-1111-1111-1111-111111111111".to_string()), + dhcp: Some(true), + network_name: Some("managed".to_string()), + network_secret: Some("secret".to_string()), + networking_method: Some(NetworkingMethod::Manual as i32), + port_forwards, + ..Default::default() + } + } + + fn port_forward(bind_port: u32, dst_port: u32) -> PortForwardConfig { + PortForwardConfig { + bind_ip: "127.0.0.1".to_string(), + bind_port, + dst_ip: "10.144.0.1".to_string(), + dst_port, + proto: "tcp".to_string(), + } + } + + #[test] + fn session_runtime_config_cache_misses_unknown_instance() { + let cache = SessionRuntimeConfigCache::default(); + let action = cache + .plan("missing", config_with_port_forwards(Vec::new())) + .expect("prepare action"); + + assert!(action.is_none()); + } + + #[test] + fn session_runtime_config_cache_skips_matching_observed_config() { + let mut cache = SessionRuntimeConfigCache::default(); + let config = config_with_port_forwards(vec![port_forward(23000, 5174)]); + + cache.remember("managed", config.clone()); + let action = cache + .plan("managed", config) + .expect("prepare action") + .expect("cached action"); + + assert!(matches!( + action, + runtime_reconcile::RuntimeReconcileAction::None + )); + } + + #[test] + fn session_runtime_config_cache_plans_patch_from_observed_config() { + let mut cache = SessionRuntimeConfigCache::default(); + let current = config_with_port_forwards(vec![port_forward(23000, 5174)]); + let desired = + config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]); + + cache.remember("managed", current); + let action = cache + .plan("managed", desired) + .expect("prepare action") + .expect("cached action"); + + let runtime_reconcile::RuntimeReconcileAction::Patch(patch) = action else { + panic!("expected cached runtime config to produce hot patch"); + }; + assert_eq!(patch.port_forwards.len(), 1); + } + + #[test] + fn session_runtime_config_cache_retain_desired_removes_stale_entries() { + let mut cache = SessionRuntimeConfigCache::default(); + let config = config_with_port_forwards(Vec::new()); + cache.remember("keep", config.clone()); + cache.remember("drop", config); + + cache.retain_desired(&HashSet::from(["keep".to_string()])); + + assert!(cache.entries.contains_key("keep")); + assert!(!cache.entries.contains_key("drop")); + } + + #[test] + fn session_runtime_config_cache_forget_removes_observed_config() { + let mut cache = SessionRuntimeConfigCache::default(); + let config = config_with_port_forwards(Vec::new()); + cache.remember("managed", config.clone()); + + cache.forget("managed"); + + let action = cache + .plan("managed", config) + .expect("prepare action after remove"); + assert!(action.is_none()); + } + + #[test] + fn missing_run_remembers_observed_config_when_it_matches_desired() { + let mut cache = SessionRuntimeConfigCache::default(); + let config = config_with_port_forwards(vec![port_forward(23000, 5174)]); + + remember_if_runtime_matches_desired("managed", &config, config.clone(), &mut cache) + .expect("remember observed config after run"); + let action = cache + .plan("managed", config) + .expect("prepare action after run") + .expect("cached action"); + + assert!(matches!( + action, + runtime_reconcile::RuntimeReconcileAction::None + )); + } + + #[test] + fn missing_run_does_not_remember_observed_config_that_still_differs() { + let mut cache = SessionRuntimeConfigCache::default(); + let current = config_with_port_forwards(vec![port_forward(23000, 5174)]); + let desired = + config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]); + + let err = remember_if_runtime_matches_desired("managed", &desired, current, &mut cache) + .expect_err("expected stale run result not to be cached"); + + assert!( + err.to_string() + .contains("runtime config still differs after managed run") + ); + let action = cache + .plan("managed", desired) + .expect("prepare action after stale run result"); + assert!(action.is_none()); + } +} diff --git a/easytier-web/src/client_manager/session/webhook_validation.rs b/easytier-web/src/client_manager/session/webhook_validation.rs new file mode 100644 index 00000000..36b0ca1d --- /dev/null +++ b/easytier-web/src/client_manager/session/webhook_validation.rs @@ -0,0 +1,402 @@ +use std::{sync::Arc, time::Duration}; + +use anyhow::Context as _; +use easytier::proto::web::HeartbeatRequest; +use tokio::sync::RwLock; + +use super::{ + SessionAuthState, SessionData, SessionRpcService, WebhookConnectNotification, + WebhookDisconnectNotification, send_webhook_connection_transition, +}; +use crate::{ + client_manager::storage::{Storage, StorageToken}, + webhook::SharedWebhookConfig, +}; + +pub(super) const VALIDATION_RETRY_MS: u64 = 60_000; + +pub(super) struct WebhookHeartbeatValidation { + pub(super) config_revision: String, + pub(super) binding_version: u64, +} + +pub(super) struct WebhookValidationInput { + pub(super) storage: Storage, + pub(super) webhook_config: SharedWebhookConfig, + pub(super) client_url: url::Url, + pub(super) applied_config_revision: Option, + pub(super) req: HeartbeatRequest, + pub(super) machine_id: uuid::Uuid, +} + +fn deterministic_machine_delay(machine_id: uuid::Uuid, max_delay_ms: u64) -> Duration { + let delay_ms = (machine_id.as_u128() % u128::from(max_delay_ms + 1)) as u64; + Duration::from_millis(delay_ms) +} + +pub(super) fn retry_delay(machine_id: uuid::Uuid) -> Duration { + Duration::from_millis(VALIDATION_RETRY_MS) + + deterministic_machine_delay(machine_id, VALIDATION_RETRY_MS) +} + +async fn request_heartbeat_validation( + webhook_config: &crate::webhook::WebhookConfig, + client_url: &url::Url, + persisted_config_revision: Option<&str>, + applied_config_revision: Option<&str>, + req: &HeartbeatRequest, + machine_id: uuid::Uuid, +) -> anyhow::Result> { + let webhook_req = crate::webhook::ValidateTokenRequest { + token: req.user_token.clone(), + machine_id: machine_id.to_string(), + public_ip: client_url.host_str().map(str::to_string), + hostname: req.hostname.clone(), + version: req.easytier_version.clone(), + os_type: req.device_os.as_ref().map(|info| info.os_type.clone()), + os_version: req.device_os.as_ref().map(|info| info.version.clone()), + os_distribution: req.device_os.as_ref().map(|info| info.distribution.clone()), + web_instance_id: webhook_config.web_instance_id.clone(), + web_instance_api_base_url: webhook_config.web_instance_api_base_url.clone(), + persisted_config_revision: persisted_config_revision.map(str::to_string), + applied_config_revision: applied_config_revision.map(str::to_string), + }; + let resp = webhook_config + .validate_token(&webhook_req) + .await + .map_err(|e| anyhow::anyhow!("Webhook token validation failed: {:?}", e))?; + + if !resp.valid { + return Ok(None); + } + + Ok(Some(WebhookHeartbeatValidation { + config_revision: resp.config_revision, + binding_version: resp.binding_version, + })) +} + +async fn resolve_user_id(storage: &Storage, token: &str) -> anyhow::Result { + let user_id = match storage + .db() + .get_user_id_by_token(token) + .await + .map_err(|e| anyhow::anyhow!("DB error: {:?}", e))? + { + Some(id) => id, + None => storage + .auto_create_user(token) + .await + .with_context(|| format!("Failed to auto-create webhook user: {:?}", token))?, + }; + + Ok(user_id) +} + +async fn persisted_config_revision_for_token( + storage: &Storage, + token: &str, + machine_id: uuid::Uuid, +) -> anyhow::Result> { + let Some(user_id) = storage + .db() + .get_user_id_by_token(token) + .await + .map_err(|e| anyhow::anyhow!("DB error: {:?}", e))? + else { + return Ok(None); + }; + storage + .db() + .get_managed_config_revision((user_id, machine_id)) + .await + .map_err(|e| anyhow::anyhow!("DB error: {:?}", e)) +} + +async fn wait_for_input( + session_data: std::sync::Weak>, +) -> Option { + loop { + let notify = { + let session_data = session_data.upgrade()?; + let mut data = session_data.write().await; + if matches!(data.auth_state, SessionAuthState::Invalid) { + data.webhook_validation_dirty = false; + tracing::info!( + client_url = %data.client_url, + "webhook validation stopped for invalid session; reconnect is required before revalidation" + ); + return None; + } + if data.webhook_validation_dirty { + data.webhook_validation_dirty = false; + let req = data.req.clone()?; + let machine_id = req.machine_id.map(Into::into)?; + let storage = Storage::try_from(data.storage.clone()).ok()?; + return Some(WebhookValidationInput { + storage, + webhook_config: data.webhook_config.clone(), + client_url: data.client_url.clone(), + applied_config_revision: data.applied_config_revision.clone(), + req, + machine_id, + }); + } + data.webhook_validation_notify.clone() + }; + notify.notified().await; + } +} + +pub(super) async fn run_worker(session_data: std::sync::Weak>) { + while let Some(input) = wait_for_input(session_data.clone()).await { + let machine_id = input.machine_id; + if let Err(error) = run_round(session_data.clone(), input).await { + tracing::warn!( + ?machine_id, + %error, + "webhook validation failed, will retry later" + ); + tokio::time::sleep(retry_delay(machine_id)).await; + mark_dirty_if_current(&session_data, machine_id).await; + } + } +} + +pub(super) async fn run_round( + session_data: std::sync::Weak>, + input: WebhookValidationInput, +) -> anyhow::Result<()> { + let persisted_config_revision = persisted_config_revision_for_token( + &input.storage, + &input.req.user_token, + input.machine_id, + ) + .await?; + let validation = request_heartbeat_validation( + &input.webhook_config, + &input.client_url, + persisted_config_revision.as_deref(), + input.applied_config_revision.as_deref(), + &input.req, + input.machine_id, + ) + .await?; + + let Some(validation) = validation else { + apply_rejected(&session_data, &input).await; + return Ok(()); + }; + + let user_id = resolve_user_id(&input.storage, &input.req.user_token).await?; + apply_success(&session_data, input, validation, user_id).await; + Ok(()) +} + +async fn mark_dirty_if_current( + session_data: &std::sync::Weak>, + machine_id: uuid::Uuid, +) { + let Some(session_data) = session_data.upgrade() else { + return; + }; + let notify = { + let mut data = session_data.write().await; + let Some(req) = data.req.as_ref() else { + return; + }; + if req.machine_id.map(uuid::Uuid::from) != Some(machine_id) { + return; + } + if matches!(data.auth_state, SessionAuthState::Invalid) { + data.webhook_validation_dirty = false; + tracing::debug!( + %machine_id, + "skip webhook validation retry for invalid session" + ); + return; + } + SessionRpcService::mark_webhook_validation_dirty_locked(&mut data) + }; + notify.notify_one(); +} + +pub(super) async fn apply_rejected( + session_data: &std::sync::Weak>, + input: &WebhookValidationInput, +) { + let Some(session_data) = session_data.upgrade() else { + return; + }; + let (storage_token, disconnect_notification) = { + let mut data = session_data.write().await; + if !data.req.as_ref().is_some_and(|req| { + SessionRpcService::heartbeat_matches_identity( + req, + &input.req.user_token, + input.machine_id, + ) + }) { + return; + } + tracing::info!( + machine_id = %input.machine_id, + client_url = %data.client_url, + "webhook token rejected; marking session invalid and requiring client reconnect" + ); + data.auth_state = SessionAuthState::Invalid; + data.webhook_validation_dirty = false; + data.binding_version = None; + data.applied_config_revision = None; + let storage_token = data.storage_token.clone(); + let disconnect_notification = storage_token.as_ref().and_then(|storage_token| { + data.webhook_connected_binding_version + .take() + .map(|binding_version| WebhookDisconnectNotification { + webhook: data.webhook_config.clone(), + storage_token: storage_token.clone(), + binding_version, + }) + }); + (storage_token, disconnect_notification) + }; + if let Some(storage_token) = storage_token { + let report_time = SessionRpcService::heartbeat_report_timestamp(&input.req); + input + .storage + .update_client(storage_token, report_time, false); + } + if disconnect_notification.is_some() { + wait_webhook_connection_transition( + Arc::downgrade(&session_data), + disconnect_notification, + None, + ) + .await; + } +} + +pub(super) async fn apply_success( + session_data: &std::sync::Weak>, + input: WebhookValidationInput, + validation: WebhookHeartbeatValidation, + user_id: i32, +) { + let WebhookHeartbeatValidation { + config_revision: _, + binding_version, + } = validation; + + let Some(session_data) = session_data.upgrade() else { + return; + }; + let (storage_token, notifier, disconnect_notification, connect_notification, runtime_req) = { + let mut data = session_data.write().await; + let Some(runtime_req) = data.req.clone() else { + return; + }; + if !SessionRpcService::heartbeat_matches_identity( + &runtime_req, + &input.req.user_token, + input.machine_id, + ) { + return; + } + if matches!(data.auth_state, SessionAuthState::Invalid) { + tracing::info!( + machine_id = %input.machine_id, + client_url = %data.client_url, + "ignore webhook validation success for invalid session; reconnect is required before revalidation" + ); + return; + } + + let previous_connected_binding_version = data.webhook_connected_binding_version; + let client_url = data.client_url.clone(); + let storage_token = data.storage_token.get_or_insert_with(|| StorageToken { + token: runtime_req.user_token.clone(), + client_url, + machine_id: input.machine_id, + user_id, + }); + let storage_token = storage_token.clone(); + data.auth_state = SessionAuthState::Authorized; + data.binding_version = Some(binding_version); + let should_notify_connected = previous_connected_binding_version != Some(binding_version); + let disconnect_notification = previous_connected_binding_version + .filter(|previous_binding_version| *previous_binding_version != binding_version) + .map(|previous_binding_version| { + data.webhook_connected_binding_version = None; + WebhookDisconnectNotification { + webhook: data.webhook_config.clone(), + storage_token: storage_token.clone(), + binding_version: previous_binding_version, + } + }); + + let connect_notification = should_notify_connected.then(|| WebhookConnectNotification { + webhook: data.webhook_config.clone(), + storage_token: storage_token.clone(), + binding_version, + req: crate::webhook::NodeConnectedRequest { + machine_id: input.machine_id.to_string(), + token: runtime_req.user_token.clone(), + user_id: Some(user_id), + hostname: runtime_req.hostname.clone(), + version: runtime_req.easytier_version.clone(), + os_type: runtime_req + .device_os + .as_ref() + .map(|info| info.os_type.clone()), + os_version: runtime_req + .device_os + .as_ref() + .map(|info| info.version.clone()), + os_distribution: runtime_req + .device_os + .as_ref() + .map(|info| info.distribution.clone()), + web_instance_id: data.webhook_config.web_instance_id.clone(), + binding_version: Some(binding_version), + }, + }); + + ( + storage_token, + data.notifier.clone(), + disconnect_notification, + connect_notification, + runtime_req, + ) + }; + + if disconnect_notification.is_some() || connect_notification.is_some() { + wait_webhook_connection_transition( + Arc::downgrade(&session_data), + disconnect_notification, + connect_notification, + ) + .await; + } + + let report_time = SessionRpcService::heartbeat_report_timestamp(&runtime_req); + input + .storage + .update_client(storage_token, report_time, true); + let _ = notifier.send(runtime_req); +} + +async fn wait_webhook_connection_transition( + session_data: std::sync::Weak>, + disconnect: Option, + connect: Option, +) { + let transition = tokio::spawn(send_webhook_connection_transition( + session_data, + disconnect, + connect, + )); + if let Err(error) = transition.await { + tracing::warn!(%error, "webhook connection transition task failed"); + } +} diff --git a/easytier-web/src/client_manager/storage.rs b/easytier-web/src/client_manager/storage.rs index 9a8ba0d5..297d8666 100644 --- a/easytier-web/src/client_manager/storage.rs +++ b/easytier-web/src/client_manager/storage.rs @@ -17,6 +17,7 @@ pub struct StorageToken { struct ClientInfo { storage_token: StorageToken, report_time: i64, + authorized: bool, } #[derive(Debug)] @@ -55,7 +56,19 @@ impl Storage { fn update_client_info_map(map: &DashMap, client_info: &ClientInfo) { map.entry(client_info.storage_token.machine_id) .and_modify(|e| { - if e.report_time < client_info.report_time { + let same_client = e.storage_token.client_url + == client_info.storage_token.client_url + && e.storage_token.user_id == client_info.storage_token.user_id; + let should_replace = if (same_client && e.authorized != client_info.authorized) + || (!e.authorized && client_info.authorized) + { + true + } else if e.authorized && !client_info.authorized && !same_client { + false + } else { + e.report_time < client_info.report_time + }; + if should_replace { assert_eq!( e.storage_token.machine_id, client_info.storage_token.machine_id @@ -66,12 +79,13 @@ impl Storage { .or_insert(client_info.clone()); } - pub fn update_client(&self, stoken: StorageToken, report_time: i64) { + pub fn update_client(&self, stoken: StorageToken, report_time: i64, authorized: bool) { let inner = self.0.user_clients_map.entry(stoken.user_id).or_default(); let client_info = ClientInfo { storage_token: stoken.clone(), report_time, + authorized, }; Self::update_client_info_map(&inner, &client_info); } @@ -93,11 +107,21 @@ impl Storage { &self, user_id: UserIdInDb, machine_id: &uuid::Uuid, + ) -> Option { + self.get_client_url_by_machine_id_with_auth(user_id, machine_id, true) + } + + pub fn get_client_url_by_machine_id_with_auth( + &self, + user_id: UserIdInDb, + machine_id: &uuid::Uuid, + require_authorized: bool, ) -> Option { self.0.user_clients_map.get(&user_id).and_then(|info_map| { - info_map - .get(machine_id) - .map(|info| info.storage_token.client_url.clone()) + info_map.get(machine_id).and_then(|info| { + (!require_authorized || info.authorized) + .then(|| info.storage_token.client_url.clone()) + }) }) } @@ -108,6 +132,7 @@ impl Storage { .map(|info_map| { info_map .iter() + .filter(|info| info.value().authorized) .map(|info| info.value().storage_token.client_url.clone()) .collect() }) @@ -115,6 +140,14 @@ impl Storage { } pub fn list_clients(&self) -> Vec { + self.list_clients_with_auth(true) + } + + pub fn list_all_clients(&self) -> Vec { + self.list_clients_with_auth(false) + } + + fn list_clients_with_auth(&self, require_authorized: bool) -> Vec { self.0 .user_clients_map .iter() @@ -122,6 +155,7 @@ impl Storage { user_clients .value() .iter() + .filter(|info| !require_authorized || info.value().authorized) .map(|info| info.value().storage_token.clone()) .collect::>() }) @@ -164,8 +198,8 @@ mod tests { let user1_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001"); let user2_token = make_storage_token(2, machine_id, "tcp://127.0.0.1:1002"); - storage.update_client(user1_token.clone(), 10); - storage.update_client(user2_token.clone(), 20); + storage.update_client(user1_token.clone(), 10, true); + storage.update_client(user2_token.clone(), 20, true); assert_eq!( storage.get_client_url_by_machine_id(1, &machine_id), @@ -195,8 +229,8 @@ mod tests { let user1_token = make_storage_token(1, uuid::Uuid::new_v4(), "tcp://127.0.0.1:1001"); let user2_token = make_storage_token(2, uuid::Uuid::new_v4(), "tcp://127.0.0.1:1002"); - storage.update_client(user1_token.clone(), 10); - storage.update_client(user2_token.clone(), 20); + storage.update_client(user1_token.clone(), 10, true); + storage.update_client(user2_token.clone(), 20, true); let tokens = storage.list_clients(); assert_eq!(tokens.len(), 2); @@ -209,4 +243,84 @@ mod tests { assert_eq!(tokens.len(), 1); assert_eq!(tokens[0].token, user2_token.token); } + + #[tokio::test] + async fn pending_client_is_listed_but_not_authorized_for_machine_lookup() { + let storage = Storage::new(Db::memory_db().await); + let machine_id = uuid::Uuid::new_v4(); + let token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001"); + + storage.update_client(token.clone(), 10, false); + + assert_eq!(storage.list_clients().len(), 0); + assert_eq!(storage.list_all_clients().len(), 1); + assert_eq!(storage.list_user_clients(1), Vec::::new()); + assert_eq!(storage.get_client_url_by_machine_id(1, &machine_id), None); + assert_eq!( + storage.get_client_url_by_machine_id_with_auth(1, &machine_id, false), + Some(token.client_url.clone()) + ); + + storage.update_client(token.clone(), 11, true); + + assert_eq!( + storage.get_client_url_by_machine_id(1, &machine_id), + Some(token.client_url.clone()) + ); + + storage.update_client(token.clone(), 11, false); + + assert_eq!(storage.get_client_url_by_machine_id(1, &machine_id), None); + assert_eq!(storage.list_clients().len(), 0); + assert_eq!(storage.list_all_clients().len(), 1); + } + + #[tokio::test] + async fn stale_client_authorization_update_does_not_replace_newer_client() { + let storage = Storage::new(Db::memory_db().await); + let machine_id = uuid::Uuid::new_v4(); + let old_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001"); + let new_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002"); + + storage.update_client(old_token.clone(), 10, true); + storage.update_client(new_token.clone(), 20, true); + storage.update_client(old_token, 10, false); + + assert_eq!( + storage.get_client_url_by_machine_id(1, &machine_id), + Some(new_token.client_url) + ); + } + + #[tokio::test] + async fn pending_client_does_not_replace_authorized_route() { + let storage = Storage::new(Db::memory_db().await); + let machine_id = uuid::Uuid::new_v4(); + let authorized_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001"); + let pending_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002"); + + storage.update_client(authorized_token.clone(), 10, true); + storage.update_client(pending_token, i64::MAX, false); + + assert_eq!( + storage.get_client_url_by_machine_id(1, &machine_id), + Some(authorized_token.client_url) + ); + } + + #[tokio::test] + async fn authorized_client_replaces_pending_route_regardless_of_report_time() { + let storage = Storage::new(Db::memory_db().await); + let machine_id = uuid::Uuid::new_v4(); + let pending_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001"); + let authorized_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002"); + + storage.update_client(pending_token, i64::MAX, false); + storage.update_client(authorized_token.clone(), 10, true); + + assert_eq!( + storage.get_client_url_by_machine_id(1, &machine_id), + Some(authorized_token.client_url) + ); + } } diff --git a/easytier-web/src/db/entity/managed_config_revisions.rs b/easytier-web/src/db/entity/managed_config_revisions.rs new file mode 100644 index 00000000..b0cd00c7 --- /dev/null +++ b/easytier-web/src/db/entity/managed_config_revisions.rs @@ -0,0 +1,38 @@ +//! `SeaORM` Entity, hand-written to match the generated entity style. + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "managed_config_revisions")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub user_id: i32, + #[sea_orm(column_type = "Text")] + pub device_id: String, + #[sea_orm(column_type = "Text")] + pub config_revision: String, + pub create_time: DateTimeWithTimeZone, + pub update_time: DateTimeWithTimeZone, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::users::Entity", + from = "Column::UserId", + to = "super::users::Column::Id", + on_update = "Cascade", + on_delete = "Cascade" + )] + Users, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Users.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/easytier-web/src/db/entity/mod.rs b/easytier-web/src/db/entity/mod.rs index 1cda5cf8..e3ff4fd6 100644 --- a/easytier-web/src/db/entity/mod.rs +++ b/easytier-web/src/db/entity/mod.rs @@ -4,6 +4,7 @@ pub mod prelude; pub mod groups; pub mod groups_permissions; +pub mod managed_config_revisions; pub mod permissions; pub mod tower_sessions; pub mod user_running_network_configs; diff --git a/easytier-web/src/db/entity/prelude.rs b/easytier-web/src/db/entity/prelude.rs index 81917839..fb897cd9 100644 --- a/easytier-web/src/db/entity/prelude.rs +++ b/easytier-web/src/db/entity/prelude.rs @@ -2,6 +2,7 @@ pub use super::groups::Entity as Groups; pub use super::groups_permissions::Entity as GroupsPermissions; +pub use super::managed_config_revisions::Entity as ManagedConfigRevisions; pub use super::permissions::Entity as Permissions; pub use super::tower_sessions::Entity as TowerSessions; pub use super::user_running_network_configs::Entity as UserRunningNetworkConfigs; diff --git a/easytier-web/src/db/mod.rs b/easytier-web/src/db/mod.rs index 33cb3c8e..a07de22c 100644 --- a/easytier-web/src/db/mod.rs +++ b/easytier-web/src/db/mod.rs @@ -141,6 +141,110 @@ impl Db { ) -> Result, DbErr> { self.get_user_id(token).await } + + pub async fn get_managed_config_revision( + &self, + (user_id, device_id): (UserIdInDb, Uuid), + ) -> Result, DbErr> { + use entity::managed_config_revisions as mcr; + + let revision = mcr::Entity::find() + .filter(mcr::Column::UserId.eq(user_id)) + .filter(mcr::Column::DeviceId.eq(device_id.to_string())) + .one(self.orm_db()) + .await?; + + Ok(revision.map(|row| row.config_revision)) + } + + pub async fn set_managed_config_revision( + &self, + (user_id, device_id): (UserIdInDb, Uuid), + config_revision: &str, + ) -> Result<(), DbErr> { + use entity::managed_config_revisions as mcr; + + let now = chrono::Local::now().fixed_offset(); + let on_conflict = OnConflict::columns([mcr::Column::UserId, mcr::Column::DeviceId]) + .update_columns([mcr::Column::ConfigRevision, mcr::Column::UpdateTime]) + .to_owned(); + let insert_m = mcr::ActiveModel { + user_id: Set(user_id), + device_id: Set(device_id.to_string()), + config_revision: Set(config_revision.to_string()), + create_time: Set(now), + update_time: Set(now), + ..Default::default() + }; + + mcr::Entity::insert(insert_m) + .on_conflict(on_conflict) + .do_nothing() + .exec(self.orm_db()) + .await?; + Ok(()) + } + + pub async fn insert_or_update_web_network_config( + &self, + (user_id, device_id): (UserIdInDb, Uuid), + network_inst_id: Uuid, + network_config: NetworkConfig, + ) -> Result { + let now = chrono::Local::now().fixed_offset(); + let network_config = + serde_json::to_string(&network_config).map_err(|e| DbErr::Json(e.to_string()))?; + let source = ConfigSource::Web.as_str(); + let result = sqlx::query( + r#" + INSERT INTO user_running_network_configs ( + user_id, device_id, network_instance_id, network_config, + source, disabled, create_time, update_time + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(user_id, device_id, network_instance_id) DO UPDATE SET + network_config = excluded.network_config, + source = excluded.source, + disabled = excluded.disabled, + update_time = excluded.update_time + WHERE user_running_network_configs.source = ? + "#, + ) + .bind(user_id) + .bind(device_id.to_string()) + .bind(network_inst_id.to_string()) + .bind(network_config) + .bind(source) + .bind(false) + .bind(now) + .bind(now) + .bind(source) + .execute(&self.db) + .await + .map_err(|e| DbErr::Custom(e.to_string()))?; + + Ok(result.rows_affected() > 0) + } + + pub async fn delete_web_network_configs( + &self, + (user_id, device_id): (UserIdInDb, Uuid), + network_inst_ids: &[Uuid], + ) -> Result<(), DbErr> { + use entity::user_running_network_configs as urnc; + + urnc::Entity::delete_many() + .filter(urnc::Column::UserId.eq(user_id)) + .filter(urnc::Column::DeviceId.eq(device_id.to_string())) + .filter(urnc::Column::Source.eq(ConfigSource::Web.as_str())) + .filter( + urnc::Column::NetworkInstanceId + .is_in(network_inst_ids.iter().map(|id| id.to_string())), + ) + .exec(self.orm_db()) + .await?; + + Ok(()) + } } #[async_trait] @@ -468,4 +572,46 @@ mod tests { assert_eq!(device1_configs.len(), 1); assert_eq!(device2_configs.len(), 1); } + + #[tokio::test] + async fn test_web_network_config_does_not_replace_user_owned_config() { + let db = Db::memory_db().await; + let user_id = db.auto_create_user("user-web-race").await.unwrap().id; + let device_id = uuid::Uuid::new_v4(); + let inst_id = uuid::Uuid::new_v4(); + + db.insert_or_update_user_network_config( + (user_id, device_id), + inst_id, + NetworkConfig { + network_name: Some("user-owned".to_string()), + ..Default::default() + }, + ConfigSource::User, + ) + .await + .unwrap(); + + let updated = db + .insert_or_update_web_network_config( + (user_id, device_id), + inst_id, + NetworkConfig { + network_name: Some("web-owned".to_string()), + ..Default::default() + }, + ) + .await + .unwrap(); + + assert!(!updated); + let saved = db + .get_network_config((user_id, device_id), &inst_id.to_string()) + .await + .unwrap() + .unwrap(); + assert_eq!(saved.get_network_config_source(), ConfigSource::User); + let saved_config = saved.get_network_config().unwrap(); + assert_eq!(saved_config.network_name.as_deref(), Some("user-owned")); + } } diff --git a/easytier-web/src/main.rs b/easytier-web/src/main.rs index 4ea06100..6bddbf62 100644 --- a/easytier-web/src/main.rs +++ b/easytier-web/src/main.rs @@ -3,8 +3,8 @@ #[macro_use] extern crate rust_i18n; -use std::net::IpAddr; use std::sync::Arc; +use std::{net::IpAddr, time::Duration}; use clap::Parser; use easytier::tunnel::websocket::WsTunnelListener; @@ -113,6 +113,14 @@ struct Cli { )] geoip_db: Option, + #[arg( + long, + env = "ET_HEARTBEAT_MIN_RESPONSE_MS", + default_value = "0", + help = t!("cli.heartbeat_min_response_ms").to_string(), + )] + heartbeat_min_response_ms: u64, + #[cfg(feature = "embed")] #[arg( long, @@ -312,6 +320,7 @@ async fn main() { let mut mgr = client_manager::ClientManager::new( db.clone(), cli.geoip_db, + Duration::from_millis(cli.heartbeat_min_response_ms), feature_flags.clone(), webhook_config.clone(), ); diff --git a/easytier-web/src/migrator/m20260619_000005_managed_config_revisions.rs b/easytier-web/src/migrator/m20260619_000005_managed_config_revisions.rs new file mode 100644 index 00000000..b7e1473b --- /dev/null +++ b/easytier-web/src/migrator/m20260619_000005_managed_config_revisions.rs @@ -0,0 +1,46 @@ +use sea_orm_migration::prelude::*; + +pub struct Migration; + +impl MigrationName for Migration { + fn name(&self) -> &str { + "m20260619_000005_managed_config_revisions" + } +} + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .get_connection() + .execute_unprepared( + r#" + CREATE TABLE managed_config_revisions ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + user_id INTEGER NOT NULL, + device_id TEXT NOT NULL, + config_revision TEXT NOT NULL, + create_time TEXT NOT NULL, + update_time TEXT NOT NULL, + CONSTRAINT fk_managed_config_revisions_user_id_to_users_id + FOREIGN KEY (user_id) REFERENCES users(id) + ON DELETE CASCADE + ON UPDATE CASCADE + ); + + CREATE UNIQUE INDEX idx_managed_config_revisions_scope + ON managed_config_revisions(user_id, device_id); + "#, + ) + .await?; + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .get_connection() + .execute_unprepared("DROP TABLE managed_config_revisions;") + .await?; + Ok(()) + } +} diff --git a/easytier-web/src/migrator/mod.rs b/easytier-web/src/migrator/mod.rs index 6eea9f11..f1412282 100644 --- a/easytier-web/src/migrator/mod.rs +++ b/easytier-web/src/migrator/mod.rs @@ -4,6 +4,7 @@ mod m20241029_000001_init; mod m20260403_000002_scope_network_config_unique; mod m20260421_000003_add_network_config_source; mod m20260514_000004_rename_web_config_source; +mod m20260619_000005_managed_config_revisions; pub struct Migrator; @@ -15,6 +16,7 @@ impl MigratorTrait for Migrator { Box::new(m20260403_000002_scope_network_config_unique::Migration), Box::new(m20260421_000003_add_network_config_source::Migration), Box::new(m20260514_000004_rename_web_config_source::Migration), + Box::new(m20260619_000005_managed_config_revisions::Migration), ] } } diff --git a/easytier-web/src/restful/mod.rs b/easytier-web/src/restful/mod.rs index 91fc1996..f759e067 100644 --- a/easytier-web/src/restful/mod.rs +++ b/easytier-web/src/restful/mod.rs @@ -307,7 +307,7 @@ impl RestfulServer { async fn handle_list_all_sessions_internal( State(client_mgr): AppState, ) -> Result, HttpHandleError> { - let ret = client_mgr.list_sessions().await; + let ret = client_mgr.list_all_sessions().await; Ok(ListSessionJsonResp(ret).into()) } diff --git a/easytier-web/src/restful/network.rs b/easytier-web/src/restful/network.rs index 46c65aab..d437f122 100644 --- a/easytier-web/src/restful/network.rs +++ b/easytier-web/src/restful/network.rs @@ -93,6 +93,8 @@ struct ManagedNetworkConfigJson { #[derive(Debug, serde::Deserialize, serde::Serialize)] struct ReconcileManagedNetworkConfigsJsonReq { managed_network_configs: Vec, + config_revision: Option, + expected_config_revision: Option, } #[derive(Debug, serde::Deserialize, serde::Serialize)] @@ -357,13 +359,21 @@ impl NetworkApi { }) .collect(); client_mgr - .reconcile_managed_network_configs(user_id, machine_id, desired) + .reconcile_managed_network_configs( + user_id, + machine_id, + desired, + payload.config_revision, + payload.expected_config_revision, + ) .await .map_err(|err| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - other_error(err.to_string()).into(), - ) + let status = if crate::client_manager::is_managed_config_revision_conflict(&err) { + StatusCode::CONFLICT + } else { + StatusCode::INTERNAL_SERVER_ERROR + }; + (status, other_error(err.to_string()).into()) })?; Ok(Void::default().into()) } diff --git a/easytier-web/src/webhook.rs b/easytier-web/src/webhook.rs index ce65b10f..59e41b38 100644 --- a/easytier-web/src/webhook.rs +++ b/easytier-web/src/webhook.rs @@ -1,6 +1,248 @@ -use std::sync::Arc; +use std::{ + cmp::Ordering, + collections::VecDeque, + fmt, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; use serde::{Deserialize, Serialize}; +use tokio::sync::oneshot; + +const VALIDATE_TOKEN_INITIAL_CONCURRENCY: usize = 8; +const VALIDATE_TOKEN_MIN_CONCURRENCY: usize = 2; +const VALIDATE_TOKEN_MAX_CONCURRENCY: usize = 64; +const VALIDATE_TOKEN_ADJUST_WINDOW: Duration = Duration::from_secs(1); +const VALIDATE_TOKEN_SLOW_THRESHOLD: Duration = Duration::from_secs(2); +const WEBHOOK_HTTP_TIMEOUT: Duration = Duration::from_secs(10); + +struct AdaptiveValidateLimiter { + state: Mutex, +} + +impl fmt::Debug for AdaptiveValidateLimiter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AdaptiveValidateLimiter") + .field("state", &self.lock_state()) + .finish_non_exhaustive() + } +} + +struct AdaptiveValidateLimiterState { + limit: usize, + in_flight: usize, + waiters: VecDeque>, + window_started_at: Instant, + samples: usize, + slow_samples: usize, + failures: usize, + had_queue: bool, +} + +impl fmt::Debug for AdaptiveValidateLimiterState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AdaptiveValidateLimiterState") + .field("limit", &self.limit) + .field("in_flight", &self.in_flight) + .field("waiters", &self.waiters.len()) + .field("window_started_at", &self.window_started_at) + .field("samples", &self.samples) + .field("slow_samples", &self.slow_samples) + .field("failures", &self.failures) + .field("had_queue", &self.had_queue) + .finish() + } +} + +struct AdaptiveValidatePermit { + limiter: Arc, + started_at: Instant, + completed: bool, +} + +struct AdaptiveValidateGrant { + limiter: Arc, + active: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LimitAdjustment { + Unchanged, + Increased, + Decreased, +} + +impl AdaptiveValidateLimiter { + fn new() -> Arc { + Arc::new(Self { + state: Mutex::new(AdaptiveValidateLimiterState::new(Instant::now())), + }) + } + + async fn acquire(self: &Arc) -> AdaptiveValidatePermit { + loop { + let receiver = { + let mut state = self.lock_state(); + state.complete_window_if_due(Instant::now()); + if state.waiters.is_empty() && state.in_flight < state.limit { + state.in_flight += 1; + return AdaptiveValidatePermit::new(self.clone()); + } + + let (sender, receiver) = oneshot::channel(); + state.had_queue = true; + state.waiters.push_back(sender); + self.grant_waiters(&mut state); + receiver + }; + + if let Ok(grant) = receiver.await { + return grant.into_permit(); + } + } + } + + fn grant_waiters(self: &Arc, state: &mut AdaptiveValidateLimiterState) { + while state.in_flight < state.limit { + let Some(waiter) = state.waiters.pop_front() else { + break; + }; + state.in_flight += 1; + if let Err(mut grant) = waiter.send(AdaptiveValidateGrant::new(self.clone())) { + grant.disarm(); + state.in_flight -= 1; + } + } + } + + fn record_sample(self: &Arc, elapsed: Duration, success: bool) { + let mut state = self.lock_state(); + let adjustment = state.record_sample(Instant::now(), elapsed, success); + if adjustment == LimitAdjustment::Increased { + self.grant_waiters(&mut state); + } + } + + fn release_slot(self: &Arc) { + let mut state = self.lock_state(); + state.in_flight = state.in_flight.saturating_sub(1); + self.grant_waiters(&mut state); + } + + fn lock_state(&self) -> std::sync::MutexGuard<'_, AdaptiveValidateLimiterState> { + self.state + .lock() + .expect("adaptive validate limiter state should not be poisoned") + } +} + +impl AdaptiveValidateLimiterState { + fn new(now: Instant) -> Self { + Self { + limit: VALIDATE_TOKEN_INITIAL_CONCURRENCY, + in_flight: 0, + waiters: VecDeque::new(), + window_started_at: now, + samples: 0, + slow_samples: 0, + failures: 0, + had_queue: false, + } + } + + fn record_sample(&mut self, now: Instant, elapsed: Duration, success: bool) -> LimitAdjustment { + self.samples += 1; + if elapsed > VALIDATE_TOKEN_SLOW_THRESHOLD { + self.slow_samples += 1; + } + if !success { + self.failures += 1; + } + self.complete_window_if_due(now) + } + + fn complete_window_if_due(&mut self, now: Instant) -> LimitAdjustment { + if now.duration_since(self.window_started_at) < VALIDATE_TOKEN_ADJUST_WINDOW { + return LimitAdjustment::Unchanged; + } + + let old_limit = self.limit; + if self.samples > 0 { + if self.failures > 0 || self.is_p95_slow() { + self.limit = (self.limit / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY); + } else if self.had_queue { + self.limit = (self.limit + 1).min(VALIDATE_TOKEN_MAX_CONCURRENCY); + } + } + + self.window_started_at = now; + self.samples = 0; + self.slow_samples = 0; + self.failures = 0; + self.had_queue = false; + + match self.limit.cmp(&old_limit) { + Ordering::Greater => LimitAdjustment::Increased, + Ordering::Less => LimitAdjustment::Decreased, + Ordering::Equal => LimitAdjustment::Unchanged, + } + } + + fn is_p95_slow(&self) -> bool { + self.slow_samples > 0 && self.slow_samples * 20 >= self.samples + } +} + +impl AdaptiveValidatePermit { + fn new(limiter: Arc) -> Self { + Self { + limiter, + started_at: Instant::now(), + completed: false, + } + } + + fn complete(mut self, success: bool) { + self.limiter + .record_sample(self.started_at.elapsed(), success); + self.completed = true; + } +} + +impl AdaptiveValidateGrant { + fn new(limiter: Arc) -> Self { + Self { + limiter, + active: true, + } + } + + fn into_permit(mut self) -> AdaptiveValidatePermit { + self.active = false; + AdaptiveValidatePermit::new(self.limiter.clone()) + } + + fn disarm(&mut self) { + self.active = false; + } +} + +impl Drop for AdaptiveValidateGrant { + fn drop(&mut self) { + if self.active { + self.limiter.release_slot(); + } + } +} + +impl Drop for AdaptiveValidatePermit { + fn drop(&mut self) { + if !self.completed { + self.limiter.record_sample(self.started_at.elapsed(), false); + } + self.limiter.release_slot(); + } +} /// Webhook configuration for external integrations. #[derive(Debug, Clone)] @@ -11,6 +253,7 @@ pub struct WebhookConfig { pub web_instance_id: Option, pub web_instance_api_base_url: Option, + validate_limiter: Arc, client: reqwest::Client, } @@ -28,7 +271,11 @@ impl WebhookConfig { internal_auth_token, web_instance_id, web_instance_api_base_url, - client: reqwest::Client::new(), + validate_limiter: AdaptiveValidateLimiter::new(), + client: reqwest::Client::builder() + .timeout(WEBHOOK_HTTP_TIMEOUT) + .build() + .expect("webhook HTTP client should be valid"), } } @@ -58,6 +305,8 @@ pub struct ValidateTokenRequest { pub web_instance_id: Option, pub web_instance_api_base_url: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub persisted_config_revision: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub applied_config_revision: Option, } @@ -69,7 +318,6 @@ pub struct ValidateTokenResponse { #[serde(default)] pub binding_version: u64, #[serde(default)] - pub managed_network_configs: Option>, pub config_revision: String, } @@ -125,21 +373,40 @@ impl WebhookConfig { pub async fn validate_token( &self, req: &ValidateTokenRequest, + ) -> anyhow::Result { + self.validate_token_with_http_timeout(req, WEBHOOK_HTTP_TIMEOUT) + .await + } + + async fn validate_token_with_http_timeout( + &self, + req: &ValidateTokenRequest, + http_timeout: Duration, ) -> anyhow::Result { let url = self.webhook_endpoint("validate-token")?; - let resp = self - .client - .post(&url) - .header("X-Internal-Auth", self.webhook_auth_secret()) - .json(req) - .send() - .await?; + let permit = self.validate_limiter.acquire().await; + let ret = match tokio::time::timeout(http_timeout, async { + let resp = self + .client + .post(&url) + .header("X-Internal-Auth", self.webhook_auth_secret()) + .json(req) + .send() + .await?; - if !resp.status().is_success() { - anyhow::bail!("webhook validate-token returned status {}", resp.status()); - } + if !resp.status().is_success() { + anyhow::bail!("webhook validate-token returned status {}", resp.status()); + } - Ok(resp.json().await?) + Ok(resp.json().await?) + }) + .await + { + Ok(ret) => ret, + Err(_) => Err(anyhow::anyhow!("webhook validate-token timed out")), + }; + permit.complete(ret.is_ok()); + ret } /// Notify the webhook receiver that a node has connected. @@ -191,13 +458,319 @@ pub type SharedWebhookConfig = Arc; #[cfg(test)] mod tests { use super::*; + use axum::{Json, Router, routing::post}; + use serde_json::json; #[test] - fn validate_token_response_allows_missing_managed_configs() { + fn adaptive_validate_limiter_increases_under_queue_pressure() { + let now = Instant::now(); + let mut state = AdaptiveValidateLimiterState::new(now); + state.had_queue = true; + + for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY { + state.record_sample(now, Duration::from_millis(50), true); + } + + assert_eq!( + state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW), + LimitAdjustment::Increased + ); + assert_eq!(state.limit, VALIDATE_TOKEN_INITIAL_CONCURRENCY + 1); + } + + #[test] + fn adaptive_validate_limiter_does_not_increase_without_queue_pressure() { + let now = Instant::now(); + let mut state = AdaptiveValidateLimiterState::new(now); + + for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY { + state.record_sample(now, Duration::from_millis(50), true); + } + + assert_eq!( + state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW), + LimitAdjustment::Unchanged + ); + assert_eq!(state.limit, VALIDATE_TOKEN_INITIAL_CONCURRENCY); + } + + #[test] + fn adaptive_validate_limiter_reduces_on_failure() { + let now = Instant::now(); + let mut state = AdaptiveValidateLimiterState::new(now); + + state.record_sample(now, Duration::from_millis(50), false); + + assert_eq!( + state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW), + LimitAdjustment::Decreased + ); + assert_eq!( + state.limit, + (VALIDATE_TOKEN_INITIAL_CONCURRENCY / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY) + ); + } + + #[test] + fn adaptive_validate_limiter_reduces_on_slow_latency() { + let now = Instant::now(); + let mut state = AdaptiveValidateLimiterState::new(now); + + state.record_sample( + now, + VALIDATE_TOKEN_SLOW_THRESHOLD + Duration::from_millis(1), + true, + ); + + assert_eq!( + state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW), + LimitAdjustment::Decreased + ); + assert_eq!( + state.limit, + (VALIDATE_TOKEN_INITIAL_CONCURRENCY / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY) + ); + } + + #[tokio::test] + async fn adaptive_validate_limiter_waiter_acquires_after_release() { + let limiter = AdaptiveValidateLimiter::new(); + let mut permits = Vec::new(); + for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY { + permits.push(limiter.acquire().await); + } + + let waiter_limiter = limiter.clone(); + let waiter = tokio::spawn(async move { + let permit = waiter_limiter.acquire().await; + permit.complete(true); + }); + + tokio::time::sleep(Duration::from_millis(10)).await; + assert!(!waiter.is_finished()); + + permits.pop().unwrap().complete(true); + tokio::time::timeout(Duration::from_secs(1), waiter) + .await + .unwrap() + .unwrap(); + + for permit in permits { + permit.complete(true); + } + } + + #[tokio::test] + async fn adaptive_validate_limiter_releases_when_permit_is_dropped() { + let limiter = AdaptiveValidateLimiter::new(); + let mut permits = Vec::new(); + for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY { + permits.push(limiter.acquire().await); + } + + let waiter_limiter = limiter.clone(); + let waiter = tokio::spawn(async move { + let permit = waiter_limiter.acquire().await; + permit.complete(true); + }); + + tokio::time::sleep(Duration::from_millis(10)).await; + assert!(!waiter.is_finished()); + + drop(permits.pop().unwrap()); + tokio::time::timeout(Duration::from_secs(1), waiter) + .await + .unwrap() + .unwrap(); + + for permit in permits { + permit.complete(true); + } + } + + #[tokio::test] + async fn adaptive_validate_limiter_skips_canceled_waiters() { + let limiter = AdaptiveValidateLimiter::new(); + let mut permits = Vec::new(); + for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY { + permits.push(limiter.acquire().await); + } + + let waiter_limiter = limiter.clone(); + let waiter = tokio::spawn(async move { + let permit = waiter_limiter.acquire().await; + permit.complete(true); + }); + + tokio::time::sleep(Duration::from_millis(10)).await; + assert!(!waiter.is_finished()); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + + permits.pop().unwrap().complete(true); + tokio::time::sleep(Duration::from_millis(10)).await; + + let state = limiter.lock_state(); + assert_eq!(state.samples, 1); + assert_eq!(state.failures, 0); + drop(state); + + for permit in permits { + permit.complete(true); + } + } + + #[test] + fn adaptive_validate_limiter_releases_dropped_grant_without_failure_sample() { + let limiter = AdaptiveValidateLimiter::new(); + { + let mut state = limiter.lock_state(); + state.in_flight = 1; + } + + drop(AdaptiveValidateGrant::new(limiter.clone())); + + let state = limiter.lock_state(); + assert_eq!(state.in_flight, 0); + assert_eq!(state.samples, 0); + assert_eq!(state.failures, 0); + } + + #[tokio::test] + async fn adaptive_validate_limiter_wakes_multiple_waiters_in_order() { + let limiter = AdaptiveValidateLimiter::new(); + let mut permits = Vec::new(); + for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY { + permits.push(limiter.acquire().await); + } + + let (first_acquired_tx, first_acquired_rx) = oneshot::channel(); + let (first_release_tx, first_release_rx) = oneshot::channel(); + let first = { + let limiter = limiter.clone(); + tokio::spawn(async move { + let permit = limiter.acquire().await; + first_acquired_tx.send(()).unwrap(); + first_release_rx.await.unwrap(); + permit.complete(true); + }) + }; + let (second_acquired_tx, mut second_acquired_rx) = oneshot::channel(); + let second = { + let limiter = limiter.clone(); + tokio::spawn(async move { + let permit = limiter.acquire().await; + second_acquired_tx.send(()).unwrap(); + permit.complete(true); + }) + }; + + tokio::time::sleep(Duration::from_millis(10)).await; + assert!(!first.is_finished()); + assert!(!second.is_finished()); + + permits.pop().unwrap().complete(true); + tokio::time::timeout(Duration::from_secs(1), first_acquired_rx) + .await + .unwrap() + .unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut second_acquired_rx) + .await + .is_err() + ); + + first_release_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(1), first) + .await + .unwrap() + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), &mut second_acquired_rx) + .await + .unwrap() + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), second) + .await + .unwrap() + .unwrap(); + + for permit in permits { + permit.complete(true); + } + } + + #[tokio::test] + async fn validate_token_http_timeout_starts_after_limiter_permit() { + let app = Router::new().route( + "/validate-token", + post(|| async { + Json(json!({ + "valid": true, + "config_revision": "rev-1" + })) + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + let webhook = WebhookConfig::new(Some(format!("http://{addr}")), None, None, None, None); + let mut permits = Vec::new(); + for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY { + permits.push(webhook.validate_limiter.acquire().await); + } + + let validate_webhook = webhook.clone(); + let validate = tokio::spawn(async move { + let req = ValidateTokenRequest { + token: "token".to_string(), + machine_id: uuid::Uuid::new_v4().to_string(), + public_ip: None, + hostname: String::new(), + version: String::new(), + os_type: None, + os_version: None, + os_distribution: None, + web_instance_id: None, + web_instance_api_base_url: None, + persisted_config_revision: None, + applied_config_revision: None, + }; + validate_webhook + .validate_token_with_http_timeout(&req, Duration::from_millis(20)) + .await + }); + + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!validate.is_finished()); + + permits.pop().unwrap().complete(true); + let resp = tokio::time::timeout(Duration::from_secs(1), validate) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(resp.valid); + + for permit in permits { + permit.complete(true); + } + server.abort(); + } + + #[test] + fn validate_token_response_deserializes_config_revision() { let resp: ValidateTokenResponse = serde_json::from_str(r#"{"valid":true,"config_revision":"rev-1"}"#).unwrap(); assert!(resp.valid); assert_eq!(resp.config_revision, "rev-1"); - assert!(resp.managed_network_configs.is_none()); + } + + #[test] + fn validate_token_response_allows_missing_config_revision() { + let resp: ValidateTokenResponse = serde_json::from_str(r#"{"valid":true}"#).unwrap(); + assert!(resp.valid); + assert!(resp.config_revision.is_empty()); } } diff --git a/easytier/src/instance/instance.rs b/easytier/src/instance/instance.rs index d6c2f156..22e22018 100644 --- a/easytier/src/instance/instance.rs +++ b/easytier/src/instance/instance.rs @@ -483,18 +483,22 @@ impl InstanceConfigPatcher { } let global_ctx = weak_upgrade(&self.global_ctx)?; for proxy_network_patch in proxy_networks { - let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else { - tracing::warn!("Proxy network cidr is None, skipping."); - continue; - }; - let mapped_cidr: Option = - proxy_network_patch.mapped_cidr.map(|s| s.into()); match ConfigPatchAction::try_from(proxy_network_patch.action) { Ok(ConfigPatchAction::Add) => { + let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else { + tracing::warn!("Proxy network cidr is None, skipping add."); + continue; + }; + let mapped_cidr: Option = + proxy_network_patch.mapped_cidr.map(|s| s.into()); tracing::info!("Proxy network added: {}", cidr); global_ctx.config.add_proxy_cidr(cidr, mapped_cidr)?; } Ok(ConfigPatchAction::Remove) => { + let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else { + tracing::warn!("Proxy network cidr is None, skipping remove."); + continue; + }; tracing::info!("Proxy network removed: {}", cidr); global_ctx.config.remove_proxy_cidr(cidr); } diff --git a/easytier/src/launcher.rs b/easytier/src/launcher.rs index 3c26b18c..127fd254 100644 --- a/easytier/src/launcher.rs +++ b/easytier/src/launcher.rs @@ -1116,6 +1116,7 @@ impl NetworkConfig { .get_credential_file() .map(|path| path.to_string_lossy().into_owned()); let flags = config.get_flags(); + let default_flags = default_config.get_flags(); result.latency_first = Some(flags.latency_first); result.dev_name = Some(flags.dev_name.clone()); result.use_smoltcp = Some(flags.use_smoltcp); @@ -1144,6 +1145,11 @@ impl NetworkConfig { result.disable_sym_hole_punching = Some(flags.disable_sym_hole_punching); result.enable_magic_dns = Some(flags.accept_dns); result.mtu = Some(flags.mtu as i32); + result.data_compress_algo = (flags.data_compress_algo != default_flags.data_compress_algo) + .then_some(flags.data_compress_algo); + result.encryption_algorithm = (flags.encryption_algorithm + != default_flags.encryption_algorithm) + .then_some(flags.encryption_algorithm.clone()); result.instance_recv_bps_limit = (flags.instance_recv_bps_limit != u64::MAX).then_some(flags.instance_recv_bps_limit); result.enable_private_mode = Some(flags.private_mode); @@ -1173,7 +1179,7 @@ impl NetworkConfig { mod tests { use crate::{ common::config::{ConfigLoader, process_secure_mode_cfg}, - proto::common::SecureModeConfig, + proto::common::{CompressionAlgoPb, SecureModeConfig}, }; use base64::prelude::{BASE64_STANDARD, Engine as _}; use rand::Rng; @@ -1534,4 +1540,37 @@ mod tests { Ok(()) } + + #[test] + fn test_network_config_conversion_preserves_runtime_algorithm_flags() + -> Result<(), anyhow::Error> { + let config = gen_default_config(); + let mut flags = config.get_flags(); + flags.data_compress_algo = CompressionAlgoPb::Zstd.into(); + flags.encryption_algorithm = "managed-test-algo".to_string(); + config.set_flags(flags.clone()); + + let network_config = super::NetworkConfig::new_from_config(&config)?; + + assert_eq!( + network_config.data_compress_algo, + Some(CompressionAlgoPb::Zstd as i32) + ); + assert_eq!( + network_config.encryption_algorithm.as_deref(), + Some("managed-test-algo") + ); + + let generated_config = network_config.gen_config()?; + assert_eq!( + generated_config.get_flags().data_compress_algo, + flags.data_compress_algo + ); + assert_eq!( + generated_config.get_flags().encryption_algorithm, + flags.encryption_algorithm + ); + + Ok(()) + } } diff --git a/easytier/src/tests/three_node.rs b/easytier/src/tests/three_node.rs index c5ac99b9..5fa89640 100644 --- a/easytier/src/tests/three_node.rs +++ b/easytier/src/tests/three_node.rs @@ -3653,6 +3653,25 @@ pub async fn config_patch_test() { true }, ); + let patch = InstanceConfigPatch { + proxy_networks: vec![ProxyNetworkPatch { + action: ConfigPatchAction::Clear as i32, + ..Default::default() + }], + ..Default::default() + }; + insts[1] + .get_config_patcher() + .apply_patch(patch) + .await + .unwrap(); + assert!( + insts[1] + .get_global_ctx() + .config + .get_proxy_cidrs() + .is_empty() + ); // 测试1.1：修改公网 IPv6 provider 相关配置 let public_prefix = "2001:db8:100::/64";