diff --git a/easytier-gui/src-tauri/src/lib.rs b/easytier-gui/src-tauri/src/lib.rs index b72a5025..007df2f9 100644 --- a/easytier-gui/src-tauri/src/lib.rs +++ b/easytier-gui/src-tauri/src/lib.rs @@ -191,7 +191,7 @@ async fn remove_network_instance(app: AppHandle, instance_id: String) -> Result< .await .map_err(|e| e.to_string())?; client_manager - .post_remove_network_instances_hook(&app, &[instance_id]) + .post_stop_network_instances_hook(&app) .await?; Ok(()) @@ -214,7 +214,7 @@ async fn update_network_config_state( if disabled { client_manager - .post_remove_network_instances_hook(&app, &[instance_id]) + .post_stop_network_instances_hook(&app) .await?; } @@ -613,7 +613,7 @@ mod manager { async fn post_remove_network_instances(&self, ids: &[uuid::Uuid]) -> Result<(), String> { let client_manager = get_client_manager!()?; client_manager - .post_remove_network_instances_hook(&self.app, ids) + .post_remote_remove_network_instances_hook(&self.app, ids) .await } } @@ -696,7 +696,9 @@ mod manager { self.network_configs.remove(network_inst_id); self.enabled_networks.remove(network_inst_id); } - self.save_configs(&app) + self.save_configs(&app)?; + self.save_enabled_networks(&app)?; + Ok(()) } async fn update_network_config_state( @@ -897,14 +899,23 @@ mod manager { Ok(()) } - pub(super) async fn post_remove_network_instances_hook( + pub(super) async fn post_remote_remove_network_instances_hook( &self, app: &AppHandle, - _ids: &[uuid::Uuid], + ids: &[uuid::Uuid], ) -> Result<(), String> { self.storage - .enabled_networks - .retain(|id| !_ids.contains(id)); + .delete_network_configs(app.clone(), ids) + .await + .map_err(|e| e.to_string())?; + self.notify_vpn_stop_if_no_tun(app)?; + Ok(()) + } + + pub(super) async fn post_stop_network_instances_hook( + &self, + app: &AppHandle, + ) -> Result<(), String> { self.notify_vpn_stop_if_no_tun(app)?; Ok(()) } diff --git a/easytier-web/src/client_manager/session.rs b/easytier-web/src/client_manager/session.rs index c6e14b23..9c695fe1 100644 --- a/easytier-web/src/client_manager/session.rs +++ b/easytier-web/src/client_manager/session.rs @@ -1,4 +1,4 @@ -use std::{fmt::Debug, str::FromStr as _, sync::Arc}; +use std::{collections::HashSet, fmt::Debug, str::FromStr as _, sync::Arc}; use anyhow::Context; use easytier::{ @@ -399,6 +399,7 @@ impl Session { storage: WeakRefStorage, rpc_client: SessionRpcClient, ) { + let mut cleaned_web_managed_instances = false; loop { heartbeat_waiter = heartbeat_waiter.resubscribe(); let req = heartbeat_waiter.recv().await; @@ -420,7 +421,7 @@ impl Session { .running_network_instances .iter() .map(|x| x.to_string()) - .collect::>(); + .collect::>(); let Some(storage) = storage.upgrade() else { tracing::error!("Failed to get storage"); return; @@ -456,6 +457,60 @@ impl Session { let mut has_failed = false; + if !cleaned_web_managed_instances { + let all_local_configs = match storage + .db + .list_network_configs((user_id, machine_id.into()), ListNetworkProps::All) + .await + { + Ok(configs) => configs, + Err(e) => { + tracing::error!("Failed to list all network configs, error: {:?}", e); + return; + } + }; + + let all_inst_ids = all_local_configs + .iter() + .map(|cfg| cfg.network_instance_id.clone()) + .collect::>(); + + let should_be_alive_inst_ids = local_configs + .iter() + .map(|cfg| cfg.network_instance_id.clone()) + .collect::>(); + + let should_delete_ids = running_inst_ids + .iter() + .chain(all_inst_ids.iter()) + .filter(|inst_id| !should_be_alive_inst_ids.contains(*inst_id)) + .filter_map(|inst_id| uuid::Uuid::parse_str(inst_id).ok()) + .map(Into::into) + .collect::>(); + + if !should_delete_ids.is_empty() { + let ret = rpc_client + .delete_network_instance( + BaseController::default(), + easytier::proto::api::manage::DeleteNetworkInstanceRequest { + inst_ids: should_delete_ids, + }, + ) + .await; + tracing::info!( + ?user_id, + "Clean non-web-managed network instances on start: {:?}, user_token: {:?}", + ret, + req.user_token + ); + has_failed |= ret.is_err(); + } + + if !has_failed { + cleaned_web_managed_instances = true; + } + } + for c in local_configs { if running_inst_ids.contains(&c.network_instance_id) { continue; diff --git a/easytier/src/common/global_ctx.rs b/easytier/src/common/global_ctx.rs index ef6eafea..bb1a6558 100644 --- a/easytier/src/common/global_ctx.rs +++ b/easytier/src/common/global_ctx.rs @@ -148,6 +148,24 @@ impl TrustedKeyMapManager { !metadata.is_expired() } + + pub fn list_trusted_keys(&self, network_name: &str) -> Vec<(Vec, TrustedKeyMetadata)> { + let Some(trusted_keys) = self + .network_trusted_keys + .get(network_name) + .map(|v| v.load_full()) + else { + return Vec::new(); + }; + + let mut items = trusted_keys + .iter() + .filter(|(_, metadata)| !metadata.is_expired()) + .map(|(pubkey, metadata)| (pubkey.clone(), metadata.clone())) + .collect::>(); + items.sort_by(|left, right| left.0.cmp(&right.0)); + items + } } pub struct GlobalCtx { @@ -534,6 +552,10 @@ impl GlobalCtx { self.trusted_keys.remove_trusted_keys(network_name); } + pub fn list_trusted_keys(&self, network_name: &str) -> Vec<(Vec, TrustedKeyMetadata)> { + self.trusted_keys.list_trusted_keys(network_name) + } + pub fn get_acl_groups(&self, peer_id: PeerId) -> Vec { use std::collections::HashSet; self.config diff --git a/easytier/src/easytier-cli.rs b/easytier/src/easytier-cli.rs index 052362f8..51032641 100644 --- a/easytier/src/easytier-cli.rs +++ b/easytier/src/easytier-cli.rs @@ -12,6 +12,8 @@ use std::{ }; use anyhow::Context; +use base64::prelude::BASE64_STANDARD; +use base64::Engine as _; use cidr::Ipv4Inet; use clap::{Args, CommandFactory, Parser, Subcommand}; use dashmap::DashMap; @@ -56,8 +58,8 @@ use easytier::{ PeerManageRpcClientFactory, PortForwardManageRpc, PortForwardManageRpcClientFactory, RevokeCredentialRequest, ShowNodeInfoRequest, StatsRpc, StatsRpcClientFactory, TcpProxyEntryState, TcpProxyEntryTransportType, - TcpProxyRpc, TcpProxyRpcClientFactory, VpnPortalInfo, VpnPortalRpc, - VpnPortalRpcClientFactory, + TcpProxyRpc, TcpProxyRpcClientFactory, TrustedKeySourcePb, VpnPortalInfo, + VpnPortalRpc, VpnPortalRpcClientFactory, }, logger::{ GetLoggerConfigRequest, LogLevel, LoggerRpc, LoggerRpcClientFactory, @@ -193,7 +195,14 @@ enum PeerSubCommand { Add, Remove, List, - ListForeign, + ListForeign { + #[arg( + long, + default_value = "false", + help = "include trusted keys for each foreign network" + )] + trusted_keys: bool, + }, ListGlobalForeign, } @@ -901,7 +910,10 @@ impl<'a> CommandHandler<'a> { .result) } - async fn fetch_foreign_networks(&self) -> Result { + async fn fetch_foreign_networks( + &self, + include_trusted_keys: bool, + ) -> Result { Ok(self .get_peer_manager_client() .await? @@ -909,6 +921,7 @@ impl<'a> CommandHandler<'a> { BaseController::default(), ListForeignNetworkRequest { instance: Some(self.instance_selector.clone()), + include_trusted_keys, }, ) .await? @@ -1316,9 +1329,11 @@ impl<'a> CommandHandler<'a> { }) } - async fn handle_foreign_network_list(&self) -> Result<(), Error> { + async fn handle_foreign_network_list(&self, include_trusted_keys: bool) -> Result<(), Error> { let results = self - .collect_instance_results(|handler| Box::pin(handler.fetch_foreign_networks())) + .collect_instance_results(|handler| { + Box::pin(handler.fetch_foreign_networks(include_trusted_keys)) + }) .await?; if self.verbose || *self.output_format == OutputFormat::Json { return self.print_json_results(results); @@ -1351,6 +1366,24 @@ impl<'a> CommandHandler<'a> { .join("; ") ); } + if include_trusted_keys { + println!(" trusted_keys:"); + for trusted_key in &v.trusted_keys { + let source = TrustedKeySourcePb::try_from(trusted_key.source) + .map(|source| source.as_str_name()) + .unwrap_or("TRUSTED_KEY_SOURCE_PB_UNSPECIFIED"); + let expiry = trusted_key + .expiry_unix + .map(|value| value.to_string()) + .unwrap_or_else(|| "-".to_string()); + println!( + " source: {}, expiry_unix: {}, pubkey: {}", + source, + expiry, + BASE64_STANDARD.encode(&trusted_key.pubkey), + ); + } + } } Ok(()) }) @@ -2548,8 +2581,8 @@ async fn main() -> Result<(), Error> { Some(PeerSubCommand::List) => { handler.handle_peer_list().await?; } - Some(PeerSubCommand::ListForeign) => { - handler.handle_foreign_network_list().await?; + Some(PeerSubCommand::ListForeign { trusted_keys }) => { + handler.handle_foreign_network_list(*trusted_keys).await?; } Some(PeerSubCommand::ListGlobalForeign) => { handler.handle_global_foreign_network_list().await?; diff --git a/easytier/src/peers/foreign_network_manager.rs b/easytier/src/peers/foreign_network_manager.rs index 7d97f7af..8a308c43 100644 --- a/easytier/src/peers/foreign_network_manager.rs +++ b/easytier/src/peers/foreign_network_manager.rs @@ -23,7 +23,7 @@ use crate::{ common::{ config::{ConfigLoader, TomlConfigLoader}, error::Error, - global_ctx::{ArcGlobalCtx, GlobalCtx, GlobalCtxEvent, NetworkIdentity}, + global_ctx::{ArcGlobalCtx, GlobalCtx, GlobalCtxEvent, NetworkIdentity, TrustedKeySource}, join_joinset_background, shrink_dashmap, stats_manager::{LabelSet, LabelType, MetricName, StatsManager}, token_bucket::TokenBucket, @@ -32,7 +32,10 @@ use crate::{ peer_center::instance::{PeerCenterInstance, PeerMapWithPeerRpcManager}, peers::route_trait::{Route, RouteInterface}, proto::{ - api::instance::{ForeignNetworkEntryPb, ListForeignNetworkResponse, PeerInfo}, + api::instance::{ + ForeignNetworkEntryPb, ListForeignNetworkResponse, PeerInfo, TrustedKeyInfoPb, + TrustedKeySourcePb, + }, common::LimiterConfig, peer_rpc::{DirectConnectorRpcServer, PeerIdentityType}, }, @@ -627,6 +630,22 @@ impl ForeignNetworkManager { .is_pubkey_trusted(remote_static_pubkey, &entry.network.network_name) } + fn build_trusted_key_items(entry: &ForeignNetworkEntry) -> Vec { + entry + .global_ctx + .list_trusted_keys(&entry.network.network_name) + .into_iter() + .map(|(pubkey, metadata)| TrustedKeyInfoPb { + pubkey, + source: match metadata.source { + TrustedKeySource::OspfNode => TrustedKeySourcePb::OspfNode.into(), + TrustedKeySource::OspfCredential => TrustedKeySourcePb::OspfCredential.into(), + }, + expiry_unix: metadata.expiry_unix, + }) + .collect() + } + pub fn new( my_peer_id: PeerId, global_ctx: ArcGlobalCtx, @@ -775,6 +794,13 @@ impl ForeignNetworkManager { } pub async fn list_foreign_networks(&self) -> ListForeignNetworkResponse { + self.list_foreign_networks_with_options(false).await + } + + pub async fn list_foreign_networks_with_options( + &self, + include_trusted_keys: bool, + ) -> ListForeignNetworkResponse { let mut ret = ListForeignNetworkResponse::default(); let networks = self .data @@ -801,6 +827,11 @@ impl ForeignNetworkManager { .to_vec(), my_peer_id_for_this_network: item.my_peer_id, peers: Default::default(), + trusted_keys: if include_trusted_keys { + Self::build_trusted_key_items(&item) + } else { + Default::default() + }, }; for peer in item.peer_map.list_peers() { let peer_info = PeerInfo { @@ -872,6 +903,7 @@ pub mod tests { create_mock_peer_manager_with_mock_stun, replace_stun_info_collector, }, peers::{ + peer_conn::tests::set_secure_mode_cfg, peer_manager::{PeerManager, RouteAlgoType}, tests::{connect_peer_manager, wait_route_appear}, }, @@ -905,6 +937,21 @@ pub mod tests { create_mock_peer_manager_for_foreign_network_ext(network, network).await } + pub async fn create_mock_peer_manager_for_secure_foreign_network( + network: &str, + ) -> Arc { + let (s, _r) = create_packet_recv_chan(); + let global_ctx = get_mock_global_ctx_with_network(Some(NetworkIdentity::new( + network.to_string(), + network.to_string(), + ))); + set_secure_mode_cfg(&global_ctx, true); + let peer_mgr = Arc::new(PeerManager::new(RouteAlgoType::Ospf, global_ctx, s)); + replace_stun_info_collector(peer_mgr.clone(), NatType::Unknown); + peer_mgr.run().await.unwrap(); + peer_mgr + } + #[tokio::test] async fn foreign_network_basic() { let pm_center = create_mock_peer_manager_with_mock_stun(NatType::Unknown).await; @@ -935,6 +982,51 @@ pub mod tests { assert_eq!(2, rpc_resp.foreign_networks["net1"].peers.len()); } + #[tokio::test] + async fn foreign_network_list_can_include_trusted_keys() { + let pm_center = create_mock_peer_manager_with_mock_stun(NatType::Unknown).await; + set_secure_mode_cfg(&pm_center.get_global_ctx(), true); + + let pma_net1 = create_mock_peer_manager_for_secure_foreign_network("net1").await; + let pmb_net1 = create_mock_peer_manager_for_secure_foreign_network("net1").await; + connect_peer_manager(pma_net1.clone(), pm_center.clone()).await; + connect_peer_manager(pmb_net1.clone(), pm_center.clone()).await; + wait_route_appear(pma_net1.clone(), pmb_net1.clone()) + .await + .unwrap(); + + let without_trusted_keys = pm_center + .get_foreign_network_manager() + .list_foreign_networks() + .await; + assert!(without_trusted_keys.foreign_networks["net1"] + .trusted_keys + .is_empty()); + + let foreign_mgr = pm_center.get_foreign_network_manager(); + wait_for_condition( + || { + let foreign_mgr = foreign_mgr.clone(); + async move { + foreign_mgr + .list_foreign_networks_with_options(true) + .await + .foreign_networks + .get("net1") + .map(|entry| !entry.trusted_keys.is_empty()) + .unwrap_or(false) + } + }, + Duration::from_secs(5), + ) + .await; + + let with_trusted_keys = foreign_mgr.list_foreign_networks_with_options(true).await; + assert!(!with_trusted_keys.foreign_networks["net1"] + .trusted_keys + .is_empty()); + } + async fn foreign_network_whitelist_helper(name: String) { let pm_center = create_mock_peer_manager_with_mock_stun(NatType::Unknown).await; tracing::debug!("pm_center: {:?}", pm_center.my_peer_id()); diff --git a/easytier/src/peers/rpc_service.rs b/easytier/src/peers/rpc_service.rs index 354d5d47..7a95cfa8 100644 --- a/easytier/src/peers/rpc_service.rs +++ b/easytier/src/peers/rpc_service.rs @@ -124,11 +124,11 @@ impl PeerManageRpc for PeerManagerRpcService { async fn list_foreign_network( &self, _: BaseController, - _request: ListForeignNetworkRequest, // Accept request of type HelloRequest + request: ListForeignNetworkRequest, ) -> Result { let reply = weak_upgrade(&self.peer_manager)? .get_foreign_network_manager() - .list_foreign_networks() + .list_foreign_networks_with_options(request.include_trusted_keys) .await; Ok(reply) } diff --git a/easytier/src/proto/api_instance.proto b/easytier/src/proto/api_instance.proto index 16042fbb..5f5f2460 100644 --- a/easytier/src/proto/api_instance.proto +++ b/easytier/src/proto/api_instance.proto @@ -114,12 +114,28 @@ message DumpRouteRequest { InstanceIdentifier instance = 1; } message DumpRouteResponse { string result = 1; } -message ListForeignNetworkRequest { InstanceIdentifier instance = 1; } +message ListForeignNetworkRequest { + InstanceIdentifier instance = 1; + bool include_trusted_keys = 2; +} + +enum TrustedKeySourcePb { + TRUSTED_KEY_SOURCE_PB_UNSPECIFIED = 0; + TRUSTED_KEY_SOURCE_PB_OSPF_NODE = 1; + TRUSTED_KEY_SOURCE_PB_OSPF_CREDENTIAL = 2; +} + +message TrustedKeyInfoPb { + bytes pubkey = 1; + TrustedKeySourcePb source = 2; + optional int64 expiry_unix = 3; +} message ForeignNetworkEntryPb { repeated PeerInfo peers = 1; bytes network_secret_digest = 2; uint32 my_peer_id_for_this_network = 3; + repeated TrustedKeyInfoPb trusted_keys = 4; } message ListForeignNetworkResponse { diff --git a/easytier/src/tests/credential_tests.rs b/easytier/src/tests/credential_tests.rs index 82eb0d4f..fbbf59ef 100644 --- a/easytier/src/tests/credential_tests.rs +++ b/easytier/src/tests/credential_tests.rs @@ -193,6 +193,128 @@ fn create_shared_config( config } +fn create_generated_credential_config( + admin_inst: &Instance, + inst_name: &str, + ns: Option<&str>, + ipv4: &str, + ipv6: &str, +) -> (TomlConfigLoader, String) { + use base64::Engine as _; + + let (cred_id, cred_secret) = admin_inst + .get_global_ctx() + .get_credential_manager() + .generate_credential(vec![], false, vec![], Duration::from_secs(3600)); + let privkey_bytes: [u8; 32] = base64::prelude::BASE64_STANDARD + .decode(&cred_secret) + .unwrap() + .try_into() + .unwrap(); + let private = x25519_dalek::StaticSecret::from(privkey_bytes); + + let config = TomlConfigLoader::default(); + config.set_inst_name(inst_name.to_owned()); + config.set_netns(ns.map(|s| s.to_owned())); + config.set_ipv4(Some(ipv4.parse().unwrap())); + config.set_ipv6(Some(ipv6.parse().unwrap())); + config.set_listeners(vec![]); + config.set_network_identity(NetworkIdentity::new_credential( + admin_inst + .get_global_ctx() + .get_network_identity() + .network_name + .clone(), + )); + config.set_secure_mode(Some(generate_secure_mode_config_with_key(&private))); + + (config, cred_id) +} + +async fn wait_ping_reachability(src_ns: &str, dst_ip: &str, reachable: bool, timeout: Duration) { + wait_for_condition( + || async { + let ping_result = ping_test(src_ns, dst_ip, None).await; + if reachable { + ping_result + } else { + !ping_result + } + }, + timeout, + ) + .await; +} + +async fn wait_route_presence_on_admins( + admin_a_inst: &Instance, + admin_c_inst: &Instance, + peer_id: u32, + should_exist: bool, + timeout: Duration, +) { + wait_for_condition( + || async { + let admin_a_routes = admin_a_inst.get_peer_manager().list_routes().await; + let admin_c_routes = admin_c_inst.get_peer_manager().list_routes().await; + let admin_a_has = admin_a_routes.iter().any(|r| r.peer_id == peer_id); + let admin_c_has = admin_c_routes.iter().any(|r| r.peer_id == peer_id); + if should_exist { + admin_a_has || admin_c_has + } else { + !admin_a_has && !admin_c_has + } + }, + timeout, + ) + .await; +} + +async fn assert_shared_visibility_stable( + admin_a_inst: &Instance, + admin_c_inst: &Instance, + peer_id: u32, + peer_ip: &str, + should_exist: bool, + label: &str, +) { + for _ in 0..5 { + let admin_a_routes = admin_a_inst.get_peer_manager().list_routes().await; + let admin_c_routes = admin_c_inst.get_peer_manager().list_routes().await; + let admin_a_has = admin_a_routes.iter().any(|r| r.peer_id == peer_id); + let admin_c_has = admin_c_routes.iter().any(|r| r.peer_id == peer_id); + if should_exist { + assert!( + admin_a_has || admin_c_has, + "{} should exist via shared path but routes are admin_a={:?} admin_c={:?}", + label, + admin_a_routes.iter().map(|r| r.peer_id).collect::>(), + admin_c_routes.iter().map(|r| r.peer_id).collect::>() + ); + } else { + assert!( + !admin_a_has && !admin_c_has, + "{} should be absent via shared path but routes are admin_a={:?} admin_c={:?}", + label, + admin_a_routes.iter().map(|r| r.peer_id).collect::>(), + admin_c_routes.iter().map(|r| r.peer_id).collect::>() + ); + } + + let ping_from_admin_a = ping_test("ns_adm", peer_ip, None).await; + let ping_from_admin_c = ping_test("ns_c3", peer_ip, None).await; + if should_exist { + assert!(ping_from_admin_a, "admin_a should reach {}", label); + assert!(ping_from_admin_c, "admin_c should reach {}", label); + } else { + assert!(!ping_from_admin_a, "admin_a should not reach {}", label); + assert!(!ping_from_admin_c, "admin_c should not reach {}", label); + } + + tokio::time::sleep(Duration::from_secs(1)).await; + } +} + /// Test 1: Basic credential node connectivity /// Topology: Admin ← Credential /// Verifies that a credential node can connect to an admin node and appears in routes @@ -834,9 +956,10 @@ async fn credential_unknown_rejected() { /// Regression test: an unknown credential must still be rejected when it first connects via a /// shared node. If this fails, the shared path is incorrectly admitting the node into the target /// network's route domain. +#[rstest::rstest] #[tokio::test] #[serial_test::serial] -async fn credential_unknown_via_shared_rejected() { +async fn credential_unknown_via_shared_rejected(#[values(true, false)] test_revoke: bool) { prepare_credential_network(); let admin_a_config = @@ -877,18 +1000,32 @@ async fn credential_unknown_via_shared_rejected() { ) .await; - let unknown_config = create_unknown_credential_config( - admin_a_inst - .get_global_ctx() - .get_network_identity() - .network_name - .clone(), - "unknown_d", - Some("ns_c2"), - "10.144.144.5", - "fd00::5/64", - ); - let mut unknown_inst = Instance::new(unknown_config); + let (credential_config, credential_id) = if test_revoke { + let (config, cred_id) = create_generated_credential_config( + &admin_a_inst, + "cred_d", + Some("ns_c2"), + "10.144.144.5", + "fd00::5/64", + ); + (config, Some(cred_id)) + } else { + ( + create_unknown_credential_config( + admin_a_inst + .get_global_ctx() + .get_network_identity() + .network_name + .clone(), + "unknown_d", + Some("ns_c2"), + "10.144.144.5", + "fd00::5/64", + ), + None, + ) + }; + let mut unknown_inst = Instance::new(credential_config); unknown_inst.run().await.unwrap(); unknown_inst @@ -901,30 +1038,65 @@ async fn credential_unknown_via_shared_rejected() { println!("unknown_peer_id: {:?}", unknown_peer_id); - for _ in 0..5 { - let admin_a_routes = admin_a_inst.get_peer_manager().list_routes().await; - let admin_c_routes = admin_c_inst.get_peer_manager().list_routes().await; + if test_revoke { + wait_route_presence_on_admins( + &admin_a_inst, + &admin_c_inst, + unknown_peer_id, + true, + Duration::from_secs(30), + ) + .await; + wait_ping_reachability("ns_adm", "10.144.144.5", true, Duration::from_secs(20)).await; + wait_ping_reachability("ns_c3", "10.144.144.5", true, Duration::from_secs(20)).await; assert!( - !admin_a_routes.iter().any(|r| r.peer_id == unknown_peer_id), - "unknown credential unexpectedly appeared in admin_a routes via shared path: {:?}", - admin_a_routes.iter().map(|r| r.peer_id).collect::>() - ); - assert!( - !admin_c_routes.iter().any(|r| r.peer_id == unknown_peer_id), - "unknown credential unexpectedly appeared in admin_c routes via shared path: {:?}", - admin_c_routes.iter().map(|r| r.peer_id).collect::>() - ); - assert!( - !ping_test("ns_adm", "10.144.144.5", None).await, - "admin_a unexpectedly reached unknown credential via shared path" - ); - assert!( - !ping_test("ns_c3", "10.144.144.5", None).await, - "admin_c unexpectedly reached unknown credential via shared path" + admin_a_inst + .get_global_ctx() + .get_credential_manager() + .revoke_credential(credential_id.as_ref().unwrap()), + "credential should be revoked successfully" ); + admin_a_inst + .get_global_ctx() + .issue_event(GlobalCtxEvent::CredentialChanged); - tokio::time::sleep(Duration::from_secs(1)).await; + wait_route_presence_on_admins( + &admin_a_inst, + &admin_c_inst, + unknown_peer_id, + false, + Duration::from_secs(30), + ) + .await; + wait_ping_reachability("ns_adm", "10.144.144.5", false, Duration::from_secs(5)).await; + wait_ping_reachability("ns_c3", "10.144.144.5", false, Duration::from_secs(5)).await; + + unknown_inst + .get_conn_manager() + .add_connector(TcpTunnelConnector::new( + "tcp://10.1.1.2:11010".parse().unwrap(), + )); + + assert_shared_visibility_stable( + &admin_a_inst, + &admin_c_inst, + unknown_peer_id, + "10.144.144.5", + false, + "revoked credential", + ) + .await; + } else { + assert_shared_visibility_stable( + &admin_a_inst, + &admin_c_inst, + unknown_peer_id, + "10.144.144.5", + false, + "unknown credential", + ) + .await; } println!("drop all");