feat: string deserialization for prost enums (#2316)

Use pbjson to support string deserialization for enum fields

This allows TOML configs like:
    chainType = "Inbound"
instead of:
    chainType = 1

- Maintain backward compatibility with integer values
- Default serialization format is now string

easytier/Cargo.toml

@@ -127,9 +127,11 @@ uuid = { version = "1.5.0", features = [
 once_cell = "1.18.0"
 
 # for rpc
-prost = "0.13.5"
-prost-wkt = "0.6"
-prost-wkt-types = "0.6"
+prost = "0.14.3"
+prost-reflect = { version = "0.16.4", default-features = false, features = ["derive"] }
+prost-wkt-types = "0.7.1"
+pbjson = "0.9.0"
+
 anyhow = "1.0"
 
 url = { version = "2.5", features = ["serde"] }
@@ -226,10 +228,6 @@ zstd = { version = "0.13", optional = true }
 
 kcp-sys = { git = "https://github.com/EasyTier/kcp-sys", rev = "d7427c22d764deb1860a7d37acc446ed5033464c", optional = true }
 
-prost-reflect = { version = "0.14.5", default-features = false, features = [
-    "derive",
-] }
-
 # for http connector
 http_req = { git = "https://github.com/EasyTier/http_req.git", default-features = false, features = [
     "rust-tls",
@@ -320,9 +318,9 @@ cfg_aliases = "0.2.1"
 indoc = "2.0"
 globwalk = "0.8.1"
 regex = "1"
-prost-build = "0.13.5"
-prost-wkt-build = "0.6"
-prost-reflect-build = { version = "0.14.0" }
+prost-build = "0.14.3"
+prost-reflect-build = "0.16.0"
+pbjson-build = "0.9.0"
 proc-macro2 = "1"
 quote = "1"
 thunk-rs = { git = "https://github.com/easytier/thunk.git", default-features = false, features = [

easytier/build/main.rs

@@ -2,7 +2,6 @@ mod rpc;
 
 use crate::rpc::ServiceGenerator;
 use cfg_aliases::cfg_aliases;
-use prost_wkt_build::{FileDescriptorSet, Message as _};
 #[cfg(target_os = "windows")]
 use std::io::Cursor;
 use std::{env, path::PathBuf};
@@ -174,32 +173,15 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         println!("cargo:rerun-if-changed={proto_file}");
     }
 
-    let out = PathBuf::from(env::var("OUT_DIR").unwrap());
-    let descriptor_file = out.join("descriptors.bin");
+    let out = PathBuf::from(env::var("OUT_DIR")?);
+    let descriptor = out.join("descriptors.bin");
 
     let mut config = prost_build::Config::new();
     config
-        .type_attribute(".", "#[derive(serde::Serialize,serde::Deserialize)]")
         .extern_path(".google.protobuf.Any", "::prost_wkt_types::Any")
         .extern_path(".google.protobuf.Timestamp", "::prost_wkt_types::Timestamp")
         .extern_path(".google.protobuf.Value", "::prost_wkt_types::Value")
-        .file_descriptor_set_path(&descriptor_file)
-        .protoc_arg("--experimental_allow_proto3_optional")
-        .type_attribute("peer_rpc.DirectConnectedPeerInfo", "#[derive(Hash)]")
-        .type_attribute("peer_rpc.PeerInfoForGlobalMap", "#[derive(Hash)]")
-        .type_attribute("peer_rpc.ForeignNetworkRouteInfoKey", "#[derive(Hash, Eq)]")
-        .type_attribute(
-            "peer_rpc.RouteForeignNetworkSummary.Info",
-            "#[derive(Hash, Eq)]",
-        )
-        .type_attribute("peer_rpc.RouteForeignNetworkSummary", "#[derive(Hash, Eq)]")
-        .type_attribute("common.RpcDescriptor", "#[derive(Hash, Eq)]")
-        .type_attribute("acl.Acl", "#[serde(default)]")
-        .type_attribute("acl.AclV1", "#[serde(default)]")
-        .type_attribute("acl.Chain", "#[serde(default)]")
-        .type_attribute("acl.Rule", "#[serde(default)]")
-        .type_attribute("acl.GroupInfo", "#[serde(default)]")
-        .field_attribute(".api.manage.NetworkConfig", "#[serde(default)]")
+        .file_descriptor_set_path(&descriptor)
         .service_generator(Box::new(ServiceGenerator::default()))
         .btree_map(["."])
         .skip_debug([".common.Ipv4Addr", ".common.Ipv6Addr", ".common.UUID"]);
@@ -210,9 +192,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .file_descriptor_set_bytes("crate::proto::DESCRIPTOR_POOL_BYTES")
         .compile_protos_with_config(config, &proto_files_reflect, &["src/proto/"])?;
 
-    let descriptor_bytes = std::fs::read(descriptor_file).unwrap();
-    let descriptor = FileDescriptorSet::decode(&descriptor_bytes[..]).unwrap();
-    prost_wkt_build::add_serde(out, descriptor);
+    let descriptor = std::fs::read(descriptor)?;
+    pbjson_build::Builder::new()
+        .register_descriptors(&descriptor)?
+        .preserve_proto_field_names()
+        .btree_map(["."])
+        .build(&["."])?;
 
     check_locale();
     Ok(())

easytier/src/common/config.rs

@@ -636,21 +636,12 @@ impl TomlConfigLoader {
         Ok(ret)
     }
 
-    fn gen_flags(mut flags_hashmap: serde_json::Map<String, serde_json::Value>) -> Flags {
-        let default_flags_json = serde_json::to_string(&gen_default_flags()).unwrap();
-        let default_flags_hashmap =
-            serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(&default_flags_json)
-                .unwrap();
-
-        let mut merged_hashmap = serde_json::Map::new();
-        for (key, value) in default_flags_hashmap {
-            if let Some(v) = flags_hashmap.remove(&key) {
-                merged_hashmap.insert(key, v);
-            } else {
-                merged_hashmap.insert(key, value);
-            }
-        }
-
+    fn gen_flags(flags_hashmap: serde_json::Map<String, serde_json::Value>) -> Flags {
+        let mut merged_hashmap = match serde_json::to_value(gen_default_flags()) {
+            Ok(serde_json::Value::Object(map)) => map,
+            _ => serde_json::Map::new(),
+        };
+        merged_hashmap.extend(flags_hashmap);
         serde_json::from_value(serde_json::Value::Object(merged_hashmap)).unwrap()
     }
 }

easytier/src/peers/peer_conn.rs

@@ -654,7 +654,7 @@ impl PeerConn {
             .and_then(|p| p.peer_public_key)
     }
 
-    async fn send_noise_msg<Msg: prost::Message>(
+    async fn send_noise_msg<Msg: prost::Message + Debug>(
         &self,
         pb: Msg,
         packet_type: PacketType,

easytier/src/peers/peer_manager.rs

@@ -1397,7 +1397,7 @@ impl PeerManager {
             let f = OneForeignNetwork {
                 network_name: info.key.as_ref().unwrap().network_name.clone(),
                 peer_ids: route_info.foreign_peer_ids.clone(),
-                last_updated: format!("{}", route_info.last_update.unwrap()),
+                last_updated: serde_json::to_string(&route_info.last_update.unwrap()).unwrap(),
                 version: route_info.version,
             };
 

easytier/src/peers/peer_ospf_route.rs

@@ -73,7 +73,9 @@ use super::{
     },
 };
 
+use crate::proto::common::TimestampExt;
 use atomic_shim::AtomicU64;
+use prost_wkt_types::Timestamp;
 
 static SERVICE_ID: u32 = 7;
 static UPDATE_PEER_INFO_PERIOD: Duration = Duration::from_secs(3600);
@@ -757,7 +759,7 @@ impl SyncedRouteInfo {
             if !guard.contains_key(peer_id) {
                 let mut peer_info = RoutePeerInfo::new();
                 let mut guard = RwLockUpgradableReadGuard::upgrade(guard);
-                peer_info.last_update = Some(SystemTime::now().into());
+                peer_info.last_update = Some(Timestamp::now());
                 guard.insert(*peer_id, peer_info);
                 need_inc_version = true;
             } else {
@@ -867,7 +869,7 @@ impl SyncedRouteInfo {
             let mut guard = self.peer_infos.write();
             // time between peers may not be synchronized, so update last_update to local now.
             // note only last_update with larger version will be updated to local saved peer info.
-            route_info.last_update = Some(SystemTime::now().into());
+            route_info.last_update = Some(Timestamp::now());
             if guard
                 .get_mut(&route_info.peer_id)
                 .is_none_or(|old| route_info.version > old.version)
@@ -962,7 +964,7 @@ impl SyncedRouteInfo {
                 continue;
             };
 
-            entry.last_update = Some(SystemTime::now().into());
+            entry.last_update = Some(Timestamp::now());
 
             self.foreign_network
                 .entry(key.clone())
@@ -1010,7 +1012,7 @@ impl SyncedRouteInfo {
             };
 
             guard.with_upgraded(|peer_infos| {
-                new.last_update = Some(SystemTime::now().into());
+                new.last_update = Some(Timestamp::now());
                 new.version = new_version;
                 peer_infos.insert(my_peer_id, new)
             });
@@ -1086,7 +1088,7 @@ impl SyncedRouteInfo {
                 foreign_networks.remove(key).unwrap();
             } else if !item.foreign_peer_ids.is_empty() {
                 item.foreign_peer_ids.clear();
-                item.last_update = Some(SystemTime::now().into());
+                item.last_update = Some(Timestamp::now());
                 item.version = std::cmp::max(item.version + 1, now_version);
                 updated = true;
             }
@@ -2542,7 +2544,7 @@ impl PeerRouteServiceImpl {
         for (peer_id, peer_info) in peer_infos.iter().rev() {
             // stop iter if last_update of peer info is older than session.last_sync_succ_timestamp
             if let Some(last_update) = peer_info.last_update {
-                let last_update = TryInto::<SystemTime>::try_into(last_update).unwrap();
+                let last_update = SystemTime::try_from(last_update).unwrap();
                 if last_sync_succ_timestamp.is_some_and(|t| last_update < t) {
                     break;
                 }
@@ -4158,7 +4160,9 @@ mod tests {
     use dashmap::DashMap;
     use parking_lot::Mutex;
     use prefix_trie::PrefixMap;
+    use prost::Message;
     use prost_reflect::{DynamicMessage, ReflectMessage};
+    use prost_wkt_types::Timestamp;
     use std::net::IpAddr;
     use std::{
         collections::{BTreeSet, HashMap},
@@ -4170,6 +4174,7 @@ mod tests {
     };
 
     use super::{NextHopInfo, PeerRoute, REMOVE_DEAD_PEER_INFO_AFTER, RouteConnInfo};
+    use crate::proto::common::TimestampExt;
     use crate::{
         common::{
             PeerId,
@@ -4198,7 +4203,7 @@ mod tests {
         },
         tunnel::common::tests::wait_for_condition,
     };
-    use prost::Message;
+
     struct AuthOnlyInterface {
         my_peer_id: PeerId,
         identity_type: DashMap<PeerId, PeerIdentityType>,
@@ -5489,7 +5494,7 @@ mod tests {
         );
         let mut self_info = self_info;
         self_info.version = 1;
-        self_info.last_update = Some(SystemTime::now().into());
+        self_info.last_update = Some(Timestamp::now());
         {
             let mut guard = service_impl.synced_route_info.peer_infos.write();
             guard.insert(service_impl.my_peer_id, self_info);

easytier/src/proto/acl.rs

@@ -1,6 +1,7 @@
 use std::fmt::Display;
 
 include!(concat!(env!("OUT_DIR"), "/acl.rs"));
+include!(concat!(env!("OUT_DIR"), "/acl.serde.rs"));
 
 impl Acl {
     pub fn is_empty(&self) -> bool {

easytier/src/proto/api.rs

@@ -1,5 +1,7 @@
 pub mod config {
     include!(concat!(env!("OUT_DIR"), "/api.config.rs"));
+    include!(concat!(env!("OUT_DIR"), "/api.config.serde.rs"));
+
     pub struct Patchable<T> {
         pub action: Option<ConfigPatchAction>,
         pub value: Option<T>,
@@ -77,6 +79,7 @@ pub mod config {
 
 pub mod instance {
     include!(concat!(env!("OUT_DIR"), "/api.instance.rs"));
+    include!(concat!(env!("OUT_DIR"), "/api.instance.serde.rs"));
 
     impl PeerRoutePair {
         pub fn get_latency_ms(&self) -> Option<f64> {
@@ -229,10 +232,12 @@ pub mod instance {
 
 pub mod logger {
     include!(concat!(env!("OUT_DIR"), "/api.logger.rs"));
+    include!(concat!(env!("OUT_DIR"), "/api.logger.serde.rs"));
 }
 
 pub mod manage {
     include!(concat!(env!("OUT_DIR"), "/api.manage.rs"));
+    include!(concat!(env!("OUT_DIR"), "/api.manage.serde.rs"));
 }
 
 #[cfg(test)]

easytier/src/proto/common.rs

@@ -1,15 +1,26 @@
+use anyhow::Context;
+use base64::{Engine as _, prelude::BASE64_STANDARD};
+use std::time::SystemTime;
 use std::{
     fmt::{self, Display},
     str::FromStr,
 };
-
-use anyhow::Context;
-use base64::{Engine as _, prelude::BASE64_STANDARD};
 use strum::VariantArray;
 
 use crate::tunnel::{IpScheme, packet_def::CompressorAlgo};
 
 include!(concat!(env!("OUT_DIR"), "/common.rs"));
+include!(concat!(env!("OUT_DIR"), "/common.serde.rs"));
+
+pub trait TimestampExt {
+    fn now() -> Self;
+}
+
+impl TimestampExt for prost_wkt_types::Timestamp {
+    fn now() -> Self {
+        SystemTime::now().into()
+    }
+}
 
 impl From<uuid::Uuid> for Uuid {
     fn from(uuid: uuid::Uuid) -> Self {

easytier/src/proto/error.rs

@@ -1,10 +1,9 @@
 #![allow(clippy::module_inception)]
 
-use prost::DecodeError;
-
 use super::rpc_types;
 
 include!(concat!(env!("OUT_DIR"), "/error.rs"));
+include!(concat!(env!("OUT_DIR"), "/error.serde.rs"));
 
 impl From<&rpc_types::error::Error> for Error {
     fn from(e: &rpc_types::error::Error) -> Self {
@@ -15,10 +14,10 @@ impl From<&rpc_types::error::Error> for Error {
                     error_message: format!("{:?}", e),
                 })),
             },
-            rpc_types::error::Error::DecodeError(_) => Self {
+            rpc_types::error::Error::DecodeError => Self {
                 error_kind: Some(ProtoError::ProstDecodeError(ProstDecodeError {})),
             },
-            rpc_types::error::Error::EncodeError(_) => Self {
+            rpc_types::error::Error::EncodeError => Self {
                 error_kind: Some(ProtoError::ProstEncodeError(ProstEncodeError {})),
             },
             rpc_types::error::Error::InvalidMethodIndex(m, s) => Self {
@@ -59,12 +58,8 @@ impl From<&Error> for rpc_types::error::Error {
             Some(ProtoError::ExecuteError(e)) => {
                 Self::ExecutionError(anyhow::anyhow!(e.error_message.clone()))
             }
-            Some(ProtoError::ProstDecodeError(_)) => {
-                Self::DecodeError(DecodeError::new("decode error"))
-            }
-            Some(ProtoError::ProstEncodeError(_)) => {
-                Self::DecodeError(DecodeError::new("encode error"))
-            }
+            Some(ProtoError::ProstDecodeError(_)) => Self::DecodeError,
+            Some(ProtoError::ProstEncodeError(_)) => Self::EncodeError,
             Some(ProtoError::InvalidMethodIndex(e)) => {
                 Self::InvalidMethodIndex(e.method_index as u8, e.service_name.clone())
             }

easytier/src/proto/magic_dns.rs

@@ -1 +1,2 @@
 include!(concat!(env!("OUT_DIR"), "/magic_dns.rs"));
+include!(concat!(env!("OUT_DIR"), "/magic_dns.serde.rs"));

easytier/src/proto/peer_rpc.rs

@@ -5,6 +5,7 @@ use sha2::Sha256;
 use crate::common::PeerId;
 
 include!(concat!(env!("OUT_DIR"), "/peer_rpc.rs"));
+include!(concat!(env!("OUT_DIR"), "/peer_rpc.serde.rs"));
 
 impl PeerGroupInfo {
     pub fn generate_with_proof(group_name: String, group_secret: String, peer_id: PeerId) -> Self {

easytier/src/proto/rpc_types/error.rs

@@ -9,11 +9,11 @@ pub enum Error {
     #[error("Rust error: {0}")]
     ExecutionError(#[from] anyhow::Error),
 
-    #[error("Decode error: {0}")]
-    DecodeError(#[from] prost::DecodeError),
+    #[error("Decode error")]
+    DecodeError,
 
-    #[error("Encode error: {0}")]
-    EncodeError(#[from] prost::EncodeError),
+    #[error("Encode error")]
+    EncodeError,
 
     #[error("Invalid method index: {0}, service: {1}")]
     InvalidMethodIndex(u8, String),
@@ -34,4 +34,16 @@ pub enum Error {
     Shutdown,
 }
 
+impl From<prost::DecodeError> for Error {
+    fn from(_: prost::DecodeError) -> Self {
+        Error::DecodeError
+    }
+}
+
+impl From<prost::EncodeError> for Error {
+    fn from(_: prost::EncodeError) -> Self {
+        Error::EncodeError
+    }
+}
+
 pub type Result<T> = result::Result<T, Error>;

easytier/src/proto/tests.rs

@@ -1,4 +1,5 @@
 include!(concat!(env!("OUT_DIR"), "/tests.rs"));
+include!(concat!(env!("OUT_DIR"), "/tests.serde.rs"));
 
 use std::sync::{Arc, Mutex};
 

easytier/src/proto/web.rs

@@ -1 +1,2 @@
 include!(concat!(env!("OUT_DIR"), "/web.rs"));
+include!(concat!(env!("OUT_DIR"), "/web.serde.rs"));