refactor: rpc build (#2244)

rewrite rpc build with quota crate
This commit is contained in:
Luna Yao
2026-05-15 08:01:56 +02:00
committed by GitHub
parent 8428a89d2d
commit 811f151155
9 changed files with 730 additions and 502 deletions
+219
View File
@@ -0,0 +1,219 @@
mod rpc;
use crate::rpc::ServiceGenerator;
use cfg_aliases::cfg_aliases;
use prost_wkt_build::{FileDescriptorSet, Message as _};
#[cfg(target_os = "windows")]
use std::io::Cursor;
use std::{env, path::PathBuf};
#[cfg(target_os = "windows")]
struct WindowsBuild {}
#[cfg(target_os = "windows")]
impl WindowsBuild {
fn check_protoc_exist() -> Option<PathBuf> {
let path = env::var_os("PROTOC").map(PathBuf::from);
if path.is_some() && path.as_ref().unwrap().exists() {
return path;
}
let path = env::var_os("PATH").unwrap_or_default();
for p in env::split_paths(&path) {
let p = p.join("protoc.exe");
if p.exists() && p.is_file() {
return Some(p);
}
}
None
}
fn get_cargo_target_dir() -> Result<std::path::PathBuf, Box<dyn std::error::Error>> {
let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR")?);
let profile = std::env::var("PROFILE")?;
let mut target_dir = None;
let mut sub_path = out_dir.as_path();
while let Some(parent) = sub_path.parent() {
if parent.ends_with(&profile) {
target_dir = Some(parent);
break;
}
sub_path = parent;
}
let target_dir = target_dir.ok_or("not found")?;
Ok(target_dir.to_path_buf())
}
fn download_protoc() -> PathBuf {
println!("cargo:info=use exist protoc: {:?}", "k");
let out_dir = Self::get_cargo_target_dir().unwrap().join("protobuf");
let fname = out_dir.join("bin/protoc.exe");
if fname.exists() {
println!("cargo:info=use exist protoc: {:?}", fname);
return fname;
}
println!("cargo:info=need download protoc, please wait...");
let url = "https://github.com/protocolbuffers/protobuf/releases/download/v26.0-rc1/protoc-26.0-rc-1-win64.zip";
let response = reqwest::blocking::get(url).unwrap();
println!("{:?}", response);
let mut content = response
.bytes()
.map(|v| v.to_vec())
.map(Cursor::new)
.map(zip::ZipArchive::new)
.unwrap()
.unwrap();
content.extract(out_dir).unwrap();
fname
}
pub fn check_for_win() {
// add third_party dir to link search path
let target = std::env::var("TARGET").unwrap_or_default();
if target.contains("x86_64") {
println!("cargo:rustc-link-search=native=easytier/third_party/x86_64/");
} else if target.contains("i686") {
println!("cargo:rustc-link-search=native=easytier/third_party/i686/");
} else if target.contains("aarch64") {
println!("cargo:rustc-link-search=native=easytier/third_party/arm64/");
}
let protoc_path = if let Some(o) = Self::check_protoc_exist() {
println!("cargo:info=use os exist protoc: {:?}", o);
o
} else {
Self::download_protoc()
};
unsafe {
std::env::set_var("PROTOC", protoc_path);
}
}
}
fn workdir() -> Option<String> {
if let Ok(cargo_manifest_dir) = std::env::var("CARGO_MANIFEST_DIR") {
return Some(cargo_manifest_dir);
}
let dest = std::env::var("OUT_DIR");
if dest.is_err() {
return None;
}
let dest = dest.unwrap();
let seperator = regex::Regex::new(r"(/target/(.+?)/build/)|(\\target\\(.+?)\\build\\)")
.expect("Invalid regex");
let parts = seperator.split(dest.as_str()).collect::<Vec<_>>();
if parts.len() >= 2 {
return Some(parts[0].to_string());
}
None
}
fn check_locale() {
let workdir = workdir().unwrap_or("./".to_string());
let locale_path = format!("{workdir}/**/locales/**/*");
if let Ok(globs) = globwalk::glob(locale_path) {
for entry in globs {
if let Err(e) = entry {
println!("cargo:i18n-error={e}");
continue;
}
let entry = entry.unwrap().into_path();
println!("cargo:rerun-if-changed={}", entry.display());
}
}
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
cfg_aliases! {
mobile: {
any(
target_os = "android",
target_os = "ios",
all(target_os = "macos", feature = "macos-ne"),
target_env = "ohos"
)
}
}
let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default();
let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default();
// enable thunk-rs when target os is windows and arch is x86_64 or i686
if target_os == "windows" && (target_arch == "x86" || target_arch == "x86_64") {
thunk::thunk();
}
#[cfg(target_os = "windows")]
WindowsBuild::check_for_win();
let proto_files_reflect = ["src/proto/peer_rpc.proto", "src/proto/common.proto"];
let proto_files = [
"src/proto/error.proto",
"src/proto/tests.proto",
"src/proto/api_instance.proto",
"src/proto/api_logger.proto",
"src/proto/api_config.proto",
"src/proto/api_manage.proto",
"src/proto/web.proto",
"src/proto/magic_dns.proto",
"src/proto/acl.proto",
];
for proto_file in proto_files.iter().chain(proto_files_reflect.iter()) {
println!("cargo:rerun-if-changed={proto_file}");
}
let out = PathBuf::from(env::var("OUT_DIR").unwrap());
let descriptor_file = out.join("descriptors.bin");
let mut config = prost_build::Config::new();
config
.type_attribute(".", "#[derive(serde::Serialize,serde::Deserialize)]")
.extern_path(".google.protobuf.Any", "::prost_wkt_types::Any")
.extern_path(".google.protobuf.Timestamp", "::prost_wkt_types::Timestamp")
.extern_path(".google.protobuf.Value", "::prost_wkt_types::Value")
.file_descriptor_set_path(&descriptor_file)
.protoc_arg("--experimental_allow_proto3_optional")
.type_attribute("peer_rpc.DirectConnectedPeerInfo", "#[derive(Hash)]")
.type_attribute("peer_rpc.PeerInfoForGlobalMap", "#[derive(Hash)]")
.type_attribute("peer_rpc.ForeignNetworkRouteInfoKey", "#[derive(Hash, Eq)]")
.type_attribute(
"peer_rpc.RouteForeignNetworkSummary.Info",
"#[derive(Hash, Eq)]",
)
.type_attribute("peer_rpc.RouteForeignNetworkSummary", "#[derive(Hash, Eq)]")
.type_attribute("common.RpcDescriptor", "#[derive(Hash, Eq)]")
.type_attribute("acl.Acl", "#[serde(default)]")
.type_attribute("acl.AclV1", "#[serde(default)]")
.type_attribute("acl.Chain", "#[serde(default)]")
.type_attribute("acl.Rule", "#[serde(default)]")
.type_attribute("acl.GroupInfo", "#[serde(default)]")
.field_attribute(".api.manage.NetworkConfig", "#[serde(default)]")
.service_generator(Box::new(ServiceGenerator::default()))
.btree_map(["."])
.skip_debug([".common.Ipv4Addr", ".common.Ipv6Addr", ".common.UUID"]);
config.compile_protos(&proto_files, &["src/proto/"])?;
prost_reflect_build::Builder::new()
.file_descriptor_set_bytes("crate::proto::DESCRIPTOR_POOL_BYTES")
.compile_protos_with_config(config, &proto_files_reflect, &["src/proto/"])?;
let descriptor_bytes = std::fs::read(descriptor_file).unwrap();
let descriptor = FileDescriptorSet::decode(&descriptor_bytes[..]).unwrap();
prost_wkt_build::add_serde(out, descriptor);
check_locale();
Ok(())
}
+720
View File
@@ -0,0 +1,720 @@
#![allow(non_snake_case)]
use indoc::formatdoc;
use proc_macro2::{Ident, TokenStream};
use quote::{format_ident, quote};
use std::str::FromStr;
fn parse(value: &str) -> TokenStream {
TokenStream::from_str(value)
.unwrap_or_else(|err| panic!("Failed to parse tokens: {} ({})", value, err))
}
fn doc(comments: &prost_build::Comments) -> TokenStream {
let doc = comments
.leading
.iter()
.flat_map(|c| c.lines().filter(|s| !s.is_empty()));
quote! { #( #[doc = #doc] )* }
}
const NAMESPACE: &str = "crate::proto::rpc_types";
struct Method {
index: u8,
doc: TokenStream,
method: Ident,
method_inner: Ident,
method_str: String,
method_proto: Ident,
method_proto_str: String,
Input: TokenStream,
Input_proto_str: String,
Output: TokenStream,
Output_proto_str: String,
}
impl Method {
fn new(index: u8, method: prost_build::Method) -> Self {
assert!(
!method.client_streaming,
"Client streaming not yet supported for method {}",
method.proto_name
);
assert!(
!method.server_streaming,
"Server streaming not yet supported for method {}",
method.proto_name
);
Self {
index,
doc: doc(&method.comments),
method: format_ident!("{}", method.name),
method_inner: format_ident!("{}_inner", method.name),
method_str: method.name,
method_proto: format_ident!("{}", method.proto_name),
method_proto_str: method.proto_name,
Input: parse(&method.input_type),
Input_proto_str: method.input_proto_type,
Output: parse(&method.output_type),
Output_proto_str: method.output_proto_type,
}
}
}
struct Service {
namespace: TokenStream,
doc: TokenStream,
Service: Ident,
ServiceDescriptor: Ident,
ServiceServer: Ident,
ServiceClient: Ident,
ServiceClientFactory: Ident,
ServiceMethodDescriptor: Ident,
Service_str: String,
Service_proto_str: String,
Service_package_str: String,
methods: Vec<Method>,
}
impl Service {
fn new(service: prost_build::Service) -> Self {
let methods = service
.methods
.into_iter()
.enumerate()
.map(|(i, method)| Method::new((i + 1) as u8, method))
.collect();
Self {
namespace: parse(NAMESPACE),
doc: doc(&service.comments),
Service: format_ident!("{}", service.name),
ServiceDescriptor: format_ident!("{}Descriptor", service.name),
ServiceServer: format_ident!("{}Server", service.name),
ServiceClient: format_ident!("{}Client", service.name),
ServiceClientFactory: format_ident!("{}ClientFactory", service.name),
ServiceMethodDescriptor: format_ident!("{}MethodDescriptor", service.name),
Service_str: service.name,
Service_proto_str: service.proto_name,
Service_package_str: service.package,
methods,
}
}
fn trait_Service(&self) -> TokenStream {
let Self {
namespace,
doc,
Service,
methods,
..
} = self;
let match_json_call_method = methods.iter().map(
|Method {
method,
method_str,
method_proto_str,
Input,
..
}| {
quote! {
#method_str | #method_proto_str => {
let req: #Input = ::serde_json::from_value(json)
.map_err(|e| #namespace::error::Error::MalformatRpcPacket(format!("json error: {}", e)))?;
let resp = self.#method(ctrl, req).await?;
Ok(::serde_json::to_value(resp)
.map_err(|e| #namespace::error::Error::MalformatRpcPacket(format!("json error: {}", e)))?)
}
}
},
);
let methods = methods.iter().map(
|Method {
doc,
method,
Input,
Output,
..
}| {
quote! {
#doc
async fn #method(&self, ctrl: Self::Controller, input: #Input) -> #namespace::error::Result<#Output>;
}
},
);
quote! {
#doc
#[async_trait::async_trait]
#[auto_impl::auto_impl(&, Arc, Box)]
pub trait #Service {
type Controller: #namespace::controller::Controller;
#(#methods)*
async fn json_call_method(
&self,
ctrl: Self::Controller,
method: &str,
json: ::serde_json::Value,
) -> #namespace::error::Result<::serde_json::Value> {
match method {
#(#match_json_call_method)*
_ => Err(#namespace::error::Error::InvalidMethodIndex(0, method.to_string())),
}
}
}
}
}
fn impl_Service_for_Weak(&self) -> TokenStream {
let Self {
namespace,
Service,
methods,
..
} = self;
let methods = methods.iter().map(
|Method {
method,
Input,
Output,
..
}| {
quote! {
async fn #method(&self, ctrl: Self::Controller, input: #Input) -> #namespace::error::Result<#Output> {
let Some(service) = self.upgrade() else {
return Err(#namespace::error::Error::Shutdown);
};
service.#method(ctrl, input).await
}
}
},
);
quote! {
#[async_trait::async_trait]
impl<T> #Service for ::std::sync::Weak<T>
where
T: Send + Sync + 'static,
::std::sync::Arc<T>: #Service,
{
type Controller = <::std::sync::Arc<T> as #Service>::Controller;
#(#methods)*
}
}
}
fn struct_ServiceDescriptor(&self) -> TokenStream {
let Self {
namespace,
ServiceDescriptor,
ServiceMethodDescriptor,
Service_str,
Service_proto_str,
Service_package_str,
methods,
..
} = self;
let doc = format!("A service descriptor for a `{}`.", Service_str);
let methods = methods.iter().map(|Method { method_proto, .. }| {
quote! { #ServiceMethodDescriptor::#method_proto, }
});
quote! {
#[doc = #doc]
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Default)]
pub struct #ServiceDescriptor;
impl #namespace::descriptor::ServiceDescriptor for #ServiceDescriptor {
type Method = #ServiceMethodDescriptor;
fn name(&self) -> &'static str { #Service_str }
fn proto_name(&self) -> &'static str { #Service_proto_str }
fn package(&self) -> &'static str { #Service_package_str }
fn methods(&self) -> &'static [Self::Method] {
&[ #(#methods)* ]
}
}
}
}
fn enum_ServiceMethodDescriptor(&self) -> TokenStream {
let Self {
ServiceMethodDescriptor,
Service_str,
methods,
..
} = self;
let doc = formatdoc! {"
Methods available on a `{Service_str}`.
This can be used as a key when routing requests for servers/clients of a `{Service_str}`.
"};
let variants = methods.iter().map(
|Method {
method_proto,
index,
..
}| {
quote! { #method_proto = #index, }
},
);
let impl_MethodDescriptor = self.impl_MethodDescriptor_for_ServiceMethodDescriptor();
let impl_TryFrom = self.impl_TryFrom_for_ServiceMethodDescriptor();
quote! {
#[doc = #doc]
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
#[repr(u8)]
pub enum #ServiceMethodDescriptor {
#(#variants)*
}
#impl_MethodDescriptor
#impl_TryFrom
}
}
fn impl_MethodDescriptor_for_ServiceMethodDescriptor(&self) -> TokenStream {
let Self {
namespace,
ServiceMethodDescriptor,
methods,
..
} = self;
let name = {
let arms = methods.iter().map(
|Method {
method_proto,
method_str,
..
}| {
quote! { #ServiceMethodDescriptor::#method_proto => #method_str, }
},
);
quote! {
fn name(&self) -> &'static str {
match *self {
#(#arms)*
}
}
}
};
let proto_name = {
let arms = methods.iter().map(
|Method {
method_proto,
method_proto_str,
..
}| {
quote! { #ServiceMethodDescriptor::#method_proto => #method_proto_str, }
},
);
quote! {
fn proto_name(&self) -> &'static str {
match *self {
#(#arms)*
}
}
}
};
let input_type = {
let arms = methods.iter().map(|Method { method_proto, Input, .. }| {
quote! { #ServiceMethodDescriptor::#method_proto => ::std::any::TypeId::of::<#Input>(), }
});
quote! {
fn input_type(&self) -> ::std::any::TypeId {
match *self {
#(#arms)*
}
}
}
};
let input_proto_type = {
let arms = methods.iter().map(
|Method {
method_proto,
Input_proto_str,
..
}| {
quote! { #ServiceMethodDescriptor::#method_proto => #Input_proto_str, }
},
);
quote! {
fn input_proto_type(&self) -> &'static str {
match *self {
#(#arms)*
}
}
}
};
let output_type = {
let arms = methods.iter().map(|Method { method_proto, Output, .. }| {
quote! { #ServiceMethodDescriptor::#method_proto => ::std::any::TypeId::of::<#Output>(), }
});
quote! {
fn output_type(&self) -> ::std::any::TypeId {
match *self {
#(#arms)*
}
}
}
};
let output_proto_type = {
let arms = methods.iter().map(
|Method {
method_proto,
Output_proto_str,
..
}| {
quote! { #ServiceMethodDescriptor::#method_proto => #Output_proto_str, }
},
);
quote! {
fn output_proto_type(&self) -> &'static str {
match *self {
#(#arms)*
}
}
}
};
quote! {
impl #namespace::descriptor::MethodDescriptor for #ServiceMethodDescriptor {
#name
#proto_name
#input_type
#input_proto_type
#output_type
#output_proto_type
fn index(&self) -> u8 {
*self as u8
}
}
}
}
fn impl_TryFrom_for_ServiceMethodDescriptor(&self) -> TokenStream {
let Self {
namespace,
ServiceMethodDescriptor,
Service_str,
methods,
..
} = self;
let arms = methods.iter().map(
|Method {
method_proto,
index,
..
}| {
quote! { #index => Ok(#ServiceMethodDescriptor::#method_proto), }
},
);
quote! {
impl std::convert::TryFrom<u8> for #ServiceMethodDescriptor {
type Error = #namespace::error::Error;
fn try_from(value: u8) -> #namespace::error::Result<Self> {
match value {
#(#arms)*
_ => Err(#namespace::error::Error::InvalidMethodIndex(value, #Service_str.to_string())),
}
}
}
}
}
fn struct_ServiceClient(&self) -> TokenStream {
let Self {
namespace,
ServiceDescriptor,
ServiceClient,
Service_str,
..
} = self;
let doc = formatdoc! {"
A client for a `{Service_str}`.
This implements the `{Service_str}` trait by dispatching all method calls to the supplied `Handler`.
"};
let impl_service_client = self.impl_ServiceClient();
let impl_service_for_client = self.impl_Service_for_ServiceClient();
quote! {
#[doc = #doc]
#[derive(Clone, Debug)]
pub struct #ServiceClient<H>(H) where H: #namespace::handler::Handler;
impl<H> #ServiceClient<H> where H: #namespace::handler::Handler<Descriptor = #ServiceDescriptor> {
/// Creates a new client instance that delegates all method calls to the supplied handler.
pub fn new(handler: H) -> Self {
Self(handler)
}
}
#impl_service_client
#impl_service_for_client
}
}
fn impl_ServiceClient(&self) -> TokenStream {
let Self {
namespace,
ServiceClient,
ServiceDescriptor,
ServiceMethodDescriptor,
methods,
..
} = self;
let methods = methods.iter().map(
|Method {
method_inner,
method_proto,
Input,
Output,
..
}| {
quote! {
async fn #method_inner(handler: H, ctrl: H::Controller, input: #Input) -> #namespace::error::Result<#Output> {
#namespace::__rt::call_method(handler, ctrl, #ServiceMethodDescriptor::#method_proto, input).await
}
}
},
);
quote! {
impl<H> #ServiceClient<H> where H: #namespace::handler::Handler<Descriptor = #ServiceDescriptor> {
#(#methods)*
}
}
}
fn impl_Service_for_ServiceClient(&self) -> TokenStream {
let Self {
namespace,
Service,
ServiceClient,
ServiceDescriptor,
methods,
..
} = self;
let methods = methods.iter().map(
|Method {
method,
method_inner,
Input,
Output,
..
}| {
quote! {
async fn #method(&self, ctrl: H::Controller, input: #Input) -> #namespace::error::Result<#Output> {
#ServiceClient::#method_inner(self.0.clone(), ctrl, input).await
}
}
},
);
quote! {
#[async_trait::async_trait]
impl<H> #Service for #ServiceClient<H> where H: #namespace::handler::Handler<Descriptor = #ServiceDescriptor> {
type Controller = H::Controller;
#(#methods)*
}
}
}
fn struct_ServiceClientFactory(&self) -> TokenStream {
let Self {
namespace,
Service,
ServiceClient,
ServiceClientFactory,
ServiceDescriptor,
..
} = self;
quote! {
pub struct #ServiceClientFactory<C: #namespace::controller::Controller>(std::marker::PhantomData<C>);
impl<C: #namespace::controller::Controller> Clone for #ServiceClientFactory<C> {
fn clone(&self) -> Self {
Self(std::marker::PhantomData)
}
}
impl<C> #namespace::__rt::RpcClientFactory for #ServiceClientFactory<C> where C: #namespace::controller::Controller {
type Descriptor = #ServiceDescriptor;
type ClientImpl = Box<dyn #Service<Controller = C> + Send + Sync + 'static>;
type Controller = C;
fn new(handler: impl #namespace::handler::Handler<Descriptor = Self::Descriptor, Controller = Self::Controller>) -> Self::ClientImpl {
Box::new(#ServiceClient::new(handler))
}
}
}
}
fn struct_ServiceServer(&self) -> TokenStream {
let Self {
namespace,
Service,
ServiceDescriptor,
ServiceServer,
ServiceMethodDescriptor,
Service_str,
methods,
..
} = self;
let doc = formatdoc! {"
A server for a `{Service_str}`.
This implements the `Server` trait by handling requests and dispatch them to methods on the
supplied `{Service_str}`.
"};
let arms = methods.iter().map(
|Method {
method_proto,
method,
Input,
..
}| {
quote! {
#ServiceMethodDescriptor::#method_proto => {
let decoded: #Input = #namespace::__rt::decode(input)?;
let ret = service.#method(ctrl, decoded).await?;
#namespace::__rt::encode(ret)
}
}
},
);
quote! {
#[doc = #doc]
#[derive(Clone, Debug)]
pub struct #ServiceServer<A>(A) where A: #Service + Clone + Send + 'static;
impl<T> #ServiceServer<::std::sync::Weak<T>>
where
T: Send + Sync + 'static,
::std::sync::Arc<T>: #Service,
{
pub fn new_arc(service: ::std::sync::Arc<T>) -> #ServiceServer<::std::sync::Weak<T>> {
#ServiceServer(::std::sync::Arc::downgrade(&service))
}
}
impl<A> #ServiceServer<A> where A: #Service + Clone + Send + 'static {
/// Creates a new server instance that dispatches all calls to the supplied service.
pub fn new(service: A) -> #ServiceServer<A> {
#ServiceServer(service)
}
async fn call_inner(
service: A,
method: #ServiceMethodDescriptor,
ctrl: A::Controller,
input: ::bytes::Bytes)
-> #namespace::error::Result<::bytes::Bytes> {
match method {
#(#arms)*
}
}
}
#[async_trait::async_trait]
impl<A> #namespace::handler::Handler for #ServiceServer<A>
where
A: #Service + Clone + Send + Sync + 'static {
type Descriptor = #ServiceDescriptor;
type Controller = A::Controller;
async fn call(
&self,
ctrl: A::Controller,
method: #ServiceMethodDescriptor,
input: ::bytes::Bytes)
-> #namespace::error::Result<::bytes::Bytes> {
#ServiceServer::call_inner(self.0.clone(), method, ctrl, input).await
}
}
}
}
}
/// The service generator to be used with `prost-build` to generate RPC implementations for
/// `prost-simple-rpc`.
///
/// See the crate-level documentation for more info.
#[non_exhaustive]
#[derive(Debug, Default)]
pub struct ServiceGenerator;
impl prost_build::ServiceGenerator for ServiceGenerator {
fn generate(&mut self, service: prost_build::Service, buf: &mut String) {
let info = Service::new(service);
let trait_Service = info.trait_Service();
let impl_Service_for_Weak = info.impl_Service_for_Weak();
let struct_ServiceDescriptor = info.struct_ServiceDescriptor();
let enum_ServiceMethodDescriptor = info.enum_ServiceMethodDescriptor();
let struct_ServiceClient = info.struct_ServiceClient();
let struct_ServiceClientFactory = info.struct_ServiceClientFactory();
let struct_ServiceServer = info.struct_ServiceServer();
let tokens = quote! {
#trait_Service
#impl_Service_for_Weak
#struct_ServiceDescriptor
#enum_ServiceMethodDescriptor
#struct_ServiceClient
#struct_ServiceClientFactory
#struct_ServiceServer
};
buf.push('\n');
buf.push_str(&tokens.to_string());
buf.push('\n');
}
}