Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add autogenerated protos for v1.0 #1381

Closed
wants to merge 10 commits into from
Closed

Add autogenerated protos for v1.0 #1381

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
99e961d
Pre: auto changes
sergio-mena Nov 17, 2023
a935338
Added cometbft v1_0 to protobuf generation
sergio-mena Nov 17, 2023
95a2621
Auto generated protobuf code
sergio-mena Nov 17, 2023
659ad9c
Adapted custom attributes for cometbft types
sergio-mena Nov 17, 2023
f60e631
Auto generated protobuf code after adapting custom attributes
sergio-mena Nov 17, 2023
0e70404
Extended mod generation to support several sub-mods
sergio-mena Nov 18, 2023
42a3555
Auto generated protobuf code after extended mod generation
sergio-mena Nov 18, 2023
dd1fcb8
Fix build of protos crate
sergio-mena Nov 19, 2023
03e3aab
Changelog
sergio-mena Nov 19, 2023
f5ddeb1
Addressed @mzabaluev's comments
sergio-mena Nov 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
159 changes: 159 additions & 0 deletions tools/proto-compiler/src/constants.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ pub struct TendermintVersion {
/// - Branch: `main`
/// - Commit ID (full length): `d7d0ffea13c60c98b812d243ba5a2c375f341c15`
pub commitish: &'static str,
/// Project name
pub project: &'static str,
}

/// All Tendermint versions to generate code for
Expand All @@ -20,16 +22,25 @@ pub const TENDERMINT_VERSIONS: &[TendermintVersion] = &[
repo: "https://github.com/cometbft/cometbft",
ident: "v0_34",
commitish: "v0.34.29",
project:"tendermint",
},
TendermintVersion {
repo: "https://github.com/cometbft/cometbft",
ident: "v0_37",
commitish: "v0.37.2",
project:"tendermint",
},
TendermintVersion {
repo: "https://github.com/cometbft/cometbft",
ident: "v0_38",
commitish: "v0.38.0",
project:"tendermint",
},
TendermintVersion {
repo: "https://github.com/cometbft/cometbft",
ident: "v1_0",
commitish: "feature/proto-upgrade",
project:"cometbft",
},
];

Expand Down Expand Up @@ -109,6 +120,40 @@ pub static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[
(".tendermint.types.BlockMeta", SERIALIZED),
(".tendermint.types.TxProof", SERIALIZED),
(".tendermint.crypto.Proof", SERIALIZED),
(".cometbft.libs.bits.BitArray", SERIALIZED),//TODO Revisit
(".cometbft.types.BlockIDFlag", PRIMITIVE_ENUM),
(".cometbft.types.Block", SERIALIZED),
(".cometbft.types.Data", SERIALIZED),
(".cometbft.types.EvidenceParams", SERIALIZED),
(".cometbft.types.Evidence.sum", SERIALIZED),
(".cometbft.types.Evidence.sum", TYPE_TAG),
(".cometbft.types.EvidenceList", SERIALIZED),
(".cometbft.types.DuplicateVoteEvidence", SERIALIZED),
(".cometbft.types.Vote", SERIALIZED),
(".cometbft.types.BlockID", SERIALIZED),
(".cometbft.types.PartSetHeader", SERIALIZED),
(".cometbft.types.LightClientAttackEvidence", SERIALIZED),
(
".cometbft.types.LightClientAttackEvidence",
RENAME_ALL_PASCALCASE,
),
(".cometbft.types.LightBlock", SERIALIZED),
(".cometbft.types.SignedHeader", SERIALIZED),
(".cometbft.types.Header", SERIALIZED),
(".cometbft.version.Consensus", SERIALIZED),
(".cometbft.types.Commit", SERIALIZED),
(".cometbft.types.CommitSig", SERIALIZED),
(".cometbft.types.ValidatorSet", SERIALIZED),
(".cometbft.crypto.PublicKey.sum", SERIALIZED),
(".cometbft.crypto.PublicKey.sum", TYPE_TAG),
(".cometbft.abci.ResponseInfo", SERIALIZED),
(".cometbft.types.CanonicalBlockID", SERIALIZED),
(".cometbft.types.CanonicalPartSetHeader", SERIALIZED),
(".cometbft.types.Validator", SERIALIZED),
(".cometbft.types.CanonicalVote", SERIALIZED),
(".cometbft.types.BlockMeta", SERIALIZED),
(".cometbft.types.TxProof", SERIALIZED),
(".cometbft.crypto.Proof", SERIALIZED),
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please ignore these changes: review them in Commit 4

];

/// Custom field attributes applied on top of protobuf fields in (a) struct(s)
Expand Down Expand Up @@ -231,4 +276,118 @@ pub static CUSTOM_FIELD_ATTRIBUTES: &[(&str, &str)] = &[
(".tendermint.crypto.Proof.total", QUOTED),
(".tendermint.crypto.Proof.aunts", VEC_BASE64STRING),
(".tendermint.crypto.Proof.leaf_hash", BASE64STRING),
(
".cometbft.types.EvidenceParams.max_bytes",
QUOTED_WITH_DEFAULT,
), //TODO Revisit
(
".cometbft.types.EvidenceParams.max_age_num_blocks",
QUOTED_WITH_DEFAULT,
),
(".cometbft.version.Consensus.block", QUOTED),
(".cometbft.version.Consensus.app", QUOTED_WITH_DEFAULT),
(".cometbft.abci.ResponseInfo.data", DEFAULT),
(".cometbft.abci.ResponseInfo.version", DEFAULT),
(
".cometbft.abci.ResponseInfo.app_version",
QUOTED_WITH_DEFAULT,
),
(
".cometbft.abci.ResponseInfo.last_block_height",
QUOTED_WITH_DEFAULT,
),
(".cometbft.abci.ResponseInfo.last_block_app_hash", DEFAULT),
(
".cometbft.abci.ResponseInfo.last_block_app_hash",
BYTES_SKIP_IF_EMPTY,
),
(".cometbft.types.BlockID.hash", HEXSTRING),
(".cometbft.types.BlockID.part_set_header", RENAME_PARTS),
(
".cometbft.types.PartSetHeader.total",
PART_SET_HEADER_TOTAL,
),
(".cometbft.types.PartSetHeader.hash", HEXSTRING),
(".cometbft.types.Header.height", QUOTED),
(".cometbft.types.Header.time", OPTIONAL),
(".cometbft.types.Header.last_commit_hash", HEXSTRING),
(".cometbft.types.Header.data_hash", HEXSTRING),
(".cometbft.types.Header.validators_hash", HEXSTRING),
(".cometbft.types.Header.next_validators_hash", HEXSTRING),
(".cometbft.types.Header.consensus_hash", HEXSTRING),
(".cometbft.types.Header.app_hash", HEXSTRING),
(".cometbft.types.Header.last_results_hash", HEXSTRING),
(".cometbft.types.Header.evidence_hash", HEXSTRING),
(".cometbft.types.Header.proposer_address", HEXSTRING),
(".cometbft.types.Data.txs", NULLABLEVECARRAY),
(".cometbft.types.EvidenceList.evidence", NULLABLE),
(".cometbft.types.Commit.height", QUOTED),
(".cometbft.types.Commit.signatures", NULLABLE),
(".cometbft.types.CommitSig.validator_address", HEXSTRING),
(".cometbft.types.CommitSig.timestamp", OPTIONAL),
(".cometbft.types.CommitSig.signature", BASE64STRING),
(
".cometbft.types.DuplicateVoteEvidence.total_voting_power",
RENAME_TOTAL_VOTING_POWER_QUOTED,
),
(
".cometbft.types.DuplicateVoteEvidence.validator_power",
RENAME_VALIDATOR_POWER_QUOTED,
),
(
".cometbft.types.DuplicateVoteEvidence.timestamp",
RENAME_TIMESTAMP,
),
(
".cometbft.types.LightClientAttackEvidence.common_height",
QUOTED,
),
(
".cometbft.types.LightClientAttackEvidence.total_voting_power",
QUOTED,
),
(".cometbft.types.Vote.height", QUOTED),
(".cometbft.types.Vote.validator_address", HEXSTRING),
(".cometbft.types.Vote.signature", BASE64STRING),
(".cometbft.types.Vote.timestamp", OPTIONAL),
(".cometbft.types.Validator.address", HEXSTRING),
(
".cometbft.types.Validator.voting_power",
ALIAS_POWER_QUOTED,
), // https://github.com/tendermint/tendermint/issues/5549
(
".cometbft.types.Validator.proposer_priority",
QUOTED_ALLOW_NULL,
), // null occurs in some LightBlock data
(".cometbft.types.Validator.proposer_priority", DEFAULT), // Default is for /genesis deserialization
(
".cometbft.types.ValidatorSet.total_voting_power",
QUOTED_WITH_DEFAULT,
),
(
".cometbft.types.ValidatorSet.total_voting_power",
SKIP_SERIALIZING,
),
(".cometbft.types.BlockMeta.block_size", QUOTED),
(".cometbft.types.BlockMeta.num_txs", QUOTED),
(".cometbft.crypto.PublicKey.sum.ed25519", RENAME_EDPUBKEY),
(
".cometbft.crypto.PublicKey.sum.secp256k1",
RENAME_SECPPUBKEY,
),
(".cometbft.crypto.PublicKey.sum.sr25519", RENAME_SRPUBKEY),
(
".cometbft.types.Evidence.sum.duplicate_vote_evidence",
RENAME_DUPLICATEVOTE,
),
(
".cometbft.types.Evidence.sum.light_client_attack_evidence",
RENAME_LIGHTCLIENTATTACK,
),
(".cometbft.types.TxProof.data", BASE64STRING),
(".cometbft.types.TxProof.root_hash", HEXSTRING),
(".cometbft.crypto.Proof.index", QUOTED),
(".cometbft.crypto.Proof.total", QUOTED),
(".cometbft.crypto.Proof.aunts", VEC_BASE64STRING),
(".cometbft.crypto.Proof.leaf_hash", BASE64STRING),
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please ignore these changes: review them in Commit 4

];
22 changes: 14 additions & 8 deletions tools/proto-compiler/src/functions.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ fn find_reference_or_commit<'a>(
}

/// Copy generated files to target folder
pub fn copy_files(src_dir: &Path, target_dir: &Path) {
pub fn copy_files(src_dir: &Path, target_dir: &Path, project:&str) {
// Remove old compiled files
remove_dir_all(target_dir).unwrap_or_default();
create_dir_all(target_dir).unwrap();
Expand All @@ -172,7 +172,7 @@ pub fn copy_files(src_dir: &Path, target_dir: &Path) {
e.file_type().is_file()
&& e.file_name()
.to_str()
.map(|name| name.starts_with("tendermint."))
.map(|name| name.starts_with(&format!("{project}.")))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Performance nit: it would be better if the prefix string was formatted once, not with every invocation of the closure.

.unwrap_or(false)
})
.map(|res| {
Expand Down Expand Up @@ -212,12 +212,14 @@ pub fn find_proto_files(proto_path: &Path) -> Vec<PathBuf> {
/// Tendermint source version.
pub fn generate_tendermint_mod(prost_dir: &Path, version: &TendermintVersion, target_dir: &Path) {
create_dir_all(target_dir).unwrap();

let project_dot = format!("{}.", version.project);
let file_names = WalkDir::new(prost_dir)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| {
e.file_type().is_file()
&& e.file_name().to_str().unwrap().starts_with("tendermint.")
&& e.file_name().to_str().unwrap().starts_with(&project_dot)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right, like here.

&& e.file_name().to_str().unwrap().ends_with(".rs")
})
.map(|d| d.file_name().to_str().unwrap().to_owned())
Expand All @@ -230,7 +232,7 @@ pub fn generate_tendermint_mod(prost_dir: &Path, version: &TendermintVersion, ta

for file_name in file_names {
let parts: Vec<_> = file_name
.strip_prefix("tendermint.")
.strip_prefix(&project_dot)
.unwrap()
.strip_suffix(".rs")
.unwrap()
Expand Down Expand Up @@ -276,12 +278,16 @@ pub fn generate_tendermint_mod(prost_dir: &Path, version: &TendermintVersion, ta
.expect("tendermint module file write failed");
}

pub fn generate_tendermint_lib(versions: &[TendermintVersion], tendermint_lib_target: &Path) {
pub fn generate_tendermint_lib(versions: &[TendermintVersion], tendermint_lib_target_dir: &Path, project: &str, write_use: bool) {
let tendermint_lib_target = tendermint_lib_target_dir.join(format!("{project}.rs"));
let mut file =
File::create(tendermint_lib_target).expect("tendermint library file create failed");
for version in versions {
let project_versions = versions.iter().filter(|v| v.project == project).collect::<Vec<_>>();
for version in &project_versions {
writeln!(&mut file, "pub mod {};", version.ident).unwrap();
}
let last_version = versions.last().unwrap();
writeln!(&mut file, "pub use {}::*;", last_version.ident).unwrap();
if write_use {
let last_version = project_versions.last().unwrap();
writeln!(&mut file, "pub use {}::*;", last_version.ident).unwrap();
}
}
28 changes: 16 additions & 12 deletions tools/proto-compiler/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,25 +21,25 @@ use constants::{CUSTOM_FIELD_ATTRIBUTES, CUSTOM_TYPE_ATTRIBUTES, TENDERMINT_VERS
fn main() {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let target_dir = ["..", "..", "proto", "src"].iter().collect::<PathBuf>();
let tendermint_dir = PathBuf::from(var("TENDERMINT_DIR").unwrap_or_else(|_| {
let comet_dir = PathBuf::from(var("COMETBFT_DIR").unwrap_or_else(|_| {
root.join("..")
.join("target")
.join("tendermint")
.join("cometbft")
.to_str()
.unwrap()
.to_string()
}));

for version in TENDERMINT_VERSIONS {
println!(
"[info] => Fetching {} at {} into {tendermint_dir:?}",
"[info] => Fetching {} at {} into {comet_dir:?}",
version.repo, version.commitish,
);
get_commitish(&tendermint_dir, &version.repo, &version.commitish); // This panics if it fails.
get_commitish(&comet_dir, &version.repo, &version.commitish); // This panics if it fails.

let proto_path = tendermint_dir.join("proto");
let proto_path = comet_dir.join("proto");

let mut proto_includes_paths = vec![tendermint_dir.join("proto")];
let mut proto_includes_paths = vec![comet_dir.join("proto")];

let buf_lock_path = proto_path.join("buf.lock");
let _temp_dirs = if fs::metadata(&buf_lock_path).is_ok() {
Expand Down Expand Up @@ -68,15 +68,15 @@ fn main() {
}
} else {
// Old school, assume the dependency protos are bundled in the tree.
proto_includes_paths.push(tendermint_dir.join("third_party").join("proto"));
proto_includes_paths.push(comet_dir.join("third_party").join("proto"));
vec![]
};

// List available proto files
let protos = find_proto_files(&proto_path);

let ver_target_dir = target_dir.join("prost").join(version.ident);
let ver_module_dir = target_dir.join("tendermint");
let ver_module_dir = target_dir.join(version.project);

let out_dir = var("OUT_DIR")
.map(|d| Path::new(&d).join(version.ident))
Expand All @@ -86,7 +86,7 @@ fn main() {
let mut pb = prost_build::Config::new();

// Use shared Bytes buffers for ABCI messages:
pb.bytes([".tendermint.abci"]);
pb.bytes([".tendermint.abci", ".cometbft.abci"]);

// Compile proto files with added annotations, exchange prost_types to our own
pb.out_dir(&out_dir);
Expand All @@ -113,6 +113,8 @@ fn main() {
.out_dir(&out_dir)
.build_server(true)
.build_client(false)
.server_mod_attribute("cometbft.abci", "#[cfg(feature = \"grpc-server\")]") //TODO Revisit
.server_mod_attribute("cometbft.rpc.grpc", "#[cfg(feature = \"grpc-server\")]")
.server_mod_attribute("tendermint.abci", "#[cfg(feature = \"grpc-server\")]")
.server_mod_attribute("tendermint.rpc.grpc", "#[cfg(feature = \"grpc-server\")]");
// TODO: this is tracked in https://github.com/informalsystems/tendermint-rs/issues/1134
Expand All @@ -130,10 +132,12 @@ fn main() {
"[info] => Removing old structs and copying new structs to {}",
ver_target_dir.to_string_lossy(),
);
copy_files(&out_dir, &ver_target_dir); // This panics if it fails.
generate_tendermint_mod(&out_dir, version, &ver_module_dir);
copy_files(&out_dir, &ver_target_dir, version.project); // This panics if it fails.
generate_tendermint_mod(&out_dir, version, &ver_module_dir)
;
}
generate_tendermint_lib(TENDERMINT_VERSIONS, &target_dir.join("tendermint.rs"));
generate_tendermint_lib(TENDERMINT_VERSIONS, &target_dir, "tendermint", false);
generate_tendermint_lib(TENDERMINT_VERSIONS, &target_dir, "cometbft", true);

println!("[info] => Done!");
}