From 13cc70d9cdf970e1b9e3e0d53da79166d520cecc Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 16 Apr 2026 14:18:58 -0700 Subject: [PATCH 01/18] Expose generator_interned_weight() to Python Split the interned tree cost calculation into two parts: - interned_weight(): returns atom_bytes + 2*atoms + 3*pairs - total_cost_from_tree(): interned_weight() * COST_PER_BYTE The new generator_interned_weight() Python function deserializes a generator program, interns the tree, and returns the raw weight. Callers multiply by the COST_PER_BYTE consensus constant themselves, avoiding a hardcoded multiplier on the Python side. Made-with: Cursor --- crates/chia-consensus/src/generator_cost.rs | 28 +++++++++++++++------ wheel/python/chia_rs/chia_rs.pyi | 2 ++ wheel/src/api.rs | 4 ++- wheel/src/run_generator.rs | 19 ++++++++++++++ 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/crates/chia-consensus/src/generator_cost.rs b/crates/chia-consensus/src/generator_cost.rs index b02c55423..0f11cbd2e 100644 --- a/crates/chia-consensus/src/generator_cost.rs +++ b/crates/chia-consensus/src/generator_cost.rs @@ -1,6 +1,11 @@ //! Chia-specific generator cost calculation. //! -//! Pure storage model: cost = (atom_bytes + 2*atoms + 3*pairs) * COST_PER_BYTE. +//! Pure storage model: cost = interned_weight(tree) * COST_PER_BYTE. +//! The weight `atom_bytes + 2*atoms + 3*pairs` is an upper bound on the +//! serialized byte count. COST_PER_BYTE (12000) comes from the consensus +//! constants, so `total_cost_from_tree` hardcodes it only for internal use; +//! callers who need the pre-multiplied weight can use `interned_weight`. +//! //! SHA tree-hash cost is not charged separately — it is structurally bounded //! by the size component (worst-case ratio <= 3.33, ~37ms SHA CPU on a 2012 //! Celeron). See PR #1371 for the alternative split model (6000/4500). @@ -9,13 +14,14 @@ use clvmr::serde::InternedTree; const COST_PER_BYTE: u64 = 12000; -/// Compute total generator cost from an interned tree. +/// Return the byte-weight-equivalent of an interned tree: +/// `atom_bytes + 2*atom_count + 3*pair_count`. /// -/// The size formula `atom_bytes + 2*atom_count + 3*pair_count` is proven to -/// be an upper bound on the serialized byte count (P=3 accounts for pair -/// opcodes and back-reference overhead). +/// Multiply by `COST_PER_BYTE` (consensus constant, currently 12000) to get +/// the full generator size cost. This is deliberately separated so Python +/// callers can reuse the consensus constant instead of hardcoding a multiplier. #[inline] -pub fn total_cost_from_tree(tree: &InternedTree) -> u64 { +pub fn interned_weight(tree: &InternedTree) -> u64 { let atom_count = tree.atoms.len() as u64; let pair_count = tree.pairs.len() as u64; @@ -24,7 +30,15 @@ pub fn total_cost_from_tree(tree: &InternedTree) -> u64 { atom_bytes += tree.allocator.atom_len(atom) as u64; } - (atom_bytes + 2 * atom_count + 3 * pair_count) * COST_PER_BYTE + atom_bytes + 2 * atom_count + 3 * pair_count +} + +/// Compute total generator cost from an interned tree. +/// +/// Equivalent to `interned_weight(tree) * COST_PER_BYTE`. +#[inline] +pub fn total_cost_from_tree(tree: &InternedTree) -> u64 { + interned_weight(tree) * COST_PER_BYTE } #[cfg(test)] diff --git a/wheel/python/chia_rs/chia_rs.pyi b/wheel/python/chia_rs/chia_rs.pyi index b94a2b803..b72660b30 100644 --- a/wheel/python/chia_rs/chia_rs.pyi +++ b/wheel/python/chia_rs/chia_rs.pyi @@ -31,6 +31,8 @@ def run_block_generator2( program: ReadableBuffer, block_refs: list[ReadableBuffer], max_cost: int, flags: int, signature: G2Element, bls_cache: Optional[BLSCache], constants: ConsensusConstants ) -> tuple[Optional[int], Optional[SpendBundleConditions]]: ... +def generator_interned_weight(program: ReadableBuffer) -> int: ... + def additions_and_removals( program: ReadableBuffer, block_refs: list[ReadableBuffer], flags: int, constants: ConsensusConstants ) -> tuple[list[tuple[Coin, Optional[bytes]]], list[tuple[bytes32, Coin]]]: ... diff --git a/wheel/src/api.rs b/wheel/src/api.rs index 71928611b..7485ffd85 100644 --- a/wheel/src/api.rs +++ b/wheel/src/api.rs @@ -1,6 +1,7 @@ use crate::error::{map_pyerr, map_pyerr_w_ptr}; use crate::run_generator::{ - additions_and_removals, py_to_slice, run_block_generator, run_block_generator2, + additions_and_removals, generator_interned_weight, py_to_slice, run_block_generator, + run_block_generator2, }; use chia_consensus::allocator::make_allocator; use chia_consensus::build_compressed_block::BlockBuilder; @@ -762,6 +763,7 @@ pub fn chia_rs(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { // generator functions m.add_function(wrap_pyfunction!(run_block_generator, m)?)?; m.add_function(wrap_pyfunction!(run_block_generator2, m)?)?; + m.add_function(wrap_pyfunction!(generator_interned_weight, m)?)?; m.add_function(wrap_pyfunction!(additions_and_removals, m)?)?; m.add_function(wrap_pyfunction!(solution_generator, m)?)?; m.add_function(wrap_pyfunction!(solution_generator_backrefs, m)?)?; diff --git a/wheel/src/run_generator.rs b/wheel/src/run_generator.rs index 2f8336b8b..8d076ade9 100644 --- a/wheel/src/run_generator.rs +++ b/wheel/src/run_generator.rs @@ -2,13 +2,16 @@ use chia_bls::{BlsCache, Signature}; use chia_consensus::additions_and_removals::additions_and_removals as native_additions_and_removals; use chia_consensus::consensus_constants::ConsensusConstants; use chia_consensus::flags::ConsensusFlags; +use chia_consensus::generator_cost::interned_weight; use chia_consensus::owned_conditions::OwnedSpendBundleConditions; use chia_consensus::run_block_generator::run_block_generator as native_run_block_generator; use chia_consensus::run_block_generator::run_block_generator2 as native_run_block_generator2; use chia_consensus::validation_error::ValidationErr; use chia_protocol::{Bytes, Bytes32, Coin}; +use clvmr::allocator::Allocator; use clvmr::cost::Cost; +use clvmr::serde::{intern_tree_limited, node_from_bytes_backrefs}; use pyo3::PyResult; use pyo3::buffer::PyBuffer; @@ -138,3 +141,19 @@ pub fn additions_and_removals<'a>( }) }) } + +/// Return the byte-weight-equivalent of a serialized generator program. +/// +/// Deserializes (with back-refs), interns the tree, and returns +/// `atom_bytes + 2*atom_count + 3*pair_count`. Multiply by +/// `COST_PER_BYTE` to get the full generator size cost. +#[pyfunction] +pub fn generator_interned_weight(program: PyBuffer) -> PyResult { + let program = py_to_slice(program); + let mut a = Allocator::new(); + let node = node_from_bytes_backrefs(&mut a, program) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("bad generator: {e}")))?; + let tree = intern_tree_limited(&a, node, u32::MAX as usize) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("intern failed: {e}")))?; + Ok(interned_weight(&tree)) +} From 98c31bd13ae83cddc03da0355da0a9358479fb91 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 21 Apr 2026 21:13:03 -0700 Subject: [PATCH 02/18] Add generator_interned_weight to type stub generator The .pyi had it manually but generate_type_stubs.py didn't, causing the Check chia_rs.pyi CI job to fail. Made-with: Cursor --- wheel/generate_type_stubs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wheel/generate_type_stubs.py b/wheel/generate_type_stubs.py index da88de9d8..feca5f53b 100644 --- a/wheel/generate_type_stubs.py +++ b/wheel/generate_type_stubs.py @@ -338,6 +338,8 @@ def run_block_generator2( program: ReadableBuffer, block_refs: list[ReadableBuffer], max_cost: int, flags: int, signature: G2Element, bls_cache: Optional[BLSCache], constants: ConsensusConstants ) -> tuple[Optional[int], Optional[SpendBundleConditions]]: ... +def generator_interned_weight(program: ReadableBuffer) -> int: ... + def additions_and_removals( program: ReadableBuffer, block_refs: list[ReadableBuffer], flags: int, constants: ConsensusConstants ) -> tuple[list[tuple[Coin, Optional[bytes]]], list[tuple[bytes32, Coin]]]: ... From 48b8bc91d00a9bde367b027b18bb5227ae86c33d Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 31 Mar 2026 12:31:22 -0700 Subject: [PATCH 03/18] add InternedBlockBuilder for INTERNED_GENERATOR cost model A clean, separate builder that avoids the Serializer/sentinel/restore complexity of BlockBuilder. Cost is computed from total_cost_from_tree on the interned quoted generator tree after each add; serialization happens once in finalize() via node_to_bytes_backrefs. Includes Python bindings matching the BlockBuilder interface. Made-with: Cursor --- .../src/build_interned_block.rs | 203 ++++++++++++++++++ crates/chia-consensus/src/lib.rs | 1 + 2 files changed, 204 insertions(+) create mode 100644 crates/chia-consensus/src/build_interned_block.rs diff --git a/crates/chia-consensus/src/build_interned_block.rs b/crates/chia-consensus/src/build_interned_block.rs new file mode 100644 index 000000000..140a644c6 --- /dev/null +++ b/crates/chia-consensus/src/build_interned_block.rs @@ -0,0 +1,203 @@ +use crate::consensus_constants::ConsensusConstants; +use crate::error::Result; +use crate::generator_cost::total_cost_from_tree; +use chia_bls::Signature; +use chia_protocol::SpendBundle; +use clvmr::allocator::{Allocator, NodePtr}; +use clvmr::serde::{intern_tree_limited, node_from_bytes_backrefs, node_to_bytes_backrefs}; +use std::borrow::Borrow; + +#[cfg(feature = "py-bindings")] +use pyo3::prelude::*; +#[cfg(feature = "py-bindings")] +use pyo3::types::PyList; + +const MAX_SKIPPED_ITEMS: u32 = 6; +const MIN_COST_THRESHOLD: u64 = 6_000_000; + +#[derive(PartialEq)] +pub enum BuildBlockResult { + KeepGoing, + Done, +} + +fn result(num_skipped: u32) -> BuildBlockResult { + if num_skipped > MAX_SKIPPED_ITEMS { + BuildBlockResult::Done + } else { + BuildBlockResult::KeepGoing + } +} + +/// Builds a block generator under the INTERNED_GENERATOR cost model. +/// +/// Unlike `BlockBuilder`, serialization cost is not computed incrementally. +/// Cost comes from `total_cost_from_tree(intern_tree_limited(...))` on the +/// full quoted generator tree. Serialization happens once in `finalize()`. +#[cfg_attr(feature = "py-bindings", pyclass)] +pub struct InternedBlockBuilder { + allocator: Allocator, + signature: Signature, + spend_list: NodePtr, + block_cost: u64, + generator_cost: u64, + num_skipped: u32, +} + +impl InternedBlockBuilder { + pub fn new() -> Result { + let a = Allocator::new(); + let spend_list = a.nil(); + Ok(Self { + allocator: a, + signature: Signature::default(), + spend_list, + block_cost: 20, + generator_cost: 0, + num_skipped: 0, + }) + } + + fn compute_generator_cost(allocator: &mut Allocator, spend_list: NodePtr) -> Result { + // Build (q . ((spend_list))) + let inner = allocator.new_pair(spend_list, allocator.nil())?; + let outer = allocator.new_pair(allocator.one(), inner)?; + let interned = intern_tree_limited(allocator, outer, usize::MAX)?; + Ok(total_cost_from_tree(&interned)) + } + + /// Add a batch of spend bundles. `cost` must be execution + conditions cost + /// only (no byte cost). Returns `(added, BuildBlockResult)`. + pub fn add_spend_bundles( + &mut self, + bundles: T, + cost: u64, + constants: &ConsensusConstants, + ) -> Result<(bool, BuildBlockResult)> + where + T: IntoIterator, + S: Borrow, + { + if self.generator_cost + self.block_cost + MIN_COST_THRESHOLD + > constants.max_block_cost_clvm + { + self.num_skipped += 1; + return Ok((false, BuildBlockResult::Done)); + } + + if self.generator_cost + self.block_cost + cost > constants.max_block_cost_clvm { + self.num_skipped += 1; + return Ok((false, result(self.num_skipped))); + } + + let saved_spend_list = self.spend_list; + let a = &mut self.allocator; + + let mut cumulative_signature = Signature::default(); + for bundle in bundles { + for spend in &bundle.borrow().coin_spends { + let solution = node_from_bytes_backrefs(a, spend.solution.as_ref())?; + let item = a.new_pair(solution, NodePtr::NIL)?; + let amount = a.new_number(spend.coin.amount.into())?; + let item = a.new_pair(amount, item)?; + let puzzle = node_from_bytes_backrefs(a, spend.puzzle_reveal.as_ref())?; + let item = a.new_pair(puzzle, item)?; + let parent_id = a.new_atom(&spend.coin.parent_coin_info)?; + let item = a.new_pair(parent_id, item)?; + self.spend_list = a.new_pair(item, self.spend_list)?; + } + cumulative_signature.aggregate(&bundle.borrow().aggregated_signature); + } + + let new_generator_cost = + Self::compute_generator_cost(&mut self.allocator, self.spend_list)?; + + if new_generator_cost + self.block_cost + cost > constants.max_block_cost_clvm { + // Restore: the allocator is not reset (dead nodes are acceptable). + self.spend_list = saved_spend_list; + self.num_skipped += 1; + return Ok((false, result(self.num_skipped))); + } + + self.generator_cost = new_generator_cost; + self.block_cost += cost; + self.signature.aggregate(&cumulative_signature); + + let done = self.generator_cost + self.block_cost + MIN_COST_THRESHOLD + > constants.max_block_cost_clvm; + Ok(( + true, + if done { + BuildBlockResult::Done + } else { + BuildBlockResult::KeepGoing + }, + )) + } + + pub fn cost(&self) -> u64 { + self.generator_cost + self.block_cost + } + + /// Serialize the generator once and return `(bytes, signature, total_cost)`. + pub fn finalize(mut self, constants: &ConsensusConstants) -> Result<(Vec, Signature, u64)> { + let inner = self + .allocator + .new_pair(self.spend_list, self.allocator.nil())?; + let root = self.allocator.new_pair(self.allocator.one(), inner)?; + + let serialized = node_to_bytes_backrefs(&self.allocator, root)?; + + let generator_cost = Self::compute_generator_cost(&mut self.allocator, self.spend_list)?; + let total_cost = generator_cost + self.block_cost; + + assert!(total_cost <= constants.max_block_cost_clvm); + Ok((serialized, self.signature, total_cost)) + } +} + +#[cfg(feature = "py-bindings")] +#[pymethods] +impl InternedBlockBuilder { + #[new] + pub fn py_new() -> PyResult { + Ok(Self::new()?) + } + + #[pyo3(name = "add_spend_bundles")] + pub fn py_add_spend_bundle( + &mut self, + bundles: &Bound<'_, PyList>, + cost: u64, + constants: &ConsensusConstants, + ) -> PyResult<(bool, bool)> { + let (added, result) = self.add_spend_bundles( + bundles.iter().map(|item| { + item.extract::>() + .expect("spend bundle") + .get() + .clone() + }), + cost, + constants, + )?; + let done = matches!(result, BuildBlockResult::Done); + Ok((added, done)) + } + + #[pyo3(name = "cost")] + pub fn py_cost(&self) -> u64 { + self.cost() + } + + #[pyo3(name = "finalize")] + pub fn py_finalize( + &mut self, + constants: &ConsensusConstants, + ) -> PyResult<(Vec, Signature, u64)> { + let mut temp = InternedBlockBuilder::new()?; + std::mem::swap(self, &mut temp); + let (generator, sig, cost) = temp.finalize(constants)?; + Ok((generator, sig, cost)) + } +} diff --git a/crates/chia-consensus/src/lib.rs b/crates/chia-consensus/src/lib.rs index a7df7b7ec..044234211 100644 --- a/crates/chia-consensus/src/lib.rs +++ b/crates/chia-consensus/src/lib.rs @@ -4,6 +4,7 @@ pub mod additions_and_removals; pub mod allocator; pub mod build_compressed_block; +pub mod build_interned_block; pub mod check_time_locks; mod coin_id; mod condition_sanitizers; From 993538f5321c0801c28c510e3e9209876db968f0 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 31 Mar 2026 20:21:15 -0700 Subject: [PATCH 04/18] Wire serde_2026 serialization into chia_rs - Patch workspace clvmr to Chia-Network/clvm_rs@serde_2026 (v0.17.1, feature ser-2026) and lower version requirement from 0.17.4 to 0.17 - Add solution_generator_2026() using node_to_bytes_serde_2026 - Switch INTERNED_GENERATOR path in run_block_generator2 from node_from_bytes_backrefs to node_from_bytes_auto (auto-detect format) - Replace intern_tree_limited / intern_tree (not in serde_2026 branch) with intern() in run_block_generator, spendbundle_conditions, generator_cost tests - Drop ClvmFlags::LIMITS / ClvmFlags::MALACHITE mappings in flags.rs; those flags were added in clvmr 0.17.4 which post-dates serde_2026 - Add Python binding solution_generator_2026 + pyi / stub stubs Made-with: Cursor --- Cargo.toml | 5 +++++ crates/chia-consensus/src/flags.rs | 6 ------ crates/chia-consensus/src/generator_cost.rs | 10 +++++----- crates/chia-consensus/src/run_block_generator.rs | 9 ++++++--- crates/chia-consensus/src/solution_generator.rs | 14 +++++++++++++- .../chia-consensus/src/spendbundle_conditions.rs | 4 ++-- wheel/generate_type_stubs.py | 1 + wheel/python/chia_rs/chia_rs.pyi | 1 + wheel/src/api.rs | 11 +++++++++++ 9 files changed, 44 insertions(+), 17 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 08e5bd7fc..3dcbc6a8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,6 +110,11 @@ clvm-utils = ["dep:clvm-utils"] openssl = ["chia-sha2/openssl", "clvmr/openssl"] +[patch.crates-io] +# Pin clvmr to clvm_rs PR #708 (serde_2026) until the format ships in a +# crates.io release. Bump this rev in lockstep with that branch. +clvmr = { git = "https://github.com/Chia-Network/clvm_rs", rev = "2104989d4a9fc6e21e1789c71bf8cb4599ab31cf" } + [profile.release] lto = "thin" diff --git a/crates/chia-consensus/src/flags.rs b/crates/chia-consensus/src/flags.rs index 2c19a873b..56940a958 100644 --- a/crates/chia-consensus/src/flags.rs +++ b/crates/chia-consensus/src/flags.rs @@ -95,9 +95,6 @@ impl ConsensusFlags { if clvm.contains(ClvmFlags::ENABLE_SECP_OPS) { out = out.union(ConsensusFlags::ENABLE_SECP_OPS); } - if clvm.contains(ClvmFlags::MALACHITE) { - out = out.union(ConsensusFlags::MALACHITE); - } out } @@ -136,9 +133,6 @@ impl ConsensusFlags { if self.contains(ConsensusFlags::ENABLE_SECP_OPS) { out.insert(ClvmFlags::ENABLE_SECP_OPS); } - if self.contains(ConsensusFlags::MALACHITE) { - out.insert(ClvmFlags::MALACHITE); - } out } } diff --git a/crates/chia-consensus/src/generator_cost.rs b/crates/chia-consensus/src/generator_cost.rs index 0f11cbd2e..90709de8c 100644 --- a/crates/chia-consensus/src/generator_cost.rs +++ b/crates/chia-consensus/src/generator_cost.rs @@ -45,13 +45,13 @@ pub fn total_cost_from_tree(tree: &InternedTree) -> u64 { mod tests { use super::*; use clvmr::allocator::Allocator; - use clvmr::serde::intern_tree; + use clvmr::serde::intern; #[test] fn test_empty_atom() { let allocator = Allocator::new(); let node = allocator.nil(); - let tree = intern_tree(&allocator, node).unwrap(); + let tree = intern(&allocator, node).unwrap(); assert_eq!(total_cost_from_tree(&tree), 24_000); } @@ -61,7 +61,7 @@ mod tests { let left = allocator.new_atom(&[1, 2, 3]).unwrap(); let right = allocator.new_atom(&[4, 5, 6]).unwrap(); let node = allocator.new_pair(left, right).unwrap(); - let tree = intern_tree(&allocator, node).unwrap(); + let tree = intern(&allocator, node).unwrap(); assert_eq!(total_cost_from_tree(&tree), 156_000); } @@ -70,7 +70,7 @@ mod tests { let mut allocator = Allocator::new(); let atom = allocator.new_atom(&[42]).unwrap(); let node = allocator.new_pair(atom, atom).unwrap(); - let tree = intern_tree(&allocator, node).unwrap(); + let tree = intern(&allocator, node).unwrap(); assert_eq!(total_cost_from_tree(&tree), 72_000); } @@ -79,7 +79,7 @@ mod tests { let mut allocator = Allocator::new(); let atom = allocator.new_atom(&[1, 2, 3, 4, 5]).unwrap(); let node = allocator.new_pair(atom, allocator.nil()).unwrap(); - let tree = intern_tree(&allocator, node).unwrap(); + let tree = intern(&allocator, node).unwrap(); assert_eq!(total_cost_from_tree(&tree), 144_000); } } diff --git a/crates/chia-consensus/src/run_block_generator.rs b/crates/chia-consensus/src/run_block_generator.rs index e2c5220c2..dde47cfbb 100644 --- a/crates/chia-consensus/src/run_block_generator.rs +++ b/crates/chia-consensus/src/run_block_generator.rs @@ -24,7 +24,9 @@ use clvmr::chia_dialect::ChiaDialect; use clvmr::cost::Cost; use clvmr::reduction::Reduction; use clvmr::run_program::run_program; -use clvmr::serde::{InternedTree, intern_tree_limited, node_from_bytes, node_from_bytes_backrefs}; +use clvmr::serde::{ + InternedTree, intern, node_from_bytes, node_from_bytes_auto, node_from_bytes_backrefs, +}; pub fn subtract_cost( a: &Allocator, @@ -233,8 +235,9 @@ where let (mut a, base_cost, program) = if flags.contains(ConsensusFlags::INTERNED_GENERATOR) { let mut decode_allocator = Allocator::new(); - let program_node = node_from_bytes_backrefs(&mut decode_allocator, program)?; - let interned = intern_tree_limited(&decode_allocator, program_node, u32::MAX as usize) + let program_node = node_from_bytes_auto(&mut decode_allocator, program) + .map_err(|_| ValidationErr(NodePtr::NIL, ErrorCode::GeneratorRuntimeError))?; + let interned = intern(&decode_allocator, program_node) .map_err(|_| ValidationErr(NodePtr::NIL, ErrorCode::GeneratorRuntimeError))?; let cost = total_cost_from_tree(&interned); let InternedTree { diff --git a/crates/chia-consensus/src/solution_generator.rs b/crates/chia-consensus/src/solution_generator.rs index f13c933a4..b360f74fb 100644 --- a/crates/chia-consensus/src/solution_generator.rs +++ b/crates/chia-consensus/src/solution_generator.rs @@ -2,7 +2,9 @@ use crate::error::Result; use chia_protocol::Coin; use chia_protocol::CoinSpend; use clvmr::allocator::{Allocator, NodePtr}; -use clvmr::serde::{node_from_bytes_backrefs, node_to_bytes, node_to_bytes_backrefs}; +use clvmr::serde::{ + node_from_bytes_backrefs, node_to_bytes, node_to_bytes_backrefs, node_to_bytes_serde_2026, +}; /// the tuple has the Coin, puzzle-reveal and solution pub(crate) fn build_generator(a: &mut Allocator, spends: I) -> Result @@ -106,6 +108,16 @@ where Ok(node_to_bytes_backrefs(&a, generator)?) } +pub fn solution_generator_2026(spends: I) -> Result> +where + BufRef: AsRef<[u8]>, + I: IntoIterator, +{ + let mut a = Allocator::new(); + let generator = build_generator(&mut a, spends)?; + Ok(node_to_bytes_serde_2026(&a, generator)?) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/chia-consensus/src/spendbundle_conditions.rs b/crates/chia-consensus/src/spendbundle_conditions.rs index 8f8e860af..f4ea177ff 100644 --- a/crates/chia-consensus/src/spendbundle_conditions.rs +++ b/crates/chia-consensus/src/spendbundle_conditions.rs @@ -21,7 +21,7 @@ use clvmr::allocator::Allocator; use clvmr::chia_dialect::ChiaDialect; use clvmr::reduction::Reduction; use clvmr::run_program::run_program; -use clvmr::serde::intern_tree_limited; +use clvmr::serde::intern; use clvmr::serde::node_from_bytes; const QUOTE_BYTES: usize = 2; @@ -64,7 +64,7 @@ fn calculate_base_cost( .map(|cs| (cs.coin, cs.puzzle_reveal.as_slice(), cs.solution.as_slice())), ) .map_err(|_| ValidationErr(a.nil(), ErrorCode::GeneratorRuntimeError))?; - let interned = intern_tree_limited(&gen_allocator, generator, u32::MAX as usize) + let interned = intern(&gen_allocator, generator) .map_err(|_| ValidationErr(NodePtr::NIL, ErrorCode::GeneratorRuntimeError))?; Ok(total_cost_from_tree(&interned)) } else { diff --git a/wheel/generate_type_stubs.py b/wheel/generate_type_stubs.py index feca5f53b..675c4dec8 100644 --- a/wheel/generate_type_stubs.py +++ b/wheel/generate_type_stubs.py @@ -322,6 +322,7 @@ class _Unspec: def solution_generator(spends: Sequence[tuple[Coin, bytes, bytes]]) -> bytes: ... def solution_generator_backrefs(spends: Sequence[tuple[Coin, bytes, bytes]]) -> bytes: ... +def solution_generator_2026(spends: Sequence[tuple[Coin, bytes, bytes]]) -> bytes: ... def is_canonical_serialization(buf: bytes) -> bool: ... diff --git a/wheel/python/chia_rs/chia_rs.pyi b/wheel/python/chia_rs/chia_rs.pyi index b72660b30..d0f3bcc46 100644 --- a/wheel/python/chia_rs/chia_rs.pyi +++ b/wheel/python/chia_rs/chia_rs.pyi @@ -15,6 +15,7 @@ class _Unspec: def solution_generator(spends: Sequence[tuple[Coin, bytes, bytes]]) -> bytes: ... def solution_generator_backrefs(spends: Sequence[tuple[Coin, bytes, bytes]]) -> bytes: ... +def solution_generator_2026(spends: Sequence[tuple[Coin, bytes, bytes]]) -> bytes: ... def is_canonical_serialization(buf: bytes) -> bool: ... diff --git a/wheel/src/api.rs b/wheel/src/api.rs index 7485ffd85..26a615dcc 100644 --- a/wheel/src/api.rs +++ b/wheel/src/api.rs @@ -16,6 +16,7 @@ use chia_consensus::run_block_generator::{ get_coinspends_for_trusted_block, get_coinspends_with_conditions_for_trusted_block, }; use chia_consensus::solution_generator::solution_generator as native_solution_generator; +use chia_consensus::solution_generator::solution_generator_2026 as native_solution_generator_2026; use chia_consensus::solution_generator::solution_generator_backrefs as native_solution_generator_backrefs; use chia_consensus::spendbundle_conditions::get_conditions_from_spendbundle; use chia_consensus::spendbundle_validation::{ @@ -304,6 +305,15 @@ fn solution_generator_backrefs<'p>( )) } +#[pyfunction] +fn solution_generator_2026<'p>( + py: Python<'p>, + spends: &Bound<'_, PyAny>, +) -> PyResult> { + let spends = convert_list_of_tuples(spends)?; + Ok(PyBytes::new(py, &native_solution_generator_2026(spends)?)) +} + #[pyclass] struct AugSchemeMPL {} @@ -767,6 +777,7 @@ pub fn chia_rs(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(additions_and_removals, m)?)?; m.add_function(wrap_pyfunction!(solution_generator, m)?)?; m.add_function(wrap_pyfunction!(solution_generator_backrefs, m)?)?; + m.add_function(wrap_pyfunction!(solution_generator_2026, m)?)?; m.add_function(wrap_pyfunction!(supports_fast_forward, m)?)?; m.add_function(wrap_pyfunction!(fast_forward_singleton, m)?)?; m.add_class::()?; From 54a32a05829b770cbefaa6f0a5f9f5228fdbfce4 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 23 Apr 2026 13:09:14 -0700 Subject: [PATCH 05/18] Add Block2026Builder + switch non-consensus readers to node_from_bytes_auto Block2026Builder: anytime block builder for post-HF2 (serde_2026). Accepts candidates in priority order, packs greedily with upper-bound cost estimates, validates with exact interned_weight costing, then refines in a background thread. Caller retrieves best block instantly via best() which also returns included_indices for metadata tracking. Usable as Python context manager. Non-consensus CLVM readers (additions_and_removals, get_puzzle_and_solution, get_coinspends_for_trusted_block, run_chia_program, generator_interned_weight) switched from node_from_bytes_backrefs to node_from_bytes_auto so they accept classic, backrefs, and serde_2026 formats transparently. Consensus-critical paths unchanged (gated on INTERNED_GENERATOR flag). Made-with: Cursor --- .../src/additions_and_removals.rs | 4 +- crates/chia-consensus/src/build_block_2026.rs | 740 ++++++++++++++++++ crates/chia-consensus/src/lib.rs | 1 + .../chia-consensus/src/run_block_generator.rs | 4 +- wheel/generate_type_stubs.py | 16 + wheel/src/api.rs | 10 +- wheel/src/run_generator.rs | 4 +- wheel/src/run_program.rs | 6 +- 8 files changed, 772 insertions(+), 13 deletions(-) create mode 100644 crates/chia-consensus/src/build_block_2026.rs diff --git a/crates/chia-consensus/src/additions_and_removals.rs b/crates/chia-consensus/src/additions_and_removals.rs index dea123f29..781667b4e 100644 --- a/crates/chia-consensus/src/additions_and_removals.rs +++ b/crates/chia-consensus/src/additions_and_removals.rs @@ -14,7 +14,7 @@ use clvmr::allocator::NodePtr; use clvmr::chia_dialect::ChiaDialect; use clvmr::reduction::Reduction; use clvmr::run_program::run_program; -use clvmr::serde::node_from_bytes_backrefs; +use clvmr::serde::node_from_bytes_auto; /// Run a *trusted* block generator and return its additions and removals. This /// function does not validate the block, it is assumed to be valid. @@ -36,7 +36,7 @@ where let mut cost_left = constants.max_block_cost_clvm; - let program = node_from_bytes_backrefs(&mut a, program)?; + let program = node_from_bytes_auto(&mut a, program)?; let args = setup_generator_args(&mut a, block_refs, flags)?; let dialect = ChiaDialect::new(flags.to_clvm_flags()); diff --git a/crates/chia-consensus/src/build_block_2026.rs b/crates/chia-consensus/src/build_block_2026.rs new file mode 100644 index 000000000..f51c2d00f --- /dev/null +++ b/crates/chia-consensus/src/build_block_2026.rs @@ -0,0 +1,740 @@ +//! Anytime block builder for the INTERNED_GENERATOR (HF2) cost model. +//! +//! Accepts candidate spends in priority order, then optimizes the block in a +//! background thread. The caller retrieves the best block when ready. +//! +//! ```ignore +//! let mut builder = Block2026Builder::new(&constants); +//! for (bundle, exec_cost) in mempool_items_by_priority { +//! builder.add_candidate(bundle, exec_cost)?; +//! } +//! builder.start(); +//! +//! // ... sleep, wait for timelord, do other work ... +//! +//! let (gen, sig, cost) = builder.best(); // instant — grabs current best, signals stop +//! // broadcast the block +//! builder.close(); // join thread (after broadcast) +//! ``` +//! +//! Python callers can use the context-manager pattern: +//! +//! ```python +//! with Block2026Builder(constants) as builder: +//! for bundle, cost in mempool_items: +//! builder.add_candidate(bundle, cost) +//! builder.start() +//! # ... wait for timelord ... +//! gen, sig, cost = builder.best() +//! # __exit__ joins the thread +//! ``` +//! +//! # Cost model +//! +//! Generator cost = `interned_weight(tree) × COST_PER_BYTE`. This is a +//! property of the tree shape (after deduplication), not the serialized bytes. +//! Serialization with serde_2026 happens once per validation, not incrementally. +//! +//! # Two-pool cost estimation +//! +//! Each candidate's cost splits into: +//! - **Irreducible** — execution + conditions cost. Known exactly, fixed. +//! - **Compressible** — generator weight × `COST_PER_BYTE`. Upper-bounded by +//! the candidate's solo interned weight (no sharing assumed). Actual cost is +//! lower when spends share puzzle subtrees. + +use crate::consensus_constants::ConsensusConstants; +use crate::error::Result; +use crate::generator_cost::interned_weight; +use crate::solution_generator::build_generator; +use chia_bls::Signature; +use chia_protocol::SpendBundle; +use clvmr::allocator::Allocator; +use clvmr::serde::{intern, node_to_bytes_serde_2026}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; + +#[cfg(feature = "py-bindings")] +use pyo3::prelude::*; + +const QUOTE_COST: u64 = 20; +const MAX_SKIPPED_ITEMS: u32 = 6; + +/// Weight of the generator wrapper `(q . ((... . nil)))`: +/// q atom (1 byte): 3, outer pair: 3, inner pair: 3, nil: 2 = 11 +const WRAPPER_WEIGHT: u64 = 11; + +/// Weight added per spend for the list cons cell. +const LIST_CONS_WEIGHT: u64 = 3; + +// ── Internal types ───────────────────────────────────────────────────── + +struct Candidate { + bundle: SpendBundle, + irreducible_cost: u64, + spend_weight: u64, + original_index: usize, +} + +#[derive(Clone)] +struct BestBlock { + generator: Vec, + signature: Signature, + total_cost: u64, + included_indices: Vec, +} + +impl BestBlock { + fn empty() -> Self { + Self { + generator: Vec::new(), + signature: Signature::default(), + total_cost: 0, + included_indices: Vec::new(), + } + } + +} + +struct SharedState { + best: Mutex, + stop: AtomicBool, +} + +// ── BuilderInner (owns all optimization state, lives on the bg thread) ─ + +struct BuilderInner { + candidates: Vec, + best: BestBlock, + included_count: usize, + included_irreducible: u64, + included_weight_sum: u64, + headroom_cursor: usize, + phase: Phase, + max_cost: u64, + cost_per_byte: u64, +} + +#[derive(Clone, Copy, PartialEq)] +enum Phase { + NeedsPack, + NeedsValidation, + Validated, + Done, +} + +fn compute_spend_weight(bundle: &SpendBundle) -> Result { + let mut a = Allocator::new(); + let spends: Vec<(_, &[u8], &[u8])> = bundle + .coin_spends + .iter() + .map(|cs| (cs.coin, cs.puzzle_reveal.as_ref(), cs.solution.as_ref())) + .collect(); + let generator = build_generator(&mut a, spends)?; + let interned = intern(&a, generator)?; + let total = interned_weight(&interned); + Ok(total.saturating_sub(WRAPPER_WEIGHT)) +} + +#[inline] +fn upper_bound_cost(irreducible: u64, weight_sum: u64, cost_per_byte: u64) -> u64 { + irreducible + (WRAPPER_WEIGHT + weight_sum) * cost_per_byte +} + +impl BuilderInner { + fn new(max_cost: u64, cost_per_byte: u64) -> Self { + Self { + candidates: Vec::new(), + best: BestBlock::empty(), + included_count: 0, + included_irreducible: QUOTE_COST, + included_weight_sum: 0, + headroom_cursor: 0, + phase: Phase::NeedsPack, + max_cost, + cost_per_byte, + } + } + + fn add_candidate(&mut self, bundle: SpendBundle, irreducible_cost: u64) -> Result<()> { + let original_index = self.candidates.len(); + let spend_weight = compute_spend_weight(&bundle)?; + self.candidates.push(Candidate { + bundle, + irreducible_cost, + spend_weight, + original_index, + }); + Ok(()) + } + + fn improve(&mut self) -> Result { + match self.phase { + Phase::NeedsPack => { + self.greedy_pack(); + self.phase = Phase::NeedsValidation; + Ok(true) + } + Phase::NeedsValidation => { + self.validate_working_set()?; + self.headroom_cursor = self.included_count; + self.phase = if self.headroom_cursor < self.candidates.len() { + Phase::Validated + } else { + Phase::Done + }; + Ok(self.phase != Phase::Done) + } + Phase::Validated => { + if self.try_fill_headroom()? { + Ok(true) + } else { + self.phase = Phase::Done; + Ok(false) + } + } + Phase::Done => Ok(false), + } + } + + fn greedy_pack(&mut self) { + let mut num_skipped: u32 = 0; + for idx in 0..self.candidates.len() { + if num_skipped > MAX_SKIPPED_ITEMS { + break; + } + let c = &self.candidates[idx]; + let marginal_weight = c.spend_weight + LIST_CONS_WEIGHT; + let new_irreducible = self.included_irreducible + c.irreducible_cost; + let new_weight_sum = self.included_weight_sum + marginal_weight; + let new_cost = upper_bound_cost(new_irreducible, new_weight_sum, self.cost_per_byte); + + if new_cost <= self.max_cost { + self.candidates.swap(idx, self.included_count); + self.included_count += 1; + self.included_irreducible = new_irreducible; + self.included_weight_sum = new_weight_sum; + num_skipped = 0; + } else { + num_skipped += 1; + } + } + } + + fn validate_working_set(&mut self) -> Result<()> { + if self.included_count == 0 { + self.best = BestBlock::empty(); + return Ok(()); + } + + let mut a = Allocator::new(); + let mut signature = Signature::default(); + let mut spend_tuples: Vec<(_, &[u8], &[u8])> = Vec::new(); + + for c in &self.candidates[..self.included_count] { + signature.aggregate(&c.bundle.aggregated_signature); + for cs in &c.bundle.coin_spends { + spend_tuples.push(( + cs.coin, + cs.puzzle_reveal.as_ref(), + cs.solution.as_ref(), + )); + } + } + + let generator = build_generator(&mut a, spend_tuples)?; + let interned = intern(&a, generator)?; + let exact_weight = interned_weight(&interned); + let generator_cost = exact_weight * self.cost_per_byte; + let total_cost = self.included_irreducible + generator_cost; + + if total_cost > self.max_cost { + return Err(crate::error::Error::Custom( + "block cost exceeds limit after exact costing".into(), + )); + } + + let serialized = node_to_bytes_serde_2026(&a, generator)?; + + let included_indices = self.candidates[..self.included_count] + .iter() + .map(|c| c.original_index) + .collect(); + + self.best = BestBlock { + generator: serialized, + signature, + total_cost, + included_indices, + }; + self.included_weight_sum = exact_weight.saturating_sub(WRAPPER_WEIGHT); + + Ok(()) + } + + fn try_fill_headroom(&mut self) -> Result { + let headroom = self.max_cost.saturating_sub(self.best.total_cost); + if headroom == 0 { + return Ok(false); + } + + let mut added_any = false; + while self.headroom_cursor < self.candidates.len() { + let idx = self.headroom_cursor; + self.headroom_cursor += 1; + + let c = &self.candidates[idx]; + let marginal_upper = + c.irreducible_cost + (c.spend_weight + LIST_CONS_WEIGHT) * self.cost_per_byte; + if marginal_upper > headroom { + continue; + } + + self.included_irreducible += c.irreducible_cost; + self.included_weight_sum += c.spend_weight + LIST_CONS_WEIGHT; + self.candidates.swap(idx, self.included_count); + self.included_count += 1; + added_any = true; + } + + if added_any { + self.validate_working_set()?; + self.headroom_cursor = self.included_count; + Ok(true) + } else { + Ok(false) + } + } + + /// Run the headroom-filling loop, publishing improvements to shared state. + /// Called on the background thread AFTER the initial pack+validate has + /// already been done synchronously in `start()`. + fn run_headroom(mut self, shared: &SharedState) { + let mut last_cost = self.best.total_cost; + while self.phase != Phase::Done { + if shared.stop.load(Ordering::Relaxed) { + return; + } + match self.improve() { + Ok(true) => { + if self.best.total_cost != last_cost { + *shared.best.lock().unwrap() = self.best.clone(); + last_cost = self.best.total_cost; + } + } + _ => return, + } + } + } +} + +// ── Public API ───────────────────────────────────────────────────────── + +/// Anytime block builder for post-HF2 blocks. +/// +/// Feed candidates, then start a background thread. Call [`best`](Self::best) +/// to instantly grab the best block found so far (and signal the thread to +/// stop). Call [`close`](Self::close) to join the thread — typically after +/// broadcasting. +/// +/// Also usable as a Python context manager (`with Block2026Builder(...) as b:`), +/// which calls `close()` on exit. +#[cfg_attr(feature = "py-bindings", pyclass)] +pub struct Block2026Builder { + inner: Option, + shared: Option>, + thread: Option>, +} + +impl Block2026Builder { + pub fn new(constants: &ConsensusConstants) -> Self { + Self { + inner: Some(BuilderInner::new( + constants.max_block_cost_clvm, + constants.cost_per_byte, + )), + shared: None, + thread: None, + } + } + + /// Add a candidate spend bundle. Must be called before [`start`]. + /// Candidates should be in priority order (highest fee-per-cost first). + pub fn add_candidate( + &mut self, + bundle: SpendBundle, + irreducible_cost: u64, + ) -> Result<()> { + self.inner + .as_mut() + .expect("add_candidate called after start()") + .add_candidate(bundle, irreducible_cost) + } + + /// Pack and validate the initial block synchronously, then spawn a + /// background thread for headroom refinement. + /// + /// After this returns, [`best`] is guaranteed to return a valid block + /// instantly. The background thread may further improve it by + /// discovering sharing headroom. + pub fn start(&mut self) { + let mut inner = self.inner.take().expect("already started or finished"); + + // Initial pack + validate runs synchronously so `best()` is + // immediately useful. + inner.greedy_pack(); + inner.phase = Phase::NeedsValidation; + let _ = inner.improve(); // NeedsValidation → Validated|Done + + let shared = Arc::new(SharedState { + best: Mutex::new(inner.best.clone()), + stop: AtomicBool::new(false), + }); + + if inner.phase != Phase::Done { + let shared_clone = shared.clone(); + let thread = std::thread::spawn(move || { + inner.run_headroom(&shared_clone); + }); + self.thread = Some(thread); + } + + self.shared = Some(shared); + } + + /// Instantly return the best block found so far and signal the thread + /// to stop. + /// + /// Returns `(generator_bytes, aggregate_signature, total_cost, included_indices)`. + /// Generator is empty with cost 0 if no candidates fit or none were added. + /// + /// Can be called multiple times (idempotent stop signal; returns same block + /// once the thread has stopped improving). + pub fn best(&self) -> (Vec, Signature, u64, Vec) { + let block = if let Some(ref shared) = self.shared { + shared.stop.store(true, Ordering::Relaxed); + shared.best.lock().unwrap().clone() + } else if let Some(ref inner) = self.inner { + inner.best.clone() + } else { + BestBlock::empty() + }; + ( + block.generator, + block.signature, + block.total_cost, + block.included_indices, + ) + } + + /// Join the background thread. Call this after broadcasting to ensure + /// clean shutdown. Safe to call multiple times or without `start`. + pub fn close(&mut self) { + if let Some(ref shared) = self.shared { + shared.stop.store(true, Ordering::Relaxed); + } + if let Some(thread) = self.thread.take() { + thread.join().expect("builder thread panicked"); + } + self.shared = None; + } + + // ── Manual (non-threaded) API ────────────────────────────────────── + + /// One step of improvement (non-threaded path). + pub fn improve(&mut self) -> Result { + self.inner + .as_mut() + .expect("improve() called after start()") + .improve() + } + + pub fn num_included(&self) -> usize { + self.inner.as_ref().map_or(0, |i| i.included_count) + } +} + +impl Drop for Block2026Builder { + fn drop(&mut self) { + self.close(); + } +} + +#[cfg(feature = "py-bindings")] +#[pymethods] +impl Block2026Builder { + #[new] + pub fn py_new(constants: &ConsensusConstants) -> Self { + Self::new(constants) + } + + #[pyo3(name = "add_candidate")] + pub fn py_add_candidate( + &mut self, + bundle: SpendBundle, + irreducible_cost: u64, + ) -> PyResult<()> { + Ok(self.add_candidate(bundle, irreducible_cost)?) + } + + #[pyo3(name = "start")] + pub fn py_start(&mut self) { + self.start(); + } + + /// Instantly return the best block found so far and signal the thread + /// to stop. + #[pyo3(name = "best")] + pub fn py_best(&self) -> (Vec, Signature, u64, Vec) { + self.best() + } + + /// Join the background thread. Releases the GIL while waiting. + #[pyo3(name = "close")] + pub fn py_close(&mut self, py: Python<'_>) { + py.detach(|| self.close()); + } + + fn __enter__(slf: Py) -> Py { + slf + } + + #[pyo3(signature = (_exc_type=None, _exc_val=None, _exc_tb=None))] + fn __exit__( + &mut self, + py: Python<'_>, + _exc_type: Option<&Bound<'_, PyAny>>, + _exc_val: Option<&Bound<'_, PyAny>>, + _exc_tb: Option<&Bound<'_, PyAny>>, + ) -> bool { + py.detach(|| self.close()); + false + } + + #[pyo3(name = "improve")] + pub fn py_improve(&mut self) -> PyResult { + Ok(self.improve()?) + } + + #[pyo3(name = "num_included")] + pub fn py_num_included(&self) -> usize { + self.num_included() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::consensus_constants::TEST_CONSTANTS; + use chia_protocol::{Coin, CoinSpend, Program}; + use hex_literal::hex; + + fn make_bundle(puzzle: &[u8], solution: &[u8], parent: [u8; 32], amount: u64) -> SpendBundle { + let coin = Coin::new( + parent.into(), + hex!("fcc78a9e396df6ceebc217d2446bc016e0b3d5922fb32e5783ec5a85d490cfb6").into(), + amount, + ); + SpendBundle::new( + vec![CoinSpend::new( + coin, + Program::from(puzzle), + Program::from(solution), + )], + Signature::default(), + ) + } + + const STANDARD_PUZZLE: [u8; 291] = hex!( + "ff02ffff01ff02ffff01ff02ffff03ff0bffff01ff02ffff03ffff09ff05ffff" + "1dff0bffff1effff0bff0bffff02ff06ffff04ff02ffff04ff17ff8080808080" + "808080ffff01ff02ff17ff2f80ffff01ff088080ff0180ffff01ff04ffff04ff" + "04ffff04ff05ffff04ffff02ff06ffff04ff02ffff04ff17ff80808080ff8080" + "8080ffff02ff17ff2f808080ff0180ffff04ffff01ff32ff02ffff03ffff07ff" + "0580ffff01ff0bffff0102ffff02ff06ffff04ff02ffff04ff09ff80808080ff" + "ff02ff06ffff04ff02ffff04ff0dff8080808080ffff01ff0bffff0101ff0580" + "80ff0180ff018080ffff04ffff01b08cf5533a94afae0f4613d3ea565e47abc5" + "373415967ef5824fd009c602cb629e259908ce533c21de7fd7a68eb96c52d0ff" + "018080" + ); + + #[test] + fn test_empty_block() { + let builder = Block2026Builder::new(&TEST_CONSTANTS); + let (generator, _sig, cost, indices) = builder.best(); + assert!(generator.is_empty()); + assert_eq!(cost, 0); + assert!(indices.is_empty()); + } + + #[test] + fn test_single_spend_manual() { + let puzzle = hex!("ff01ff8080"); + let solution = hex!("80"); + let bundle = make_bundle(&puzzle, &solution, [1u8; 32], 100); + + let mut builder = Block2026Builder::new(&TEST_CONSTANTS); + builder.add_candidate(bundle, 1_000_000).unwrap(); + while builder.improve().unwrap() {} + + let (generator, _sig, cost, indices) = builder.best(); + assert!(!generator.is_empty()); + assert!(cost > 0); + assert!(cost <= TEST_CONSTANTS.max_block_cost_clvm); + assert_eq!(indices, vec![0]); + } + + #[test] + fn test_single_spend_threaded() { + let puzzle = hex!("ff01ff8080"); + let solution = hex!("80"); + let bundle = make_bundle(&puzzle, &solution, [1u8; 32], 100); + + let mut builder = Block2026Builder::new(&TEST_CONSTANTS); + builder.add_candidate(bundle, 1_000_000).unwrap(); + builder.start(); + + let (generator, _sig, cost, indices) = builder.best(); + builder.close(); + + assert!(!generator.is_empty()); + assert!(cost > 0); + assert!(cost <= TEST_CONSTANTS.max_block_cost_clvm); + assert_eq!(indices, vec![0]); + } + + #[test] + fn test_shared_puzzles_threaded() { + let solution = hex!("80"); + let mut builder = Block2026Builder::new(&TEST_CONSTANTS); + + for i in 0..5u64 { + let bundle = make_bundle( + STANDARD_PUZZLE.as_ref(), + solution.as_ref(), + [i as u8; 32], + 100 + i, + ); + builder.add_candidate(bundle, 1_000_000).unwrap(); + } + + builder.start(); + let (_gen, _sig, cost, indices) = builder.best(); + builder.close(); + assert!(cost > 0); + assert!(!indices.is_empty()); + } + + #[test] + fn test_cost_limit_respected() { + let puzzle = hex!("ff01ff8080"); + let solution = hex!("80"); + let mut builder = Block2026Builder::new(&TEST_CONSTANTS); + + let cost_per_spend = TEST_CONSTANTS.max_block_cost_clvm / 3; + for i in 0..10u64 { + let bundle = make_bundle(puzzle.as_ref(), solution.as_ref(), [i as u8; 32], 100); + builder.add_candidate(bundle, cost_per_spend).unwrap(); + } + + while builder.improve().unwrap() {} + + let (_gen, _sig, cost, _indices) = builder.best(); + assert!(cost <= TEST_CONSTANTS.max_block_cost_clvm); + assert!(builder.num_included() >= 2); + assert!(builder.num_included() <= 3); + } + + #[test] + fn test_headroom_fills_extra_spends() { + let mut constants = TEST_CONSTANTS.clone(); + let solution = hex!("80"); + + let sample = make_bundle(STANDARD_PUZZLE.as_ref(), solution.as_ref(), [0u8; 32], 100); + let solo_w = compute_spend_weight(&sample).unwrap(); + let exec_cost = 1_000_000u64; + let cpb = constants.cost_per_byte; + let ub3 = + QUOTE_COST + 3 * exec_cost + (WRAPPER_WEIGHT + 3 * (solo_w + LIST_CONS_WEIGHT)) * cpb; + let ub4 = + QUOTE_COST + 4 * exec_cost + (WRAPPER_WEIGHT + 4 * (solo_w + LIST_CONS_WEIGHT)) * cpb; + constants.max_block_cost_clvm = (ub3 + ub4) / 2; + + let mut builder = Block2026Builder::new(&constants); + for i in 0..6u64 { + let bundle = make_bundle( + STANDARD_PUZZLE.as_ref(), + solution.as_ref(), + [i as u8; 32], + 100 + i, + ); + builder.add_candidate(bundle, exec_cost).unwrap(); + } + + while builder.improve().unwrap() {} + + assert!( + builder.num_included() > 3, + "expected headroom to allow >3 spends, got {}", + builder.num_included() + ); + let (_gen, _sig, cost, _indices) = builder.best(); + assert!(cost <= constants.max_block_cost_clvm); + } + + #[test] + fn test_threaded_headroom() { + let mut constants = TEST_CONSTANTS.clone(); + let solution = hex!("80"); + + let sample = make_bundle(STANDARD_PUZZLE.as_ref(), solution.as_ref(), [0u8; 32], 100); + let solo_w = compute_spend_weight(&sample).unwrap(); + let exec_cost = 1_000_000u64; + let cpb = constants.cost_per_byte; + let ub3 = + QUOTE_COST + 3 * exec_cost + (WRAPPER_WEIGHT + 3 * (solo_w + LIST_CONS_WEIGHT)) * cpb; + let ub4 = + QUOTE_COST + 4 * exec_cost + (WRAPPER_WEIGHT + 4 * (solo_w + LIST_CONS_WEIGHT)) * cpb; + constants.max_block_cost_clvm = (ub3 + ub4) / 2; + + let mut builder = Block2026Builder::new(&constants); + for i in 0..6u64 { + let bundle = make_bundle( + STANDARD_PUZZLE.as_ref(), + solution.as_ref(), + [i as u8; 32], + 100 + i, + ); + builder.add_candidate(bundle, exec_cost).unwrap(); + } + + builder.start(); + let (_gen, _sig, cost, _indices) = builder.best(); + builder.close(); + + assert!(cost > 0); + assert!(cost <= constants.max_block_cost_clvm); + } + + #[test] + fn test_close_idempotent() { + let mut builder = Block2026Builder::new(&TEST_CONSTANTS); + builder.close(); + builder.close(); + + let mut builder = Block2026Builder::new(&TEST_CONSTANTS); + builder.start(); + builder.close(); + builder.close(); + } + + #[test] + fn test_drop_joins_thread() { + let mut builder = Block2026Builder::new(&TEST_CONSTANTS); + let bundle = make_bundle(&hex!("ff01ff8080"), &hex!("80"), [1u8; 32], 100); + builder.add_candidate(bundle, 1_000_000).unwrap(); + builder.start(); + drop(builder); + } +} diff --git a/crates/chia-consensus/src/lib.rs b/crates/chia-consensus/src/lib.rs index 044234211..fafb484d6 100644 --- a/crates/chia-consensus/src/lib.rs +++ b/crates/chia-consensus/src/lib.rs @@ -3,6 +3,7 @@ pub mod additions_and_removals; pub mod allocator; +pub mod build_block_2026; pub mod build_compressed_block; pub mod build_interned_block; pub mod check_time_locks; diff --git a/crates/chia-consensus/src/run_block_generator.rs b/crates/chia-consensus/src/run_block_generator.rs index dde47cfbb..9958078a3 100644 --- a/crates/chia-consensus/src/run_block_generator.rs +++ b/crates/chia-consensus/src/run_block_generator.rs @@ -353,7 +353,7 @@ where check_generator_quote(generator.as_ref(), flags)?; let mut output = Vec::::new(); - let program = node_from_bytes_backrefs(&mut a, generator)?; + let program = node_from_bytes_auto(&mut a, generator)?; check_generator_node(&a, program, flags)?; let args = setup_generator_args(&mut a, refs, flags)?; let dialect = ChiaDialect::new(flags.to_clvm_flags()); @@ -452,7 +452,7 @@ where check_generator_quote(generator.as_ref(), flags)?; let mut output = Vec::<(CoinSpend, Vec<(u32, Vec>)>)>::new(); - let program = node_from_bytes_backrefs(&mut a, generator)?; + let program = node_from_bytes_auto(&mut a, generator)?; check_generator_node(&a, program, flags)?; let args = setup_generator_args(&mut a, refs, flags)?; let dialect = ChiaDialect::new(flags.to_clvm_flags()); diff --git a/wheel/generate_type_stubs.py b/wheel/generate_type_stubs.py index 675c4dec8..2a69965a0 100644 --- a/wheel/generate_type_stubs.py +++ b/wheel/generate_type_stubs.py @@ -503,6 +503,22 @@ def add_spend_bundles(self, bundles: Sequence[SpendBundle], cost: uint64, consta def cost(self) -> uint64: ... def finalize(self, constants: ConsensusConstants) -> tuple[bytes, G2Element, uint64]: ... +@final +class Block2026Builder: + """Anytime block builder for post-HF2 (INTERNED_GENERATOR). + + Usable as a context manager: ``__exit__`` calls ``close()``. + """ + def __new__(cls, constants: ConsensusConstants) -> Self: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type: object = None, exc_val: object = None, exc_tb: object = None) -> bool: ... + def add_candidate(self, bundle: SpendBundle, irreducible_cost: uint64) -> None: ... + def start(self) -> None: ... + def best(self) -> tuple[bytes, G2Element, uint64, list[int]]: ... + def close(self) -> None: ... + def improve(self) -> bool: ... + def num_included(self) -> int: ... + @final class MerkleSet: def get_root(self) -> bytes32: ... diff --git a/wheel/src/api.rs b/wheel/src/api.rs index 26a615dcc..ea52def20 100644 --- a/wheel/src/api.rs +++ b/wheel/src/api.rs @@ -4,6 +4,7 @@ use crate::run_generator::{ run_block_generator2, }; use chia_consensus::allocator::make_allocator; +use chia_consensus::build_block_2026::Block2026Builder; use chia_consensus::build_compressed_block::BlockBuilder; use chia_consensus::check_time_locks::py_check_time_locks; use chia_consensus::consensus_constants::ConsensusConstants; @@ -82,7 +83,7 @@ use clvmr::error::EvalErr; use clvmr::reduction::Reduction; use clvmr::run_program; use clvmr::serde::is_canonical_serialization; -use clvmr::serde::{node_from_bytes, node_from_bytes_backrefs, node_to_bytes}; +use clvmr::serde::{node_from_bytes, node_from_bytes_auto, node_to_bytes}; use chia_bls::{ BlsCache, DerivableKey, G1Element, GTElement, PublicKey, SecretKey, Signature, @@ -178,9 +179,9 @@ pub fn get_puzzle_and_solution_for_coin<'a>( let program = py_to_slice::<'a>(program); let args = py_to_slice::<'a>(args); - let program = node_from_bytes_backrefs(&mut allocator, program) + let program = node_from_bytes_auto(&mut allocator, program) .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; - let args = node_from_bytes_backrefs(&mut allocator, args) + let args = node_from_bytes_auto(&mut allocator, args) .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; let dialect = &ChiaDialect::new(flags.to_clvm_flags()); @@ -238,7 +239,7 @@ pub fn get_puzzle_and_solution_for_coin2<'a>( py_to_slice::<'a>(buf) }); - let generator = node_from_bytes_backrefs(&mut allocator, generator.as_ref()) + let generator = node_from_bytes_auto(&mut allocator, generator.as_ref()) .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; let args = setup_generator_args(&mut allocator, refs, flags)?; let dialect = &ChiaDialect::new(flags.to_clvm_flags()); @@ -782,6 +783,7 @@ pub fn chia_rs(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(fast_forward_singleton, m)?)?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add( "ELIGIBLE_FOR_DEDUP", chia_consensus::conditions::ELIGIBLE_FOR_DEDUP, diff --git a/wheel/src/run_generator.rs b/wheel/src/run_generator.rs index 8d076ade9..0458d9483 100644 --- a/wheel/src/run_generator.rs +++ b/wheel/src/run_generator.rs @@ -11,7 +11,7 @@ use chia_protocol::{Bytes, Bytes32, Coin}; use clvmr::allocator::Allocator; use clvmr::cost::Cost; -use clvmr::serde::{intern_tree_limited, node_from_bytes_backrefs}; +use clvmr::serde::{intern_tree_limited, node_from_bytes_auto}; use pyo3::PyResult; use pyo3::buffer::PyBuffer; @@ -151,7 +151,7 @@ pub fn additions_and_removals<'a>( pub fn generator_interned_weight(program: PyBuffer) -> PyResult { let program = py_to_slice(program); let mut a = Allocator::new(); - let node = node_from_bytes_backrefs(&mut a, program) + let node = node_from_bytes_auto(&mut a, program) .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("bad generator: {e}")))?; let tree = intern_tree_limited(&a, node, u32::MAX as usize) .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("intern failed: {e}")))?; diff --git a/wheel/src/run_program.rs b/wheel/src/run_program.rs index d79a8e648..57143b3b2 100644 --- a/wheel/src/run_program.rs +++ b/wheel/src/run_program.rs @@ -7,7 +7,7 @@ use clvmr::cost::Cost; use clvmr::reduction::Response; use clvmr::run_program::run_program; use clvmr::serde::{ - node_from_bytes_backrefs, serialized_length_from_bytes, serialized_length_from_bytes_trusted, + node_from_bytes_auto, serialized_length_from_bytes, serialized_length_from_bytes_trusted, }; use pyo3::buffer::PyBuffer; use pyo3::prelude::*; @@ -44,9 +44,9 @@ pub fn run_chia_program( let flags = flags.to_clvm_flags(); let reduction = (|| -> PyResult { - let program = node_from_bytes_backrefs(&mut allocator, program) + let program = node_from_bytes_auto(&mut allocator, program) .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; - let args = node_from_bytes_backrefs(&mut allocator, args) + let args = node_from_bytes_auto(&mut allocator, args) .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; let dialect = ChiaDialect::new(flags); From 60c412c7ff344565dc3cabbb7bf666515c0a71e0 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 23 Apr 2026 17:48:30 -0700 Subject: [PATCH 06/18] add Program.from_program_bytes() + Block2026Builder stubs Program.from_program_bytes(blob) wraps raw bytes as a Program without CLVM structure validation. Needed for serde_2026 format generators which are validated at execution time by run_block_generator / node_from_bytes_auto, not at construction time. Also adds Block2026Builder type stubs to the .pyi file. Made-with: Cursor --- crates/chia-protocol/src/program.rs | 8 ++++++++ wheel/generate_type_stubs.py | 1 + wheel/python/chia_rs/chia_rs.pyi | 13 +++++++++++++ 3 files changed, 22 insertions(+) diff --git a/crates/chia-protocol/src/program.rs b/crates/chia-protocol/src/program.rs index eebc41b94..0ba7cc44b 100644 --- a/crates/chia-protocol/src/program.rs +++ b/crates/chia-protocol/src/program.rs @@ -467,6 +467,14 @@ impl Program { "This class does not support from_parent().", )) } + + /// Wrap raw bytes as a Program without CLVM structure validation. + /// Use for non-standard serializations (e.g. serde_2026) that will be + /// validated at execution time by run_block_generator / node_from_bytes_auto. + #[staticmethod] + fn from_program_bytes(blob: &[u8]) -> Self { + Self(blob.into()) + } } #[cfg(feature = "py-bindings")] diff --git a/wheel/generate_type_stubs.py b/wheel/generate_type_stubs.py index 2a69965a0..3fb14c6e9 100644 --- a/wheel/generate_type_stubs.py +++ b/wheel/generate_type_stubs.py @@ -259,6 +259,7 @@ def parse_rust_source(filename: str, upper_case: bool) -> list[tuple[str, list[s "def get_tree_hash(self) -> bytes32: ...", "@staticmethod\n def default() -> Program: ...", "@staticmethod\n def fromhex(h: str) -> Program: ...", + "@staticmethod\n def from_program_bytes(blob: bytes) -> Program: ...", "@staticmethod\n def to(o: object) -> Program: ...", "def run_rust(self, max_cost: int, flags: int, args: object) -> tuple[int, LazyNode]: ...", "def uncurry_rust(self) -> tuple[LazyNode, LazyNode]: ...", diff --git a/wheel/python/chia_rs/chia_rs.pyi b/wheel/python/chia_rs/chia_rs.pyi index d0f3bcc46..0f063ff98 100644 --- a/wheel/python/chia_rs/chia_rs.pyi +++ b/wheel/python/chia_rs/chia_rs.pyi @@ -196,6 +196,17 @@ class BlockBuilder: def cost(self) -> uint64: ... def finalize(self, constants: ConsensusConstants) -> tuple[bytes, G2Element, uint64]: ... +class Block2026Builder: + def __new__(cls, constants: ConsensusConstants) -> Self: ... + def add_candidate(self, bundle: SpendBundle, irreducible_cost: int) -> None: ... + def start(self) -> None: ... + def best(self) -> tuple[bytes, G2Element, int, list[int]]: ... + def close(self) -> None: ... + def improve(self) -> bool: ... + def num_included(self) -> int: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type: Any = None, exc_val: Any = None, exc_tb: Any = None) -> bool: ... + @final class MerkleSet: def get_root(self) -> bytes32: ... @@ -2198,6 +2209,8 @@ class Program: @staticmethod def fromhex(h: str) -> Program: ... @staticmethod + def from_program_bytes(blob: bytes) -> Program: ... + @staticmethod def to(o: object) -> Program: ... def run_rust(self, max_cost: int, flags: int, args: object) -> tuple[int, LazyNode]: ... def uncurry_rust(self) -> tuple[LazyNode, LazyNode]: ... From d7831f576ee1d00c1c28360462f4a4d2e02c9a31 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 23 Apr 2026 18:00:08 -0700 Subject: [PATCH 07/18] add tree_hash_auto() for serde_2026 / backrefs programs tree_hash_auto() deserializes using node_from_bytes_auto (handles standard CLVM, backrefs, and serde_2026) then computes the tree hash. Needed for generator_root computation on serde_2026 blocks. Made-with: Cursor --- wheel/python/chia_rs/chia_rs.pyi | 1 + wheel/src/api.rs | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/wheel/python/chia_rs/chia_rs.pyi b/wheel/python/chia_rs/chia_rs.pyi index 0f063ff98..ad2222c78 100644 --- a/wheel/python/chia_rs/chia_rs.pyi +++ b/wheel/python/chia_rs/chia_rs.pyi @@ -157,6 +157,7 @@ class LazyNode: def serialized_length(program: ReadableBuffer) -> int: ... def serialized_length_trusted(program: ReadableBuffer) -> int: ... def tree_hash(blob: ReadableBuffer) -> bytes32: ... +def tree_hash_auto(blob: ReadableBuffer) -> bytes32: ... def get_puzzle_and_solution_for_coin(program: ReadableBuffer, args: ReadableBuffer, max_cost: int, find_parent: bytes32, find_amount: int, find_ph: bytes32, flags: int) -> tuple[bytes, bytes]: ... def get_puzzle_and_solution_for_coin2(generator: Program, block_refs: list[ReadableBuffer], max_cost: int, find_coin: Coin, flags: int) -> tuple[Program, Program]: ... diff --git a/wheel/src/api.rs b/wheel/src/api.rs index ea52def20..b030f00b1 100644 --- a/wheel/src/api.rs +++ b/wheel/src/api.rs @@ -132,6 +132,16 @@ pub fn tree_hash<'a>(py: Python<'a>, blob: PyBuffer) -> PyResult(py: Python<'a>, blob: PyBuffer) -> PyResult> { + use clvmr::serde::node_from_bytes_auto; + let slice = py_to_slice::<'a>(blob); + let mut a = clvmr::Allocator::new(); + let node = node_from_bytes_auto(&mut a, slice).map_err(map_pyerr)?; + let hash = clvm_utils::tree_hash(&a, node); + ChiaToPython::to_python(&Bytes32::from(&hash.into()), py) +} + #[pyfunction] fn compute_plot_id_v1( plot_pk: G1Element, @@ -1007,6 +1017,7 @@ pub fn chia_rs(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(serialized_length_trusted, m)?)?; m.add_function(wrap_pyfunction!(compute_merkle_set_root, m)?)?; m.add_function(wrap_pyfunction!(tree_hash, m)?)?; + m.add_function(wrap_pyfunction!(tree_hash_auto, m)?)?; m.add_function(wrap_pyfunction!(get_puzzle_and_solution_for_coin, m)?)?; m.add_function(wrap_pyfunction!(get_puzzle_and_solution_for_coin2, m)?)?; From 59b695949c9d4dc09e8dc3bbf8af9eb7d278a86c Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 23 Apr 2026 18:25:31 -0700 Subject: [PATCH 08/18] skip SIMPLE_GENERATOR quote check for INTERNED_GENERATOR blocks serde_2026 blocks don't start with [0xff, 0x01] (standard CLVM quote prefix) since the serialization format is different. When INTERNED_GENERATOR is active, bypass the SIMPLE_GENERATOR quote check in both the raw bytes and deserialized node validators. Made-with: Cursor --- crates/chia-consensus/src/run_block_generator.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/chia-consensus/src/run_block_generator.rs b/crates/chia-consensus/src/run_block_generator.rs index 9958078a3..c8833beca 100644 --- a/crates/chia-consensus/src/run_block_generator.rs +++ b/crates/chia-consensus/src/run_block_generator.rs @@ -177,6 +177,10 @@ fn extract_n( // this is required after the SIMPLE_GENERATOR fork is active #[inline] pub fn check_generator_quote(program: &[u8], flags: ConsensusFlags) -> Result<(), ValidationErr> { + if flags.contains(ConsensusFlags::INTERNED_GENERATOR) { + // serde_2026 blocks have a different serialization header + return Ok(()); + } if !flags.contains(ConsensusFlags::SIMPLE_GENERATOR) || program.starts_with(&[0xff, 0x01]) { Ok(()) } else { @@ -195,7 +199,9 @@ pub fn check_generator_node( program: NodePtr, flags: ConsensusFlags, ) -> Result<(), ValidationErr> { - if !flags.contains(ConsensusFlags::SIMPLE_GENERATOR) { + if !flags.contains(ConsensusFlags::SIMPLE_GENERATOR) + || flags.contains(ConsensusFlags::INTERNED_GENERATOR) + { return Ok(()); } // this expects an atom with a single byte value of 1 as the first value in the list From d32b8636c2709042195e633040268b450ea20acb Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 23 Apr 2026 18:36:44 -0700 Subject: [PATCH 09/18] accept serde_2026 format in Program.from_json_dict When deserializing a Program from JSON, try standard CLVM serialized_length first. If that fails, fall back to node_from_bytes_auto which handles backrefs and serde_2026. This allows FullBlock JSON round-trips for blocks with serde_2026 generators (e.g. via RPC). Made-with: Cursor --- crates/chia-protocol/src/program.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/crates/chia-protocol/src/program.rs b/crates/chia-protocol/src/program.rs index 0ba7cc44b..ed74c783e 100644 --- a/crates/chia-protocol/src/program.rs +++ b/crates/chia-protocol/src/program.rs @@ -480,16 +480,18 @@ impl Program { #[cfg(feature = "py-bindings")] impl FromJsonDict for Program { fn from_json_dict(o: &Bound<'_, PyAny>) -> PyResult { + use clvmr::serde::node_from_bytes_auto; let bytes = Bytes::from_json_dict(o)?; - let len = - serialized_length_from_bytes(bytes.as_slice()).map_err(|_e| Error::EndOfBuffer)?; - if len as usize != bytes.len() { - // If the bytes in the JSON string is not a valid CLVM - // serialization, or if it has garbage at the end of the string, - // reject it - return Err(Error::InvalidClvm)?; + match serialized_length_from_bytes(bytes.as_slice()) { + Ok(len) if len as usize == bytes.len() => Ok(Self(bytes)), + _ => { + // Fall back to auto-detection for backrefs / serde_2026 + let mut a = Allocator::new(); + node_from_bytes_auto(&mut a, bytes.as_slice()) + .map_err(|_e| >::into(Error::EndOfBuffer))?; + Ok(Self(bytes)) + } } - Ok(Self(bytes)) } } From 205344829628ad48fe1aeb7365c994035561d4c7 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 23 Apr 2026 18:55:54 -0700 Subject: [PATCH 10/18] add tests: SIMPLE_GENERATOR guard, tree_hash_auto equivalence - 5 tests for check_generator_quote/check_generator_node: verify SIMPLE_GENERATOR still rejects non-quote generators when INTERNED_GENERATOR is NOT set (pre-HF2 safety) - 1 test for tree_hash_auto: verify it produces the same hash as tree_hash_from_bytes for standard CLVM and backrefs, and also works for serde_2026 (which tree_hash_from_bytes rejects) Made-with: Cursor --- .../chia-consensus/src/run_block_generator.rs | 54 +++++++++++++++++++ crates/clvm-utils/src/tree_hash.rs | 38 +++++++++++++ 2 files changed, 92 insertions(+) diff --git a/crates/chia-consensus/src/run_block_generator.rs b/crates/chia-consensus/src/run_block_generator.rs index c8833beca..69cb4b5dd 100644 --- a/crates/chia-consensus/src/run_block_generator.rs +++ b/crates/chia-consensus/src/run_block_generator.rs @@ -699,4 +699,58 @@ mod tests { assert_eq!(without.execution_cost, with.execution_cost); } + + #[test] + fn test_check_generator_quote_simple_only_rejects_non_quote() { + let flags = ConsensusFlags::SIMPLE_GENERATOR; + // Non-quote generator: starts with 0x80 (nil atom), not [0xff, 0x01] + assert_eq!( + check_generator_quote(&[0x80], flags).unwrap_err().1, + ErrorCode::ComplexGeneratorReceived, + ); + // Valid quote generator: starts with [0xff, 0x01] + assert!(check_generator_quote(&[0xff, 0x01, 0x80], flags).is_ok()); + } + + #[test] + fn test_check_generator_quote_interned_bypasses_check() { + let flags = ConsensusFlags::SIMPLE_GENERATOR | ConsensusFlags::INTERNED_GENERATOR; + // With INTERNED_GENERATOR, even non-quote bytes are accepted + assert!(check_generator_quote(&[0x80], flags).is_ok()); + assert!(check_generator_quote(&[0x00, 0x42], flags).is_ok()); + } + + #[test] + fn test_check_generator_quote_pre_simple_always_passes() { + let flags = ConsensusFlags::empty(); + // Before SIMPLE_GENERATOR, anything is accepted + assert!(check_generator_quote(&[0x80], flags).is_ok()); + assert!(check_generator_quote(&[0xff, 0x01, 0x80], flags).is_ok()); + } + + #[test] + fn test_check_generator_node_simple_only_rejects_non_quote() { + let flags = ConsensusFlags::SIMPLE_GENERATOR; + let mut a = Allocator::new(); + // Build a non-quote tree: just an atom (not (1 . rest)) + let atom = a.new_atom(&[42]).unwrap(); + assert_eq!( + check_generator_node(&a, atom, flags).unwrap_err().1, + ErrorCode::ComplexGeneratorReceived, + ); + // Build a valid (1 . nil) tree + let one = a.new_atom(&[1]).unwrap(); + let nil = a.nil(); + let pair = a.new_pair(one, nil).unwrap(); + assert!(check_generator_node(&a, pair, flags).is_ok()); + } + + #[test] + fn test_check_generator_node_interned_bypasses_check() { + let flags = ConsensusFlags::SIMPLE_GENERATOR | ConsensusFlags::INTERNED_GENERATOR; + let mut a = Allocator::new(); + let atom = a.new_atom(&[42]).unwrap(); + // INTERNED_GENERATOR bypasses the node check even for non-quote trees + assert!(check_generator_node(&a, atom, flags).is_ok()); + } } diff --git a/crates/clvm-utils/src/tree_hash.rs b/crates/clvm-utils/src/tree_hash.rs index f493535ac..5566080d9 100644 --- a/crates/clvm-utils/src/tree_hash.rs +++ b/crates/clvm-utils/src/tree_hash.rs @@ -404,6 +404,44 @@ fn test_tree_hash_from_bytes() { assert_eq!(hash1, hash3); } +#[test] +fn test_tree_hash_auto_matches_tree_hash_for_all_formats() { + use clvmr::serde::{ + node_from_bytes_auto, node_to_bytes, node_to_bytes_backrefs, node_to_bytes_serde_2026, + }; + + let mut a = Allocator::new(); + let atom1 = a.new_atom(&[1, 2, 3]).unwrap(); + let atom2 = a.new_atom(&[4, 5, 6]).unwrap(); + let node1 = a.new_pair(atom1, atom2).unwrap(); + let node2 = a.new_pair(atom2, atom1).unwrap(); + let node1 = a.new_pair(node1, node1).unwrap(); + let node2 = a.new_pair(node2, node2).unwrap(); + let root = a.new_pair(node1, node2).unwrap(); + + let canonical_hash = tree_hash(&a, root); + + let standard = node_to_bytes(&a, root).unwrap(); + let backrefs = node_to_bytes_backrefs(&a, root).unwrap(); + let serde_2026 = node_to_bytes_serde_2026(&a, root).unwrap(); + + // tree_hash_from_bytes only handles standard + backrefs + assert_eq!(tree_hash_from_bytes(&standard).unwrap(), canonical_hash); + assert_eq!(tree_hash_from_bytes(&backrefs).unwrap(), canonical_hash); + // tree_hash_from_bytes rejects serde_2026 + assert!(tree_hash_from_bytes(&serde_2026).is_err()); + + // node_from_bytes_auto + tree_hash works for ALL formats (the + // approach used by tree_hash_auto in the Python binding) + for (label, bytes) in [("standard", &standard), ("backrefs", &backrefs), ("serde_2026", &serde_2026)] { + let mut a2 = Allocator::new(); + let node = node_from_bytes_auto(&mut a2, bytes) + .unwrap_or_else(|e| panic!("{label}: node_from_bytes_auto failed: {e}")); + let hash = tree_hash(&a2, node); + assert_eq!(hash, canonical_hash, "{label}: tree_hash mismatch"); + } +} + #[cfg(test)] use rstest::rstest; From 2d99df9cd47b668e276f679c6841ce4dc1dc0be2 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 27 Apr 2026 12:44:08 -0700 Subject: [PATCH 11/18] fix Streamable round-trip for serde_2026 generators Program::parse() now auto-detects serde_2026 magic prefix and uses serialized_length_serde_2026() to delimit the blob. Without this, FullBlock serialization/deserialization fails for post-HF2 blocks that use serde_2026 generators. Also updates clvmr dep to pick up the new function, and adapts to API renames (intern -> intern_tree, node_from_bytes_auto gains DeserializeLimits parameter). Made-with: Cursor --- Cargo.lock | 3 +-- .../src/additions_and_removals.rs | 4 ++-- crates/chia-consensus/src/build_block_2026.rs | 8 ++++---- .../src/build_interned_block.rs | 4 ++-- crates/chia-consensus/src/generator_cost.rs | 10 +++++----- .../chia-consensus/src/run_block_generator.rs | 14 ++++++++------ .../chia-consensus/src/solution_generator.rs | 5 +++-- .../src/spendbundle_conditions.rs | 4 ++-- crates/chia-protocol/src/program.rs | 10 ++++++---- crates/clvm-utils/src/tree_hash.rs | 5 +++-- wheel/src/api.rs | 19 +++++++++++-------- wheel/src/run_generator.rs | 6 +++--- wheel/src/run_program.rs | 10 ++++++---- 13 files changed, 56 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4785155fa..b0ee14894 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -866,8 +866,7 @@ dependencies = [ [[package]] name = "clvmr" version = "0.17.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3060bcd64cb8cf2b32fe6ee3a82698835c03361c8e1da446d2e9d058fbfffd5f" +source = "git+https://github.com/Chia-Network/clvm_rs?rev=2104989d4a9fc6e21e1789c71bf8cb4599ab31cf#2104989d4a9fc6e21e1789c71bf8cb4599ab31cf" dependencies = [ "bitflags", "bitvec", diff --git a/crates/chia-consensus/src/additions_and_removals.rs b/crates/chia-consensus/src/additions_and_removals.rs index 781667b4e..43d941210 100644 --- a/crates/chia-consensus/src/additions_and_removals.rs +++ b/crates/chia-consensus/src/additions_and_removals.rs @@ -14,7 +14,7 @@ use clvmr::allocator::NodePtr; use clvmr::chia_dialect::ChiaDialect; use clvmr::reduction::Reduction; use clvmr::run_program::run_program; -use clvmr::serde::node_from_bytes_auto; +use clvmr::serde::{DeserializeOptions, node_from_bytes_auto}; /// Run a *trusted* block generator and return its additions and removals. This /// function does not validate the block, it is assumed to be valid. @@ -36,7 +36,7 @@ where let mut cost_left = constants.max_block_cost_clvm; - let program = node_from_bytes_auto(&mut a, program)?; + let program = node_from_bytes_auto(&mut a, program, DeserializeOptions::default())?; let args = setup_generator_args(&mut a, block_refs, flags)?; let dialect = ChiaDialect::new(flags.to_clvm_flags()); diff --git a/crates/chia-consensus/src/build_block_2026.rs b/crates/chia-consensus/src/build_block_2026.rs index f51c2d00f..d14a89549 100644 --- a/crates/chia-consensus/src/build_block_2026.rs +++ b/crates/chia-consensus/src/build_block_2026.rs @@ -50,7 +50,7 @@ use crate::solution_generator::build_generator; use chia_bls::Signature; use chia_protocol::SpendBundle; use clvmr::allocator::Allocator; -use clvmr::serde::{intern, node_to_bytes_serde_2026}; +use clvmr::serde::{Compression, intern_tree, node_to_bytes_serde_2026}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; @@ -132,7 +132,7 @@ fn compute_spend_weight(bundle: &SpendBundle) -> Result { .map(|cs| (cs.coin, cs.puzzle_reveal.as_ref(), cs.solution.as_ref())) .collect(); let generator = build_generator(&mut a, spends)?; - let interned = intern(&a, generator)?; + let interned = intern_tree(&a, generator)?; let total = interned_weight(&interned); Ok(total.saturating_sub(WRAPPER_WEIGHT)) } @@ -244,7 +244,7 @@ impl BuilderInner { } let generator = build_generator(&mut a, spend_tuples)?; - let interned = intern(&a, generator)?; + let interned = intern_tree(&a, generator)?; let exact_weight = interned_weight(&interned); let generator_cost = exact_weight * self.cost_per_byte; let total_cost = self.included_irreducible + generator_cost; @@ -255,7 +255,7 @@ impl BuilderInner { )); } - let serialized = node_to_bytes_serde_2026(&a, generator)?; + let serialized = node_to_bytes_serde_2026(&a, generator, Compression::default())?; let included_indices = self.candidates[..self.included_count] .iter() diff --git a/crates/chia-consensus/src/build_interned_block.rs b/crates/chia-consensus/src/build_interned_block.rs index 140a644c6..590be6e1e 100644 --- a/crates/chia-consensus/src/build_interned_block.rs +++ b/crates/chia-consensus/src/build_interned_block.rs @@ -4,7 +4,7 @@ use crate::generator_cost::total_cost_from_tree; use chia_bls::Signature; use chia_protocol::SpendBundle; use clvmr::allocator::{Allocator, NodePtr}; -use clvmr::serde::{intern_tree_limited, node_from_bytes_backrefs, node_to_bytes_backrefs}; +use clvmr::serde::{intern_tree, node_from_bytes_backrefs, node_to_bytes_backrefs}; use std::borrow::Borrow; #[cfg(feature = "py-bindings")] @@ -62,7 +62,7 @@ impl InternedBlockBuilder { // Build (q . ((spend_list))) let inner = allocator.new_pair(spend_list, allocator.nil())?; let outer = allocator.new_pair(allocator.one(), inner)?; - let interned = intern_tree_limited(allocator, outer, usize::MAX)?; + let interned = intern_tree(allocator, outer)?; Ok(total_cost_from_tree(&interned)) } diff --git a/crates/chia-consensus/src/generator_cost.rs b/crates/chia-consensus/src/generator_cost.rs index 90709de8c..0f11cbd2e 100644 --- a/crates/chia-consensus/src/generator_cost.rs +++ b/crates/chia-consensus/src/generator_cost.rs @@ -45,13 +45,13 @@ pub fn total_cost_from_tree(tree: &InternedTree) -> u64 { mod tests { use super::*; use clvmr::allocator::Allocator; - use clvmr::serde::intern; + use clvmr::serde::intern_tree; #[test] fn test_empty_atom() { let allocator = Allocator::new(); let node = allocator.nil(); - let tree = intern(&allocator, node).unwrap(); + let tree = intern_tree(&allocator, node).unwrap(); assert_eq!(total_cost_from_tree(&tree), 24_000); } @@ -61,7 +61,7 @@ mod tests { let left = allocator.new_atom(&[1, 2, 3]).unwrap(); let right = allocator.new_atom(&[4, 5, 6]).unwrap(); let node = allocator.new_pair(left, right).unwrap(); - let tree = intern(&allocator, node).unwrap(); + let tree = intern_tree(&allocator, node).unwrap(); assert_eq!(total_cost_from_tree(&tree), 156_000); } @@ -70,7 +70,7 @@ mod tests { let mut allocator = Allocator::new(); let atom = allocator.new_atom(&[42]).unwrap(); let node = allocator.new_pair(atom, atom).unwrap(); - let tree = intern(&allocator, node).unwrap(); + let tree = intern_tree(&allocator, node).unwrap(); assert_eq!(total_cost_from_tree(&tree), 72_000); } @@ -79,7 +79,7 @@ mod tests { let mut allocator = Allocator::new(); let atom = allocator.new_atom(&[1, 2, 3, 4, 5]).unwrap(); let node = allocator.new_pair(atom, allocator.nil()).unwrap(); - let tree = intern(&allocator, node).unwrap(); + let tree = intern_tree(&allocator, node).unwrap(); assert_eq!(total_cost_from_tree(&tree), 144_000); } } diff --git a/crates/chia-consensus/src/run_block_generator.rs b/crates/chia-consensus/src/run_block_generator.rs index 69cb4b5dd..655f921f7 100644 --- a/crates/chia-consensus/src/run_block_generator.rs +++ b/crates/chia-consensus/src/run_block_generator.rs @@ -25,7 +25,8 @@ use clvmr::cost::Cost; use clvmr::reduction::Reduction; use clvmr::run_program::run_program; use clvmr::serde::{ - InternedTree, intern, node_from_bytes, node_from_bytes_auto, node_from_bytes_backrefs, + DeserializeOptions, InternedTree, intern_tree, node_from_bytes, node_from_bytes_auto, + node_from_bytes_backrefs, }; pub fn subtract_cost( @@ -241,9 +242,10 @@ where let (mut a, base_cost, program) = if flags.contains(ConsensusFlags::INTERNED_GENERATOR) { let mut decode_allocator = Allocator::new(); - let program_node = node_from_bytes_auto(&mut decode_allocator, program) - .map_err(|_| ValidationErr(NodePtr::NIL, ErrorCode::GeneratorRuntimeError))?; - let interned = intern(&decode_allocator, program_node) + let program_node = + node_from_bytes_auto(&mut decode_allocator, program, DeserializeOptions::default()) + .map_err(|_| ValidationErr(NodePtr::NIL, ErrorCode::GeneratorRuntimeError))?; + let interned = intern_tree(&decode_allocator, program_node) .map_err(|_| ValidationErr(NodePtr::NIL, ErrorCode::GeneratorRuntimeError))?; let cost = total_cost_from_tree(&interned); let InternedTree { @@ -359,7 +361,7 @@ where check_generator_quote(generator.as_ref(), flags)?; let mut output = Vec::::new(); - let program = node_from_bytes_auto(&mut a, generator)?; + let program = node_from_bytes_auto(&mut a, generator, DeserializeOptions::default())?; check_generator_node(&a, program, flags)?; let args = setup_generator_args(&mut a, refs, flags)?; let dialect = ChiaDialect::new(flags.to_clvm_flags()); @@ -458,7 +460,7 @@ where check_generator_quote(generator.as_ref(), flags)?; let mut output = Vec::<(CoinSpend, Vec<(u32, Vec>)>)>::new(); - let program = node_from_bytes_auto(&mut a, generator)?; + let program = node_from_bytes_auto(&mut a, generator, DeserializeOptions::default())?; check_generator_node(&a, program, flags)?; let args = setup_generator_args(&mut a, refs, flags)?; let dialect = ChiaDialect::new(flags.to_clvm_flags()); diff --git a/crates/chia-consensus/src/solution_generator.rs b/crates/chia-consensus/src/solution_generator.rs index b360f74fb..7fc405d10 100644 --- a/crates/chia-consensus/src/solution_generator.rs +++ b/crates/chia-consensus/src/solution_generator.rs @@ -3,7 +3,8 @@ use chia_protocol::Coin; use chia_protocol::CoinSpend; use clvmr::allocator::{Allocator, NodePtr}; use clvmr::serde::{ - node_from_bytes_backrefs, node_to_bytes, node_to_bytes_backrefs, node_to_bytes_serde_2026, + Compression, node_from_bytes_backrefs, node_to_bytes, node_to_bytes_backrefs, + node_to_bytes_serde_2026, }; /// the tuple has the Coin, puzzle-reveal and solution @@ -115,7 +116,7 @@ where { let mut a = Allocator::new(); let generator = build_generator(&mut a, spends)?; - Ok(node_to_bytes_serde_2026(&a, generator)?) + Ok(node_to_bytes_serde_2026(&a, generator, Compression::default())?) } #[cfg(test)] diff --git a/crates/chia-consensus/src/spendbundle_conditions.rs b/crates/chia-consensus/src/spendbundle_conditions.rs index f4ea177ff..e6224999a 100644 --- a/crates/chia-consensus/src/spendbundle_conditions.rs +++ b/crates/chia-consensus/src/spendbundle_conditions.rs @@ -21,7 +21,7 @@ use clvmr::allocator::Allocator; use clvmr::chia_dialect::ChiaDialect; use clvmr::reduction::Reduction; use clvmr::run_program::run_program; -use clvmr::serde::intern; +use clvmr::serde::intern_tree; use clvmr::serde::node_from_bytes; const QUOTE_BYTES: usize = 2; @@ -64,7 +64,7 @@ fn calculate_base_cost( .map(|cs| (cs.coin, cs.puzzle_reveal.as_slice(), cs.solution.as_slice())), ) .map_err(|_| ValidationErr(a.nil(), ErrorCode::GeneratorRuntimeError))?; - let interned = intern(&gen_allocator, generator) + let interned = intern_tree(&gen_allocator, generator) .map_err(|_| ValidationErr(NodePtr::NIL, ErrorCode::GeneratorRuntimeError))?; Ok(total_cost_from_tree(&interned)) } else { diff --git a/crates/chia-protocol/src/program.rs b/crates/chia-protocol/src/program.rs index ed74c783e..ebad311d0 100644 --- a/crates/chia-protocol/src/program.rs +++ b/crates/chia-protocol/src/program.rs @@ -12,7 +12,7 @@ use clvmr::error::EvalErr; use clvmr::run_program; use clvmr::serde::{ node_from_bytes, node_from_bytes_backrefs, node_to_bytes, serialized_length_from_bytes, - serialized_length_from_bytes_trusted, + serialized_length_from_bytes_trusted, serialized_length_serde_2026, SERDE_2026_MAGIC_PREFIX, }; use clvmr::{Allocator, ChiaDialect, ClvmFlags, NodePtr}; #[cfg(feature = "py-bindings")] @@ -436,7 +436,9 @@ impl Streamable for Program { fn parse(input: &mut Cursor<&[u8]>) -> Result { let pos = input.position(); let buf: &[u8] = &input.get_ref()[pos as usize..]; - let len = if TRUSTED { + let len = if buf.starts_with(&SERDE_2026_MAGIC_PREFIX) { + serialized_length_serde_2026(buf).map_err(|_e| Error::EndOfBuffer)? + } else if TRUSTED { serialized_length_from_bytes_trusted(buf).map_err(|_e| Error::EndOfBuffer)? } else { serialized_length_from_bytes(buf).map_err(|_e| Error::EndOfBuffer)? @@ -480,14 +482,14 @@ impl Program { #[cfg(feature = "py-bindings")] impl FromJsonDict for Program { fn from_json_dict(o: &Bound<'_, PyAny>) -> PyResult { - use clvmr::serde::node_from_bytes_auto; + use clvmr::serde::{DeserializeOptions, node_from_bytes_auto}; let bytes = Bytes::from_json_dict(o)?; match serialized_length_from_bytes(bytes.as_slice()) { Ok(len) if len as usize == bytes.len() => Ok(Self(bytes)), _ => { // Fall back to auto-detection for backrefs / serde_2026 let mut a = Allocator::new(); - node_from_bytes_auto(&mut a, bytes.as_slice()) + node_from_bytes_auto(&mut a, bytes.as_slice(), DeserializeOptions::default()) .map_err(|_e| >::into(Error::EndOfBuffer))?; Ok(Self(bytes)) } diff --git a/crates/clvm-utils/src/tree_hash.rs b/crates/clvm-utils/src/tree_hash.rs index 5566080d9..cf8bb47db 100644 --- a/crates/clvm-utils/src/tree_hash.rs +++ b/crates/clvm-utils/src/tree_hash.rs @@ -407,7 +407,8 @@ fn test_tree_hash_from_bytes() { #[test] fn test_tree_hash_auto_matches_tree_hash_for_all_formats() { use clvmr::serde::{ - node_from_bytes_auto, node_to_bytes, node_to_bytes_backrefs, node_to_bytes_serde_2026, + DeserializeOptions, node_from_bytes_auto, node_to_bytes, node_to_bytes_backrefs, + node_to_bytes_serde_2026, }; let mut a = Allocator::new(); @@ -435,7 +436,7 @@ fn test_tree_hash_auto_matches_tree_hash_for_all_formats() { // approach used by tree_hash_auto in the Python binding) for (label, bytes) in [("standard", &standard), ("backrefs", &backrefs), ("serde_2026", &serde_2026)] { let mut a2 = Allocator::new(); - let node = node_from_bytes_auto(&mut a2, bytes) + let node = node_from_bytes_auto(&mut a2, bytes, DeserializeOptions::default()) .unwrap_or_else(|e| panic!("{label}: node_from_bytes_auto failed: {e}")); let hash = tree_hash(&a2, node); assert_eq!(hash, canonical_hash, "{label}: tree_hash mismatch"); diff --git a/wheel/src/api.rs b/wheel/src/api.rs index b030f00b1..b04139c06 100644 --- a/wheel/src/api.rs +++ b/wheel/src/api.rs @@ -83,7 +83,7 @@ use clvmr::error::EvalErr; use clvmr::reduction::Reduction; use clvmr::run_program; use clvmr::serde::is_canonical_serialization; -use clvmr::serde::{node_from_bytes, node_from_bytes_auto, node_to_bytes}; +use clvmr::serde::{DeserializeOptions, node_from_bytes, node_from_bytes_auto, node_to_bytes}; use chia_bls::{ BlsCache, DerivableKey, G1Element, GTElement, PublicKey, SecretKey, Signature, @@ -134,10 +134,11 @@ pub fn tree_hash<'a>(py: Python<'a>, blob: PyBuffer) -> PyResult(py: Python<'a>, blob: PyBuffer) -> PyResult> { - use clvmr::serde::node_from_bytes_auto; + use clvmr::serde::{DeserializeOptions, node_from_bytes_auto}; let slice = py_to_slice::<'a>(blob); let mut a = clvmr::Allocator::new(); - let node = node_from_bytes_auto(&mut a, slice).map_err(map_pyerr)?; + let node = + node_from_bytes_auto(&mut a, slice, DeserializeOptions::default()).map_err(map_pyerr)?; let hash = clvm_utils::tree_hash(&a, node); ChiaToPython::to_python(&Bytes32::from(&hash.into()), py) } @@ -189,9 +190,10 @@ pub fn get_puzzle_and_solution_for_coin<'a>( let program = py_to_slice::<'a>(program); let args = py_to_slice::<'a>(args); - let program = node_from_bytes_auto(&mut allocator, program) - .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; - let args = node_from_bytes_auto(&mut allocator, args) + let program = + node_from_bytes_auto(&mut allocator, program, DeserializeOptions::default()) + .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; + let args = node_from_bytes_auto(&mut allocator, args, DeserializeOptions::default()) .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; let dialect = &ChiaDialect::new(flags.to_clvm_flags()); @@ -249,8 +251,9 @@ pub fn get_puzzle_and_solution_for_coin2<'a>( py_to_slice::<'a>(buf) }); - let generator = node_from_bytes_auto(&mut allocator, generator.as_ref()) - .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; + let generator = + node_from_bytes_auto(&mut allocator, generator.as_ref(), DeserializeOptions::default()) + .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; let args = setup_generator_args(&mut allocator, refs, flags)?; let dialect = &ChiaDialect::new(flags.to_clvm_flags()); diff --git a/wheel/src/run_generator.rs b/wheel/src/run_generator.rs index 0458d9483..b8b01ae84 100644 --- a/wheel/src/run_generator.rs +++ b/wheel/src/run_generator.rs @@ -11,7 +11,7 @@ use chia_protocol::{Bytes, Bytes32, Coin}; use clvmr::allocator::Allocator; use clvmr::cost::Cost; -use clvmr::serde::{intern_tree_limited, node_from_bytes_auto}; +use clvmr::serde::{DeserializeOptions, intern_tree, node_from_bytes_auto}; use pyo3::PyResult; use pyo3::buffer::PyBuffer; @@ -151,9 +151,9 @@ pub fn additions_and_removals<'a>( pub fn generator_interned_weight(program: PyBuffer) -> PyResult { let program = py_to_slice(program); let mut a = Allocator::new(); - let node = node_from_bytes_auto(&mut a, program) + let node = node_from_bytes_auto(&mut a, program, DeserializeOptions::default()) .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("bad generator: {e}")))?; - let tree = intern_tree_limited(&a, node, u32::MAX as usize) + let tree = intern_tree(&a, node) .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("intern failed: {e}")))?; Ok(interned_weight(&tree)) } diff --git a/wheel/src/run_program.rs b/wheel/src/run_program.rs index 57143b3b2..806cb8de2 100644 --- a/wheel/src/run_program.rs +++ b/wheel/src/run_program.rs @@ -7,7 +7,8 @@ use clvmr::cost::Cost; use clvmr::reduction::Response; use clvmr::run_program::run_program; use clvmr::serde::{ - node_from_bytes_auto, serialized_length_from_bytes, serialized_length_from_bytes_trusted, + DeserializeOptions, node_from_bytes_auto, serialized_length_from_bytes, + serialized_length_from_bytes_trusted, }; use pyo3::buffer::PyBuffer; use pyo3::prelude::*; @@ -44,9 +45,10 @@ pub fn run_chia_program( let flags = flags.to_clvm_flags(); let reduction = (|| -> PyResult { - let program = node_from_bytes_auto(&mut allocator, program) - .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; - let args = node_from_bytes_auto(&mut allocator, args) + let program = + node_from_bytes_auto(&mut allocator, program, DeserializeOptions::default()) + .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; + let args = node_from_bytes_auto(&mut allocator, args, DeserializeOptions::default()) .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; let dialect = ChiaDialect::new(flags); From 7e3d5605e89b9015f5e832090ce3e7afefbc4740 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Sat, 2 May 2026 12:17:53 -0700 Subject: [PATCH 12/18] fix CI: fmt, clippy, stubtest, and trailing-bytes JSON parse - cargo fmt across files modified by serde_2026 wiring - clippy::manual_midpoint -> u64::midpoint in build_block_2026.rs - supply Compression::default() to node_to_bytes_serde_2026 in tree_hash.rs test - generate_type_stubs.py: replace embedded triple-quoted docstring with comments (was a Python SyntaxError when parsed by 3.14) - generate_type_stubs.py: add tree_hash_auto stub (matches public API) - Block2026Builder: add @disjoint_base + emit @final for stubtest - Program.from_json_dict: detect serde_2026 prefix and require all bytes consumed; raise InvalidClvm on trailing garbage (restoring chia/main behavior the test_streamable test relies on) - regenerate chia_rs.pyi --- crates/chia-consensus/src/build_block_2026.rs | 23 ++++-------------- .../chia-consensus/src/run_block_generator.rs | 9 ++++--- .../chia-consensus/src/solution_generator.rs | 6 ++++- crates/chia-protocol/src/program.rs | 24 +++++++++---------- crates/clvm-utils/src/tree_hash.rs | 12 ++++++---- wheel/generate_type_stubs.py | 8 +++---- wheel/python/chia_rs/chia_rs.pyi | 12 ++++++---- wheel/src/api.rs | 14 ++++++----- wheel/src/run_program.rs | 5 ++-- 9 files changed, 58 insertions(+), 55 deletions(-) diff --git a/crates/chia-consensus/src/build_block_2026.rs b/crates/chia-consensus/src/build_block_2026.rs index d14a89549..f12e8ecba 100644 --- a/crates/chia-consensus/src/build_block_2026.rs +++ b/crates/chia-consensus/src/build_block_2026.rs @@ -94,7 +94,6 @@ impl BestBlock { included_indices: Vec::new(), } } - } struct SharedState { @@ -235,11 +234,7 @@ impl BuilderInner { for c in &self.candidates[..self.included_count] { signature.aggregate(&c.bundle.aggregated_signature); for cs in &c.bundle.coin_spends { - spend_tuples.push(( - cs.coin, - cs.puzzle_reveal.as_ref(), - cs.solution.as_ref(), - )); + spend_tuples.push((cs.coin, cs.puzzle_reveal.as_ref(), cs.solution.as_ref())); } } @@ -361,11 +356,7 @@ impl Block2026Builder { /// Add a candidate spend bundle. Must be called before [`start`]. /// Candidates should be in priority order (highest fee-per-cost first). - pub fn add_candidate( - &mut self, - bundle: SpendBundle, - irreducible_cost: u64, - ) -> Result<()> { + pub fn add_candidate(&mut self, bundle: SpendBundle, irreducible_cost: u64) -> Result<()> { self.inner .as_mut() .expect("add_candidate called after start()") @@ -470,11 +461,7 @@ impl Block2026Builder { } #[pyo3(name = "add_candidate")] - pub fn py_add_candidate( - &mut self, - bundle: SpendBundle, - irreducible_cost: u64, - ) -> PyResult<()> { + pub fn py_add_candidate(&mut self, bundle: SpendBundle, irreducible_cost: u64) -> PyResult<()> { Ok(self.add_candidate(bundle, irreducible_cost)?) } @@ -659,7 +646,7 @@ mod tests { QUOTE_COST + 3 * exec_cost + (WRAPPER_WEIGHT + 3 * (solo_w + LIST_CONS_WEIGHT)) * cpb; let ub4 = QUOTE_COST + 4 * exec_cost + (WRAPPER_WEIGHT + 4 * (solo_w + LIST_CONS_WEIGHT)) * cpb; - constants.max_block_cost_clvm = (ub3 + ub4) / 2; + constants.max_block_cost_clvm = u64::midpoint(ub3, ub4); let mut builder = Block2026Builder::new(&constants); for i in 0..6u64 { @@ -696,7 +683,7 @@ mod tests { QUOTE_COST + 3 * exec_cost + (WRAPPER_WEIGHT + 3 * (solo_w + LIST_CONS_WEIGHT)) * cpb; let ub4 = QUOTE_COST + 4 * exec_cost + (WRAPPER_WEIGHT + 4 * (solo_w + LIST_CONS_WEIGHT)) * cpb; - constants.max_block_cost_clvm = (ub3 + ub4) / 2; + constants.max_block_cost_clvm = u64::midpoint(ub3, ub4); let mut builder = Block2026Builder::new(&constants); for i in 0..6u64 { diff --git a/crates/chia-consensus/src/run_block_generator.rs b/crates/chia-consensus/src/run_block_generator.rs index 655f921f7..2ebd01094 100644 --- a/crates/chia-consensus/src/run_block_generator.rs +++ b/crates/chia-consensus/src/run_block_generator.rs @@ -242,9 +242,12 @@ where let (mut a, base_cost, program) = if flags.contains(ConsensusFlags::INTERNED_GENERATOR) { let mut decode_allocator = Allocator::new(); - let program_node = - node_from_bytes_auto(&mut decode_allocator, program, DeserializeOptions::default()) - .map_err(|_| ValidationErr(NodePtr::NIL, ErrorCode::GeneratorRuntimeError))?; + let program_node = node_from_bytes_auto( + &mut decode_allocator, + program, + DeserializeOptions::default(), + ) + .map_err(|_| ValidationErr(NodePtr::NIL, ErrorCode::GeneratorRuntimeError))?; let interned = intern_tree(&decode_allocator, program_node) .map_err(|_| ValidationErr(NodePtr::NIL, ErrorCode::GeneratorRuntimeError))?; let cost = total_cost_from_tree(&interned); diff --git a/crates/chia-consensus/src/solution_generator.rs b/crates/chia-consensus/src/solution_generator.rs index 7fc405d10..136c8eca9 100644 --- a/crates/chia-consensus/src/solution_generator.rs +++ b/crates/chia-consensus/src/solution_generator.rs @@ -116,7 +116,11 @@ where { let mut a = Allocator::new(); let generator = build_generator(&mut a, spends)?; - Ok(node_to_bytes_serde_2026(&a, generator, Compression::default())?) + Ok(node_to_bytes_serde_2026( + &a, + generator, + Compression::default(), + )?) } #[cfg(test)] diff --git a/crates/chia-protocol/src/program.rs b/crates/chia-protocol/src/program.rs index ebad311d0..38cffb41b 100644 --- a/crates/chia-protocol/src/program.rs +++ b/crates/chia-protocol/src/program.rs @@ -11,8 +11,9 @@ use clvmr::cost::Cost; use clvmr::error::EvalErr; use clvmr::run_program; use clvmr::serde::{ - node_from_bytes, node_from_bytes_backrefs, node_to_bytes, serialized_length_from_bytes, - serialized_length_from_bytes_trusted, serialized_length_serde_2026, SERDE_2026_MAGIC_PREFIX, + SERDE_2026_MAGIC_PREFIX, node_from_bytes, node_from_bytes_backrefs, node_to_bytes, + serialized_length_from_bytes, serialized_length_from_bytes_trusted, + serialized_length_serde_2026, }; use clvmr::{Allocator, ChiaDialect, ClvmFlags, NodePtr}; #[cfg(feature = "py-bindings")] @@ -482,18 +483,17 @@ impl Program { #[cfg(feature = "py-bindings")] impl FromJsonDict for Program { fn from_json_dict(o: &Bound<'_, PyAny>) -> PyResult { - use clvmr::serde::{DeserializeOptions, node_from_bytes_auto}; let bytes = Bytes::from_json_dict(o)?; - match serialized_length_from_bytes(bytes.as_slice()) { - Ok(len) if len as usize == bytes.len() => Ok(Self(bytes)), - _ => { - // Fall back to auto-detection for backrefs / serde_2026 - let mut a = Allocator::new(); - node_from_bytes_auto(&mut a, bytes.as_slice(), DeserializeOptions::default()) - .map_err(|_e| >::into(Error::EndOfBuffer))?; - Ok(Self(bytes)) - } + let buf = bytes.as_slice(); + let len = if buf.starts_with(&SERDE_2026_MAGIC_PREFIX) { + serialized_length_serde_2026(buf).map_err(|_e| Error::EndOfBuffer)? + } else { + serialized_length_from_bytes(buf).map_err(|_e| Error::EndOfBuffer)? + }; + if len as usize != buf.len() { + return Err(Error::InvalidClvm)?; } + Ok(Self(bytes)) } } diff --git a/crates/clvm-utils/src/tree_hash.rs b/crates/clvm-utils/src/tree_hash.rs index cf8bb47db..eeafe74d0 100644 --- a/crates/clvm-utils/src/tree_hash.rs +++ b/crates/clvm-utils/src/tree_hash.rs @@ -407,8 +407,8 @@ fn test_tree_hash_from_bytes() { #[test] fn test_tree_hash_auto_matches_tree_hash_for_all_formats() { use clvmr::serde::{ - DeserializeOptions, node_from_bytes_auto, node_to_bytes, node_to_bytes_backrefs, - node_to_bytes_serde_2026, + Compression, DeserializeOptions, node_from_bytes_auto, node_to_bytes, + node_to_bytes_backrefs, node_to_bytes_serde_2026, }; let mut a = Allocator::new(); @@ -424,7 +424,7 @@ fn test_tree_hash_auto_matches_tree_hash_for_all_formats() { let standard = node_to_bytes(&a, root).unwrap(); let backrefs = node_to_bytes_backrefs(&a, root).unwrap(); - let serde_2026 = node_to_bytes_serde_2026(&a, root).unwrap(); + let serde_2026 = node_to_bytes_serde_2026(&a, root, Compression::default()).unwrap(); // tree_hash_from_bytes only handles standard + backrefs assert_eq!(tree_hash_from_bytes(&standard).unwrap(), canonical_hash); @@ -434,7 +434,11 @@ fn test_tree_hash_auto_matches_tree_hash_for_all_formats() { // node_from_bytes_auto + tree_hash works for ALL formats (the // approach used by tree_hash_auto in the Python binding) - for (label, bytes) in [("standard", &standard), ("backrefs", &backrefs), ("serde_2026", &serde_2026)] { + for (label, bytes) in [ + ("standard", &standard), + ("backrefs", &backrefs), + ("serde_2026", &serde_2026), + ] { let mut a2 = Allocator::new(); let node = node_from_bytes_auto(&mut a2, bytes, DeserializeOptions::default()) .unwrap_or_else(|e| panic!("{label}: node_from_bytes_auto failed: {e}")); diff --git a/wheel/generate_type_stubs.py b/wheel/generate_type_stubs.py index 3fb14c6e9..89d396b08 100644 --- a/wheel/generate_type_stubs.py +++ b/wheel/generate_type_stubs.py @@ -465,6 +465,7 @@ class LazyNode: def serialized_length(program: ReadableBuffer) -> int: ... def serialized_length_trusted(program: ReadableBuffer) -> int: ... def tree_hash(blob: ReadableBuffer) -> bytes32: ... +def tree_hash_auto(blob: ReadableBuffer) -> bytes32: ... def get_puzzle_and_solution_for_coin(program: ReadableBuffer, args: ReadableBuffer, max_cost: int, find_parent: bytes32, find_amount: int, find_ph: bytes32, flags: int) -> tuple[bytes, bytes]: ... def get_puzzle_and_solution_for_coin2(generator: Program, block_refs: list[ReadableBuffer], max_cost: int, find_coin: Coin, flags: int) -> tuple[Program, Program]: ... @@ -505,11 +506,10 @@ def cost(self) -> uint64: ... def finalize(self, constants: ConsensusConstants) -> tuple[bytes, G2Element, uint64]: ... @final +@disjoint_base class Block2026Builder: - """Anytime block builder for post-HF2 (INTERNED_GENERATOR). - - Usable as a context manager: ``__exit__`` calls ``close()``. - """ + # Anytime block builder for post-HF2 (INTERNED_GENERATOR). + # Usable as a context manager: ``__exit__`` calls ``close()``. def __new__(cls, constants: ConsensusConstants) -> Self: ... def __enter__(self) -> Self: ... def __exit__(self, exc_type: object = None, exc_val: object = None, exc_tb: object = None) -> bool: ... diff --git a/wheel/python/chia_rs/chia_rs.pyi b/wheel/python/chia_rs/chia_rs.pyi index ad2222c78..a4e2bc14a 100644 --- a/wheel/python/chia_rs/chia_rs.pyi +++ b/wheel/python/chia_rs/chia_rs.pyi @@ -197,16 +197,20 @@ class BlockBuilder: def cost(self) -> uint64: ... def finalize(self, constants: ConsensusConstants) -> tuple[bytes, G2Element, uint64]: ... +@final +@disjoint_base class Block2026Builder: + # Anytime block builder for post-HF2 (INTERNED_GENERATOR). + # Usable as a context manager: ``__exit__`` calls ``close()``. def __new__(cls, constants: ConsensusConstants) -> Self: ... - def add_candidate(self, bundle: SpendBundle, irreducible_cost: int) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type: object = None, exc_val: object = None, exc_tb: object = None) -> bool: ... + def add_candidate(self, bundle: SpendBundle, irreducible_cost: uint64) -> None: ... def start(self) -> None: ... - def best(self) -> tuple[bytes, G2Element, int, list[int]]: ... + def best(self) -> tuple[bytes, G2Element, uint64, list[int]]: ... def close(self) -> None: ... def improve(self) -> bool: ... def num_included(self) -> int: ... - def __enter__(self) -> Self: ... - def __exit__(self, exc_type: Any = None, exc_val: Any = None, exc_tb: Any = None) -> bool: ... @final class MerkleSet: diff --git a/wheel/src/api.rs b/wheel/src/api.rs index b04139c06..d80fe49e5 100644 --- a/wheel/src/api.rs +++ b/wheel/src/api.rs @@ -190,9 +190,8 @@ pub fn get_puzzle_and_solution_for_coin<'a>( let program = py_to_slice::<'a>(program); let args = py_to_slice::<'a>(args); - let program = - node_from_bytes_auto(&mut allocator, program, DeserializeOptions::default()) - .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; + let program = node_from_bytes_auto(&mut allocator, program, DeserializeOptions::default()) + .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; let args = node_from_bytes_auto(&mut allocator, args, DeserializeOptions::default()) .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; let dialect = &ChiaDialect::new(flags.to_clvm_flags()); @@ -251,9 +250,12 @@ pub fn get_puzzle_and_solution_for_coin2<'a>( py_to_slice::<'a>(buf) }); - let generator = - node_from_bytes_auto(&mut allocator, generator.as_ref(), DeserializeOptions::default()) - .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; + let generator = node_from_bytes_auto( + &mut allocator, + generator.as_ref(), + DeserializeOptions::default(), + ) + .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; let args = setup_generator_args(&mut allocator, refs, flags)?; let dialect = &ChiaDialect::new(flags.to_clvm_flags()); diff --git a/wheel/src/run_program.rs b/wheel/src/run_program.rs index 806cb8de2..d51ff9bbc 100644 --- a/wheel/src/run_program.rs +++ b/wheel/src/run_program.rs @@ -45,9 +45,8 @@ pub fn run_chia_program( let flags = flags.to_clvm_flags(); let reduction = (|| -> PyResult { - let program = - node_from_bytes_auto(&mut allocator, program, DeserializeOptions::default()) - .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; + let program = node_from_bytes_auto(&mut allocator, program, DeserializeOptions::default()) + .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; let args = node_from_bytes_auto(&mut allocator, args, DeserializeOptions::default()) .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; let dialect = ChiaDialect::new(flags); From 290035dd53c4b2c9f851a5be553d4b22b7eeb997 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Sat, 2 May 2026 12:28:10 -0700 Subject: [PATCH 13/18] fix CI: map MALACHITE flag and allowlist 3.9 stubtest conflict - ConsensusFlags::from_clvm_flags / to_clvm_flags: add MALACHITE branch so the round-trip test (and ClvmFlags::FLAGS coverage check) passes against clvmr 0.17.7 which now defines that flag. - Block2026Builder uses both @final + @disjoint_base, which 3.10+ accepts but 3.9 stubtest rejects ("remove @disjoint_base"). Add it to the disjoint-base allowlist (already wired to 3.9 only) to silence that. --- crates/chia-consensus/src/flags.rs | 6 ++++++ wheel/stubtest.allowlist.disjoint-base | 3 +++ 2 files changed, 9 insertions(+) diff --git a/crates/chia-consensus/src/flags.rs b/crates/chia-consensus/src/flags.rs index 56940a958..2c19a873b 100644 --- a/crates/chia-consensus/src/flags.rs +++ b/crates/chia-consensus/src/flags.rs @@ -95,6 +95,9 @@ impl ConsensusFlags { if clvm.contains(ClvmFlags::ENABLE_SECP_OPS) { out = out.union(ConsensusFlags::ENABLE_SECP_OPS); } + if clvm.contains(ClvmFlags::MALACHITE) { + out = out.union(ConsensusFlags::MALACHITE); + } out } @@ -133,6 +136,9 @@ impl ConsensusFlags { if self.contains(ConsensusFlags::ENABLE_SECP_OPS) { out.insert(ClvmFlags::ENABLE_SECP_OPS); } + if self.contains(ConsensusFlags::MALACHITE) { + out.insert(ClvmFlags::MALACHITE); + } out } } diff --git a/wheel/stubtest.allowlist.disjoint-base b/wheel/stubtest.allowlist.disjoint-base index af7a461a9..232e5c8ba 100644 --- a/wheel/stubtest.allowlist.disjoint-base +++ b/wheel/stubtest.allowlist.disjoint-base @@ -2,3 +2,6 @@ # or .__disjoint_base__ is not present in stub when adding the decorator chia_rs\.sized_byte_class\.SizedBytes chia_rs\.struct_stream\.StructStream + +# 3.9 stubtest rejects @final + @disjoint_base together; we keep both for 3.10+ +chia_rs\.Block2026Builder From 4658cbe0057028a62060ae629c5f72297cfa7f69 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Sat, 2 May 2026 12:36:13 -0700 Subject: [PATCH 14/18] fix CI: move Block2026Builder stubtest allowlist to 3-9 only The 3.10/3.11 disjoint-base allowlist would flag chia_rs.Block2026Builder as an unused entry, since those Python versions accept @final + @disjoint_base together. Only 3.9 needs to silence the "remove @disjoint_base" complaint. --- wheel/stubtest.allowlist.3-9 | 3 +++ wheel/stubtest.allowlist.disjoint-base | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/wheel/stubtest.allowlist.3-9 b/wheel/stubtest.allowlist.3-9 index 5aeab69f3..aaba942b3 100644 --- a/wheel/stubtest.allowlist.3-9 +++ b/wheel/stubtest.allowlist.3-9 @@ -23,3 +23,6 @@ chia_rs\.SpendBundle\.from_parent # disjoint base (3.9 only) chia_rs\.sized_bytes\.SizedBytes chia_rs\.sized_ints\.StructStream + +# 3.9 stubtest rejects @final + @disjoint_base together; we keep both for 3.10+ +chia_rs\.Block2026Builder diff --git a/wheel/stubtest.allowlist.disjoint-base b/wheel/stubtest.allowlist.disjoint-base index 232e5c8ba..af7a461a9 100644 --- a/wheel/stubtest.allowlist.disjoint-base +++ b/wheel/stubtest.allowlist.disjoint-base @@ -2,6 +2,3 @@ # or .__disjoint_base__ is not present in stub when adding the decorator chia_rs\.sized_byte_class\.SizedBytes chia_rs\.struct_stream\.StructStream - -# 3.9 stubtest rejects @final + @disjoint_base together; we keep both for 3.10+ -chia_rs\.Block2026Builder From 50b5665d2391a44a6ff56e72934bcbd054e2bec8 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 4 May 2026 13:33:48 -0700 Subject: [PATCH 15/18] serde_2026: track upstream level: u32 API, drop Compression imports clvm_rs PR #708 replaced the public `Compression` enum with a saturating `level: u32` API. Update the three call sites here and bump the patched clvmr pin to a67ee12a269000bb6e20266bfa40769bc50ab993: - crates/chia-consensus/src/build_block_2026.rs - crates/chia-consensus/src/solution_generator.rs - crates/clvm-utils/src/tree_hash.rs Each previously called the function as `..._serde_2026(a, n, Compression::default())`; now just `..._serde_2026(a, n)`. The default level (0) is unchanged. Made-with: Cursor --- Cargo.lock | 2 +- Cargo.toml | 2 +- crates/chia-consensus/src/build_block_2026.rs | 4 ++-- crates/chia-consensus/src/solution_generator.rs | 9 ++------- crates/clvm-utils/src/tree_hash.rs | 6 +++--- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0ee14894..3a3ae161c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -866,7 +866,7 @@ dependencies = [ [[package]] name = "clvmr" version = "0.17.7" -source = "git+https://github.com/Chia-Network/clvm_rs?rev=2104989d4a9fc6e21e1789c71bf8cb4599ab31cf#2104989d4a9fc6e21e1789c71bf8cb4599ab31cf" +source = "git+https://github.com/Chia-Network/clvm_rs?rev=a67ee12a269000bb6e20266bfa40769bc50ab993#a67ee12a269000bb6e20266bfa40769bc50ab993" dependencies = [ "bitflags", "bitvec", diff --git a/Cargo.toml b/Cargo.toml index 3dcbc6a8c..13a7cdec7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -113,7 +113,7 @@ openssl = ["chia-sha2/openssl", "clvmr/openssl"] [patch.crates-io] # Pin clvmr to clvm_rs PR #708 (serde_2026) until the format ships in a # crates.io release. Bump this rev in lockstep with that branch. -clvmr = { git = "https://github.com/Chia-Network/clvm_rs", rev = "2104989d4a9fc6e21e1789c71bf8cb4599ab31cf" } +clvmr = { git = "https://github.com/Chia-Network/clvm_rs", rev = "a67ee12a269000bb6e20266bfa40769bc50ab993" } [profile.release] lto = "thin" diff --git a/crates/chia-consensus/src/build_block_2026.rs b/crates/chia-consensus/src/build_block_2026.rs index f12e8ecba..eb73157bf 100644 --- a/crates/chia-consensus/src/build_block_2026.rs +++ b/crates/chia-consensus/src/build_block_2026.rs @@ -50,7 +50,7 @@ use crate::solution_generator::build_generator; use chia_bls::Signature; use chia_protocol::SpendBundle; use clvmr::allocator::Allocator; -use clvmr::serde::{Compression, intern_tree, node_to_bytes_serde_2026}; +use clvmr::serde::{intern_tree, node_to_bytes_serde_2026}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; @@ -250,7 +250,7 @@ impl BuilderInner { )); } - let serialized = node_to_bytes_serde_2026(&a, generator, Compression::default())?; + let serialized = node_to_bytes_serde_2026(&a, generator)?; let included_indices = self.candidates[..self.included_count] .iter() diff --git a/crates/chia-consensus/src/solution_generator.rs b/crates/chia-consensus/src/solution_generator.rs index 136c8eca9..b360f74fb 100644 --- a/crates/chia-consensus/src/solution_generator.rs +++ b/crates/chia-consensus/src/solution_generator.rs @@ -3,8 +3,7 @@ use chia_protocol::Coin; use chia_protocol::CoinSpend; use clvmr::allocator::{Allocator, NodePtr}; use clvmr::serde::{ - Compression, node_from_bytes_backrefs, node_to_bytes, node_to_bytes_backrefs, - node_to_bytes_serde_2026, + node_from_bytes_backrefs, node_to_bytes, node_to_bytes_backrefs, node_to_bytes_serde_2026, }; /// the tuple has the Coin, puzzle-reveal and solution @@ -116,11 +115,7 @@ where { let mut a = Allocator::new(); let generator = build_generator(&mut a, spends)?; - Ok(node_to_bytes_serde_2026( - &a, - generator, - Compression::default(), - )?) + Ok(node_to_bytes_serde_2026(&a, generator)?) } #[cfg(test)] diff --git a/crates/clvm-utils/src/tree_hash.rs b/crates/clvm-utils/src/tree_hash.rs index eeafe74d0..bd31e5a4c 100644 --- a/crates/clvm-utils/src/tree_hash.rs +++ b/crates/clvm-utils/src/tree_hash.rs @@ -407,8 +407,8 @@ fn test_tree_hash_from_bytes() { #[test] fn test_tree_hash_auto_matches_tree_hash_for_all_formats() { use clvmr::serde::{ - Compression, DeserializeOptions, node_from_bytes_auto, node_to_bytes, - node_to_bytes_backrefs, node_to_bytes_serde_2026, + DeserializeOptions, node_from_bytes_auto, node_to_bytes, node_to_bytes_backrefs, + node_to_bytes_serde_2026, }; let mut a = Allocator::new(); @@ -424,7 +424,7 @@ fn test_tree_hash_auto_matches_tree_hash_for_all_formats() { let standard = node_to_bytes(&a, root).unwrap(); let backrefs = node_to_bytes_backrefs(&a, root).unwrap(); - let serde_2026 = node_to_bytes_serde_2026(&a, root, Compression::default()).unwrap(); + let serde_2026 = node_to_bytes_serde_2026(&a, root).unwrap(); // tree_hash_from_bytes only handles standard + backrefs assert_eq!(tree_hash_from_bytes(&standard).unwrap(), canonical_hash); From 17af0417876a1793e3b1f5351192f73626814411 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 4 May 2026 15:02:12 -0700 Subject: [PATCH 16/18] serde_2026: track upstream API drop of DeserializeOptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit clvm_rs no longer ships DeserializeOptions / DEFAULT_MAX_ATOM_LEN / DEFAULT_MAX_INPUT_BYTES / node_from_bytes_auto — those defaults were consensus-flavored without being consensus, and Arvid pushed back on having them in clvm_rs. They live here now, where they belong. Adds chia_consensus::serde_2026 with: - CONSENSUS_MAX_ATOM_LEN (= 1 MiB, matches the old default exactly) - node_from_bytes_auto(a, &[u8]) — sniffs SERDE_2026_MAGIC_PREFIX and dispatches to deserialize_2026 (with consensus caps) or node_from_bytes_backrefs All 8 in-tree call sites switch to the new helper: - chia-consensus: additions_and_removals, run_block_generator (×3) - wheel: api (tree_hash_auto, get_puzzle_and_solution_*), run_generator, run_program clvm-utils stays decoupled: its tree_hash test inlines the same sniff-and-dispatch with its own 1 MiB constant. Bumps clvmr to 37e09884 (the matching clvm_rs PR head). Made-with: Cursor --- Cargo.lock | 2 +- Cargo.toml | 2 +- .../src/additions_and_removals.rs | 4 +-- crates/chia-consensus/src/lib.rs | 1 + .../chia-consensus/src/run_block_generator.rs | 19 +++++------- crates/chia-consensus/src/serde_2026.rs | 31 +++++++++++++++++++ crates/clvm-utils/src/tree_hash.rs | 22 +++++++++---- wheel/src/api.rs | 21 +++++-------- wheel/src/run_generator.rs | 5 +-- wheel/src/run_program.rs | 10 +++--- 10 files changed, 74 insertions(+), 43 deletions(-) create mode 100644 crates/chia-consensus/src/serde_2026.rs diff --git a/Cargo.lock b/Cargo.lock index 3a3ae161c..8b8bd9aaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -866,7 +866,7 @@ dependencies = [ [[package]] name = "clvmr" version = "0.17.7" -source = "git+https://github.com/Chia-Network/clvm_rs?rev=a67ee12a269000bb6e20266bfa40769bc50ab993#a67ee12a269000bb6e20266bfa40769bc50ab993" +source = "git+https://github.com/Chia-Network/clvm_rs?rev=37e098844512f7f8e976aa4326b766b21b911601#37e098844512f7f8e976aa4326b766b21b911601" dependencies = [ "bitflags", "bitvec", diff --git a/Cargo.toml b/Cargo.toml index 13a7cdec7..5db2c8a13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -113,7 +113,7 @@ openssl = ["chia-sha2/openssl", "clvmr/openssl"] [patch.crates-io] # Pin clvmr to clvm_rs PR #708 (serde_2026) until the format ships in a # crates.io release. Bump this rev in lockstep with that branch. -clvmr = { git = "https://github.com/Chia-Network/clvm_rs", rev = "a67ee12a269000bb6e20266bfa40769bc50ab993" } +clvmr = { git = "https://github.com/Chia-Network/clvm_rs", rev = "37e098844512f7f8e976aa4326b766b21b911601" } [profile.release] lto = "thin" diff --git a/crates/chia-consensus/src/additions_and_removals.rs b/crates/chia-consensus/src/additions_and_removals.rs index 43d941210..a95924a73 100644 --- a/crates/chia-consensus/src/additions_and_removals.rs +++ b/crates/chia-consensus/src/additions_and_removals.rs @@ -6,6 +6,7 @@ use chia_protocol::Coin; use crate::allocator::make_allocator; use crate::consensus_constants::ConsensusConstants; use crate::flags::ConsensusFlags; +use crate::serde_2026::node_from_bytes_auto; use crate::validation_error::{ErrorCode, ValidationErr, atom, first, next, rest}; use chia_protocol::{Bytes, Bytes32}; use clvm_traits::FromClvm; @@ -14,7 +15,6 @@ use clvmr::allocator::NodePtr; use clvmr::chia_dialect::ChiaDialect; use clvmr::reduction::Reduction; use clvmr::run_program::run_program; -use clvmr::serde::{DeserializeOptions, node_from_bytes_auto}; /// Run a *trusted* block generator and return its additions and removals. This /// function does not validate the block, it is assumed to be valid. @@ -36,7 +36,7 @@ where let mut cost_left = constants.max_block_cost_clvm; - let program = node_from_bytes_auto(&mut a, program, DeserializeOptions::default())?; + let program = node_from_bytes_auto(&mut a, program)?; let args = setup_generator_args(&mut a, block_refs, flags)?; let dialect = ChiaDialect::new(flags.to_clvm_flags()); diff --git a/crates/chia-consensus/src/lib.rs b/crates/chia-consensus/src/lib.rs index fafb484d6..fb6efcc97 100644 --- a/crates/chia-consensus/src/lib.rs +++ b/crates/chia-consensus/src/lib.rs @@ -25,6 +25,7 @@ pub mod owned_conditions; pub mod puzzle_fingerprint; pub mod run_block_generator; pub mod sanitize_int; +pub mod serde_2026; pub mod solution_generator; pub mod spend_visitor; pub mod spendbundle_conditions; diff --git a/crates/chia-consensus/src/run_block_generator.rs b/crates/chia-consensus/src/run_block_generator.rs index 2ebd01094..2c0ebc86e 100644 --- a/crates/chia-consensus/src/run_block_generator.rs +++ b/crates/chia-consensus/src/run_block_generator.rs @@ -24,10 +24,9 @@ use clvmr::chia_dialect::ChiaDialect; use clvmr::cost::Cost; use clvmr::reduction::Reduction; use clvmr::run_program::run_program; -use clvmr::serde::{ - DeserializeOptions, InternedTree, intern_tree, node_from_bytes, node_from_bytes_auto, - node_from_bytes_backrefs, -}; +use clvmr::serde::{InternedTree, intern_tree, node_from_bytes, node_from_bytes_backrefs}; + +use crate::serde_2026::node_from_bytes_auto; pub fn subtract_cost( a: &Allocator, @@ -242,12 +241,8 @@ where let (mut a, base_cost, program) = if flags.contains(ConsensusFlags::INTERNED_GENERATOR) { let mut decode_allocator = Allocator::new(); - let program_node = node_from_bytes_auto( - &mut decode_allocator, - program, - DeserializeOptions::default(), - ) - .map_err(|_| ValidationErr(NodePtr::NIL, ErrorCode::GeneratorRuntimeError))?; + let program_node = node_from_bytes_auto(&mut decode_allocator, program) + .map_err(|_| ValidationErr(NodePtr::NIL, ErrorCode::GeneratorRuntimeError))?; let interned = intern_tree(&decode_allocator, program_node) .map_err(|_| ValidationErr(NodePtr::NIL, ErrorCode::GeneratorRuntimeError))?; let cost = total_cost_from_tree(&interned); @@ -364,7 +359,7 @@ where check_generator_quote(generator.as_ref(), flags)?; let mut output = Vec::::new(); - let program = node_from_bytes_auto(&mut a, generator, DeserializeOptions::default())?; + let program = node_from_bytes_auto(&mut a, generator)?; check_generator_node(&a, program, flags)?; let args = setup_generator_args(&mut a, refs, flags)?; let dialect = ChiaDialect::new(flags.to_clvm_flags()); @@ -463,7 +458,7 @@ where check_generator_quote(generator.as_ref(), flags)?; let mut output = Vec::<(CoinSpend, Vec<(u32, Vec>)>)>::new(); - let program = node_from_bytes_auto(&mut a, generator, DeserializeOptions::default())?; + let program = node_from_bytes_auto(&mut a, generator)?; check_generator_node(&a, program, flags)?; let args = setup_generator_args(&mut a, refs, flags)?; let dialect = ChiaDialect::new(flags.to_clvm_flags()); diff --git a/crates/chia-consensus/src/serde_2026.rs b/crates/chia-consensus/src/serde_2026.rs new file mode 100644 index 000000000..409daf650 --- /dev/null +++ b/crates/chia-consensus/src/serde_2026.rs @@ -0,0 +1,31 @@ +//! Consensus-tuned wrappers around the `clvm_rs::serde_2026` deserializer. +//! +//! `clvm_rs` deliberately makes the caller pick `max_atom_len` and `strict`, +//! since those are policy and clvm_rs has no consensus opinion. This module +//! supplies the values chia consensus expects and exposes the +//! "sniff the magic prefix and dispatch" convenience that callers used to get +//! from `clvm_rs::serde::node_from_bytes_auto`. + +use clvmr::allocator::{Allocator, NodePtr}; +use clvmr::error::Result; +use clvmr::serde::{SERDE_2026_MAGIC_PREFIX, deserialize_2026, node_from_bytes_backrefs}; + +/// Per-atom byte cap used by chia consensus when deserializing CLVM blobs. +/// +/// Matches the historical `clvm_rs` default. CLVM atoms above this size are +/// uneconomical to construct under cost limits anyway; this is a defensive +/// pre-allocation cap, not a hard consensus rule. +pub const CONSENSUS_MAX_ATOM_LEN: usize = 1 << 20; + +/// Deserialize CLVM bytes, auto-detecting classic / backrefs / serde_2026. +/// +/// Sniffs `SERDE_2026_MAGIC_PREFIX` at the head of `bytes`; if present, +/// dispatches to [`deserialize_2026`] with consensus caps. Otherwise falls +/// back to [`node_from_bytes_backrefs`] (which also accepts plain classic). +pub fn node_from_bytes_auto(allocator: &mut Allocator, bytes: &[u8]) -> Result { + if let Some(body) = bytes.strip_prefix(SERDE_2026_MAGIC_PREFIX.as_slice()) { + deserialize_2026(allocator, body, CONSENSUS_MAX_ATOM_LEN, false) + } else { + node_from_bytes_backrefs(allocator, bytes) + } +} diff --git a/crates/clvm-utils/src/tree_hash.rs b/crates/clvm-utils/src/tree_hash.rs index bd31e5a4c..b46fb0ccb 100644 --- a/crates/clvm-utils/src/tree_hash.rs +++ b/crates/clvm-utils/src/tree_hash.rs @@ -407,8 +407,18 @@ fn test_tree_hash_from_bytes() { #[test] fn test_tree_hash_auto_matches_tree_hash_for_all_formats() { use clvmr::serde::{ - DeserializeOptions, node_from_bytes_auto, node_to_bytes, node_to_bytes_backrefs, - node_to_bytes_serde_2026, + SERDE_2026_MAGIC_PREFIX, deserialize_2026, node_from_bytes_backrefs, node_to_bytes, + node_to_bytes_backrefs, node_to_bytes_serde_2026, + }; + + // 1 MiB matches the legacy clvm_rs default; this test isn't consensus. + const TEST_MAX_ATOM_LEN: usize = 1 << 20; + let auto = |a: &mut Allocator, bytes: &[u8]| { + if let Some(body) = bytes.strip_prefix(SERDE_2026_MAGIC_PREFIX.as_slice()) { + deserialize_2026(a, body, TEST_MAX_ATOM_LEN, false) + } else { + node_from_bytes_backrefs(a, bytes) + } }; let mut a = Allocator::new(); @@ -432,16 +442,16 @@ fn test_tree_hash_auto_matches_tree_hash_for_all_formats() { // tree_hash_from_bytes rejects serde_2026 assert!(tree_hash_from_bytes(&serde_2026).is_err()); - // node_from_bytes_auto + tree_hash works for ALL formats (the - // approach used by tree_hash_auto in the Python binding) + // sniff-and-dispatch + tree_hash works for ALL formats (mirrors + // chia_rs::serde_2026::node_from_bytes_auto, which clvm-utils can't + // depend on without pulling in chia-consensus). for (label, bytes) in [ ("standard", &standard), ("backrefs", &backrefs), ("serde_2026", &serde_2026), ] { let mut a2 = Allocator::new(); - let node = node_from_bytes_auto(&mut a2, bytes, DeserializeOptions::default()) - .unwrap_or_else(|e| panic!("{label}: node_from_bytes_auto failed: {e}")); + let node = auto(&mut a2, bytes).unwrap_or_else(|e| panic!("{label}: auto failed: {e}")); let hash = tree_hash(&a2, node); assert_eq!(hash, canonical_hash, "{label}: tree_hash mismatch"); } diff --git a/wheel/src/api.rs b/wheel/src/api.rs index d80fe49e5..bebae9bc5 100644 --- a/wheel/src/api.rs +++ b/wheel/src/api.rs @@ -75,6 +75,7 @@ use crate::run_program::{run_chia_program, serialized_length, serialized_length_ use chia_consensus::fast_forward::fast_forward_singleton as native_ff; use chia_consensus::get_puzzle_and_solution::get_puzzle_and_solution_for_coin as parse_puzzle_solution; +use chia_consensus::serde_2026::node_from_bytes_auto; use chia_consensus::validation_error::ValidationErr; use clvmr::ChiaDialect; use clvmr::allocator::NodePtr; @@ -83,7 +84,7 @@ use clvmr::error::EvalErr; use clvmr::reduction::Reduction; use clvmr::run_program; use clvmr::serde::is_canonical_serialization; -use clvmr::serde::{DeserializeOptions, node_from_bytes, node_from_bytes_auto, node_to_bytes}; +use clvmr::serde::{node_from_bytes, node_to_bytes}; use chia_bls::{ BlsCache, DerivableKey, G1Element, GTElement, PublicKey, SecretKey, Signature, @@ -134,11 +135,9 @@ pub fn tree_hash<'a>(py: Python<'a>, blob: PyBuffer) -> PyResult(py: Python<'a>, blob: PyBuffer) -> PyResult> { - use clvmr::serde::{DeserializeOptions, node_from_bytes_auto}; let slice = py_to_slice::<'a>(blob); let mut a = clvmr::Allocator::new(); - let node = - node_from_bytes_auto(&mut a, slice, DeserializeOptions::default()).map_err(map_pyerr)?; + let node = node_from_bytes_auto(&mut a, slice).map_err(map_pyerr)?; let hash = clvm_utils::tree_hash(&a, node); ChiaToPython::to_python(&Bytes32::from(&hash.into()), py) } @@ -190,10 +189,10 @@ pub fn get_puzzle_and_solution_for_coin<'a>( let program = py_to_slice::<'a>(program); let args = py_to_slice::<'a>(args); - let program = node_from_bytes_auto(&mut allocator, program, DeserializeOptions::default()) - .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; - let args = node_from_bytes_auto(&mut allocator, args, DeserializeOptions::default()) + let program = node_from_bytes_auto(&mut allocator, program) .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; + let args = + node_from_bytes_auto(&mut allocator, args).map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; let dialect = &ChiaDialect::new(flags.to_clvm_flags()); let (puzzle, solution) = py @@ -250,12 +249,8 @@ pub fn get_puzzle_and_solution_for_coin2<'a>( py_to_slice::<'a>(buf) }); - let generator = node_from_bytes_auto( - &mut allocator, - generator.as_ref(), - DeserializeOptions::default(), - ) - .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; + let generator = node_from_bytes_auto(&mut allocator, generator.as_ref()) + .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; let args = setup_generator_args(&mut allocator, refs, flags)?; let dialect = &ChiaDialect::new(flags.to_clvm_flags()); diff --git a/wheel/src/run_generator.rs b/wheel/src/run_generator.rs index b8b01ae84..18b3e4e66 100644 --- a/wheel/src/run_generator.rs +++ b/wheel/src/run_generator.rs @@ -9,9 +9,10 @@ use chia_consensus::run_block_generator::run_block_generator2 as native_run_bloc use chia_consensus::validation_error::ValidationErr; use chia_protocol::{Bytes, Bytes32, Coin}; +use chia_consensus::serde_2026::node_from_bytes_auto; use clvmr::allocator::Allocator; use clvmr::cost::Cost; -use clvmr::serde::{DeserializeOptions, intern_tree, node_from_bytes_auto}; +use clvmr::serde::intern_tree; use pyo3::PyResult; use pyo3::buffer::PyBuffer; @@ -151,7 +152,7 @@ pub fn additions_and_removals<'a>( pub fn generator_interned_weight(program: PyBuffer) -> PyResult { let program = py_to_slice(program); let mut a = Allocator::new(); - let node = node_from_bytes_auto(&mut a, program, DeserializeOptions::default()) + let node = node_from_bytes_auto(&mut a, program) .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("bad generator: {e}")))?; let tree = intern_tree(&a, node) .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("intern failed: {e}")))?; diff --git a/wheel/src/run_program.rs b/wheel/src/run_program.rs index d51ff9bbc..f682c68a0 100644 --- a/wheel/src/run_program.rs +++ b/wheel/src/run_program.rs @@ -1,15 +1,13 @@ use crate::error::{map_pyerr, map_pyerr_w_ptr}; use chia_consensus::allocator::make_allocator; use chia_consensus::flags::ConsensusFlags; +use chia_consensus::serde_2026::node_from_bytes_auto; use chia_protocol::LazyNode; use clvmr::chia_dialect::ChiaDialect; use clvmr::cost::Cost; use clvmr::reduction::Response; use clvmr::run_program::run_program; -use clvmr::serde::{ - DeserializeOptions, node_from_bytes_auto, serialized_length_from_bytes, - serialized_length_from_bytes_trusted, -}; +use clvmr::serde::{serialized_length_from_bytes, serialized_length_from_bytes_trusted}; use pyo3::buffer::PyBuffer; use pyo3::prelude::*; use std::rc::Rc; @@ -45,9 +43,9 @@ pub fn run_chia_program( let flags = flags.to_clvm_flags(); let reduction = (|| -> PyResult { - let program = node_from_bytes_auto(&mut allocator, program, DeserializeOptions::default()) + let program = node_from_bytes_auto(&mut allocator, program) .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; - let args = node_from_bytes_auto(&mut allocator, args, DeserializeOptions::default()) + let args = node_from_bytes_auto(&mut allocator, args) .map_err(|e| map_pyerr_w_ptr(&e, &allocator))?; let dialect = ChiaDialect::new(flags); From df025b91add5238fa6167c4b67665f6ec9495280 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 4 May 2026 16:15:41 -0700 Subject: [PATCH 17/18] serde_2026: track upstream rename + new node_from_bytes_serde_2026 clvm_rs renamed `deserialize_2026{,_from_stream}` to make the body-only contract visible (`_body` suffix), added `node_from_bytes_serde_2026` as the prefix-aware counterpart to `node_to_bytes_serde_2026`, and parametrized `serialized_length_serde_2026` so it mirrors every header-time validation the body deserializer does. Updates: - chia_consensus::serde_2026::node_from_bytes_auto: use the new `node_from_bytes_serde_2026` (no manual prefix stripping). - chia-protocol::Program::{parse, from_json_dict}: pass `(usize::MAX, false)` to the length helper. Framing-only callers don't claim a consensus opinion; max_atom_len is enforced later, at run_block_generator time. - clvm-utils::tree_hash test: use `node_from_bytes_serde_2026`. Bumps clvmr to af8cb91b. Made-with: Cursor --- Cargo.lock | 2 +- Cargo.toml | 2 +- crates/chia-consensus/src/serde_2026.rs | 11 ++++++----- crates/chia-protocol/src/program.rs | 9 +++++++-- crates/clvm-utils/src/tree_hash.rs | 8 ++++---- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8b8bd9aaf..9865b7d9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -866,7 +866,7 @@ dependencies = [ [[package]] name = "clvmr" version = "0.17.7" -source = "git+https://github.com/Chia-Network/clvm_rs?rev=37e098844512f7f8e976aa4326b766b21b911601#37e098844512f7f8e976aa4326b766b21b911601" +source = "git+https://github.com/Chia-Network/clvm_rs?rev=af8cb91b132c2341d0bd34d093bc133f89b3b16b#af8cb91b132c2341d0bd34d093bc133f89b3b16b" dependencies = [ "bitflags", "bitvec", diff --git a/Cargo.toml b/Cargo.toml index 5db2c8a13..df2db1b80 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -113,7 +113,7 @@ openssl = ["chia-sha2/openssl", "clvmr/openssl"] [patch.crates-io] # Pin clvmr to clvm_rs PR #708 (serde_2026) until the format ships in a # crates.io release. Bump this rev in lockstep with that branch. -clvmr = { git = "https://github.com/Chia-Network/clvm_rs", rev = "37e098844512f7f8e976aa4326b766b21b911601" } +clvmr = { git = "https://github.com/Chia-Network/clvm_rs", rev = "af8cb91b132c2341d0bd34d093bc133f89b3b16b" } [profile.release] lto = "thin" diff --git a/crates/chia-consensus/src/serde_2026.rs b/crates/chia-consensus/src/serde_2026.rs index 409daf650..7e2559da4 100644 --- a/crates/chia-consensus/src/serde_2026.rs +++ b/crates/chia-consensus/src/serde_2026.rs @@ -8,7 +8,7 @@ use clvmr::allocator::{Allocator, NodePtr}; use clvmr::error::Result; -use clvmr::serde::{SERDE_2026_MAGIC_PREFIX, deserialize_2026, node_from_bytes_backrefs}; +use clvmr::serde::{SERDE_2026_MAGIC_PREFIX, node_from_bytes_backrefs, node_from_bytes_serde_2026}; /// Per-atom byte cap used by chia consensus when deserializing CLVM blobs. /// @@ -20,11 +20,12 @@ pub const CONSENSUS_MAX_ATOM_LEN: usize = 1 << 20; /// Deserialize CLVM bytes, auto-detecting classic / backrefs / serde_2026. /// /// Sniffs `SERDE_2026_MAGIC_PREFIX` at the head of `bytes`; if present, -/// dispatches to [`deserialize_2026`] with consensus caps. Otherwise falls -/// back to [`node_from_bytes_backrefs`] (which also accepts plain classic). +/// dispatches to [`node_from_bytes_serde_2026`] with consensus caps. +/// Otherwise falls back to [`node_from_bytes_backrefs`] (which also accepts +/// plain classic). pub fn node_from_bytes_auto(allocator: &mut Allocator, bytes: &[u8]) -> Result { - if let Some(body) = bytes.strip_prefix(SERDE_2026_MAGIC_PREFIX.as_slice()) { - deserialize_2026(allocator, body, CONSENSUS_MAX_ATOM_LEN, false) + if bytes.starts_with(&SERDE_2026_MAGIC_PREFIX) { + node_from_bytes_serde_2026(allocator, bytes, CONSENSUS_MAX_ATOM_LEN, false) } else { node_from_bytes_backrefs(allocator, bytes) } diff --git a/crates/chia-protocol/src/program.rs b/crates/chia-protocol/src/program.rs index 38cffb41b..efceb84ef 100644 --- a/crates/chia-protocol/src/program.rs +++ b/crates/chia-protocol/src/program.rs @@ -438,7 +438,11 @@ impl Streamable for Program { let pos = input.position(); let buf: &[u8] = &input.get_ref()[pos as usize..]; let len = if buf.starts_with(&SERDE_2026_MAGIC_PREFIX) { - serialized_length_serde_2026(buf).map_err(|_e| Error::EndOfBuffer)? + // Framing-only walk: chia-protocol just needs to find the byte + // boundary. Consensus caps (max_atom_len) are enforced later, at + // run_block_generator time, so accept anything that's structurally + // walkable here. + serialized_length_serde_2026(buf, usize::MAX, false).map_err(|_e| Error::EndOfBuffer)? } else if TRUSTED { serialized_length_from_bytes_trusted(buf).map_err(|_e| Error::EndOfBuffer)? } else { @@ -486,7 +490,8 @@ impl FromJsonDict for Program { let bytes = Bytes::from_json_dict(o)?; let buf = bytes.as_slice(); let len = if buf.starts_with(&SERDE_2026_MAGIC_PREFIX) { - serialized_length_serde_2026(buf).map_err(|_e| Error::EndOfBuffer)? + // Framing-only walk; see Streamable::parse for rationale. + serialized_length_serde_2026(buf, usize::MAX, false).map_err(|_e| Error::EndOfBuffer)? } else { serialized_length_from_bytes(buf).map_err(|_e| Error::EndOfBuffer)? }; diff --git a/crates/clvm-utils/src/tree_hash.rs b/crates/clvm-utils/src/tree_hash.rs index b46fb0ccb..2e1c21c3a 100644 --- a/crates/clvm-utils/src/tree_hash.rs +++ b/crates/clvm-utils/src/tree_hash.rs @@ -407,15 +407,15 @@ fn test_tree_hash_from_bytes() { #[test] fn test_tree_hash_auto_matches_tree_hash_for_all_formats() { use clvmr::serde::{ - SERDE_2026_MAGIC_PREFIX, deserialize_2026, node_from_bytes_backrefs, node_to_bytes, - node_to_bytes_backrefs, node_to_bytes_serde_2026, + SERDE_2026_MAGIC_PREFIX, node_from_bytes_backrefs, node_from_bytes_serde_2026, + node_to_bytes, node_to_bytes_backrefs, node_to_bytes_serde_2026, }; // 1 MiB matches the legacy clvm_rs default; this test isn't consensus. const TEST_MAX_ATOM_LEN: usize = 1 << 20; let auto = |a: &mut Allocator, bytes: &[u8]| { - if let Some(body) = bytes.strip_prefix(SERDE_2026_MAGIC_PREFIX.as_slice()) { - deserialize_2026(a, body, TEST_MAX_ATOM_LEN, false) + if bytes.starts_with(&SERDE_2026_MAGIC_PREFIX) { + node_from_bytes_serde_2026(a, bytes, TEST_MAX_ATOM_LEN, false) } else { node_from_bytes_backrefs(a, bytes) } From 241aa8800c20813d8bf6f6ffa7a158e41ddc234b Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 5 May 2026 13:30:48 -0700 Subject: [PATCH 18/18] expose SERDE_2026_MAGIC_PREFIX as Python constant chia-blockchain needs the magic prefix to gate serde_2026 generators behind HF2 (reject pre-HF2, accept after). The constant lives canonically in clvmr::serde, but PyPI clvm_rs is capped at 0.2.x by chia-puzzles-py and chia-base, so importing from clvm_rs Python isn't viable yet. Re-export through chia_rs (which already pulls clvmr from git) until serde_2026 ships and downstream caps relax. Co-authored-by: Cursor --- wheel/generate_type_stubs.py | 2 ++ wheel/python/chia_rs/chia_rs.pyi | 2 ++ wheel/src/api.rs | 6 +++++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/wheel/generate_type_stubs.py b/wheel/generate_type_stubs.py index 89d396b08..97a59f1c4 100644 --- a/wheel/generate_type_stubs.py +++ b/wheel/generate_type_stubs.py @@ -451,6 +451,8 @@ def compute_plot_id_v2(strength: uint8, plot_pk: G1Element, pool_pk: G1Element | ELIGIBLE_FOR_DEDUP: int = ... ELIGIBLE_FOR_FF: int = ... +SERDE_2026_MAGIC_PREFIX: bytes = ... + NO_UNKNOWN_OPS: int = ... def run_chia_program( diff --git a/wheel/python/chia_rs/chia_rs.pyi b/wheel/python/chia_rs/chia_rs.pyi index a4e2bc14a..6dd5fdcfe 100644 --- a/wheel/python/chia_rs/chia_rs.pyi +++ b/wheel/python/chia_rs/chia_rs.pyi @@ -143,6 +143,8 @@ MALACHITE: int = ... ELIGIBLE_FOR_DEDUP: int = ... ELIGIBLE_FOR_FF: int = ... +SERDE_2026_MAGIC_PREFIX: bytes = ... + NO_UNKNOWN_OPS: int = ... def run_chia_program( diff --git a/wheel/src/api.rs b/wheel/src/api.rs index bebae9bc5..73fcf2735 100644 --- a/wheel/src/api.rs +++ b/wheel/src/api.rs @@ -84,7 +84,7 @@ use clvmr::error::EvalErr; use clvmr::reduction::Reduction; use clvmr::run_program; use clvmr::serde::is_canonical_serialization; -use clvmr::serde::{node_from_bytes, node_to_bytes}; +use clvmr::serde::{SERDE_2026_MAGIC_PREFIX, node_from_bytes, node_to_bytes}; use chia_bls::{ BlsCache, DerivableKey, G1Element, GTElement, PublicKey, SecretKey, Signature, @@ -803,6 +803,10 @@ pub fn chia_rs(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { chia_consensus::conditions::ELIGIBLE_FOR_FF, )?; m.add_class::()?; + m.add( + "SERDE_2026_MAGIC_PREFIX", + PyBytes::new(py, &SERDE_2026_MAGIC_PREFIX), + )?; // pot functions m.add_function(wrap_pyfunction!(py_calculate_sp_interval_iters, m)?)?;