From b7698a9e6d628cf2ced291e0dac61797bbe7b06e Mon Sep 17 00:00:00 2001 From: KaffinPX Date: Tue, 1 Sep 2026 01:56:31 +0300 Subject: [PATCH 01/14] feat(standards): Generalize notes and deprecate old types --- node/src/application/config/cli_args.rs | 2 +- standards/src/wallet/keys/address.rs | 44 +---- standards/src/wallet/keys/mod.rs | 25 ++- .../src/wallet/keys/schemes/generation.rs | 35 +--- .../src/wallet/keys/schemes/symmetric.rs | 39 +--- .../src/wallet/notes/announcement_flag.rs | 69 ------- standards/src/wallet/notes/content.rs | 83 +++++++++ .../notes/encrypted_utxo_notification.rs | 111 ----------- standards/src/wallet/notes/mod.rs | 5 +- standards/src/wallet/notes/note.rs | 176 ++++++++++++++++++ .../src/wallet/notes/utxo_notification.rs | 36 ---- 11 files changed, 298 insertions(+), 327 deletions(-) delete mode 100644 standards/src/wallet/notes/announcement_flag.rs create mode 100644 standards/src/wallet/notes/content.rs delete mode 100644 standards/src/wallet/notes/encrypted_utxo_notification.rs create mode 100644 standards/src/wallet/notes/note.rs delete mode 100644 standards/src/wallet/notes/utxo_notification.rs diff --git a/node/src/application/config/cli_args.rs b/node/src/application/config/cli_args.rs index d7463bb..18591d7 100644 --- a/node/src/application/config/cli_args.rs +++ b/node/src/application/config/cli_args.rs @@ -11,8 +11,8 @@ use clap::builder::TypedValueParser; use clap::Parser; use libp2p::multiaddr::Protocol; use libp2p::Multiaddr; -use nyks_consensus::transaction::transaction_proof::TransactionProofQuality; use nyks_consensus::network::Network; +use nyks_consensus::transaction::transaction_proof::TransactionProofQuality; use nyks_consensus::type_scripts::native_currency_amount::NativeCurrencyAmount; use nyks_rpc_core::api::ops::Namespace; use tracing::error; diff --git a/standards/src/wallet/keys/address.rs b/standards/src/wallet/keys/address.rs index 2dd0c23..4216ecd 100644 --- a/standards/src/wallet/keys/address.rs +++ b/standards/src/wallet/keys/address.rs @@ -1,7 +1,6 @@ use nyks_consensus::BFieldElement; use nyks_consensus::network::Network; use nyks_consensus::tasm_lib::prelude::Digest; -use nyks_consensus::transaction::announcement::Announcement; use nyks_consensus::transaction::lock_script::LockScript; use serde::Deserialize; use serde::Serialize; @@ -9,7 +8,8 @@ use thiserror::Error; use crate::wallet::keys::schemes::generation::GenerationAddress; use crate::wallet::keys::schemes::symmetric::SymmetricAddress; -use crate::wallet::notes::utxo_notification::UtxoNotificationPayload; +use crate::wallet::notes::content::NoteContent; +use crate::wallet::notes::note::Note; #[derive(Debug, Error)] pub enum Bech32mDecodeError { @@ -54,27 +54,8 @@ pub trait Recipient: Sync { /// the transaction. fn lock_script(&self) -> LockScript; - /// Generates an [Announcement] for a UTXO notification. - /// - /// The announcement typically contains: - /// - /// - type flag - /// - receiver identifier - /// - encrypted payload - /// - /// These fields allow the receiver to determine whether decryption should - /// be attempted. - fn create_note_announcement( - &self, - utxo_notification_payload: &UtxoNotificationPayload, - ) -> Announcement; - - // Generates a redeemable note (the data someone needs to claim and use an UTXO) intended for off-chain use. - fn create_note( - &self, - utxo_notification_payload: &UtxoNotificationPayload, - network: Network, - ) -> String; + /// TODO: comment + fn create_private_note(&self, content: &NoteContent) -> Note; } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -128,21 +109,10 @@ impl Recipient for Address { } } - fn create_note_announcement(&self, payload: &UtxoNotificationPayload) -> Announcement { - match self { - Address::Generation(a) => a.create_note_announcement(payload), - Address::Symmetric(a) => a.create_note_announcement(payload), - } - } - - fn create_note( - &self, - utxo_notification_payload: &UtxoNotificationPayload, - network: Network, - ) -> String { + fn create_private_note(&self, content: &NoteContent) -> Note { match self { - Address::Generation(a) => a.create_note(utxo_notification_payload, network), - Address::Symmetric(a) => a.create_note(utxo_notification_payload, network), + Address::Generation(a) => a.create_private_note(content), + Address::Symmetric(a) => a.create_private_note(content), } } } diff --git a/standards/src/wallet/keys/mod.rs b/standards/src/wallet/keys/mod.rs index 8776149..a2d01e1 100644 --- a/standards/src/wallet/keys/mod.rs +++ b/standards/src/wallet/keys/mod.rs @@ -7,7 +7,7 @@ use sha3::Shake256; use sha3::digest::ExtendableOutput; use sha3::digest::Update; -use crate::wallet::notes::utxo_notification::UtxoNotificationPayload; +use crate::wallet::notes::content::NoteContent; pub mod address; pub mod key; @@ -29,17 +29,22 @@ pub(crate) fn network_hrp_char(network: Network) -> char { /// reuse proofs for tests. These values are used in the encryption /// step. pub(crate) fn deterministically_derive_seed_and_nonce( - payload: &UtxoNotificationPayload, + content: &NoteContent, ) -> ([u8; 32], BFieldElement) { - let combined = Tip5::hash_pair(payload.sender_randomness, payload.utxo.lock_script_hash()); - let [e0, e1, e2, e3, e4] = combined.values(); - let e0: [u8; 8] = e0.into(); - let e1: [u8; 8] = e1.into(); - let e2: [u8; 8] = e2.into(); - let e3: [u8; 8] = e3.into(); - let seed: [u8; 32] = [e0, e1, e2, e3].concat().try_into().unwrap(); + match content { + NoteContent::Utxo(u) => { + let combined = Tip5::hash_pair(u.sender_randomness, Tip5::hash(&u.utxo)); + let [e0, e1, e2, e3, e4] = combined.values(); + let e0: [u8; 8] = e0.into(); + let e1: [u8; 8] = e1.into(); + let e2: [u8; 8] = e2.into(); + let e3: [u8; 8] = e3.into(); + let seed: [u8; 32] = [e0, e1, e2, e3].concat().try_into().unwrap(); - (seed, e4) + (seed, e4) + } + NoteContent::Message(_) => todo!(), + } } // note: copied from twenty_first::math::lattice::kem::shake256() diff --git a/standards/src/wallet/keys/schemes/generation.rs b/standards/src/wallet/keys/schemes/generation.rs index f89c207..739e66b 100644 --- a/standards/src/wallet/keys/schemes/generation.rs +++ b/standards/src/wallet/keys/schemes/generation.rs @@ -9,7 +9,6 @@ use nyks_consensus::BFieldElement; use nyks_consensus::network::Network; use nyks_consensus::tasm_lib::prelude::Digest; use nyks_consensus::tasm_lib::prelude::Tip5; -use nyks_consensus::transaction::announcement::Announcement; use nyks_consensus::transaction::lock_script::LockScript; use nyks_consensus::transaction::lock_script::LockScriptAndWitness; use nyks_consensus::transaction::utxo::Utxo; @@ -32,8 +31,9 @@ use crate::wallet::keys::key::Spender; use crate::wallet::keys::network_hrp_char; use crate::wallet::keys::shake256; use crate::wallet::keys::viewing_key::Decryptor; -use crate::wallet::notes::encrypted_utxo_notification::EncryptedUtxoNotification; -use crate::wallet::notes::utxo_notification::UtxoNotificationPayload; +use crate::wallet::notes::content::NoteContent; +use crate::wallet::notes::note::Note; +use crate::wallet::notes::note::PrivateNote; pub(crate) const GENERATION_FLAG_U8: u8 = 79; pub const GENERATION_FLAG: BFieldElement = BFieldElement::new(GENERATION_FLAG_U8 as u64); @@ -60,7 +60,7 @@ impl GenerationAddress { } // Used beneath private_note etc. - fn encrypt(&self, payload: &UtxoNotificationPayload) -> Vec { + fn encrypt(&self, payload: &NoteContent) -> Vec { let (randomness, nonce_bfe) = deterministically_derive_seed_and_nonce(payload); let (shared_key, kem_ctxt) = lattice::kem::enc(self.encryption_key, randomness); @@ -121,31 +121,8 @@ impl Recipient for GenerationAddress { LockScript::standard_hash_lock_from_after_image(self.lock_postimage) } - fn create_note_announcement( - &self, - utxo_notification_payload: &UtxoNotificationPayload, - ) -> Announcement { - let encrypted_utxo_notification = EncryptedUtxoNotification { - flag: GENERATION_FLAG_U8.into(), - receiver_identifier: self.receiver_identifier(), - ciphertext: self.encrypt(utxo_notification_payload), - }; - - encrypted_utxo_notification.into_announcement() - } - - fn create_note( - &self, - utxo_notification_payload: &UtxoNotificationPayload, - network: Network, - ) -> String { - let encrypted_utxo_notification = EncryptedUtxoNotification { - flag: GENERATION_FLAG_U8.into(), - receiver_identifier: self.receiver_identifier(), - ciphertext: self.encrypt(utxo_notification_payload), - }; - - encrypted_utxo_notification.into_bech32m(network) + fn create_private_note(&self, content: &NoteContent) -> Note { + PrivateNote::new(self.receiver_identifier(), self.encrypt(content)).into() } } diff --git a/standards/src/wallet/keys/schemes/symmetric.rs b/standards/src/wallet/keys/schemes/symmetric.rs index e2ecdd5..2a6d012 100644 --- a/standards/src/wallet/keys/schemes/symmetric.rs +++ b/standards/src/wallet/keys/schemes/symmetric.rs @@ -10,7 +10,6 @@ use nyks_consensus::BFieldElement; use nyks_consensus::network::Network; use nyks_consensus::tasm_lib::prelude::Digest; use nyks_consensus::tasm_lib::prelude::Tip5; -use nyks_consensus::transaction::announcement::Announcement; use nyks_consensus::transaction::lock_script::LockScript; use nyks_consensus::transaction::lock_script::LockScriptAndWitness; use nyks_consensus::transaction::utxo::Utxo; @@ -29,8 +28,9 @@ use crate::wallet::keys::key::Spender; use crate::wallet::keys::network_hrp_char; use crate::wallet::keys::shake256; use crate::wallet::keys::viewing_key::Decryptor; -use crate::wallet::notes::encrypted_utxo_notification::EncryptedUtxoNotification; -use crate::wallet::notes::utxo_notification::UtxoNotificationPayload; +use crate::wallet::notes::content::NoteContent; +use crate::wallet::notes::note::Note; +use crate::wallet::notes::note::PrivateNote; pub(crate) const SYMMETRIC_FLAG_U8: u8 = 80; pub const SYMMETRIC_FLAG: BFieldElement = BFieldElement::new(SYMMETRIC_FLAG_U8 as u64); @@ -54,15 +54,15 @@ impl SymmetricAddress { hrp } - fn encrypt(&self, payload: &UtxoNotificationPayload) -> Vec { + fn encrypt(&self, content: &NoteContent) -> Vec { // 1. derive nonce deterministically - let (_randomness, nonce_bfe) = deterministically_derive_seed_and_nonce(payload); + let (_randomness, nonce_bfe) = deterministically_derive_seed_and_nonce(content); let nonce_bytes = [&nonce_bfe.value().to_be_bytes(), [0u8; 4].as_slice()].concat(); let nonce = Nonce::from_slice(&nonce_bytes); // 2. serialize payload - let plaintext = bincode::serialize(payload).unwrap(); + let plaintext = bincode::serialize(content).unwrap(); // 3. encrypt let cipher = Aes256Gcm::new(&derive_encryption_secret(&self.receiver_postimage)); @@ -113,31 +113,8 @@ impl Recipient for SymmetricAddress { LockScript::standard_hash_lock_from_after_image(self.lock_postimage) } - fn create_note_announcement( - &self, - utxo_notification_payload: &UtxoNotificationPayload, - ) -> Announcement { - let encrypted_utxo_notification = EncryptedUtxoNotification { - flag: SYMMETRIC_FLAG_U8.into(), - receiver_identifier: self.receiver_identifier(), - ciphertext: self.encrypt(utxo_notification_payload), - }; - - encrypted_utxo_notification.into_announcement() - } - - fn create_note( - &self, - utxo_notification_payload: &UtxoNotificationPayload, - network: Network, - ) -> String { - let encrypted_utxo_notification = EncryptedUtxoNotification { - flag: SYMMETRIC_FLAG_U8.into(), - receiver_identifier: self.receiver_identifier(), - ciphertext: self.encrypt(utxo_notification_payload), - }; - - encrypted_utxo_notification.into_bech32m(network) + fn create_private_note(&self, content: &NoteContent) -> Note { + PrivateNote::new(self.receiver_identifier(), self.encrypt(content)).into() } } diff --git a/standards/src/wallet/notes/announcement_flag.rs b/standards/src/wallet/notes/announcement_flag.rs deleted file mode 100644 index 881bb10..0000000 --- a/standards/src/wallet/notes/announcement_flag.rs +++ /dev/null @@ -1,69 +0,0 @@ -use std::cmp::Ordering; - -use nyks_consensus::BFieldElement; -use nyks_consensus::transaction::announcement::Announcement; -use serde::Deserialize; -use serde::Serialize; - -use crate::wallet::keys::address::Address; -use crate::wallet::keys::address::Recipient; - -/// Announcement meta-information, intended for use in combination with -/// [`ReceivingAddress`]. Can be used to quickly identify if the announcement -/// relates to a specific [`ReceivingAddress`]. Contains the first two elements -/// of an announcement, as these are interpreted as purpose, receiver ID, -/// respectively. -/// -/// [`ReceivingAddress`]: crate::api::export::ReceivingAddress -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] -pub struct AnnouncementFlag { - /// Purpose of the announcement. E.g.: announcement for generational - /// address, or for symmetric key address. - pub flag: BFieldElement, - - /// An ID identifying the receiver. - pub receiver_id: BFieldElement, -} - -impl Ord for AnnouncementFlag { - // Ordering is implemented to allow for idempotent and deterministic lookup - // tables. - fn cmp(&self, other: &Self) -> Ordering { - match self.flag.value().cmp(&other.flag.value()) { - Ordering::Equal => self.receiver_id.value().cmp(&other.receiver_id.value()), - non_eq => non_eq, - } - } -} - -impl PartialOrd for AnnouncementFlag { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl From<&Address> for AnnouncementFlag { - fn from(address: &Address) -> Self { - Self { - flag: address.flag(), - receiver_id: address.receiver_identifier(), - } - } -} - -impl TryFrom<&Announcement> for AnnouncementFlag { - // Only possible converstion error is that announcement message is too - // short. - type Error = (); - - fn try_from(value: &Announcement) -> Result { - if value.message.len() < 2 { - return Err(()); - } - - Ok(AnnouncementFlag { - flag: value.message[0], - receiver_id: value.message[1], - }) - } -} diff --git a/standards/src/wallet/notes/content.rs b/standards/src/wallet/notes/content.rs new file mode 100644 index 0000000..c573d5e --- /dev/null +++ b/standards/src/wallet/notes/content.rs @@ -0,0 +1,83 @@ +use std::fmt::Debug; + +use anyhow::Result; +use anyhow::bail; +use nyks_consensus::BFieldElement; +use nyks_consensus::transaction::utxo::Utxo; +use nyks_consensus::twenty_first::math::bfield_codec::BFieldCodec; +use nyks_consensus::twenty_first::tip5::Digest; +use serde::Deserialize; +use serde::Serialize; + +pub trait Content: + Clone + Debug + PartialEq + Eq + Send + Sync + BFieldCodec + for<'de> Deserialize<'de> + Serialize +{ + /// Unique discriminant used in the note header. + const DISCRIMINANT: u64; +} + +/// Plain message content: an arbitrary list of BFieldElements. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, BFieldCodec)] +pub struct MessageContent(pub Vec); + +impl Content for MessageContent { + const DISCRIMINANT: u64 = 0; +} + +/// UTXO notification payload. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, BFieldCodec)] +pub struct UtxoContent { + pub utxo: Utxo, + pub sender_randomness: Digest, +} + +impl Content for UtxoContent { + const DISCRIMINANT: u64 = 1; +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum NoteContent { + Message(MessageContent), + Utxo(UtxoContent), +} + +impl NoteContent { + /// Returns the discriminant of the contained content. + pub fn discriminant(&self) -> u64 { + match self { + Self::Message(_) => MessageContent::DISCRIMINANT, + Self::Utxo(_) => UtxoContent::DISCRIMINANT, + } + } + + /// Encodes the content into a vector of BFieldElements. + pub fn encode(&self) -> Vec { + match self { + Self::Message(m) => m.encode(), + Self::Utxo(u) => u.encode(), + } + } + + /// Decodes content from a discriminant and a data slice. + pub fn decode(disc: u64, data: &[BFieldElement]) -> Result { + match disc { + d if d == MessageContent::DISCRIMINANT => { + Ok(Self::Message(*MessageContent::decode(data)?)) + } + d if d == UtxoContent::DISCRIMINANT => Ok(Self::Utxo(*UtxoContent::decode(data)?)), + _ => bail!("Unknown content discriminant: {disc}"), + } + } +} + +impl From for NoteContent { + fn from(m: MessageContent) -> Self { + Self::Message(m) + } +} + +impl From for NoteContent { + fn from(u: UtxoContent) -> Self { + Self::Utxo(u) + } +} diff --git a/standards/src/wallet/notes/encrypted_utxo_notification.rs b/standards/src/wallet/notes/encrypted_utxo_notification.rs deleted file mode 100644 index 7e766e9..0000000 --- a/standards/src/wallet/notes/encrypted_utxo_notification.rs +++ /dev/null @@ -1,111 +0,0 @@ -use anyhow::Result; -use anyhow::anyhow; -use anyhow::ensure; -use bech32::FromBase32; -use bech32::ToBase32; -use nyks_consensus::BFieldElement; -use nyks_consensus::network::Network; -use nyks_consensus::transaction::announcement::Announcement; -use nyks_consensus::triton_vm::prelude::BFieldCodec; -use serde::Deserialize; -use serde::Serialize; -use thiserror::Error; - -use crate::wallet::keys::network_hrp_char; - -/// an encrypted wrapper for UTXO notifications. -/// -/// This type is intended to be serialized and actually transferred between -/// parties. -/// -/// note: bech32m encoding of this type is considered standard and is -/// recommended over serde serialization. -/// -/// the receiver_identifier enables the receiver to find the matching -/// `SpendingKey` in their wallet. -#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, BFieldCodec)] -pub struct EncryptedUtxoNotification { - /// Describes the type of encoding used here - pub(crate) flag: BFieldElement, - - /// enables the receiver to find the matching `SpendingKey` in their wallet. - pub(crate) receiver_identifier: BFieldElement, - - /// Encrypted UTXO notification payload. - pub(crate) ciphertext: Vec, -} - -#[derive(Debug, Copy, Clone, Error)] -pub enum ConversionFromMessageError { - #[error("message too short: length is {0}, minimum required is 2")] - MessageTooShort(usize), -} - -impl EncryptedUtxoNotification { - fn into_message(self) -> Vec { - [vec![self.flag, self.receiver_identifier], self.ciphertext].concat() - } - - fn from_message(message: Vec) -> Result { - if message.len() < 2 { - Err(ConversionFromMessageError::MessageTooShort(message.len())) - } else { - Ok(Self { - flag: message[0], - receiver_identifier: message[1], - ciphertext: message[2..].to_vec(), - }) - } - } - - /// Convert an encrypted UTXO notification to a announcement. Leaks - /// privacy in the form of `receiver_identifier` is addresses are reused. - /// Never leaks actual UTXO info such as amount transferred. - pub(crate) fn into_announcement(self) -> Announcement { - // We could use `BfieldCodec` encode here. But it might be a bit faster - // to filter out irrelevant announcement if we don't have to - // attempt a decoding to a specific data type first but can instead just - // read out b-field elements and skip items based on that. - Announcement::new(self.into_message()) - } - - pub fn into_bech32m(self, network: Network) -> String { - let hrp = Self::get_hrp(network); - let message = self.into_message(); - let payload = bincode::serialize(&message).unwrap_or_else(|e| { - panic!("Serialization shouldn't fail. Message was: {message:?}\nerror: {e}") - }); - let payload_base_32 = payload.to_base32(); - let variant = bech32::Variant::Bech32m; - bech32::encode(&hrp, payload_base_32, variant).unwrap_or_else(|e| panic!( - "bech32 encoding shouldn't fail. Arguments were:\n\n{hrp}\n\n{payload:?}\n\n{variant:?}\n\nerror: {e}" - )) - } - - /// decodes from a bech32m string and verifies it matches `network` - pub fn from_bech32m(encoded: &str, network: Network) -> Result { - let (hrp, data, variant) = bech32::decode(encoded)?; - - ensure!( - variant == bech32::Variant::Bech32m, - "Can only decode bech32m addresses." - ); - ensure!( - hrp == *Self::get_hrp(network), - "Could not decode bech32m address because of invalid prefix", - ); - - let payload = Vec::::from_base32(&data)?; - let message = bincode::deserialize(&payload) - .map_err(|e| anyhow!("Could not decode bech32m because of error: {e}"))?; - let encrypted_utxo_notification = Self::from_message(message) - .map_err(|e| anyhow!("conversion from bech32m failed: {e}"))?; - - Ok(encrypted_utxo_notification) - } - - /// returns human readable prefix (hrp) of a utxo-transfer-encrypted, specific to `network` - pub(crate) fn get_hrp(network: Network) -> String { - format!("utxo{}", network_hrp_char(network)) - } -} diff --git a/standards/src/wallet/notes/mod.rs b/standards/src/wallet/notes/mod.rs index 8ce44e4..b9491d2 100644 --- a/standards/src/wallet/notes/mod.rs +++ b/standards/src/wallet/notes/mod.rs @@ -1,3 +1,2 @@ -pub mod announcement_flag; -pub mod encrypted_utxo_notification; -pub mod utxo_notification; +pub mod content; +pub mod note; diff --git a/standards/src/wallet/notes/note.rs b/standards/src/wallet/notes/note.rs new file mode 100644 index 0000000..b73bdd0 --- /dev/null +++ b/standards/src/wallet/notes/note.rs @@ -0,0 +1,176 @@ +use anyhow::Result; +use anyhow::bail; +use anyhow::ensure; +use bech32::FromBase32; +use bech32::ToBase32; +use nyks_consensus::BFieldElement; +use nyks_consensus::network::Network; +use nyks_consensus::transaction::announcement::Announcement; +use serde::Deserialize; +use serde::Serialize; + +use crate::wallet::keys::network_hrp_char; +use crate::wallet::notes::content::NoteContent; + +pub(crate) const TAG_PUBLIC: u64 = 0; +pub(crate) const TAG_PRIVATE: u64 = 1; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PublicNote { + pub receiver_id: BFieldElement, + pub content: NoteContent, +} + +impl PublicNote { + pub fn new(receiver_id: BFieldElement, content: NoteContent) -> Self { + Self { + receiver_id, + content, + } + } + + pub fn into_message(&self) -> Vec { + let mut msg = vec![ + BFieldElement::new(TAG_PUBLIC), + self.receiver_id, + BFieldElement::new(self.content.discriminant()), + ]; + msg.extend(self.content.encode()); + msg + } + + pub fn from_message(data: &[BFieldElement]) -> Result { + if data.len() < 3 { + bail!("Public note too short"); + } + if data[0].value() != TAG_PUBLIC { + bail!("Expected public tag, got {}", data[0].value()); + } + let receiver_id = data[1]; + let disc = data[2].value(); + let content = NoteContent::decode(disc, &data[3..])?; + Ok(Self { + receiver_id, + content, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PrivateNote { + pub receiver_id: BFieldElement, + pub ciphertext: Vec, +} + +impl PrivateNote { + pub fn new(receiver_id: BFieldElement, ciphertext: Vec) -> Self { + Self { + receiver_id, + ciphertext, + } + } + + pub fn into_message(&self) -> Vec { + let mut msg = vec![BFieldElement::new(TAG_PRIVATE), self.receiver_id]; + msg.extend(self.ciphertext.clone()); + msg + } + + pub fn from_message(data: &[BFieldElement]) -> Result { + if data.len() < 2 { + bail!("Private note too short"); + } + if data[0].value() != TAG_PRIVATE { + bail!("Expected private tag, got {}", data[0].value()); + } + let receiver_id = data[1]; + let ciphertext = data[2..].to_vec(); + Ok(Self { + receiver_id, + ciphertext, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum Note { + Public(PublicNote), + Private(PrivateNote), +} + +impl Note { + pub fn receiver_id(&self) -> BFieldElement { + match self { + Self::Public(p) => p.receiver_id, + Self::Private(p) => p.receiver_id, + } + } + + pub fn is_public(&self) -> bool { + matches!(self, Self::Public(_)) + } + + pub fn is_private(&self) -> bool { + matches!(self, Self::Private(_)) + } + + pub fn into_announcement(self) -> Announcement { + let msg = match self { + Self::Public(p) => p.into_message(), + Self::Private(p) => p.into_message(), + }; + Announcement::new(msg) + } + + pub fn try_from_announcement(ann: &Announcement) -> Result { + let msg = &ann.message; + if msg.is_empty() { + bail!("Empty announcement"); + } + match msg[0].value() { + TAG_PUBLIC => Ok(Self::Public(PublicNote::from_message(msg)?)), + TAG_PRIVATE => Ok(Self::Private(PrivateNote::from_message(msg)?)), + other => bail!("Unknown tag: {other}"), + } + } + + pub fn into_bech32m(self, network: Network) -> String { + let hrp = Self::get_hrp(network); + let msg = self.into_announcement().message; + let payload = + bincode::serialize(&msg).expect("BFieldElement vec serialization never fails"); + let payload_base32 = payload.to_base32(); + bech32::encode(&hrp, payload_base32, bech32::Variant::Bech32m) + .expect("bech32m encoding never fails") + } + + pub fn from_bech32m(encoded: &str, network: Network) -> Result { + let (hrp, data, variant) = bech32::decode(encoded)?; + ensure!( + variant == bech32::Variant::Bech32m, + "Only bech32m is supported" + ); + ensure!(hrp == Self::get_hrp(network), "Invalid HRP for network"); + let payload = Vec::::from_base32(&data)?; + let msg: Vec = bincode::deserialize(&payload) + .map_err(|e| anyhow::anyhow!("Failed to deserialize bech32 payload: {e}"))?; + let ann = Announcement::new(msg); + Self::try_from_announcement(&ann) + } + + fn get_hrp(network: Network) -> String { + format!("note{}", network_hrp_char(network)) + } +} + +impl From for Note { + fn from(n: PublicNote) -> Self { + Self::Public(n) + } +} + +impl From for Note { + fn from(n: PrivateNote) -> Self { + Self::Private(n) + } +} diff --git a/standards/src/wallet/notes/utxo_notification.rs b/standards/src/wallet/notes/utxo_notification.rs deleted file mode 100644 index e598ea2..0000000 --- a/standards/src/wallet/notes/utxo_notification.rs +++ /dev/null @@ -1,36 +0,0 @@ -use nyks_consensus::transaction::utxo::Utxo; -use nyks_consensus::twenty_first::tip5::Digest; -use serde::Deserialize; -use serde::Serialize; - -use crate::wallet::keys::address::Address; - -/// The payload of a UTXO notification, containing all information necessary -/// to claim it, provided that the decryptor already has access to the -/// associated spending key. -/// -/// future work: -/// we should consider adding functionality that would facilitate passing -/// these payloads from sender to receiver off-chain for lower-fee transfers -/// between trusted parties or eg wallets owned by the same person/org. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct UtxoNotificationPayload { - pub(crate) utxo: Utxo, - pub(crate) sender_randomness: Digest, -} - -impl UtxoNotificationPayload { - pub fn new(utxo: Utxo, sender_randomness: Digest) -> Self { - Self { - utxo, - sender_randomness, - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct PrivateNotificationData { - pub cleartext: UtxoNotificationPayload, - pub ciphertext: String, - pub recipient_address: Address, -} From bb53b013299988c453794df221dec8fd991129b4 Mon Sep 17 00:00:00 2001 From: KaffinPX Date: Wed, 2 Sep 2026 23:28:55 +0300 Subject: [PATCH 02/14] feat: Remove UTXO index --- .cargo/config.toml | 2 - node/src/application/config/data_directory.rs | 11 - node/src/application/rpc/server.rs | 10 - node/src/application/rpc/service.rs | 109 ----- node/src/state/archival_state.rs | 385 ++-------------- .../state/archival_state/rusty_utxo_index.rs | 431 ------------------ node/src/state/mod.rs | 4 - rpc/core/src/api/ops.rs | 15 - rpc/core/src/api/rpc.rs | 78 ---- rpc/core/src/model/message.rs | 52 --- rpc/core/src/model/wallet/mod.rs | 4 - rpc/core/src/model/wallet/transaction.rs | 61 --- 12 files changed, 32 insertions(+), 1130 deletions(-) delete mode 100644 node/src/state/archival_state/rusty_utxo_index.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index a0e9bbd..71e377f 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -12,5 +12,3 @@ RUST_BACKTRACE = "1" # workaround for dependency `leveldb-sys v2.0.9` CMAKE_POLICY_VERSION_MINIMUM = "3.5" -CC = "clang" -CXX = "clang++" diff --git a/node/src/application/config/data_directory.rs b/node/src/application/config/data_directory.rs index 2cbd4fd..77294d3 100644 --- a/node/src/application/config/data_directory.rs +++ b/node/src/application/config/data_directory.rs @@ -11,7 +11,6 @@ use serde::Serialize; use crate::state::archival_state::ARCHIVAL_BLOCK_MMR_DIRECTORY_NAME; use crate::state::archival_state::BLOCK_INDEX_DB_NAME; use crate::state::archival_state::MUTATOR_SET_DIRECTORY_NAME; -use crate::state::archival_state::UTXO_INDEX_DIRECTORY_NAME; use crate::state::database::DATABASE_DIRECTORY_ROOT_NAME; use crate::state::networking_state::BANNED_IPS_DB_NAME; use crate::state::shared::BLOCK_FILENAME_EXTENSION; @@ -128,16 +127,6 @@ impl DataDirectory { .join(Path::new(ARCHIVAL_BLOCK_MMR_DIRECTORY_NAME)) } - /////////////////////////////////////////////////////////////////////////// - /// - /// The UTXO index database directory path - /// - /// This directory lives within `DataDirectory::database_dir_path()`. - pub(crate) fn utxo_index_dir_path(&self) -> PathBuf { - self.database_dir_path() - .join(Path::new(UTXO_INDEX_DIRECTORY_NAME)) - } - /////////////////////////////////////////////////////////////////////////// /// /// The block body directory. diff --git a/node/src/application/rpc/server.rs b/node/src/application/rpc/server.rs index 1849951..ee991d8 100644 --- a/node/src/application/rpc/server.rs +++ b/node/src/application/rpc/server.rs @@ -54,16 +54,6 @@ impl RpcServer { } } - if namespaces.contains(&Namespace::Utxoindex) { - let has_utxo_index = - state.chain.is_archival_node() && state.chain.archival_state().utxo_index.is_some(); - - if !has_utxo_index { - namespaces.remove(&Namespace::Utxoindex); - error!("Node does not maintain a UTXO index, cannot enable utxoindex namespace."); - } - } - if !self.unrestricted && namespaces.contains(&Namespace::Network) { warn!("Networking module is enabled without unsafe mode - this may expose sensitive data.") } diff --git a/node/src/application/rpc/service.rs b/node/src/application/rpc/service.rs index 53c6210..6081389 100644 --- a/node/src/application/rpc/service.rs +++ b/node/src/application/rpc/service.rs @@ -1,11 +1,8 @@ -use std::collections::HashSet; use async_trait::async_trait; use itertools::Itertools; use nyks_consensus::block::Block; use nyks_consensus::block::FUTUREDATING_LIMIT; -use nyks_consensus::mutator_set::addition_record::AdditionRecord; -use nyks_consensus::mutator_set::removal_record::absolute_index_set::AbsoluteIndexSet; use nyks_consensus::proof_abstractions::timestamp::Timestamp; use nyks_consensus::transaction::Transaction; use nyks_rpc_core::api::rpc::*; @@ -650,112 +647,6 @@ impl RpcApi for RpcServer { Ok(SubmitBlockResponse { success }) } - /* Utxoindex */ - - async fn block_heights_by_flags_call( - &self, - request: BlockHeightsByFlagsRequest, - ) -> RpcResult { - let announcement_flags: HashSet<_> = request.announcement_flags.into_iter().collect(); - let heights = self - .state - .lock_guard() - .await - .chain - .archival_state() - .utxo_index - .as_ref() - .expect("Utxo index namespace can only be active when UTXO index is present") - .blocks_by_announcement_flags(&announcement_flags) - .await; - - let block_heights = BlockHeightsByFlagsResponse { - block_heights: heights.into_iter().collect(), - }; - - Ok(block_heights) - } - - async fn block_heights_by_addition_records_call( - &self, - request: BlockHeightsByAdditionRecordsRequest, - ) -> RpcResult { - let addition_records: HashSet = request - .addition_records - .into_iter() - .map(|x| x.into()) - .collect(); - - let block_heights = self - .state - .lock_guard() - .await - .chain - .archival_state() - .addition_records_to_block_height(addition_records) - .await - .expect("Utxo index namespace can only be active when UTXO index is present"); - - let block_heights = BlockHeightsByAdditionRecordsResponse { - block_heights: block_heights.into_iter().collect(), - }; - - Ok(block_heights) - } - - async fn block_heights_by_absolute_index_sets_call( - &self, - request: BlockHeightsByAbsoluteIndexSetsRequest, - ) -> RpcResult { - let absolute_index_sets: HashSet = - request.absolute_index_sets.into_iter().collect(); - - let block_heights = self - .state - .lock_guard() - .await - .chain - .archival_state() - .absolute_index_sets_to_block_heights(absolute_index_sets) - .await - .expect("Utxo index namespace can only be active when UTXO index is present"); - - let block_heights = BlockHeightsByAbsoluteIndexSetsResponse { - block_heights: block_heights.into_iter().collect(), - }; - - Ok(block_heights) - } - - async fn was_mined_call(&self, request: WasMinedRequest) -> RpcResult { - if request.addition_records.is_empty() && request.absolute_index_sets.is_empty() { - return Err(RpcError::EmptyFilteringConditions); - } - - let addition_records = request - .addition_records - .into_iter() - .map(|x| x.into()) - .collect(); - let absolute_index_sets = request.absolute_index_sets.into_iter().collect(); - - let blocks = self - .state - .lock_guard() - .await - .chain - .archival_state() - .canonical_block_heights_with_puts(absolute_index_sets, addition_records) - .await - .expect("UTXO index namespace is only active if UTXO index is maintained"); - - let res = WasMinedResponse { - block_heights: blocks.into_iter().collect(), - }; - - Ok(res) - } - /* Mempool */ async fn transactions_call(&self, _: TransactionsRequest) -> RpcResult { diff --git a/node/src/state/archival_state.rs b/node/src/state/archival_state.rs index e8d5c95..5ef216c 100644 --- a/node/src/state/archival_state.rs +++ b/node/src/state/archival_state.rs @@ -1,40 +1,21 @@ +pub(crate) mod import_blocks_from_files; + use std::collections::HashMap; -use std::collections::HashSet; use std::ops::DerefMut; use std::path::PathBuf; use anyhow::bail; -use anyhow::ensure; use anyhow::Result; use itertools::Itertools; use memmap2::MmapOptions; use num_traits::Zero; -use tasm_lib::prelude::Tip5; use tasm_lib::twenty_first::prelude::Mmr; use tasm_lib::twenty_first::tip5::digest::Digest; use tokio::io::AsyncSeekExt; use tokio::io::AsyncWriteExt; use tokio::io::SeekFrom; use tracing::debug; -use tracing::info; use tracing::warn; - -pub(crate) mod import_blocks_from_files; -pub mod rusty_utxo_index; - -use super::shared::new_block_file_is_needed; -use super::StorageVecBase; -use crate::application::config::cli_args::Args; -use crate::application::config::data_directory::DataDirectory; -use crate::state::archival_state::rusty_utxo_index::RustyUtxoIndex; -use crate::state::database::BlockFileLocation; -use crate::state::database::BlockIndexKey; -use crate::state::database::BlockIndexValue; -use crate::state::database::BlockRecord; -use crate::state::database::FileRecord; -use crate::state::database::LastFileRecord; -use crate::util_types::rusty_archival_block_mmr::RustyArchivalBlockMmr; -use crate::util_types::rusty_archival_mutator_set::RustyArchivalMutatorSet; use nyks_consensus::block::block_header::BlockHeader; use nyks_consensus::block::block_header::BlockHeaderWithBlockHashWitness; use nyks_consensus::block::block_header::HeaderToBlockHashWitness; @@ -53,10 +34,22 @@ use nyks_database::storage::storage_schema::traits::*; use nyks_database::NeptuneLevelDb; use nyks_database::WriteBatchAsync; +use super::shared::new_block_file_is_needed; +use super::StorageVecBase; +use crate::application::config::cli_args::Args; +use crate::application::config::data_directory::DataDirectory; +use crate::state::database::BlockFileLocation; +use crate::state::database::BlockIndexKey; +use crate::state::database::BlockIndexValue; +use crate::state::database::BlockRecord; +use crate::state::database::FileRecord; +use crate::state::database::LastFileRecord; +use crate::util_types::rusty_archival_block_mmr::RustyArchivalBlockMmr; +use crate::util_types::rusty_archival_mutator_set::RustyArchivalMutatorSet; + pub(crate) const BLOCK_INDEX_DB_NAME: &str = "block_index"; pub(crate) const MUTATOR_SET_DIRECTORY_NAME: &str = "mutator_set"; pub(crate) const ARCHIVAL_BLOCK_MMR_DIRECTORY_NAME: &str = "archival_block_mmr"; -pub(crate) const UTXO_INDEX_DIRECTORY_NAME: &str = "utxo_index"; /// Provides interface to historic blockchain data which consists of /// * block-data stored in individual files (append-only) @@ -91,13 +84,6 @@ pub struct ArchivalState { /// Archival-MMR of the block digests belonging to the canonical chain. pub archival_block_mmr: RustyArchivalBlockMmr, - /// Mapping from block digest to a list of (flag, receiver_id) pairs for all - /// announcement in the block, and other indexing data related to historical - /// blocks. This index is only maintained if the node has been started with - /// the CLI flag `--utxo-index`, which implies that this value is Some(T). - /// If the node is not started with this flag, this value is `None`. - pub(crate) utxo_index: Option, - /// The network that this node is on. Used to simplify method interfaces. network: Network, } @@ -312,22 +298,6 @@ impl ArchivalState { .expect("Must be able to initialize block index database"); debug!("Got block index database"); - // UTXO index is always initialized. But only populated with blocks if - // this index is activated. - let utxo_index = if cli.utxo_index { - let mut utxo_index = RustyUtxoIndex::initialize(&data_dir) - .await - .expect("Must be able to initialize utxo index database"); - - if utxo_index.is_empty().await { - utxo_index.index_block(&genesis_block).await; - } - debug!("UTXO index populated"); - Some(utxo_index) - } else { - None - }; - let network = cli.network; let genesis_block = Box::new(genesis_block); @@ -338,7 +308,6 @@ impl ArchivalState { archival_mutator_set, archival_block_mmr, network, - utxo_index, } } @@ -559,7 +528,6 @@ impl ArchivalState { self.write_block_as_tip(block).await?; self.append_to_archival_block_mmr(block).await; self.update_mutator_set(block).await?; - self.update_utxo_index(block).await; Ok(()) } @@ -658,77 +626,6 @@ impl ArchivalState { .await; } - /// Apply a new block to the UTXO index. Does nothing if no UTXO index is - /// maintained by this archival state. - /// - /// This method handles reorganizations, but all predecessors of this block - /// must be known and stored in the block index database for it to work. - /// Reorganizations leaves ophaned blocks in the index though. So this must - /// be accounted for when reading from the index. - /// - /// # Panics - /// - If any of the predecessor blocks have not been applied to the block - /// index database. - async fn update_utxo_index(&mut self, new_block: &Block) { - if self.utxo_index.is_none() { - return; - } - - let current_sync = self.utxo_index.as_ref().unwrap().sync_label().await; - let new_block_hash = new_block.hash(); - - // Index all not-yet-indexed blocks preceding the new block. In the - // common case, where the new block is the direct descendant of the - // block that was previously applied, only one block will be processed - // here. This path-finding logic allows for an efficient common-case - // processing, and an effcient "catchup" behavior where the UTXO index - // is many block behind the rest of the archival state. - let (_, _, missing_blocks) = self.find_path(current_sync, new_block_hash).await; - - // Inform user if this will take a long time. - let num_missing_blocks = missing_blocks.len(); - debug!("Applying {num_missing_blocks} missing blocks to UTXO index."); - if num_missing_blocks > 10 { - info!("Applying {num_missing_blocks} missing blocks to UTXO index. This may take a while.") - } - for missing in missing_blocks { - // This optimization means that we don't have to read the full - // blocks from disk in case it was already processed. - if self - .utxo_index - .as_ref() - .unwrap() - .block_was_indexed(missing) - .await - { - continue; - } - - if missing == new_block_hash { - // Avoid reading the new block from disk if it's already in - // memory. - self.utxo_index - .as_mut() - .unwrap() - .index_block(new_block) - .await; - } else { - let missing = self - .get_block(missing) - .await - .expect("Fetching block must succeed") - .expect("missing block must exist before processed by UTXO index"); - self.utxo_index - .as_mut() - .unwrap() - .index_block(&missing) - .await; - } - } - - debug!("Done updating UTXO index"); - } - async fn get_block_from_block_record(&self, block_record: BlockRecord) -> Result { let block_file_path: PathBuf = self .data_dir @@ -894,51 +791,23 @@ impl ArchivalState { ) } - /// Returns the 1st block hash containing this addition record. Returns - /// `None` if no canonical block with this output is known. - /// - /// searches max `max_search_depth` from tip for a matching transaction - /// output. Unless the node maintain a UTXO index in which case all blocks - /// are searched and this parameter is ignored. - /// - /// If `max_search_depth` is set to `None`, then all blocks are searched - /// until a match is found. A `max_search_depth` of `Some(0)` will only - /// consider the tip. - /// - /// Never loads blocks from disk, so performance should be good. async fn find_canonical_block_hash_with_output( &self, output: AdditionRecord, max_search_depth: Option, ) -> Option { - let block_heights = match &self.utxo_index { - Some(utxo_index) => { - let heights = utxo_index - .blocks_by_addition_record(output) - .await - .into_iter() - .map(|x| x.value()) - .sorted_unstable(); - itertools::Either::Left(heights) - } - None => { - let tip_height = self.tip_header().await.height.value(); + let tip_height = self.tip_header().await.height.value(); - let end = match max_search_depth { - Some(num) => tip_height.saturating_sub(num), - None => 0, - }; - - let heights = (end..=tip_height).rev(); - itertools::Either::Right(heights) - } + let end = match max_search_depth { + Some(num) => tip_height.saturating_sub(num), + None => 0, }; - for block_height in block_heights { + for block_height in (end..=tip_height).rev() { let (addition_records, block_hash) = self .addition_record_indices_for_block_by_height(block_height) .await - .expect("Block height from UTXO index must be known"); + .expect("Block height in search range must be known"); if addition_records.contains_key(&output) { return Some(block_hash); @@ -951,41 +820,25 @@ impl ArchivalState { /// Returns the block containing this input. Returns `None` if no canonical /// block with this input is known. /// - /// searches max `max_search_depth` from tip for a matching transaction - /// input. + /// Searches max `max_search_depth` blocks back from tip for a matching + /// transaction input. /// - /// searches max `max_search_depth` from tip for a matching transaction - /// input. Unless the node maintain a UTXO index in which case all blocks - /// are searched and this parameter is ignored. + /// If `max_search_depth` is set to `None`, then all blocks are searched + /// until a match is found. A `max_search_depth` of `Some(0)` will only + /// consider the tip. pub(crate) async fn find_canonical_block_with_input( &self, input: AbsoluteIndexSet, max_search_depth: Option, ) -> Option { - let block_heights = match &self.utxo_index { - Some(utxo_index) => { - let heights = utxo_index - .block_by_index_set(&input) - .await - .into_iter() - .map(|x| x.value()) - .sorted_unstable(); - itertools::Either::Left(heights) - } - None => { - let tip_height = self.tip_header().await.height.value(); + let tip_height = self.tip_header().await.height.value(); - let end = match max_search_depth { - Some(num) => tip_height.saturating_sub(num), - None => 0, - }; - - let heights = (end..=tip_height).rev(); - itertools::Either::Right(heights) - } + let end = match max_search_depth { + Some(num) => tip_height.saturating_sub(num), + None => 0, }; - for block_height in block_heights { + for block_height in (end..=tip_height).rev() { let block = self .canonical_block_by_height(block_height.into()) .await @@ -1004,180 +857,6 @@ impl ArchivalState { None } - /// Return all block heights of blocks belonging to the canonical chain - /// containing any of the requested addition records. - /// - /// Never loads the entire block from disk. Only reads from the database, so - /// performace should be good. - /// - /// Only works if a UTXO index is maintained. - pub(crate) async fn addition_records_to_block_height( - &self, - addition_records: HashSet, - ) -> Result> { - ensure!( - self.utxo_index.is_some(), - "Only works a UTXO index is maintained." - ); - - let mut ret = HashSet::new(); - for addition_record in addition_records { - let maybe_matching_blocks = self - .utxo_index - .as_ref() - .unwrap() - .blocks_by_addition_record(addition_record) - .await; - - // Verify reported block height matches a block in the canonical - // chain. - // An addition record can (theoretically) be present in mutiple - // blocks, and even multiple times in the same block. This loop - // handles that case. Common case is that returned list has length - // zero or one. - for height in maybe_matching_blocks { - let (actual_ars_in_canonical_block, _) = self - .addition_record_indices_for_block_by_height(height.into()) - .await - .expect("Height reported by UTXO index must be known by archival state"); - if actual_ars_in_canonical_block.contains_key(&addition_record) { - ret.insert(height); - } - } - } - - Ok(ret) - } - - /// Return all block heights of blocks belonging to the canonical chain - /// containing any of the requested absolute index sets. - /// - /// Never loads the entire block from disk. Only reads from the database, so - /// performace should be good. - /// - /// Only works if a UTXO index is maintained. - pub(crate) async fn absolute_index_sets_to_block_heights( - &self, - absolute_index_sets: HashSet, - ) -> Result> { - ensure!( - self.utxo_index.is_some(), - "Only works a UTXO index is maintained." - ); - - let mut ret = HashSet::new(); - for index_set in absolute_index_sets { - let maybe_matching_block = self - .utxo_index - .as_ref() - .unwrap() - .block_by_index_set(&index_set) - .await; - - let Some(maybe_matching_block) = maybe_matching_block else { - continue; - }; - - // No use to add same block height twice. - if ret.contains(&maybe_matching_block) { - continue; - } - - // Verify that the block height in question has not been reorganized - // out of canonicity. - let Some(block_hash) = self - .archival_block_mmr - .ammr() - .try_get_leaf(maybe_matching_block.into()) - .await - else { - // Reorganization to shorter chain. Very unlikely. - continue; - }; - - let block_index_set_digests = self - .utxo_index - .as_ref() - .unwrap() - .index_set_digests(block_hash) - .await - .expect("Canonical block must have been indexed by UTXO index"); - - let index_set_digest = Tip5::hash(&index_set); - if block_index_set_digests.contains(&index_set_digest) { - ret.insert(maybe_matching_block); - } - } - - Ok(ret) - } - - /// Return the block heights for blocks matching *all* elements in the - /// specified input/output lists, for blocks belonging to the canonical - /// chain. Will not return block heights were e.g. only one of the outputs - /// was included if more than one output is included in the outputs list. - /// - /// Can return multiple blocks in the case where blocks are selected only - /// based on addition records and multiple blocks contain the same addition - /// records. - /// - /// Only works if a UTXO index is maintained, otherwise an error is - /// returned. - /// - /// # Panics - /// - If no filtering is applied, i.e. if both input and output lists are - /// empty. - pub(crate) async fn canonical_block_heights_with_puts( - &self, - absolute_index_sets: HashSet, - addition_records: HashSet, - ) -> Result> { - ensure!( - self.utxo_index.is_some(), - "Only works a UTXO index is maintained." - ); - - assert!( - !addition_records.is_empty() || !absolute_index_sets.is_empty(), - "No filtering was applied" - ); - - let mut block_matches: Option> = None; - for index_set in absolute_index_sets { - let block_heights = self - .absolute_index_sets_to_block_heights(HashSet::from([index_set])) - .await - .expect("Utxo index namespace can only be active when UTXO index is present"); - - match block_matches { - Some(bmatches) => { - block_matches = Some(bmatches.intersection(&block_heights).copied().collect()); - } - None => { - block_matches = Some(block_heights); - } - } - } - - for addition_record in addition_records { - let block_heights = self - .addition_records_to_block_height(HashSet::from([addition_record])) - .await - .expect("Utxo index namespace can only be active when UTXO index is present"); - - match block_matches { - Some(bmatches) => { - block_matches = Some(bmatches.intersection(&block_heights).copied().collect()); - } - None => { - block_matches = Some(block_heights); - } - } - } - - Ok(block_matches.expect("At least one filtering criteria was set")) - } - /// Return latest block from database, or genesis block if no other block /// is known. pub async fn get_tip(&self) -> Block { @@ -1342,7 +1021,7 @@ impl ArchivalState { ) } - /// Returns a [`HashMap`] of [`AdditionRecord`] to AOCL leaf indices for + /// Returns a [`HashMap`] of [`AdditionRecord`] to AOCL leaf indices for /// all outputs in a given block, including guesser rewards. Also returns /// the block hash. Returns `None` if no block at the specified height is /// known. AOCL leaf indices have list type since a block can contain the diff --git a/node/src/state/archival_state/rusty_utxo_index.rs b/node/src/state/archival_state/rusty_utxo_index.rs deleted file mode 100644 index 3d4154f..0000000 --- a/node/src/state/archival_state/rusty_utxo_index.rs +++ /dev/null @@ -1,431 +0,0 @@ -use std::collections::HashSet; - -use anyhow::Result; -use itertools::Itertools; -use serde::Deserialize; -use serde::Serialize; -use tasm_lib::prelude::Digest; -use tasm_lib::prelude::Tip5; -use tracing::warn; - -use crate::application::config::data_directory::DataDirectory; -use nyks_consensus::block::block_height::BlockHeight; -use nyks_consensus::block::Block; -use nyks_consensus::mutator_set::addition_record::AdditionRecord; -use nyks_consensus::mutator_set::removal_record::absolute_index_set::AbsoluteIndexSet; -use nyks_database::create_db_if_missing; -use nyks_database::storage::storage_schema::traits::*; -use nyks_database::NeptuneLevelDb; -use nyks_database::WriteBatchAsync; -use nyks_standards::wallet::notes::announcement_flag::AnnouncementFlag; - -/// The maximum number of blocks stored for each [`AnnouncementFlag`]. Wallets -/// with incoming UTXOs in more than this number of blocks cannot rely on the -/// mapping from announcement flags to block heights to restore a wallet. They -/// must instead use other methods. Also used to cap mapping from addition -/// records to block heights. -pub const MAX_NUM_BLOCKS_IN_LOOKUP_LIST: usize = 10_000; - -/// The purpose of the UTXO index is to speed up the rescanning of historical -/// blocks, and to serve 3rd parties with information required to detect -/// incoming and outgoing UTXOs as quickly as possible. It assumes the presence -/// of an [`ArchivalState`]. Any decision about tables in the UTXO index should -/// be made in the light of allowing clients or 3rd parties to discover balance- -/// affecting input or output UTXOs in historical blocks as quickly as possible, -/// and to minimize storage requirements for the UTXO index. -/// -/// The tables of the UTXO index database. Does not include a mapping from -/// block digest to the block's addition records since that mapping can be found -/// from the [`ArchivalMutatorSet`] which is assumed to be part of the state of -/// all nodes that maintain a UTXO index. -/// -/// Block heights are often preferred over block digests due to their smaller -/// serialized size (8 bytes vs. 40). -/// -/// [`ArchivalMutatorSet`]: nyks_consensus::mutator_set::archival_mutator_set::ArchivalMutatorSet -/// [`ArchivalState`]: crate::state::archival_state::ArchivalState -#[derive(Debug)] -pub(crate) struct RustyUtxoIndex { - db: NeptuneLevelDb, -} - -/// The key types used by the UTXO index database. -#[derive(Debug, Copy, Clone, Serialize, Deserialize)] -enum UtxoIndexKey { - /// Latest block handled by this database. Any initialized database must - /// have a sync label set. The default value indicates that no blocks have - /// been processed by the UTXO index. - SyncLabel, - - /// Mapping from block hash to the list of announcement flags contained in - /// the block. - /// - /// Can be used to speed up the scanning for incoming, announced UTXOs. - AnnouncementsByBlock(Digest), - - /// Mapping from block hash to the list of digests of the absolute indices - /// being set in the block. - /// - /// Can be used to speed up the scanning for spent UTXOs, i.e. expenditures. - IndexSetDigestsByBlock(Digest), - - /// Mapping from announcement flag to block height for all blocks in which - /// announcements with this flag are present. Length of list of block - /// heights is capped by [MAX_NUM_BLOCKS_IN_LOOKUP_LIST] in order to foil - /// certain DOS attacks. This means that extremely active wallets/smart - /// contracts that have received announced UTXOs in more than - /// [MAX_NUM_BLOCKS_IN_LOOKUP_LIST] blocks, cannot use this index to fully - /// restore a wallet. But in their case, they might as well scan all blocks - /// anyway. - /// - /// Since the indexed blocks are not guaranteed to be canonical, this - /// mapping may contain entries for blocks that are not part of the - /// canonical chain. - /// - /// Can be used to speed up the scanning for incoming, announced UTXOs, and - /// to serve RPC requests from external wallet programs. - BlocksByAnnouncementFlag(AnnouncementFlag), - - /// Mapping from addition record to block height for all blocks containing - /// the specific addition record. Length of list of block heights is capped - /// by [MAX_NUM_BLOCKS_IN_LOOKUP_LIST] in order to foil certain DOS attacks. - /// This means that if this addition record is present in more than - /// [MAX_NUM_BLOCKS_IN_LOOKUP_LIST] different blocks, the list will be - /// capped. - /// - /// This mapping includes guesser-reward addition records. - /// - /// Since the indexed blocks are not guaranteed to be canonical, this - /// mapping may contain entries for blocks that are not part of the - /// canonical chain. - /// - /// Can be used to serve an RPC endpoint that maps addition records to block - /// heights. - BlocksByAdditionRecord(AdditionRecord), - - /// Mapping from hash of absolute index set to block height. - /// - /// Can be used to serve an RPC endpoint that maps absolute index sets to - /// block heights. - BlockByIndexSetDigest(Digest), -} - -/// The values used by the UTXO index database. -/// -/// See documentstion in [`UtxoIndexKey`] for each variant of this enum. -#[derive(Debug, Clone, Serialize, Deserialize)] -enum UtxoIndexValue { - SyncLabel(Digest), - AnnouncementsByBlock(Vec), - IndexSetDigestsByBlock(Vec), - BlocksByAnnouncementFlag(Vec), - BlocksByAdditionRecord(Vec), - BlockByIndexSetDigest(BlockHeight), -} - -impl UtxoIndexValue { - fn expect_sync_label(self) -> Digest { - match self { - UtxoIndexValue::SyncLabel(digest) => digest, - _ => panic!("Expected SyncLabel found {:?}", self), - } - } - - fn expect_index_set_digests_by_block(self) -> Vec { - match self { - UtxoIndexValue::IndexSetDigestsByBlock(index_set_digests) => index_set_digests, - _ => panic!("Expected IndexSetDigestsByBlock found {:?}", self), - } - } - - fn expect_blocks_by_announcements(self) -> Vec { - match self { - UtxoIndexValue::BlocksByAnnouncementFlag(block_heights) => block_heights, - _ => panic!("Expected BlocksByAnnouncementFlag found {:?}", self), - } - } - - fn expect_blocks_by_addition_records(self) -> Vec { - match self { - UtxoIndexValue::BlocksByAdditionRecord(block_heights) => block_heights, - _ => panic!("Expected BlocksByAdditionRecord found {:?}", self), - } - } - - fn expect_block_by_index_set_digest(self) -> BlockHeight { - match self { - UtxoIndexValue::BlockByIndexSetDigest(height) => height, - _ => panic!("Expected BlockByIndexSetDigest found {:?}", self), - } - } -} - -impl RustyUtxoIndex { - /// Returns true iff no blocks have been indexed. - pub(super) async fn is_empty(&self) -> bool { - self.sync_label().await == Default::default() - } - - /// Returns true if the block was already indexed. - pub(crate) async fn block_was_indexed(&self, block_hash: Digest) -> bool { - self.db - .get(UtxoIndexKey::AnnouncementsByBlock(block_hash)) - .await - .is_some() - } - - /// Initialize a UTXO index. Does not apply the genesis block to the index. - pub(super) async fn initialize(data_dir: &DataDirectory) -> Result { - let utxo_index_db_dir_path = data_dir.utxo_index_dir_path(); - DataDirectory::create_dir_if_not_exists(&utxo_index_db_dir_path).await?; - - let utxo_index = NeptuneLevelDb::::new( - &utxo_index_db_dir_path, - &create_db_if_missing(), - ) - .await?; - - let mut utxo_index = RustyUtxoIndex { db: utxo_index }; - - // After initialization a value for sync label must always be set. - if utxo_index.db.get(UtxoIndexKey::SyncLabel).await.is_none() { - utxo_index - .db - .put( - UtxoIndexKey::SyncLabel, - UtxoIndexValue::SyncLabel(Digest::default()), - ) - .await; - } - - Ok(utxo_index) - } - - /// Return the digests of all absolute index sets of the removal records in - /// this block. Returns `None` if the block is not known to this index. - pub(crate) async fn index_set_digests(&self, block_hash: Digest) -> Option> { - let key = UtxoIndexKey::IndexSetDigestsByBlock(block_hash); - self.db - .get(key) - .await - .map(|x| x.expect_index_set_digests_by_block()) - } - - /// Return the block heights for blocks containing announcements matching - /// the [`AnnouncementFlag`]s. The referenced blocks are not guaranteed to - /// be canonical. - /// - /// # Warning - /// - /// For each announcement flag, the returned list is capped in length (for - /// DOS reasons) by [`MAX_NUM_BLOCKS_IN_LOOKUP_LIST`] so extremely active - /// wallets cannot rely on this method for wallet recovery. They should - /// instead use [`UtxoIndexKey::AnnouncementsByBlock`] to scan through - /// each block. - pub(crate) async fn blocks_by_announcement_flags( - &self, - announcement_flags: &HashSet, - ) -> HashSet { - let mut block_heights = HashSet::new(); - for flag in announcement_flags { - let key = UtxoIndexKey::BlocksByAnnouncementFlag(*flag); - let matching_blocks = self - .db - .get(key) - .await - .map(|x| x.expect_blocks_by_announcements()) - .unwrap_or_default(); - block_heights.extend(matching_blocks); - } - - block_heights - } - - /// Return all block heights for blocks containing the requested - /// addition record. The referenced blocks are not guaranteed to be - /// canonical. - /// # Warning - /// - /// For each addition record, the returned list is capped in length (for - /// DOS reasons) by [`MAX_NUM_BLOCKS_IN_LOOKUP_LIST`]. But since addition - /// records are unlikely to be repeated in large numbers, this truncation - /// is probably never met. - pub(crate) async fn blocks_by_addition_record( - &self, - addition_record: AdditionRecord, - ) -> HashSet { - let key = UtxoIndexKey::BlocksByAdditionRecord(addition_record); - let blocks = self - .db - .get(key) - .await - .map(|x| x.expect_blocks_by_addition_records()) - .unwrap_or_default(); - - blocks.into_iter().collect() - } - - /// Return the block height of the block containing the specified - /// transaction input. Any referenced block is not guaranteed to be - /// canonical. - /// - /// Returns `None` if the UTXO index has never seen this absolute index set. - pub(crate) async fn block_by_index_set( - &self, - index_set: &AbsoluteIndexSet, - ) -> Option { - let index_set_digest = Tip5::hash(index_set); - let key = UtxoIndexKey::BlockByIndexSetDigest(index_set_digest); - self.db - .get(key) - .await - .map(|x| x.expect_block_by_index_set_digest()) - } - - /// Add block to UTXO index. Adds all announcements, addition records, and - /// index set digests to the UTXO index. - /// - /// This method is idempotent, meaning that it does not alter the index if - /// the same block is indexed twice, apart from the [`Self::sync_label`] - /// which always points to the latest blocks that was indexed. - pub(crate) async fn index_block(&mut self, block: &Block) { - let hash = block.hash(); - let height = block.header().height; - - let tx_kernel = &block.body().transaction_kernel; - - // Get flags for all announcements in block, removing duplicates - let announcement_flags: HashSet = tx_kernel - .announcements - .iter() - .filter_map(|ann| AnnouncementFlag::try_from(ann).ok()) - .collect(); - - // sort announcement flags to ensure idempotency - let mut announcement_flags = announcement_flags.iter().copied().collect_vec(); - announcement_flags.sort_unstable(); - let mut batch_writes = WriteBatchAsync::new(); - batch_writes.op_write( - UtxoIndexKey::AnnouncementsByBlock(hash), - UtxoIndexValue::AnnouncementsByBlock(announcement_flags.clone()), - ); - - // Loop over all announcement flags to maintain flag to block mapping - for announcement_flag in announcement_flags { - let announcement_flag = UtxoIndexKey::BlocksByAnnouncementFlag(announcement_flag); - let mut block_heights = self - .db - .get(announcement_flag) - .await - .map(|x| x.expect_blocks_by_announcements()) - .unwrap_or_default(); - - // Ensure same block is not added twice, to ensure function's - // idempotency. - if block_heights.contains(&height) { - continue; - } - - // DOS protection: Do not allow list to grow indefinitely as list is - // stored in RAM during this function call. - if block_heights.len() >= MAX_NUM_BLOCKS_IN_LOOKUP_LIST { - warn!( - "List of block heights matching announcement flag exceeds max.\ - Not adding new block to list." - ); - continue; - } - - block_heights.push(height); - - batch_writes.op_write( - announcement_flag, - UtxoIndexValue::BlocksByAnnouncementFlag(block_heights), - ); - } - - // Loop over all addition records to maintain addition record to block - // mapping. - for addition_record in block - .all_addition_records() - .expect("Block must have mutator set update") - { - let addition_record = UtxoIndexKey::BlocksByAdditionRecord(addition_record); - let mut block_heights = self - .db - .get(addition_record) - .await - .map(|x| x.expect_blocks_by_addition_records()) - .unwrap_or_default(); - - // Ensure same block is not added twice, to ensure function's - // idempotency. - if block_heights.contains(&height) { - continue; - } - - // DOS protection: Do not allow list to grow indefinitely as list is - // stored in RAM during this function call. Very unlikely this is - // ever hit. - if block_heights.len() >= MAX_NUM_BLOCKS_IN_LOOKUP_LIST { - warn!( - "List of block heights matching addition record exceeds max.\ - Not adding new block to list." - ); - continue; - } - - block_heights.push(height); - - batch_writes.op_write( - addition_record, - UtxoIndexValue::BlocksByAdditionRecord(block_heights), - ); - } - - // Loop over all inputs to maintain hash(absolute index set) to block - // mapping. - let index_set_digests = tx_kernel - .inputs - .iter() - .map(|rr| Tip5::hash(&rr.absolute_indices)) - .collect_vec(); - for index_set_digest in &index_set_digests { - // All absolute index sets are assumed to be unique, so no - // duplication removal is needed here. - batch_writes.op_write( - UtxoIndexKey::BlockByIndexSetDigest(*index_set_digest), - UtxoIndexValue::BlockByIndexSetDigest(height), - ); - } - - batch_writes.op_write( - UtxoIndexKey::IndexSetDigestsByBlock(hash), - UtxoIndexValue::IndexSetDigestsByBlock(index_set_digests), - ); - - batch_writes.op_write(UtxoIndexKey::SyncLabel, UtxoIndexValue::SyncLabel(hash)); - - self.db.batch_write(batch_writes).await; - } - - /// Return the hash of the latest block indexed. The default value means - /// that no blocks have been indexed. - pub(crate) async fn sync_label(&self) -> Digest { - self.db - .get(UtxoIndexKey::SyncLabel) - .await - .expect("UTXO index must have a SyncLabel set") - .expect_sync_label() - } -} - -impl StorageWriter for RustyUtxoIndex { - async fn persist(&mut self) { - self.db.flush().await; - } - - async fn drop_unpersisted(&mut self) { - unimplemented!("utxo index does not need it") - } -} diff --git a/node/src/state/mod.rs b/node/src/state/mod.rs index 7a253ac..2a6c0d5 100644 --- a/node/src/state/mod.rs +++ b/node/src/state/mod.rs @@ -649,10 +649,6 @@ impl GlobalState { .persist() .await; - if let Some(utxo_index) = &mut self.chain.archival_state_mut().utxo_index { - utxo_index.persist().await; - } - // flush peer_standings self.net.peer_databases.peer_standings_by_ip.flush().await; diff --git a/rpc/core/src/api/ops.rs b/rpc/core/src/api/ops.rs index e114e68..381419d 100644 --- a/rpc/core/src/api/ops.rs +++ b/rpc/core/src/api/ops.rs @@ -43,9 +43,6 @@ pub enum Namespace { /// Endpoints for serving external wallets Wallet, - - /// Endpoints relating to and requiring a UTXO index - Utxoindex, } #[derive( @@ -141,18 +138,6 @@ pub enum RpcMethods { #[namespace(Namespace::Mining)] SubmitBlock, - #[namespace(Namespace::Utxoindex)] - BlockHeightsByFlags, - - #[namespace(Namespace::Utxoindex)] - BlockHeightsByAdditionRecords, - - #[namespace(Namespace::Utxoindex)] - BlockHeightsByAbsoluteIndexSets, - - #[namespace(Namespace::Utxoindex)] - WasMined, - #[namespace(Namespace::Mempool)] Transactions, diff --git a/rpc/core/src/api/rpc.rs b/rpc/core/src/api/rpc.rs index 23f0668..ac2059d 100644 --- a/rpc/core/src/api/rpc.rs +++ b/rpc/core/src/api/rpc.rs @@ -1,5 +1,4 @@ use async_trait::async_trait; -use nyks_standards::wallet::notes::announcement_flag::AnnouncementFlag; use serde::Deserialize; use serde::Serialize; use tasm_lib::prelude::Digest; @@ -365,83 +364,6 @@ pub trait RpcApi: Sync + Send { request: SubmitBlockRequest, ) -> RpcResult; - /* Utxoindex */ - - /// Return block heights for blocks containing announcements with specified - /// announcement flags. May return results from orphaned blocks. - async fn block_heights_by_flags( - &self, - announcement_flags: Vec, - ) -> RpcResult { - self.block_heights_by_flags_call(BlockHeightsByFlagsRequest { announcement_flags }) - .await - } - - async fn block_heights_by_flags_call( - &self, - request: BlockHeightsByFlagsRequest, - ) -> RpcResult; - - /// Return block heights for blocks containing specified addition records. - /// Returned block heights are guaranteed to reference blocks belonging to - /// the canonical chain. - async fn block_heights_by_addition_records( - &self, - addition_records: Vec, - ) -> RpcResult { - self.block_heights_by_addition_records_call(BlockHeightsByAdditionRecordsRequest { - addition_records, - }) - .await - } - - async fn block_heights_by_addition_records_call( - &self, - request: BlockHeightsByAdditionRecordsRequest, - ) -> RpcResult; - - async fn block_heights_by_absolute_index_sets( - &self, - absolute_index_sets: Vec, - ) -> RpcResult { - self.block_heights_by_absolute_index_sets_call(BlockHeightsByAbsoluteIndexSetsRequest { - absolute_index_sets, - }) - .await - } - - /// Return block heights for blocks containing specified absolute index - /// sets. Returned block heights are guaranteed to reference blocks - /// belonging to the canonical chain. - async fn block_heights_by_absolute_index_sets_call( - &self, - request: BlockHeightsByAbsoluteIndexSetsRequest, - ) -> RpcResult; - - /// Return the block heights for blocks matching *all* elements in the - /// specified input/output lists, for blocks belonging to the canonical - /// chain. Will not return block heights were e.g. only one of the outputs - /// was included if more than one output is included in the outputs list. - /// - /// Can return multiple blocks in the case where blocks are selected only - /// based on addition records and multiple blocks contain the same addition - /// records. - /// - /// Returns an error if no filtering conditions are set. - async fn was_mined( - &self, - inputs: Vec, - outputs: Vec, - ) -> RpcResult { - self.was_mined_call(WasMinedRequest { - absolute_index_sets: inputs, - addition_records: outputs, - }) - .await - } - - async fn was_mined_call(&self, request: WasMinedRequest) -> RpcResult; - /* Mempool */ async fn transactions(&self) -> RpcResult { diff --git a/rpc/core/src/model/message.rs b/rpc/core/src/model/message.rs index bb7c7c5..ddf322e 100644 --- a/rpc/core/src/model/message.rs +++ b/rpc/core/src/model/message.rs @@ -12,7 +12,6 @@ use crate::model::block::*; use crate::model::common::*; use crate::model::mining::mempool::RpcMempoolMetadata; use crate::model::mining::template::RpcBlockTemplate; -use crate::model::wallet::RpcAnnouncementFlag; use crate::model::wallet::block::*; use crate::model::wallet::mutator_set::*; use crate::model::wallet::transaction::RpcTransaction; @@ -355,57 +354,6 @@ pub struct SubmitBlockResponse { pub success: bool, } -/* Utxo Index */ - -#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple)] -#[serde(rename_all = "camelCase")] -pub struct BlockHeightsByFlagsRequest { - pub announcement_flags: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct BlockHeightsByFlagsResponse { - pub block_heights: Vec, -} - -#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple)] -#[serde(rename_all = "camelCase")] -pub struct BlockHeightsByAdditionRecordsRequest { - pub addition_records: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct BlockHeightsByAdditionRecordsResponse { - pub block_heights: Vec, -} - -#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple)] -#[serde(rename_all = "camelCase")] -pub struct BlockHeightsByAbsoluteIndexSetsRequest { - pub absolute_index_sets: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct BlockHeightsByAbsoluteIndexSetsResponse { - pub block_heights: Vec, -} - -#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple)] -#[serde(rename_all = "camelCase")] -pub struct WasMinedRequest { - pub absolute_index_sets: Vec, - pub addition_records: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct WasMinedResponse { - pub block_heights: Vec, -} - /* Mempool */ #[derive(Clone, Copy, Debug, Serialize_tuple, Deserialize_tuple)] diff --git a/rpc/core/src/model/wallet/mod.rs b/rpc/core/src/model/wallet/mod.rs index 076025d..07d6569 100644 --- a/rpc/core/src/model/wallet/mod.rs +++ b/rpc/core/src/model/wallet/mod.rs @@ -1,7 +1,3 @@ -use nyks_standards::wallet::notes::announcement_flag::AnnouncementFlag; - pub mod block; pub mod mutator_set; pub mod transaction; - -pub type RpcAnnouncementFlag = AnnouncementFlag; diff --git a/rpc/core/src/model/wallet/transaction.rs b/rpc/core/src/model/wallet/transaction.rs index c334a11..142396d 100644 --- a/rpc/core/src/model/wallet/transaction.rs +++ b/rpc/core/src/model/wallet/transaction.rs @@ -6,8 +6,6 @@ use crate::model::block::transaction_kernel::RpcTransactionKernel; use crate::model::common::RpcBFieldElements; use nyks_consensus::transaction::Transaction; use nyks_consensus::transaction::TransactionProof; -use nyks_consensus::transaction::utxo::Coin; -use nyks_consensus::transaction::utxo::Utxo; use nyks_consensus::transaction::validity::nyks_proof::NyksProof; use nyks_consensus::transaction::validity::proof_collection::ProofCollection; @@ -134,62 +132,3 @@ impl From for RpcTransaction { } } } - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct RpcCoin { - pub type_script_hash: Digest, - pub state: RpcBFieldElements, -} - -impl From for RpcCoin { - fn from(value: Coin) -> Self { - Self { - type_script_hash: value.type_script_hash, - state: value.state.into(), - } - } -} - -impl From<&Coin> for RpcCoin { - fn from(value: &Coin) -> Self { - Self { - type_script_hash: value.type_script_hash, - state: value.state.to_owned().into(), - } - } -} - -impl From for Coin { - fn from(value: RpcCoin) -> Self { - Self { - type_script_hash: value.type_script_hash, - state: value.state.into(), - } - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct RpcUtxo { - lock_script_hash: Digest, - coins: Vec, -} - -impl From for RpcUtxo { - fn from(value: Utxo) -> Self { - Self { - lock_script_hash: value.lock_script_hash(), - coins: value.coins().iter().map(|x| x.into()).collect(), - } - } -} - -impl From for Utxo { - fn from(value: RpcUtxo) -> Self { - Self::new( - value.lock_script_hash, - value.coins.into_iter().map(|x| x.into()).collect(), - ) - } -} From e814e014f52889cec936d81068da7a8c0970520d Mon Sep 17 00:00:00 2001 From: KaffinPX Date: Thu, 3 Sep 2026 00:57:48 +0300 Subject: [PATCH 03/14] style(node): Remove weird comment "boxes" --- node/src/application/config/data_directory.rs | 13 ------------- node/src/application/rpc/service.rs | 1 - node/src/state/archival_state.rs | 14 +++++++------- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/node/src/application/config/data_directory.rs b/node/src/application/config/data_directory.rs index 77294d3..f96a6d4 100644 --- a/node/src/application/config/data_directory.rs +++ b/node/src/application/config/data_directory.rs @@ -73,15 +73,11 @@ impl DataDirectory { .context("open_ensure_parent_dir_exists") } - /////////////////////////////////////////////////////////////////////////// - /// /// The root data directory path pub fn root_dir_path(&self) -> PathBuf { self.data_dir.clone() } - /////////////////////////////////////////////////////////////////////////// - /// /// The block database directory path pub fn database_dir_path(&self) -> PathBuf { self.data_dir.join(Path::new(DATABASE_DIRECTORY_ROOT_NAME)) @@ -91,8 +87,6 @@ impl DataDirectory { self.data_dir.join(Path::new(NETWORK_SUBDIRECTORY_NAME)) } - /////////////////////////////////////////////////////////////////////////// - /// /// The banned IPs database directory path. /// /// This directory lives within `DataDirectory::database_dir_path()`. @@ -100,15 +94,12 @@ impl DataDirectory { self.database_dir_path().join(Path::new(BANNED_IPS_DB_NAME)) } - /////////////////////////////////////////////////////////////////////////// /// directory for storing database backups before migrating schema to newer version pub fn db_migration_backups_dir_path(&self) -> PathBuf { self.database_dir_path() .join(Path::new(DB_MIGRATION_BACKUPS_DIR)) } - /////////////////////////////////////////////////////////////////////////// - /// /// The mutator set database directory path. /// /// This directory lives within `DataDirectory::database_dir_path()`. @@ -117,8 +108,6 @@ impl DataDirectory { .join(Path::new(MUTATOR_SET_DIRECTORY_NAME)) } - /////////////////////////////////////////////////////////////////////////// - /// /// The archival block MMR database directory path /// /// This directory lives within `DataDirectory::database_dir_path()`. @@ -127,8 +116,6 @@ impl DataDirectory { .join(Path::new(ARCHIVAL_BLOCK_MMR_DIRECTORY_NAME)) } - /////////////////////////////////////////////////////////////////////////// - /// /// The block body directory. /// /// This directory lives within `DataDirectory::root_dir_path()`. diff --git a/node/src/application/rpc/service.rs b/node/src/application/rpc/service.rs index 6081389..0673429 100644 --- a/node/src/application/rpc/service.rs +++ b/node/src/application/rpc/service.rs @@ -1,4 +1,3 @@ - use async_trait::async_trait; use itertools::Itertools; use nyks_consensus::block::Block; diff --git a/node/src/state/archival_state.rs b/node/src/state/archival_state.rs index 5ef216c..6f9d860 100644 --- a/node/src/state/archival_state.rs +++ b/node/src/state/archival_state.rs @@ -9,13 +9,6 @@ use anyhow::Result; use itertools::Itertools; use memmap2::MmapOptions; use num_traits::Zero; -use tasm_lib::twenty_first::prelude::Mmr; -use tasm_lib::twenty_first::tip5::digest::Digest; -use tokio::io::AsyncSeekExt; -use tokio::io::AsyncWriteExt; -use tokio::io::SeekFrom; -use tracing::debug; -use tracing::warn; use nyks_consensus::block::block_header::BlockHeader; use nyks_consensus::block::block_header::BlockHeaderWithBlockHashWitness; use nyks_consensus::block::block_header::HeaderToBlockHashWitness; @@ -33,6 +26,13 @@ use nyks_database::create_db_if_missing; use nyks_database::storage::storage_schema::traits::*; use nyks_database::NeptuneLevelDb; use nyks_database::WriteBatchAsync; +use tasm_lib::twenty_first::prelude::Mmr; +use tasm_lib::twenty_first::tip5::digest::Digest; +use tokio::io::AsyncSeekExt; +use tokio::io::AsyncWriteExt; +use tokio::io::SeekFrom; +use tracing::debug; +use tracing::warn; use super::shared::new_block_file_is_needed; use super::StorageVecBase; From 9449ce79fefea224c09bbbf1799ba7a22f49155f Mon Sep 17 00:00:00 2001 From: KaffinPX Date: Thu, 3 Sep 2026 01:03:43 +0300 Subject: [PATCH 04/14] feat(wallet-core): Adapt tx builder SDK outputs to new format --- standards/src/wallet/notes/content.rs | 15 +++++ wallet/core/src/transaction/builder/output.rs | 60 +++++-------------- 2 files changed, 30 insertions(+), 45 deletions(-) diff --git a/standards/src/wallet/notes/content.rs b/standards/src/wallet/notes/content.rs index c573d5e..d82da2b 100644 --- a/standards/src/wallet/notes/content.rs +++ b/standards/src/wallet/notes/content.rs @@ -20,6 +20,12 @@ pub trait Content: #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, BFieldCodec)] pub struct MessageContent(pub Vec); +impl MessageContent { + pub fn new(content: Vec) -> Self { + Self(content) + } +} + impl Content for MessageContent { const DISCRIMINANT: u64 = 0; } @@ -31,6 +37,15 @@ pub struct UtxoContent { pub sender_randomness: Digest, } +impl UtxoContent { + pub fn new(utxo: Utxo, sender_randomness: Digest) -> Self { + Self { + utxo, + sender_randomness, + } + } +} + impl Content for UtxoContent { const DISCRIMINANT: u64 = 1; } diff --git a/wallet/core/src/transaction/builder/output.rs b/wallet/core/src/transaction/builder/output.rs index ae72ab1..d910cbd 100644 --- a/wallet/core/src/transaction/builder/output.rs +++ b/wallet/core/src/transaction/builder/output.rs @@ -3,21 +3,18 @@ use std::ops::Deref; use std::ops::DerefMut; -use nyks_consensus::tasm_lib::prelude::Digest; -use nyks_standards::wallet::keys::address::Address; -use nyks_standards::wallet::keys::address::Recipient; -use nyks_standards::wallet::notes::utxo_notification::PrivateNotificationData; -use nyks_standards::wallet::notes::utxo_notification::UtxoNotificationPayload; -use serde::Deserialize; -use serde::Serialize; - use nyks_consensus::mutator_set::addition_record::AdditionRecord; -use nyks_consensus::network::Network; use nyks_consensus::proof_abstractions::timestamp::Timestamp; +use nyks_consensus::tasm_lib::prelude::Digest; use nyks_consensus::transaction::announcement::Announcement; use nyks_consensus::transaction::utxo::Utxo; use nyks_consensus::transaction::utxo_triple::UtxoTriple; use nyks_consensus::type_scripts::native_currency_amount::NativeCurrencyAmount; +use nyks_standards::wallet::keys::address::Address; +use nyks_standards::wallet::keys::address::Recipient; +use nyks_standards::wallet::notes::content::UtxoContent; +use serde::Deserialize; +use serde::Serialize; use crate::transaction::utxo::notifications::UtxoNotificationMedium; use crate::transaction::utxo::notifications::UtxoNotificationMethod; @@ -56,8 +53,8 @@ impl TxOutput { } } - fn notification_payload(&self) -> UtxoNotificationPayload { - UtxoNotificationPayload::new(self.utxo(), self.sender_randomness()) + fn note_content(&self) -> UtxoContent { + UtxoContent::new(self.utxo(), self.sender_randomness()) } /// retrieve native currency amount @@ -197,29 +194,19 @@ impl TxOutput { } /// Retrieve on-chain UTXO notification announcement, if any. + /// TODO: redesign this potentially pub fn announcement(&self) -> Option { match &self.notification_method { UtxoNotificationMethod::None => None, UtxoNotificationMethod::OffChain(_) => None, UtxoNotificationMethod::OnChain(receiving_address) => { - let notification_payload = self.notification_payload(); - Some(receiving_address.create_note_announcement(¬ification_payload)) - } - } - } - - pub(crate) fn offchain_notification(&self, network: Network) -> Option<(String, Address)> { - match &self.notification_method { - UtxoNotificationMethod::OnChain(_) => None, - UtxoNotificationMethod::OffChain(recipient) => { - let notification_payload = self.notification_payload(); - - Some(( - recipient.create_note(¬ification_payload, network), - recipient.to_owned(), - )) + let utxo_content = self.note_content(); + Some( + receiving_address + .create_private_note(&utxo_content.into()) + .into_announcement(), + ) } - UtxoNotificationMethod::None => None, } } @@ -343,23 +330,6 @@ impl TxOutputList { announcements } - pub fn offchain_notifications( - &self, - network: Network, - ) -> impl Iterator + use<'_> { - self.0.iter().filter_map(move |tx_output| { - if let Some((ciphertext, receiver_address)) = tx_output.offchain_notification(network) { - Some(PrivateNotificationData { - cleartext: tx_output.notification_payload(), - ciphertext, - recipient_address: receiver_address, - }) - } else { - None - } - }) - } - /// indicates if any offchain notifications exist pub fn has_offchain(&self) -> bool { self.0.iter().any(|u| u.is_offchain()) From 163f7eafff77fd84416a6a5075a1a2981117f607 Mon Sep 17 00:00:00 2001 From: KaffinPX Date: Thu, 3 Sep 2026 01:33:40 +0300 Subject: [PATCH 05/14] chore: Fix Makefile --- Makefile | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 42a37af..329e72e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: clean help stats bench all install run test build doc check format bench-no-run pretty-log ensure-clang -prog :=neptune-core +prog :=nyks-node # Passive dependency-checker. Informs and exits. check-clang: @@ -41,7 +41,7 @@ build: ensure-clang doc: cargo doc --no-deps - xdg-open "target/doc/neptune-core/index.html" + xdg-open "target/doc/nyks_node/index.html" check: check-clang cargo check @@ -59,12 +59,11 @@ happy: clippy format cargo test --doc install: ensure-clang - cargo install --force --locked --path neptune-core/ - cargo install --force --locked --path neptune-core-cli/ - cargo install --force --locked --path neptune-dashboard/ - -install-linux: install - @echo "\n\nPlease run:\n./scripts/linux/install-bash-completions.sh\nto install bash-completions for Neptune-core's CLI." + cargo install --force --locked --path nyks-node/ + cargo install --force --locked --path nyks-prover/ + cargo install --force --locked --path nyks-wallet/ + cargo install --force --locked --path nyks-composer/ + cargo install --force --locked --path nyks-upgrader/ clippy: cargo clippy --all-targets -- -D warnings From 484a4be40cb895b6ff2f516c5fada2daf38be904 Mon Sep 17 00:00:00 2001 From: KaffinPX Date: Thu, 3 Sep 2026 02:15:52 +0300 Subject: [PATCH 06/14] feat(node): Remove unused utxoindex parameter --- node/src/application/config/cli_args.rs | 13 ------------- node/src/state/archival_state.rs | 3 +-- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/node/src/application/config/cli_args.rs b/node/src/application/config/cli_args.rs index 18591d7..858983b 100644 --- a/node/src/application/config/cli_args.rs +++ b/node/src/application/config/cli_args.rs @@ -271,19 +271,6 @@ pub struct Args { #[clap(long, default_value = "1800", value_parser = duration_from_seconds_str)] pub reconnect_cooldown: Duration, - /// Construct and maintain a UTXO index - /// - /// If set, all announcements and inputs in all processed blocks will be - /// indexed in a database that enables a fast rescan for the discovery of - /// all balance-affecting inputs and outputs of blocks. - /// - /// If blocks have already been processed without this flag active, and the - /// flag is later activated, all blocks up to the current tip will be - /// indexed, when a new block is set as tip. This process might take some - /// time (tens of minutes). - #[clap(long)] - pub utxo_index: bool, - /// Enable JSON/HTTP RPC. /// You can optionally specify an address and port (default: 127.0.0.1:9797). /// If not given, RPC is disabled. diff --git a/node/src/state/archival_state.rs b/node/src/state/archival_state.rs index 6f9d860..b2c6d97 100644 --- a/node/src/state/archival_state.rs +++ b/node/src/state/archival_state.rs @@ -769,8 +769,7 @@ impl ArchivalState { /// `None` if no canonical block with this output is known. /// /// searches max `max_search_depth` from tip for a matching transaction - /// output. Unless the node maintain a UTXO index in which case all blocks - /// are searched and this parameter is ignored. + /// output. /// /// If `max_search_depth` is set to `None`, then all blocks are searched /// until a match is found. A `max_search_depth` of `Some(0)` will only From a3c41fe6c2878dd801e407d87367ad523a518360 Mon Sep 17 00:00:00 2001 From: KaffinPX Date: Thu, 3 Sep 2026 19:05:24 +0300 Subject: [PATCH 07/14] feat(standards): Remove MessageContent for now --- standards/src/wallet/keys/mod.rs | 1 - standards/src/wallet/notes/content.rs | 28 +-------------------------- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/standards/src/wallet/keys/mod.rs b/standards/src/wallet/keys/mod.rs index a2d01e1..8333402 100644 --- a/standards/src/wallet/keys/mod.rs +++ b/standards/src/wallet/keys/mod.rs @@ -43,7 +43,6 @@ pub(crate) fn deterministically_derive_seed_and_nonce( (seed, e4) } - NoteContent::Message(_) => todo!(), } } diff --git a/standards/src/wallet/notes/content.rs b/standards/src/wallet/notes/content.rs index d82da2b..8fecd2d 100644 --- a/standards/src/wallet/notes/content.rs +++ b/standards/src/wallet/notes/content.rs @@ -16,20 +16,6 @@ pub trait Content: const DISCRIMINANT: u64; } -/// Plain message content: an arbitrary list of BFieldElements. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, BFieldCodec)] -pub struct MessageContent(pub Vec); - -impl MessageContent { - pub fn new(content: Vec) -> Self { - Self(content) - } -} - -impl Content for MessageContent { - const DISCRIMINANT: u64 = 0; -} - /// UTXO notification payload. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, BFieldCodec)] pub struct UtxoContent { @@ -47,12 +33,11 @@ impl UtxoContent { } impl Content for UtxoContent { - const DISCRIMINANT: u64 = 1; + const DISCRIMINANT: u64 = 0; } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum NoteContent { - Message(MessageContent), Utxo(UtxoContent), } @@ -60,7 +45,6 @@ impl NoteContent { /// Returns the discriminant of the contained content. pub fn discriminant(&self) -> u64 { match self { - Self::Message(_) => MessageContent::DISCRIMINANT, Self::Utxo(_) => UtxoContent::DISCRIMINANT, } } @@ -68,7 +52,6 @@ impl NoteContent { /// Encodes the content into a vector of BFieldElements. pub fn encode(&self) -> Vec { match self { - Self::Message(m) => m.encode(), Self::Utxo(u) => u.encode(), } } @@ -76,21 +59,12 @@ impl NoteContent { /// Decodes content from a discriminant and a data slice. pub fn decode(disc: u64, data: &[BFieldElement]) -> Result { match disc { - d if d == MessageContent::DISCRIMINANT => { - Ok(Self::Message(*MessageContent::decode(data)?)) - } d if d == UtxoContent::DISCRIMINANT => Ok(Self::Utxo(*UtxoContent::decode(data)?)), _ => bail!("Unknown content discriminant: {disc}"), } } } -impl From for NoteContent { - fn from(m: MessageContent) -> Self { - Self::Message(m) - } -} - impl From for NoteContent { fn from(u: UtxoContent) -> Self { Self::Utxo(u) From 12b21da9d6581523ba5c8d722ed129b217448c1b Mon Sep 17 00:00:00 2001 From: KaffinPX Date: Thu, 3 Sep 2026 23:43:58 +0300 Subject: [PATCH 08/14] feat(standards): Consume self on into_message for convention --- standards/src/wallet/notes/note.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/standards/src/wallet/notes/note.rs b/standards/src/wallet/notes/note.rs index b73bdd0..e4650f6 100644 --- a/standards/src/wallet/notes/note.rs +++ b/standards/src/wallet/notes/note.rs @@ -29,7 +29,7 @@ impl PublicNote { } } - pub fn into_message(&self) -> Vec { + pub fn into_message(self) -> Vec { let mut msg = vec![ BFieldElement::new(TAG_PUBLIC), self.receiver_id, @@ -70,9 +70,9 @@ impl PrivateNote { } } - pub fn into_message(&self) -> Vec { + pub fn into_message(self) -> Vec { let mut msg = vec![BFieldElement::new(TAG_PRIVATE), self.receiver_id]; - msg.extend(self.ciphertext.clone()); + msg.extend(self.ciphertext); msg } From 5d0338d31573e0bf591fb6232fc4f5dcc75a8af5 Mon Sep 17 00:00:00 2001 From: KaffinPX Date: Fri, 4 Sep 2026 03:20:56 +0300 Subject: [PATCH 09/14] style(standards): Note::`get_hrp` is renamed to `hrp` --- standards/src/wallet/notes/note.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/standards/src/wallet/notes/note.rs b/standards/src/wallet/notes/note.rs index e4650f6..9bdf041 100644 --- a/standards/src/wallet/notes/note.rs +++ b/standards/src/wallet/notes/note.rs @@ -135,7 +135,7 @@ impl Note { } pub fn into_bech32m(self, network: Network) -> String { - let hrp = Self::get_hrp(network); + let hrp = Self::hrp(network); let msg = self.into_announcement().message; let payload = bincode::serialize(&msg).expect("BFieldElement vec serialization never fails"); @@ -150,7 +150,7 @@ impl Note { variant == bech32::Variant::Bech32m, "Only bech32m is supported" ); - ensure!(hrp == Self::get_hrp(network), "Invalid HRP for network"); + ensure!(hrp == Self::hrp(network), "Invalid HRP for network"); let payload = Vec::::from_base32(&data)?; let msg: Vec = bincode::deserialize(&payload) .map_err(|e| anyhow::anyhow!("Failed to deserialize bech32 payload: {e}"))?; @@ -158,7 +158,7 @@ impl Note { Self::try_from_announcement(&ann) } - fn get_hrp(network: Network) -> String { + fn hrp(network: Network) -> String { format!("note{}", network_hrp_char(network)) } } From 0b855fde662331a2bbbea7f405d62a08279b328b Mon Sep 17 00:00:00 2001 From: KaffinPX Date: Fri, 4 Sep 2026 16:53:03 +0300 Subject: [PATCH 10/14] feat(standards): Return proper result with ``NoteContent::decode`` --- standards/src/wallet/notes/content.rs | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/standards/src/wallet/notes/content.rs b/standards/src/wallet/notes/content.rs index 8fecd2d..40f8e7a 100644 --- a/standards/src/wallet/notes/content.rs +++ b/standards/src/wallet/notes/content.rs @@ -1,13 +1,12 @@ use std::fmt::Debug; -use anyhow::Result; -use anyhow::bail; use nyks_consensus::BFieldElement; use nyks_consensus::transaction::utxo::Utxo; use nyks_consensus::twenty_first::math::bfield_codec::BFieldCodec; use nyks_consensus::twenty_first::tip5::Digest; use serde::Deserialize; use serde::Serialize; +use thiserror::Error; pub trait Content: Clone + Debug + PartialEq + Eq + Send + Sync + BFieldCodec + for<'de> Deserialize<'de> + Serialize @@ -16,6 +15,18 @@ pub trait Content: const DISCRIMINANT: u64; } +/// Errors that can occur when working with [`NoteContent`]. +#[derive(Debug, Error)] +pub enum NoteContentError { + #[error("unknown content discriminant: {0}")] + UnknownDiscriminant(u64), + + // Boxed because each Content impl has its own BFieldCodec::Error type. + // One Decode variant can then handle all of them. + #[error("failed to decode content: {0}")] + Decode(Box), +} + /// UTXO notification payload. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, BFieldCodec)] pub struct UtxoContent { @@ -57,10 +68,14 @@ impl NoteContent { } /// Decodes content from a discriminant and a data slice. - pub fn decode(disc: u64, data: &[BFieldElement]) -> Result { + pub fn decode(disc: u64, data: &[BFieldElement]) -> Result { match disc { - d if d == UtxoContent::DISCRIMINANT => Ok(Self::Utxo(*UtxoContent::decode(data)?)), - _ => bail!("Unknown content discriminant: {disc}"), + d if d == UtxoContent::DISCRIMINANT => { + let content = + UtxoContent::decode(data).map_err(|e| NoteContentError::Decode(e.into()))?; + Ok(Self::Utxo(*content)) + } + _ => Err(NoteContentError::UnknownDiscriminant(disc)), } } } From df548bd1fe2dfaa5c4cc86da557f7e10d6ccacde Mon Sep 17 00:00:00 2001 From: KaffinPX Date: Fri, 4 Sep 2026 18:05:01 +0300 Subject: [PATCH 11/14] feat(standards): Manual implementation of BFieldCodec for NoteContent --- standards/src/wallet/notes/content.rs | 67 ++++++++++++++++----------- standards/src/wallet/notes/note.rs | 6 +-- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/standards/src/wallet/notes/content.rs b/standards/src/wallet/notes/content.rs index 40f8e7a..22430a9 100644 --- a/standards/src/wallet/notes/content.rs +++ b/standards/src/wallet/notes/content.rs @@ -8,20 +8,16 @@ use serde::Deserialize; use serde::Serialize; use thiserror::Error; -pub trait Content: - Clone + Debug + PartialEq + Eq + Send + Sync + BFieldCodec + for<'de> Deserialize<'de> + Serialize -{ - /// Unique discriminant used in the note header. - const DISCRIMINANT: u64; -} - /// Errors that can occur when working with [`NoteContent`]. #[derive(Debug, Error)] pub enum NoteContentError { + #[error("empty sequence: expected at least a discriminant element")] + EmptySequence, + #[error("unknown content discriminant: {0}")] UnknownDiscriminant(u64), - // Boxed because each Content impl has its own BFieldCodec::Error type. + // Boxed because each content type has its own BFieldCodec::Error type. // One Decode variant can then handle all of them. #[error("failed to decode content: {0}")] Decode(Box), @@ -35,6 +31,8 @@ pub struct UtxoContent { } impl UtxoContent { + pub const DISCRIMINANT: u64 = 0; + pub fn new(utxo: Utxo, sender_randomness: Digest) -> Self { Self { utxo, @@ -43,10 +41,6 @@ impl UtxoContent { } } -impl Content for UtxoContent { - const DISCRIMINANT: u64 = 0; -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum NoteContent { Utxo(UtxoContent), @@ -59,29 +53,46 @@ impl NoteContent { Self::Utxo(_) => UtxoContent::DISCRIMINANT, } } +} - /// Encodes the content into a vector of BFieldElements. - pub fn encode(&self) -> Vec { - match self { - Self::Utxo(u) => u.encode(), - } +impl From for NoteContent { + fn from(u: UtxoContent) -> Self { + Self::Utxo(u) + } +} + +impl BFieldCodec for NoteContent { + type Error = NoteContentError; + + fn encode(&self) -> Vec { + let (discriminant, mut payload) = match self { + Self::Utxo(u) => (UtxoContent::DISCRIMINANT, u.encode()), + }; + + let mut out = Vec::with_capacity(1 + payload.len()); + out.push(BFieldElement::new(discriminant)); + out.append(&mut payload); + out } - /// Decodes content from a discriminant and a data slice. - pub fn decode(disc: u64, data: &[BFieldElement]) -> Result { - match disc { + fn decode(sequence: &[BFieldElement]) -> Result, Self::Error> { + let (disc_elem, rest) = sequence + .split_first() + .ok_or(NoteContentError::EmptySequence)?; + let discriminant = disc_elem.value(); + + match discriminant { d if d == UtxoContent::DISCRIMINANT => { - let content = - UtxoContent::decode(data).map_err(|e| NoteContentError::Decode(e.into()))?; - Ok(Self::Utxo(*content)) + let content = UtxoContent::decode(rest) + .map_err(|e| NoteContentError::Decode(e.into()))?; + Ok(Box::new(Self::Utxo(*content))) } - _ => Err(NoteContentError::UnknownDiscriminant(disc)), + _ => Err(NoteContentError::UnknownDiscriminant(discriminant)), } } -} -impl From for NoteContent { - fn from(u: UtxoContent) -> Self { - Self::Utxo(u) + fn static_length() -> Option { + // Variable-length: total size depends on which variant is encoded. + None } } diff --git a/standards/src/wallet/notes/note.rs b/standards/src/wallet/notes/note.rs index 9bdf041..5ef6639 100644 --- a/standards/src/wallet/notes/note.rs +++ b/standards/src/wallet/notes/note.rs @@ -6,6 +6,7 @@ use bech32::ToBase32; use nyks_consensus::BFieldElement; use nyks_consensus::network::Network; use nyks_consensus::transaction::announcement::Announcement; +use nyks_consensus::twenty_first::math::bfield_codec::BFieldCodec; use serde::Deserialize; use serde::Serialize; @@ -40,15 +41,14 @@ impl PublicNote { } pub fn from_message(data: &[BFieldElement]) -> Result { - if data.len() < 3 { + if data.len() < 2 { bail!("Public note too short"); } if data[0].value() != TAG_PUBLIC { bail!("Expected public tag, got {}", data[0].value()); } let receiver_id = data[1]; - let disc = data[2].value(); - let content = NoteContent::decode(disc, &data[3..])?; + let content = *NoteContent::decode(&data[2..])?; Ok(Self { receiver_id, content, From 5b6d3024e1c03d55e7026ecbcf6c8358457242b8 Mon Sep 17 00:00:00 2001 From: KaffinPX Date: Fri, 4 Sep 2026 23:10:11 +0300 Subject: [PATCH 12/14] feat(standards): Move deserialization out of ViewKey Decryptor and solidify UTXO scanner --- standards/src/wallet/keys/schemes/generation.rs | 11 ++--------- standards/src/wallet/keys/schemes/symmetric.rs | 8 ++------ standards/src/wallet/keys/viewing_key.rs | 5 ++--- standards/src/wallet/notes/content.rs | 4 ++-- wallet/sdk/Cargo.toml | 1 + wallet/sdk/src/scanners/chain.rs | 9 ++++++++- 6 files changed, 17 insertions(+), 21 deletions(-) diff --git a/standards/src/wallet/keys/schemes/generation.rs b/standards/src/wallet/keys/schemes/generation.rs index 739e66b..67410c4 100644 --- a/standards/src/wallet/keys/schemes/generation.rs +++ b/standards/src/wallet/keys/schemes/generation.rs @@ -11,7 +11,6 @@ use nyks_consensus::tasm_lib::prelude::Digest; use nyks_consensus::tasm_lib::prelude::Tip5; use nyks_consensus::transaction::lock_script::LockScript; use nyks_consensus::transaction::lock_script::LockScriptAndWitness; -use nyks_consensus::transaction::utxo::Utxo; use nyks_consensus::twenty_first::math::lattice; use nyks_consensus::twenty_first::math::lattice::kem::CIPHERTEXT_SIZE_IN_BFES; use nyks_consensus::twenty_first::math::lattice::kem::PublicKey; @@ -188,9 +187,6 @@ pub enum GenerationDecryptError { #[error("Failed to convert BFieldElements to bytes")] BfeToBytes, - - #[error("Deserialization failed")] - Deserialization(#[from] bincode::Error), } impl Zeroize for GenerationKey { @@ -238,7 +234,7 @@ impl Decryptor for GenerationViewingKey { self.privacy_preimage } - fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result<(Utxo, Digest), Self::Error> { + fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result, Self::Error> { // parse ciphertext if ciphertext.len() <= CIPHERTEXT_SIZE_IN_BFES { return Err(GenerationDecryptError::MissingNonce); @@ -273,10 +269,7 @@ impl Decryptor for GenerationViewingKey { .decrypt(nonce, ciphertext_bytes.as_ref()) .map_err(|_| GenerationDecryptError::SymmetricDecryptionFailed)?; - // convert plaintext to utxo and digest - let result = bincode::deserialize(&plaintext)?; // uses #[from] - - Ok(result) + Ok(plaintext) } } diff --git a/standards/src/wallet/keys/schemes/symmetric.rs b/standards/src/wallet/keys/schemes/symmetric.rs index 2a6d012..05098d2 100644 --- a/standards/src/wallet/keys/schemes/symmetric.rs +++ b/standards/src/wallet/keys/schemes/symmetric.rs @@ -12,7 +12,6 @@ use nyks_consensus::tasm_lib::prelude::Digest; use nyks_consensus::tasm_lib::prelude::Tip5; use nyks_consensus::transaction::lock_script::LockScript; use nyks_consensus::transaction::lock_script::LockScriptAndWitness; -use nyks_consensus::transaction::utxo::Utxo; use serde::Deserialize; use serde::Serialize; use thiserror::Error; @@ -172,9 +171,6 @@ pub enum SymmetricDecryptError { #[error("Decryption failed")] Decryption(#[from] aes_gcm::Error), - - #[error("Deserialization failed")] - Deserialization(#[from] bincode::Error), } impl Zeroize for SymmetricKey { @@ -222,7 +218,7 @@ impl Decryptor for SymmetricViewingKey { self.privacy_preimage } - fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result<(Utxo, Digest), Self::Error> { + fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result, Self::Error> { const NONCE_LEN: usize = 1; if ciphertext.len() <= NONCE_LEN { @@ -240,7 +236,7 @@ impl Decryptor for SymmetricViewingKey { let cipher = Aes256Gcm::new(&self.key); let plaintext = cipher.decrypt(nonce, ciphertext_bytes.as_ref())?; - Ok(bincode::deserialize(&plaintext)?) + Ok(plaintext) } } diff --git a/standards/src/wallet/keys/viewing_key.rs b/standards/src/wallet/keys/viewing_key.rs index 052ed8e..314c5a0 100644 --- a/standards/src/wallet/keys/viewing_key.rs +++ b/standards/src/wallet/keys/viewing_key.rs @@ -1,5 +1,4 @@ use nyks_consensus::BFieldElement; -use nyks_consensus::transaction::utxo::Utxo; use nyks_consensus::twenty_first::tip5::Digest; use zeroize::Zeroize; use zeroize::ZeroizeOnDrop; @@ -19,7 +18,7 @@ pub trait Decryptor { // Needed to extract indices of an UTXO and see if it is/was part of mutator set. fn privacy_preimage(&self) -> Digest; - fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result<(Utxo, Digest), Self::Error>; + fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result, Self::Error>; } #[derive(Debug)] @@ -52,7 +51,7 @@ impl Decryptor for ViewingKey { } } - fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result<(Utxo, Digest), Self::Error> { + fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result, Self::Error> { match self { ViewingKey::Generation(k) => k.decrypt(ciphertext).map_err(ViewingKeyError::Generation), ViewingKey::Symmetric(k) => k.decrypt(ciphertext).map_err(ViewingKeyError::Symmetric), diff --git a/standards/src/wallet/notes/content.rs b/standards/src/wallet/notes/content.rs index 22430a9..d99d8d5 100644 --- a/standards/src/wallet/notes/content.rs +++ b/standards/src/wallet/notes/content.rs @@ -83,8 +83,8 @@ impl BFieldCodec for NoteContent { match discriminant { d if d == UtxoContent::DISCRIMINANT => { - let content = UtxoContent::decode(rest) - .map_err(|e| NoteContentError::Decode(e.into()))?; + let content = + UtxoContent::decode(rest).map_err(|e| NoteContentError::Decode(e.into()))?; Ok(Box::new(Self::Utxo(*content))) } _ => Err(NoteContentError::UnknownDiscriminant(discriminant)), diff --git a/wallet/sdk/Cargo.toml b/wallet/sdk/Cargo.toml index e17d3c6..8552354 100644 --- a/wallet/sdk/Cargo.toml +++ b/wallet/sdk/Cargo.toml @@ -18,6 +18,7 @@ num-traits = "0.2.19" serde = "1.0.228" thiserror = "1.0.65" zeroize = "1.8.1" +bincode = "1.3" tokio = { version = "1.45.1", features = ["sync"] } [dev-dependencies] diff --git a/wallet/sdk/src/scanners/chain.rs b/wallet/sdk/src/scanners/chain.rs index 438a969..1d5a700 100644 --- a/wallet/sdk/src/scanners/chain.rs +++ b/wallet/sdk/src/scanners/chain.rs @@ -269,9 +269,16 @@ impl ChainScanner { }) .filter_map(|a| extract_ciphertext(a)) .filter_map(|ciphertext| key.decrypt(&ciphertext).ok()) - .map(|(utxo, sender_randomness)| { + .map(|decrypted: Vec| { + let (utxo, sender_randomness): (Utxo, Digest) = + bincode::deserialize(&decrypted).unwrap(); (utxo, sender_randomness, key.privacy_preimage()) }) + // A third party can create an announcement that decrypts under this key but + // contains a UTXO that is unspendable by us. + .filter(|(utxo, _, _)| { + utxo.lock_script_hash() == key.address().lock_script().hash() + }) .collect::>() }) .collect() From 46e9cfd7cc4fab454fa6d318d8c779bbce4cf748 Mon Sep 17 00:00:00 2001 From: KaffinPX Date: Sat, 5 Sep 2026 02:42:45 +0300 Subject: [PATCH 13/14] fix(standards): Note content discriminant being encoded twice --- .cargo/config.toml | 2 ++ standards/src/wallet/notes/note.rs | 6 +----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 71e377f..891fa3c 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -12,3 +12,5 @@ RUST_BACKTRACE = "1" # workaround for dependency `leveldb-sys v2.0.9` CMAKE_POLICY_VERSION_MINIMUM = "3.5" +CC = "clang" +CXX = "clang++" \ No newline at end of file diff --git a/standards/src/wallet/notes/note.rs b/standards/src/wallet/notes/note.rs index 5ef6639..97d8436 100644 --- a/standards/src/wallet/notes/note.rs +++ b/standards/src/wallet/notes/note.rs @@ -31,11 +31,7 @@ impl PublicNote { } pub fn into_message(self) -> Vec { - let mut msg = vec![ - BFieldElement::new(TAG_PUBLIC), - self.receiver_id, - BFieldElement::new(self.content.discriminant()), - ]; + let mut msg = vec![BFieldElement::new(TAG_PUBLIC), self.receiver_id]; msg.extend(self.content.encode()); msg } From 53f4f651a3d2f2875816eeec9f6898e5c7e1b3b3 Mon Sep 17 00:00:00 2001 From: KaffinPX Date: Sat, 5 Sep 2026 17:00:52 +0300 Subject: [PATCH 14/14] style: Fix lints --- composer/src/composer/prover/transaction.rs | 1 - consensus/src/block/block_transaction.rs | 2 +- .../src/mutator_set/removal_record/removal_record_list.rs | 8 ++++---- consensus/src/transaction/mod.rs | 2 +- consensus/src/transaction/transaction_proof.rs | 1 + 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/composer/src/composer/prover/transaction.rs b/composer/src/composer/prover/transaction.rs index 20af963..92f743c 100644 --- a/composer/src/composer/prover/transaction.rs +++ b/composer/src/composer/prover/transaction.rs @@ -100,7 +100,6 @@ impl TransactionProver { rng.random(), consensus_rule_set, ) - .await .unwrap() .into(); diff --git a/consensus/src/block/block_transaction.rs b/consensus/src/block/block_transaction.rs index dadbb9a..874b47e 100644 --- a/consensus/src/block/block_transaction.rs +++ b/consensus/src/block/block_transaction.rs @@ -159,7 +159,7 @@ impl BlockTransaction { /// See also: [`Transaction::merge_with`], which should be used if /// - a) the arguments are two regular [`Transaction`]s; and /// - b) the result must be a regular [`Transaction`] as well. - pub async fn merge( + pub fn merge( coinbase: BlockOrRegularTransaction, other: Transaction, shuffle_seed: [u8; 32], diff --git a/consensus/src/mutator_set/removal_record/removal_record_list.rs b/consensus/src/mutator_set/removal_record/removal_record_list.rs index 97b161f..d0bc285 100644 --- a/consensus/src/mutator_set/removal_record/removal_record_list.rs +++ b/consensus/src/mutator_set/removal_record/removal_record_list.rs @@ -564,10 +564,10 @@ impl RemovalRecordList { { if *tree_height < Self::ENCODING_TREE_HEIGHT_OFFSET { // Verify that tree heights are sorted correctly - if let Some(previous) = tree_heights.last() { - if *previous > *tree_height { - return Err(RemovalRecordListUnpackError::IncorrectlySortedTreeHeights); - } + if let Some(previous) = tree_heights.last() + && *previous > *tree_height + { + return Err(RemovalRecordListUnpackError::IncorrectlySortedTreeHeights); } // use both authentication structure and chunk diff --git a/consensus/src/transaction/mod.rs b/consensus/src/transaction/mod.rs index fa0b0f0..fef2ab4 100644 --- a/consensus/src/transaction/mod.rs +++ b/consensus/src/transaction/mod.rs @@ -82,7 +82,7 @@ impl Transaction { /// set hashes are not the same, if both transactions have coinbase a /// coinbase UTXO, if either of the transactions are *not* a single /// proof, or if the RHS (`other`) has a negative fee. - pub async fn merge_with( + pub fn merge_with( self, other: Transaction, shuffle_seed: [u8; 32], diff --git a/consensus/src/transaction/transaction_proof.rs b/consensus/src/transaction/transaction_proof.rs index 1ab5025..842aefa 100644 --- a/consensus/src/transaction/transaction_proof.rs +++ b/consensus/src/transaction/transaction_proof.rs @@ -68,6 +68,7 @@ impl TransactionProofType { /// represents a transaction proof, which can be of different types. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, GetSize, BFieldCodec)] +#[allow(clippy::large_enum_variant)] pub enum TransactionProof { /// a strong proof. required for confirming a transaction into a block. SingleProof(NyksProof),