diff --git a/.cargo/config.toml b/.cargo/config.toml index a0e9bbd..891fa3c 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -13,4 +13,4 @@ RUST_BACKTRACE = "1" # workaround for dependency `leveldb-sys v2.0.9` CMAKE_POLICY_VERSION_MINIMUM = "3.5" CC = "clang" -CXX = "clang++" +CXX = "clang++" \ No newline at end of file diff --git a/Makefile b/Makefile index 42a37af..329e72e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: clean help stats bench all install run test build doc check format bench-no-run pretty-log ensure-clang -prog :=neptune-core +prog :=nyks-node # Passive dependency-checker. Informs and exits. check-clang: @@ -41,7 +41,7 @@ build: ensure-clang doc: cargo doc --no-deps - xdg-open "target/doc/neptune-core/index.html" + xdg-open "target/doc/nyks_node/index.html" check: check-clang cargo check @@ -59,12 +59,11 @@ happy: clippy format cargo test --doc install: ensure-clang - cargo install --force --locked --path neptune-core/ - cargo install --force --locked --path neptune-core-cli/ - cargo install --force --locked --path neptune-dashboard/ - -install-linux: install - @echo "\n\nPlease run:\n./scripts/linux/install-bash-completions.sh\nto install bash-completions for Neptune-core's CLI." + cargo install --force --locked --path nyks-node/ + cargo install --force --locked --path nyks-prover/ + cargo install --force --locked --path nyks-wallet/ + cargo install --force --locked --path nyks-composer/ + cargo install --force --locked --path nyks-upgrader/ clippy: cargo clippy --all-targets -- -D warnings diff --git a/composer/src/composer/prover/transaction.rs b/composer/src/composer/prover/transaction.rs index 20af963..92f743c 100644 --- a/composer/src/composer/prover/transaction.rs +++ b/composer/src/composer/prover/transaction.rs @@ -100,7 +100,6 @@ impl TransactionProver { rng.random(), consensus_rule_set, ) - .await .unwrap() .into(); diff --git a/consensus/src/block/block_transaction.rs b/consensus/src/block/block_transaction.rs index dadbb9a..874b47e 100644 --- a/consensus/src/block/block_transaction.rs +++ b/consensus/src/block/block_transaction.rs @@ -159,7 +159,7 @@ impl BlockTransaction { /// See also: [`Transaction::merge_with`], which should be used if /// - a) the arguments are two regular [`Transaction`]s; and /// - b) the result must be a regular [`Transaction`] as well. - pub async fn merge( + pub fn merge( coinbase: BlockOrRegularTransaction, other: Transaction, shuffle_seed: [u8; 32], diff --git a/consensus/src/mutator_set/removal_record/removal_record_list.rs b/consensus/src/mutator_set/removal_record/removal_record_list.rs index 97b161f..d0bc285 100644 --- a/consensus/src/mutator_set/removal_record/removal_record_list.rs +++ b/consensus/src/mutator_set/removal_record/removal_record_list.rs @@ -564,10 +564,10 @@ impl RemovalRecordList { { if *tree_height < Self::ENCODING_TREE_HEIGHT_OFFSET { // Verify that tree heights are sorted correctly - if let Some(previous) = tree_heights.last() { - if *previous > *tree_height { - return Err(RemovalRecordListUnpackError::IncorrectlySortedTreeHeights); - } + if let Some(previous) = tree_heights.last() + && *previous > *tree_height + { + return Err(RemovalRecordListUnpackError::IncorrectlySortedTreeHeights); } // use both authentication structure and chunk diff --git a/consensus/src/transaction/mod.rs b/consensus/src/transaction/mod.rs index fa0b0f0..fef2ab4 100644 --- a/consensus/src/transaction/mod.rs +++ b/consensus/src/transaction/mod.rs @@ -82,7 +82,7 @@ impl Transaction { /// set hashes are not the same, if both transactions have coinbase a /// coinbase UTXO, if either of the transactions are *not* a single /// proof, or if the RHS (`other`) has a negative fee. - pub async fn merge_with( + pub fn merge_with( self, other: Transaction, shuffle_seed: [u8; 32], diff --git a/consensus/src/transaction/transaction_proof.rs b/consensus/src/transaction/transaction_proof.rs index 1ab5025..842aefa 100644 --- a/consensus/src/transaction/transaction_proof.rs +++ b/consensus/src/transaction/transaction_proof.rs @@ -68,6 +68,7 @@ impl TransactionProofType { /// represents a transaction proof, which can be of different types. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, GetSize, BFieldCodec)] +#[allow(clippy::large_enum_variant)] pub enum TransactionProof { /// a strong proof. required for confirming a transaction into a block. SingleProof(NyksProof), diff --git a/node/src/application/config/cli_args.rs b/node/src/application/config/cli_args.rs index d7463bb..858983b 100644 --- a/node/src/application/config/cli_args.rs +++ b/node/src/application/config/cli_args.rs @@ -11,8 +11,8 @@ use clap::builder::TypedValueParser; use clap::Parser; use libp2p::multiaddr::Protocol; use libp2p::Multiaddr; -use nyks_consensus::transaction::transaction_proof::TransactionProofQuality; use nyks_consensus::network::Network; +use nyks_consensus::transaction::transaction_proof::TransactionProofQuality; use nyks_consensus::type_scripts::native_currency_amount::NativeCurrencyAmount; use nyks_rpc_core::api::ops::Namespace; use tracing::error; @@ -271,19 +271,6 @@ pub struct Args { #[clap(long, default_value = "1800", value_parser = duration_from_seconds_str)] pub reconnect_cooldown: Duration, - /// Construct and maintain a UTXO index - /// - /// If set, all announcements and inputs in all processed blocks will be - /// indexed in a database that enables a fast rescan for the discovery of - /// all balance-affecting inputs and outputs of blocks. - /// - /// If blocks have already been processed without this flag active, and the - /// flag is later activated, all blocks up to the current tip will be - /// indexed, when a new block is set as tip. This process might take some - /// time (tens of minutes). - #[clap(long)] - pub utxo_index: bool, - /// Enable JSON/HTTP RPC. /// You can optionally specify an address and port (default: 127.0.0.1:9797). /// If not given, RPC is disabled. diff --git a/node/src/application/config/data_directory.rs b/node/src/application/config/data_directory.rs index 2cbd4fd..f96a6d4 100644 --- a/node/src/application/config/data_directory.rs +++ b/node/src/application/config/data_directory.rs @@ -11,7 +11,6 @@ use serde::Serialize; use crate::state::archival_state::ARCHIVAL_BLOCK_MMR_DIRECTORY_NAME; use crate::state::archival_state::BLOCK_INDEX_DB_NAME; use crate::state::archival_state::MUTATOR_SET_DIRECTORY_NAME; -use crate::state::archival_state::UTXO_INDEX_DIRECTORY_NAME; use crate::state::database::DATABASE_DIRECTORY_ROOT_NAME; use crate::state::networking_state::BANNED_IPS_DB_NAME; use crate::state::shared::BLOCK_FILENAME_EXTENSION; @@ -74,15 +73,11 @@ impl DataDirectory { .context("open_ensure_parent_dir_exists") } - /////////////////////////////////////////////////////////////////////////// - /// /// The root data directory path pub fn root_dir_path(&self) -> PathBuf { self.data_dir.clone() } - /////////////////////////////////////////////////////////////////////////// - /// /// The block database directory path pub fn database_dir_path(&self) -> PathBuf { self.data_dir.join(Path::new(DATABASE_DIRECTORY_ROOT_NAME)) @@ -92,8 +87,6 @@ impl DataDirectory { self.data_dir.join(Path::new(NETWORK_SUBDIRECTORY_NAME)) } - /////////////////////////////////////////////////////////////////////////// - /// /// The banned IPs database directory path. /// /// This directory lives within `DataDirectory::database_dir_path()`. @@ -101,15 +94,12 @@ impl DataDirectory { self.database_dir_path().join(Path::new(BANNED_IPS_DB_NAME)) } - /////////////////////////////////////////////////////////////////////////// /// directory for storing database backups before migrating schema to newer version pub fn db_migration_backups_dir_path(&self) -> PathBuf { self.database_dir_path() .join(Path::new(DB_MIGRATION_BACKUPS_DIR)) } - /////////////////////////////////////////////////////////////////////////// - /// /// The mutator set database directory path. /// /// This directory lives within `DataDirectory::database_dir_path()`. @@ -118,8 +108,6 @@ impl DataDirectory { .join(Path::new(MUTATOR_SET_DIRECTORY_NAME)) } - /////////////////////////////////////////////////////////////////////////// - /// /// The archival block MMR database directory path /// /// This directory lives within `DataDirectory::database_dir_path()`. @@ -128,18 +116,6 @@ impl DataDirectory { .join(Path::new(ARCHIVAL_BLOCK_MMR_DIRECTORY_NAME)) } - /////////////////////////////////////////////////////////////////////////// - /// - /// The UTXO index database directory path - /// - /// This directory lives within `DataDirectory::database_dir_path()`. - pub(crate) fn utxo_index_dir_path(&self) -> PathBuf { - self.database_dir_path() - .join(Path::new(UTXO_INDEX_DIRECTORY_NAME)) - } - - /////////////////////////////////////////////////////////////////////////// - /// /// The block body directory. /// /// This directory lives within `DataDirectory::root_dir_path()`. diff --git a/node/src/application/rpc/server.rs b/node/src/application/rpc/server.rs index 1849951..ee991d8 100644 --- a/node/src/application/rpc/server.rs +++ b/node/src/application/rpc/server.rs @@ -54,16 +54,6 @@ impl RpcServer { } } - if namespaces.contains(&Namespace::Utxoindex) { - let has_utxo_index = - state.chain.is_archival_node() && state.chain.archival_state().utxo_index.is_some(); - - if !has_utxo_index { - namespaces.remove(&Namespace::Utxoindex); - error!("Node does not maintain a UTXO index, cannot enable utxoindex namespace."); - } - } - if !self.unrestricted && namespaces.contains(&Namespace::Network) { warn!("Networking module is enabled without unsafe mode - this may expose sensitive data.") } diff --git a/node/src/application/rpc/service.rs b/node/src/application/rpc/service.rs index 53c6210..0673429 100644 --- a/node/src/application/rpc/service.rs +++ b/node/src/application/rpc/service.rs @@ -1,11 +1,7 @@ -use std::collections::HashSet; - use async_trait::async_trait; use itertools::Itertools; use nyks_consensus::block::Block; use nyks_consensus::block::FUTUREDATING_LIMIT; -use nyks_consensus::mutator_set::addition_record::AdditionRecord; -use nyks_consensus::mutator_set::removal_record::absolute_index_set::AbsoluteIndexSet; use nyks_consensus::proof_abstractions::timestamp::Timestamp; use nyks_consensus::transaction::Transaction; use nyks_rpc_core::api::rpc::*; @@ -650,112 +646,6 @@ impl RpcApi for RpcServer { Ok(SubmitBlockResponse { success }) } - /* Utxoindex */ - - async fn block_heights_by_flags_call( - &self, - request: BlockHeightsByFlagsRequest, - ) -> RpcResult { - let announcement_flags: HashSet<_> = request.announcement_flags.into_iter().collect(); - let heights = self - .state - .lock_guard() - .await - .chain - .archival_state() - .utxo_index - .as_ref() - .expect("Utxo index namespace can only be active when UTXO index is present") - .blocks_by_announcement_flags(&announcement_flags) - .await; - - let block_heights = BlockHeightsByFlagsResponse { - block_heights: heights.into_iter().collect(), - }; - - Ok(block_heights) - } - - async fn block_heights_by_addition_records_call( - &self, - request: BlockHeightsByAdditionRecordsRequest, - ) -> RpcResult { - let addition_records: HashSet = request - .addition_records - .into_iter() - .map(|x| x.into()) - .collect(); - - let block_heights = self - .state - .lock_guard() - .await - .chain - .archival_state() - .addition_records_to_block_height(addition_records) - .await - .expect("Utxo index namespace can only be active when UTXO index is present"); - - let block_heights = BlockHeightsByAdditionRecordsResponse { - block_heights: block_heights.into_iter().collect(), - }; - - Ok(block_heights) - } - - async fn block_heights_by_absolute_index_sets_call( - &self, - request: BlockHeightsByAbsoluteIndexSetsRequest, - ) -> RpcResult { - let absolute_index_sets: HashSet = - request.absolute_index_sets.into_iter().collect(); - - let block_heights = self - .state - .lock_guard() - .await - .chain - .archival_state() - .absolute_index_sets_to_block_heights(absolute_index_sets) - .await - .expect("Utxo index namespace can only be active when UTXO index is present"); - - let block_heights = BlockHeightsByAbsoluteIndexSetsResponse { - block_heights: block_heights.into_iter().collect(), - }; - - Ok(block_heights) - } - - async fn was_mined_call(&self, request: WasMinedRequest) -> RpcResult { - if request.addition_records.is_empty() && request.absolute_index_sets.is_empty() { - return Err(RpcError::EmptyFilteringConditions); - } - - let addition_records = request - .addition_records - .into_iter() - .map(|x| x.into()) - .collect(); - let absolute_index_sets = request.absolute_index_sets.into_iter().collect(); - - let blocks = self - .state - .lock_guard() - .await - .chain - .archival_state() - .canonical_block_heights_with_puts(absolute_index_sets, addition_records) - .await - .expect("UTXO index namespace is only active if UTXO index is maintained"); - - let res = WasMinedResponse { - block_heights: blocks.into_iter().collect(), - }; - - Ok(res) - } - /* Mempool */ async fn transactions_call(&self, _: TransactionsRequest) -> RpcResult { diff --git a/node/src/state/archival_state.rs b/node/src/state/archival_state.rs index e8d5c95..b2c6d97 100644 --- a/node/src/state/archival_state.rs +++ b/node/src/state/archival_state.rs @@ -1,32 +1,43 @@ +pub(crate) mod import_blocks_from_files; + use std::collections::HashMap; -use std::collections::HashSet; use std::ops::DerefMut; use std::path::PathBuf; use anyhow::bail; -use anyhow::ensure; use anyhow::Result; use itertools::Itertools; use memmap2::MmapOptions; use num_traits::Zero; -use tasm_lib::prelude::Tip5; +use nyks_consensus::block::block_header::BlockHeader; +use nyks_consensus::block::block_header::BlockHeaderWithBlockHashWitness; +use nyks_consensus::block::block_header::HeaderToBlockHashWitness; +use nyks_consensus::block::block_height::BlockHeight; +use nyks_consensus::block::block_kernel::BlockKernel; +use nyks_consensus::block::mutator_set_update::MutatorSetUpdate; +use nyks_consensus::block::Block; +use nyks_consensus::mutator_set::addition_record::AdditionRecord; +use nyks_consensus::mutator_set::mutator_set_accumulator::MutatorSetAccumulator; +use nyks_consensus::mutator_set::removal_record::absolute_index_set::AbsoluteIndexSet; +use nyks_consensus::mutator_set::removal_record::RemovalRecord; +use nyks_consensus::network::Network; +use nyks_consensus::transaction::transaction_kernel::TransactionKernelProxy; +use nyks_database::create_db_if_missing; +use nyks_database::storage::storage_schema::traits::*; +use nyks_database::NeptuneLevelDb; +use nyks_database::WriteBatchAsync; use tasm_lib::twenty_first::prelude::Mmr; use tasm_lib::twenty_first::tip5::digest::Digest; use tokio::io::AsyncSeekExt; use tokio::io::AsyncWriteExt; use tokio::io::SeekFrom; use tracing::debug; -use tracing::info; use tracing::warn; -pub(crate) mod import_blocks_from_files; -pub mod rusty_utxo_index; - use super::shared::new_block_file_is_needed; use super::StorageVecBase; use crate::application::config::cli_args::Args; use crate::application::config::data_directory::DataDirectory; -use crate::state::archival_state::rusty_utxo_index::RustyUtxoIndex; use crate::state::database::BlockFileLocation; use crate::state::database::BlockIndexKey; use crate::state::database::BlockIndexValue; @@ -35,28 +46,10 @@ use crate::state::database::FileRecord; use crate::state::database::LastFileRecord; use crate::util_types::rusty_archival_block_mmr::RustyArchivalBlockMmr; use crate::util_types::rusty_archival_mutator_set::RustyArchivalMutatorSet; -use nyks_consensus::block::block_header::BlockHeader; -use nyks_consensus::block::block_header::BlockHeaderWithBlockHashWitness; -use nyks_consensus::block::block_header::HeaderToBlockHashWitness; -use nyks_consensus::block::block_height::BlockHeight; -use nyks_consensus::block::block_kernel::BlockKernel; -use nyks_consensus::block::mutator_set_update::MutatorSetUpdate; -use nyks_consensus::block::Block; -use nyks_consensus::mutator_set::addition_record::AdditionRecord; -use nyks_consensus::mutator_set::mutator_set_accumulator::MutatorSetAccumulator; -use nyks_consensus::mutator_set::removal_record::absolute_index_set::AbsoluteIndexSet; -use nyks_consensus::mutator_set::removal_record::RemovalRecord; -use nyks_consensus::network::Network; -use nyks_consensus::transaction::transaction_kernel::TransactionKernelProxy; -use nyks_database::create_db_if_missing; -use nyks_database::storage::storage_schema::traits::*; -use nyks_database::NeptuneLevelDb; -use nyks_database::WriteBatchAsync; pub(crate) const BLOCK_INDEX_DB_NAME: &str = "block_index"; pub(crate) const MUTATOR_SET_DIRECTORY_NAME: &str = "mutator_set"; pub(crate) const ARCHIVAL_BLOCK_MMR_DIRECTORY_NAME: &str = "archival_block_mmr"; -pub(crate) const UTXO_INDEX_DIRECTORY_NAME: &str = "utxo_index"; /// Provides interface to historic blockchain data which consists of /// * block-data stored in individual files (append-only) @@ -91,13 +84,6 @@ pub struct ArchivalState { /// Archival-MMR of the block digests belonging to the canonical chain. pub archival_block_mmr: RustyArchivalBlockMmr, - /// Mapping from block digest to a list of (flag, receiver_id) pairs for all - /// announcement in the block, and other indexing data related to historical - /// blocks. This index is only maintained if the node has been started with - /// the CLI flag `--utxo-index`, which implies that this value is Some(T). - /// If the node is not started with this flag, this value is `None`. - pub(crate) utxo_index: Option, - /// The network that this node is on. Used to simplify method interfaces. network: Network, } @@ -312,22 +298,6 @@ impl ArchivalState { .expect("Must be able to initialize block index database"); debug!("Got block index database"); - // UTXO index is always initialized. But only populated with blocks if - // this index is activated. - let utxo_index = if cli.utxo_index { - let mut utxo_index = RustyUtxoIndex::initialize(&data_dir) - .await - .expect("Must be able to initialize utxo index database"); - - if utxo_index.is_empty().await { - utxo_index.index_block(&genesis_block).await; - } - debug!("UTXO index populated"); - Some(utxo_index) - } else { - None - }; - let network = cli.network; let genesis_block = Box::new(genesis_block); @@ -338,7 +308,6 @@ impl ArchivalState { archival_mutator_set, archival_block_mmr, network, - utxo_index, } } @@ -559,7 +528,6 @@ impl ArchivalState { self.write_block_as_tip(block).await?; self.append_to_archival_block_mmr(block).await; self.update_mutator_set(block).await?; - self.update_utxo_index(block).await; Ok(()) } @@ -658,77 +626,6 @@ impl ArchivalState { .await; } - /// Apply a new block to the UTXO index. Does nothing if no UTXO index is - /// maintained by this archival state. - /// - /// This method handles reorganizations, but all predecessors of this block - /// must be known and stored in the block index database for it to work. - /// Reorganizations leaves ophaned blocks in the index though. So this must - /// be accounted for when reading from the index. - /// - /// # Panics - /// - If any of the predecessor blocks have not been applied to the block - /// index database. - async fn update_utxo_index(&mut self, new_block: &Block) { - if self.utxo_index.is_none() { - return; - } - - let current_sync = self.utxo_index.as_ref().unwrap().sync_label().await; - let new_block_hash = new_block.hash(); - - // Index all not-yet-indexed blocks preceding the new block. In the - // common case, where the new block is the direct descendant of the - // block that was previously applied, only one block will be processed - // here. This path-finding logic allows for an efficient common-case - // processing, and an effcient "catchup" behavior where the UTXO index - // is many block behind the rest of the archival state. - let (_, _, missing_blocks) = self.find_path(current_sync, new_block_hash).await; - - // Inform user if this will take a long time. - let num_missing_blocks = missing_blocks.len(); - debug!("Applying {num_missing_blocks} missing blocks to UTXO index."); - if num_missing_blocks > 10 { - info!("Applying {num_missing_blocks} missing blocks to UTXO index. This may take a while.") - } - for missing in missing_blocks { - // This optimization means that we don't have to read the full - // blocks from disk in case it was already processed. - if self - .utxo_index - .as_ref() - .unwrap() - .block_was_indexed(missing) - .await - { - continue; - } - - if missing == new_block_hash { - // Avoid reading the new block from disk if it's already in - // memory. - self.utxo_index - .as_mut() - .unwrap() - .index_block(new_block) - .await; - } else { - let missing = self - .get_block(missing) - .await - .expect("Fetching block must succeed") - .expect("missing block must exist before processed by UTXO index"); - self.utxo_index - .as_mut() - .unwrap() - .index_block(&missing) - .await; - } - } - - debug!("Done updating UTXO index"); - } - async fn get_block_from_block_record(&self, block_record: BlockRecord) -> Result { let block_file_path: PathBuf = self .data_dir @@ -872,8 +769,7 @@ impl ArchivalState { /// `None` if no canonical block with this output is known. /// /// searches max `max_search_depth` from tip for a matching transaction - /// output. Unless the node maintain a UTXO index in which case all blocks - /// are searched and this parameter is ignored. + /// output. /// /// If `max_search_depth` is set to `None`, then all blocks are searched /// until a match is found. A `max_search_depth` of `Some(0)` will only @@ -894,51 +790,23 @@ impl ArchivalState { ) } - /// Returns the 1st block hash containing this addition record. Returns - /// `None` if no canonical block with this output is known. - /// - /// searches max `max_search_depth` from tip for a matching transaction - /// output. Unless the node maintain a UTXO index in which case all blocks - /// are searched and this parameter is ignored. - /// - /// If `max_search_depth` is set to `None`, then all blocks are searched - /// until a match is found. A `max_search_depth` of `Some(0)` will only - /// consider the tip. - /// - /// Never loads blocks from disk, so performance should be good. async fn find_canonical_block_hash_with_output( &self, output: AdditionRecord, max_search_depth: Option, ) -> Option { - let block_heights = match &self.utxo_index { - Some(utxo_index) => { - let heights = utxo_index - .blocks_by_addition_record(output) - .await - .into_iter() - .map(|x| x.value()) - .sorted_unstable(); - itertools::Either::Left(heights) - } - None => { - let tip_height = self.tip_header().await.height.value(); - - let end = match max_search_depth { - Some(num) => tip_height.saturating_sub(num), - None => 0, - }; + let tip_height = self.tip_header().await.height.value(); - let heights = (end..=tip_height).rev(); - itertools::Either::Right(heights) - } + let end = match max_search_depth { + Some(num) => tip_height.saturating_sub(num), + None => 0, }; - for block_height in block_heights { + for block_height in (end..=tip_height).rev() { let (addition_records, block_hash) = self .addition_record_indices_for_block_by_height(block_height) .await - .expect("Block height from UTXO index must be known"); + .expect("Block height in search range must be known"); if addition_records.contains_key(&output) { return Some(block_hash); @@ -951,41 +819,25 @@ impl ArchivalState { /// Returns the block containing this input. Returns `None` if no canonical /// block with this input is known. /// - /// searches max `max_search_depth` from tip for a matching transaction - /// input. + /// Searches max `max_search_depth` blocks back from tip for a matching + /// transaction input. /// - /// searches max `max_search_depth` from tip for a matching transaction - /// input. Unless the node maintain a UTXO index in which case all blocks - /// are searched and this parameter is ignored. + /// If `max_search_depth` is set to `None`, then all blocks are searched + /// until a match is found. A `max_search_depth` of `Some(0)` will only + /// consider the tip. pub(crate) async fn find_canonical_block_with_input( &self, input: AbsoluteIndexSet, max_search_depth: Option, ) -> Option { - let block_heights = match &self.utxo_index { - Some(utxo_index) => { - let heights = utxo_index - .block_by_index_set(&input) - .await - .into_iter() - .map(|x| x.value()) - .sorted_unstable(); - itertools::Either::Left(heights) - } - None => { - let tip_height = self.tip_header().await.height.value(); + let tip_height = self.tip_header().await.height.value(); - let end = match max_search_depth { - Some(num) => tip_height.saturating_sub(num), - None => 0, - }; - - let heights = (end..=tip_height).rev(); - itertools::Either::Right(heights) - } + let end = match max_search_depth { + Some(num) => tip_height.saturating_sub(num), + None => 0, }; - for block_height in block_heights { + for block_height in (end..=tip_height).rev() { let block = self .canonical_block_by_height(block_height.into()) .await @@ -1004,180 +856,6 @@ impl ArchivalState { None } - /// Return all block heights of blocks belonging to the canonical chain - /// containing any of the requested addition records. - /// - /// Never loads the entire block from disk. Only reads from the database, so - /// performace should be good. - /// - /// Only works if a UTXO index is maintained. - pub(crate) async fn addition_records_to_block_height( - &self, - addition_records: HashSet, - ) -> Result> { - ensure!( - self.utxo_index.is_some(), - "Only works a UTXO index is maintained." - ); - - let mut ret = HashSet::new(); - for addition_record in addition_records { - let maybe_matching_blocks = self - .utxo_index - .as_ref() - .unwrap() - .blocks_by_addition_record(addition_record) - .await; - - // Verify reported block height matches a block in the canonical - // chain. - // An addition record can (theoretically) be present in mutiple - // blocks, and even multiple times in the same block. This loop - // handles that case. Common case is that returned list has length - // zero or one. - for height in maybe_matching_blocks { - let (actual_ars_in_canonical_block, _) = self - .addition_record_indices_for_block_by_height(height.into()) - .await - .expect("Height reported by UTXO index must be known by archival state"); - if actual_ars_in_canonical_block.contains_key(&addition_record) { - ret.insert(height); - } - } - } - - Ok(ret) - } - - /// Return all block heights of blocks belonging to the canonical chain - /// containing any of the requested absolute index sets. - /// - /// Never loads the entire block from disk. Only reads from the database, so - /// performace should be good. - /// - /// Only works if a UTXO index is maintained. - pub(crate) async fn absolute_index_sets_to_block_heights( - &self, - absolute_index_sets: HashSet, - ) -> Result> { - ensure!( - self.utxo_index.is_some(), - "Only works a UTXO index is maintained." - ); - - let mut ret = HashSet::new(); - for index_set in absolute_index_sets { - let maybe_matching_block = self - .utxo_index - .as_ref() - .unwrap() - .block_by_index_set(&index_set) - .await; - - let Some(maybe_matching_block) = maybe_matching_block else { - continue; - }; - - // No use to add same block height twice. - if ret.contains(&maybe_matching_block) { - continue; - } - - // Verify that the block height in question has not been reorganized - // out of canonicity. - let Some(block_hash) = self - .archival_block_mmr - .ammr() - .try_get_leaf(maybe_matching_block.into()) - .await - else { - // Reorganization to shorter chain. Very unlikely. - continue; - }; - - let block_index_set_digests = self - .utxo_index - .as_ref() - .unwrap() - .index_set_digests(block_hash) - .await - .expect("Canonical block must have been indexed by UTXO index"); - - let index_set_digest = Tip5::hash(&index_set); - if block_index_set_digests.contains(&index_set_digest) { - ret.insert(maybe_matching_block); - } - } - - Ok(ret) - } - - /// Return the block heights for blocks matching *all* elements in the - /// specified input/output lists, for blocks belonging to the canonical - /// chain. Will not return block heights were e.g. only one of the outputs - /// was included if more than one output is included in the outputs list. - /// - /// Can return multiple blocks in the case where blocks are selected only - /// based on addition records and multiple blocks contain the same addition - /// records. - /// - /// Only works if a UTXO index is maintained, otherwise an error is - /// returned. - /// - /// # Panics - /// - If no filtering is applied, i.e. if both input and output lists are - /// empty. - pub(crate) async fn canonical_block_heights_with_puts( - &self, - absolute_index_sets: HashSet, - addition_records: HashSet, - ) -> Result> { - ensure!( - self.utxo_index.is_some(), - "Only works a UTXO index is maintained." - ); - - assert!( - !addition_records.is_empty() || !absolute_index_sets.is_empty(), - "No filtering was applied" - ); - - let mut block_matches: Option> = None; - for index_set in absolute_index_sets { - let block_heights = self - .absolute_index_sets_to_block_heights(HashSet::from([index_set])) - .await - .expect("Utxo index namespace can only be active when UTXO index is present"); - - match block_matches { - Some(bmatches) => { - block_matches = Some(bmatches.intersection(&block_heights).copied().collect()); - } - None => { - block_matches = Some(block_heights); - } - } - } - - for addition_record in addition_records { - let block_heights = self - .addition_records_to_block_height(HashSet::from([addition_record])) - .await - .expect("Utxo index namespace can only be active when UTXO index is present"); - - match block_matches { - Some(bmatches) => { - block_matches = Some(bmatches.intersection(&block_heights).copied().collect()); - } - None => { - block_matches = Some(block_heights); - } - } - } - - Ok(block_matches.expect("At least one filtering criteria was set")) - } - /// Return latest block from database, or genesis block if no other block /// is known. pub async fn get_tip(&self) -> Block { @@ -1342,7 +1020,7 @@ impl ArchivalState { ) } - /// Returns a [`HashMap`] of [`AdditionRecord`] to AOCL leaf indices for + /// Returns a [`HashMap`] of [`AdditionRecord`] to AOCL leaf indices for /// all outputs in a given block, including guesser rewards. Also returns /// the block hash. Returns `None` if no block at the specified height is /// known. AOCL leaf indices have list type since a block can contain the diff --git a/node/src/state/archival_state/rusty_utxo_index.rs b/node/src/state/archival_state/rusty_utxo_index.rs deleted file mode 100644 index 3d4154f..0000000 --- a/node/src/state/archival_state/rusty_utxo_index.rs +++ /dev/null @@ -1,431 +0,0 @@ -use std::collections::HashSet; - -use anyhow::Result; -use itertools::Itertools; -use serde::Deserialize; -use serde::Serialize; -use tasm_lib::prelude::Digest; -use tasm_lib::prelude::Tip5; -use tracing::warn; - -use crate::application::config::data_directory::DataDirectory; -use nyks_consensus::block::block_height::BlockHeight; -use nyks_consensus::block::Block; -use nyks_consensus::mutator_set::addition_record::AdditionRecord; -use nyks_consensus::mutator_set::removal_record::absolute_index_set::AbsoluteIndexSet; -use nyks_database::create_db_if_missing; -use nyks_database::storage::storage_schema::traits::*; -use nyks_database::NeptuneLevelDb; -use nyks_database::WriteBatchAsync; -use nyks_standards::wallet::notes::announcement_flag::AnnouncementFlag; - -/// The maximum number of blocks stored for each [`AnnouncementFlag`]. Wallets -/// with incoming UTXOs in more than this number of blocks cannot rely on the -/// mapping from announcement flags to block heights to restore a wallet. They -/// must instead use other methods. Also used to cap mapping from addition -/// records to block heights. -pub const MAX_NUM_BLOCKS_IN_LOOKUP_LIST: usize = 10_000; - -/// The purpose of the UTXO index is to speed up the rescanning of historical -/// blocks, and to serve 3rd parties with information required to detect -/// incoming and outgoing UTXOs as quickly as possible. It assumes the presence -/// of an [`ArchivalState`]. Any decision about tables in the UTXO index should -/// be made in the light of allowing clients or 3rd parties to discover balance- -/// affecting input or output UTXOs in historical blocks as quickly as possible, -/// and to minimize storage requirements for the UTXO index. -/// -/// The tables of the UTXO index database. Does not include a mapping from -/// block digest to the block's addition records since that mapping can be found -/// from the [`ArchivalMutatorSet`] which is assumed to be part of the state of -/// all nodes that maintain a UTXO index. -/// -/// Block heights are often preferred over block digests due to their smaller -/// serialized size (8 bytes vs. 40). -/// -/// [`ArchivalMutatorSet`]: nyks_consensus::mutator_set::archival_mutator_set::ArchivalMutatorSet -/// [`ArchivalState`]: crate::state::archival_state::ArchivalState -#[derive(Debug)] -pub(crate) struct RustyUtxoIndex { - db: NeptuneLevelDb, -} - -/// The key types used by the UTXO index database. -#[derive(Debug, Copy, Clone, Serialize, Deserialize)] -enum UtxoIndexKey { - /// Latest block handled by this database. Any initialized database must - /// have a sync label set. The default value indicates that no blocks have - /// been processed by the UTXO index. - SyncLabel, - - /// Mapping from block hash to the list of announcement flags contained in - /// the block. - /// - /// Can be used to speed up the scanning for incoming, announced UTXOs. - AnnouncementsByBlock(Digest), - - /// Mapping from block hash to the list of digests of the absolute indices - /// being set in the block. - /// - /// Can be used to speed up the scanning for spent UTXOs, i.e. expenditures. - IndexSetDigestsByBlock(Digest), - - /// Mapping from announcement flag to block height for all blocks in which - /// announcements with this flag are present. Length of list of block - /// heights is capped by [MAX_NUM_BLOCKS_IN_LOOKUP_LIST] in order to foil - /// certain DOS attacks. This means that extremely active wallets/smart - /// contracts that have received announced UTXOs in more than - /// [MAX_NUM_BLOCKS_IN_LOOKUP_LIST] blocks, cannot use this index to fully - /// restore a wallet. But in their case, they might as well scan all blocks - /// anyway. - /// - /// Since the indexed blocks are not guaranteed to be canonical, this - /// mapping may contain entries for blocks that are not part of the - /// canonical chain. - /// - /// Can be used to speed up the scanning for incoming, announced UTXOs, and - /// to serve RPC requests from external wallet programs. - BlocksByAnnouncementFlag(AnnouncementFlag), - - /// Mapping from addition record to block height for all blocks containing - /// the specific addition record. Length of list of block heights is capped - /// by [MAX_NUM_BLOCKS_IN_LOOKUP_LIST] in order to foil certain DOS attacks. - /// This means that if this addition record is present in more than - /// [MAX_NUM_BLOCKS_IN_LOOKUP_LIST] different blocks, the list will be - /// capped. - /// - /// This mapping includes guesser-reward addition records. - /// - /// Since the indexed blocks are not guaranteed to be canonical, this - /// mapping may contain entries for blocks that are not part of the - /// canonical chain. - /// - /// Can be used to serve an RPC endpoint that maps addition records to block - /// heights. - BlocksByAdditionRecord(AdditionRecord), - - /// Mapping from hash of absolute index set to block height. - /// - /// Can be used to serve an RPC endpoint that maps absolute index sets to - /// block heights. - BlockByIndexSetDigest(Digest), -} - -/// The values used by the UTXO index database. -/// -/// See documentstion in [`UtxoIndexKey`] for each variant of this enum. -#[derive(Debug, Clone, Serialize, Deserialize)] -enum UtxoIndexValue { - SyncLabel(Digest), - AnnouncementsByBlock(Vec), - IndexSetDigestsByBlock(Vec), - BlocksByAnnouncementFlag(Vec), - BlocksByAdditionRecord(Vec), - BlockByIndexSetDigest(BlockHeight), -} - -impl UtxoIndexValue { - fn expect_sync_label(self) -> Digest { - match self { - UtxoIndexValue::SyncLabel(digest) => digest, - _ => panic!("Expected SyncLabel found {:?}", self), - } - } - - fn expect_index_set_digests_by_block(self) -> Vec { - match self { - UtxoIndexValue::IndexSetDigestsByBlock(index_set_digests) => index_set_digests, - _ => panic!("Expected IndexSetDigestsByBlock found {:?}", self), - } - } - - fn expect_blocks_by_announcements(self) -> Vec { - match self { - UtxoIndexValue::BlocksByAnnouncementFlag(block_heights) => block_heights, - _ => panic!("Expected BlocksByAnnouncementFlag found {:?}", self), - } - } - - fn expect_blocks_by_addition_records(self) -> Vec { - match self { - UtxoIndexValue::BlocksByAdditionRecord(block_heights) => block_heights, - _ => panic!("Expected BlocksByAdditionRecord found {:?}", self), - } - } - - fn expect_block_by_index_set_digest(self) -> BlockHeight { - match self { - UtxoIndexValue::BlockByIndexSetDigest(height) => height, - _ => panic!("Expected BlockByIndexSetDigest found {:?}", self), - } - } -} - -impl RustyUtxoIndex { - /// Returns true iff no blocks have been indexed. - pub(super) async fn is_empty(&self) -> bool { - self.sync_label().await == Default::default() - } - - /// Returns true if the block was already indexed. - pub(crate) async fn block_was_indexed(&self, block_hash: Digest) -> bool { - self.db - .get(UtxoIndexKey::AnnouncementsByBlock(block_hash)) - .await - .is_some() - } - - /// Initialize a UTXO index. Does not apply the genesis block to the index. - pub(super) async fn initialize(data_dir: &DataDirectory) -> Result { - let utxo_index_db_dir_path = data_dir.utxo_index_dir_path(); - DataDirectory::create_dir_if_not_exists(&utxo_index_db_dir_path).await?; - - let utxo_index = NeptuneLevelDb::::new( - &utxo_index_db_dir_path, - &create_db_if_missing(), - ) - .await?; - - let mut utxo_index = RustyUtxoIndex { db: utxo_index }; - - // After initialization a value for sync label must always be set. - if utxo_index.db.get(UtxoIndexKey::SyncLabel).await.is_none() { - utxo_index - .db - .put( - UtxoIndexKey::SyncLabel, - UtxoIndexValue::SyncLabel(Digest::default()), - ) - .await; - } - - Ok(utxo_index) - } - - /// Return the digests of all absolute index sets of the removal records in - /// this block. Returns `None` if the block is not known to this index. - pub(crate) async fn index_set_digests(&self, block_hash: Digest) -> Option> { - let key = UtxoIndexKey::IndexSetDigestsByBlock(block_hash); - self.db - .get(key) - .await - .map(|x| x.expect_index_set_digests_by_block()) - } - - /// Return the block heights for blocks containing announcements matching - /// the [`AnnouncementFlag`]s. The referenced blocks are not guaranteed to - /// be canonical. - /// - /// # Warning - /// - /// For each announcement flag, the returned list is capped in length (for - /// DOS reasons) by [`MAX_NUM_BLOCKS_IN_LOOKUP_LIST`] so extremely active - /// wallets cannot rely on this method for wallet recovery. They should - /// instead use [`UtxoIndexKey::AnnouncementsByBlock`] to scan through - /// each block. - pub(crate) async fn blocks_by_announcement_flags( - &self, - announcement_flags: &HashSet, - ) -> HashSet { - let mut block_heights = HashSet::new(); - for flag in announcement_flags { - let key = UtxoIndexKey::BlocksByAnnouncementFlag(*flag); - let matching_blocks = self - .db - .get(key) - .await - .map(|x| x.expect_blocks_by_announcements()) - .unwrap_or_default(); - block_heights.extend(matching_blocks); - } - - block_heights - } - - /// Return all block heights for blocks containing the requested - /// addition record. The referenced blocks are not guaranteed to be - /// canonical. - /// # Warning - /// - /// For each addition record, the returned list is capped in length (for - /// DOS reasons) by [`MAX_NUM_BLOCKS_IN_LOOKUP_LIST`]. But since addition - /// records are unlikely to be repeated in large numbers, this truncation - /// is probably never met. - pub(crate) async fn blocks_by_addition_record( - &self, - addition_record: AdditionRecord, - ) -> HashSet { - let key = UtxoIndexKey::BlocksByAdditionRecord(addition_record); - let blocks = self - .db - .get(key) - .await - .map(|x| x.expect_blocks_by_addition_records()) - .unwrap_or_default(); - - blocks.into_iter().collect() - } - - /// Return the block height of the block containing the specified - /// transaction input. Any referenced block is not guaranteed to be - /// canonical. - /// - /// Returns `None` if the UTXO index has never seen this absolute index set. - pub(crate) async fn block_by_index_set( - &self, - index_set: &AbsoluteIndexSet, - ) -> Option { - let index_set_digest = Tip5::hash(index_set); - let key = UtxoIndexKey::BlockByIndexSetDigest(index_set_digest); - self.db - .get(key) - .await - .map(|x| x.expect_block_by_index_set_digest()) - } - - /// Add block to UTXO index. Adds all announcements, addition records, and - /// index set digests to the UTXO index. - /// - /// This method is idempotent, meaning that it does not alter the index if - /// the same block is indexed twice, apart from the [`Self::sync_label`] - /// which always points to the latest blocks that was indexed. - pub(crate) async fn index_block(&mut self, block: &Block) { - let hash = block.hash(); - let height = block.header().height; - - let tx_kernel = &block.body().transaction_kernel; - - // Get flags for all announcements in block, removing duplicates - let announcement_flags: HashSet = tx_kernel - .announcements - .iter() - .filter_map(|ann| AnnouncementFlag::try_from(ann).ok()) - .collect(); - - // sort announcement flags to ensure idempotency - let mut announcement_flags = announcement_flags.iter().copied().collect_vec(); - announcement_flags.sort_unstable(); - let mut batch_writes = WriteBatchAsync::new(); - batch_writes.op_write( - UtxoIndexKey::AnnouncementsByBlock(hash), - UtxoIndexValue::AnnouncementsByBlock(announcement_flags.clone()), - ); - - // Loop over all announcement flags to maintain flag to block mapping - for announcement_flag in announcement_flags { - let announcement_flag = UtxoIndexKey::BlocksByAnnouncementFlag(announcement_flag); - let mut block_heights = self - .db - .get(announcement_flag) - .await - .map(|x| x.expect_blocks_by_announcements()) - .unwrap_or_default(); - - // Ensure same block is not added twice, to ensure function's - // idempotency. - if block_heights.contains(&height) { - continue; - } - - // DOS protection: Do not allow list to grow indefinitely as list is - // stored in RAM during this function call. - if block_heights.len() >= MAX_NUM_BLOCKS_IN_LOOKUP_LIST { - warn!( - "List of block heights matching announcement flag exceeds max.\ - Not adding new block to list." - ); - continue; - } - - block_heights.push(height); - - batch_writes.op_write( - announcement_flag, - UtxoIndexValue::BlocksByAnnouncementFlag(block_heights), - ); - } - - // Loop over all addition records to maintain addition record to block - // mapping. - for addition_record in block - .all_addition_records() - .expect("Block must have mutator set update") - { - let addition_record = UtxoIndexKey::BlocksByAdditionRecord(addition_record); - let mut block_heights = self - .db - .get(addition_record) - .await - .map(|x| x.expect_blocks_by_addition_records()) - .unwrap_or_default(); - - // Ensure same block is not added twice, to ensure function's - // idempotency. - if block_heights.contains(&height) { - continue; - } - - // DOS protection: Do not allow list to grow indefinitely as list is - // stored in RAM during this function call. Very unlikely this is - // ever hit. - if block_heights.len() >= MAX_NUM_BLOCKS_IN_LOOKUP_LIST { - warn!( - "List of block heights matching addition record exceeds max.\ - Not adding new block to list." - ); - continue; - } - - block_heights.push(height); - - batch_writes.op_write( - addition_record, - UtxoIndexValue::BlocksByAdditionRecord(block_heights), - ); - } - - // Loop over all inputs to maintain hash(absolute index set) to block - // mapping. - let index_set_digests = tx_kernel - .inputs - .iter() - .map(|rr| Tip5::hash(&rr.absolute_indices)) - .collect_vec(); - for index_set_digest in &index_set_digests { - // All absolute index sets are assumed to be unique, so no - // duplication removal is needed here. - batch_writes.op_write( - UtxoIndexKey::BlockByIndexSetDigest(*index_set_digest), - UtxoIndexValue::BlockByIndexSetDigest(height), - ); - } - - batch_writes.op_write( - UtxoIndexKey::IndexSetDigestsByBlock(hash), - UtxoIndexValue::IndexSetDigestsByBlock(index_set_digests), - ); - - batch_writes.op_write(UtxoIndexKey::SyncLabel, UtxoIndexValue::SyncLabel(hash)); - - self.db.batch_write(batch_writes).await; - } - - /// Return the hash of the latest block indexed. The default value means - /// that no blocks have been indexed. - pub(crate) async fn sync_label(&self) -> Digest { - self.db - .get(UtxoIndexKey::SyncLabel) - .await - .expect("UTXO index must have a SyncLabel set") - .expect_sync_label() - } -} - -impl StorageWriter for RustyUtxoIndex { - async fn persist(&mut self) { - self.db.flush().await; - } - - async fn drop_unpersisted(&mut self) { - unimplemented!("utxo index does not need it") - } -} diff --git a/node/src/state/mod.rs b/node/src/state/mod.rs index 7a253ac..2a6c0d5 100644 --- a/node/src/state/mod.rs +++ b/node/src/state/mod.rs @@ -649,10 +649,6 @@ impl GlobalState { .persist() .await; - if let Some(utxo_index) = &mut self.chain.archival_state_mut().utxo_index { - utxo_index.persist().await; - } - // flush peer_standings self.net.peer_databases.peer_standings_by_ip.flush().await; diff --git a/rpc/core/src/api/ops.rs b/rpc/core/src/api/ops.rs index e114e68..381419d 100644 --- a/rpc/core/src/api/ops.rs +++ b/rpc/core/src/api/ops.rs @@ -43,9 +43,6 @@ pub enum Namespace { /// Endpoints for serving external wallets Wallet, - - /// Endpoints relating to and requiring a UTXO index - Utxoindex, } #[derive( @@ -141,18 +138,6 @@ pub enum RpcMethods { #[namespace(Namespace::Mining)] SubmitBlock, - #[namespace(Namespace::Utxoindex)] - BlockHeightsByFlags, - - #[namespace(Namespace::Utxoindex)] - BlockHeightsByAdditionRecords, - - #[namespace(Namespace::Utxoindex)] - BlockHeightsByAbsoluteIndexSets, - - #[namespace(Namespace::Utxoindex)] - WasMined, - #[namespace(Namespace::Mempool)] Transactions, diff --git a/rpc/core/src/api/rpc.rs b/rpc/core/src/api/rpc.rs index 23f0668..ac2059d 100644 --- a/rpc/core/src/api/rpc.rs +++ b/rpc/core/src/api/rpc.rs @@ -1,5 +1,4 @@ use async_trait::async_trait; -use nyks_standards::wallet::notes::announcement_flag::AnnouncementFlag; use serde::Deserialize; use serde::Serialize; use tasm_lib::prelude::Digest; @@ -365,83 +364,6 @@ pub trait RpcApi: Sync + Send { request: SubmitBlockRequest, ) -> RpcResult; - /* Utxoindex */ - - /// Return block heights for blocks containing announcements with specified - /// announcement flags. May return results from orphaned blocks. - async fn block_heights_by_flags( - &self, - announcement_flags: Vec, - ) -> RpcResult { - self.block_heights_by_flags_call(BlockHeightsByFlagsRequest { announcement_flags }) - .await - } - - async fn block_heights_by_flags_call( - &self, - request: BlockHeightsByFlagsRequest, - ) -> RpcResult; - - /// Return block heights for blocks containing specified addition records. - /// Returned block heights are guaranteed to reference blocks belonging to - /// the canonical chain. - async fn block_heights_by_addition_records( - &self, - addition_records: Vec, - ) -> RpcResult { - self.block_heights_by_addition_records_call(BlockHeightsByAdditionRecordsRequest { - addition_records, - }) - .await - } - - async fn block_heights_by_addition_records_call( - &self, - request: BlockHeightsByAdditionRecordsRequest, - ) -> RpcResult; - - async fn block_heights_by_absolute_index_sets( - &self, - absolute_index_sets: Vec, - ) -> RpcResult { - self.block_heights_by_absolute_index_sets_call(BlockHeightsByAbsoluteIndexSetsRequest { - absolute_index_sets, - }) - .await - } - - /// Return block heights for blocks containing specified absolute index - /// sets. Returned block heights are guaranteed to reference blocks - /// belonging to the canonical chain. - async fn block_heights_by_absolute_index_sets_call( - &self, - request: BlockHeightsByAbsoluteIndexSetsRequest, - ) -> RpcResult; - - /// Return the block heights for blocks matching *all* elements in the - /// specified input/output lists, for blocks belonging to the canonical - /// chain. Will not return block heights were e.g. only one of the outputs - /// was included if more than one output is included in the outputs list. - /// - /// Can return multiple blocks in the case where blocks are selected only - /// based on addition records and multiple blocks contain the same addition - /// records. - /// - /// Returns an error if no filtering conditions are set. - async fn was_mined( - &self, - inputs: Vec, - outputs: Vec, - ) -> RpcResult { - self.was_mined_call(WasMinedRequest { - absolute_index_sets: inputs, - addition_records: outputs, - }) - .await - } - - async fn was_mined_call(&self, request: WasMinedRequest) -> RpcResult; - /* Mempool */ async fn transactions(&self) -> RpcResult { diff --git a/rpc/core/src/model/message.rs b/rpc/core/src/model/message.rs index bb7c7c5..ddf322e 100644 --- a/rpc/core/src/model/message.rs +++ b/rpc/core/src/model/message.rs @@ -12,7 +12,6 @@ use crate::model::block::*; use crate::model::common::*; use crate::model::mining::mempool::RpcMempoolMetadata; use crate::model::mining::template::RpcBlockTemplate; -use crate::model::wallet::RpcAnnouncementFlag; use crate::model::wallet::block::*; use crate::model::wallet::mutator_set::*; use crate::model::wallet::transaction::RpcTransaction; @@ -355,57 +354,6 @@ pub struct SubmitBlockResponse { pub success: bool, } -/* Utxo Index */ - -#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple)] -#[serde(rename_all = "camelCase")] -pub struct BlockHeightsByFlagsRequest { - pub announcement_flags: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct BlockHeightsByFlagsResponse { - pub block_heights: Vec, -} - -#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple)] -#[serde(rename_all = "camelCase")] -pub struct BlockHeightsByAdditionRecordsRequest { - pub addition_records: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct BlockHeightsByAdditionRecordsResponse { - pub block_heights: Vec, -} - -#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple)] -#[serde(rename_all = "camelCase")] -pub struct BlockHeightsByAbsoluteIndexSetsRequest { - pub absolute_index_sets: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct BlockHeightsByAbsoluteIndexSetsResponse { - pub block_heights: Vec, -} - -#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple)] -#[serde(rename_all = "camelCase")] -pub struct WasMinedRequest { - pub absolute_index_sets: Vec, - pub addition_records: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct WasMinedResponse { - pub block_heights: Vec, -} - /* Mempool */ #[derive(Clone, Copy, Debug, Serialize_tuple, Deserialize_tuple)] diff --git a/rpc/core/src/model/wallet/mod.rs b/rpc/core/src/model/wallet/mod.rs index 076025d..07d6569 100644 --- a/rpc/core/src/model/wallet/mod.rs +++ b/rpc/core/src/model/wallet/mod.rs @@ -1,7 +1,3 @@ -use nyks_standards::wallet::notes::announcement_flag::AnnouncementFlag; - pub mod block; pub mod mutator_set; pub mod transaction; - -pub type RpcAnnouncementFlag = AnnouncementFlag; diff --git a/rpc/core/src/model/wallet/transaction.rs b/rpc/core/src/model/wallet/transaction.rs index c334a11..142396d 100644 --- a/rpc/core/src/model/wallet/transaction.rs +++ b/rpc/core/src/model/wallet/transaction.rs @@ -6,8 +6,6 @@ use crate::model::block::transaction_kernel::RpcTransactionKernel; use crate::model::common::RpcBFieldElements; use nyks_consensus::transaction::Transaction; use nyks_consensus::transaction::TransactionProof; -use nyks_consensus::transaction::utxo::Coin; -use nyks_consensus::transaction::utxo::Utxo; use nyks_consensus::transaction::validity::nyks_proof::NyksProof; use nyks_consensus::transaction::validity::proof_collection::ProofCollection; @@ -134,62 +132,3 @@ impl From for RpcTransaction { } } } - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct RpcCoin { - pub type_script_hash: Digest, - pub state: RpcBFieldElements, -} - -impl From for RpcCoin { - fn from(value: Coin) -> Self { - Self { - type_script_hash: value.type_script_hash, - state: value.state.into(), - } - } -} - -impl From<&Coin> for RpcCoin { - fn from(value: &Coin) -> Self { - Self { - type_script_hash: value.type_script_hash, - state: value.state.to_owned().into(), - } - } -} - -impl From for Coin { - fn from(value: RpcCoin) -> Self { - Self { - type_script_hash: value.type_script_hash, - state: value.state.into(), - } - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct RpcUtxo { - lock_script_hash: Digest, - coins: Vec, -} - -impl From for RpcUtxo { - fn from(value: Utxo) -> Self { - Self { - lock_script_hash: value.lock_script_hash(), - coins: value.coins().iter().map(|x| x.into()).collect(), - } - } -} - -impl From for Utxo { - fn from(value: RpcUtxo) -> Self { - Self::new( - value.lock_script_hash, - value.coins.into_iter().map(|x| x.into()).collect(), - ) - } -} diff --git a/standards/src/wallet/keys/address.rs b/standards/src/wallet/keys/address.rs index 2dd0c23..4216ecd 100644 --- a/standards/src/wallet/keys/address.rs +++ b/standards/src/wallet/keys/address.rs @@ -1,7 +1,6 @@ use nyks_consensus::BFieldElement; use nyks_consensus::network::Network; use nyks_consensus::tasm_lib::prelude::Digest; -use nyks_consensus::transaction::announcement::Announcement; use nyks_consensus::transaction::lock_script::LockScript; use serde::Deserialize; use serde::Serialize; @@ -9,7 +8,8 @@ use thiserror::Error; use crate::wallet::keys::schemes::generation::GenerationAddress; use crate::wallet::keys::schemes::symmetric::SymmetricAddress; -use crate::wallet::notes::utxo_notification::UtxoNotificationPayload; +use crate::wallet::notes::content::NoteContent; +use crate::wallet::notes::note::Note; #[derive(Debug, Error)] pub enum Bech32mDecodeError { @@ -54,27 +54,8 @@ pub trait Recipient: Sync { /// the transaction. fn lock_script(&self) -> LockScript; - /// Generates an [Announcement] for a UTXO notification. - /// - /// The announcement typically contains: - /// - /// - type flag - /// - receiver identifier - /// - encrypted payload - /// - /// These fields allow the receiver to determine whether decryption should - /// be attempted. - fn create_note_announcement( - &self, - utxo_notification_payload: &UtxoNotificationPayload, - ) -> Announcement; - - // Generates a redeemable note (the data someone needs to claim and use an UTXO) intended for off-chain use. - fn create_note( - &self, - utxo_notification_payload: &UtxoNotificationPayload, - network: Network, - ) -> String; + /// TODO: comment + fn create_private_note(&self, content: &NoteContent) -> Note; } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -128,21 +109,10 @@ impl Recipient for Address { } } - fn create_note_announcement(&self, payload: &UtxoNotificationPayload) -> Announcement { - match self { - Address::Generation(a) => a.create_note_announcement(payload), - Address::Symmetric(a) => a.create_note_announcement(payload), - } - } - - fn create_note( - &self, - utxo_notification_payload: &UtxoNotificationPayload, - network: Network, - ) -> String { + fn create_private_note(&self, content: &NoteContent) -> Note { match self { - Address::Generation(a) => a.create_note(utxo_notification_payload, network), - Address::Symmetric(a) => a.create_note(utxo_notification_payload, network), + Address::Generation(a) => a.create_private_note(content), + Address::Symmetric(a) => a.create_private_note(content), } } } diff --git a/standards/src/wallet/keys/mod.rs b/standards/src/wallet/keys/mod.rs index 8776149..8333402 100644 --- a/standards/src/wallet/keys/mod.rs +++ b/standards/src/wallet/keys/mod.rs @@ -7,7 +7,7 @@ use sha3::Shake256; use sha3::digest::ExtendableOutput; use sha3::digest::Update; -use crate::wallet::notes::utxo_notification::UtxoNotificationPayload; +use crate::wallet::notes::content::NoteContent; pub mod address; pub mod key; @@ -29,17 +29,21 @@ pub(crate) fn network_hrp_char(network: Network) -> char { /// reuse proofs for tests. These values are used in the encryption /// step. pub(crate) fn deterministically_derive_seed_and_nonce( - payload: &UtxoNotificationPayload, + content: &NoteContent, ) -> ([u8; 32], BFieldElement) { - let combined = Tip5::hash_pair(payload.sender_randomness, payload.utxo.lock_script_hash()); - let [e0, e1, e2, e3, e4] = combined.values(); - let e0: [u8; 8] = e0.into(); - let e1: [u8; 8] = e1.into(); - let e2: [u8; 8] = e2.into(); - let e3: [u8; 8] = e3.into(); - let seed: [u8; 32] = [e0, e1, e2, e3].concat().try_into().unwrap(); + match content { + NoteContent::Utxo(u) => { + let combined = Tip5::hash_pair(u.sender_randomness, Tip5::hash(&u.utxo)); + let [e0, e1, e2, e3, e4] = combined.values(); + let e0: [u8; 8] = e0.into(); + let e1: [u8; 8] = e1.into(); + let e2: [u8; 8] = e2.into(); + let e3: [u8; 8] = e3.into(); + let seed: [u8; 32] = [e0, e1, e2, e3].concat().try_into().unwrap(); - (seed, e4) + (seed, e4) + } + } } // note: copied from twenty_first::math::lattice::kem::shake256() diff --git a/standards/src/wallet/keys/schemes/generation.rs b/standards/src/wallet/keys/schemes/generation.rs index f89c207..67410c4 100644 --- a/standards/src/wallet/keys/schemes/generation.rs +++ b/standards/src/wallet/keys/schemes/generation.rs @@ -9,10 +9,8 @@ use nyks_consensus::BFieldElement; use nyks_consensus::network::Network; use nyks_consensus::tasm_lib::prelude::Digest; use nyks_consensus::tasm_lib::prelude::Tip5; -use nyks_consensus::transaction::announcement::Announcement; use nyks_consensus::transaction::lock_script::LockScript; use nyks_consensus::transaction::lock_script::LockScriptAndWitness; -use nyks_consensus::transaction::utxo::Utxo; use nyks_consensus::twenty_first::math::lattice; use nyks_consensus::twenty_first::math::lattice::kem::CIPHERTEXT_SIZE_IN_BFES; use nyks_consensus::twenty_first::math::lattice::kem::PublicKey; @@ -32,8 +30,9 @@ use crate::wallet::keys::key::Spender; use crate::wallet::keys::network_hrp_char; use crate::wallet::keys::shake256; use crate::wallet::keys::viewing_key::Decryptor; -use crate::wallet::notes::encrypted_utxo_notification::EncryptedUtxoNotification; -use crate::wallet::notes::utxo_notification::UtxoNotificationPayload; +use crate::wallet::notes::content::NoteContent; +use crate::wallet::notes::note::Note; +use crate::wallet::notes::note::PrivateNote; pub(crate) const GENERATION_FLAG_U8: u8 = 79; pub const GENERATION_FLAG: BFieldElement = BFieldElement::new(GENERATION_FLAG_U8 as u64); @@ -60,7 +59,7 @@ impl GenerationAddress { } // Used beneath private_note etc. - fn encrypt(&self, payload: &UtxoNotificationPayload) -> Vec { + fn encrypt(&self, payload: &NoteContent) -> Vec { let (randomness, nonce_bfe) = deterministically_derive_seed_and_nonce(payload); let (shared_key, kem_ctxt) = lattice::kem::enc(self.encryption_key, randomness); @@ -121,31 +120,8 @@ impl Recipient for GenerationAddress { LockScript::standard_hash_lock_from_after_image(self.lock_postimage) } - fn create_note_announcement( - &self, - utxo_notification_payload: &UtxoNotificationPayload, - ) -> Announcement { - let encrypted_utxo_notification = EncryptedUtxoNotification { - flag: GENERATION_FLAG_U8.into(), - receiver_identifier: self.receiver_identifier(), - ciphertext: self.encrypt(utxo_notification_payload), - }; - - encrypted_utxo_notification.into_announcement() - } - - fn create_note( - &self, - utxo_notification_payload: &UtxoNotificationPayload, - network: Network, - ) -> String { - let encrypted_utxo_notification = EncryptedUtxoNotification { - flag: GENERATION_FLAG_U8.into(), - receiver_identifier: self.receiver_identifier(), - ciphertext: self.encrypt(utxo_notification_payload), - }; - - encrypted_utxo_notification.into_bech32m(network) + fn create_private_note(&self, content: &NoteContent) -> Note { + PrivateNote::new(self.receiver_identifier(), self.encrypt(content)).into() } } @@ -211,9 +187,6 @@ pub enum GenerationDecryptError { #[error("Failed to convert BFieldElements to bytes")] BfeToBytes, - - #[error("Deserialization failed")] - Deserialization(#[from] bincode::Error), } impl Zeroize for GenerationKey { @@ -261,7 +234,7 @@ impl Decryptor for GenerationViewingKey { self.privacy_preimage } - fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result<(Utxo, Digest), Self::Error> { + fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result, Self::Error> { // parse ciphertext if ciphertext.len() <= CIPHERTEXT_SIZE_IN_BFES { return Err(GenerationDecryptError::MissingNonce); @@ -296,10 +269,7 @@ impl Decryptor for GenerationViewingKey { .decrypt(nonce, ciphertext_bytes.as_ref()) .map_err(|_| GenerationDecryptError::SymmetricDecryptionFailed)?; - // convert plaintext to utxo and digest - let result = bincode::deserialize(&plaintext)?; // uses #[from] - - Ok(result) + Ok(plaintext) } } diff --git a/standards/src/wallet/keys/schemes/symmetric.rs b/standards/src/wallet/keys/schemes/symmetric.rs index e2ecdd5..05098d2 100644 --- a/standards/src/wallet/keys/schemes/symmetric.rs +++ b/standards/src/wallet/keys/schemes/symmetric.rs @@ -10,10 +10,8 @@ use nyks_consensus::BFieldElement; use nyks_consensus::network::Network; use nyks_consensus::tasm_lib::prelude::Digest; use nyks_consensus::tasm_lib::prelude::Tip5; -use nyks_consensus::transaction::announcement::Announcement; use nyks_consensus::transaction::lock_script::LockScript; use nyks_consensus::transaction::lock_script::LockScriptAndWitness; -use nyks_consensus::transaction::utxo::Utxo; use serde::Deserialize; use serde::Serialize; use thiserror::Error; @@ -29,8 +27,9 @@ use crate::wallet::keys::key::Spender; use crate::wallet::keys::network_hrp_char; use crate::wallet::keys::shake256; use crate::wallet::keys::viewing_key::Decryptor; -use crate::wallet::notes::encrypted_utxo_notification::EncryptedUtxoNotification; -use crate::wallet::notes::utxo_notification::UtxoNotificationPayload; +use crate::wallet::notes::content::NoteContent; +use crate::wallet::notes::note::Note; +use crate::wallet::notes::note::PrivateNote; pub(crate) const SYMMETRIC_FLAG_U8: u8 = 80; pub const SYMMETRIC_FLAG: BFieldElement = BFieldElement::new(SYMMETRIC_FLAG_U8 as u64); @@ -54,15 +53,15 @@ impl SymmetricAddress { hrp } - fn encrypt(&self, payload: &UtxoNotificationPayload) -> Vec { + fn encrypt(&self, content: &NoteContent) -> Vec { // 1. derive nonce deterministically - let (_randomness, nonce_bfe) = deterministically_derive_seed_and_nonce(payload); + let (_randomness, nonce_bfe) = deterministically_derive_seed_and_nonce(content); let nonce_bytes = [&nonce_bfe.value().to_be_bytes(), [0u8; 4].as_slice()].concat(); let nonce = Nonce::from_slice(&nonce_bytes); // 2. serialize payload - let plaintext = bincode::serialize(payload).unwrap(); + let plaintext = bincode::serialize(content).unwrap(); // 3. encrypt let cipher = Aes256Gcm::new(&derive_encryption_secret(&self.receiver_postimage)); @@ -113,31 +112,8 @@ impl Recipient for SymmetricAddress { LockScript::standard_hash_lock_from_after_image(self.lock_postimage) } - fn create_note_announcement( - &self, - utxo_notification_payload: &UtxoNotificationPayload, - ) -> Announcement { - let encrypted_utxo_notification = EncryptedUtxoNotification { - flag: SYMMETRIC_FLAG_U8.into(), - receiver_identifier: self.receiver_identifier(), - ciphertext: self.encrypt(utxo_notification_payload), - }; - - encrypted_utxo_notification.into_announcement() - } - - fn create_note( - &self, - utxo_notification_payload: &UtxoNotificationPayload, - network: Network, - ) -> String { - let encrypted_utxo_notification = EncryptedUtxoNotification { - flag: SYMMETRIC_FLAG_U8.into(), - receiver_identifier: self.receiver_identifier(), - ciphertext: self.encrypt(utxo_notification_payload), - }; - - encrypted_utxo_notification.into_bech32m(network) + fn create_private_note(&self, content: &NoteContent) -> Note { + PrivateNote::new(self.receiver_identifier(), self.encrypt(content)).into() } } @@ -195,9 +171,6 @@ pub enum SymmetricDecryptError { #[error("Decryption failed")] Decryption(#[from] aes_gcm::Error), - - #[error("Deserialization failed")] - Deserialization(#[from] bincode::Error), } impl Zeroize for SymmetricKey { @@ -245,7 +218,7 @@ impl Decryptor for SymmetricViewingKey { self.privacy_preimage } - fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result<(Utxo, Digest), Self::Error> { + fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result, Self::Error> { const NONCE_LEN: usize = 1; if ciphertext.len() <= NONCE_LEN { @@ -263,7 +236,7 @@ impl Decryptor for SymmetricViewingKey { let cipher = Aes256Gcm::new(&self.key); let plaintext = cipher.decrypt(nonce, ciphertext_bytes.as_ref())?; - Ok(bincode::deserialize(&plaintext)?) + Ok(plaintext) } } diff --git a/standards/src/wallet/keys/viewing_key.rs b/standards/src/wallet/keys/viewing_key.rs index 052ed8e..314c5a0 100644 --- a/standards/src/wallet/keys/viewing_key.rs +++ b/standards/src/wallet/keys/viewing_key.rs @@ -1,5 +1,4 @@ use nyks_consensus::BFieldElement; -use nyks_consensus::transaction::utxo::Utxo; use nyks_consensus::twenty_first::tip5::Digest; use zeroize::Zeroize; use zeroize::ZeroizeOnDrop; @@ -19,7 +18,7 @@ pub trait Decryptor { // Needed to extract indices of an UTXO and see if it is/was part of mutator set. fn privacy_preimage(&self) -> Digest; - fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result<(Utxo, Digest), Self::Error>; + fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result, Self::Error>; } #[derive(Debug)] @@ -52,7 +51,7 @@ impl Decryptor for ViewingKey { } } - fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result<(Utxo, Digest), Self::Error> { + fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result, Self::Error> { match self { ViewingKey::Generation(k) => k.decrypt(ciphertext).map_err(ViewingKeyError::Generation), ViewingKey::Symmetric(k) => k.decrypt(ciphertext).map_err(ViewingKeyError::Symmetric), diff --git a/standards/src/wallet/notes/announcement_flag.rs b/standards/src/wallet/notes/announcement_flag.rs deleted file mode 100644 index 881bb10..0000000 --- a/standards/src/wallet/notes/announcement_flag.rs +++ /dev/null @@ -1,69 +0,0 @@ -use std::cmp::Ordering; - -use nyks_consensus::BFieldElement; -use nyks_consensus::transaction::announcement::Announcement; -use serde::Deserialize; -use serde::Serialize; - -use crate::wallet::keys::address::Address; -use crate::wallet::keys::address::Recipient; - -/// Announcement meta-information, intended for use in combination with -/// [`ReceivingAddress`]. Can be used to quickly identify if the announcement -/// relates to a specific [`ReceivingAddress`]. Contains the first two elements -/// of an announcement, as these are interpreted as purpose, receiver ID, -/// respectively. -/// -/// [`ReceivingAddress`]: crate::api::export::ReceivingAddress -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] -pub struct AnnouncementFlag { - /// Purpose of the announcement. E.g.: announcement for generational - /// address, or for symmetric key address. - pub flag: BFieldElement, - - /// An ID identifying the receiver. - pub receiver_id: BFieldElement, -} - -impl Ord for AnnouncementFlag { - // Ordering is implemented to allow for idempotent and deterministic lookup - // tables. - fn cmp(&self, other: &Self) -> Ordering { - match self.flag.value().cmp(&other.flag.value()) { - Ordering::Equal => self.receiver_id.value().cmp(&other.receiver_id.value()), - non_eq => non_eq, - } - } -} - -impl PartialOrd for AnnouncementFlag { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl From<&Address> for AnnouncementFlag { - fn from(address: &Address) -> Self { - Self { - flag: address.flag(), - receiver_id: address.receiver_identifier(), - } - } -} - -impl TryFrom<&Announcement> for AnnouncementFlag { - // Only possible converstion error is that announcement message is too - // short. - type Error = (); - - fn try_from(value: &Announcement) -> Result { - if value.message.len() < 2 { - return Err(()); - } - - Ok(AnnouncementFlag { - flag: value.message[0], - receiver_id: value.message[1], - }) - } -} diff --git a/standards/src/wallet/notes/content.rs b/standards/src/wallet/notes/content.rs new file mode 100644 index 0000000..d99d8d5 --- /dev/null +++ b/standards/src/wallet/notes/content.rs @@ -0,0 +1,98 @@ +use std::fmt::Debug; + +use nyks_consensus::BFieldElement; +use nyks_consensus::transaction::utxo::Utxo; +use nyks_consensus::twenty_first::math::bfield_codec::BFieldCodec; +use nyks_consensus::twenty_first::tip5::Digest; +use serde::Deserialize; +use serde::Serialize; +use thiserror::Error; + +/// Errors that can occur when working with [`NoteContent`]. +#[derive(Debug, Error)] +pub enum NoteContentError { + #[error("empty sequence: expected at least a discriminant element")] + EmptySequence, + + #[error("unknown content discriminant: {0}")] + UnknownDiscriminant(u64), + + // Boxed because each content type has its own BFieldCodec::Error type. + // One Decode variant can then handle all of them. + #[error("failed to decode content: {0}")] + Decode(Box), +} + +/// UTXO notification payload. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, BFieldCodec)] +pub struct UtxoContent { + pub utxo: Utxo, + pub sender_randomness: Digest, +} + +impl UtxoContent { + pub const DISCRIMINANT: u64 = 0; + + pub fn new(utxo: Utxo, sender_randomness: Digest) -> Self { + Self { + utxo, + sender_randomness, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum NoteContent { + Utxo(UtxoContent), +} + +impl NoteContent { + /// Returns the discriminant of the contained content. + pub fn discriminant(&self) -> u64 { + match self { + Self::Utxo(_) => UtxoContent::DISCRIMINANT, + } + } +} + +impl From for NoteContent { + fn from(u: UtxoContent) -> Self { + Self::Utxo(u) + } +} + +impl BFieldCodec for NoteContent { + type Error = NoteContentError; + + fn encode(&self) -> Vec { + let (discriminant, mut payload) = match self { + Self::Utxo(u) => (UtxoContent::DISCRIMINANT, u.encode()), + }; + + let mut out = Vec::with_capacity(1 + payload.len()); + out.push(BFieldElement::new(discriminant)); + out.append(&mut payload); + out + } + + fn decode(sequence: &[BFieldElement]) -> Result, Self::Error> { + let (disc_elem, rest) = sequence + .split_first() + .ok_or(NoteContentError::EmptySequence)?; + let discriminant = disc_elem.value(); + + match discriminant { + d if d == UtxoContent::DISCRIMINANT => { + let content = + UtxoContent::decode(rest).map_err(|e| NoteContentError::Decode(e.into()))?; + Ok(Box::new(Self::Utxo(*content))) + } + _ => Err(NoteContentError::UnknownDiscriminant(discriminant)), + } + } + + fn static_length() -> Option { + // Variable-length: total size depends on which variant is encoded. + None + } +} diff --git a/standards/src/wallet/notes/encrypted_utxo_notification.rs b/standards/src/wallet/notes/encrypted_utxo_notification.rs deleted file mode 100644 index 7e766e9..0000000 --- a/standards/src/wallet/notes/encrypted_utxo_notification.rs +++ /dev/null @@ -1,111 +0,0 @@ -use anyhow::Result; -use anyhow::anyhow; -use anyhow::ensure; -use bech32::FromBase32; -use bech32::ToBase32; -use nyks_consensus::BFieldElement; -use nyks_consensus::network::Network; -use nyks_consensus::transaction::announcement::Announcement; -use nyks_consensus::triton_vm::prelude::BFieldCodec; -use serde::Deserialize; -use serde::Serialize; -use thiserror::Error; - -use crate::wallet::keys::network_hrp_char; - -/// an encrypted wrapper for UTXO notifications. -/// -/// This type is intended to be serialized and actually transferred between -/// parties. -/// -/// note: bech32m encoding of this type is considered standard and is -/// recommended over serde serialization. -/// -/// the receiver_identifier enables the receiver to find the matching -/// `SpendingKey` in their wallet. -#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, BFieldCodec)] -pub struct EncryptedUtxoNotification { - /// Describes the type of encoding used here - pub(crate) flag: BFieldElement, - - /// enables the receiver to find the matching `SpendingKey` in their wallet. - pub(crate) receiver_identifier: BFieldElement, - - /// Encrypted UTXO notification payload. - pub(crate) ciphertext: Vec, -} - -#[derive(Debug, Copy, Clone, Error)] -pub enum ConversionFromMessageError { - #[error("message too short: length is {0}, minimum required is 2")] - MessageTooShort(usize), -} - -impl EncryptedUtxoNotification { - fn into_message(self) -> Vec { - [vec![self.flag, self.receiver_identifier], self.ciphertext].concat() - } - - fn from_message(message: Vec) -> Result { - if message.len() < 2 { - Err(ConversionFromMessageError::MessageTooShort(message.len())) - } else { - Ok(Self { - flag: message[0], - receiver_identifier: message[1], - ciphertext: message[2..].to_vec(), - }) - } - } - - /// Convert an encrypted UTXO notification to a announcement. Leaks - /// privacy in the form of `receiver_identifier` is addresses are reused. - /// Never leaks actual UTXO info such as amount transferred. - pub(crate) fn into_announcement(self) -> Announcement { - // We could use `BfieldCodec` encode here. But it might be a bit faster - // to filter out irrelevant announcement if we don't have to - // attempt a decoding to a specific data type first but can instead just - // read out b-field elements and skip items based on that. - Announcement::new(self.into_message()) - } - - pub fn into_bech32m(self, network: Network) -> String { - let hrp = Self::get_hrp(network); - let message = self.into_message(); - let payload = bincode::serialize(&message).unwrap_or_else(|e| { - panic!("Serialization shouldn't fail. Message was: {message:?}\nerror: {e}") - }); - let payload_base_32 = payload.to_base32(); - let variant = bech32::Variant::Bech32m; - bech32::encode(&hrp, payload_base_32, variant).unwrap_or_else(|e| panic!( - "bech32 encoding shouldn't fail. Arguments were:\n\n{hrp}\n\n{payload:?}\n\n{variant:?}\n\nerror: {e}" - )) - } - - /// decodes from a bech32m string and verifies it matches `network` - pub fn from_bech32m(encoded: &str, network: Network) -> Result { - let (hrp, data, variant) = bech32::decode(encoded)?; - - ensure!( - variant == bech32::Variant::Bech32m, - "Can only decode bech32m addresses." - ); - ensure!( - hrp == *Self::get_hrp(network), - "Could not decode bech32m address because of invalid prefix", - ); - - let payload = Vec::::from_base32(&data)?; - let message = bincode::deserialize(&payload) - .map_err(|e| anyhow!("Could not decode bech32m because of error: {e}"))?; - let encrypted_utxo_notification = Self::from_message(message) - .map_err(|e| anyhow!("conversion from bech32m failed: {e}"))?; - - Ok(encrypted_utxo_notification) - } - - /// returns human readable prefix (hrp) of a utxo-transfer-encrypted, specific to `network` - pub(crate) fn get_hrp(network: Network) -> String { - format!("utxo{}", network_hrp_char(network)) - } -} diff --git a/standards/src/wallet/notes/mod.rs b/standards/src/wallet/notes/mod.rs index 8ce44e4..b9491d2 100644 --- a/standards/src/wallet/notes/mod.rs +++ b/standards/src/wallet/notes/mod.rs @@ -1,3 +1,2 @@ -pub mod announcement_flag; -pub mod encrypted_utxo_notification; -pub mod utxo_notification; +pub mod content; +pub mod note; diff --git a/standards/src/wallet/notes/note.rs b/standards/src/wallet/notes/note.rs new file mode 100644 index 0000000..97d8436 --- /dev/null +++ b/standards/src/wallet/notes/note.rs @@ -0,0 +1,172 @@ +use anyhow::Result; +use anyhow::bail; +use anyhow::ensure; +use bech32::FromBase32; +use bech32::ToBase32; +use nyks_consensus::BFieldElement; +use nyks_consensus::network::Network; +use nyks_consensus::transaction::announcement::Announcement; +use nyks_consensus::twenty_first::math::bfield_codec::BFieldCodec; +use serde::Deserialize; +use serde::Serialize; + +use crate::wallet::keys::network_hrp_char; +use crate::wallet::notes::content::NoteContent; + +pub(crate) const TAG_PUBLIC: u64 = 0; +pub(crate) const TAG_PRIVATE: u64 = 1; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PublicNote { + pub receiver_id: BFieldElement, + pub content: NoteContent, +} + +impl PublicNote { + pub fn new(receiver_id: BFieldElement, content: NoteContent) -> Self { + Self { + receiver_id, + content, + } + } + + pub fn into_message(self) -> Vec { + let mut msg = vec![BFieldElement::new(TAG_PUBLIC), self.receiver_id]; + msg.extend(self.content.encode()); + msg + } + + pub fn from_message(data: &[BFieldElement]) -> Result { + if data.len() < 2 { + bail!("Public note too short"); + } + if data[0].value() != TAG_PUBLIC { + bail!("Expected public tag, got {}", data[0].value()); + } + let receiver_id = data[1]; + let content = *NoteContent::decode(&data[2..])?; + Ok(Self { + receiver_id, + content, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PrivateNote { + pub receiver_id: BFieldElement, + pub ciphertext: Vec, +} + +impl PrivateNote { + pub fn new(receiver_id: BFieldElement, ciphertext: Vec) -> Self { + Self { + receiver_id, + ciphertext, + } + } + + pub fn into_message(self) -> Vec { + let mut msg = vec![BFieldElement::new(TAG_PRIVATE), self.receiver_id]; + msg.extend(self.ciphertext); + msg + } + + pub fn from_message(data: &[BFieldElement]) -> Result { + if data.len() < 2 { + bail!("Private note too short"); + } + if data[0].value() != TAG_PRIVATE { + bail!("Expected private tag, got {}", data[0].value()); + } + let receiver_id = data[1]; + let ciphertext = data[2..].to_vec(); + Ok(Self { + receiver_id, + ciphertext, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum Note { + Public(PublicNote), + Private(PrivateNote), +} + +impl Note { + pub fn receiver_id(&self) -> BFieldElement { + match self { + Self::Public(p) => p.receiver_id, + Self::Private(p) => p.receiver_id, + } + } + + pub fn is_public(&self) -> bool { + matches!(self, Self::Public(_)) + } + + pub fn is_private(&self) -> bool { + matches!(self, Self::Private(_)) + } + + pub fn into_announcement(self) -> Announcement { + let msg = match self { + Self::Public(p) => p.into_message(), + Self::Private(p) => p.into_message(), + }; + Announcement::new(msg) + } + + pub fn try_from_announcement(ann: &Announcement) -> Result { + let msg = &ann.message; + if msg.is_empty() { + bail!("Empty announcement"); + } + match msg[0].value() { + TAG_PUBLIC => Ok(Self::Public(PublicNote::from_message(msg)?)), + TAG_PRIVATE => Ok(Self::Private(PrivateNote::from_message(msg)?)), + other => bail!("Unknown tag: {other}"), + } + } + + pub fn into_bech32m(self, network: Network) -> String { + let hrp = Self::hrp(network); + let msg = self.into_announcement().message; + let payload = + bincode::serialize(&msg).expect("BFieldElement vec serialization never fails"); + let payload_base32 = payload.to_base32(); + bech32::encode(&hrp, payload_base32, bech32::Variant::Bech32m) + .expect("bech32m encoding never fails") + } + + pub fn from_bech32m(encoded: &str, network: Network) -> Result { + let (hrp, data, variant) = bech32::decode(encoded)?; + ensure!( + variant == bech32::Variant::Bech32m, + "Only bech32m is supported" + ); + ensure!(hrp == Self::hrp(network), "Invalid HRP for network"); + let payload = Vec::::from_base32(&data)?; + let msg: Vec = bincode::deserialize(&payload) + .map_err(|e| anyhow::anyhow!("Failed to deserialize bech32 payload: {e}"))?; + let ann = Announcement::new(msg); + Self::try_from_announcement(&ann) + } + + fn hrp(network: Network) -> String { + format!("note{}", network_hrp_char(network)) + } +} + +impl From for Note { + fn from(n: PublicNote) -> Self { + Self::Public(n) + } +} + +impl From for Note { + fn from(n: PrivateNote) -> Self { + Self::Private(n) + } +} diff --git a/standards/src/wallet/notes/utxo_notification.rs b/standards/src/wallet/notes/utxo_notification.rs deleted file mode 100644 index e598ea2..0000000 --- a/standards/src/wallet/notes/utxo_notification.rs +++ /dev/null @@ -1,36 +0,0 @@ -use nyks_consensus::transaction::utxo::Utxo; -use nyks_consensus::twenty_first::tip5::Digest; -use serde::Deserialize; -use serde::Serialize; - -use crate::wallet::keys::address::Address; - -/// The payload of a UTXO notification, containing all information necessary -/// to claim it, provided that the decryptor already has access to the -/// associated spending key. -/// -/// future work: -/// we should consider adding functionality that would facilitate passing -/// these payloads from sender to receiver off-chain for lower-fee transfers -/// between trusted parties or eg wallets owned by the same person/org. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct UtxoNotificationPayload { - pub(crate) utxo: Utxo, - pub(crate) sender_randomness: Digest, -} - -impl UtxoNotificationPayload { - pub fn new(utxo: Utxo, sender_randomness: Digest) -> Self { - Self { - utxo, - sender_randomness, - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct PrivateNotificationData { - pub cleartext: UtxoNotificationPayload, - pub ciphertext: String, - pub recipient_address: Address, -} diff --git a/wallet/core/src/transaction/builder/output.rs b/wallet/core/src/transaction/builder/output.rs index ae72ab1..d910cbd 100644 --- a/wallet/core/src/transaction/builder/output.rs +++ b/wallet/core/src/transaction/builder/output.rs @@ -3,21 +3,18 @@ use std::ops::Deref; use std::ops::DerefMut; -use nyks_consensus::tasm_lib::prelude::Digest; -use nyks_standards::wallet::keys::address::Address; -use nyks_standards::wallet::keys::address::Recipient; -use nyks_standards::wallet::notes::utxo_notification::PrivateNotificationData; -use nyks_standards::wallet::notes::utxo_notification::UtxoNotificationPayload; -use serde::Deserialize; -use serde::Serialize; - use nyks_consensus::mutator_set::addition_record::AdditionRecord; -use nyks_consensus::network::Network; use nyks_consensus::proof_abstractions::timestamp::Timestamp; +use nyks_consensus::tasm_lib::prelude::Digest; use nyks_consensus::transaction::announcement::Announcement; use nyks_consensus::transaction::utxo::Utxo; use nyks_consensus::transaction::utxo_triple::UtxoTriple; use nyks_consensus::type_scripts::native_currency_amount::NativeCurrencyAmount; +use nyks_standards::wallet::keys::address::Address; +use nyks_standards::wallet::keys::address::Recipient; +use nyks_standards::wallet::notes::content::UtxoContent; +use serde::Deserialize; +use serde::Serialize; use crate::transaction::utxo::notifications::UtxoNotificationMedium; use crate::transaction::utxo::notifications::UtxoNotificationMethod; @@ -56,8 +53,8 @@ impl TxOutput { } } - fn notification_payload(&self) -> UtxoNotificationPayload { - UtxoNotificationPayload::new(self.utxo(), self.sender_randomness()) + fn note_content(&self) -> UtxoContent { + UtxoContent::new(self.utxo(), self.sender_randomness()) } /// retrieve native currency amount @@ -197,29 +194,19 @@ impl TxOutput { } /// Retrieve on-chain UTXO notification announcement, if any. + /// TODO: redesign this potentially pub fn announcement(&self) -> Option { match &self.notification_method { UtxoNotificationMethod::None => None, UtxoNotificationMethod::OffChain(_) => None, UtxoNotificationMethod::OnChain(receiving_address) => { - let notification_payload = self.notification_payload(); - Some(receiving_address.create_note_announcement(¬ification_payload)) - } - } - } - - pub(crate) fn offchain_notification(&self, network: Network) -> Option<(String, Address)> { - match &self.notification_method { - UtxoNotificationMethod::OnChain(_) => None, - UtxoNotificationMethod::OffChain(recipient) => { - let notification_payload = self.notification_payload(); - - Some(( - recipient.create_note(¬ification_payload, network), - recipient.to_owned(), - )) + let utxo_content = self.note_content(); + Some( + receiving_address + .create_private_note(&utxo_content.into()) + .into_announcement(), + ) } - UtxoNotificationMethod::None => None, } } @@ -343,23 +330,6 @@ impl TxOutputList { announcements } - pub fn offchain_notifications( - &self, - network: Network, - ) -> impl Iterator + use<'_> { - self.0.iter().filter_map(move |tx_output| { - if let Some((ciphertext, receiver_address)) = tx_output.offchain_notification(network) { - Some(PrivateNotificationData { - cleartext: tx_output.notification_payload(), - ciphertext, - recipient_address: receiver_address, - }) - } else { - None - } - }) - } - /// indicates if any offchain notifications exist pub fn has_offchain(&self) -> bool { self.0.iter().any(|u| u.is_offchain()) diff --git a/wallet/sdk/Cargo.toml b/wallet/sdk/Cargo.toml index e17d3c6..8552354 100644 --- a/wallet/sdk/Cargo.toml +++ b/wallet/sdk/Cargo.toml @@ -18,6 +18,7 @@ num-traits = "0.2.19" serde = "1.0.228" thiserror = "1.0.65" zeroize = "1.8.1" +bincode = "1.3" tokio = { version = "1.45.1", features = ["sync"] } [dev-dependencies] diff --git a/wallet/sdk/src/scanners/chain.rs b/wallet/sdk/src/scanners/chain.rs index 438a969..1d5a700 100644 --- a/wallet/sdk/src/scanners/chain.rs +++ b/wallet/sdk/src/scanners/chain.rs @@ -269,9 +269,16 @@ impl ChainScanner { }) .filter_map(|a| extract_ciphertext(a)) .filter_map(|ciphertext| key.decrypt(&ciphertext).ok()) - .map(|(utxo, sender_randomness)| { + .map(|decrypted: Vec| { + let (utxo, sender_randomness): (Utxo, Digest) = + bincode::deserialize(&decrypted).unwrap(); (utxo, sender_randomness, key.privacy_preimage()) }) + // A third party can create an announcement that decrypts under this key but + // contains a UTXO that is unspendable by us. + .filter(|(utxo, _, _)| { + utxo.lock_script_hash() == key.address().lock_script().hash() + }) .collect::>() }) .collect()