From f9b15f57e672eed049586e0a9dcf72d58ab02515 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Mon, 21 Mar 2022 12:44:39 +0700 Subject: [PATCH 01/25] Simultaneous activation of coins in two modes (Iguana and Trezor) * Refactor `CryptoCtx` to be a structure that contains `IguanaArc` always and a constructible `HwArc` * Add `init_trezor` RPC call that initializes `HwArc` * Rename `KeyPairCtx` to `IguanaCtx` * Comment out `mm_init_task`, `rpc_command` modules, `mm_init_status`, `mm_init_user_action` RPC calls --- Cargo.lock | 1 + Cargo.toml | 1 + mm2src/coins/eth/eth_wasm_tests.rs | 2 +- mm2src/coins/hd_pubkey.rs | 34 ++-- mm2src/coins/hd_wallet.rs | 8 +- mm2src/coins/hd_wallet_storage/mod.rs | 1 + mm2src/coins/init_create_account.rs | 2 +- mm2src/coins/lp_coins.rs | 46 +++-- mm2src/coins/utxo.rs | 20 +- .../utxo/utxo_builder/utxo_coin_builder.rs | 55 ++++-- mm2src/coins/utxo/utxo_common.rs | 7 +- mm2src/coins/utxo/utxo_withdraw.rs | 105 ++++------- .../standalone_coin/init_standalone_coin.rs | 11 +- .../src/utxo_activation/common_impl.rs | 12 +- .../utxo_activation/init_qtum_activation.rs | 10 +- .../init_utxo_standard_activation.rs | 10 +- mm2src/crypto/Cargo.toml | 1 + mm2src/crypto/src/crypto_ctx.rs | 126 ++++++------- mm2src/crypto/src/hw_ctx.rs | 44 +++-- mm2src/crypto/src/key_pair_ctx.rs | 18 +- mm2src/crypto/src/lib.rs | 4 +- mm2src/lp_init/init_context.rs | 15 +- mm2src/lp_init/init_hw.rs | 175 ++++++++++++++++++ mm2src/lp_native_dex.rs | 32 +--- mm2src/rpc/dispatcher/dispatcher.rs | 9 +- 25 files changed, 474 insertions(+), 275 deletions(-) create mode 100644 mm2src/lp_init/init_hw.rs diff --git a/Cargo.lock b/Cargo.lock index 15e3469c42..0c8b2c7012 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1171,6 +1171,7 @@ checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" name = "crypto" version = "0.1.0" dependencies = [ + "async-std", "async-trait", "bip32", "bitcrypto", diff --git a/Cargo.toml b/Cargo.toml index c2fd24f6c6..f407011530 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -144,6 +144,7 @@ regex = "1" [workspace] members = [ "mm2src/coins", + "mm2src/common/shared_ref_counter", "mm2src/coins/lightning_persister", "mm2src/coins/lightning_background_processor", "mm2src/coins/utxo_signer", diff --git a/mm2src/coins/eth/eth_wasm_tests.rs b/mm2src/coins/eth/eth_wasm_tests.rs index c57269552d..3edb58d0bc 100644 --- a/mm2src/coins/eth/eth_wasm_tests.rs +++ b/mm2src/coins/eth/eth_wasm_tests.rs @@ -78,7 +78,7 @@ async fn test_init_eth_coin() { }); let ctx = MmCtxBuilder::new().with_conf(conf).into_mm_arc(); - CryptoCtx::init_with_passphrase( + CryptoCtx::init_with_iguana_passphrase( ctx.clone(), "spice describe gravity federal blast come thank unfair canal monkey style afraid", ) diff --git a/mm2src/coins/hd_pubkey.rs b/mm2src/coins/hd_pubkey.rs index ae019a4a86..d5a6402d40 100644 --- a/mm2src/coins/hd_pubkey.rs +++ b/mm2src/coins/hd_pubkey.rs @@ -13,7 +13,7 @@ use std::convert::TryInto; #[derive(Clone)] pub enum HDExtractPubkeyError { - HDWalletUnavailable, + HwContextNotInitialized, CoinDoesntSupportTrezor, RpcTaskError(RpcTaskError), HardwareWalletError(HwError), @@ -54,7 +54,7 @@ impl From> for HDExtractPubkeyError { impl From for NewAccountCreatingError { fn from(e: HDExtractPubkeyError) -> Self { match e { - HDExtractPubkeyError::HDWalletUnavailable => NewAccountCreatingError::HDWalletUnavailable, + HDExtractPubkeyError::HwContextNotInitialized => NewAccountCreatingError::HwContextNotInitialized, HDExtractPubkeyError::CoinDoesntSupportTrezor => NewAccountCreatingError::CoinDoesntSupportTrezor, HDExtractPubkeyError::RpcTaskError(rpc) => NewAccountCreatingError::RpcTaskError(rpc), HDExtractPubkeyError::HardwareWalletError(hw) => NewAccountCreatingError::HardwareWalletError(hw), @@ -69,7 +69,7 @@ impl From for NewAccountCreatingError { impl From for HDWalletRpcError { fn from(e: HDExtractPubkeyError) -> Self { match e { - HDExtractPubkeyError::HDWalletUnavailable => HDWalletRpcError::CoinIsActivatedNotWithHDWallet, + HDExtractPubkeyError::HwContextNotInitialized => HDWalletRpcError::HwContextNotInitialized, HDExtractPubkeyError::CoinDoesntSupportTrezor => HDWalletRpcError::CoinDoesntSupportTrezor, HDExtractPubkeyError::RpcTaskError(rpc) => HDWalletRpcError::from(rpc), HDExtractPubkeyError::HardwareWalletError(hw) => HDWalletRpcError::from(hw), @@ -138,30 +138,30 @@ where Task: RpcTask, Task::UserAction: TryInto + Send, { - pub fn new( + pub async fn new( ctx: &MmArc, task_handle: &'task RpcTaskHandle, statuses: HwConnectStatuses, - ) -> MmResult { + ) -> MmResult, HDExtractPubkeyError> { let crypto_ctx = CryptoCtx::from_ctx(ctx)?; - // Don't use [`CryptoCtx::hw_ctx`] because we are planning to support HD master key. - match *crypto_ctx { - CryptoCtx::HardwareWallet(ref hw_ctx) => Ok(RpcTaskXPubExtractor::Trezor { - hw_ctx: hw_ctx.clone(), - task_handle, - statuses, - }), - CryptoCtx::KeyPair(_) => MmError::err(HDExtractPubkeyError::HDWalletUnavailable), - } + let hw_ctx = crypto_ctx + .hw_ctx() + .await + .or_mm_err(|| HDExtractPubkeyError::HwContextNotInitialized)?; + Ok(RpcTaskXPubExtractor::Trezor { + hw_ctx, + task_handle, + statuses, + }) } /// Constructs an Xpub extractor without checking if the MarketMaker is initialized with a hardware wallet. - pub fn new_unchecked( + pub async fn new_unchecked( ctx: &MmArc, task_handle: &'task RpcTaskHandle, statuses: HwConnectStatuses, - ) -> XPubExtractorUnchecked { - XPubExtractorUnchecked(Self::new(ctx, task_handle, statuses)) + ) -> XPubExtractorUnchecked> { + XPubExtractorUnchecked(Self::new(ctx, task_handle, statuses).await) } async fn extract_utxo_xpub_from_trezor( diff --git a/mm2src/coins/hd_wallet.rs b/mm2src/coins/hd_wallet.rs index 69f5d820ae..118f7ebf53 100644 --- a/mm2src/coins/hd_wallet.rs +++ b/mm2src/coins/hd_wallet.rs @@ -82,6 +82,8 @@ impl From for NewAddressDerivingError { #[derive(Display)] pub enum NewAccountCreatingError { + #[display(fmt = "Hardware Wallet context is not initialized")] + HwContextNotInitialized, #[display(fmt = "HD wallet is unavailable")] HDWalletUnavailable, #[display( @@ -120,6 +122,7 @@ impl From for NewAccountCreatingError { impl From for HDWalletRpcError { fn from(e: NewAccountCreatingError) -> Self { match e { + NewAccountCreatingError::HwContextNotInitialized => HDWalletRpcError::HwContextNotInitialized, NewAccountCreatingError::HDWalletUnavailable => HDWalletRpcError::CoinIsActivatedNotWithHDWallet, NewAccountCreatingError::CoinDoesntSupportTrezor => HDWalletRpcError::CoinDoesntSupportTrezor, NewAccountCreatingError::RpcTaskError(rpc) => HDWalletRpcError::from(rpc), @@ -192,9 +195,11 @@ pub enum HDWalletRpcError { /* */ /* ----------- HD Wallet RPC error ------------ */ /* */ + #[display(fmt = "Hardware Wallet context is not initialized")] + HwContextNotInitialized, #[display(fmt = "No such coin {}", coin)] NoSuchCoin { coin: String }, - #[display(fmt = "Withdraw timed out {:?}", _0)] + #[display(fmt = "RPC timed out {:?}", _0)] Timeout(Duration), #[display(fmt = "Coin is expected to be activated with the HD wallet derivation method")] CoinIsActivatedNotWithHDWallet, @@ -304,6 +309,7 @@ impl HttpStatusCode for HDWalletRpcError { fn status_code(&self) -> StatusCode { match self { HDWalletRpcError::CoinDoesntSupportTrezor + | HDWalletRpcError::HwContextNotInitialized | HDWalletRpcError::NoSuchCoin { .. } | HDWalletRpcError::CoinIsActivatedNotWithHDWallet | HDWalletRpcError::UnknownAccount { .. } diff --git a/mm2src/coins/hd_wallet_storage/mod.rs b/mm2src/coins/hd_wallet_storage/mod.rs index 5baaf700bc..32301d8fea 100644 --- a/mm2src/coins/hd_wallet_storage/mod.rs +++ b/mm2src/coins/hd_wallet_storage/mod.rs @@ -226,6 +226,7 @@ impl HDWalletCoinStorage { let crypto_ctx = CryptoCtx::from_ctx(ctx)?; let hd_wallet_rmd160 = crypto_ctx .hd_wallet_rmd160() + .await .or_mm_err(|| HDWalletStorageError::HDWalletUnavailable)?; Ok(HDWalletCoinStorage { coin, diff --git a/mm2src/coins/init_create_account.rs b/mm2src/coins/init_create_account.rs index 39d56addba..eac422e7c2 100644 --- a/mm2src/coins/init_create_account.rs +++ b/mm2src/coins/init_create_account.rs @@ -93,7 +93,7 @@ impl RpcTask for InitCreateAccountTask { on_pin_request: CreateAccountAwaitingStatus::WaitForTrezorPin, on_ready: CreateAccountInProgressStatus::RequestingAccountBalance, }; - let xpub_extractor = CreateAccountXPubExtractor::new(ctx, task_handle, hw_statuses)?; + let xpub_extractor = CreateAccountXPubExtractor::new(ctx, task_handle, hw_statuses).await?; coin.init_create_account_rpc(params, &xpub_extractor).await } diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 4208ca4ee1..2192a5d66b 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -1578,17 +1578,34 @@ impl CoinsContext { } } +/// This enum is used in coin activation requests. +#[derive(Copy, Clone, Debug, Deserialize, Serialize)] +pub enum PrivKeyActivationPolicy { + IguanaPrivKey, + Trezor, +} + +impl PrivKeyActivationPolicy { + /// The function can be used as a default deserialization constructor: + /// `#[serde(default = "PrivKeyActivationPolicy::iguana_priv_key")]` + pub fn iguana_priv_key() -> PrivKeyActivationPolicy { PrivKeyActivationPolicy::IguanaPrivKey } + + /// The function can be used as a default deserialization constructor: + /// `#[serde(default = "PrivKeyActivationPolicy::trezor")]` + pub fn trezor() -> PrivKeyActivationPolicy { PrivKeyActivationPolicy::Trezor } +} + #[derive(Debug)] pub enum PrivKeyPolicy { KeyPair(T), - HardwareWallet, + Trezor, } impl PrivKeyPolicy { pub fn key_pair(&self) -> Option<&T> { match self { PrivKeyPolicy::KeyPair(key_pair) => Some(key_pair), - PrivKeyPolicy::HardwareWallet => None, + PrivKeyPolicy::Trezor => None, } } @@ -1601,17 +1618,12 @@ impl PrivKeyPolicy { #[derive(Clone)] pub enum PrivKeyBuildPolicy<'a> { IguanaPrivKey(&'a [u8]), - HardwareWallet, + Trezor, } impl<'a> PrivKeyBuildPolicy<'a> { - pub fn from_crypto_ctx(crypto_ctx: &'a CryptoCtx) -> PrivKeyBuildPolicy<'a> { - match crypto_ctx { - CryptoCtx::KeyPair(key_pair_ctx) => { - PrivKeyBuildPolicy::IguanaPrivKey(key_pair_ctx.secp256k1_privkey_bytes()) - }, - CryptoCtx::HardwareWallet(_) => PrivKeyBuildPolicy::HardwareWallet, - } + pub fn iguana_priv_key(crypto_ctx: &'a CryptoCtx) -> Self { + PrivKeyBuildPolicy::IguanaPrivKey(crypto_ctx.iguana_ctx().secp256k1_privkey_bytes()) } } @@ -1658,6 +1670,10 @@ pub trait CoinWithDerivationMethod { type HDWallet; fn derivation_method(&self) -> &DerivationMethod; + + fn has_hd_wallet_derivation_method(&self) -> bool { + matches!(self.derivation_method(), DerivationMethod::HDWallet(_)) + } } #[allow(clippy::upper_case_acronyms)] @@ -1869,12 +1885,10 @@ pub async fn lp_coininit(ctx: &MmArc, ticker: &str, req: &Json) -> Result key_pair_ctx.secp256k1_privkey_bytes().to_vec(), - CryptoCtx::HardwareWallet(_) => { - return ERR!("'enable' and 'electrum' RPC calls don't support coins activation with a Hardware Wallet") - }, - }; + let secret = try_s!(CryptoCtx::from_ctx(ctx)) + .iguana_ctx() + .secp256k1_privkey_bytes() + .to_vec(); if coins_en["protocol"].is_null() { return ERR!( diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 725211b6cb..afe0b563af 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -89,9 +89,10 @@ use self::rpc_clients::{electrum_script_hash, ElectrumClient, ElectrumRpcRequest NativeClient, UnspentInfo, UtxoRpcClientEnum, UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; use super::{BalanceError, BalanceFut, BalanceResult, CoinsContext, DerivationMethod, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, KmdRewardsDetails, MarketCoinOps, MmCoin, NumConversError, NumConversResult, - PrivKeyNotAllowed, PrivKeyPolicy, RpcTransportEventHandler, RpcTransportEventHandlerShared, TradeFee, - TradePreimageError, TradePreimageFut, TradePreimageResult, Transaction, TransactionDetails, - TransactionEnum, TransactionFut, UnexpectedDerivationMethod, WithdrawError, WithdrawRequest}; + PrivKeyActivationPolicy, PrivKeyNotAllowed, PrivKeyPolicy, RpcTransportEventHandler, + RpcTransportEventHandlerShared, TradeFee, TradePreimageError, TradePreimageFut, TradePreimageResult, + Transaction, TransactionDetails, TransactionEnum, TransactionFut, UnexpectedDerivationMethod, + WithdrawError, WithdrawRequest}; use crate::coin_balance::{EnableCoinScanPolicy, HDAddressBalanceScanner}; use crate::hd_wallet::{HDAccountOps, HDAccountsMutex, HDAddress, HDWalletCoinOps, HDWalletOps, InvalidBip44ChainError}; use crate::hd_wallet_storage::{HDAccountStorageItem, HDWalletCoinStorage, HDWalletStorageError, HDWalletStorageResult}; @@ -1043,6 +1044,8 @@ pub struct UtxoActivationParams { pub gap_limit: Option, #[serde(default)] pub scan_policy: EnableCoinScanPolicy, + #[serde(default = "PrivKeyActivationPolicy::iguana_priv_key")] + pub priv_key_policy: PrivKeyActivationPolicy, /// The flag determines whether to use mature unspent outputs *only* to generate transactions. /// https://github.com/KomodoPlatform/atomicDEX-API/issues/1181 pub check_utxo_maturity: Option, @@ -1057,6 +1060,8 @@ pub enum UtxoFromLegacyReqErr { InvalidRequiresNota(json::Error), InvalidAddressFormat(json::Error), InvalidCheckUtxoMaturity(json::Error), + InvalidScanPolicy(json::Error), + InvalidPrivKeyPolicy(json::Error), } impl UtxoActivationParams { @@ -1082,7 +1087,12 @@ impl UtxoActivationParams { json::from_value(req["address_format"].clone()).map_to_mm(UtxoFromLegacyReqErr::InvalidAddressFormat)?; let check_utxo_maturity = json::from_value(req["check_utxo_maturity"].clone()) .map_to_mm(UtxoFromLegacyReqErr::InvalidCheckUtxoMaturity)?; - let scan_policy = EnableCoinScanPolicy::default(); + let scan_policy = json::from_value::>(req["scan_policy"].clone()) + .map_to_mm(UtxoFromLegacyReqErr::InvalidScanPolicy)? + .unwrap_or_default(); + let priv_key_policy = json::from_value::>(req["priv_key_policy"].clone()) + .map_to_mm(UtxoFromLegacyReqErr::InvalidPrivKeyPolicy)? + .unwrap_or(PrivKeyActivationPolicy::IguanaPrivKey); Ok(UtxoActivationParams { mode, @@ -1093,6 +1103,7 @@ impl UtxoActivationParams { address_format, gap_limit: None, scan_policy, + priv_key_policy, check_utxo_maturity, }) } @@ -1482,6 +1493,7 @@ pub fn address_by_conf_and_pubkey_str( address_format: None, gap_limit: None, scan_policy: EnableCoinScanPolicy::default(), + priv_key_policy: PrivKeyActivationPolicy::IguanaPrivKey, check_utxo_maturity: None, }; let conf_builder = UtxoConfBuilder::new(conf, ¶ms, coin); diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 332114588e..9642d51c64 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -4,7 +4,8 @@ use crate::utxo::rpc_clients::{ElectrumClient, ElectrumClientImpl, ElectrumRpcRe UtxoRpcClientEnum}; use crate::utxo::utxo_builder::utxo_conf_builder::{UtxoConfBuilder, UtxoConfError, UtxoConfResult}; use crate::utxo::{output_script, utxo_common, ElectrumBuilderArgs, ElectrumProtoVerifier, RecentlySpentOutPoints, - TxFee, UtxoCoinFields, UtxoHDAccount, UtxoHDWallet, UtxoRpcMode, DEFAULT_GAP_LIMIT, UTXO_DUST_AMOUNT}; + TxFee, UtxoCoinConf, UtxoCoinFields, UtxoHDAccount, UtxoHDWallet, UtxoRpcMode, DEFAULT_GAP_LIMIT, + UTXO_DUST_AMOUNT}; use crate::{BlockchainNetwork, CoinTransportMetrics, DerivationMethod, HistorySyncState, PrivKeyBuildPolicy, PrivKeyPolicy, RpcClientType, UtxoActivationParams}; use async_trait::async_trait; @@ -13,8 +14,7 @@ use common::executor::{spawn, Timer}; use common::mm_ctx::MmArc; use common::mm_error::prelude::*; use common::small_rng; -use crypto::trezor::TrezorError; -use crypto::{Bip32DerPathError, Bip44DerPathError, Bip44PathToCoin, CryptoInitError, HwError}; +use crypto::{Bip32DerPathError, Bip44DerPathError, Bip44PathToCoin, CryptoCtx, CryptoInitError, HwWalletType}; use derive_more::Display; use futures::channel::mpsc; use futures::compat::Future01CompatExt; @@ -62,12 +62,13 @@ pub enum UtxoCoinBuildError { CantDetectUserHome, #[display(fmt = "Unexpected derivation method: {}", _0)] UnexpectedDerivationMethod(String), - HardwareWalletError(HwError), + #[display(fmt = "Hardware Wallet context is not initialized")] + HwContextNotInitialized, HDWalletStorageError(HDWalletStorageError), - #[display(fmt = "Error processing Hardware Wallet request: {}", _0)] - ErrorProcessingHwRequest(String), - #[display(fmt = "Cannot extract an extended public key from an Iguana key pair")] - HDWalletUnavailable, + #[display( + fmt = "Coin should be activated with Hardware Wallet. Please consider using `\"priv_key_policy\": \"Trezor\"` in the activation request" + )] + CoinShouldBeActivatedWithHw, #[display( fmt = "Coin doesn't support Trezor hardware wallet. Please consider adding the 'trezor_coin' field to the coins config" )] @@ -80,14 +81,6 @@ impl From for UtxoCoinBuildError { fn from(e: UtxoConfError) -> Self { UtxoCoinBuildError::ConfError(e) } } -impl From for UtxoCoinBuildError { - fn from(hw_err: HwError) -> Self { UtxoCoinBuildError::HardwareWalletError(hw_err) } -} - -impl From for UtxoCoinBuildError { - fn from(trezor_err: TrezorError) -> Self { UtxoCoinBuildError::HardwareWalletError(HwError::from(trezor_err)) } -} - impl From for UtxoCoinBuildError { /// `CryptoCtx` is expected to be initialized already. fn from(crypto_err: CryptoInitError) -> Self { UtxoCoinBuildError::Internal(crypto_err.to_string()) } @@ -113,7 +106,7 @@ pub trait UtxoCoinBuilder: UtxoFieldsWithIguanaPrivKeyBuilder + UtxoFieldsWithHa async fn build_utxo_fields(&self) -> UtxoCoinBuildResult { match self.priv_key_policy() { PrivKeyBuildPolicy::IguanaPrivKey(priv_key) => self.build_utxo_fields_with_iguana_priv_key(priv_key).await, - PrivKeyBuildPolicy::HardwareWallet => self.build_utxo_fields_with_xpub().await, + PrivKeyBuildPolicy::Trezor => self.build_utxo_fields_with_trezor().await, } } } @@ -133,6 +126,10 @@ pub trait UtxoFieldsWithIguanaPrivKeyBuilder: UtxoCoinBuilderCommonOps { async fn build_utxo_fields_with_iguana_priv_key(&self, priv_key: &[u8]) -> UtxoCoinBuildResult { let conf = UtxoConfBuilder::new(self.conf(), self.activation_params(), self.ticker()).build()?; + if self.is_hw_coin(&conf) { + return MmError::err(UtxoCoinBuildError::CoinShouldBeActivatedWithHw); + } + let private = Private { prefix: conf.wif_prefix, secret: H256::from(priv_key), @@ -184,10 +181,15 @@ pub trait UtxoFieldsWithIguanaPrivKeyBuilder: UtxoCoinBuilderCommonOps { #[async_trait] pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { - async fn build_utxo_fields_with_xpub(&self) -> UtxoCoinBuildResult { + async fn build_utxo_fields_with_trezor(&self) -> UtxoCoinBuildResult { let ticker = self.ticker().to_owned(); let conf = UtxoConfBuilder::new(self.conf(), self.activation_params(), &ticker).build()?; + if !self.supports_trezor(&conf) { + return MmError::err(UtxoCoinBuildError::CoinDoesntSupportTrezor); + } + self.check_if_trezor_is_initialized().await?; + // For now, use a default script pubkey. // TODO change the type of `recently_spent_outpoints` to `AsyncMutex>` let my_script_pubkey = Bytes::new(); @@ -225,7 +227,7 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { decimals, dust_amount, rpc_client, - priv_key_policy: PrivKeyPolicy::HardwareWallet, + priv_key_policy: PrivKeyPolicy::Trezor, derivation_method: DerivationMethod::HDWallet(hd_wallet), history_sync_state: Mutex::new(initial_history_state), tx_cache_directory, @@ -256,6 +258,19 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { } fn gap_limit(&self) -> u32 { self.activation_params().gap_limit.unwrap_or(DEFAULT_GAP_LIMIT) } + + fn supports_trezor(&self, conf: &UtxoCoinConf) -> bool { conf.trezor_coin.is_some() } + + async fn check_if_trezor_is_initialized(&self) -> UtxoCoinBuildResult<()> { + let crypto_ctx = CryptoCtx::from_ctx(self.ctx())?; + let hw_ctx = crypto_ctx + .hw_ctx() + .await + .or_mm_err(|| UtxoCoinBuildError::HwContextNotInitialized)?; + match hw_ctx.hw_wallet_type() { + HwWalletType::Trezor => Ok(()), + } + } } #[async_trait] @@ -530,6 +545,8 @@ pub trait UtxoCoinBuilderCommonOps { } fn check_utxo_maturity(&self) -> bool { self.activation_params().check_utxo_maturity.unwrap_or_default() } + + fn is_hw_coin(&self, conf: &UtxoCoinConf) -> bool { conf.trezor_coin.is_some() } } /// Attempts to parse native daemon conf file and return rpcport, rpcuser and rpcpassword diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 7d1b4b385c..7969576555 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -522,7 +522,7 @@ pub fn my_public_key(coin: &UtxoCoinFields) -> Result<&Public, MmError Ok(key_pair.public()), // Hardware Wallets requires BIP39/BIP44 derivation path to extract a public key. - PrivKeyPolicy::HardwareWallet => MmError::err(UnexpectedDerivationMethod::IguanaPrivKeyUnavailable), + PrivKeyPolicy::Trezor => MmError::err(UnexpectedDerivationMethod::IguanaPrivKeyUnavailable), } } @@ -1767,7 +1767,7 @@ pub fn current_block(coin: &UtxoCoinFields) -> Box Result { match coin.priv_key_policy { PrivKeyPolicy::KeyPair(ref key_pair) => Ok(key_pair.private().to_string()), - PrivKeyPolicy::HardwareWallet => ERR!("'display_priv_key' doesn't support Hardware Wallets"), + PrivKeyPolicy::Trezor => ERR!("'display_priv_key' doesn't support Hardware Wallets"), } } @@ -1803,6 +1803,7 @@ where + UtxoCommonOps + MarketCoinOps + UtxoSignerOps + + CoinWithDerivationMethod + GetWithdrawSenderAddress
+ Send + Sync @@ -3469,7 +3470,7 @@ where { match &coin.as_ref().priv_key_policy { PrivKeyPolicy::KeyPair(key_pair) => key_pair.clone(), - PrivKeyPolicy::HardwareWallet => KeyPair::random_compressed(), + PrivKeyPolicy::Trezor => KeyPair::random_compressed(), } } diff --git a/mm2src/coins/utxo/utxo_withdraw.rs b/mm2src/coins/utxo/utxo_withdraw.rs index 3bac44f458..1cf723bc2c 100644 --- a/mm2src/coins/utxo/utxo_withdraw.rs +++ b/mm2src/coins/utxo/utxo_withdraw.rs @@ -2,8 +2,8 @@ use crate::init_withdraw::{WithdrawAwaitingStatus, WithdrawInProgressStatus, Wit use crate::utxo::utxo_common::{big_decimal_from_sat, UtxoTxBuilder}; use crate::utxo::{output_script, sat_from_big_decimal, ActualTxFee, Address, FeePolicy, PrivKeyPolicy, UtxoAddressFormat, UtxoCoinFields, UtxoCommonOps, UtxoFeeDetails, UtxoTx, UTXO_LOCK}; -use crate::{GetWithdrawSenderAddress, MarketCoinOps, TransactionDetails, WithdrawError, WithdrawFee, WithdrawRequest, - WithdrawResult}; +use crate::{CoinWithDerivationMethod, GetWithdrawSenderAddress, MarketCoinOps, TransactionDetails, WithdrawError, + WithdrawFee, WithdrawRequest, WithdrawResult}; use async_trait::async_trait; use chain::TransactionOutput; use common::log::info; @@ -13,8 +13,7 @@ use common::now_ms; use crypto::hw_rpc_task::{HwConnectStatuses, TrezorRpcTaskConnectProcessor}; use crypto::trezor::client::TrezorClient; use crypto::trezor::{TrezorError, TrezorProcessingError}; -use crypto::{Bip32Error, CryptoCtx, CryptoInitError, DerivationPath, HardwareWalletArc, HwError, HwProcessingError, - HwWalletType}; +use crypto::{Bip32Error, CryptoCtx, CryptoInitError, DerivationPath, HwError, HwProcessingError}; use keys::{Public as PublicKey, Type as ScriptType}; use rpc_task::RpcTaskError; use script::{Builder, Script, SignatureVersion, TransactionInputSigner}; @@ -228,6 +227,7 @@ where } pub struct InitUtxoWithdraw<'a, Coin> { + ctx: MmArc, coin: Coin, task_handle: &'a WithdrawTaskHandle, req: WithdrawRequest, @@ -238,7 +238,6 @@ pub struct InitUtxoWithdraw<'a, Coin> { from_derivation_path: DerivationPath, /// Public key corresponding to [`InitUtxoWithdraw::from_address`]. from_pubkey: PublicKey, - trezor: Option, } #[async_trait] @@ -315,16 +314,12 @@ where let sign_policy = match self.coin.as_ref().priv_key_policy { PrivKeyPolicy::KeyPair(ref key_pair) => SignPolicy::WithKeyPair(key_pair), - PrivKeyPolicy::HardwareWallet => match self.trezor { - Some(ref trezor) => SignPolicy::WithTrezor(trezor.clone()), - None => { - let error = "'InitUtxoWithdraw::trezor' is expected to be set".to_owned(); - return MmError::err(WithdrawError::InternalError(error)); - }, + PrivKeyPolicy::Trezor => { + let trezor_client = self.trezor_client().await?; + SignPolicy::WithTrezor(trezor_client) }, }; - // TODO refactor [`UtxoSignerOps::sign_tx`] to use [`TrezorInteractWithUser::interact_with_user_if_required`]. self.task_handle .update_in_progress_status(WithdrawInProgressStatus::WaitingForUserToConfirmSigning)?; let signed = self.coin.sign_tx(sign_params, sign_policy).await?; @@ -333,42 +328,37 @@ where } } -impl<'a, Coin> InitUtxoWithdraw<'a, Coin> -where - Coin: AsRef - + UtxoCommonOps - + MarketCoinOps - + UtxoSignerOps - + GetWithdrawSenderAddress
, -{ +impl<'a, Coin> InitUtxoWithdraw<'a, Coin> { pub async fn new( ctx: MmArc, coin: Coin, req: WithdrawRequest, task_handle: &'a WithdrawTaskHandle, - ) -> Result, MmError> { - let crypto_ctx = CryptoCtx::from_ctx(&ctx)?; - match *crypto_ctx { - CryptoCtx::KeyPair(_) => Self::new_with_key_pair(coin, req, task_handle).await, - CryptoCtx::HardwareWallet(ref hw_ctx) => match hw_ctx.hw_wallet_type() { - HwWalletType::Trezor => Self::new_with_trezor(hw_ctx, coin, req, task_handle).await, - }, - } - } - - async fn new_with_key_pair( - coin: Coin, - req: WithdrawRequest, - task_handle: &'a WithdrawTaskHandle, - ) -> Result, MmError> { + ) -> Result, MmError> + where + Coin: AsRef + + UtxoCommonOps + + MarketCoinOps + + UtxoSignerOps + + CoinWithDerivationMethod + + GetWithdrawSenderAddress
, + { let from = coin.get_withdraw_sender_address(&req).await?; let from_address_string = from.address.display_address().map_to_mm(WithdrawError::InternalError)?; + let from_derivation_path = match from.derivation_path { Some(der_path) => der_path, + // [`WithdrawSenderAddress::derivation_path`] is not set, but the coin is initialized with an HD wallet derivation method. + None if coin.has_hd_wallet_derivation_method() => { + let error = "Cannot determine 'from' address derivation path".to_owned(); + return MmError::err(WithdrawError::UnexpectedFromAddress(error)); + }, // Temporary initialize the derivation path by default since this field is not used without Trezor. None => DerivationPath::default(), }; + Ok(InitUtxoWithdraw { + ctx, coin, task_handle, req, @@ -376,27 +366,20 @@ where from_address_string, from_derivation_path, from_pubkey: from.pubkey, - trezor: None, }) } - async fn new_with_trezor( - hw_ctx: &HardwareWalletArc, - coin: Coin, - req: WithdrawRequest, - task_handle: &'a WithdrawTaskHandle, - ) -> Result, MmError> { - let from = coin.get_withdraw_sender_address(&req).await?; - let from_derivation_path = match from.derivation_path { - Some(der_path) => der_path, - None => { - let error = "Cannot determine 'from' address derivation path".to_owned(); - return MmError::err(WithdrawError::UnexpectedFromAddress(error)); - }, - }; - let from_address_string = from.address.display_address().map_to_mm(WithdrawError::InternalError)?; - - let trezor_connect_processor = TrezorRpcTaskConnectProcessor::new(task_handle, HwConnectStatuses { + /// # Fail + /// + /// The method fails if [`CryptoCtx::hw_ctx`] is not initialized yet. + async fn trezor_client(&self) -> MmResult { + let crypto_ctx = CryptoCtx::from_ctx(&self.ctx)?; + let hw_ctx = crypto_ctx + .hw_ctx() + .await + .or_mm_err(|| WithdrawError::NoTrezorDeviceAvailable)?; + + let trezor_connect_processor = TrezorRpcTaskConnectProcessor::new(self.task_handle, HwConnectStatuses { on_connect: WithdrawInProgressStatus::WaitingForTrezorToConnect, on_connected: WithdrawInProgressStatus::Preparing, on_connection_failed: WithdrawInProgressStatus::Finishing, @@ -407,18 +390,10 @@ where .with_connect_timeout(TREZOR_CONNECT_TIMEOUT) .with_pin_timeout(TREZOR_PIN_TIMEOUT); - let trezor_client = hw_ctx.trezor(&trezor_connect_processor).await?; - - Ok(InitUtxoWithdraw { - coin, - task_handle, - req, - from_address: from.address, - from_address_string, - from_derivation_path, - from_pubkey: from.pubkey, - trezor: Some(trezor_client), - }) + hw_ctx + .trezor(&trezor_connect_processor) + .await + .mm_err(WithdrawError::from) } } diff --git a/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs b/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs index 3368fb90b2..b118d0d514 100644 --- a/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs +++ b/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs @@ -3,17 +3,15 @@ use crate::prelude::{coin_conf_with_protocol, TryFromCoinProtocol}; use crate::standalone_coin::init_standalone_coin_error::{InitStandaloneCoinError, InitStandaloneCoinStatusError, InitStandaloneCoinUserActionError}; use async_trait::async_trait; -use coins::{lp_coinfind, MmCoinEnum, PrivKeyBuildPolicy}; +use coins::{lp_coinfind, MmCoinEnum}; use common::mm_ctx::MmArc; use common::mm_error::prelude::*; use common::{NotSame, SuccessResponse}; use crypto::trezor::trezor_rpc_task::RpcTaskHandle; -use crypto::CryptoCtx; use rpc_task::rpc_common::{InitRpcTaskResponse, RpcTaskStatusRequest, RpcTaskUserActionRequest}; use rpc_task::{RpcTask, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, RpcTaskTypes}; use serde_derive::Deserialize; use serde_json::Value as Json; -use std::sync::Arc; pub type InitStandaloneCoinResponse = InitRpcTaskResponse; pub type InitStandaloneCoinStatusRequest = RpcTaskStatusRequest; @@ -47,7 +45,6 @@ pub trait InitStandaloneCoinActivationOps: Into + Send + Sync + 'sta coin_conf: Json, activation_request: &Self::ActivationRequest, protocol_info: Self::StandaloneProtocol, - priv_key_policy: PrivKeyBuildPolicy<'_>, task_handle: &InitStandaloneCoinTaskHandle, ) -> Result>; @@ -69,8 +66,6 @@ where InitStandaloneCoinError: From, (Standalone::ActivationError, InitStandaloneCoinError): NotSame, { - let crypto_ctx = CryptoCtx::from_ctx(&ctx).mm_err(|e| InitStandaloneCoinError::Internal(e.to_string()))?; - if let Ok(Some(_)) = lp_coinfind(&ctx, &request.ticker).await { return MmError::err(InitStandaloneCoinError::CoinIsAlreadyActivated { ticker: request.ticker }); } @@ -80,7 +75,6 @@ where let coins_act_ctx = CoinsActivationContext::from_ctx(&ctx).map_to_mm(InitStandaloneCoinError::Internal)?; let task = InitStandaloneCoinTask:: { ctx, - crypto_ctx, request, coin_conf, protocol_info, @@ -133,7 +127,6 @@ pub async fn init_standalone_coin_user_action { ctx: MmArc, - crypto_ctx: Arc, request: InitStandaloneCoinReq, coin_conf: Json, protocol_info: Standalone::StandaloneProtocol, @@ -157,14 +150,12 @@ where } async fn run(self, task_handle: &RpcTaskHandle) -> Result> { - let priv_key_policy = PrivKeyBuildPolicy::from_crypto_ctx(&self.crypto_ctx); let coin = Standalone::init_standalone_coin( self.ctx.clone(), self.request.ticker, self.coin_conf, &self.request.activation_params, self.protocol_info, - priv_key_policy, task_handle, ) .await?; diff --git a/mm2src/coins_activation/src/utxo_activation/common_impl.rs b/mm2src/coins_activation/src/utxo_activation/common_impl.rs index 058b22ff0d..8914d508c0 100644 --- a/mm2src/coins_activation/src/utxo_activation/common_impl.rs +++ b/mm2src/coins_activation/src/utxo_activation/common_impl.rs @@ -6,10 +6,11 @@ use crate::utxo_activation::utxo_standard_activation_result::UtxoStandardActivat use coins::coin_balance::EnableCoinBalanceOps; use coins::hd_pubkey::RpcTaskXPubExtractor; use coins::utxo::UtxoActivationParams; -use coins::MarketCoinOps; +use coins::{MarketCoinOps, PrivKeyActivationPolicy, PrivKeyBuildPolicy}; use common::mm_ctx::MmArc; use common::mm_error::prelude::*; use crypto::hw_rpc_task::HwConnectStatuses; +use crypto::CryptoCtx; use futures::compat::Future01CompatExt; pub async fn get_activation_result( @@ -39,7 +40,7 @@ where // Construct an Xpub extractor without checking if the MarketMaker supports HD wallet ops. // [`EnableCoinBalanceOps::enable_coin_balance`] won't just use `xpub_extractor` // if the coin has been initialized with an Iguana priv key. - let xpub_extractor = RpcTaskXPubExtractor::new_unchecked(ctx, task_handle, xpub_extractor_rpc_statuses()); + let xpub_extractor = RpcTaskXPubExtractor::new_unchecked(ctx, task_handle, xpub_extractor_rpc_statuses()).await; task_handle.update_in_progress_status(UtxoStandardInProgressStatus::RequestingWalletBalance)?; let wallet_balance = coin .enable_coin_balance(&xpub_extractor, activation_params.scan_policy) @@ -67,3 +68,10 @@ pub fn xpub_extractor_rpc_statuses() -> HwConnectStatuses PrivKeyBuildPolicy { + match activation_policy { + PrivKeyActivationPolicy::IguanaPrivKey => PrivKeyBuildPolicy::iguana_priv_key(crypto_ctx), + PrivKeyActivationPolicy::Trezor => PrivKeyBuildPolicy::Trezor, + } +} diff --git a/mm2src/coins_activation/src/utxo_activation/init_qtum_activation.rs b/mm2src/coins_activation/src/utxo_activation/init_qtum_activation.rs index 73fcaa009c..d7f3dac5a3 100644 --- a/mm2src/coins_activation/src/utxo_activation/init_qtum_activation.rs +++ b/mm2src/coins_activation/src/utxo_activation/init_qtum_activation.rs @@ -2,7 +2,7 @@ use crate::context::CoinsActivationContext; use crate::prelude::TryFromCoinProtocol; use crate::standalone_coin::{InitStandaloneCoinActivationOps, InitStandaloneCoinTaskHandle, InitStandaloneCoinTaskManagerShared}; -use crate::utxo_activation::common_impl::get_activation_result; +use crate::utxo_activation::common_impl::{get_activation_result, priv_key_build_policy}; use crate::utxo_activation::init_utxo_standard_activation_error::InitUtxoStandardError; use crate::utxo_activation::init_utxo_standard_statuses::{UtxoStandardAwaitingStatus, UtxoStandardInProgressStatus, UtxoStandardUserAction}; @@ -11,9 +11,10 @@ use async_trait::async_trait; use coins::utxo::qtum::{QtumCoin, QtumCoinBuilder}; use coins::utxo::utxo_builder::UtxoCoinBuilder; use coins::utxo::UtxoActivationParams; -use coins::{lp_register_coin, CoinProtocol, MmCoinEnum, PrivKeyBuildPolicy, RegisterCoinParams}; +use coins::{lp_register_coin, CoinProtocol, MmCoinEnum, RegisterCoinParams}; use common::mm_ctx::MmArc; use common::mm_error::prelude::*; +use crypto::CryptoCtx; use serde_json::Value as Json; pub type QtumTaskManagerShared = InitStandaloneCoinTaskManagerShared; @@ -53,10 +54,13 @@ impl InitStandaloneCoinActivationOps for QtumCoin { coin_conf: Json, activation_request: &Self::ActivationRequest, _protocol_info: Self::StandaloneProtocol, - priv_key_policy: PrivKeyBuildPolicy<'_>, _task_handle: &QtumRpcTaskHandle, ) -> Result> { + let crypto_ctx = CryptoCtx::from_ctx(&ctx)?; + let tx_history = activation_request.tx_history; + let priv_key_policy = priv_key_build_policy(&crypto_ctx, activation_request.priv_key_policy); + let coin = QtumCoinBuilder::new(&ctx, &ticker, &coin_conf, activation_request, priv_key_policy) .build() .await diff --git a/mm2src/coins_activation/src/utxo_activation/init_utxo_standard_activation.rs b/mm2src/coins_activation/src/utxo_activation/init_utxo_standard_activation.rs index 63c5d1a1b7..7bbd0e4b68 100644 --- a/mm2src/coins_activation/src/utxo_activation/init_utxo_standard_activation.rs +++ b/mm2src/coins_activation/src/utxo_activation/init_utxo_standard_activation.rs @@ -2,7 +2,7 @@ use crate::context::CoinsActivationContext; use crate::prelude::TryFromCoinProtocol; use crate::standalone_coin::{InitStandaloneCoinActivationOps, InitStandaloneCoinTaskHandle, InitStandaloneCoinTaskManagerShared}; -use crate::utxo_activation::common_impl::get_activation_result; +use crate::utxo_activation::common_impl::{get_activation_result, priv_key_build_policy}; use crate::utxo_activation::init_utxo_standard_activation_error::InitUtxoStandardError; use crate::utxo_activation::init_utxo_standard_statuses::{UtxoStandardAwaitingStatus, UtxoStandardInProgressStatus, UtxoStandardUserAction}; @@ -11,9 +11,10 @@ use async_trait::async_trait; use coins::utxo::utxo_builder::{UtxoArcBuilder, UtxoCoinBuilder}; use coins::utxo::utxo_standard::UtxoStandardCoin; use coins::utxo::UtxoActivationParams; -use coins::{lp_register_coin, CoinProtocol, MmCoinEnum, PrivKeyBuildPolicy, RegisterCoinParams}; +use coins::{lp_register_coin, CoinProtocol, MmCoinEnum, RegisterCoinParams}; use common::mm_ctx::MmArc; use common::mm_error::prelude::*; +use crypto::CryptoCtx; use serde_json::Value as Json; pub type UtxoStandardTaskManagerShared = InitStandaloneCoinTaskManagerShared; @@ -53,10 +54,13 @@ impl InitStandaloneCoinActivationOps for UtxoStandardCoin { coin_conf: Json, activation_request: &Self::ActivationRequest, _protocol_info: Self::StandaloneProtocol, - priv_key_policy: PrivKeyBuildPolicy<'_>, _task_handle: &UtxoStandardRpcTaskHandle, ) -> MmResult { + let crypto_ctx = CryptoCtx::from_ctx(&ctx)?; + let tx_history = activation_request.tx_history; + let priv_key_policy = priv_key_build_policy(&crypto_ctx, activation_request.priv_key_policy); + let coin = UtxoArcBuilder::new( &ctx, &ticker, diff --git a/mm2src/crypto/Cargo.toml b/mm2src/crypto/Cargo.toml index 488fa0d4a9..1b96889161 100644 --- a/mm2src/crypto/Cargo.toml +++ b/mm2src/crypto/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +async-std = { version = "1.5", features = ["unstable"] } async-trait = "0.1" bip32 = { version = "0.2.2", default-features = false, features = ["alloc", "secp256k1-ffi"] } bitcrypto = { path = "../mm2_bitcoin/crypto" } diff --git a/mm2src/crypto/src/crypto_ctx.rs b/mm2src/crypto/src/crypto_ctx.rs index 21a32b0ec0..73f697f99e 100644 --- a/mm2src/crypto/src/crypto_ctx.rs +++ b/mm2src/crypto/src/crypto_ctx.rs @@ -1,16 +1,15 @@ -use crate::hw_client::{HwClient, HwError, HwProcessingError, TrezorConnectProcessor}; +use crate::hw_client::{HwError, HwProcessingError, TrezorConnectProcessor}; use crate::hw_ctx::{HardwareWalletArc, HardwareWalletCtx}; -use crate::key_pair_ctx::KeyPairArc; -use crate::HwResult; -use bitcrypto::dhash160; -use common::mm_ctx::{MmArc, MmWeak}; +use crate::key_pair_ctx::IguanaArc; +use async_std::sync::RwLock as AsyncRwLock; +use common::mm_ctx::MmArc; use common::mm_error::prelude::*; use common::privkey::{key_pair_from_seed, PrivKeyError}; +use common::NotSame; use derive_more::Display; use hw_common::primitives::EcdsaCurve; use keys::Public as PublicKey; -use parking_lot::Mutex as PaMutex; -use primitives::hash::{H160, H264}; +use primitives::hash::H160; use std::ops::Deref; use std::sync::Arc; @@ -41,9 +40,32 @@ impl From for CryptoInitError { fn from(e: PrivKeyError) -> Self { CryptoInitError::InvalidPassphrase(e) } } -pub enum CryptoCtx { - KeyPair(KeyPairArc), - HardwareWallet(HardwareWalletArc), +#[derive(Debug)] +pub enum HwCtxInitError { + InitializedAlready, + HwError(HwError), + ProcessorError(ProcessorError), +} + +impl From> for HwCtxInitError { + fn from(e: HwProcessingError) -> Self { + match e { + HwProcessingError::HwError(hw_error) => HwCtxInitError::HwError(hw_error), + HwProcessingError::ProcessorError(processor_error) => HwCtxInitError::ProcessorError(processor_error), + } + } +} + +/// This is required for converting `MmError>` into `MmError>`. +impl NotSame for HwCtxInitError {} + +pub struct CryptoCtx { + iguana_ctx: IguanaArc, + /// Can be initialized on [`CryptoCtx::init_hw_ctx_with_trezor`]. + /// Please note that it's preferred to use `AsyncRwLock` here + /// because [`CryptoCtx::hw_ctx`] can be locked for a long time while `HardwareWalletCtx` is initializing, + /// and `AsyncRwLock` allows a runtime to poll other futures even on the same thread. + hw_ctx: AsyncRwLock>, } impl CryptoCtx { @@ -61,33 +83,20 @@ impl CryptoCtx { .map_err(|_| MmError::new(CryptoInitError::Internal("Error casting the context field".to_owned()))) } - pub fn secp256k1_pubkey(&self) -> PublicKey { - match self { - CryptoCtx::KeyPair(key_pair_ctx) => key_pair_ctx.secp256k1_pubkey(), - CryptoCtx::HardwareWallet(hw_ctx) => hw_ctx.secp256k1_pubkey(), - } - } + pub fn iguana_ctx(&self) -> &IguanaArc { &self.iguana_ctx } + + pub fn secp256k1_pubkey(&self) -> PublicKey { self.iguana_ctx.secp256k1_pubkey() } pub fn secp256k1_pubkey_hex(&self) -> String { hex::encode(&*self.secp256k1_pubkey()) } - pub fn hw_ctx(&self) -> Option<&HardwareWalletCtx> { - match self { - CryptoCtx::KeyPair(_) => None, - CryptoCtx::HardwareWallet(hw_ctx) => Some(hw_ctx.deref()), - } - } + pub async fn hw_ctx(&self) -> Option { self.hw_ctx.read().await.clone() } /// Returns an `RIPEMD160(SHA256(x))` where x is secp256k1 pubkey that identifies a Hardware Wallet device or an HD master private key. - /// We're planning to allow the user to launch mm2 with Iguana private key and enable coin with a Hardware Wallet device, - /// so the `hd_wallet_rmd160` may be different from [`MmCtx::rmd160`] soon. - pub fn hd_wallet_rmd160(&self) -> Option { - match self { - CryptoCtx::HardwareWallet(hw_ctx) => Some(hw_ctx.rmd160()), - CryptoCtx::KeyPair(_) => None, - } + pub async fn hd_wallet_rmd160(&self) -> Option { + self.hw_ctx.read().await.as_ref().map(|hw_ctx| hw_ctx.rmd160()) } - pub fn init_with_passphrase(ctx: MmArc, passphrase: &str) -> CryptoInitResult<()> { + pub fn init_with_iguana_passphrase(ctx: MmArc, passphrase: &str) -> CryptoInitResult<()> { let mut ctx_field = ctx .crypto_ctx .lock() @@ -105,7 +114,10 @@ impl CryptoCtx { let secp256k1_key_pair_for_legacy = key_pair_from_seed(passphrase)?; let rmd160 = secp256k1_key_pair.public().address_hash(); - let crypto_ctx = CryptoCtx::KeyPair(KeyPairArc::from(secp256k1_key_pair)); + let crypto_ctx = CryptoCtx { + iguana_ctx: IguanaArc::from(secp256k1_key_pair), + hw_ctx: AsyncRwLock::default(), + }; *ctx_field = Some(Arc::new(crypto_ctx)); // TODO remove initializing legacy fields when lp_swap and lp_ordermatch support CryptoCtx. @@ -117,54 +129,20 @@ impl CryptoCtx { Ok(()) } - pub async fn init_with_trezor( - ctx_weak: MmWeak, + pub async fn init_hw_ctx_with_trezor( + &self, processor: &Processor, - ) -> MmResult<(), HwProcessingError> + ) -> MmResult> where Processor: TrezorConnectProcessor + Sync, { - let trezor = HwClient::trezor(processor).await?; - let hw_internal_pubkey = { - let mut session = trezor.session().await?; - HardwareWalletCtx::trezor_mm_internal_pubkey(&mut session, processor).await? - }; - - Ok(CryptoCtx::init_with_hw_wallet_internal_xpub( - ctx_weak, - HwClient::from(trezor), - hw_internal_pubkey, - )?) - } - - fn init_with_hw_wallet_internal_xpub( - ctx_weak: MmWeak, - hw_client: HwClient, - hw_internal_pubkey: H264, - ) -> HwResult<()> { - let ctx = match MmArc::from_weak(&ctx_weak) { - Some(ctx) => ctx, - None => return MmError::err(HwError::Internal("MmArc is dropped".to_owned())), - }; - - let mut ctx_field = ctx - .crypto_ctx - .lock() - .map_to_mm(|poison| HwError::Internal(poison.to_string()))?; - if ctx_field.is_some() { - return MmError::err(HwError::Internal("'crypto_ctx' is initialized already".to_owned())); + let mut guard = self.hw_ctx.write().await; + if guard.is_some() { + return MmError::err(HwCtxInitError::InitializedAlready); } - // TODO remove initializing legacy fields when lp_swap and lp_ordermatch support CryptoCtx. - let rmd160 = dhash160(hw_internal_pubkey.as_slice()); - ctx.rmd160.pin(rmd160).map_to_mm(HwError::Internal)?; - - let crypto_ctx = CryptoCtx::HardwareWallet(HardwareWalletArc::new(HardwareWalletCtx { - hw_internal_pubkey, - hw_wallet_type: hw_client.hw_wallet_type(), - hw_wallet: PaMutex::new(Some(hw_client)), - })); - *ctx_field = Some(Arc::new(crypto_ctx)); - Ok(()) + let hw_ctx = HardwareWalletCtx::init_with_trezor(processor).await?; + *guard = Some(hw_ctx.clone()); + Ok(hw_ctx) } } diff --git a/mm2src/crypto/src/hw_ctx.rs b/mm2src/crypto/src/hw_ctx.rs index c81bffc49c..c2327536a0 100644 --- a/mm2src/crypto/src/hw_ctx.rs +++ b/mm2src/crypto/src/hw_ctx.rs @@ -5,9 +5,9 @@ use crate::HwWalletType; use bitcrypto::dhash160; use common::log::warn; use common::mm_error::prelude::*; +use futures::lock::Mutex as AsyncMutex; use hw_common::primitives::{DerivationPath, Secp256k1ExtendedPublicKey}; use keys::Public as PublicKey; -use parking_lot::Mutex as PaMutex; use primitives::hash::{H160, H264}; use std::ops::Deref; use std::str::FromStr; @@ -36,13 +36,32 @@ pub struct HardwareWalletCtx { pub(crate) hw_internal_pubkey: H264, pub(crate) hw_wallet_type: HwWalletType, /// Please avoid locking multiple mutexes. - /// The mutex hasn't be locked while the wallet is used - /// because every variant of the Hardware Wallet client uses an internal mutex to operate with the device. - /// Clone the `Option` instance instead. - pub(crate) hw_wallet: PaMutex>, + /// The mutex hasn't to be locked while the client is used + /// because every variant of `HwClient` uses an internal mutex to operate with the device. + /// But it has to be locked while the client is initialized. + pub(crate) hw_wallet: AsyncMutex>, } impl HardwareWalletCtx { + pub(crate) async fn init_with_trezor( + processor: &Processor, + ) -> MmResult> + where + Processor: TrezorConnectProcessor + Sync, + { + let trezor = HwClient::trezor(processor).await?; + let hw_internal_pubkey = { + let mut session = trezor.session().await?; + HardwareWalletCtx::trezor_mm_internal_pubkey(&mut session, processor).await? + }; + let hw_client = HwClient::Trezor(trezor); + Ok(HardwareWalletArc::new(HardwareWalletCtx { + hw_internal_pubkey, + hw_wallet_type: hw_client.hw_wallet_type(), + hw_wallet: AsyncMutex::new(Some(hw_client)), + })) + } + pub fn hw_wallet_type(&self) -> HwWalletType { self.hw_wallet_type } /// Connects to a Trezor device and checks if MM was initialized from this particular device. @@ -54,12 +73,12 @@ impl HardwareWalletCtx { Processor: TrezorConnectProcessor + Sync, Processor::Error: std::fmt::Display, { - let hw_wallet = self.hw_wallet.lock().clone(); - if let Some(HwClient::Trezor(connected_trezor)) = hw_wallet { - match self.check_trezor(&connected_trezor, processor).await { - Ok(()) => return Ok(connected_trezor), + let mut hw_client = self.hw_wallet.lock().await; + if let Some(HwClient::Trezor(connected_trezor)) = hw_client.deref() { + match self.check_trezor(connected_trezor, processor).await { + Ok(()) => return Ok(connected_trezor.clone()), // The device could be unplugged. We should try to reconnect to the device. - Err(e) => warn!("Error checking a connected device: '{}'. Trying to reconnect...", e), + Err(e) => warn!("Error checking hardware wallet device: '{}'. Trying to reconnect...", e), } } // Connect to a device. @@ -68,7 +87,7 @@ impl HardwareWalletCtx { self.check_trezor(&trezor, processor).await?; // Reinitialize the field to avoid reconnecting next time. - *self.hw_wallet.lock() = Some(HwClient::Trezor(trezor.clone())); + *hw_client = Some(HwClient::Trezor(trezor.clone())); Ok(trezor) } @@ -88,8 +107,7 @@ impl HardwareWalletCtx { .expect("'MM2_INTERNAL_DERIVATION_PATH' is expected to be valid derivation path"); let mm2_internal_xpub = trezor .get_public_key(path, MM2_TREZOR_INTERNAL_COIN, MM2_INTERNAL_ECDSA_CURVE) - .await - .mm_err(HwError::from)? + .await? .process(processor) .await?; let extended_pubkey = Secp256k1ExtendedPublicKey::from_str(&mm2_internal_xpub).map_to_mm(HwError::from)?; diff --git a/mm2src/crypto/src/key_pair_ctx.rs b/mm2src/crypto/src/key_pair_ctx.rs index 84d10c8357..8026b6b9a8 100644 --- a/mm2src/crypto/src/key_pair_ctx.rs +++ b/mm2src/crypto/src/key_pair_ctx.rs @@ -3,29 +3,29 @@ use std::ops::Deref; use std::sync::Arc; #[derive(Clone)] -pub struct KeyPairArc(Arc); +pub struct IguanaArc(Arc); -impl Deref for KeyPairArc { - type Target = KeyPairCtx; +impl Deref for IguanaArc { + type Target = IguanaCtx; fn deref(&self) -> &Self::Target { &self.0 } } -impl From for KeyPairArc { - fn from(secp256k1_key_pair: KeyPair) -> Self { KeyPairArc::new(KeyPairCtx { secp256k1_key_pair }) } +impl From for IguanaArc { + fn from(secp256k1_key_pair: KeyPair) -> Self { IguanaArc::new(IguanaCtx { secp256k1_key_pair }) } } -impl KeyPairArc { - pub fn new(ctx: KeyPairCtx) -> KeyPairArc { KeyPairArc(Arc::new(ctx)) } +impl IguanaArc { + pub fn new(ctx: IguanaCtx) -> IguanaArc { IguanaArc(Arc::new(ctx)) } } -pub struct KeyPairCtx { +pub struct IguanaCtx { /// secp256k1 key pair derived from passphrase. /// cf. `key_pair_from_seed`. pub(crate) secp256k1_key_pair: KeyPair, } -impl KeyPairCtx { +impl IguanaCtx { pub fn secp256k1_pubkey(&self) -> PublicKey { *self.secp256k1_key_pair.public() } pub fn secp256k1_privkey(&self) -> &Private { self.secp256k1_key_pair.private() } diff --git a/mm2src/crypto/src/lib.rs b/mm2src/crypto/src/lib.rs index e722dc2815..c7e99fad37 100644 --- a/mm2src/crypto/src/lib.rs +++ b/mm2src/crypto/src/lib.rs @@ -11,13 +11,13 @@ mod key_pair_ctx; pub use bip32_child::{Bip32Child, Bip32DerPathError, Bip32DerPathOps, Bip44Tail}; pub use bip44::{Bip44Chain, Bip44DerPathError, Bip44DerivationPath, Bip44PathToAccount, Bip44PathToCoin, UnkownBip44ChainError, BIP44_PURPOSE}; -pub use crypto_ctx::{CryptoCtx, CryptoInitError, CryptoInitResult}; +pub use crypto_ctx::{CryptoCtx, CryptoInitError, CryptoInitResult, HwCtxInitError}; pub use hw_client::TrezorConnectProcessor; pub use hw_client::{HwClient, HwError, HwProcessingError, HwResult, HwWalletType}; pub use hw_common::primitives::{Bip32Error, ChildNumber, DerivationPath, EcdsaCurve, ExtendedPublicKey, Secp256k1ExtendedPublicKey, XPub}; pub use hw_ctx::{HardwareWalletArc, HardwareWalletCtx}; -pub use key_pair_ctx::{KeyPairArc, KeyPairCtx}; +pub use key_pair_ctx::{IguanaArc, IguanaCtx}; pub use trezor; use serde::de::Error; diff --git a/mm2src/lp_init/init_context.rs b/mm2src/lp_init/init_context.rs index 45fe9aee2c..6e55584536 100644 --- a/mm2src/lp_init/init_context.rs +++ b/mm2src/lp_init/init_context.rs @@ -1,12 +1,12 @@ -use crate::mm2::lp_native_dex::mm_init_task::MmInitTaskManagerShared; +use crate::mm2::lp_native_dex::init_hw::InitHwTaskManagerShared; use common::mm_ctx::{from_ctx, MmArc}; -use gstuff::Constructible; -use rpc_task::{RpcTaskManager, TaskId}; +use rpc_task::RpcTaskManager; use std::sync::Arc; pub struct MmInitContext { - pub mm_init_task_id: Constructible, - pub mm_init_task_manager: MmInitTaskManagerShared, + // pub mm_init_task_id: Constructible, + // pub mm_init_task_manager: MmInitTaskManagerShared, + pub init_hw_task_manager: InitHwTaskManagerShared, } impl MmInitContext { @@ -14,8 +14,9 @@ impl MmInitContext { pub fn from_ctx(ctx: &MmArc) -> Result, String> { from_ctx(&ctx.mm_init_ctx, move || { Ok(MmInitContext { - mm_init_task_id: Constructible::default(), - mm_init_task_manager: RpcTaskManager::new_shared(), + // mm_init_task_id: Constructible::default(), + // mm_init_task_manager: RpcTaskManager::new_shared(), + init_hw_task_manager: RpcTaskManager::new_shared(), }) }) } diff --git a/mm2src/lp_init/init_hw.rs b/mm2src/lp_init/init_hw.rs new file mode 100644 index 0000000000..49b942c032 --- /dev/null +++ b/mm2src/lp_init/init_hw.rs @@ -0,0 +1,175 @@ +use crate::mm2::lp_native_dex::init_context::MmInitContext; +use async_trait::async_trait; +use common::mm_ctx::MmArc; +use common::mm_error::prelude::*; +use common::{HttpStatusCode, SuccessResponse}; +use crypto::hw_rpc_task::{HwConnectStatuses, HwRpcTaskAwaitingStatus, HwRpcTaskUserAction, HwRpcTaskUserActionRequest, + TrezorRpcTaskConnectProcessor}; +use crypto::{CryptoCtx, CryptoInitError, HwCtxInitError, HwError, HwWalletType}; +use derive_more::Display; +use http::StatusCode; +use rpc_task::rpc_common::{InitRpcTaskResponse, RpcTaskStatusError, RpcTaskStatusRequest, RpcTaskUserActionError}; +use rpc_task::{RpcTask, RpcTaskError, RpcTaskHandle, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, RpcTaskTypes}; +use std::time::Duration; + +const TREZOR_CONNECT_TIMEOUT: Duration = Duration::from_secs(300); +const TREZOR_PIN_TIMEOUT: Duration = Duration::from_secs(600); + +pub type InitHwAwaitingStatus = HwRpcTaskAwaitingStatus; +pub type InitHwUserAction = HwRpcTaskUserAction; + +pub type InitHwTaskManagerShared = RpcTaskManagerShared; +pub type InitHwStatus = RpcTaskStatus; +type InitHwTaskHandle = RpcTaskHandle; + +#[derive(Clone, Display, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum InitHwError { + /* */ + /* ----------- Trezor device errors ----------- */ + /* */ + #[display(fmt = "Trezor internal error: {}", _0)] + TrezorInternal(String), + #[display(fmt = "No Trezor device available")] + NoTrezorDeviceAvailable, + /* */ + /* ---------------- RPC error ----------------- */ + /* */ + #[display(fmt = "Hardware Wallet context is initialized already")] + HwContextInitializedAlready, + #[display(fmt = "RPC timed out {:?}", _0)] + Timeout(Duration), + #[display(fmt = "Internal: {}", _0)] + Internal(String), +} + +impl From for InitHwError { + fn from(e: CryptoInitError) -> Self { InitHwError::Internal(e.to_string()) } +} + +impl From> for InitHwError { + fn from(e: HwCtxInitError) -> Self { + match e { + HwCtxInitError::InitializedAlready => InitHwError::HwContextInitializedAlready, + HwCtxInitError::HwError(hw_error) => InitHwError::from(hw_error), + HwCtxInitError::ProcessorError(rpc) => InitHwError::from(rpc), + } + } +} + +impl From for InitHwError { + fn from(e: HwError) -> Self { + match e { + HwError::NoTrezorDeviceAvailable => InitHwError::NoTrezorDeviceAvailable, + trezor => InitHwError::TrezorInternal(trezor.to_string()), + } + } +} + +impl From for InitHwError { + fn from(e: RpcTaskError) -> Self { + let error = e.to_string(); + match e { + RpcTaskError::Canceled => InitHwError::Internal("Canceled".to_owned()), + RpcTaskError::Timeout(timeout) => InitHwError::Timeout(timeout), + RpcTaskError::NoSuchTask(_) | RpcTaskError::UnexpectedTaskStatus { .. } => InitHwError::Internal(error), + RpcTaskError::Internal(internal) => InitHwError::Internal(internal), + } + } +} + +impl HttpStatusCode for InitHwError { + fn status_code(&self) -> StatusCode { + match self { + InitHwError::HwContextInitializedAlready => StatusCode::BAD_REQUEST, + InitHwError::Timeout(_) => StatusCode::REQUEST_TIMEOUT, + InitHwError::TrezorInternal(_) | InitHwError::NoTrezorDeviceAvailable | InitHwError::Internal(_) => { + StatusCode::INTERNAL_SERVER_ERROR + }, + } + } +} + +#[derive(Clone, Deserialize, Serialize)] +pub enum InitHwInProgressStatus { + Initializing, + WaitingForTrezorToConnect, + ReadPublicKeyFromTrezor, +} + +pub struct InitHwTask { + ctx: MmArc, + hw_wallet_type: HwWalletType, +} + +impl RpcTaskTypes for InitHwTask { + type Item = SuccessResponse; + type Error = InitHwError; + type InProgressStatus = InitHwInProgressStatus; + type AwaitingStatus = InitHwAwaitingStatus; + type UserAction = InitHwUserAction; +} + +#[async_trait] +impl RpcTask for InitHwTask { + fn initial_status(&self) -> Self::InProgressStatus { InitHwInProgressStatus::Initializing } + + async fn run(self, task_handle: &InitHwTaskHandle) -> Result> { + let crypto_ctx = CryptoCtx::from_ctx(&self.ctx)?; + + match self.hw_wallet_type { + HwWalletType::Trezor => { + let trezor_connect_processor = TrezorRpcTaskConnectProcessor::new(task_handle, HwConnectStatuses { + on_connect: InitHwInProgressStatus::WaitingForTrezorToConnect, + on_connected: InitHwInProgressStatus::Initializing, + on_connection_failed: InitHwInProgressStatus::Initializing, + on_button_request: InitHwInProgressStatus::ReadPublicKeyFromTrezor, + on_pin_request: InitHwAwaitingStatus::WaitForTrezorPin, + on_ready: InitHwInProgressStatus::Initializing, + }) + .with_connect_timeout(TREZOR_CONNECT_TIMEOUT) + .with_pin_timeout(TREZOR_PIN_TIMEOUT); + + crypto_ctx.init_hw_ctx_with_trezor(&trezor_connect_processor).await?; + }, + } + Ok(SuccessResponse::new()) + } +} + +#[derive(Deserialize)] +pub struct InitTrezorRequest; + +pub async fn init_trezor(ctx: MmArc, _req: InitTrezorRequest) -> MmResult { + let init_ctx = MmInitContext::from_ctx(&ctx).map_to_mm(InitHwError::Internal)?; + let task = InitHwTask { + ctx, + hw_wallet_type: HwWalletType::Trezor, + }; + let task_id = RpcTaskManager::spawn_rpc_task(&init_ctx.init_hw_task_manager, task)?; + Ok(InitRpcTaskResponse { task_id }) +} + +pub async fn init_trezor_status(ctx: MmArc, req: RpcTaskStatusRequest) -> MmResult { + let coins_ctx = MmInitContext::from_ctx(&ctx).map_to_mm(RpcTaskStatusError::Internal)?; + let mut task_manager = coins_ctx + .init_hw_task_manager + .lock() + .map_to_mm(|e| RpcTaskStatusError::Internal(e.to_string()))?; + task_manager + .task_status(req.task_id, req.forget_if_finished) + .or_mm_err(|| RpcTaskStatusError::NoSuchTask(req.task_id)) +} + +pub async fn init_trezor_user_action( + ctx: MmArc, + req: HwRpcTaskUserActionRequest, +) -> MmResult { + let coins_ctx = MmInitContext::from_ctx(&ctx).map_to_mm(RpcTaskUserActionError::Internal)?; + let mut task_manager = coins_ctx + .init_hw_task_manager + .lock() + .map_to_mm(|e| RpcTaskUserActionError::Internal(e.to_string()))?; + task_manager.on_user_action(req.task_id, req.user_action)?; + Ok(SuccessResponse::new()) +} diff --git a/mm2src/lp_native_dex.rs b/mm2src/lp_native_dex.rs index 8b3c0c8287..24e633aba0 100644 --- a/mm2src/lp_native_dex.rs +++ b/mm2src/lp_native_dex.rs @@ -52,10 +52,9 @@ cfg_native! { } #[path = "lp_init/init_context.rs"] mod init_context; -#[path = "lp_init/mm_init_task.rs"] mod mm_init_task; -#[path = "lp_init/rpc_command.rs"] pub mod rpc_command; - -use mm_init_task::MmInitTask; +#[path = "lp_init/init_hw.rs"] pub mod init_hw; +// #[path = "lp_init/mm_init_task.rs"] mod mm_init_task; +// #[path = "lp_init/rpc_command.rs"] pub mod rpc_command; const NETID_7777_SEEDNODES: [&str; 3] = ["seed1.defimania.live", "seed2.defimania.live", "seed3.defimania.live"]; @@ -394,24 +393,13 @@ pub async fn lp_init(ctx: MmArc) -> MmInitResult<()> { }); } - if ctx.conf["passphrase"].is_null() { - // TODO - // Currently, `MmInitTask` initializes `CryptoCtx` with Hardware Wallet only. - // Later I'm going to change it. - // The main blocker is that if an error occurs while MarketMaker initializing, - // we should exit with an error since our users are used to this behaviour. - // But `MmInitTask` inserts the error into [`MmCtx::rpc_task_manager`] and doesn't stop anything. - let task = MmInitTask::new(ctx.clone()); - task.spawn()?; - } else { - let passphrase: String = - json::from_value(ctx.conf["passphrase"].clone()).map_to_mm(|e| MmInitError::ErrorDeserializingConfig { - field: "passphrase".to_owned(), - error: e.to_string(), - })?; - CryptoCtx::init_with_passphrase(ctx.clone(), &passphrase)?; - lp_init_continue(ctx.clone()).await?; - } + let passphrase: String = + json::from_value(ctx.conf["passphrase"].clone()).map_to_mm(|e| MmInitError::ErrorDeserializingConfig { + field: "passphrase".to_owned(), + error: e.to_string(), + })?; + CryptoCtx::init_with_iguana_passphrase(ctx.clone(), &passphrase)?; + lp_init_continue(ctx.clone()).await?; let ctx_id = ctx.ffi_handle().map_to_mm(MmInitError::Internal)?; diff --git a/mm2src/rpc/dispatcher/dispatcher.rs b/mm2src/rpc/dispatcher/dispatcher.rs index c4fe2a6851..a6d18425d7 100644 --- a/mm2src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/rpc/dispatcher/dispatcher.rs @@ -1,5 +1,5 @@ use super::{DispatcherError, DispatcherResult, PUBLIC_METHODS}; -use crate::mm2::lp_native_dex::rpc_command::{mm_init_status, mm_init_user_action}; +use crate::mm2::lp_native_dex::init_hw::{init_trezor, init_trezor_status, init_trezor_user_action}; use crate::mm2::lp_ordermatch::{start_simple_market_maker_bot, stop_simple_market_maker_bot}; use crate::mm2::rpc::rate_limiter::{process_rate_limit, RateLimitContext}; use crate::{mm2::lp_stats::{add_node_to_version_stat, remove_node_from_version_stat, start_version_stat_collection, @@ -129,14 +129,17 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, init_standalone_coin::).await, "init_qtum_status" => handle_mmrpc(ctx, request, init_standalone_coin_status::).await, "init_qtum_user_action" => handle_mmrpc(ctx, request, init_standalone_coin_user_action::).await, + "init_trezor" => handle_mmrpc(ctx, request, init_trezor).await, + "init_trezor_status" => handle_mmrpc(ctx, request, init_trezor_status).await, + "init_trezor_user_action" => handle_mmrpc(ctx, request, init_trezor_user_action).await, "init_utxo" => handle_mmrpc(ctx, request, init_standalone_coin::).await, "init_utxo_status" => handle_mmrpc(ctx, request, init_standalone_coin_status::).await, "init_utxo_user_action" => { handle_mmrpc(ctx, request, init_standalone_coin_user_action::).await }, "init_withdraw" => handle_mmrpc(ctx, request, init_withdraw).await, - "mm_init_status" => handle_mmrpc(ctx, request, mm_init_status).await, - "mm_init_user_action" => handle_mmrpc(ctx, request, mm_init_user_action).await, + // "mm_init_status" => handle_mmrpc(ctx, request, mm_init_status).await, + // "mm_init_user_action" => handle_mmrpc(ctx, request, mm_init_user_action).await, "my_tx_history" => handle_mmrpc(ctx, request, my_tx_history_v2_rpc).await, "recreate_swap_data" => handle_mmrpc(ctx, request, recreate_swap_data).await, "remove_delegation" => handle_mmrpc(ctx, request, remove_delegation).await, From bb6f421b0998e2bd86bea77d2992ac4166cf916d Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Mon, 21 Mar 2022 23:00:35 +0700 Subject: [PATCH 02/25] WIP Add JSONRPC batch requests --- mm2src/coins/utxo/rpc_clients.rs | 143 ++++++++++++++++++------------- mm2src/common/jsonrpc_client.rs | 36 +++++++- 2 files changed, 119 insertions(+), 60 deletions(-) diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index 11a65c3976..947ee86e76 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -8,8 +8,9 @@ use bigdecimal::BigDecimal; use chain::{BlockHeader, BlockHeaderBits, BlockHeaderNonce, OutPoint, Transaction as UtxoTx}; use common::custom_futures::{select_ok_sequential, FutureTimerExt}; use common::executor::{spawn, Timer}; -use common::jsonrpc_client::{JsonRpcClient, JsonRpcError, JsonRpcErrorType, JsonRpcMultiClient, JsonRpcRemoteAddr, - JsonRpcRequest, JsonRpcResponse, JsonRpcResponseFut, RpcRes}; +use common::jsonrpc_client::{JsonRpcBatchIds, JsonRpcBatchResponses, JsonRpcClient, JsonRpcError, JsonRpcErrorType, + JsonRpcMultiClient, JsonRpcRemoteAddr, JsonRpcRequest, JsonRpcResponse, + JsonRpcResponseFut, RpcRes}; use common::log::{error, info, warn}; use common::mm_error::prelude::*; use common::mm_number::{BigInt, MmNumber}; @@ -1294,6 +1295,30 @@ pub fn spawn_electrum( Ok(electrum_connect(url, config, event_handlers)) } +pub type JsonRpcPendingRequestsShared = Arc>; + +#[derive(Default)] +pub struct JsonRpcPendingRequests { + single: HashMap>, + batch: HashMap>, +} + +impl JsonRpcPendingRequests { + fn add_single(&mut self, id: String, response_sender: async_oneshot::Sender) { + self.single.insert(id, response_sender); + } + + fn add_batch(&mut self, ids: JsonRpcBatchIds, response_sender: async_oneshot::Sender) { + self.batch.insert(ids, response_sender); + } + + fn remove_single(&mut self, id: &str) -> Option> { self.single.remove(id) } + + fn remove_batch(&mut self, ids: &JsonRpcBatchIds) -> Option> { + self.batch.remove(ids) + } +} + #[derive(Debug)] /// Represents the active Electrum connection to selected address pub struct ElectrumConnection { @@ -1307,7 +1332,7 @@ pub struct ElectrumConnection { /// The Sender used to shutdown the background connection loop when ElectrumConnection is dropped shutdown_tx: Option>, /// Responses are stored here - responses: Arc>>>, + responses: JsonRpcPendingRequestsShared, /// Selected protocol version. The value is initialized after the server.version RPC call. protocol_version: AsyncMutex>, } @@ -1865,62 +1890,64 @@ fn rx_to_stream(rx: mpsc::Receiver>) -> impl Stream, Erro rx.map_err(|_| panic!("errors not possible on rx")) } -async fn electrum_process_json( - raw_json: Json, - arc: &Arc>>>, -) { +async fn electrum_process_json(raw_json: Json, arc: &JsonRpcPendingRequestsShared) { // detect if we got standard JSONRPC response or subscription response as JSONRPC request - if raw_json["method"].is_null() && raw_json["params"].is_null() { - let response: JsonRpcResponse = match json::from_value(raw_json) { - Ok(res) => res, - Err(e) => { - error!("{}", e); - return; - }, - }; - let mut resp = arc.lock().await; - // the corresponding sender may not exist, receiver may be dropped - // these situations are not considered as errors so we just silently skip them - if let Some(tx) = resp.remove(&response.id.to_string()) { - tx.send(response).unwrap_or(()) - } - drop(resp); - } else { - let request: JsonRpcRequest = match json::from_value(raw_json) { - Ok(res) => res, - Err(e) => { - error!("{}", e); - return; - }, - }; - let id = match request.method.as_ref() { - BLOCKCHAIN_HEADERS_SUB_ID => BLOCKCHAIN_HEADERS_SUB_ID, - _ => { - error!("Couldn't get id of request {:?}", request); - return; - }, - }; + #[derive(Deserialize)] + #[serde(untagged)] + enum ElectrumRpcResponseEnum { + /// The standard JSONRPC single response. + SingleResponse(JsonRpcResponse), + /// The batch of standard JSONRPC responses. + BatchResponses(JsonRpcBatchResponses), + /// The subscription response as JSONRPC request. + SubscriptionNotification(JsonRpcRequest), + } + + let response: ElectrumRpcResponseEnum = match json::from_value(raw_json) { + Ok(res) => res, + Err(e) => { + error!("{}", e); + return; + }, + }; - let response = JsonRpcResponse { - id: id.into(), - jsonrpc: "2.0".into(), - result: request.params[0].clone(), - error: Json::Null, - }; - let mut resp = arc.lock().await; - // the corresponding sender may not exist, receiver may be dropped - // these situations are not considered as errors so we just silently skip them - if let Some(tx) = resp.remove(&response.id.to_string()) { - tx.send(response).unwrap_or(()) - } - drop(resp); + let mut pending = arc.lock().await; + + // the corresponding sender may not exist, receiver may be dropped + // these situations are not considered as errors so we just silently skip them + match response { + ElectrumRpcResponseEnum::SingleResponse(resp) => { + if let Some(tx) = pending.remove_single(&resp.id.to_string()) { + tx.send(resp).ok(); + } + }, + ElectrumRpcResponseEnum::BatchResponses(batch) => { + if let Some(tx) = pending.remove_batch(&batch.ids()) { + tx.send(batch).ok(); + } + }, + ElectrumRpcResponseEnum::SubscriptionNotification(req) => { + let id = match req.method.as_ref() { + BLOCKCHAIN_HEADERS_SUB_ID => BLOCKCHAIN_HEADERS_SUB_ID, + _ => { + error!("Couldn't get id of request {:?}", req); + return; + }, + }; + let resp = JsonRpcResponse { + id: id.into(), + jsonrpc: "2.0".into(), + result: req.params[0].clone(), + error: Json::Null, + }; + if let Some(tx) = pending.remove_single(&resp.id.to_string()) { + tx.send(resp).ok(); + } + }, } } -async fn electrum_process_chunk( - chunk: &[u8], - arc: &Arc>>>, -) { +async fn electrum_process_chunk(chunk: &[u8], arc: &JsonRpcPendingRequestsShared) { // we should split the received chunk because we can get several responses in 1 chunk. let split = chunk.split(|item| *item == b'\n'); for chunk in split { @@ -2029,7 +2056,7 @@ async fn electrum_last_chunk_loop(last_chunk: Arc) { async fn connect_loop( config: ElectrumConfig, addr: String, - responses: Arc>>>, + responses: JsonRpcPendingRequestsShared, connection_tx: Arc>>>>, event_handlers: Vec, ) -> Result<(), ()> { @@ -2257,7 +2284,7 @@ fn electrum_connect( event_handlers: Vec, ) -> ElectrumConnection { let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); - let responses = Arc::new(AsyncMutex::new(HashMap::new())); + let responses = Arc::new(AsyncMutex::new(JsonRpcPendingRequests::default())); let tx = Arc::new(AsyncMutex::new(None)); let connect_loop = connect_loop( @@ -2283,7 +2310,7 @@ fn electrum_connect( fn electrum_request( request: JsonRpcRequest, tx: mpsc::Sender>, - responses: Arc>>>, + responses: JsonRpcPendingRequestsShared, timeout: u64, ) -> Box + Send + 'static> { let send_fut = async move { @@ -2297,7 +2324,7 @@ fn electrum_request( let request_id = request.get_id().to_string(); let (req_tx, resp_rx) = async_oneshot::channel(); - responses.lock().await.insert(request_id, req_tx); + responses.lock().await.add_single(request_id, req_tx); try_s!(tx.send(json.into_bytes()).compat().await); let response = try_s!(resp_rx.await); Ok(response) diff --git a/mm2src/common/jsonrpc_client.rs b/mm2src/common/jsonrpc_client.rs index b94cd6fb27..6bb88676d4 100644 --- a/mm2src/common/jsonrpc_client.rs +++ b/mm2src/common/jsonrpc_client.rs @@ -60,8 +60,18 @@ impl From for JsonRpcRemoteAddr { fn from(addr: String) -> Self { JsonRpcRemoteAddr(addr) } } +#[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct JsonRpcBatchIds(Vec); + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(untagged)] +pub enum JsonRpcRequestEnum { + Single(JsonRpcRequest), + Batch(JsonRpcBatchRequests), +} + /// Serializable RPC request -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct JsonRpcRequest { pub jsonrpc: String, #[serde(default)] @@ -74,7 +84,22 @@ impl JsonRpcRequest { pub fn get_id(&self) -> &str { &self.id } } -#[derive(Deserialize, Debug, Clone)] +/// Serializable RPC request +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct JsonRpcBatchRequests(Vec); + +impl JsonRpcBatchRequests { + pub fn ids(&self) -> JsonRpcBatchIds { JsonRpcBatchIds(self.0.iter().map(|req| req.id.clone()).collect()) } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub enum JsonRpcResponseEnum { + Single(JsonRpcResponse), + Batch(JsonRpcBatchResponses), +} + +#[derive(Clone, Debug, Deserialize)] pub struct JsonRpcResponse { #[serde(default)] pub jsonrpc: String, @@ -86,6 +111,13 @@ pub struct JsonRpcResponse { pub error: Json, } +#[derive(Clone, Debug, Deserialize)] +pub struct JsonRpcBatchResponses(Vec); + +impl JsonRpcBatchResponses { + pub fn ids(&self) -> JsonRpcBatchIds { JsonRpcBatchIds(self.0.iter().map(|res| res.id.clone()).collect()) } +} + #[derive(Clone, Debug)] pub struct JsonRpcError { /// Additional member contains an instance info that implements the JsonRpcClient trait. From 94af4fd50fef8f0af44f847d821ea6d0b794caf5 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Tue, 22 Mar 2022 18:32:03 +0700 Subject: [PATCH 03/25] Add `JsonRpcClient::send_batch_request` --- mm2src/coins/utxo/rpc_clients.rs | 124 +++++++++------------- mm2src/common/jsonrpc_client.rs | 172 +++++++++++++++++++++++++++---- 2 files changed, 198 insertions(+), 98 deletions(-) diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index 947ee86e76..0ef88849fb 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -8,9 +8,9 @@ use bigdecimal::BigDecimal; use chain::{BlockHeader, BlockHeaderBits, BlockHeaderNonce, OutPoint, Transaction as UtxoTx}; use common::custom_futures::{select_ok_sequential, FutureTimerExt}; use common::executor::{spawn, Timer}; -use common::jsonrpc_client::{JsonRpcBatchIds, JsonRpcBatchResponses, JsonRpcClient, JsonRpcError, JsonRpcErrorType, - JsonRpcMultiClient, JsonRpcRemoteAddr, JsonRpcRequest, JsonRpcResponse, - JsonRpcResponseFut, RpcRes}; +use common::jsonrpc_client::{JsonRpcBatchResponse, JsonRpcClient, JsonRpcError, JsonRpcErrorType, JsonRpcId, + JsonRpcMultiClient, JsonRpcRemoteAddr, JsonRpcRequest, JsonRpcRequestEnum, + JsonRpcResponse, JsonRpcResponseEnum, JsonRpcResponseFut, RpcRes}; use common::log::{error, info, warn}; use common::mm_error::prelude::*; use common::mm_number::{BigInt, MmNumber}; @@ -58,6 +58,8 @@ cfg_native! { } pub type AddressesByLabelResult = HashMap; +pub type JsonRpcPendingRequestsShared = Arc>; +pub type JsonRpcPendingRequests = HashMap>; #[derive(Debug, Deserialize)] #[allow(dead_code)] @@ -558,14 +560,14 @@ impl JsonRpcClient for NativeClientImpl { fn client_info(&self) -> String { UtxoJsonRpcClientInfo::client_info(self) } #[cfg(target_arch = "wasm32")] - fn transport(&self, _request: JsonRpcRequest) -> JsonRpcResponseFut { + fn transport(&self, _request: JsonRpcRequestEnum) -> JsonRpcResponseFut { Box::new(futures01::future::err(ERRL!( "'NativeClientImpl' must be used in native mode only" ))) } #[cfg(not(target_arch = "wasm32"))] - fn transport(&self, request: JsonRpcRequest) -> JsonRpcResponseFut { + fn transport(&self, request: JsonRpcRequestEnum) -> JsonRpcResponseFut { use common::transport::slurp_req; let request_body = try_fus!(json::to_string(&request)); @@ -582,7 +584,7 @@ impl JsonRpcClient for NativeClientImpl { let event_handles = self.event_handlers.clone(); Box::new(slurp_req(http_request).boxed().compat().then( - move |result| -> Result<(JsonRpcRemoteAddr, JsonRpcResponse), String> { + move |result| -> Result<(JsonRpcRemoteAddr, JsonRpcResponseEnum), String> { let res = try_s!(result); // measure now only body length, because the `hyper` crate doesn't allow to get total HTTP packet length event_handles.on_incoming_response(&res.2); @@ -1295,30 +1297,6 @@ pub fn spawn_electrum( Ok(electrum_connect(url, config, event_handlers)) } -pub type JsonRpcPendingRequestsShared = Arc>; - -#[derive(Default)] -pub struct JsonRpcPendingRequests { - single: HashMap>, - batch: HashMap>, -} - -impl JsonRpcPendingRequests { - fn add_single(&mut self, id: String, response_sender: async_oneshot::Sender) { - self.single.insert(id, response_sender); - } - - fn add_batch(&mut self, ids: JsonRpcBatchIds, response_sender: async_oneshot::Sender) { - self.batch.insert(ids, response_sender); - } - - fn remove_single(&mut self, id: &str) -> Option> { self.single.remove(id) } - - fn remove_batch(&mut self, ids: &JsonRpcBatchIds) -> Option> { - self.batch.remove(ids) - } -} - #[derive(Debug)] /// Represents the active Electrum connection to selected address pub struct ElectrumConnection { @@ -1425,8 +1403,8 @@ pub struct ElectrumClientImpl { async fn electrum_request_multi( client: ElectrumClient, - request: JsonRpcRequest, -) -> Result<(JsonRpcRemoteAddr, JsonRpcResponse), String> { + request: JsonRpcRequestEnum, +) -> Result<(JsonRpcRemoteAddr, JsonRpcResponseEnum), String> { let mut futures = vec![]; let connections = client.connections.lock().await; for (i, connection) in connections.iter().enumerate() { @@ -1449,31 +1427,32 @@ async fn electrum_request_multi( if futures.is_empty() { return ERR!("All electrums are currently disconnected"); } - if request.method != "server.ping" { - match select_ok_sequential(futures).compat().await { - Ok((res, no_of_failed_requests)) => { - client.clone().rotate_servers(no_of_failed_requests).await; - Ok(res) - }, - Err(e) => return ERR!("{:?}", e), - } - } else { - // server.ping must be sent to all servers to keep all connections alive - Ok(try_s!( - select_ok(futures) + + match request { + JsonRpcRequestEnum::Single(single) if single.method == "server.ping" => { + // server.ping must be sent to all servers to keep all connections alive + return select_ok(futures) .map(|(result, _)| result) .map_err(|e| ERRL!("{:?}", e)) .compat() - .await - )) + .await; + }, + _ => (), } + + let (res, no_of_failed_requests) = select_ok_sequential(futures) + .compat() + .await + .map_err(|e| ERRL!("{:?}", e))?; + client.rotate_servers(no_of_failed_requests).await; + Ok(res) } async fn electrum_request_to( client: ElectrumClient, - request: JsonRpcRequest, + request: JsonRpcRequestEnum, to_addr: String, -) -> Result<(JsonRpcRemoteAddr, JsonRpcResponse), String> { +) -> Result<(JsonRpcRemoteAddr, JsonRpcResponseEnum), String> { let (tx, responses) = { let connections = client.connections.lock().await; let connection = connections @@ -1582,13 +1561,13 @@ impl JsonRpcClient for ElectrumClient { fn client_info(&self) -> String { UtxoJsonRpcClientInfo::client_info(self) } - fn transport(&self, request: JsonRpcRequest) -> JsonRpcResponseFut { + fn transport(&self, request: JsonRpcRequestEnum) -> JsonRpcResponseFut { Box::new(electrum_request_multi(self.clone(), request).boxed().compat()) } } impl JsonRpcMultiClient for ElectrumClient { - fn transport_exact(&self, to_addr: String, request: JsonRpcRequest) -> JsonRpcResponseFut { + fn transport_exact(&self, to_addr: String, request: JsonRpcRequestEnum) -> JsonRpcResponseFut { Box::new(electrum_request_to(self.clone(), request, to_addr).boxed().compat()) } } @@ -1898,7 +1877,7 @@ async fn electrum_process_json(raw_json: Json, arc: &JsonRpcPendingRequestsShare /// The standard JSONRPC single response. SingleResponse(JsonRpcResponse), /// The batch of standard JSONRPC responses. - BatchResponses(JsonRpcBatchResponses), + BatchResponses(JsonRpcBatchResponse), /// The subscription response as JSONRPC request. SubscriptionNotification(JsonRpcRequest), } @@ -1911,21 +1890,9 @@ async fn electrum_process_json(raw_json: Json, arc: &JsonRpcPendingRequestsShare }, }; - let mut pending = arc.lock().await; - - // the corresponding sender may not exist, receiver may be dropped - // these situations are not considered as errors so we just silently skip them - match response { - ElectrumRpcResponseEnum::SingleResponse(resp) => { - if let Some(tx) = pending.remove_single(&resp.id.to_string()) { - tx.send(resp).ok(); - } - }, - ElectrumRpcResponseEnum::BatchResponses(batch) => { - if let Some(tx) = pending.remove_batch(&batch.ids()) { - tx.send(batch).ok(); - } - }, + let response = match response { + ElectrumRpcResponseEnum::SingleResponse(single) => JsonRpcResponseEnum::Single(single), + ElectrumRpcResponseEnum::BatchResponses(batch) => JsonRpcResponseEnum::Batch(batch), ElectrumRpcResponseEnum::SubscriptionNotification(req) => { let id = match req.method.as_ref() { BLOCKCHAIN_HEADERS_SUB_ID => BLOCKCHAIN_HEADERS_SUB_ID, @@ -1934,16 +1901,20 @@ async fn electrum_process_json(raw_json: Json, arc: &JsonRpcPendingRequestsShare return; }, }; - let resp = JsonRpcResponse { + JsonRpcResponseEnum::Single(JsonRpcResponse { id: id.into(), jsonrpc: "2.0".into(), result: req.params[0].clone(), error: Json::Null, - }; - if let Some(tx) = pending.remove_single(&resp.id.to_string()) { - tx.send(resp).ok(); - } + }) }, + }; + + // the corresponding sender may not exist, receiver may be dropped + // these situations are not considered as errors so we just silently skip them + let mut pending = arc.lock().await; + if let Some(tx) = pending.remove(&response.rpc_id()) { + tx.send(response).ok(); } } @@ -2308,11 +2279,11 @@ fn electrum_connect( } fn electrum_request( - request: JsonRpcRequest, + request: JsonRpcRequestEnum, tx: mpsc::Sender>, responses: JsonRpcPendingRequestsShared, timeout: u64, -) -> Box + Send + 'static> { +) -> Box + Send + 'static> { let send_fut = async move { let mut json = try_s!(json::to_string(&request)); #[cfg(not(target_arch = "wasm"))] @@ -2322,12 +2293,11 @@ fn electrum_request( json.push('\n'); } - let request_id = request.get_id().to_string(); let (req_tx, resp_rx) = async_oneshot::channel(); - responses.lock().await.add_single(request_id, req_tx); + responses.lock().await.insert(request.rpc_id(), req_tx); try_s!(tx.send(json.into_bytes()).compat().await); - let response = try_s!(resp_rx.await); - Ok(response) + let resps = try_s!(resp_rx.await); + Ok(resps) }; let send_fut = send_fut .boxed() diff --git a/mm2src/common/jsonrpc_client.rs b/mm2src/common/jsonrpc_client.rs index 6bb88676d4..81e10aa705 100644 --- a/mm2src/common/jsonrpc_client.rs +++ b/mm2src/common/jsonrpc_client.rs @@ -61,13 +61,34 @@ impl From for JsonRpcRemoteAddr { } #[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] -pub struct JsonRpcBatchIds(Vec); +pub enum JsonRpcId { + Single(String), + Batch(Vec), +} -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Deserialize, Serialize)] #[serde(untagged)] pub enum JsonRpcRequestEnum { Single(JsonRpcRequest), - Batch(JsonRpcBatchRequests), + Batch(JsonRpcBatchRequest), +} + +impl JsonRpcRequestEnum { + pub fn rpc_id(&self) -> JsonRpcId { + match self { + JsonRpcRequestEnum::Single(single) => single.rpc_id(), + JsonRpcRequestEnum::Batch(batch) => batch.rpc_id(), + } + } +} + +impl fmt::Debug for JsonRpcRequestEnum { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JsonRpcRequestEnum::Single(single) => single.fmt(f), + JsonRpcRequestEnum::Batch(batch) => batch.fmt(f), + } + } } /// Serializable RPC request @@ -82,21 +103,44 @@ pub struct JsonRpcRequest { impl JsonRpcRequest { pub fn get_id(&self) -> &str { &self.id } + + pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Single(self.id.clone()) } +} + +impl From for JsonRpcRequestEnum { + fn from(single: JsonRpcRequest) -> Self { JsonRpcRequestEnum::Single(single) } } /// Serializable RPC request #[derive(Clone, Debug, Deserialize, Serialize)] -pub struct JsonRpcBatchRequests(Vec); +pub struct JsonRpcBatchRequest(Vec); + +impl JsonRpcBatchRequest { + pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Batch(self.0.iter().map(|req| req.id.clone()).collect()) } + + pub fn len(&self) -> usize { self.0.len() } + + pub fn is_empty(&self) -> bool { self.0.is_empty() } +} -impl JsonRpcBatchRequests { - pub fn ids(&self) -> JsonRpcBatchIds { JsonRpcBatchIds(self.0.iter().map(|req| req.id.clone()).collect()) } +impl From for JsonRpcRequestEnum { + fn from(batch: JsonRpcBatchRequest) -> Self { JsonRpcRequestEnum::Batch(batch) } } #[derive(Clone, Debug, Deserialize)] #[serde(untagged)] pub enum JsonRpcResponseEnum { Single(JsonRpcResponse), - Batch(JsonRpcBatchResponses), + Batch(JsonRpcBatchResponse), +} + +impl JsonRpcResponseEnum { + pub fn rpc_id(&self) -> JsonRpcId { + match self { + JsonRpcResponseEnum::Single(single) => single.rpc_id(), + JsonRpcResponseEnum::Batch(batch) => batch.rpc_id(), + } + } } #[derive(Clone, Debug, Deserialize)] @@ -111,11 +155,26 @@ pub struct JsonRpcResponse { pub error: Json, } +impl JsonRpcResponse { + pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Single(self.id.clone()) } +} + #[derive(Clone, Debug, Deserialize)] -pub struct JsonRpcBatchResponses(Vec); +pub struct JsonRpcBatchResponse(Vec); + +impl JsonRpcBatchResponse { + pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Batch(self.0.iter().map(|res| res.id.clone()).collect()) } + + pub fn len(&self) -> usize { self.0.len() } -impl JsonRpcBatchResponses { - pub fn ids(&self) -> JsonRpcBatchIds { JsonRpcBatchIds(self.0.iter().map(|res| res.id.clone()).collect()) } + pub fn is_empty(&self) -> bool { self.0.is_empty() } +} + +impl IntoIterator for JsonRpcBatchResponse { + type Item = JsonRpcResponse; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { self.0.into_iter() } } #[derive(Clone, Debug)] @@ -124,7 +183,7 @@ pub struct JsonRpcError { /// The info is used in particular to supplement the error info. pub client_info: String, /// Source Rpc request. - pub request: JsonRpcRequest, + pub request: JsonRpcRequestEnum, /// Error type. pub error: JsonRpcErrorType, } @@ -148,7 +207,7 @@ impl fmt::Display for JsonRpcError { } pub type JsonRpcResponseFut = - Box + Send + 'static>; + Box + Send + 'static>; pub type RpcRes = Box + Send + 'static>; pub trait JsonRpcClient { @@ -159,20 +218,28 @@ pub trait JsonRpcClient { /// Get info that is used in particular to supplement the error info fn client_info(&self) -> String; - fn transport(&self, request: JsonRpcRequest) -> JsonRpcResponseFut; + fn transport(&self, request: JsonRpcRequestEnum) -> JsonRpcResponseFut; fn send_request(&self, request: JsonRpcRequest) -> RpcRes { let client_info = self.client_info(); Box::new( - self.transport(request.clone()) - .then(move |result| process_transport_result(result, client_info, request)), + self.transport(JsonRpcRequestEnum::Single(request.clone())) + .then(move |result| process_transport_single_result(result, client_info, request)), + ) + } + + fn send_batch_request(&self, request: JsonRpcBatchRequest) -> RpcRes> { + let client_info = self.client_info(); + Box::new( + self.transport(JsonRpcRequestEnum::Batch(request.clone())) + .then(move |result| process_transport_batch_result(result, client_info, request)), ) } } /// The trait is used when the rpc client instance has more than one remote endpoints. pub trait JsonRpcMultiClient: JsonRpcClient { - fn transport_exact(&self, to_addr: String, request: JsonRpcRequest) -> JsonRpcResponseFut; + fn transport_exact(&self, to_addr: String, request: JsonRpcRequestEnum) -> JsonRpcResponseFut; fn send_request_to( &self, @@ -181,19 +248,57 @@ pub trait JsonRpcMultiClient: JsonRpcClient { ) -> RpcRes { let client_info = self.client_info(); Box::new( - self.transport_exact(to_addr.to_owned(), request.clone()) - .then(move |result| process_transport_result(result, client_info, request)), + self.transport_exact(to_addr.to_owned(), JsonRpcRequestEnum::Single(request.clone())) + .then(move |result| process_transport_single_result(result, client_info, request)), ) } } -fn process_transport_result( - result: Result<(JsonRpcRemoteAddr, JsonRpcResponse), String>, +fn process_transport_single_result( + result: Result<(JsonRpcRemoteAddr, JsonRpcResponseEnum), String>, client_info: String, request: JsonRpcRequest, ) -> Result { + let request = JsonRpcRequestEnum::Single(request); + + match result { + Ok((remote_addr, JsonRpcResponseEnum::Single(single))) => { + process_single_response(client_info, remote_addr, request, single) + }, + Ok((remote_addr, JsonRpcResponseEnum::Batch(batch))) => { + let error = ERRL!("Expeced single response, found batch response: {:?}", batch); + Err(JsonRpcError { + client_info, + request, + error: JsonRpcErrorType::Parse(remote_addr, error), + }) + }, + Err(e) => Err(JsonRpcError { + client_info, + request, + error: JsonRpcErrorType::Transport(e), + }), + } +} + +fn process_transport_batch_result( + result: Result<(JsonRpcRemoteAddr, JsonRpcResponseEnum), String>, + client_info: String, + request: JsonRpcBatchRequest, +) -> Result, JsonRpcError> { + let expected_len = request.len(); + let request = JsonRpcRequestEnum::Batch(request); + let (remote_addr, response) = match result { - Ok(r) => r, + Ok((remote_addr, JsonRpcResponseEnum::Batch(batch))) => (remote_addr, batch), + Ok((remote_addr, JsonRpcResponseEnum::Single(single))) => { + let error = ERRL!("Expected batch response, found single response: {:?}", single); + return Err(JsonRpcError { + client_info, + request, + error: JsonRpcErrorType::Parse(remote_addr, error), + }); + }, Err(e) => { return Err(JsonRpcError { client_info, @@ -203,6 +308,31 @@ fn process_transport_result( }, }; + if response.len() != expected_len { + let error = ERRL!( + "Expected '{}' elements in batch response, found '{}'", + expected_len, + response.len() + ); + return Err(JsonRpcError { + client_info, + request, + error: JsonRpcErrorType::Parse(remote_addr, error), + }); + } + + response + .into_iter() + .map(|resp| process_single_response(client_info.clone(), remote_addr.clone(), request.clone(), resp)) + .collect() +} + +fn process_single_response( + client_info: String, + remote_addr: JsonRpcRemoteAddr, + request: JsonRpcRequestEnum, + response: JsonRpcResponse, +) -> Result { if !response.error.is_null() { return Err(JsonRpcError { client_info, From 5d23039fb9de938f863a02c8fab249639b543a48 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Fri, 25 Mar 2022 15:20:17 +0700 Subject: [PATCH 04/25] Add and implement `UtxoRpcClientOps::display_balances` * Process batch response in the same order in which the requests were sent --- Cargo.lock | 13 +-- mm2src/coins/Cargo.toml | 2 +- mm2src/coins/qrc20/history.rs | 4 +- mm2src/coins/utxo/rpc_clients.rs | 66 ++++++++++++++- mm2src/coins/utxo/utxo_common.rs | 4 +- mm2src/coins/utxo/utxo_tests.rs | 85 ++++++++++++++++++- mm2src/common/Cargo.toml | 2 +- mm2src/common/common.rs | 22 ++--- mm2src/common/jsonrpc_client.rs | 137 +++++++++++++++++++++++-------- 9 files changed, 272 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0c8b2c7012..8e54881ded 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -830,7 +830,7 @@ dependencies = [ "gstuff", "hex 0.4.3", "http 0.2.1", - "itertools 0.9.0", + "itertools 0.10.1", "js-sys", "jsonrpc-core", "keys", @@ -938,7 +938,7 @@ dependencies = [ "hyper", "hyper-rustls", "indexmap", - "itertools 0.8.2", + "itertools 0.10.1", "js-sys", "keys", "lazy_static", @@ -2454,15 +2454,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" -[[package]] -name = "itertools" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.9.0" diff --git a/mm2src/coins/Cargo.toml b/mm2src/coins/Cargo.toml index b7f1ea816d..fa60d9821d 100644 --- a/mm2src/coins/Cargo.toml +++ b/mm2src/coins/Cargo.toml @@ -41,7 +41,7 @@ futures = { version = "0.3", package = "futures", features = ["compat", "async-a gstuff = { version = "0.7", features = ["nightly"] } hex = "0.4.2" http = "0.2" -itertools = "0.9" +itertools = { version = "0.10", features = ["use_std"] } jsonrpc-core = "8.0.1" keys = { path = "../mm2_bitcoin/keys" } lazy_static = "1.4" diff --git a/mm2src/coins/qrc20/history.rs b/mm2src/coins/qrc20/history.rs index acc1152a92..82b1647094 100644 --- a/mm2src/coins/qrc20/history.rs +++ b/mm2src/coins/qrc20/history.rs @@ -359,7 +359,9 @@ impl Qrc20Coin { } } }, - JsonRpcErrorType::Transport(err) | JsonRpcErrorType::Parse(_, err) => { + JsonRpcErrorType::InvalidRequest(err) + | JsonRpcErrorType::Transport(err) + | JsonRpcErrorType::Parse(_, err) => { return RequestTxHistoryResult::Retry { error: ERRL!("Error {} on blockchain_contract_event_get_history", err), }; diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index 0ef88849fb..bb4a138be5 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -8,8 +8,8 @@ use bigdecimal::BigDecimal; use chain::{BlockHeader, BlockHeaderBits, BlockHeaderNonce, OutPoint, Transaction as UtxoTx}; use common::custom_futures::{select_ok_sequential, FutureTimerExt}; use common::executor::{spawn, Timer}; -use common::jsonrpc_client::{JsonRpcBatchResponse, JsonRpcClient, JsonRpcError, JsonRpcErrorType, JsonRpcId, - JsonRpcMultiClient, JsonRpcRemoteAddr, JsonRpcRequest, JsonRpcRequestEnum, +use common::jsonrpc_client::{BatchRequest, JsonRpcBatchResponse, JsonRpcClient, JsonRpcError, JsonRpcErrorType, + JsonRpcId, JsonRpcMultiClient, JsonRpcRemoteAddr, JsonRpcRequest, JsonRpcRequestEnum, JsonRpcResponse, JsonRpcResponseEnum, JsonRpcResponseFut, RpcRes}; use common::log::{error, info, warn}; use common::mm_error::prelude::*; @@ -25,6 +25,7 @@ use futures01::future::select_ok; use futures01::sync::{mpsc, oneshot}; use futures01::{Future, Sink, Stream}; use http::Uri; +use itertools::Itertools; use keys::hash::H256; use keys::{Address, Type as ScriptType}; #[cfg(test)] use mocktopus::macros::*; @@ -238,6 +239,7 @@ pub enum UtxoRpcError { impl From for UtxoRpcError { fn from(e: JsonRpcError) -> Self { match e.error { + JsonRpcErrorType::InvalidRequest(_) => UtxoRpcError::Internal(e.to_string()), JsonRpcErrorType::Transport(_) => UtxoRpcError::Transport(e), JsonRpcErrorType::Parse(_, _) | JsonRpcErrorType::Response(_, _) => UtxoRpcError::ResponseParseError(e), } @@ -269,6 +271,8 @@ pub trait UtxoRpcClientOps: fmt::Debug + Send + Sync + 'static { fn display_balance(&self, address: Address, decimals: u8) -> RpcRes; + fn display_balances(&self, addresses: Vec
, decimals: u8) -> RpcRes>; + /// returns fee estimation per KByte in satoshis fn estimate_fee_sat( &self, @@ -300,6 +304,7 @@ pub trait UtxoRpcClientOps: fmt::Debug + Send + Sync + 'static { } #[derive(Clone, Deserialize, Debug)] +#[cfg_attr(test, derive(Default))] pub struct NativeUnspent { pub txid: H256Json, pub vout: u32, @@ -671,6 +676,35 @@ impl UtxoRpcClientOps for NativeClient { ) } + fn display_balances(&self, addresses: Vec
, _decimals: u8) -> RpcRes> { + let addresses_str: Vec<_> = addresses.iter().map(ToString::to_string).collect(); + Box::new( + self.list_unspent_impl(0, i32::MAX, addresses_str.clone()) + .map(move |unspents| { + // First, calculate the balance of each address returned from `NativeClient::list_unspent_impl`. + let balances: HashMap = unspents + .into_iter() + // Group `NativeUnspent` by the addresses. + .into_grouping_map_by(|unspent| unspent.address.clone()) + // Fold `Vec` related to one particular address to get the total balance of that address. + .fold(BigDecimal::from(0), |sum, _key, unspent| { + sum + unspent.amount.to_decimal() + }); + // Secondly, iterate over the requested `addresses` and try to find corresponding balance. + addresses + .into_iter() + .zip(addresses_str) + .map(|(address, address_str)| { + // If `balances` doesn't contain `address`, there are no unspents related to the address. + // Consider the balance of that address equal to 0. + let balance = balances.get(&address_str).cloned().unwrap_or_default(); + (address, balance) + }) + .collect() + }), + ) + } + fn estimate_fee_sat( &self, decimals: u8, @@ -1629,6 +1663,16 @@ impl ElectrumClient { Box::new(fut.boxed().compat()) } + pub fn scripthash_get_balances(&self, hashes: I) -> RpcRes> + where + I: IntoIterator, + { + hashes + .into_iter() + .map(|hash| rpc_req!(self, "blockchain.scripthash.get_balance", &hash)) + .batch_rpc(self) + } + /// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#blockchain-headers-subscribe pub fn blockchain_headers_subscribe(&self) -> RpcRes { rpc_func!(self, "blockchain.headers.subscribe") @@ -1744,6 +1788,24 @@ impl UtxoRpcClientOps for ElectrumClient { })) } + fn display_balances(&self, addresses: Vec
, decimals: u8) -> RpcRes> { + let hashes = addresses.iter().map(|address| { + let hash = electrum_script_hash(&output_script(address, ScriptType::P2PKH)); + hex::encode(hash) + }); + Box::new(self.scripthash_get_balances(hashes).map(move |results| { + results + .into_iter() + .zip(addresses) + .map(|(result, address)| { + let balance_sat = BigInt::from(result.confirmed) + BigInt::from(result.unconfirmed); + let balance_dec = BigDecimal::from(balance_sat) / BigDecimal::from(10u64.pow(decimals as u32)); + (address, balance_dec) + }) + .collect() + })) + } + fn estimate_fee_sat( &self, decimals: u8, diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 7969576555..ddc0fcdc2e 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2240,7 +2240,9 @@ where let electrum_history = match client.scripthash_get_history(&hex::encode(script_hash)).compat().await { Ok(value) => value, Err(e) => match &e.error { - JsonRpcErrorType::Transport(e) | JsonRpcErrorType::Parse(_, e) => { + JsonRpcErrorType::InvalidRequest(e) + | JsonRpcErrorType::Transport(e) + | JsonRpcErrorType::Parse(_, e) => { return RequestTxHistoryResult::Retry { error: ERRL!("Error {} on scripthash_get_history", e), }; diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 72101a374b..305c09a2ee 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -6,7 +6,8 @@ use crate::hd_wallet_storage::{HDWalletMockStorage, HDWalletStorageInternalOps}; use crate::utxo::qtum::{qtum_coin_with_priv_key, QtumCoin, QtumDelegationOps, QtumDelegationRequest}; use crate::utxo::rpc_clients::{BlockHashOrHeight, ElectrumBalance, ElectrumClient, ElectrumClientImpl, GetAddressInfoRes, ListSinceBlockRes, ListTransactionsItem, NativeClient, - NativeClientImpl, NetworkInfo, UtxoRpcClientOps, ValidateAddressRes, VerboseBlock}; + NativeClientImpl, NativeUnspent, NetworkInfo, UtxoRpcClientOps, ValidateAddressRes, + VerboseBlock}; use crate::utxo::utxo_builder::{UtxoArcBuilder, UtxoCoinBuilderCommonOps}; use crate::utxo::utxo_common::UtxoTxBuilder; use crate::utxo::utxo_standard::{utxo_standard_coin_with_priv_key, UtxoStandardCoin}; @@ -15,6 +16,7 @@ use crate::{CoinBalance, PrivKeyBuildPolicy, StakingInfosDetails, SwapOps, Trade use bigdecimal::{BigDecimal, Signed}; use chain::OutPoint; use common::executor::Timer; +use common::jsonrpc_client::JsonRpcErrorType; use common::mm_ctx::MmCtxBuilder; use common::privkey::key_pair_from_seed; use common::{block_on, now_ms, OrdRange, PagingOptionsEnum, DEX_FEE_ADDR_RAW_PUBKEY}; @@ -3850,3 +3852,84 @@ fn test_electrum_balance_deserializing() { assert_eq!(actual.confirmed, i128::MIN); assert_eq!(actual.unconfirmed, i128::MAX); } + +#[test] +fn test_electrum_display_balances() { + let rpc_client = electrum_client_for_test(RICK_ELECTRUM_ADDRS); + + let expected: Vec<(Address, BigDecimal)> = vec![ + ("RG278CfeNPFtNztFZQir8cgdWexVhViYVy".into(), BigDecimal::from(5.77699)), + ("RYPz6Lr4muj4gcFzpMdv3ks1NCGn3mkDPN".into(), BigDecimal::from(0)), + ("RJeDDtDRtKUoL8BCKdH7TNCHqUKr7kQRsi".into(), BigDecimal::from(0.77699)), + ("RQHn9VPHBqNjYwyKfJbZCiaxVrWPKGQjeF".into(), BigDecimal::from(0.99998)), + ]; + + let addresses = expected.iter().map(|(address, _)| address.clone()).collect(); + let actual = rpc_client + .display_balances(addresses, TEST_COIN_DECIMALS) + .wait() + .unwrap(); + + assert_eq!(actual, expected); + + let invalid_hashes = vec![ + "0128a4ea8c5775039d39a192f8490b35b416f2f194cb6b6ee91a41d01233c3b5".to_owned(), + "!INVALID!".to_owned(), + "457206aa039ed77b223e4623c19152f9aa63aa7845fe93633920607500766931".to_owned(), + ]; + + let rpc_err = rpc_client.scripthash_get_balances(invalid_hashes).wait().unwrap_err(); + match rpc_err.error { + JsonRpcErrorType::Response(_, json_err) => { + let expected = json!({"code": 1, "message": "!INVALID! is not a valid script hash"}); + assert_eq!(json_err, expected); + }, + _ => panic!("Unexpected `JsonRpcErrorType`"), + } +} + +#[test] +fn test_native_display_balances() { + let unspent = vec![ + NativeUnspent { + address: "RG278CfeNPFtNztFZQir8cgdWexVhViYVy".to_owned(), + amount: "4.77699".into(), + ..NativeUnspent::default() + }, + NativeUnspent { + address: "RJeDDtDRtKUoL8BCKdH7TNCHqUKr7kQRsi".to_owned(), + amount: "0.77699".into(), + ..NativeUnspent::default() + }, + NativeUnspent { + address: "RQHn9VPHBqNjYwyKfJbZCiaxVrWPKGQjeF".to_owned(), + amount: "0.99998".into(), + ..NativeUnspent::default() + }, + NativeUnspent { + address: "RG278CfeNPFtNztFZQir8cgdWexVhViYVy".to_owned(), + amount: "1".into(), + ..NativeUnspent::default() + }, + ]; + + NativeClient::list_unspent_impl + .mock_safe(move |_, _, _, _| MockResult::Return(Box::new(futures01::future::ok(unspent.clone())))); + + let rpc_client = native_client_for_test(); + + let expected: Vec<(Address, BigDecimal)> = vec![ + ("RG278CfeNPFtNztFZQir8cgdWexVhViYVy".into(), BigDecimal::from(5.77699)), + ("RYPz6Lr4muj4gcFzpMdv3ks1NCGn3mkDPN".into(), BigDecimal::from(0)), + ("RJeDDtDRtKUoL8BCKdH7TNCHqUKr7kQRsi".into(), BigDecimal::from(0.77699)), + ("RQHn9VPHBqNjYwyKfJbZCiaxVrWPKGQjeF".into(), BigDecimal::from(0.99998)), + ]; + + let addresses = expected.iter().map(|(address, _)| address.clone()).collect(); + let actual = rpc_client + .display_balances(addresses, TEST_COIN_DECIMALS) + .wait() + .unwrap(); + + assert_eq!(actual, expected); +} diff --git a/mm2src/common/Cargo.toml b/mm2src/common/Cargo.toml index 2a42690578..d679ac234d 100644 --- a/mm2src/common/Cargo.toml +++ b/mm2src/common/Cargo.toml @@ -34,7 +34,7 @@ futures-cpupool = "0.1" hex = "0.3.2" http = "0.2" http-body = "0.1" -itertools = "0.8" +itertools = "0.10" keys = { path = "../mm2_bitcoin/keys" } lazy_static = "1.4" lightning = "0.0.104" diff --git a/mm2src/common/common.rs b/mm2src/common/common.rs index 77f13fe168..7439539857 100644 --- a/mm2src/common/common.rs +++ b/mm2src/common/common.rs @@ -78,6 +78,17 @@ macro_rules! cfg_native { }; } +/// Returns a JSON error HyRes on a failure. +#[macro_export] +macro_rules! try_h { + ($e: expr) => { + match $e { + Ok(ok) => ok, + Err(err) => return $crate::rpc_err_response(500, &ERRL!("{}", err)), + } + }; +} + #[macro_use] pub mod jsonrpc_client; #[macro_use] @@ -807,17 +818,6 @@ pub mod lazy { } } -/// Returns a JSON error HyRes on a failure. -#[macro_export] -macro_rules! try_h { - ($e: expr) => { - match $e { - Ok(ok) => ok, - Err(err) => return $crate::rpc_err_response(500, &ERRL!("{}", err)), - } - }; -} - /// Wraps a JSON string into the `HyRes` RPC response future. pub fn rpc_response(status: u16, body: T) -> HyRes where diff --git a/mm2src/common/jsonrpc_client.rs b/mm2src/common/jsonrpc_client.rs index 81e10aa705..ae9a7c64d7 100644 --- a/mm2src/common/jsonrpc_client.rs +++ b/mm2src/common/jsonrpc_client.rs @@ -1,6 +1,8 @@ use futures01::Future; +use itertools::Itertools; use serde::de::DeserializeOwned; use serde_json::{self as json, Value as Json}; +use std::collections::{BTreeSet, HashMap}; use std::fmt; /// Macro generating functions for RPC requests. @@ -9,41 +11,69 @@ use std::fmt; #[macro_export] macro_rules! rpc_func { ($selff:ident, $method:expr $(, $arg_name:expr)*) => {{ - let mut params = vec![]; - $( - params.push(json::value::to_value($arg_name).unwrap()); - )* - let request = JsonRpcRequest { - jsonrpc: $selff.version().into(), - id: $selff.next_id(), - method: $method.into(), - params - }; + let request = $crate::rpc_req!($selff, $method $(, $arg_name)*); $selff.send_request(request) }} } /// Macro generating functions for RPC requests. -/// Send the RPC request to specified remote endpoint using the passed address. +/// Sends the RPC request to specified remote endpoint using the passed address. /// Args must implement/derive Serialize trait. /// Generates params vector from input args, builds the request and sends it. #[macro_export] macro_rules! rpc_func_from { - ($selff:ident, $address:expr, $method:expr $(, $arg_name:ident)*) => {{ + ($selff:ident, $address:expr, $method:expr $(, $arg_name:expr)*) => {{ + let request = $crate::rpc_req!($selff, $method $(, $arg_name)*); + $selff.send_request_to($address, request) + }} +} + +/// Macro generating functions for RPC requests. +/// Args must implement/derive Serialize trait. +/// Generates params vector from input args, builds the `JsonRpcRequest` request. +#[macro_export] +macro_rules! rpc_req { + ($selff:ident, $method:expr $(, $arg_name:expr)*) => {{ let mut params = vec![]; $( params.push(json::value::to_value($arg_name).unwrap()); )* - let request = JsonRpcRequest { + JsonRpcRequest { jsonrpc: $selff.version().into(), id: $selff.next_id(), method: $method.into(), params - }; - $selff.send_request_to($address, request) + } }} } +pub type JsonRpcResponseFut = + Box + Send + 'static>; +pub type RpcRes = Box + Send + 'static>; + +pub trait BatchRequest { + /// Sends the RPC batch request. + /// Responses are guaranteed to be in the same order in which they were requested. + fn batch_rpc(self, rpc_client: &Rpc) -> RpcRes> + where + Rpc: JsonRpcClient, + T: DeserializeOwned + Send + 'static; +} + +impl BatchRequest for I +where + I: Iterator, +{ + fn batch_rpc(self, rpc_client: &Rpc) -> RpcRes> + where + Rpc: JsonRpcClient, + T: DeserializeOwned + Send + 'static, + { + let batch = JsonRpcBatchRequest(self.collect()); + rpc_client.send_batch_request(batch) + } +} + /// Address of server from which an Rpc response was received #[derive(Clone, Default)] pub struct JsonRpcRemoteAddr(pub String); @@ -60,10 +90,12 @@ impl From for JsonRpcRemoteAddr { fn from(addr: String) -> Self { JsonRpcRemoteAddr(addr) } } +/// The identifier is designed to uniquely match outgoing requests and incoming responses. +/// Even if the batch response is sorted in a different order, `BTreeSet` allows it to be matched to the request. #[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub enum JsonRpcId { Single(String), - Batch(Vec), + Batch(BTreeSet), } #[derive(Clone, Deserialize, Serialize)] @@ -116,11 +148,15 @@ impl From for JsonRpcRequestEnum { pub struct JsonRpcBatchRequest(Vec); impl JsonRpcBatchRequest { - pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Batch(self.0.iter().map(|req| req.id.clone()).collect()) } + pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Batch(self.orig_sequence_ids().collect()) } pub fn len(&self) -> usize { self.0.len() } pub fn is_empty(&self) -> bool { self.0.is_empty() } + + /// Returns original sequence of identifiers. + /// The method is used to process batch response in the same order in which the requests were sent. + fn orig_sequence_ids(&self) -> impl Iterator + '_ { self.0.iter().map(|req| req.id.clone()) } } impl From for JsonRpcRequestEnum { @@ -188,8 +224,14 @@ pub struct JsonRpcError { pub error: JsonRpcErrorType, } +impl fmt::Display for JsonRpcError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{:?}", self) } +} + #[derive(Clone, Debug)] pub enum JsonRpcErrorType { + /// Invalid outgoing request error + InvalidRequest(String), /// Error from transport layer Transport(String), /// Response parse error @@ -202,14 +244,6 @@ impl JsonRpcErrorType { pub fn is_transport(&self) -> bool { matches!(*self, JsonRpcErrorType::Transport(_)) } } -impl fmt::Display for JsonRpcError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{:?}", self) } -} - -pub type JsonRpcResponseFut = - Box + Send + 'static>; -pub type RpcRes = Box + Send + 'static>; - pub trait JsonRpcClient { fn version(&self) -> &'static str; @@ -228,13 +262,27 @@ pub trait JsonRpcClient { ) } + /// Responses are guaranteed to be in the same order in which they were requested. fn send_batch_request(&self, request: JsonRpcBatchRequest) -> RpcRes> { + try_fu!(self.validate_batch_request(&request)); let client_info = self.client_info(); Box::new( self.transport(JsonRpcRequestEnum::Batch(request.clone())) .then(move |result| process_transport_batch_result(result, client_info, request)), ) } + + /// Validates the given batch requests if they all have unique IDs. + fn validate_batch_request(&self, request: &JsonRpcBatchRequest) -> Result<(), JsonRpcError> { + if request.orig_sequence_ids().all_unique() { + return Ok(()); + } + Err(JsonRpcError { + client_info: self.client_info(), + request: request.clone().into(), + error: JsonRpcErrorType::InvalidRequest(ERRL!("Each request in a batch must have a unique ID")), + }) + } } /// The trait is used when the rpc client instance has more than one remote endpoints. @@ -286,10 +334,10 @@ fn process_transport_batch_result( client_info: String, request: JsonRpcBatchRequest, ) -> Result, JsonRpcError> { - let expected_len = request.len(); + let orig_ids: Vec<_> = request.orig_sequence_ids().collect(); let request = JsonRpcRequestEnum::Batch(request); - let (remote_addr, response) = match result { + let (remote_addr, batch) = match result { Ok((remote_addr, JsonRpcResponseEnum::Batch(batch))) => (remote_addr, batch), Ok((remote_addr, JsonRpcResponseEnum::Single(single))) => { let error = ERRL!("Expected batch response, found single response: {:?}", single); @@ -308,11 +356,14 @@ fn process_transport_batch_result( }, }; - if response.len() != expected_len { + // Turn the vector of responses into a hashmap by their IDs to get quick access to the content of the responses. + let mut response_map: HashMap = + batch.into_iter().map(|res| (res.id.clone(), res)).collect(); + if response_map.len() != orig_ids.len() { let error = ERRL!( "Expected '{}' elements in batch response, found '{}'", - expected_len, - response.len() + orig_ids.len(), + response_map.len() ); return Err(JsonRpcError { client_info, @@ -321,10 +372,28 @@ fn process_transport_batch_result( }); } - response - .into_iter() - .map(|resp| process_single_response(client_info.clone(), remote_addr.clone(), request.clone(), resp)) - .collect() + let mut result = Vec::with_capacity(orig_ids.len()); + for id in orig_ids.iter() { + let single_resp = match response_map.remove(id) { + Some(res) => res, + None => { + let error = ERRL!("Batch response doesn't contain '{}' identifier", id); + return Err(JsonRpcError { + client_info, + request, + error: JsonRpcErrorType::Parse(remote_addr, error), + }); + }, + }; + + result.push(process_single_response( + client_info.clone(), + remote_addr.clone(), + request.clone(), + single_resp, + )?); + } + Ok(result) } fn process_single_response( From dd5a57c7f56e6ec17f9f71a7c98eaa5d4e8c6818 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Fri, 25 Mar 2022 15:41:45 +0700 Subject: [PATCH 05/25] Fix wasm build * Add `utxo_common_tests.rs` --- mm2src/coins/utxo.rs | 2 ++ mm2src/coins/utxo/rpc_clients.rs | 2 +- mm2src/coins/utxo/utxo_common_tests.rs | 36 ++++++++++++++++++++++++++ mm2src/coins/utxo/utxo_tests.rs | 33 ++--------------------- mm2src/coins/utxo/utxo_wasm_tests.rs | 7 +++++ 5 files changed, 48 insertions(+), 32 deletions(-) create mode 100644 mm2src/coins/utxo/utxo_common_tests.rs diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index afe0b563af..1d84eb3db1 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -97,6 +97,8 @@ use crate::coin_balance::{EnableCoinScanPolicy, HDAddressBalanceScanner}; use crate::hd_wallet::{HDAccountOps, HDAccountsMutex, HDAddress, HDWalletCoinOps, HDWalletOps, InvalidBip44ChainError}; use crate::hd_wallet_storage::{HDAccountStorageItem, HDWalletCoinStorage, HDWalletStorageError, HDWalletStorageResult}; +#[cfg(any(test, target_arch = "wasm32"))] +pub mod utxo_common_tests; #[cfg(test)] pub mod utxo_tests; #[cfg(target_arch = "wasm32")] pub mod utxo_wasm_tests; diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index bb4a138be5..73cee1cbca 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -2210,7 +2210,7 @@ async fn connect_loop( async fn connect_loop( _config: ElectrumConfig, addr: String, - responses: Arc>>>, + responses: JsonRpcPendingRequestsShared, connection_tx: Arc>>>>, event_handlers: Vec, ) -> Result<(), ()> { diff --git a/mm2src/coins/utxo/utxo_common_tests.rs b/mm2src/coins/utxo/utxo_common_tests.rs new file mode 100644 index 0000000000..e23cd07557 --- /dev/null +++ b/mm2src/coins/utxo/utxo_common_tests.rs @@ -0,0 +1,36 @@ +use super::*; +use crate::utxo::rpc_clients::{ElectrumClient, UtxoRpcClientOps}; +use common::jsonrpc_client::JsonRpcErrorType; + +pub async fn test_electrum_display_balances(rpc_client: &ElectrumClient) { + let expected: Vec<(Address, BigDecimal)> = vec![ + ("RG278CfeNPFtNztFZQir8cgdWexVhViYVy".into(), BigDecimal::from(5.77699)), + ("RYPz6Lr4muj4gcFzpMdv3ks1NCGn3mkDPN".into(), BigDecimal::from(0)), + ("RJeDDtDRtKUoL8BCKdH7TNCHqUKr7kQRsi".into(), BigDecimal::from(0.77699)), + ("RQHn9VPHBqNjYwyKfJbZCiaxVrWPKGQjeF".into(), BigDecimal::from(0.99998)), + ]; + + let addresses = expected.iter().map(|(address, _)| address.clone()).collect(); + let actual = rpc_client.display_balances(addresses, 8).compat().await.unwrap(); + + assert_eq!(actual, expected); + + let invalid_hashes = vec![ + "0128a4ea8c5775039d39a192f8490b35b416f2f194cb6b6ee91a41d01233c3b5".to_owned(), + "!INVALID!".to_owned(), + "457206aa039ed77b223e4623c19152f9aa63aa7845fe93633920607500766931".to_owned(), + ]; + + let rpc_err = rpc_client + .scripthash_get_balances(invalid_hashes) + .compat() + .await + .unwrap_err(); + match rpc_err.error { + JsonRpcErrorType::Response(_, json_err) => { + let expected = json!({"code": 1, "message": "!INVALID! is not a valid script hash"}); + assert_eq!(json_err, expected); + }, + ekind => panic!("Unexpected `JsonRpcErrorType`: {:?}", ekind), + } +} diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 305c09a2ee..4a0c67bb62 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -10,13 +10,13 @@ use crate::utxo::rpc_clients::{BlockHashOrHeight, ElectrumBalance, ElectrumClien VerboseBlock}; use crate::utxo::utxo_builder::{UtxoArcBuilder, UtxoCoinBuilderCommonOps}; use crate::utxo::utxo_common::UtxoTxBuilder; +use crate::utxo::utxo_common_tests; use crate::utxo::utxo_standard::{utxo_standard_coin_with_priv_key, UtxoStandardCoin}; #[cfg(not(target_arch = "wasm32"))] use crate::WithdrawFee; use crate::{CoinBalance, PrivKeyBuildPolicy, StakingInfosDetails, SwapOps, TradePreimageValue, TxFeeDetails}; use bigdecimal::{BigDecimal, Signed}; use chain::OutPoint; use common::executor::Timer; -use common::jsonrpc_client::JsonRpcErrorType; use common::mm_ctx::MmCtxBuilder; use common::privkey::key_pair_from_seed; use common::{block_on, now_ms, OrdRange, PagingOptionsEnum, DEX_FEE_ADDR_RAW_PUBKEY}; @@ -3856,36 +3856,7 @@ fn test_electrum_balance_deserializing() { #[test] fn test_electrum_display_balances() { let rpc_client = electrum_client_for_test(RICK_ELECTRUM_ADDRS); - - let expected: Vec<(Address, BigDecimal)> = vec![ - ("RG278CfeNPFtNztFZQir8cgdWexVhViYVy".into(), BigDecimal::from(5.77699)), - ("RYPz6Lr4muj4gcFzpMdv3ks1NCGn3mkDPN".into(), BigDecimal::from(0)), - ("RJeDDtDRtKUoL8BCKdH7TNCHqUKr7kQRsi".into(), BigDecimal::from(0.77699)), - ("RQHn9VPHBqNjYwyKfJbZCiaxVrWPKGQjeF".into(), BigDecimal::from(0.99998)), - ]; - - let addresses = expected.iter().map(|(address, _)| address.clone()).collect(); - let actual = rpc_client - .display_balances(addresses, TEST_COIN_DECIMALS) - .wait() - .unwrap(); - - assert_eq!(actual, expected); - - let invalid_hashes = vec![ - "0128a4ea8c5775039d39a192f8490b35b416f2f194cb6b6ee91a41d01233c3b5".to_owned(), - "!INVALID!".to_owned(), - "457206aa039ed77b223e4623c19152f9aa63aa7845fe93633920607500766931".to_owned(), - ]; - - let rpc_err = rpc_client.scripthash_get_balances(invalid_hashes).wait().unwrap_err(); - match rpc_err.error { - JsonRpcErrorType::Response(_, json_err) => { - let expected = json!({"code": 1, "message": "!INVALID! is not a valid script hash"}); - assert_eq!(json_err, expected); - }, - _ => panic!("Unexpected `JsonRpcErrorType`"), - } + block_on(utxo_common_tests::test_electrum_display_balances(&rpc_client)); } #[test] diff --git a/mm2src/coins/utxo/utxo_wasm_tests.rs b/mm2src/coins/utxo/utxo_wasm_tests.rs index 15d8d852be..4eaaf01d79 100644 --- a/mm2src/coins/utxo/utxo_wasm_tests.rs +++ b/mm2src/coins/utxo/utxo_wasm_tests.rs @@ -1,6 +1,7 @@ use super::rpc_clients::{ElectrumClient, ElectrumClientImpl, ElectrumProtocol}; use super::*; use crate::utxo::rpc_clients::UtxoRpcClientOps; +use crate::utxo::utxo_common_tests; use common::executor::Timer; use serialization::deserialize; use wasm_bindgen_test::*; @@ -52,3 +53,9 @@ async fn test_electrum_rpc_client() { let expected = UtxoTx::from("0400008085202f8902358549fe3cf9a66bf61fb57bca1b3b49434a148a4dc29450b5eefe583f2f9ecf000000006a4730440220112aa3737672f8aa16a58426f5e7656ad13d21a219390c7a0b2e266ee6b216a8022008e9f9e94db91f069f831b0d40b7f75938122cddceaa25197146dfb00fe82599012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff358549fe3cf9a66bf61fb57bca1b3b49434a148a4dc29450b5eefe583f2f9ecf010000006b483045022100d054464799246254b09f96333bf52537938abe31c24bacf41c9ef600b28155950220527ec33c4a5bef79dcabf97e38aa240fecdd14c96f698560b2f10ec2abc2e992012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0240420f00000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac66418f00000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac0e2aa85f000000000000000000000000000000"); assert_eq!(actual, expected); } + +#[wasm_bindgen_test] +async fn test_electrum_display_balances() { + let rpc_client = electrum_client_for_test(&["electrum1.cipig.net:30017", "electrum2.cipig.net:30017"]).await; + utxo_common_tests::test_electrum_display_balances(&rpc_client).await; +} From b24eec48ece71d7f36b65448db72c4e22bae2a86 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Wed, 13 Apr 2022 15:00:45 +0700 Subject: [PATCH 06/25] Use RPC batch requests to optimize requesting HD addresses balances * Refactor functions within `utxo::tx_cache` to be able to work concurrently * Refactor `UtxoCommonOps::get_verbose_transactions_from_cache_or_rpc` to be able to work concurrently * Remove `UtxoCommonOps::list_all_unspent_ordered`, `UtxoCommonOps::list_mature_unspent_ordered` * Add the `ListUtxoOps` trait and its methods `get_all_unspent_ordered_map`, `get_mature_unspent_ordered_map`, `get_unspent_ordered_map` * Move and rename `UtxoCommonOps::list_unspent_ordered` to `ListUtxoOps::get_unspent_ordered_list` --- mm2src/coins/coin_balance.rs | 50 ++- mm2src/coins/lightning.rs | 4 +- mm2src/coins/qrc20.rs | 78 +++-- mm2src/coins/qrc20/qrc20_tests.rs | 2 +- mm2src/coins/utxo.rs | 137 +++++--- mm2src/coins/utxo/bch.rs | 145 ++++++--- mm2src/coins/utxo/qtum.rs | 71 +++-- mm2src/coins/utxo/qtum_delegation.rs | 6 +- mm2src/coins/utxo/rpc_clients.rs | 208 ++++++++++-- mm2src/coins/utxo/slp.rs | 16 +- mm2src/coins/utxo/tx_cache.rs | 90 ++++-- mm2src/coins/utxo/utxo_common.rs | 372 +++++++++++++--------- mm2src/coins/utxo/utxo_common_tests.rs | 2 +- mm2src/coins/utxo/utxo_standard.rs | 71 +++-- mm2src/coins/utxo/utxo_tests.rs | 348 ++++++++------------ mm2src/coins/utxo/utxo_withdraw.rs | 2 +- mm2src/coins/z_coin.rs | 75 +++-- mm2src/common/common.rs | 1 + mm2src/common/custom_iter.rs | 76 +++++ mm2src/common/fs/fs.rs | 14 +- mm2src/common/jsonrpc_client.rs | 12 +- mm2src/docker_tests.rs | 6 +- mm2src/lp_ordermatch/my_orders_storage.rs | 8 +- mm2src/lp_swap/saved_swap.rs | 8 +- 24 files changed, 1131 insertions(+), 671 deletions(-) create mode 100644 mm2src/common/custom_iter.rs diff --git a/mm2src/coins/coin_balance.rs b/mm2src/coins/coin_balance.rs index 8067e216fe..5183b24936 100644 --- a/mm2src/coins/coin_balance.rs +++ b/mm2src/coins/coin_balance.rs @@ -3,6 +3,7 @@ use crate::hd_wallet::{AddressDerivingError, HDWalletCoinOps, InvalidBip44ChainE use crate::{lp_coinfind_or_err, BalanceError, BalanceResult, CoinBalance, CoinFindError, CoinWithDerivationMethod, DerivationMethod, HDAddress, MarketCoinOps, MmCoinEnum, UnexpectedDerivationMethod}; use async_trait::async_trait; +use common::custom_iter::TryUnzip; use common::log::{debug, info}; use common::mm_ctx::MmArc; use common::mm_error::prelude::*; @@ -299,27 +300,37 @@ pub trait HDWalletBalanceOps: HDWalletCoinOps { address_ids: Ids, ) -> BalanceResult> where - Self::Address: fmt::Display, + Self::Address: fmt::Display + Clone, Ids: Iterator + Send, { - let (lower, upper) = address_ids.size_hint(); - let max_addresses = upper.unwrap_or(lower); - - let mut balances = Vec::with_capacity(max_addresses); - for address_id in address_ids { - let HDAddress { - address, - derivation_path, - .. - } = self.derive_address(hd_account, chain, address_id)?; - let balance = self.known_address_balance(&address).await?; - balances.push(HDAddressBalance { + let (addresses, der_paths) = address_ids + .into_iter() + .map(|address_id| -> BalanceResult<_> { + let HDAddress { + address, + derivation_path, + .. + } = self.derive_address(hd_account, chain, address_id)?; + Ok((address, derivation_path)) + }) + // Try to unzip `Result<(Address, DerivationPath)>` elements into `Result<(Vec
, Vec)>`. + .try_unzip::, Vec<_>>()?; + + let balances = self + .known_addresses_balances(addresses) + .await? + .into_iter() + // [`HDWalletBalanceOps::known_addresses_balances`] returns pairs `(Address, CoinBalance)` + // that are guaranteed to be in the same order in which they were requested. + // So we can zip the derivation paths with the pairs `(Address, CoinBalance)`. + .zip(der_paths) + .map(|((address, balance), derivation_path)| HDAddressBalance { address: address.to_string(), derivation_path: RpcDerivationPath(derivation_path), chain, balance, - }); - } + }) + .collect(); Ok(balances) } @@ -328,6 +339,13 @@ pub trait HDWalletBalanceOps: HDWalletCoinOps { /// since many of RPC clients allow us to request the address balance without the history. async fn known_address_balance(&self, address: &Self::Address) -> BalanceResult; + /// Requests balances of the given `addresses`. + /// The pairs `(Address, CoinBalance)` are guaranteed to be in the same order in which they were requested. + async fn known_addresses_balances( + &self, + addresses: Vec, + ) -> BalanceResult>; + /// Checks if the address has been used by the user by checking if the transaction history of the given `address` is not empty. /// Please note the function can return zero balance even if the address has been used before. async fn is_address_used( @@ -490,7 +508,7 @@ pub mod common_impl { ) -> MmResult where Coin: HDWalletBalanceOps + CoinWithDerivationMethod::HDWallet> + Sync, - ::Address: fmt::Display, + ::Address: fmt::Display + Clone, { let hd_wallet = coin.derivation_method().hd_wallet_or_err()?; diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 38d1b49b24..5cf99b2299 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -1,7 +1,7 @@ use super::{lp_coinfind_or_err, MmCoinEnum}; use crate::utxo::rpc_clients::UtxoRpcClientEnum; use crate::utxo::utxo_common::{big_decimal_from_sat_unsigned, UtxoTxBuilder}; -use crate::utxo::{sat_from_big_decimal, BlockchainNetwork, FeePolicy, UtxoCommonOps, UtxoTxGenerationOps}; +use crate::utxo::{sat_from_big_decimal, BlockchainNetwork, FeePolicy, ListUtxoOps, UtxoTxGenerationOps}; use crate::{BalanceFut, CoinBalance, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MarketCoinOps, MmCoin, NegotiateSwapContractAddrErr, SwapOps, TradeFee, TradePreimageFut, TradePreimageResult, TradePreimageValue, TransactionEnum, TransactionFut, UtxoStandardCoin, ValidateAddressResult, @@ -590,7 +590,7 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes let platform_coin = ln_coin.platform_coin().clone(); let decimals = platform_coin.as_ref().decimals; let my_address = platform_coin.as_ref().derivation_method.iguana_or_err()?; - let (unspents, _) = platform_coin.list_unspent_ordered(my_address).await?; + let (unspents, _) = platform_coin.get_unspent_ordered_list(my_address).await?; let (value, fee_policy) = match req.amount.clone() { ChannelOpenAmount::Max => ( unspents.iter().fold(0, |sum, unspent| sum + unspent.value), diff --git a/mm2src/coins/qrc20.rs b/mm2src/coins/qrc20.rs index 21dc1e2186..fa965369ed 100644 --- a/mm2src/coins/qrc20.rs +++ b/mm2src/coins/qrc20.rs @@ -2,15 +2,15 @@ use crate::eth::{self, u256_to_big_decimal, wei_from_big_decimal, TryToAddress}; use crate::qrc20::rpc_clients::{LogEntry, Qrc20ElectrumOps, Qrc20NativeOps, Qrc20RpcOps, TopicFilter, TxReceipt, ViewContractCallType}; use crate::utxo::qtum::QtumBasedCoin; -use crate::utxo::rpc_clients::{ElectrumClient, NativeClient, UnspentInfo, UtxoRpcClientEnum, UtxoRpcClientOps, - UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; +use crate::utxo::rpc_clients::{ElectrumClient, NativeClient, UnspentInfo, UnspentMap, UtxoRpcClientEnum, + UtxoRpcClientOps, UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuildResult, UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPrivKeyBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::{self, big_decimal_from_sat, check_all_inputs_signed_by_pub, UtxoTxBuilder}; use crate::utxo::{qtum, ActualTxFee, AdditionalTxData, BroadcastTxErr, FeePolicy, GenerateTxError, HistoryUtxoTx, - HistoryUtxoTxMap, RecentlySpentOutPoints, UtxoActivationParams, UtxoAddressFormat, UtxoCoinFields, - UtxoCommonOps, UtxoFromLegacyReqErr, UtxoTx, UtxoTxBroadcastOps, UtxoTxGenerationOps, - VerboseTransactionFrom, UTXO_LOCK}; + HistoryUtxoTxMap, ListUtxoOps, MatureUnspentMap, RecentlySpentOutPointsGuard, UtxoActivationParams, + UtxoAddressFormat, UtxoCoinFields, UtxoCommonOps, UtxoFromLegacyReqErr, UtxoTx, UtxoTxBroadcastOps, + UtxoTxGenerationOps, VerboseTransactionFrom, UTXO_LOCK}; use crate::{BalanceError, BalanceFut, CoinBalance, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MarketCoinOps, MmCoin, NegotiateSwapContractAddrErr, PrivKeyNotAllowed, SwapOps, TradeFee, TradePreimageError, TradePreimageFut, TradePreimageResult, TradePreimageValue, TransactionDetails, TransactionEnum, @@ -31,7 +31,6 @@ use derive_more::Display; use ethabi::{Function, Token}; use ethereum_types::{H160, U256}; use futures::compat::Future01CompatExt; -use futures::lock::MutexGuard as AsyncMutexGuard; use futures::{FutureExt, TryFutureExt}; use futures01::Future; use keys::bytes::Bytes as ScriptBytes; @@ -42,6 +41,7 @@ use script::{Builder as ScriptBuilder, Opcode, Script, TransactionInputSigner}; use script_pubkey::generate_contract_call_script_pubkey; use serde_json::{self as json, Value as Json}; use serialization::{deserialize, serialize, CoinVariant}; +use std::collections::{HashMap, HashSet}; use std::ops::{Deref, Neg}; #[cfg(not(target_arch = "wasm32"))] use std::path::PathBuf; use std::str::FromStr; @@ -466,7 +466,7 @@ impl Qrc20Coin { contract_outputs: Vec, ) -> Result> { let my_address = self.utxo.derivation_method.iguana_or_err()?; - let (unspents, _) = self.list_unspent_ordered(my_address).await?; + let (unspents, _) = self.get_unspent_ordered_list(my_address).await?; let mut gas_fee = 0; let mut outputs = Vec::with_capacity(contract_outputs.len()); @@ -569,6 +569,38 @@ impl UtxoTxGenerationOps for Qrc20Coin { } } +#[async_trait] +#[cfg_attr(test, mockable)] +impl ListUtxoOps for Qrc20Coin { + async fn get_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_unspent_ordered_list(self, address).await + } + + async fn get_unspent_ordered_map( + &self, + addresses: Vec
, + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_unspent_ordered_map(self, addresses).await + } + + async fn get_all_unspent_ordered_map<'a>( + &'a self, + addresses: Vec
, + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard)> { + utxo_common::get_all_unspent_ordered_map(self, addresses).await + } + + async fn get_mature_unspent_ordered_map( + &self, + addresses: Vec
, + ) -> UtxoRpcResult<(MatureUnspentMap, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_mature_unspent_ordered_map(self, addresses).await + } +} + #[async_trait] #[cfg_attr(test, mockable)] impl UtxoCommonOps for Qrc20Coin { @@ -638,37 +670,15 @@ impl UtxoCommonOps for Qrc20Coin { .await } - async fn list_all_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - utxo_common::list_all_unspent_ordered(self, address).await - } - - async fn list_mature_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - utxo_common::list_mature_unspent_ordered(self, address).await - } - - fn get_verbose_transaction_from_cache_or_rpc(&self, txid: H256Json) -> UtxoRpcFut { + fn get_verbose_transactions_from_cache_or_rpc( + &self, + tx_ids: HashSet, + ) -> UtxoRpcFut> { let selfi = self.clone(); - let fut = async move { utxo_common::get_verbose_transaction_from_cache_or_rpc(&selfi.utxo, txid).await }; + let fut = async move { utxo_common::get_verbose_transactions_from_cache_or_rpc(&selfi.utxo, tx_ids).await }; Box::new(fut.boxed().compat()) } - async fn cache_transaction_if_possible(&self, tx: &RpcTransaction) -> Result<(), String> { - utxo_common::cache_transaction_if_possible(&self.utxo, tx).await - } - - async fn list_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - utxo_common::list_unspent_ordered(self, address).await - } - async fn preimage_trade_fee_required_to_send_outputs( &self, outputs: Vec, diff --git a/mm2src/coins/qrc20/qrc20_tests.rs b/mm2src/coins/qrc20/qrc20_tests.rs index c34e7e018b..94a727f91b 100644 --- a/mm2src/coins/qrc20/qrc20_tests.rs +++ b/mm2src/coins/qrc20/qrc20_tests.rs @@ -54,7 +54,7 @@ fn check_tx_fee(coin: &Qrc20Coin, expected_tx_fee: ActualTxFee) { #[test] fn test_withdraw_impl_fee_details() { - Qrc20Coin::list_mature_unspent_ordered.mock_safe(|coin, _| { + Qrc20Coin::get_unspent_ordered_list.mock_safe(|coin, _| { let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); let unspents = vec![UnspentInfo { outpoint: OutPoint { diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 1d84eb3db1..024040a820 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -86,13 +86,14 @@ use utxo_signer::with_key_pair::sign_tx; use utxo_signer::{TxProvider, TxProviderError, UtxoSignTxError, UtxoSignTxResult}; use self::rpc_clients::{electrum_script_hash, ElectrumClient, ElectrumRpcRequest, EstimateFeeMethod, EstimateFeeMode, - NativeClient, UnspentInfo, UtxoRpcClientEnum, UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; -use super::{BalanceError, BalanceFut, BalanceResult, CoinsContext, DerivationMethod, FeeApproxStage, FoundSwapTxSpend, - HistorySyncState, KmdRewardsDetails, MarketCoinOps, MmCoin, NumConversError, NumConversResult, - PrivKeyActivationPolicy, PrivKeyNotAllowed, PrivKeyPolicy, RpcTransportEventHandler, - RpcTransportEventHandlerShared, TradeFee, TradePreimageError, TradePreimageFut, TradePreimageResult, - Transaction, TransactionDetails, TransactionEnum, TransactionFut, UnexpectedDerivationMethod, - WithdrawError, WithdrawRequest}; + NativeClient, UnspentInfo, UnspentMap, UtxoRpcClientEnum, UtxoRpcError, UtxoRpcFut, + UtxoRpcResult}; +use super::{big_decimal_from_sat_unsigned, BalanceError, BalanceFut, BalanceResult, CoinBalance, CoinsContext, + DerivationMethod, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, KmdRewardsDetails, MarketCoinOps, + MmCoin, NumConversError, NumConversResult, PrivKeyActivationPolicy, PrivKeyNotAllowed, PrivKeyPolicy, + RpcTransportEventHandler, RpcTransportEventHandlerShared, TradeFee, TradePreimageError, TradePreimageFut, + TradePreimageResult, Transaction, TransactionDetails, TransactionEnum, TransactionFut, + UnexpectedDerivationMethod, WithdrawError, WithdrawRequest}; use crate::coin_balance::{EnableCoinScanPolicy, HDAddressBalanceScanner}; use crate::hd_wallet::{HDAccountOps, HDAccountsMutex, HDAddress, HDWalletCoinOps, HDWalletOps, InvalidBip44ChainError}; use crate::hd_wallet_storage::{HDAccountStorageItem, HDWalletCoinStorage, HDWalletStorageError, HDWalletStorageResult}; @@ -119,6 +120,8 @@ const DEFAULT_GAP_LIMIT: u32 = 20; pub type GenerateTxResult = Result<(TransactionInputSigner, AdditionalTxData), MmError>; pub type HistoryUtxoTxMap = HashMap; +pub type MatureUnspentMap = HashMap; +pub type RecentlySpentOutPointsGuard<'a> = AsyncMutexGuard<'a, RecentlySpentOutPoints>; #[cfg(windows)] #[cfg(not(target_arch = "wasm32"))] @@ -678,9 +681,41 @@ impl UtxoAddressScanner { } } +#[derive(Debug, Default)] +pub struct MatureUnspentList { + mature: Vec, + immature: Vec, +} + +impl MatureUnspentList { + pub fn with_capacity(capacity: usize) -> MatureUnspentList { + MatureUnspentList { + mature: Vec::with_capacity(capacity), + immature: Vec::with_capacity(capacity), + } + } + + pub fn new_mature(mature: Vec) -> MatureUnspentList { + MatureUnspentList { + mature, + immature: Vec::new(), + } + } + + pub fn into_mature(self) -> Vec { self.mature } + + pub fn to_coin_balance(&self, decimals: u8) -> CoinBalance { + let fold = |acc: BigDecimal, x: &UnspentInfo| acc + big_decimal_from_sat_unsigned(x.value, decimals); + CoinBalance { + spendable: self.mature.iter().fold(BigDecimal::default(), fold), + unspendable: self.immature.iter().fold(BigDecimal::default(), fold), + } + } +} + #[async_trait] #[cfg_attr(test, mockable)] -pub trait UtxoCommonOps: UtxoTxGenerationOps + UtxoTxBroadcastOps { +pub trait UtxoCommonOps: UtxoTxGenerationOps + UtxoTxBroadcastOps + ListUtxoOps { async fn get_htlc_spend_fee(&self, tx_size: u64) -> UtxoRpcResult; fn addresses_from_script(&self, script: &Script) -> Result, String>; @@ -728,34 +763,11 @@ pub trait UtxoCommonOps: UtxoTxGenerationOps + UtxoTxBroadcastOps { keypair: &KeyPair, ) -> Result; - /// Returns available unspents in ascending order + RecentlySpentOutPoints MutexGuard for further interaction - /// (e.g. to add new transaction to it). - /// Please consider using [`UtxoCommonOps::list_unspent_ordered`] instead. - async fn list_all_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)>; - - /// Returns available mature unspents ascending order + RecentlySpentOutPoints MutexGuard for further interaction - /// (e.g. to add new transaction to it). - /// Please consider using [`UtxoCommonOps::list_unspent_ordered`] instead. - async fn list_mature_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)>; - /// Try to load verbose transaction from cache or try to request it from Rpc client. - fn get_verbose_transaction_from_cache_or_rpc(&self, txid: H256Json) -> UtxoRpcFut; - - /// Cache transaction if the coin supports `TX_CACHE` and tx height is set and not zero. - async fn cache_transaction_if_possible(&self, tx: &RpcTransaction) -> Result<(), String>; - - /// Returns available unspents in ascending order + RecentlySpentOutPoints MutexGuard for further interaction - /// (e.g. to add new transaction to it). - async fn list_unspent_ordered( + fn get_verbose_transactions_from_cache_or_rpc( &self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'_, RecentlySpentOutPoints>)>; + tx_ids: HashSet, + ) -> UtxoRpcFut>; async fn preimage_trade_fee_required_to_send_outputs( &self, @@ -783,6 +795,51 @@ pub trait UtxoCommonOps: UtxoTxGenerationOps + UtxoTxBroadcastOps { } } +#[async_trait] +#[cfg_attr(test, mockable)] +pub trait ListUtxoOps { + /// Returns available unspents in ascending order + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). + /// The function uses either [`ListUtxoOps::get_all_unspent_ordered_map`] or [`ListUtxoOps::get_mature_unspent_ordered_map`] + /// depending on the coin configuration. + async fn get_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)>; + + /// Returns available unspents in ascending order for every given `addresses` + /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). + /// The function uses either [`ListUtxoOps::get_all_unspent_ordered_map`] or [`ListUtxoOps::get_mature_unspent_ordered_map`] + /// depending on the coin configuration. + async fn get_unspent_ordered_map( + &self, + addresses: Vec
, + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)>; + + /// Returns available unspents in ascending order for every given `addresses` + /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). + /// + /// # Important + /// + /// The function doesn't check if the unspents are mature or immature. + /// Consider using [`ListUtxoOps::get_unspent_ordered_list`] or [`ListUtxoOps::get_unspent_ordered_map`] instead. + async fn get_all_unspent_ordered_map<'a>( + &'a self, + addresses: Vec
, + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard)>; + + /// Returns available mature and immature unspents in ascending order for every given `addresses` + /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). + /// + /// # Important + /// + /// The function may request an extra data using RPC to check each unspent output whether it's mature or not. + /// It may be overhead in some cases, so consider using [`ListUtxoOps::get_unspent_ordered_list`] or [`ListUtxoOps::get_unspent_ordered_map`] instead. + async fn get_mature_unspent_ordered_map( + &self, + addresses: Vec
, + ) -> UtxoRpcResult<(MatureUnspentMap, RecentlySpentOutPointsGuard<'_>)>; +} + #[async_trait] pub trait UtxoStandardOps { /// Gets tx details by hash requesting the coin RPC if required. @@ -910,13 +967,14 @@ pub enum RequestTxHistoryResult { CriticalError(String), } +#[derive(Clone)] pub enum VerboseTransactionFrom { Cache(RpcTransaction), Rpc(RpcTransaction), } impl VerboseTransactionFrom { - fn into_inner(self) -> RpcTransaction { + fn to_inner(&self) -> &RpcTransaction { match self { VerboseTransactionFrom::Rpc(tx) | VerboseTransactionFrom::Cache(tx) => tx, } @@ -1345,9 +1403,8 @@ where let my_address = try_s!(utxo.derivation_method.iguana_or_err()); let rpc_client = &utxo.rpc_client; let mut unspents = try_s!(rpc_client.list_unspent(my_address, utxo.decimals).compat().await); - // list_unspent_ordered() returns ordered from lowest to highest by value unspent outputs. - // reverse it to reorder from highest to lowest outputs. - unspents.reverse(); + // Reorder from highest to lowest unspent outputs. + unspents.sort_unstable_by(|x, y| y.value.cmp(&x.value)); let mut result = Vec::with_capacity(unspents.len()); for unspent in unspents { @@ -1408,7 +1465,7 @@ where T: AsRef + UtxoCommonOps, { let my_address = try_s!(coin.as_ref().derivation_method.iguana_or_err()); - let (unspents, recently_sent_txs) = try_s!(coin.list_unspent_ordered(my_address).await); + let (unspents, recently_sent_txs) = try_s!(coin.get_unspent_ordered_list(my_address).await); generate_and_send_tx(&coin, unspents, None, FeePolicy::SendExact, recently_sent_txs, outputs).await } @@ -1418,7 +1475,7 @@ async fn generate_and_send_tx( unspents: Vec, required_inputs: Option>, fee_policy: FeePolicy, - mut recently_spent: AsyncMutexGuard<'_, RecentlySpentOutPoints>, + mut recently_spent: RecentlySpentOutPointsGuard<'_>, outputs: Vec, ) -> Result where diff --git a/mm2src/coins/utxo/bch.rs b/mm2src/coins/utxo/bch.rs index c09f5084fc..6b666b7c74 100644 --- a/mm2src/coins/utxo/bch.rs +++ b/mm2src/coins/utxo/bch.rs @@ -20,6 +20,8 @@ use serde_json::{self as json, Value as Json}; use serialization::{deserialize, CoinVariant}; use std::sync::MutexGuard; +pub type BchUnspentsMap = HashMap; + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct BchActivationRequest { #[serde(default)] @@ -184,20 +186,43 @@ impl BchCoin { pub fn bchd_urls(&self) -> &[String] { &self.bchd_urls } async fn utxos_into_bch_unspents(&self, utxos: Vec) -> UtxoRpcResult { + let to_verbose: HashSet = utxos + .iter() + .filter_map(|unspent| { + if unspent.outpoint.index == 0 { + // Zero output is reserved for OP_RETURN of specific protocols + // so if we get it we can safely consider this as standard BCH UTXO. + // There is no need to request verbose transaction for such UTXO. + None + } else { + Some(unspent.outpoint.hash.reversed().into()) + } + }) + .collect(); + let verbose_txs = self + .get_verbose_transactions_from_cache_or_rpc(to_verbose) + .compat() + .await?; + let mut result = BchUnspents::default(); for unspent in utxos { if unspent.outpoint.index == 0 { - // zero output is reserved for OP_RETURN of specific protocols - // so if we get it we can safely consider this as standard BCH UTXO + // Zero output is reserved for OP_RETURN of specific protocols + // so if we get it we can safely consider this as standard BCH UTXO. result.add_standard(unspent); continue; } - let prev_tx_bytes = self - .get_verbose_transaction_from_cache_or_rpc(unspent.outpoint.hash.reversed().into()) - .compat() - .await? - .into_inner(); + let prev_tx_hash = unspent.outpoint.hash.reversed().into(); + let prev_tx_bytes = verbose_txs + .get(&prev_tx_hash) + .or_mm_err(|| { + UtxoRpcError::Internal(format!( + "'get_verbose_transactions_from_cache_or_rpc' should have returned '{:?}'", + prev_tx_hash + )) + })? + .to_inner(); let prev_tx: UtxoTx = match deserialize(prev_tx_bytes.hex.as_slice()) { Ok(b) => b, Err(e) => { @@ -288,21 +313,36 @@ impl BchCoin { pub async fn bch_unspents_for_spend( &self, address: &Address, - ) -> UtxoRpcResult<(BchUnspents, AsyncMutexGuard<'_, RecentlySpentOutPoints>)> { - let (all_unspents, recently_spent) = utxo_common::list_unspent_ordered(self, address).await?; + ) -> UtxoRpcResult<(BchUnspents, RecentlySpentOutPointsGuard<'_>)> { + let (all_unspents, recently_spent) = utxo_common::get_unspent_ordered_list(self, address).await?; let result = self.utxos_into_bch_unspents(all_unspents).await?; Ok((result, recently_spent)) } + /// Locks recently spent cache to safely return UTXOs for spending + pub async fn bch_unspents_map_for_spend( + &self, + addresses: Vec
, + ) -> UtxoRpcResult<(BchUnspentsMap, RecentlySpentOutPointsGuard<'_>)> { + let (all_unspents, recently_spent) = utxo_common::get_unspent_ordered_map(self, addresses).await?; + // Get an iterator of futures: `Iterator>` + let fut_it = all_unspents.into_iter().map(|(address, unspents)| { + self.utxos_into_bch_unspents(unspents) + .map(move |res| -> UtxoRpcResult<_> { + let bch_unspents = res?; + Ok((address, bch_unspents)) + }) + }); + // Poll the `fut_it` futures concurrently. + let bch_unspents_map = futures::future::try_join_all(fut_it).await?.into_iter().collect(); + Ok((bch_unspents_map, recently_spent)) + } + pub async fn get_token_utxos_for_spend( &self, token_id: &H256, - ) -> UtxoRpcResult<( - Vec, - Vec, - AsyncMutexGuard<'_, RecentlySpentOutPoints>, - )> { + ) -> UtxoRpcResult<(Vec, Vec, RecentlySpentOutPointsGuard<'_>)> { let my_address = self .as_ref() .derivation_method @@ -693,6 +733,50 @@ impl UtxoTxGenerationOps for BchCoin { } } +#[async_trait] +#[cfg_attr(test, mockable)] +impl ListUtxoOps for BchCoin { + async fn get_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + let (bch_unspents, recently_spent) = self.bch_unspents_for_spend(address).await?; + Ok((bch_unspents.standard, recently_spent)) + } + + async fn get_unspent_ordered_map( + &self, + addresses: Vec
, + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { + let (bch_unspents_map, recently_spent) = self.bch_unspents_map_for_spend(addresses).await?; + let unspents = bch_unspents_map + .into_iter() + .map(|(address, bch_unspents)| (address, bch_unspents.standard)) + .collect(); + Ok((unspents, recently_spent)) + } + + async fn get_all_unspent_ordered_map<'a>( + &'a self, + addresses: Vec
, + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard)> { + utxo_common::get_all_unspent_ordered_map(self, addresses).await + } + + async fn get_mature_unspent_ordered_map( + &self, + addresses: Vec
, + ) -> UtxoRpcResult<(MatureUnspentMap, RecentlySpentOutPointsGuard<'_>)> { + let (unspents_map, recently_spent) = utxo_common::get_all_unspent_ordered_map(self, addresses).await?; + // Convert `UnspentMap` into `MatureUnspentMap`. + let mature_unspents_map = unspents_map + .into_iter() + .map(|(address, unspents)| (address, MatureUnspentList::new_mature(unspents))) + .collect(); + Ok((mature_unspents_map, recently_spent)) + } +} + // if mockable is placed before async_trait there is `munmap_chunk(): invalid pointer` error on async fn mocking attempt #[async_trait] #[cfg_attr(test, mockable)] @@ -759,38 +843,15 @@ impl UtxoCommonOps for BchCoin { .await } - async fn list_all_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - utxo_common::list_all_unspent_ordered(self, address).await - } - - async fn list_mature_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - utxo_common::list_all_unspent_ordered(self, address).await - } - - fn get_verbose_transaction_from_cache_or_rpc(&self, txid: H256Json) -> UtxoRpcFut { + fn get_verbose_transactions_from_cache_or_rpc( + &self, + tx_ids: HashSet, + ) -> UtxoRpcFut> { let selfi = self.clone(); - let fut = async move { utxo_common::get_verbose_transaction_from_cache_or_rpc(&selfi.utxo_arc, txid).await }; + let fut = async move { utxo_common::get_verbose_transactions_from_cache_or_rpc(&selfi.utxo_arc, tx_ids).await }; Box::new(fut.boxed().compat()) } - async fn cache_transaction_if_possible(&self, tx: &RpcTransaction) -> Result<(), String> { - utxo_common::cache_transaction_if_possible(&self.utxo_arc, tx).await - } - - async fn list_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - let (bch_unspents, recently_spent) = self.bch_unspents_for_spend(address).await?; - Ok((bch_unspents.standard, recently_spent)) - } - async fn preimage_trade_fee_required_to_send_outputs( &self, outputs: Vec, diff --git a/mm2src/coins/utxo/qtum.rs b/mm2src/coins/utxo/qtum.rs index 066b96ff55..f70efcb284 100644 --- a/mm2src/coins/utxo/qtum.rs +++ b/mm2src/coins/utxo/qtum.rs @@ -315,6 +315,38 @@ impl UtxoTxGenerationOps for QtumCoin { } } +#[async_trait] +#[cfg_attr(test, mockable)] +impl ListUtxoOps for QtumCoin { + async fn get_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_unspent_ordered_list(self, address).await + } + + async fn get_unspent_ordered_map( + &self, + addresses: Vec
, + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_unspent_ordered_map(self, addresses).await + } + + async fn get_all_unspent_ordered_map<'a>( + &'a self, + addresses: Vec
, + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard)> { + utxo_common::get_all_unspent_ordered_map(self, addresses).await + } + + async fn get_mature_unspent_ordered_map( + &self, + addresses: Vec
, + ) -> UtxoRpcResult<(MatureUnspentMap, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_mature_unspent_ordered_map(self, addresses).await + } +} + #[async_trait] #[cfg_attr(test, mockable)] impl UtxoCommonOps for QtumCoin { @@ -384,37 +416,15 @@ impl UtxoCommonOps for QtumCoin { .await } - async fn list_all_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - utxo_common::list_all_unspent_ordered(self, address).await - } - - async fn list_mature_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - utxo_common::list_mature_unspent_ordered(self, address).await - } - - fn get_verbose_transaction_from_cache_or_rpc(&self, txid: H256Json) -> UtxoRpcFut { + fn get_verbose_transactions_from_cache_or_rpc( + &self, + tx_ids: HashSet, + ) -> UtxoRpcFut> { let selfi = self.clone(); - let fut = async move { utxo_common::get_verbose_transaction_from_cache_or_rpc(&selfi.utxo_arc, txid).await }; + let fut = async move { utxo_common::get_verbose_transactions_from_cache_or_rpc(&selfi.utxo_arc, tx_ids).await }; Box::new(fut.boxed().compat()) } - async fn cache_transaction_if_possible(&self, tx: &RpcTransaction) -> Result<(), String> { - utxo_common::cache_transaction_if_possible(&self.utxo_arc, tx).await - } - - async fn list_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - utxo_common::list_unspent_ordered(self, address).await - } - async fn preimage_trade_fee_required_to_send_outputs( &self, outputs: Vec, @@ -943,6 +953,13 @@ impl HDWalletBalanceOps for QtumCoin { async fn known_address_balance(&self, address: &Self::Address) -> BalanceResult { utxo_common::address_balance(self, address).await } + + async fn known_addresses_balances( + &self, + addresses: Vec, + ) -> BalanceResult> { + utxo_common::addresses_balances(self, addresses).await + } } impl HDWalletCoinWithStorageOps for QtumCoin { diff --git a/mm2src/coins/utxo/qtum_delegation.rs b/mm2src/coins/utxo/qtum_delegation.rs index 422cf36376..889aebca22 100644 --- a/mm2src/coins/utxo/qtum_delegation.rs +++ b/mm2src/coins/utxo/qtum_delegation.rs @@ -5,7 +5,7 @@ use crate::qrc20::{contract_addr_into_rpc_format, ContractCallOutput, GenerateQr use crate::utxo::qtum::{QtumBasedCoin, QtumCoin, QtumDelegationOps, QtumDelegationRequest, QtumStakingInfosDetails}; use crate::utxo::rpc_clients::UtxoRpcClientEnum; use crate::utxo::utxo_common::{big_decimal_from_sat_unsigned, UtxoTxBuilder}; -use crate::utxo::{qtum, utxo_common, Address, UtxoCommonOps}; +use crate::utxo::{qtum, utxo_common, Address, ListUtxoOps, UtxoCommonOps}; use crate::utxo::{PrivKeyNotAllowed, UTXO_LOCK}; use crate::{DelegationError, DelegationFut, DelegationResult, MarketCoinOps, StakingInfos, StakingInfosError, StakingInfosFut, StakingInfosResult, TransactionDetails, TransactionType}; @@ -205,7 +205,7 @@ impl QtumCoin { let my_address = coin.derivation_method.iguana_or_err()?; let staker = self.am_i_currently_staking().await?; - let (unspents, _) = self.list_unspent_ordered(my_address).await?; + let (unspents, _) = self.get_unspent_ordered_list(my_address).await?; let lower_bound = QTUM_LOWER_BOUND_DELEGATION_AMOUNT.into(); let mut amount = BigDecimal::zero(); if staker.is_some() { @@ -274,7 +274,7 @@ impl QtumCoin { let key_pair = utxo.priv_key_policy.key_pair_or_err()?; let my_address = utxo.derivation_method.iguana_or_err()?; - let (unspents, _) = self.list_unspent_ordered(my_address).await?; + let (unspents, _) = self.get_unspent_ordered_list(my_address).await?; let mut gas_fee = 0; let mut outputs = Vec::with_capacity(contract_outputs.len()); for output in contract_outputs { diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index 73cee1cbca..325e4bc681 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -2,11 +2,12 @@ #![cfg_attr(target_arch = "wasm32", allow(dead_code))] use crate::utxo::{output_script, sat_from_big_decimal}; -use crate::{NumConversError, RpcTransportEventHandler, RpcTransportEventHandlerShared}; +use crate::{big_decimal_from_sat_unsigned, NumConversError, RpcTransportEventHandler, RpcTransportEventHandlerShared}; use async_trait::async_trait; use bigdecimal::BigDecimal; use chain::{BlockHeader, BlockHeaderBits, BlockHeaderNonce, OutPoint, Transaction as UtxoTx}; use common::custom_futures::{select_ok_sequential, FutureTimerExt}; +use common::custom_iter::IntoGroupMapResult; use common::executor::{spawn, Timer}; use common::jsonrpc_client::{BatchRequest, JsonRpcBatchResponse, JsonRpcClient, JsonRpcError, JsonRpcErrorType, JsonRpcId, JsonRpcMultiClient, JsonRpcRemoteAddr, JsonRpcRequest, JsonRpcRequestEnum, @@ -25,7 +26,6 @@ use futures01::future::select_ok; use futures01::sync::{mpsc, oneshot}; use futures01::{Future, Sink, Stream}; use http::Uri; -use itertools::Itertools; use keys::hash::H256; use keys::{Address, Type as ScriptType}; #[cfg(test)] use mocktopus::macros::*; @@ -34,7 +34,8 @@ use serde_json::{self as json, Value as Json}; use serialization::{deserialize, serialize, serialize_with_flags, CoinVariant, CompactInteger, Reader, SERIALIZE_TRANSACTION_WITNESS}; use sha2::{Digest, Sha256}; -use std::collections::hash_map::{Entry, HashMap}; +use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet}; use std::fmt; use std::io; use std::net::{SocketAddr, ToSocketAddrs}; @@ -61,6 +62,10 @@ cfg_native! { pub type AddressesByLabelResult = HashMap; pub type JsonRpcPendingRequestsShared = Arc>; pub type JsonRpcPendingRequests = HashMap>; +pub type UnspentMap = HashMap>; + +type ElectrumScriptHash = String; +type ScriptHashUnspents = Vec; #[derive(Debug, Deserialize)] #[allow(dead_code)] @@ -259,6 +264,8 @@ impl From for UtxoRpcError { pub trait UtxoRpcClientOps: fmt::Debug + Send + Sync + 'static { fn list_unspent(&self, address: &Address, decimals: u8) -> UtxoRpcFut>; + fn list_unspent_group(&self, addresses: Vec
, decimals: u8) -> UtxoRpcFut; + fn send_transaction(&self, tx: &UtxoTx) -> UtxoRpcFut; fn send_raw_transaction(&self, tx: BytesJson) -> UtxoRpcFut; @@ -267,11 +274,13 @@ pub trait UtxoRpcClientOps: fmt::Debug + Send + Sync + 'static { fn get_verbose_transaction(&self, txid: &H256Json) -> UtxoRpcFut; + fn get_verbose_transactions(&self, tx_ids: &[H256Json]) -> UtxoRpcFut>; + fn get_block_count(&self) -> UtxoRpcFut; fn display_balance(&self, address: Address, decimals: u8) -> RpcRes; - fn display_balances(&self, addresses: Vec
, decimals: u8) -> RpcRes>; + fn display_balances(&self, addresses: Vec
, decimals: u8) -> UtxoRpcFut>; /// returns fee estimation per KByte in satoshis fn estimate_fee_sat( @@ -639,6 +648,45 @@ impl UtxoRpcClientOps for NativeClient { Box::new(fut) } + fn list_unspent_group(&self, addresses: Vec
, decimals: u8) -> UtxoRpcFut { + let mut addresses_str = Vec::with_capacity(addresses.len()); + let mut addresses_map = HashMap::with_capacity(addresses.len()); + for addr in addresses { + let addr_str = addr.to_string(); + addresses_str.push(addr_str.clone()); + addresses_map.insert(addr_str, addr); + } + + let fut = self + .list_unspent_impl(0, std::i32::MAX, addresses_str) + .map_to_mm_fut(UtxoRpcError::from) + .and_then(move |unspents| { + unspents + .into_iter() + // Convert `Vec` into `UnspentMap`. + .map(|unspent| { + let orig_address = addresses_map + .get(&unspent.address) + .or_mm_err(|| { + UtxoRpcError::InvalidResponse(format!("Unexpected address '{}'", unspent.address)) + })? + .clone(); + let unspent_info = UnspentInfo { + outpoint: OutPoint { + hash: unspent.txid.reversed().into(), + index: unspent.vout, + }, + value: sat_from_big_decimal(&unspent.amount.to_decimal(), decimals)?, + height: None, + }; + Ok((orig_address, unspent_info)) + }) + // Collect `(Address, UnspentInfo)` items into `HashMap>` grouped by the addresses. + .into_group_map_result() + }); + Box::new(fut) + } + fn send_transaction(&self, tx: &UtxoTx) -> UtxoRpcFut { let tx_bytes = if tx.has_witness() { BytesJson::from(serialize_with_flags(tx, SERIALIZE_TRANSACTION_WITNESS)) @@ -661,6 +709,13 @@ impl UtxoRpcClientOps for NativeClient { Box::new(self.get_raw_transaction_verbose(txid).map_to_mm_fut(UtxoRpcError::from)) } + fn get_verbose_transactions(&self, tx_ids: &[H256Json]) -> UtxoRpcFut> { + Box::new( + self.get_raw_transaction_verbose_batch(tx_ids) + .map_to_mm_fut(UtxoRpcError::from), + ) + } + fn get_block_count(&self) -> UtxoRpcFut { Box::new(self.0.get_block_count().map_to_mm_fut(UtxoRpcError::from)) } @@ -676,28 +731,22 @@ impl UtxoRpcClientOps for NativeClient { ) } - fn display_balances(&self, addresses: Vec
, _decimals: u8) -> RpcRes> { - let addresses_str: Vec<_> = addresses.iter().map(ToString::to_string).collect(); + fn display_balances(&self, addresses: Vec
, decimals: u8) -> UtxoRpcFut> { Box::new( - self.list_unspent_impl(0, i32::MAX, addresses_str.clone()) - .map(move |unspents| { - // First, calculate the balance of each address returned from `NativeClient::list_unspent_impl`. - let balances: HashMap = unspents - .into_iter() - // Group `NativeUnspent` by the addresses. - .into_grouping_map_by(|unspent| unspent.address.clone()) - // Fold `Vec` related to one particular address to get the total balance of that address. - .fold(BigDecimal::from(0), |sum, _key, unspent| { - sum + unspent.amount.to_decimal() - }); - // Secondly, iterate over the requested `addresses` and try to find corresponding balance. + self.list_unspent_group(addresses.clone(), decimals) + .map(move |mut unspent_map| { addresses .into_iter() - .zip(addresses_str) - .map(|(address, address_str)| { + .map(|address| { // If `balances` doesn't contain `address`, there are no unspents related to the address. // Consider the balance of that address equal to 0. - let balance = balances.get(&address_str).cloned().unwrap_or_default(); + let balance = unspent_map + .remove(&address) + .unwrap_or_default() + .into_iter() + .fold(BigDecimal::from(0), |sum, unspent| { + sum + big_decimal_from_sat_unsigned(unspent.value, decimals) + }); (address, balance) }) .collect() @@ -904,6 +953,18 @@ impl NativeClientImpl { rpc_func!(self, "getrawtransaction", txid, verbose) } + /// https://developer.bitcoin.org/reference/rpc/getrawtransaction.html + /// Always returns verbose transactions in a batch. + fn get_raw_transaction_verbose_batch(&self, tx_ids: &[H256Json]) -> RpcRes> { + let verbose = 1; + Box::new( + tx_ids + .iter() + .map(|txid| rpc_req!(self, "getrawtransaction", txid, verbose)) + .batch_rpc(self), + ) + } + /// https://developer.bitcoin.org/reference/rpc/getrawtransaction.html /// Always returns transaction bytes pub fn get_raw_transaction_bytes(&self, txid: &H256Json) -> RpcRes { @@ -1647,6 +1708,33 @@ impl ElectrumClient { Box::new(fut.boxed().compat()) } + /// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-listunspent + /// It can return duplicates sometimes: https://github.com/artemii235/SuperNET/issues/269 + /// We should remove them to build valid transactions. + /// Please note the function returns `ScriptHashUnspents` elements in the same order in which they were requested. + pub fn scripthash_list_unspent_batch(&self, hashes: Vec) -> RpcRes> { + Box::new( + hashes + .iter() + .map(|hash| rpc_req!(self, "blockchain.scripthash.listunspent", hash)) + .batch_rpc(self) + .map(move |unspents: Vec| { + unspents + .into_iter() + .map(|hash_unspents| { + let mut set = HashSet::with_capacity(hash_unspents.len()); + let hash_unspents: Vec<_> = hash_unspents + .into_iter() + // Don't use `Itertools::unique_by` because it seems to be inefficient. + .filter(|unspent| set.insert((unspent.tx_hash, unspent.tx_pos))) + .collect(); + hash_unspents + }) + .collect() + }), + ) + } + /// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-history pub fn scripthash_get_history(&self, hash: &str) -> RpcRes> { rpc_func!(self, "blockchain.scripthash.get_history", hash) @@ -1738,6 +1826,44 @@ impl UtxoRpcClientOps for ElectrumClient { ) } + fn list_unspent_group(&self, addresses: Vec
, _decimals: u8) -> UtxoRpcFut { + let script_hashes = addresses + .iter() + .map(|addr| { + let script = output_script(addr, ScriptType::P2PKH); + let script_hash = electrum_script_hash(&script); + hex::encode(script_hash) + }) + .collect(); + let fut = self + .scripthash_list_unspent_batch(script_hashes) + .map_to_mm_fut(UtxoRpcError::from) + .map(move |unspents| { + addresses + .into_iter() + // `scripthash_list_unspent_batch` returns `ScriptHashUnspents` elements in the same order in which they were requested. + // So we can zip `addresses` and `unspents` into one iterator. + .zip(unspents) + // Map `(Address, Vec)` pairs into `(Address, Vec)`. + .map(|(address, electrum_unspents)| { + let unspents: Vec<_> = electrum_unspents + .into_iter() + .map(|electrum_unspent| UnspentInfo { + outpoint: OutPoint { + hash: electrum_unspent.tx_hash.reversed().into(), + index: electrum_unspent.tx_pos, + }, + value: electrum_unspent.value, + height: electrum_unspent.height, + }) + .collect(); + (address, unspents) + }) + .collect() + }); + Box::new(fut) + } + fn send_transaction(&self, tx: &UtxoTx) -> UtxoRpcFut { let bytes = if tx.has_witness() { BytesJson::from(serialize_with_flags(tx, SERIALIZE_TRANSACTION_WITNESS)) @@ -1771,6 +1897,19 @@ impl UtxoRpcClientOps for ElectrumClient { Box::new(rpc_func!(self, "blockchain.transaction.get", txid, verbose).map_to_mm_fut(UtxoRpcError::from)) } + /// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#blockchain-transaction-get + /// Returns verbose transactions in a batch. + fn get_verbose_transactions(&self, tx_ids: &[H256Json]) -> UtxoRpcFut> { + let verbose = true; + Box::new( + tx_ids + .iter() + .map(|txid| rpc_req!(self, "blockchain.transaction.get", txid, verbose)) + .batch_rpc(self) + .map_to_mm_fut(UtxoRpcError::from), + ) + } + fn get_block_count(&self) -> UtxoRpcFut { Box::new( self.blockchain_headers_subscribe() @@ -1788,22 +1927,27 @@ impl UtxoRpcClientOps for ElectrumClient { })) } - fn display_balances(&self, addresses: Vec
, decimals: u8) -> RpcRes> { + fn display_balances(&self, addresses: Vec
, decimals: u8) -> UtxoRpcFut> { let hashes = addresses.iter().map(|address| { let hash = electrum_script_hash(&output_script(address, ScriptType::P2PKH)); hex::encode(hash) }); - Box::new(self.scripthash_get_balances(hashes).map(move |results| { - results - .into_iter() - .zip(addresses) - .map(|(result, address)| { - let balance_sat = BigInt::from(result.confirmed) + BigInt::from(result.unconfirmed); - let balance_dec = BigDecimal::from(balance_sat) / BigDecimal::from(10u64.pow(decimals as u32)); - (address, balance_dec) + Box::new( + self.scripthash_get_balances(hashes) + .map(move |results| { + results + .into_iter() + .zip(addresses) + .map(|(result, address)| { + let balance_sat = BigInt::from(result.confirmed) + BigInt::from(result.unconfirmed); + let balance_dec = + BigDecimal::from(balance_sat) / BigDecimal::from(10u64.pow(decimals as u32)); + (address, balance_dec) + }) + .collect() }) - .collect() - })) + .map_to_mm_fut(UtxoRpcError::from), + ) } fn estimate_fee_sat( diff --git a/mm2src/coins/utxo/slp.rs b/mm2src/coins/utxo/slp.rs index 26de7ce3ba..98892825d2 100644 --- a/mm2src/coins/utxo/slp.rs +++ b/mm2src/coins/utxo/slp.rs @@ -8,8 +8,8 @@ use crate::utxo::bchd_grpc::{check_slp_transaction, validate_slp_utxos, Validate use crate::utxo::rpc_clients::{UnspentInfo, UtxoRpcClientEnum, UtxoRpcError, UtxoRpcResult}; use crate::utxo::utxo_common::{self, big_decimal_from_sat_unsigned, payment_script, UtxoTxBuilder}; use crate::utxo::{generate_and_send_tx, sat_from_big_decimal, ActualTxFee, AdditionalTxData, BroadcastTxErr, - FeePolicy, GenerateTxError, RecentlySpentOutPoints, UtxoCoinConf, UtxoCoinFields, UtxoCommonOps, - UtxoTx, UtxoTxBroadcastOps, UtxoTxGenerationOps}; + FeePolicy, GenerateTxError, RecentlySpentOutPointsGuard, UtxoCoinConf, UtxoCoinFields, + UtxoCommonOps, UtxoTx, UtxoTxBroadcastOps, UtxoTxGenerationOps}; use crate::{BalanceFut, CoinBalance, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MarketCoinOps, MmCoin, NegotiateSwapContractAddrErr, NumConversError, PrivKeyNotAllowed, SwapOps, TradeFee, TradePreimageError, TradePreimageFut, TradePreimageResult, TradePreimageValue, TransactionDetails, TransactionEnum, @@ -28,7 +28,6 @@ use common::now_ms; use common::privkey::key_pair_from_secret; use derive_more::Display; use futures::compat::Future01CompatExt; -use futures::lock::MutexGuard as AsyncMutexGuard; use futures::{FutureExt, TryFutureExt}; use futures01::Future; use hex::FromHexError; @@ -320,11 +319,7 @@ impl SlpToken { /// Returns unspents of the SLP token plus plain BCH UTXOs plus RecentlySpentOutPoints mutex guard async fn slp_unspents_for_spend( &self, - ) -> UtxoRpcResult<( - Vec, - Vec, - AsyncMutexGuard<'_, RecentlySpentOutPoints>, - )> { + ) -> UtxoRpcResult<(Vec, Vec, RecentlySpentOutPointsGuard<'_>)> { self.platform_coin.get_token_utxos_for_spend(&self.conf.token_id).await } @@ -338,7 +333,7 @@ impl SlpToken { async fn generate_slp_tx_preimage( &self, slp_outputs: Vec, - ) -> Result<(SlpTxPreimage, AsyncMutexGuard<'_, RecentlySpentOutPoints>), MmError> { + ) -> Result<(SlpTxPreimage, RecentlySpentOutPointsGuard<'_>), MmError> { // the limit is 19, but we may require the change to be added if slp_outputs.len() > 18 { return MmError::err(GenSlpSpendErr::TooManyOutputs); @@ -1764,6 +1759,7 @@ pub fn slp_addr_from_pubkey_str(pubkey: &str, prefix: &str) -> Result = MmResult; + lazy_static! { static ref TX_CACHE_LOCK: AsyncMutex<()> = AsyncMutex::new(()); } -/// Try load transaction from cache. -/// Note: tx.confirmations can be out-of-date. -pub async fn load_transaction_from_cache( - tx_cache_path: &Path, - txid: &H256Json, -) -> Result, String> { - let _lock = TX_CACHE_LOCK.lock().await; - - let path = cached_transaction_path(tx_cache_path, txid); - let data = try_s!(safe_slurp(&path)); - if data.is_empty() { - // couldn't find corresponding file - return Ok(None); +#[derive(Debug, Display)] +pub enum TxCacheError { + ErrorLoading(String), + ErrorSaving(String), + ErrorDeserializing(String), + ErrorSerializing(String), +} + +impl From for TxCacheError { + fn from(e: FsJsonError) -> Self { + match e { + FsJsonError::IoReading(loading) => TxCacheError::ErrorLoading(loading.to_string()), + FsJsonError::IoWriting(writing) => TxCacheError::ErrorSaving(writing.to_string()), + FsJsonError::Serializing(ser) => TxCacheError::ErrorSerializing(ser.to_string()), + FsJsonError::Deserializing(de) => TxCacheError::ErrorDeserializing(de.to_string()), + } } +} + +/// Tries to load transactions from cache concurrently. +/// Note 1: tx.confirmations can be out-of-date. +/// Note 2: this function locks the `TX_CACHE_LOCK` mutex to avoid reading and writing the same files at the same time. +pub async fn load_transactions_from_cache_concurrently( + tx_cache_path: &Path, + tx_ids: I, +) -> HashMap>> +where + I: IntoIterator, +{ + let _ = TX_CACHE_LOCK.lock().await; - let data = try_s!(String::from_utf8(data)); - serde_json::from_str(&data).map(Some).map_err(|e| ERRL!("{}", e)) + let it = tx_ids + .into_iter() + .map(|txid| load_transaction_from_cache(tx_cache_path, txid).map(move |res| (txid, res))); + futures::future::join_all(it).await.into_iter().collect() } -/// Upload transaction to cache. -pub async fn cache_transaction(tx_cache_path: &Path, tx: &RpcTransaction) -> Result<(), String> { - let _lock = TX_CACHE_LOCK.lock().await; - let path = cached_transaction_path(tx_cache_path, &tx.txid); - let tmp_path = format!("{}.tmp", path.display()); +/// Upload transactions to cache concurrently. +/// Note: this function locks the `TX_CACHE_LOCK` mutex and takes `txs` as the Hash map +/// to avoid reading and writing the same files at the same time. +pub async fn cache_transactions_concurrently(tx_cache_path: &Path, txs: &HashMap) { + let _ = TX_CACHE_LOCK.lock().await; + + let it = txs.iter().map(|(_txid, tx)| cache_transaction(tx_cache_path, tx)); + futures::future::join_all(it) + .await + .into_iter() + .for_each(|tx| tx.error_log()); +} + +/// Tries to load transaction from cache. +/// Note: tx.confirmations can be out-of-date. +async fn load_transaction_from_cache(tx_cache_path: &Path, txid: H256Json) -> TxCacheResult> { + let path = cached_transaction_path(tx_cache_path, &txid); + read_json(&path).await.mm_err(TxCacheError::from) +} - let content = try_s!(serde_json::to_string(tx)); +/// Upload transaction to cache. +async fn cache_transaction(tx_cache_path: &Path, tx: &RpcTransaction) -> TxCacheResult<()> { + const USE_TMP_FILE: bool = true; - try_s!(std::fs::write(&tmp_path, content)); - try_s!(std::fs::rename(tmp_path, path)); - Ok(()) + let path = cached_transaction_path(tx_cache_path, &tx.txid); + write_json(tx, &path, USE_TMP_FILE).await.mm_err(TxCacheError::from) } fn cached_transaction_path(tx_cache_path: &Path, txid: &H256Json) -> PathBuf { diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index ddc0fcdc2e..8397014513 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -5,13 +5,13 @@ use crate::hd_wallet::{AccountUpdatingError, AddressDerivingError, HDAccountMut, NewAccountCreatingError}; use crate::hd_wallet_storage::{HDWalletCoinWithStorageOps, HDWalletStorageResult}; use crate::init_withdraw::WithdrawTaskHandle; -use crate::utxo::rpc_clients::{electrum_script_hash, BlockHashOrHeight, UnspentInfo, UtxoRpcClientEnum, +use crate::utxo::rpc_clients::{electrum_script_hash, BlockHashOrHeight, UnspentInfo, UnspentMap, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcResult}; use crate::utxo::utxo_withdraw::{InitUtxoWithdraw, StandardUtxoWithdraw, UtxoWithdraw}; use crate::{CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, GetWithdrawSenderAddress, HDAddressId, TradePreimageValue, TxFeeDetails, ValidateAddressResult, ValidatePaymentInput, WithdrawFrom, WithdrawResult, WithdrawSenderAddress}; -use bigdecimal::{BigDecimal, Zero}; +use bigdecimal::BigDecimal; pub use bitcrypto::{dhash160, sha256, ChecksumType}; use chain::constants::SEQUENCE_FINAL; use chain::{OutPoint, TransactionOutput}; @@ -28,6 +28,7 @@ use crypto::{Bip32DerPathOps, Bip44Chain, Bip44DerPathError, Bip44DerivationPath use futures::compat::Future01CompatExt; use futures::future::{FutureExt, TryFutureExt}; use futures01::future::Either; +use itertools::Itertools; use keys::bytes::Bytes; use keys::{Address, AddressFormat as UtxoAddressFormat, AddressHashEnum, Public, SegwitAddress, Type as ScriptType}; use primitives::hash::H512; @@ -326,7 +327,7 @@ pub async fn all_known_addresses_balances( ) -> BalanceResult> where T: HDWalletBalanceOps + Sync, - T::Address: std::fmt::Display, + T::Address: std::fmt::Display + Clone, { let external_addresses = hd_account .known_addresses_number(Bip44Chain::External) @@ -375,6 +376,15 @@ pub async fn address_balance(coin: &T, address: &Address) -> BalanceResult + UtxoCommonOps + MarketCoinOps, { + if coin.as_ref().check_utxo_maturity { + let (mut unspents_map, _) = coin.get_mature_unspent_ordered_map(vec![address.clone()]).await?; + let unspents = unspents_map.remove(address).or_mm_err(|| { + let error = format!("'get_mature_unspent_ordered_map' should have returned '{}'", address); + BalanceError::Internal(error) + })?; + return Ok(unspents.to_coin_balance(coin.as_ref().decimals)); + } + let balance = coin .as_ref() .rpc_client @@ -382,16 +392,46 @@ where .compat() .await?; - if !coin.as_ref().check_utxo_maturity { - return Ok(CoinBalance { - spendable: balance, - unspendable: BigDecimal::from(0), - }); - } + Ok(CoinBalance { + spendable: balance, + unspendable: BigDecimal::from(0), + }) +} + +pub async fn addresses_balances(coin: &T, addresses: Vec
) -> BalanceResult> +where + T: AsRef + UtxoCommonOps + MarketCoinOps, +{ + if coin.as_ref().check_utxo_maturity { + let (unspents_map, _) = coin.get_mature_unspent_ordered_map(addresses.clone()).await?; + return addresses + .into_iter() + .map(|address| { + let unspents = unspents_map.get(&address).or_mm_err(|| { + let error = format!("'get_mature_unspent_ordered_map' should have returned '{}'", address); + BalanceError::Internal(error) + })?; + let balance = unspents.to_coin_balance(coin.as_ref().decimals); + Ok((address, balance)) + }) + .collect(); + } + + let balances = coin + .as_ref() + .rpc_client + .display_balances(addresses.clone(), coin.as_ref().decimals) + .compat() + .await? + .into_iter() + .map(|(address, spendable)| { + let unspendable = BigDecimal::from(0); + let balance = CoinBalance { spendable, unspendable }; + (address, balance) + }) + .collect(); - let unspendable = address_unspendable_balance(coin, address, &balance).await?; - let spendable = &balance - &unspendable; - Ok(CoinBalance { spendable, unspendable }) + Ok(balances) } pub fn derivation_method(coin: &UtxoCoinFields) -> &DerivationMethod { &coin.derivation_method } @@ -2593,7 +2633,7 @@ where let dynamic_fee = coin.increase_dynamic_fee_by_stage(fee, stage); let outputs_count = outputs.len(); - let (unspents, _recently_sent_txs) = coin.list_unspent_ordered(my_address).await?; + let (unspents, _recently_sent_txs) = coin.get_unspent_ordered_list(my_address).await?; let actual_tx_fee = ActualTxFee::Dynamic(dynamic_fee); @@ -2622,7 +2662,7 @@ where }, ActualTxFee::FixedPerKb(fee) => { let outputs_count = outputs.len(); - let (unspents, _recently_sent_txs) = coin.list_unspent_ordered(my_address).await?; + let (unspents, _recently_sent_txs) = coin.get_unspent_ordered_list(my_address).await?; let mut tx_builder = UtxoTxBuilder::new(coin) .add_available_inputs(unspents) @@ -2767,18 +2807,44 @@ pub fn is_coin_protocol_supported(coin: &dyn UtxoCommonOps, info: &Option( - coin: &'a T, - address: &Address, -) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> +/// [`ListUtxoOps::get_mature_unspent_ordered_map`] implementation. +/// Returns available mature and immature unspents in ascending order for every given `addresses` +/// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). +pub async fn get_mature_unspent_ordered_map( + coin: &T, + addresses: Vec
, +) -> UtxoRpcResult<(MatureUnspentMap, RecentlySpentOutPointsGuard<'_>)> +where + T: AsRef + UtxoCommonOps + ListUtxoOps, // TODO remove `AsRef` +{ + let (unspents_map, recently_spent) = coin.get_all_unspent_ordered_map(addresses).await?; + // Get an iterator of futures: `Iterator>` + let fut_it = unspents_map.into_iter().map(|(address, unspents)| { + identify_mature_unspents(coin, unspents).map(|res| -> UtxoRpcResult<(Address, MatureUnspentList)> { + let mature_unspents = res?; + Ok((address, mature_unspents)) + }) + }); + // Poll the `fut_it` futures concurrently. + let result_map = futures::future::try_join_all(fut_it).await?.into_iter().collect(); + Ok((result_map, recently_spent)) +} + +/// Separates the given `unspents` outputs into mature and immature. +/// Identifies mature and immature unspent outputs. +pub async fn identify_mature_unspents(coin: &T, unspents: Vec) -> UtxoRpcResult where T: AsRef + UtxoCommonOps, { + /// Returns `true` if the given transaction has a known non-zero height. + fn can_tx_be_cached(tx: &RpcTransaction) -> bool { tx.height > Some(0) } + + /// Calculates an actual confirmations number of the given `tx` transaction loaded from cache. fn calc_actual_cached_tx_confirmations(tx: &RpcTransaction, block_count: u64) -> UtxoRpcResult { let tx_height = tx.height.or_mm_err(|| { UtxoRpcError::Internal(format!(r#"Warning, height of cached "{:?}" tx is unknown"#, tx.txid)) })?; - // utxo_common::cache_transaction_if_possible() shouldn't cache transaction with height == 0 + // There shouldn't be cached transactions with height == 0 if tx_height == 0 { let error = format!( r#"Warning, height of cached "{:?}" tx is expected to be non-zero"#, @@ -2798,20 +2864,31 @@ where Ok(confirmations as u32) } - let (unspents, recently_spent) = coin.list_all_unspent_ordered(address).await?; let block_count = coin.as_ref().rpc_client.get_block_count().compat().await?; - let mut result = Vec::with_capacity(unspents.len()); + let to_verbose: HashSet = unspents + .iter() + .map(|unspent| unspent.outpoint.hash.reversed().into()) + .collect(); + let verbose_txs = coin + .get_verbose_transactions_from_cache_or_rpc(to_verbose) + .compat() + .await?; + // Transactions that should be cached. + let mut txs_to_cache = HashMap::with_capacity(verbose_txs.len()); + + let mut result = MatureUnspentList::with_capacity(unspents.len()); for unspent in unspents { let tx_hash: H256Json = unspent.outpoint.hash.reversed().into(); - let tx_info = match coin.get_verbose_transaction_from_cache_or_rpc(tx_hash).compat().await { - Ok(x) => x, - Err(err) => { - log!("Error " [err] " getting the transaction " [tx_hash] ", skip the unspent output"); - continue; - }, - }; - + let tx_info = verbose_txs + .get(&tx_hash) + .or_mm_err(|| { + UtxoRpcError::Internal(format!( + "'get_verbose_transactions_from_cache_or_rpc' should have returned '{:?}'", + tx_hash + )) + })? + .clone(); let tx_info = match tx_info { VerboseTransactionFrom::Cache(mut tx) => { if unspent.height.is_some() { @@ -2821,7 +2898,7 @@ where Ok(conf) => tx.confirmations = conf, // do not skip the transaction with unknown confirmations, // because the transaction can be matured - Err(e) => log!((e)), + Err(e) => error!("{}", e), } tx }, @@ -2829,19 +2906,22 @@ where if tx.height.is_none() { tx.height = unspent.height; } - if let Err(e) = coin.cache_transaction_if_possible(&tx).await { - log!((e)); + if can_tx_be_cached(&tx) { + txs_to_cache.insert(tx_hash, tx.clone()); } tx }, }; if coin.is_unspent_mature(&tx_info) { - result.push(unspent); + result.mature.push(unspent); + } else { + result.immature.push(unspent); } } - Ok((result, recently_spent)) + cache_transactions_if_possible(coin.as_ref(), &txs_to_cache).await; + Ok(result) } pub fn is_unspent_mature(mature_confirmations: u32, output: &RpcTransaction) -> bool { @@ -2849,102 +2929,82 @@ pub fn is_unspent_mature(mature_confirmations: u32, output: &RpcTransaction) -> !output.is_coinbase() || output.confirmations >= mature_confirmations } +/// [`UtxoCommonOps::get_verbose_transactions_from_cache_or_rpc`] implementation. +/// Loads a verbose `RpcTransaction` transaction from cache or requests it using RPC client. #[cfg(not(target_arch = "wasm32"))] -pub async fn get_verbose_transaction_from_cache_or_rpc( +pub async fn get_verbose_transactions_from_cache_or_rpc( coin: &UtxoCoinFields, - txid: H256Json, -) -> UtxoRpcResult { - let tx_cache_path = match &coin.tx_cache_directory { - Some(p) => p.clone(), - _ => { - // the coin doesn't support TX local cache, don't try to load from cache and don't cache it - let tx = coin.rpc_client.get_verbose_transaction(&txid).compat().await?; - return Ok(VerboseTransactionFrom::Rpc(tx)); + tx_ids: HashSet, +) -> UtxoRpcResult> { + let mut result_map = HashMap::with_capacity(tx_ids.len()); + let mut to_request = Vec::with_capacity(tx_ids.len()); + + match coin.tx_cache_directory { + Some(ref tx_cache_path) => { + tx_cache::load_transactions_from_cache_concurrently(tx_cache_path, tx_ids.iter().cloned()) + .await + .into_iter() + .for_each(|(txid, res)| match res { + Ok(Some(tx)) => { + result_map.insert(txid, VerboseTransactionFrom::Cache(tx)); + }, + // txid not found + Ok(None) => { + to_request.push(txid); + }, + Err(err) => { + log!("Error " [err] " loading the " [txid] " transaction. Trying request tx using Rpc client"); + to_request.push(txid); + }, + }); }, - }; - - match tx_cache::load_transaction_from_cache(&tx_cache_path, &txid).await { - Ok(Some(tx)) => return Ok(VerboseTransactionFrom::Cache(tx)), - Err(err) => log!("Error " [err] " loading the " [txid] " transaction. Try request tx using Rpc client"), - // txid just not found - _ => (), + // the coin doesn't support TX local cache, don't try to load from cache + None => to_request = tx_ids.iter().copied().collect(), } - let tx = coin.rpc_client.get_verbose_transaction(&txid).compat().await?; - Ok(VerboseTransactionFrom::Rpc(tx)) + result_map.extend( + coin.rpc_client + .get_verbose_transactions(&to_request) + .compat() + .await? + .into_iter() + .map(|tx| (tx.txid, VerboseTransactionFrom::Rpc(tx))), + ); + Ok(result_map) } +/// [`UtxoCommonOps::get_verbose_transactions_from_cache_or_rpc`] implementation. +/// Loads a verbose `RpcTransaction` transaction from cache or requests it using RPC client. #[cfg(target_arch = "wasm32")] -pub async fn get_verbose_transaction_from_cache_or_rpc( +pub async fn get_verbose_transactions_from_cache_or_rpc( coin: &UtxoCoinFields, - txid: H256Json, -) -> UtxoRpcResult { - let tx = coin.rpc_client.get_verbose_transaction(&txid).compat().await?; - Ok(VerboseTransactionFrom::Rpc(tx)) + tx_ids: HashSet, +) -> UtxoRpcResult> { + let tx_ids: Vec<_> = tx_ids.iter().copied().collect(); + let txs = coin + .rpc_client + .get_verbose_transactions(&tx_ids) + .compat() + .await? + .into_iter() + .map(|tx| (tx.txid, VerboseTransactionFrom::Rpc(tx))) + .collect(); + Ok(txs) } +/// Save the given `txs` transactions in a TX cache. +/// The function takes `txs` as the Hash map to avoid duplicating same transactions. #[cfg(not(target_arch = "wasm32"))] -pub async fn cache_transaction_if_possible(coin: &UtxoCoinFields, tx: &RpcTransaction) -> Result<(), String> { - let tx_cache_path = match &coin.tx_cache_directory { - Some(p) => p.clone(), - _ => { - return Ok(()); - }, - }; - // check if the transaction height is set and not zero - match tx.height { - Some(0) => return Ok(()), - Some(_) => (), - None => return Ok(()), +pub async fn cache_transactions_if_possible(coin: &UtxoCoinFields, txs: &HashMap) { + if let Some(ref tx_cache_path) = coin.tx_cache_directory { + tx_cache::cache_transactions_concurrently(tx_cache_path, txs).await } - - tx_cache::cache_transaction(&tx_cache_path, tx) - .await - .map_err(|e| ERRL!("Error {:?} on caching transaction {:?}", e, tx.txid)) } +/// Save the given `txs` transactions in a TX cache. +/// It takes `txs` as the Hash map to avoid duplicating same transactions. #[cfg(target_arch = "wasm32")] -pub async fn cache_transaction_if_possible(_coin: &UtxoCoinFields, _tx: &RpcTransaction) -> Result<(), String> { - Ok(()) -} - -pub async fn address_unspendable_balance( - coin: &T, - address: &Address, - total_balance: &BigDecimal, -) -> BalanceResult -where - T: AsRef + UtxoCommonOps + MarketCoinOps, -{ - let mut attempts = 0i32; - loop { - let (mature_unspents, _) = coin.list_mature_unspent_ordered(address).await?; - let spendable_balance = mature_unspents.iter().fold(BigDecimal::zero(), |acc, x| { - acc + big_decimal_from_sat(x.value as i64, coin.as_ref().decimals) - }); - if total_balance >= &spendable_balance { - return Ok(total_balance - spendable_balance); - } - - if attempts == 2 { - let error = format!( - "Spendable balance {} greater than total balance {}", - spendable_balance, total_balance - ); - return MmError::err(BalanceError::Internal(error)); - } - - warn!( - "Attempt N{}: spendable balance {} greater than total balance {}", - attempts, spendable_balance, total_balance - ); - - // the balance could be changed by other instance between my_balance() and list_mature_unspent_ordered() calls - // try again - attempts += 1; - Timer::sleep(0.3).await; - } -} +pub async fn cache_transactions_if_possible(coin: &UtxoCoinFields, txs: &HashMap) { Ok(()) } /// Swap contract address is not used by standard UTXO coins. pub fn swap_contract_address() -> Option { None } @@ -3227,50 +3287,78 @@ pub fn dex_fee_script(uuid: [u8; 16], time_lock: u32, watcher_pub: &Public, send .into_script() } -pub async fn list_unspent_ordered<'a, T>( +pub async fn get_unspent_ordered_list<'a, T: ListUtxoOps>( coin: &'a T, address: &Address, -) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> +) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'a>)> { + let (mut unspent_map, recently_spent) = coin.get_unspent_ordered_map(vec![address.clone()]).await?; + let unspents = unspent_map.remove(address).or_mm_err(|| { + let error = format!("'get_unspent_ordered_map' should have returned '{}'", address); + UtxoRpcError::Internal(error) + })?; + Ok((unspents, recently_spent)) +} + +/// [`ListUtxoOps::get_unspent_ordered_map`] implementation. +/// Returns available unspents in ascending order + `RecentlySpentOutPoints` MutexGuard for further interaction +/// (e.g. to add new transaction to it). +pub async fn get_unspent_ordered_map( + coin: &T, + addresses: Vec
, +) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> where - T: AsRef + UtxoCommonOps, + T: AsRef + ListUtxoOps, { if coin.as_ref().check_utxo_maturity { - coin.list_mature_unspent_ordered(address).await + coin.get_mature_unspent_ordered_map(addresses) + .await + // Convert `MatureUnspentMap` into `UnspentMap` by discarding immature unspents. + .map(|(mature_unspents_map, recently_spent)| { + let unspents_map = mature_unspents_map + .into_iter() + .map(|(address, unspents)| (address, unspents.into_mature())) + .collect(); + (unspents_map, recently_spent) + }) } else { - coin.list_all_unspent_ordered(address).await + coin.get_all_unspent_ordered_map(addresses).await } } -pub async fn list_all_unspent_ordered<'a, T>( - coin: &'a T, - address: &Address, -) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> +/// [`ListUtxoOps::get_all_unspent_ordered_map`] implementation. +/// Returns available mature and immature unspents in ascending order for every given `addresses` +/// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). +pub async fn get_all_unspent_ordered_map( + coin: &T, + addresses: Vec
, +) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> where T: AsRef, { let decimals = coin.as_ref().decimals; - let mut unspents = coin + let mut unspents_map = coin .as_ref() .rpc_client - .list_unspent(address, decimals) + .list_unspent_group(addresses, decimals) .compat() .await?; let recently_spent = coin.as_ref().recently_spent_outpoints.lock().await; - unspents = recently_spent - .replace_spent_outputs_with_cache(unspents.into_iter().collect()) - .into_iter() - .collect(); - unspents.sort_unstable_by(|a, b| { - if a.value < b.value { - Ordering::Less - } else { - Ordering::Greater - } - }); - // dedup just in case we add duplicates of same unspent out - // all duplicates will be removed because vector in sorted before dedup - unspents.dedup_by(|one, another| one.outpoint == another.outpoint); - Ok((unspents, recently_spent)) + for (_address, unspents) in unspents_map.iter_mut() { + *unspents = recently_spent + .replace_spent_outputs_with_cache(unspents.iter().cloned().collect()) + .into_iter() + // dedup just in case we add duplicates of same unspent out + .unique_by(|unspent| unspent.outpoint.clone()) + .sorted_by(|a, b| { + if a.value < b.value { + Ordering::Less + } else { + Ordering::Greater + } + }) + .collect(); + } + Ok((unspents_map, recently_spent)) } /// Increase the given `dynamic_fee` according to the fee approximation `stage` using the [`UtxoCoinFields::tx_fee_volatility_percent`]. @@ -3327,10 +3415,10 @@ pub async fn merge_utxo_loop( }; let ticker = &coin.as_ref().conf.ticker; - let (unspents, recently_spent) = match coin.list_unspent_ordered(my_address).await { + let (unspents, recently_spent) = match coin.get_unspent_ordered_list(my_address).await { Ok((unspents, recently_spent)) => (unspents, recently_spent), Err(e) => { - error!("Error {} on list_unspent_ordered of coin {}", e, ticker); + error!("Error {} on get_unspent_ordered_list of coin {}", e, ticker); continue; }, }; diff --git a/mm2src/coins/utxo/utxo_common_tests.rs b/mm2src/coins/utxo/utxo_common_tests.rs index e23cd07557..5614dda8ea 100644 --- a/mm2src/coins/utxo/utxo_common_tests.rs +++ b/mm2src/coins/utxo/utxo_common_tests.rs @@ -7,7 +7,7 @@ pub async fn test_electrum_display_balances(rpc_client: &ElectrumClient) { ("RG278CfeNPFtNztFZQir8cgdWexVhViYVy".into(), BigDecimal::from(5.77699)), ("RYPz6Lr4muj4gcFzpMdv3ks1NCGn3mkDPN".into(), BigDecimal::from(0)), ("RJeDDtDRtKUoL8BCKdH7TNCHqUKr7kQRsi".into(), BigDecimal::from(0.77699)), - ("RQHn9VPHBqNjYwyKfJbZCiaxVrWPKGQjeF".into(), BigDecimal::from(0.99998)), + ("RQHn9VPHBqNjYwyKfJbZCiaxVrWPKGQjeF".into(), BigDecimal::from(16.55398)), ]; let addresses = expected.iter().map(|(address, _)| address.clone()).collect(); diff --git a/mm2src/coins/utxo/utxo_standard.rs b/mm2src/coins/utxo/utxo_standard.rs index 6e8f2b7ab7..21451e5835 100644 --- a/mm2src/coins/utxo/utxo_standard.rs +++ b/mm2src/coins/utxo/utxo_standard.rs @@ -87,6 +87,38 @@ impl UtxoTxGenerationOps for UtxoStandardCoin { } } +#[async_trait] +#[cfg_attr(test, mockable)] +impl ListUtxoOps for UtxoStandardCoin { + async fn get_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_unspent_ordered_list(self, address).await + } + + async fn get_unspent_ordered_map( + &self, + addresses: Vec
, + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_unspent_ordered_map(self, addresses).await + } + + async fn get_all_unspent_ordered_map<'a>( + &'a self, + addresses: Vec
, + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard)> { + utxo_common::get_all_unspent_ordered_map(self, addresses).await + } + + async fn get_mature_unspent_ordered_map( + &self, + addresses: Vec
, + ) -> UtxoRpcResult<(MatureUnspentMap, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_mature_unspent_ordered_map(self, addresses).await + } +} + // if mockable is placed before async_trait there is `munmap_chunk(): invalid pointer` error on async fn mocking attempt #[async_trait] #[cfg_attr(test, mockable)] @@ -153,37 +185,15 @@ impl UtxoCommonOps for UtxoStandardCoin { .await } - async fn list_all_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - utxo_common::list_all_unspent_ordered(self, address).await - } - - async fn list_mature_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - utxo_common::list_mature_unspent_ordered(self, address).await - } - - fn get_verbose_transaction_from_cache_or_rpc(&self, txid: H256Json) -> UtxoRpcFut { + fn get_verbose_transactions_from_cache_or_rpc( + &self, + tx_ids: HashSet, + ) -> UtxoRpcFut> { let selfi = self.clone(); - let fut = async move { utxo_common::get_verbose_transaction_from_cache_or_rpc(&selfi.utxo_arc, txid).await }; + let fut = async move { utxo_common::get_verbose_transactions_from_cache_or_rpc(&selfi.utxo_arc, tx_ids).await }; Box::new(fut.boxed().compat()) } - async fn cache_transaction_if_possible(&self, tx: &RpcTransaction) -> Result<(), String> { - utxo_common::cache_transaction_if_possible(&self.utxo_arc, tx).await - } - - async fn list_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - utxo_common::list_unspent_ordered(self, address).await - } - async fn preimage_trade_fee_required_to_send_outputs( &self, outputs: Vec, @@ -728,6 +738,13 @@ impl HDWalletBalanceOps for UtxoStandardCoin { async fn known_address_balance(&self, address: &Self::Address) -> BalanceResult { utxo_common::address_balance(self, address).await } + + async fn known_addresses_balances( + &self, + addresses: Vec, + ) -> BalanceResult> { + utxo_common::addresses_balances(self, addresses).await + } } impl HDWalletCoinWithStorageOps for UtxoStandardCoin { diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 4a0c67bb62..43dbad9a0a 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -23,9 +23,11 @@ use common::{block_on, now_ms, OrdRange, PagingOptionsEnum, DEX_FEE_ADDR_RAW_PUB use crypto::{Bip44Chain, RpcDerivationPath}; use futures::future::join_all; use futures::TryFutureExt; +use itertools::Itertools; use mocktopus::mocking::*; use rpc::v1::types::H256 as H256Json; use serialization::{deserialize, CoinVariant}; +use std::iter; use std::num::NonZeroUsize; const TEST_COIN_NAME: &'static str = "RICK"; @@ -505,7 +507,7 @@ fn test_search_for_swap_tx_spend_electrum_was_refunded() { #[test] #[cfg(not(target_arch = "wasm32"))] fn test_withdraw_impl_set_fixed_fee() { - UtxoStandardCoin::list_unspent_ordered.mock_safe(|coin, _| { + UtxoStandardCoin::get_unspent_ordered_list.mock_safe(|coin, _| { let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); let unspents = vec![UnspentInfo { outpoint: OutPoint { @@ -546,7 +548,7 @@ fn test_withdraw_impl_set_fixed_fee() { #[test] #[cfg(not(target_arch = "wasm32"))] fn test_withdraw_impl_sat_per_kb_fee() { - UtxoStandardCoin::list_unspent_ordered.mock_safe(|coin, _| { + UtxoStandardCoin::get_unspent_ordered_list.mock_safe(|coin, _| { let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); let unspents = vec![UnspentInfo { outpoint: OutPoint { @@ -590,7 +592,7 @@ fn test_withdraw_impl_sat_per_kb_fee() { #[test] #[cfg(not(target_arch = "wasm32"))] fn test_withdraw_impl_sat_per_kb_fee_amount_equal_to_max() { - UtxoStandardCoin::list_unspent_ordered.mock_safe(|coin, _| { + UtxoStandardCoin::get_unspent_ordered_list.mock_safe(|coin, _| { let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); let unspents = vec![UnspentInfo { outpoint: OutPoint { @@ -636,7 +638,7 @@ fn test_withdraw_impl_sat_per_kb_fee_amount_equal_to_max() { #[test] #[cfg(not(target_arch = "wasm32"))] fn test_withdraw_impl_sat_per_kb_fee_amount_equal_to_max_dust_included_to_fee() { - UtxoStandardCoin::list_unspent_ordered.mock_safe(|coin, _| { + UtxoStandardCoin::get_unspent_ordered_list.mock_safe(|coin, _| { let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); let unspents = vec![UnspentInfo { outpoint: OutPoint { @@ -682,7 +684,7 @@ fn test_withdraw_impl_sat_per_kb_fee_amount_equal_to_max_dust_included_to_fee() #[test] #[cfg(not(target_arch = "wasm32"))] fn test_withdraw_impl_sat_per_kb_fee_amount_over_max() { - UtxoStandardCoin::list_unspent_ordered.mock_safe(|coin, _| { + UtxoStandardCoin::get_unspent_ordered_list.mock_safe(|coin, _| { let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); let unspents = vec![UnspentInfo { outpoint: OutPoint { @@ -715,7 +717,7 @@ fn test_withdraw_impl_sat_per_kb_fee_amount_over_max() { #[test] #[cfg(not(target_arch = "wasm32"))] fn test_withdraw_impl_sat_per_kb_fee_max() { - UtxoStandardCoin::list_unspent_ordered.mock_safe(|coin, _| { + UtxoStandardCoin::get_unspent_ordered_list.mock_safe(|coin, _| { let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); let unspents = vec![UnspentInfo { outpoint: OutPoint { @@ -766,7 +768,7 @@ fn test_withdraw_kmd_rewards_impl( ) { let verbose: RpcTransaction = json::from_str(verbose_serialized).unwrap(); let unspent_height = verbose.height; - UtxoStandardCoin::list_unspent_ordered.mock_safe(move |coin, _| { + UtxoStandardCoin::get_unspent_ordered_list.mock_safe(move |coin, _| { let tx: UtxoTx = tx_hex.into(); let unspents = vec![UnspentInfo { outpoint: OutPoint { @@ -849,7 +851,7 @@ fn test_withdraw_rick_rewards_none() { // https://rick.explorer.dexstats.info/tx/7181400be323acc6b5f3164240e6c4601ff4c252f40ce7649f87e81634330209 const TX_HEX: &str = "0400008085202f8901df8119c507aa61d32332cd246dbfeb3818a4f96e76492454c1fbba5aa097977e000000004847304402205a7e229ea6929c97fd6dde254c19e4eb890a90353249721701ae7a1c477d99c402206a8b7c5bf42b5095585731d6b4c589ce557f63c20aed69ff242eca22ecfcdc7a01feffffff02d04d1bffbc050000232102afdbba3e3c90db5f0f4064118f79cf308f926c68afd64ea7afc930975663e4c4ac402dd913000000001976a9143e17014eca06281ee600adffa34b4afb0922a22288ac2bdab86035a00e000000000000000000000000"; - UtxoStandardCoin::list_unspent_ordered.mock_safe(move |coin, _| { + UtxoStandardCoin::get_unspent_ordered_list.mock_safe(move |coin, _| { let tx: UtxoTx = TX_HEX.into(); let unspents = vec![UnspentInfo { outpoint: OutPoint { @@ -884,25 +886,6 @@ fn test_withdraw_rick_rewards_none() { assert_eq!(tx_details.kmd_rewards, None); } -#[test] -fn test_list_mature_unspents_ordered_without_tx_cache() { - let client = electrum_client_for_test(RICK_ELECTRUM_ADDRS); - let coin = utxo_coin_for_test( - client.into(), - Some("spice describe gravity federal blast come thank unfair canal monkey style afraid"), - false, - ); - assert!(coin.as_ref().tx_cache_directory.is_none()); - assert_ne!( - coin.my_spendable_balance().wait().unwrap(), - 0.into(), - "The test address doesn't have unspent outputs" - ); - let (unspents, _) = - block_on(coin.list_mature_unspent_ordered(&Address::from("R9o9xTocqr6CeEDGDH6mEYpwLoMz6jNjMW"))).unwrap(); - assert!(!unspents.is_empty()); -} - #[test] fn test_utxo_lock() { // send several transactions concurrently to check that they are not using same inputs @@ -1527,77 +1510,6 @@ fn test_one_unavailable_electrum_proto_version() { assert!(coin.as_ref().rpc_client.get_block_count().wait().is_ok()); } -#[test] -fn test_qtum_unspendable_balance_failed_once() { - let mut unspents = vec![ - // spendable balance (69.0) > balance (68.0) - vec![ - UnspentInfo { - outpoint: OutPoint { - hash: 1.into(), - index: 0, - }, - value: 5000000000, - height: Default::default(), - }, - UnspentInfo { - outpoint: OutPoint { - hash: 1.into(), - index: 0, - }, - value: 1900000000, - height: Default::default(), - }, - ], - // spendable balance (68.0) == balance (68.0) - vec![ - UnspentInfo { - outpoint: OutPoint { - hash: 1.into(), - index: 0, - }, - value: 5000000000, - height: Default::default(), - }, - UnspentInfo { - outpoint: OutPoint { - hash: 1.into(), - index: 0, - }, - value: 1800000000, - height: Default::default(), - }, - ], - ]; - QtumCoin::list_mature_unspent_ordered.mock_safe(move |coin, _| { - let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); - let unspents = unspents.pop().unwrap(); - MockResult::Return(Box::pin(futures::future::ok((unspents, cache)))) - }); - - let conf = json!({"coin":"tQTUM","rpcport":13889,"pubtype":120,"p2shtype":110}); - let req = json!({ - "method": "electrum", - "servers": [{"url":"electrum1.cipig.net:10071"}, {"url":"electrum2.cipig.net:10071"}, {"url":"electrum3.cipig.net:10071"}], - }); - - let ctx = MmCtxBuilder::new().into_mm_arc(); - - let priv_key = [ - 184, 199, 116, 240, 113, 222, 8, 199, 253, 143, 98, 185, 127, 26, 87, 38, 246, 206, 159, 27, 207, 20, 27, 112, - 184, 102, 137, 37, 78, 214, 113, 78, - ]; - - let params = UtxoActivationParams::from_legacy_req(&req).unwrap(); - let coin = block_on(qtum_coin_with_priv_key(&ctx, "tQTUM", &conf, ¶ms, &priv_key)).unwrap(); - - let CoinBalance { spendable, unspendable } = coin.my_balance().wait().unwrap(); - let expected_spendable = BigDecimal::from(68); - let expected_unspendable = BigDecimal::from(0); - assert_eq!(spendable, expected_spendable); - assert_eq!(unspendable, expected_unspendable); -} - #[test] fn test_qtum_generate_pod() { let priv_key = [ @@ -1743,60 +1655,13 @@ fn test_qtum_remove_delegation() { assert_eq!(res.is_err(), false); } -#[test] -fn test_qtum_unspendable_balance_failed() { - QtumCoin::list_mature_unspent_ordered.mock_safe(move |coin, _| { - let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); - // spendable balance (69.0) > balance (68.0) - let unspents = vec![ - UnspentInfo { - outpoint: OutPoint { - hash: 1.into(), - index: 0, - }, - value: 5000000000, - height: Default::default(), - }, - UnspentInfo { - outpoint: OutPoint { - hash: 1.into(), - index: 0, - }, - value: 1900000000, - height: Default::default(), - }, - ]; - MockResult::Return(Box::pin(futures::future::ok((unspents, cache)))) - }); - - let conf = json!({"coin":"tQTUM","rpcport":13889,"pubtype":120,"p2shtype":110}); - let req = json!({ - "method": "electrum", - "servers": [{"url":"electrum1.cipig.net:10071"}, {"url":"electrum2.cipig.net:10071"}, {"url":"electrum3.cipig.net:10071"}], - }); - - let ctx = MmCtxBuilder::new().into_mm_arc(); - - let priv_key = [ - 184, 199, 116, 240, 113, 222, 8, 199, 253, 143, 98, 185, 127, 26, 87, 38, 246, 206, 159, 27, 207, 20, 27, 112, - 184, 102, 137, 37, 78, 214, 113, 78, - ]; - - let params = UtxoActivationParams::from_legacy_req(&req).unwrap(); - let coin = block_on(qtum_coin_with_priv_key(&ctx, "tQTUM", &conf, ¶ms, &priv_key)).unwrap(); - - let error = coin.my_balance().wait().err().unwrap(); - log!("error: "[error]); - let expected_error = BalanceError::Internal("Spendable balance 69 greater than total balance 68".to_owned()); - assert_eq!(error.get_inner(), &expected_error); -} - #[test] fn test_qtum_my_balance() { - QtumCoin::list_mature_unspent_ordered.mock_safe(move |coin, _| { + QtumCoin::get_mature_unspent_ordered_map.mock_safe(move |coin, addresses| { + let address = addresses.into_iter().exactly_one().unwrap(); let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); - // spendable balance (66.0) < balance (68.0), then unspendable balance is expected to be (2.0) - let unspents = vec![ + // spendable balance (66.0) + let mature = vec![ UnspentInfo { outpoint: OutPoint { hash: 1.into(), @@ -1814,7 +1679,17 @@ fn test_qtum_my_balance() { height: Default::default(), }, ]; - MockResult::Return(Box::pin(futures::future::ok((unspents, cache)))) + // unspendable (2.0) + let immature = vec![UnspentInfo { + outpoint: OutPoint { + hash: 1.into(), + index: 0, + }, + value: 200000000, + height: Default::default(), + }]; + let unspents_map: MatureUnspentMap = iter::once((address, MatureUnspentList { mature, immature })).collect(); + MockResult::Return(Box::pin(futures::future::ok((unspents_map, cache)))) }); let conf = json!({"coin":"tQTUM","rpcport":13889,"pubtype":120,"p2shtype":110}); @@ -1846,8 +1721,10 @@ fn test_qtum_my_balance_with_check_utxo_maturity_false() { ElectrumClient::display_balance.mock_safe(move |_, _, _| { MockResult::Return(Box::new(futures01::future::ok(BigDecimal::from(DISPLAY_BALANCE)))) }); - QtumCoin::list_all_unspent_ordered.mock_safe(move |_, _| { - panic!("'QtumCoin::list_all_unspent_ordered' is not expected to be called when `check_utxo_maturity` is false") + QtumCoin::get_all_unspent_ordered_map.mock_safe(move |_, _| { + panic!( + "'QtumCoin::get_all_unspent_ordered_map' is not expected to be called when `check_utxo_maturity` is false" + ) }); let conf = json!({"coin":"tQTUM","rpcport":13889,"pubtype":120,"p2shtype":110}); @@ -1874,7 +1751,7 @@ fn test_qtum_my_balance_with_check_utxo_maturity_false() { assert_eq!(unspendable, expected_unspendable); } -fn test_list_mature_unspents_ordered_from_cache_impl( +fn test_get_mature_unspent_ordered_map_from_cache_impl( unspent_height: Option, cached_height: Option, cached_confs: u32, @@ -1890,7 +1767,8 @@ fn test_list_mature_unspents_ordered_from_cache_impl( verbose.height = cached_height; // prepare mocks - ElectrumClient::list_unspent.mock_safe(move |_, _, _| { + ElectrumClient::list_unspent_group.mock_safe(move |_, addresses, _| { + let address = addresses.into_iter().exactly_one().unwrap(); let unspents = vec![UnspentInfo { outpoint: OutPoint { hash: H256::from_reversed_str(TX_HASH), @@ -1899,15 +1777,15 @@ fn test_list_mature_unspents_ordered_from_cache_impl( value: 1000000000, height: unspent_height, }]; - MockResult::Return(Box::new(futures01::future::ok(unspents))) + let unspents_map: UnspentMap = iter::once((address, unspents)).collect(); + MockResult::Return(Box::new(futures01::future::ok(unspents_map))) }); ElectrumClient::get_block_count .mock_safe(move |_| MockResult::Return(Box::new(futures01::future::ok(block_count)))); - UtxoStandardCoin::get_verbose_transaction_from_cache_or_rpc.mock_safe(move |_, txid| { - assert_eq!(txid, tx_hash); - MockResult::Return(Box::new(futures01::future::ok(VerboseTransactionFrom::Cache( - verbose.clone(), - )))) + UtxoStandardCoin::get_verbose_transactions_from_cache_or_rpc.mock_safe(move |_, tx_ids| { + itertools::assert_equal(tx_ids, iter::once(tx_hash)); + let result: HashMap<_, _> = iter::once((tx_hash, VerboseTransactionFrom::Cache(verbose.clone()))).collect(); + MockResult::Return(Box::new(futures01::future::ok(result))) }); static mut IS_UNSPENT_MATURE_CALLED: bool = false; UtxoStandardCoin::is_unspent_mature.mock_safe(move |_, tx: &RpcTransaction| { @@ -1920,23 +1798,24 @@ fn test_list_mature_unspents_ordered_from_cache_impl( // run test let coin = utxo_coin_for_test(UtxoRpcClientEnum::Electrum(client), None, false); - let (unspents, _) = - block_on(coin.list_mature_unspent_ordered(&Address::from("R9o9xTocqr6CeEDGDH6mEYpwLoMz6jNjMW"))) + let (unspents_map, _) = + block_on(coin.get_mature_unspent_ordered_map(vec![Address::from("R9o9xTocqr6CeEDGDH6mEYpwLoMz6jNjMW")])) .expect("Expected an empty unspent list"); // unspents should be empty because `is_unspent_mature()` always returns false - assert!(unspents.is_empty()); + let (_address, unspents) = unspents_map.into_iter().exactly_one().unwrap(); + assert!(unspents.mature.is_empty()); assert!(unsafe { IS_UNSPENT_MATURE_CALLED == true }); } #[test] -fn test_list_mature_unspents_ordered_from_cache() { +fn test_get_mature_unspents_ordered_map_from_cache() { let unspent_height = None; let cached_height = None; let cached_confs = 0; let block_count = 1000; let expected_height = None; // is unknown let expected_confs = 0; // is not changed because height is unknown - test_list_mature_unspents_ordered_from_cache_impl( + test_get_mature_unspent_ordered_map_from_cache_impl( unspent_height, cached_height, cached_confs, @@ -1951,7 +1830,7 @@ fn test_list_mature_unspents_ordered_from_cache() { let block_count = 1000; let expected_height = None; // is unknown let expected_confs = 5; // is not changed because height is unknown - test_list_mature_unspents_ordered_from_cache_impl( + test_get_mature_unspent_ordered_map_from_cache_impl( unspent_height, cached_height, cached_confs, @@ -1966,7 +1845,7 @@ fn test_list_mature_unspents_ordered_from_cache() { let block_count = 1000; let expected_height = Some(998); // as the unspent_height let expected_confs = 3; // 1000 - 998 + 1 - test_list_mature_unspents_ordered_from_cache_impl( + test_get_mature_unspent_ordered_map_from_cache_impl( unspent_height, cached_height, cached_confs, @@ -1981,7 +1860,7 @@ fn test_list_mature_unspents_ordered_from_cache() { let block_count = 1000; let expected_height = Some(998); // as the cached_height let expected_confs = 3; // 1000 - 998 + 1 - test_list_mature_unspents_ordered_from_cache_impl( + test_get_mature_unspent_ordered_map_from_cache_impl( unspent_height, cached_height, cached_confs, @@ -1996,7 +1875,7 @@ fn test_list_mature_unspents_ordered_from_cache() { let block_count = 1000; let expected_height = Some(998); // as the unspent_height let expected_confs = 3; // 1000 - 998 + 1 - test_list_mature_unspents_ordered_from_cache_impl( + test_get_mature_unspent_ordered_map_from_cache_impl( unspent_height, cached_height, cached_confs, @@ -2012,7 +1891,7 @@ fn test_list_mature_unspents_ordered_from_cache() { let block_count = 999; let expected_height = Some(1000); // as the cached_height let expected_confs = 1; // is not changed because height cannot be calculated - test_list_mature_unspents_ordered_from_cache_impl( + test_get_mature_unspent_ordered_map_from_cache_impl( unspent_height, cached_height, cached_confs, @@ -2028,7 +1907,7 @@ fn test_list_mature_unspents_ordered_from_cache() { let block_count = 1000; let expected_height = Some(1000); // as the cached_height let expected_confs = 1; // 1000 - 1000 + 1 - test_list_mature_unspents_ordered_from_cache_impl( + test_get_mature_unspent_ordered_map_from_cache_impl( unspent_height, cached_height, cached_confs, @@ -2044,7 +1923,7 @@ fn test_list_mature_unspents_ordered_from_cache() { let block_count = 1000; let expected_height = Some(0); // as the cached_height let expected_confs = 1; // is not changed because tx_height is expected to be not zero - test_list_mature_unspents_ordered_from_cache_impl( + test_get_mature_unspent_ordered_map_from_cache_impl( unspent_height, cached_height, cached_confs, @@ -2084,11 +1963,13 @@ fn test_native_client_unspents_filtered_using_tx_cache_single_tx_in_cache() { tx.hash(), tx.outputs.clone(), ); - NativeClient::list_unspent - .mock_safe(move |_, _, _| MockResult::Return(Box::new(futures01::future::ok(spent_by_tx.clone())))); + NativeClient::list_unspent_group.mock_safe(move |_, addresses, _| { + let address = addresses.into_iter().exactly_one().unwrap(); + let unspents_map: UnspentMap = iter::once((address, spent_by_tx.clone())).collect(); + MockResult::Return(Box::new(futures01::future::ok(unspents_map))) + }); - let address: Address = "RGfFZaaNV68uVe1uMf6Y37Y8E1i2SyYZBN".into(); - let (unspents_ordered, _) = block_on(coin.list_unspent_ordered(&address)).unwrap(); + let (unspents_ordered, _) = block_on(coin.get_unspent_ordered_list(&address)).unwrap(); // output 2 is change so it must be returned let expected_unspent = UnspentInfo { outpoint: OutPoint { @@ -2179,10 +2060,13 @@ fn test_native_client_unspents_filtered_using_tx_cache_single_several_chained_tx unspents_to_return.extend(spent_by_tx_1); unspents_to_return.extend(spent_by_tx_2); - NativeClient::list_unspent - .mock_safe(move |_, _, _| MockResult::Return(Box::new(futures01::future::ok(unspents_to_return.clone())))); + NativeClient::list_unspent_group.mock_safe(move |_, addresses, _| { + let address = addresses.into_iter().exactly_one().unwrap(); + let unspents_map: UnspentMap = iter::once((address, unspents_to_return.clone())).collect(); + MockResult::Return(Box::new(futures01::future::ok(unspents_map))) + }); - let (unspents_ordered, _) = block_on(coin.list_unspent_ordered(&address)).unwrap(); + let (unspents_ordered, _) = block_on(coin.get_unspent_ordered_list(&address)).unwrap(); // output 2 is change so it must be returned let expected_unspent = UnspentInfo { @@ -3096,7 +2980,7 @@ fn tbch_electroncash_verbose_tx_unconfirmed() { #[test] #[cfg(not(target_arch = "wasm32"))] fn test_withdraw_to_p2pkh() { - UtxoStandardCoin::list_unspent_ordered.mock_safe(|coin, _| { + UtxoStandardCoin::get_unspent_ordered_list.mock_safe(|coin, _| { let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); let unspents = vec![UnspentInfo { outpoint: OutPoint { @@ -3143,7 +3027,7 @@ fn test_withdraw_to_p2pkh() { #[test] #[cfg(not(target_arch = "wasm32"))] fn test_withdraw_to_p2sh() { - UtxoStandardCoin::list_unspent_ordered.mock_safe(|coin, _| { + UtxoStandardCoin::get_unspent_ordered_list.mock_safe(|coin, _| { let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); let unspents = vec![UnspentInfo { outpoint: OutPoint { @@ -3190,7 +3074,7 @@ fn test_withdraw_to_p2sh() { #[test] #[cfg(not(target_arch = "wasm32"))] fn test_withdraw_to_p2wpkh() { - UtxoStandardCoin::list_unspent_ordered.mock_safe(|coin, _| { + UtxoStandardCoin::get_unspent_ordered_list.mock_safe(|coin, _| { let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); let unspents = vec![UnspentInfo { outpoint: OutPoint { @@ -3240,11 +3124,14 @@ fn test_withdraw_to_p2wpkh() { fn test_utxo_standard_with_check_utxo_maturity_true() { static mut LIST_MATURE_UNSPENT_ORDERED_CALLED: bool = false; - UtxoStandardCoin::list_mature_unspent_ordered.mock_safe(|coin, _| { + UtxoStandardCoin::get_mature_unspent_ordered_map.mock_safe(|coin, addresses| { unsafe { LIST_MATURE_UNSPENT_ORDERED_CALLED = true }; + + let address = addresses.into_iter().exactly_one().unwrap(); + let unspents_map: MatureUnspentMap = iter::once((address, MatureUnspentList::default())).collect(); + let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); - let unspents = Vec::new(); - MockResult::Return(Box::pin(futures::future::ok((unspents, cache)))) + MockResult::Return(Box::pin(futures::future::ok((unspents_map, cache)))) }); let conf = json!({"coin":"RICK","asset":"RICK","rpcport":25435,"txversion":4,"overwintered":1,"mm2":1,"protocol":{"type":"UTXO"}}); @@ -3267,8 +3154,8 @@ fn test_utxo_standard_with_check_utxo_maturity_true() { .unwrap(); let address = Address::from("R9o9xTocqr6CeEDGDH6mEYpwLoMz6jNjMW"); - // Don't use `block_on` here because it's used within a mock of [`UtxoStandardCoin::list_mature_unspent_ordered`]. - coin.list_unspent_ordered(&address).compat().wait().unwrap(); + // Don't use `block_on` here because it's used within a mock of [`UtxoStandardCoin::get_mature_unspent_ordered_map`]. + coin.get_unspent_ordered_list(&address).compat().wait().unwrap(); assert!(unsafe { LIST_MATURE_UNSPENT_ORDERED_CALLED }); } @@ -3276,17 +3163,20 @@ fn test_utxo_standard_with_check_utxo_maturity_true() { /// https://github.com/KomodoPlatform/atomicDEX-API/issues/1181 #[test] fn test_utxo_standard_without_check_utxo_maturity() { - static mut LIST_ALL_UNSPENT_ORDERED_CALLED: bool = false; + static mut GET_ALL_UNSPENT_ORDERED_MAP_CALLED: bool = false; + + UtxoStandardCoin::get_all_unspent_ordered_map.mock_safe(|coin, addresses| { + unsafe { GET_ALL_UNSPENT_ORDERED_MAP_CALLED = true }; + + let address = addresses.into_iter().exactly_one().unwrap(); + let unspents_map: HashMap<_, _> = iter::once((address, Vec::new())).collect(); - UtxoStandardCoin::list_all_unspent_ordered.mock_safe(|coin, _| { - unsafe { LIST_ALL_UNSPENT_ORDERED_CALLED = true }; let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); - let unspents = Vec::new(); - MockResult::Return(Box::pin(futures::future::ok((unspents, cache)))) + MockResult::Return(Box::pin(futures::future::ok((unspents_map, cache)))) }); - UtxoStandardCoin::list_mature_unspent_ordered.mock_safe(|_, _| { - panic!("'UtxoStandardCoin::list_mature_unspent_ordered' is not expected to be called when `check_utxo_maturity` is not set") + UtxoStandardCoin::get_mature_unspent_ordered_map.mock_safe(|_, _| { + panic!("'UtxoStandardCoin::get_mature_unspent_ordered_map' is not expected to be called when `check_utxo_maturity` is not set") }); let conf = json!({"coin":"RICK","asset":"RICK","rpcport":25435,"txversion":4,"overwintered":1,"mm2":1,"protocol":{"type":"UTXO"}}); @@ -3308,22 +3198,25 @@ fn test_utxo_standard_without_check_utxo_maturity() { .unwrap(); let address = Address::from("R9o9xTocqr6CeEDGDH6mEYpwLoMz6jNjMW"); - // Don't use `block_on` here because it's used within a mock of [`UtxoStandardCoin::list_mature_unspent_ordered`]. - coin.list_unspent_ordered(&address).compat().wait().unwrap(); - assert!(unsafe { LIST_ALL_UNSPENT_ORDERED_CALLED }); + // Don't use `block_on` here because it's used within a mock of [`UtxoStandardCoin::get_mature_unspent_ordered_map`]. + coin.get_unspent_ordered_list(&address).compat().wait().unwrap(); + assert!(unsafe { GET_ALL_UNSPENT_ORDERED_MAP_CALLED }); } /// `QtumCoin` has to check UTXO maturity if `check_utxo_maturity` is not set. /// https://github.com/KomodoPlatform/atomicDEX-API/issues/1181 #[test] fn test_qtum_without_check_utxo_maturity() { - static mut LIST_MATURE_UNSPENT_ORDERED_CALLED: bool = false; + static mut GET_MATURE_UNSPENT_ORDERED_MAP_CALLED: bool = false; + + QtumCoin::get_mature_unspent_ordered_map.mock_safe(|coin, addresses| { + unsafe { GET_MATURE_UNSPENT_ORDERED_MAP_CALLED = true }; + + let address = addresses.into_iter().exactly_one().unwrap(); + let unspents_map: HashMap<_, _> = iter::once((address, MatureUnspentList::default())).collect(); - QtumCoin::list_mature_unspent_ordered.mock_safe(|coin, _| { - unsafe { LIST_MATURE_UNSPENT_ORDERED_CALLED = true }; let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); - let unspents = Vec::new(); - MockResult::Return(Box::pin(futures::future::ok((unspents, cache)))) + MockResult::Return(Box::pin(futures::future::ok((unspents_map, cache)))) }); let conf = json!({"coin":"tQTUM","rpcport":13889,"pubtype":120,"p2shtype":110}); @@ -3342,26 +3235,29 @@ fn test_qtum_without_check_utxo_maturity() { let coin = block_on(qtum_coin_with_priv_key(&ctx, "QTUM", &conf, ¶ms, &[1u8; 32])).unwrap(); let address = Address::from("qcyBHeSct7Wr4mAw18iuQ1zW5mMFYmtmBE"); - // Don't use `block_on` here because it's used within a mock of [`QtumCoin::list_mature_unspent_ordered`]. - coin.list_unspent_ordered(&address).compat().wait().unwrap(); - assert!(unsafe { LIST_MATURE_UNSPENT_ORDERED_CALLED }); + // Don't use `block_on` here because it's used within a mock of [`QtumCoin::get_mature_unspent_ordered_map`]. + coin.get_unspent_ordered_list(&address).compat().wait().unwrap(); + assert!(unsafe { GET_MATURE_UNSPENT_ORDERED_MAP_CALLED }); } /// `QtumCoin` hasn't to check UTXO maturity if `check_utxo_maturity` is `false`. /// https://github.com/KomodoPlatform/atomicDEX-API/issues/1181 #[test] fn test_qtum_with_check_utxo_maturity_false() { - static mut LIST_ALL_UNSPENT_ORDERED_CALLED: bool = false; + static mut GET_ALL_UNSPENT_ORDERED_MAP_CALLED: bool = false; + + QtumCoin::get_all_unspent_ordered_map.mock_safe(|coin, addresses| { + unsafe { GET_ALL_UNSPENT_ORDERED_MAP_CALLED = true }; + + let address = addresses.into_iter().exactly_one().unwrap(); + let unspents_map: HashMap<_, _> = iter::once((address, Vec::new())).collect(); - QtumCoin::list_all_unspent_ordered.mock_safe(|coin, _| { - unsafe { LIST_ALL_UNSPENT_ORDERED_CALLED = true }; let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); - let unspents = Vec::new(); - MockResult::Return(Box::pin(futures::future::ok((unspents, cache)))) + MockResult::Return(Box::pin(futures::future::ok((unspents_map, cache)))) }); - QtumCoin::list_mature_unspent_ordered.mock_safe(|_, _| { + QtumCoin::get_mature_unspent_ordered_map.mock_safe(|_, _| { panic!( - "'QtumCoin::list_mature_unspent_ordered' is not expected to be called when `check_utxo_maturity` is false" + "'QtumCoin::get_mature_unspent_ordered_map' is not expected to be called when `check_utxo_maturity` is false" ) }); @@ -3382,9 +3278,9 @@ fn test_qtum_with_check_utxo_maturity_false() { let coin = block_on(qtum_coin_with_priv_key(&ctx, "QTUM", &conf, ¶ms, &[1u8; 32])).unwrap(); let address = Address::from("qcyBHeSct7Wr4mAw18iuQ1zW5mMFYmtmBE"); - // Don't use `block_on` here because it's used within a mock of [`QtumCoin::list_mature_unspent_ordered`]. - coin.list_unspent_ordered(&address).compat().wait().unwrap(); - assert!(unsafe { LIST_ALL_UNSPENT_ORDERED_CALLED }); + // Don't use `block_on` here because it's used within a mock of [`QtumCoin::get_all_unspent_ordered_map`]. + coin.get_unspent_ordered_list(&address).compat().wait().unwrap(); + assert!(unsafe { GET_ALL_UNSPENT_ORDERED_MAP_CALLED }); } #[test] @@ -3430,12 +3326,18 @@ fn test_account_balance_rpc() { known_address!("m/44'/141'/1'/1/0", "RGo7sYzivPtzv8aRQ4A6vRJDxoqkRRBRhZ", Bip44Chain::Internal, balance = 0); } - NativeClient::display_balance.mock_safe(move |_, address: Address, _| { - let address = address.to_string(); - let balance = addresses_map - .remove(&address) - .expect(&format!("Unexpected address: {}", address)); - MockResult::Return(Box::new(futures01::future::ok(BigDecimal::from(balance)))) + NativeClient::display_balances.mock_safe(move |_, addresses: Vec
, _| { + let result: Vec<_> = addresses + .into_iter() + .map(|address| { + let address_str = address.to_string(); + let balance = addresses_map + .remove(&address_str) + .expect(&format!("Unexpected address: {}", address_str)); + (address, BigDecimal::from(balance)) + }) + .collect(); + MockResult::Return(Box::new(futures01::future::ok(result))) }); let client = NativeClient(Arc::new(NativeClientImpl::default())); @@ -3861,7 +3763,7 @@ fn test_electrum_display_balances() { #[test] fn test_native_display_balances() { - let unspent = vec![ + let unspents = vec![ NativeUnspent { address: "RG278CfeNPFtNztFZQir8cgdWexVhViYVy".to_owned(), amount: "4.77699".into(), @@ -3885,7 +3787,7 @@ fn test_native_display_balances() { ]; NativeClient::list_unspent_impl - .mock_safe(move |_, _, _, _| MockResult::Return(Box::new(futures01::future::ok(unspent.clone())))); + .mock_safe(move |_, _, _, _| MockResult::Return(Box::new(futures01::future::ok(unspents.clone())))); let rpc_client = native_client_for_test(); diff --git a/mm2src/coins/utxo/utxo_withdraw.rs b/mm2src/coins/utxo/utxo_withdraw.rs index 1cf723bc2c..f1b934d73a 100644 --- a/mm2src/coins/utxo/utxo_withdraw.rs +++ b/mm2src/coins/utxo/utxo_withdraw.rs @@ -151,7 +151,7 @@ where let script_pubkey = output_script(&to, script_type).to_bytes(); let _utxo_lock = UTXO_LOCK.lock().await; - let (unspents, _) = coin.list_unspent_ordered(&self.from_address()).await?; + let (unspents, _) = coin.get_unspent_ordered_list(&self.from_address()).await?; let (value, fee_policy) = if req.max { ( unspents.iter().fold(0, |sum, unspent| sum + unspent.value), diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index b92250f480..8073ead522 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -1,12 +1,13 @@ -use crate::utxo::rpc_clients::{UnspentInfo, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcError, UtxoRpcFut, +use crate::utxo::rpc_clients::{UnspentInfo, UnspentMap, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; use crate::utxo::utxo_builder::{UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPrivKeyBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::{big_decimal_from_sat_unsigned, payment_script}; use crate::utxo::{sat_from_big_decimal, utxo_common, ActualTxFee, AdditionalTxData, Address, BroadcastTxErr, - FeePolicy, HistoryUtxoTx, HistoryUtxoTxMap, RecentlySpentOutPoints, UtxoActivationParams, - UtxoAddressFormat, UtxoArc, UtxoCoinFields, UtxoCommonOps, UtxoFeeDetails, UtxoTxBroadcastOps, - UtxoTxGenerationOps, UtxoWeak, VerboseTransactionFrom}; + FeePolicy, HistoryUtxoTx, HistoryUtxoTxMap, ListUtxoOps, MatureUnspentList, MatureUnspentMap, + RecentlySpentOutPoints, RecentlySpentOutPointsGuard, UtxoActivationParams, UtxoAddressFormat, + UtxoArc, UtxoCoinFields, UtxoCommonOps, UtxoFeeDetails, UtxoTxBroadcastOps, UtxoTxGenerationOps, + UtxoWeak, VerboseTransactionFrom}; use crate::{BalanceFut, CoinBalance, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MarketCoinOps, MmCoin, NegotiateSwapContractAddrErr, NumConversError, SwapOps, TradeFee, TradePreimageFut, TradePreimageResult, TradePreimageValue, TransactionDetails, TransactionEnum, TransactionFut, TxFeeDetails, @@ -35,7 +36,7 @@ use rusqlite::{Connection, Error as SqliteError, Row, ToSql, NO_PARAMS}; use script::{Builder as ScriptBuilder, Opcode, Script, TransactionInputSigner}; use serde_json::Value as Json; use serialization::{deserialize, serialize_list, CoinVariant, Reader}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering}; @@ -1311,6 +1312,38 @@ impl UtxoTxBroadcastOps for ZCoin { } } +#[async_trait] +#[cfg_attr(test, mockable)] +impl ListUtxoOps for ZCoin { + async fn get_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_unspent_ordered_list(self, address).await + } + + async fn get_unspent_ordered_map( + &self, + addresses: Vec
, + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_unspent_ordered_map(self, addresses).await + } + + async fn get_all_unspent_ordered_map<'a>( + &'a self, + addresses: Vec
, + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard)> { + utxo_common::get_all_unspent_ordered_map(self, addresses).await + } + + async fn get_mature_unspent_ordered_map( + &self, + addresses: Vec
, + ) -> UtxoRpcResult<(MatureUnspentMap, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_mature_unspent_ordered_map(self, addresses).await + } +} + #[async_trait] impl UtxoCommonOps for ZCoin { async fn get_htlc_spend_fee(&self, tx_size: u64) -> UtxoRpcResult { @@ -1381,37 +1414,15 @@ impl UtxoCommonOps for ZCoin { .await } - async fn list_all_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - utxo_common::list_all_unspent_ordered(self, address).await - } - - async fn list_mature_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - utxo_common::list_mature_unspent_ordered(self, address).await - } - - fn get_verbose_transaction_from_cache_or_rpc(&self, txid: H256Json) -> UtxoRpcFut { + fn get_verbose_transactions_from_cache_or_rpc( + &self, + tx_ids: HashSet, + ) -> UtxoRpcFut> { let selfi = self.clone(); - let fut = async move { utxo_common::get_verbose_transaction_from_cache_or_rpc(&selfi.utxo_arc, txid).await }; + let fut = async move { utxo_common::get_verbose_transactions_from_cache_or_rpc(&selfi.utxo_arc, tx_ids).await }; Box::new(fut.boxed().compat()) } - async fn cache_transaction_if_possible(&self, tx: &RpcTransaction) -> Result<(), String> { - utxo_common::cache_transaction_if_possible(&self.utxo_arc, tx).await - } - - async fn list_unspent_ordered<'a>( - &'a self, - address: &Address, - ) -> UtxoRpcResult<(Vec, AsyncMutexGuard<'a, RecentlySpentOutPoints>)> { - utxo_common::list_unspent_ordered(self, address).await - } - async fn preimage_trade_fee_required_to_send_outputs( &self, outputs: Vec, diff --git a/mm2src/common/common.rs b/mm2src/common/common.rs index 7439539857..e6cc4745b8 100644 --- a/mm2src/common/common.rs +++ b/mm2src/common/common.rs @@ -99,6 +99,7 @@ pub mod mm_metrics; pub mod big_int_str; pub mod crash_reports; pub mod custom_futures; +pub mod custom_iter; pub mod duplex_mutex; pub mod event_dispatcher; pub mod for_tests; diff --git a/mm2src/common/custom_iter.rs b/mm2src/common/custom_iter.rs new file mode 100644 index 0000000000..a944def952 --- /dev/null +++ b/mm2src/common/custom_iter.rs @@ -0,0 +1,76 @@ +use std::collections::HashMap; +use std::hash::Hash; + +pub trait IntoGroupMapResult { + /// An iterator method that unwraps the given `Result<(Key, Value), Err>` items yielded by the input iterator + /// and collects `(Key, Value)` tuple pairs into a `HashMap` of keys mapped to `Vec`s of values until an `Err` error is encountered. + fn into_group_map_result(self) -> Result>, Err> + where + Self: Iterator> + Sized, + K: Hash + Eq, + { + let (lower, upper) = self.size_hint(); + let capacity = upper.unwrap_or(lower); + + let mut lookup = HashMap::with_capacity(capacity); + for res in self { + let (key, val) = res?; + lookup.entry(key).or_insert_with(Vec::new).push(val); + } + Ok(lookup) + } +} + +impl IntoGroupMapResult for T {} + +pub trait TryUnzip +where + Self: Iterator> + Sized, +{ + /// An iterator method that unwraps the given `Result<(A, B), Err>` items yielded by the input iterator + /// and collects `(A, B)` tuple pairs into the pair of `(FromA, FromB)` containers until an `Err` error is encountered. + fn try_unzip(self) -> Result<(FromA, FromB), E> + where + FromA: Default + Extend, + FromB: Default + Extend, + { + let (mut from_a, mut from_b) = (FromA::default(), FromB::default()); + for res in self { + let (a, b) = res?; + from_a.extend(Some(a)); + from_b.extend(Some(b)); + } + Ok((from_a, from_b)) + } +} + +impl TryUnzip for T where T: Iterator> {} + +#[test] +fn test_into_group_map_result() { + let actual: Result<_, &'static str> = vec![Ok(("foo", 1)), Ok(("bar", 2)), Ok(("foo", 3))] + .into_iter() + .into_group_map_result(); + let expected: HashMap<_, _> = vec![("foo", vec![1, 3]), ("bar", vec![2])].into_iter().collect(); + assert_eq!(actual, Ok(expected)); + + let err = vec![Ok(("foo", 1)), Ok(("bar", 2)), Err("Error"), Ok(("foo", 3))] + .into_iter() + .into_group_map_result() + .unwrap_err(); + assert_eq!(err, "Error"); +} + +#[test] +fn test_try_unzip() { + let actual: Result<(Vec<_>, Vec<_>), &'static str> = vec![Ok(("foo", 1)), Ok(("bar", 2)), Ok(("foo", 3))] + .into_iter() + .try_unzip(); + assert_eq!(actual, Ok((vec!["foo", "bar", "foo"], vec![1, 2, 3]))); + + let err = vec![Ok(("foo", 1)), Ok(("bar", 2)), Err("Error"), Ok(("foo", 3))] + .into_iter() + .try_unzip::, Vec<_>>() + .unwrap_err(); + assert_eq!(err, "Error"); +} diff --git a/mm2src/common/fs/fs.rs b/mm2src/common/fs/fs.rs index 9bf99a904a..808e5abcc7 100644 --- a/mm2src/common/fs/fs.rs +++ b/mm2src/common/fs/fs.rs @@ -226,16 +226,26 @@ where Ok(result) } -pub async fn write_json(t: &T, path: &Path) -> FsJsonResult<()> +pub async fn write_json(t: &T, path: &Path, use_tmp_file: bool) -> FsJsonResult<()> where T: Serialize, { let content = json::to_vec(t).map_to_mm(FsJsonError::Serializing)?; + let path_tmp = if use_tmp_file { + path.to_path_buf() + } else { + PathBuf::from(format!("{}.tmp", path.display())) + }; + let fs_fut = async { - let mut file = async_fs::File::create(&path).await?; + let mut file = async_fs::File::create(&path_tmp).await?; file.write_all(&content).await?; file.flush().await?; + + if use_tmp_file { + async_fs::rename(path_tmp, path).await?; + } Ok(()) }; diff --git a/mm2src/common/jsonrpc_client.rs b/mm2src/common/jsonrpc_client.rs index ae9a7c64d7..5d5927edd8 100644 --- a/mm2src/common/jsonrpc_client.rs +++ b/mm2src/common/jsonrpc_client.rs @@ -69,8 +69,12 @@ where Rpc: JsonRpcClient, T: DeserializeOwned + Send + 'static, { - let batch = JsonRpcBatchRequest(self.collect()); - rpc_client.send_batch_request(batch) + let requests: Vec<_> = self.collect(); + if requests.is_empty() { + // Return an empty vector of results. + return Box::new(futures01::future::ok(Vec::new())); + } + rpc_client.send_batch_request(JsonRpcBatchRequest(requests)) } } @@ -106,6 +110,10 @@ pub enum JsonRpcRequestEnum { } impl JsonRpcRequestEnum { + pub fn new_batch(requests: Vec) -> JsonRpcRequestEnum { + JsonRpcRequestEnum::Batch(JsonRpcBatchRequest(requests)) + } + pub fn rpc_id(&self) -> JsonRpcId { match self { JsonRpcRequestEnum::Single(single) => single.rpc_id(), diff --git a/mm2src/docker_tests.rs b/mm2src/docker_tests.rs index 7984e3c0d3..83d258e83c 100644 --- a/mm2src/docker_tests.rs +++ b/mm2src/docker_tests.rs @@ -110,7 +110,7 @@ mod docker_tests { use coins::utxo::slp::{slp_genesis_output, SlpOutput}; use coins::utxo::utxo_common::send_outputs_from_my_address; use coins::utxo::utxo_standard::{utxo_standard_coin_with_priv_key, UtxoStandardCoin}; - use coins::utxo::{dhash160, UtxoActivationParams, UtxoCommonOps}; + use coins::utxo::{dhash160, ListUtxoOps, UtxoActivationParams, UtxoCommonOps}; use coins::{CoinProtocol, FoundSwapTxSpend, MarketCoinOps, MmCoin, SwapOps, Transaction, TransactionEnum, WithdrawRequest}; use common::for_tests::{check_my_swap_status_amounts, enable_electrum}; @@ -3232,7 +3232,7 @@ mod docker_tests { thread::sleep(Duration::from_secs(2)); let (unspents, _) = - block_on(coin.list_unspent_ordered(&coin.as_ref().derivation_method.unwrap_iguana())).unwrap(); + block_on(coin.get_unspent_ordered_list(&coin.as_ref().derivation_method.unwrap_iguana())).unwrap(); assert_eq!(unspents.len(), 1); } @@ -3290,7 +3290,7 @@ mod docker_tests { thread::sleep(Duration::from_secs(2)); let (unspents, _) = - block_on(coin.list_unspent_ordered(&coin.as_ref().derivation_method.unwrap_iguana())).unwrap(); + block_on(coin.get_unspent_ordered_list(&coin.as_ref().derivation_method.unwrap_iguana())).unwrap(); // 4 utxos are merged of 5 so the resulting unspents len must be 2 assert_eq!(unspents.len(), 2); } diff --git a/mm2src/lp_ordermatch/my_orders_storage.rs b/mm2src/lp_ordermatch/my_orders_storage.rs index b3dfeb4747..0082422739 100644 --- a/mm2src/lp_ordermatch/my_orders_storage.rs +++ b/mm2src/lp_ordermatch/my_orders_storage.rs @@ -212,6 +212,8 @@ mod native_impl { my_taker_order_file_path, my_taker_orders_dir}; use common::fs::{read_dir_json, read_json, remove_file_async, write_json, FsJsonError}; + const USE_TMP_FILE: bool = false; + impl From for MyOrdersError { fn from(fs: FsJsonError) -> Self { match fs { @@ -255,13 +257,13 @@ mod native_impl { async fn save_new_active_maker_order(&self, order: &MakerOrder) -> MyOrdersResult<()> { let path = my_maker_order_file_path(&self.ctx, &order.uuid); - write_json(order, &path).await?; + write_json(order, &path, USE_TMP_FILE).await?; Ok(()) } async fn save_new_active_taker_order(&self, order: &TakerOrder) -> MyOrdersResult<()> { let path = my_taker_order_file_path(&self.ctx, &order.request.uuid); - write_json(order, &path).await?; + write_json(order, &path, USE_TMP_FILE).await?; Ok(()) } @@ -294,7 +296,7 @@ mod native_impl { impl MyOrdersHistory for MyOrdersStorage { async fn save_order_in_history(&self, order: &Order) -> MyOrdersResult<()> { let path = my_order_history_file_path(&self.ctx, &order.uuid()); - write_json(order, &path).await?; + write_json(order, &path, USE_TMP_FILE).await?; Ok(()) } diff --git a/mm2src/lp_swap/saved_swap.rs b/mm2src/lp_swap/saved_swap.rs index 3b8263ad2f..6d610bbe16 100644 --- a/mm2src/lp_swap/saved_swap.rs +++ b/mm2src/lp_swap/saved_swap.rs @@ -176,6 +176,8 @@ mod native_impl { use crate::mm2::lp_swap::{my_swap_file_path, my_swaps_dir}; use common::fs::{read_dir_json, read_json, write_json, FsJsonError}; + const USE_TMP_FILE: bool = false; + impl From for SavedSwapError { fn from(fs: FsJsonError) -> Self { match fs { @@ -223,7 +225,7 @@ mod native_impl { async fn save_to_db(&self, ctx: &MmArc) -> SavedSwapResult<()> { let path = my_swap_file_path(ctx, self.uuid()); - write_json(self, &path).await?; + write_json(self, &path,USE_TMP_FILE).await?; Ok(()) } @@ -232,11 +234,11 @@ mod native_impl { match self { SavedSwap::Maker(maker) => { let path = stats_maker_swap_file_path(ctx, &maker.uuid); - write_json(self, &path).await?; + write_json(self, &path,USE_TMP_FILE).await?; }, SavedSwap::Taker(taker) => { let path = stats_taker_swap_file_path(ctx, &taker.uuid); - write_json(self, &path).await?; + write_json(self, &path,USE_TMP_FILE).await?; }, } Ok(()) From 82beca049eaaa4ea786611a3d0b9b1cc9abafe9e Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Thu, 14 Apr 2022 15:15:05 +0700 Subject: [PATCH 07/25] Move some coin's RPCs to separate modules * Move `account_balance`, `init_withdraw`, `init_create_account` RPCs from `coins` to `coins::rpc_command` * Use `RpcTask` for `init_scan_for_new_addresses` --- mm2src/coins/coin_balance.rs | 256 +----------------- mm2src/coins/lp_coins.rs | 10 +- mm2src/coins/rpc_command/account_balance.rs | 113 ++++++++ .../hd_account_balance_rpc_error.rs | 107 ++++++++ .../{ => rpc_command}/init_create_account.rs | 0 .../init_scan_for_new_addresses.rs | 152 +++++++++++ .../coins/{ => rpc_command}/init_withdraw.rs | 0 mm2src/coins/rpc_command/mod.rs | 5 + mm2src/coins/utxo/qtum.rs | 29 +- mm2src/coins/utxo/utxo_common.rs | 2 +- mm2src/coins/utxo/utxo_standard.rs | 29 +- mm2src/coins/utxo/utxo_tests.rs | 18 +- mm2src/coins/utxo/utxo_withdraw.rs | 2 +- mm2src/lp_swap/saved_swap.rs | 6 +- mm2src/rpc/dispatcher/dispatcher.rs | 12 +- 15 files changed, 442 insertions(+), 299 deletions(-) create mode 100644 mm2src/coins/rpc_command/account_balance.rs create mode 100644 mm2src/coins/rpc_command/hd_account_balance_rpc_error.rs rename mm2src/coins/{ => rpc_command}/init_create_account.rs (100%) create mode 100644 mm2src/coins/rpc_command/init_scan_for_new_addresses.rs rename mm2src/coins/{ => rpc_command}/init_withdraw.rs (100%) create mode 100644 mm2src/coins/rpc_command/mod.rs diff --git a/mm2src/coins/coin_balance.rs b/mm2src/coins/coin_balance.rs index 5183b24936..49b5f472e8 100644 --- a/mm2src/coins/coin_balance.rs +++ b/mm2src/coins/coin_balance.rs @@ -1,104 +1,19 @@ use crate::hd_pubkey::HDXPubExtractor; -use crate::hd_wallet::{AddressDerivingError, HDWalletCoinOps, InvalidBip44ChainError, NewAccountCreatingError}; -use crate::{lp_coinfind_or_err, BalanceError, BalanceResult, CoinBalance, CoinFindError, CoinWithDerivationMethod, - DerivationMethod, HDAddress, MarketCoinOps, MmCoinEnum, UnexpectedDerivationMethod}; +use crate::hd_wallet::{HDWalletCoinOps, NewAccountCreatingError}; +use crate::{BalanceError, BalanceResult, CoinBalance, CoinWithDerivationMethod, DerivationMethod, HDAddress, + MarketCoinOps}; use async_trait::async_trait; use common::custom_iter::TryUnzip; use common::log::{debug, info}; -use common::mm_ctx::MmArc; use common::mm_error::prelude::*; -use common::{HttpStatusCode, PagingOptionsEnum}; use crypto::{Bip44Chain, RpcDerivationPath}; use derive_more::Display; use futures::compat::Future01CompatExt; -use http::StatusCode; use std::fmt; use std::ops::Range; pub type AddressIdRange = Range; -#[derive(Debug, Display, Serialize, SerializeErrorType)] -#[serde(tag = "error_type", content = "error_data")] -pub enum HDAccountBalanceRpcError { - #[display(fmt = "No such coin {}", coin)] - NoSuchCoin { coin: String }, - #[display(fmt = "Coin is expected to be activated with the HD wallet derivation method")] - CoinIsActivatedNotWithHDWallet, - #[display(fmt = "HD account '{}' is not activated", account_id)] - UnknownAccount { account_id: u32 }, - #[display(fmt = "Coin doesn't support the given BIP44 chain: {:?}", chain)] - InvalidBip44Chain { chain: Bip44Chain }, - #[display(fmt = "Error deriving an address: {}", _0)] - ErrorDerivingAddress(String), - #[display(fmt = "Wallet storage error: {}", _0)] - WalletStorageError(String), - #[display(fmt = "Electrum/Native RPC invalid response: {}", _0)] - RpcInvalidResponse(String), - #[display(fmt = "Transport: {}", _0)] - Transport(String), - #[display(fmt = "Internal: {}", _0)] - Internal(String), -} - -impl HttpStatusCode for HDAccountBalanceRpcError { - fn status_code(&self) -> StatusCode { - match self { - HDAccountBalanceRpcError::NoSuchCoin { .. } - | HDAccountBalanceRpcError::CoinIsActivatedNotWithHDWallet - | HDAccountBalanceRpcError::UnknownAccount { .. } - | HDAccountBalanceRpcError::InvalidBip44Chain { .. } - | HDAccountBalanceRpcError::ErrorDerivingAddress(_) => StatusCode::BAD_REQUEST, - HDAccountBalanceRpcError::Transport(_) - | HDAccountBalanceRpcError::WalletStorageError(_) - | HDAccountBalanceRpcError::RpcInvalidResponse(_) - | HDAccountBalanceRpcError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, - } - } -} - -impl From for HDAccountBalanceRpcError { - fn from(e: CoinFindError) -> Self { - match e { - CoinFindError::NoSuchCoin { coin } => HDAccountBalanceRpcError::NoSuchCoin { coin }, - } - } -} - -impl From for HDAccountBalanceRpcError { - fn from(e: UnexpectedDerivationMethod) -> Self { - match e { - UnexpectedDerivationMethod::HDWalletUnavailable => HDAccountBalanceRpcError::CoinIsActivatedNotWithHDWallet, - unexpected_error => HDAccountBalanceRpcError::Internal(unexpected_error.to_string()), - } - } -} - -impl From for HDAccountBalanceRpcError { - fn from(e: BalanceError) -> Self { - match e { - BalanceError::Transport(transport) => HDAccountBalanceRpcError::Transport(transport), - BalanceError::InvalidResponse(rpc) => HDAccountBalanceRpcError::RpcInvalidResponse(rpc), - BalanceError::UnexpectedDerivationMethod(der_method) => HDAccountBalanceRpcError::from(der_method), - BalanceError::WalletStorageError(e) => HDAccountBalanceRpcError::Internal(e), - BalanceError::Internal(internal) => HDAccountBalanceRpcError::Internal(internal), - } - } -} - -impl From for HDAccountBalanceRpcError { - fn from(e: InvalidBip44ChainError) -> Self { HDAccountBalanceRpcError::InvalidBip44Chain { chain: e.chain } } -} - -impl From for HDAccountBalanceRpcError { - fn from(e: AddressDerivingError) -> Self { - match e { - AddressDerivingError::Bip32Error(bip32) => { - HDAccountBalanceRpcError::ErrorDerivingAddress(bip32.to_string()) - }, - } - } -} - #[derive(Display)] pub enum EnableCoinBalanceError { NewAccountCreatingError(NewAccountCreatingError), @@ -147,56 +62,6 @@ pub struct HDAddressBalance { pub balance: CoinBalance, } -#[derive(Deserialize)] -pub struct HDAccountBalanceRequest { - coin: String, - #[serde(flatten)] - params: AccountBalanceParams, -} - -#[derive(Deserialize)] -pub struct AccountBalanceParams { - pub account_index: u32, - pub chain: Bip44Chain, - #[serde(default = "common::ten")] - pub limit: usize, - #[serde(default)] - pub paging_options: PagingOptionsEnum, -} - -#[derive(Deserialize)] -pub struct CheckHDAccountBalanceRequest { - coin: String, - #[serde(flatten)] - params: CheckHDAccountBalanceParams, -} - -#[derive(Deserialize)] -pub struct CheckHDAccountBalanceParams { - pub account_index: u32, - pub gap_limit: Option, -} - -#[derive(Debug, PartialEq, Serialize)] -pub struct HDAccountBalanceResponse { - pub account_index: u32, - pub derivation_path: RpcDerivationPath, - pub addresses: Vec, - pub page_balance: CoinBalance, - pub limit: usize, - pub skipped: u32, - pub total: u32, - pub total_pages: usize, - pub paging_options: PagingOptionsEnum, -} - -#[derive(Debug, PartialEq, Serialize)] -pub struct CheckHDAccountBalanceResponse { - pub account_index: u32, - pub derivation_path: RpcDerivationPath, - pub new_addresses: Vec, -} - #[derive(Clone, Copy, Debug, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] pub enum EnableCoinScanPolicy { @@ -374,48 +239,9 @@ pub enum AddressBalanceStatus { NotUsed, } -#[async_trait] -pub trait HDWalletBalanceRpcOps { - async fn account_balance_rpc( - &self, - params: AccountBalanceParams, - ) -> MmResult; - - async fn scan_for_new_addresses_rpc( - &self, - params: CheckHDAccountBalanceParams, - ) -> MmResult; -} - -pub async fn account_balance( - ctx: MmArc, - req: HDAccountBalanceRequest, -) -> MmResult { - let coin = lp_coinfind_or_err(&ctx, &req.coin).await?; - match coin { - MmCoinEnum::UtxoCoin(utxo) => utxo.account_balance_rpc(req.params).await, - MmCoinEnum::QtumCoin(qtum) => qtum.account_balance_rpc(req.params).await, - _ => MmError::err(HDAccountBalanceRpcError::CoinIsActivatedNotWithHDWallet), - } -} - -pub async fn scan_for_new_addresses( - ctx: MmArc, - req: CheckHDAccountBalanceRequest, -) -> MmResult { - let coin = lp_coinfind_or_err(&ctx, &req.coin).await?; - match coin { - MmCoinEnum::UtxoCoin(utxo) => utxo.scan_for_new_addresses_rpc(req.params).await, - MmCoinEnum::QtumCoin(qtum) => qtum.scan_for_new_addresses_rpc(req.params).await, - _ => MmError::err(HDAccountBalanceRpcError::CoinIsActivatedNotWithHDWallet), - } -} - pub mod common_impl { use super::*; use crate::hd_wallet::{HDAccountOps, HDWalletOps}; - use common::calc_total_pages; - use std::ops::DerefMut; pub(crate) async fn enable_hd_account( coin: &Coin, @@ -501,80 +327,4 @@ pub mod common_impl { Ok(result) } - - pub async fn account_balance_rpc( - coin: &Coin, - params: AccountBalanceParams, - ) -> MmResult - where - Coin: HDWalletBalanceOps + CoinWithDerivationMethod::HDWallet> + Sync, - ::Address: fmt::Display + Clone, - { - let hd_wallet = coin.derivation_method().hd_wallet_or_err()?; - - let account_id = params.account_index; - let chain = params.chain; - let hd_account = hd_wallet - .get_account(account_id) - .await - .or_mm_err(|| HDAccountBalanceRpcError::UnknownAccount { account_id })?; - let total_addresses_number = hd_account.known_addresses_number(params.chain)?; - - let from_address_id = match params.paging_options { - PagingOptionsEnum::FromId(from_address_id) => from_address_id, - PagingOptionsEnum::PageNumber(page_number) => ((page_number.get() - 1) * params.limit) as u32, - }; - let to_address_id = std::cmp::min(from_address_id + params.limit as u32, total_addresses_number); - - let addresses = coin - .known_addresses_balances_with_ids(&hd_account, chain, from_address_id..to_address_id) - .await?; - let page_balance = addresses.iter().fold(CoinBalance::default(), |total, addr_balance| { - total + addr_balance.balance.clone() - }); - - let result = HDAccountBalanceResponse { - account_index: params.account_index, - derivation_path: RpcDerivationPath(hd_account.account_derivation_path()), - addresses, - page_balance, - limit: params.limit, - skipped: std::cmp::min(from_address_id, total_addresses_number), - total: total_addresses_number, - total_pages: calc_total_pages(total_addresses_number as usize, params.limit), - paging_options: params.paging_options, - }; - - Ok(result) - } - - pub async fn scan_for_new_addresses_rpc( - coin: &Coin, - params: CheckHDAccountBalanceParams, - ) -> MmResult - where - Coin: CoinWithDerivationMethod::HDWallet> + HDWalletBalanceOps + Sync, - ::Address: fmt::Display, - { - let hd_wallet = coin.derivation_method().hd_wallet_or_err()?; - - let account_id = params.account_index; - let mut hd_account = hd_wallet - .get_account_mut(account_id) - .await - .or_mm_err(|| HDAccountBalanceRpcError::CoinIsActivatedNotWithHDWallet)?; - let account_derivation_path = hd_account.account_derivation_path(); - let address_scanner = coin.produce_hd_address_scanner().await?; - let gap_limit = params.gap_limit.unwrap_or_else(|| hd_wallet.gap_limit()); - - let new_addresses = coin - .scan_for_new_addresses(hd_wallet, hd_account.deref_mut(), &address_scanner, gap_limit) - .await?; - - Ok(CheckHDAccountBalanceResponse { - account_index: account_id, - derivation_path: RpcDerivationPath(account_derivation_path), - new_addresses, - }) - } } diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 926eaa9504..474b73572c 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -105,12 +105,11 @@ pub mod eth; pub mod hd_pubkey; pub mod hd_wallet; pub mod hd_wallet_storage; -pub mod init_create_account; -pub mod init_withdraw; #[cfg(not(target_arch = "wasm32"))] pub mod lightning; #[cfg_attr(target_arch = "wasm32", allow(dead_code, unused_imports))] pub mod my_tx_history_v2; pub mod qrc20; +pub mod rpc_command; #[cfg(not(target_arch = "wasm32"))] pub mod sql_tx_history_storage; #[doc(hidden)] @@ -134,11 +133,12 @@ pub mod z_coin; use eth::{eth_coin_from_conf_and_request, EthCoin, EthTxFeeDetails, SignedEthTx}; use hd_wallet::{HDAddress, HDAddressId}; -use init_create_account::{CreateAccountTaskManager, CreateAccountTaskManagerShared}; -use init_withdraw::{WithdrawTaskManager, WithdrawTaskManagerShared}; use qrc20::Qrc20ActivationParams; use qrc20::{qrc20_coin_from_conf_and_params, Qrc20Coin, Qrc20FeeDetails}; use qtum::{Qrc20AddressError, ScriptHashTypeNotSupported}; +use rpc_command::init_create_account::{CreateAccountTaskManager, CreateAccountTaskManagerShared}; +use rpc_command::init_scan_for_new_addresses::{ScanAddressesTaskManager, ScanAddressesTaskManagerShared}; +use rpc_command::init_withdraw::{WithdrawTaskManager, WithdrawTaskManagerShared}; use utxo::bch::{bch_coin_from_conf_and_params, BchActivationRequest, BchCoin}; use utxo::qtum::{self, qtum_coin_with_priv_key, QtumCoin}; use utxo::qtum::{QtumDelegationOps, QtumDelegationRequest, QtumStakingInfosDetails}; @@ -1546,6 +1546,7 @@ pub struct CoinsContext { balance_update_handlers: AsyncMutex>>, withdraw_task_manager: WithdrawTaskManagerShared, create_account_manager: CreateAccountTaskManagerShared, + scan_addresses_manager: ScanAddressesTaskManagerShared, #[cfg(target_arch = "wasm32")] tx_history_db: SharedDb, #[cfg(target_arch = "wasm32")] @@ -1571,6 +1572,7 @@ impl CoinsContext { balance_update_handlers: AsyncMutex::new(vec![]), withdraw_task_manager: WithdrawTaskManager::new_shared(), create_account_manager: CreateAccountTaskManager::new_shared(), + scan_addresses_manager: ScanAddressesTaskManager::new_shared(), #[cfg(target_arch = "wasm32")] tx_history_db: ConstructibleDb::new_shared(ctx), #[cfg(target_arch = "wasm32")] diff --git a/mm2src/coins/rpc_command/account_balance.rs b/mm2src/coins/rpc_command/account_balance.rs new file mode 100644 index 0000000000..1eecd810c0 --- /dev/null +++ b/mm2src/coins/rpc_command/account_balance.rs @@ -0,0 +1,113 @@ +use crate::coin_balance::HDAddressBalance; +use crate::hd_wallet::HDWalletCoinOps; +use crate::rpc_command::hd_account_balance_rpc_error::HDAccountBalanceRpcError; +use crate::{lp_coinfind_or_err, CoinBalance, CoinWithDerivationMethod, MmCoinEnum}; +use async_trait::async_trait; +use common::mm_ctx::MmArc; +use common::mm_error::prelude::*; +use common::PagingOptionsEnum; +use crypto::{Bip44Chain, RpcDerivationPath}; +use std::fmt; + +#[derive(Deserialize)] +pub struct HDAccountBalanceRequest { + coin: String, + #[serde(flatten)] + params: AccountBalanceParams, +} + +#[derive(Deserialize)] +pub struct AccountBalanceParams { + pub account_index: u32, + pub chain: Bip44Chain, + #[serde(default = "common::ten")] + pub limit: usize, + #[serde(default)] + pub paging_options: PagingOptionsEnum, +} + +#[derive(Debug, PartialEq, Serialize)] +pub struct HDAccountBalanceResponse { + pub account_index: u32, + pub derivation_path: RpcDerivationPath, + pub addresses: Vec, + pub page_balance: CoinBalance, + pub limit: usize, + pub skipped: u32, + pub total: u32, + pub total_pages: usize, + pub paging_options: PagingOptionsEnum, +} + +#[async_trait] +pub trait AccountBalanceRpcOps { + async fn account_balance_rpc( + &self, + params: AccountBalanceParams, + ) -> MmResult; +} + +pub async fn account_balance( + ctx: MmArc, + req: HDAccountBalanceRequest, +) -> MmResult { + let coin = lp_coinfind_or_err(&ctx, &req.coin).await?; + match coin { + MmCoinEnum::UtxoCoin(utxo) => utxo.account_balance_rpc(req.params).await, + MmCoinEnum::QtumCoin(qtum) => qtum.account_balance_rpc(req.params).await, + _ => MmError::err(HDAccountBalanceRpcError::CoinIsActivatedNotWithHDWallet), + } +} + +pub mod common_impl { + use super::*; + use crate::coin_balance::HDWalletBalanceOps; + use crate::hd_wallet::{HDAccountOps, HDWalletOps}; + use common::calc_total_pages; + + pub async fn account_balance_rpc( + coin: &Coin, + params: AccountBalanceParams, + ) -> MmResult + where + Coin: HDWalletBalanceOps + CoinWithDerivationMethod::HDWallet> + Sync, + ::Address: fmt::Display + Clone, + { + let hd_wallet = coin.derivation_method().hd_wallet_or_err()?; + + let account_id = params.account_index; + let chain = params.chain; + let hd_account = hd_wallet + .get_account(account_id) + .await + .or_mm_err(|| HDAccountBalanceRpcError::UnknownAccount { account_id })?; + let total_addresses_number = hd_account.known_addresses_number(params.chain)?; + + let from_address_id = match params.paging_options { + PagingOptionsEnum::FromId(from_address_id) => from_address_id, + PagingOptionsEnum::PageNumber(page_number) => ((page_number.get() - 1) * params.limit) as u32, + }; + let to_address_id = std::cmp::min(from_address_id + params.limit as u32, total_addresses_number); + + let addresses = coin + .known_addresses_balances_with_ids(&hd_account, chain, from_address_id..to_address_id) + .await?; + let page_balance = addresses.iter().fold(CoinBalance::default(), |total, addr_balance| { + total + addr_balance.balance.clone() + }); + + let result = HDAccountBalanceResponse { + account_index: params.account_index, + derivation_path: RpcDerivationPath(hd_account.account_derivation_path()), + addresses, + page_balance, + limit: params.limit, + skipped: std::cmp::min(from_address_id, total_addresses_number), + total: total_addresses_number, + total_pages: calc_total_pages(total_addresses_number as usize, params.limit), + paging_options: params.paging_options, + }; + + Ok(result) + } +} diff --git a/mm2src/coins/rpc_command/hd_account_balance_rpc_error.rs b/mm2src/coins/rpc_command/hd_account_balance_rpc_error.rs new file mode 100644 index 0000000000..e47191293e --- /dev/null +++ b/mm2src/coins/rpc_command/hd_account_balance_rpc_error.rs @@ -0,0 +1,107 @@ +use crate::hd_wallet::{AddressDerivingError, InvalidBip44ChainError}; +use crate::{BalanceError, CoinFindError, UnexpectedDerivationMethod}; +use common::HttpStatusCode; +use crypto::Bip44Chain; +use derive_more::Display; +use http::StatusCode; +use rpc_task::RpcTaskError; +use std::time::Duration; + +#[derive(Clone, Debug, Display, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum HDAccountBalanceRpcError { + #[display(fmt = "No such coin {}", coin)] + NoSuchCoin { coin: String }, + #[display(fmt = "RPC timed out {:?}", _0)] + Timeout(Duration), + #[display(fmt = "Coin is expected to be activated with the HD wallet derivation method")] + CoinIsActivatedNotWithHDWallet, + #[display(fmt = "HD account '{}' is not activated", account_id)] + UnknownAccount { account_id: u32 }, + #[display(fmt = "Coin doesn't support the given BIP44 chain: {:?}", chain)] + InvalidBip44Chain { chain: Bip44Chain }, + #[display(fmt = "Error deriving an address: {}", _0)] + ErrorDerivingAddress(String), + #[display(fmt = "Wallet storage error: {}", _0)] + WalletStorageError(String), + #[display(fmt = "Electrum/Native RPC invalid response: {}", _0)] + RpcInvalidResponse(String), + #[display(fmt = "Transport: {}", _0)] + Transport(String), + #[display(fmt = "Internal: {}", _0)] + Internal(String), +} + +impl HttpStatusCode for HDAccountBalanceRpcError { + fn status_code(&self) -> StatusCode { + match self { + HDAccountBalanceRpcError::NoSuchCoin { .. } + | HDAccountBalanceRpcError::CoinIsActivatedNotWithHDWallet + | HDAccountBalanceRpcError::UnknownAccount { .. } + | HDAccountBalanceRpcError::InvalidBip44Chain { .. } + | HDAccountBalanceRpcError::ErrorDerivingAddress(_) => StatusCode::BAD_REQUEST, + HDAccountBalanceRpcError::Timeout(_) => StatusCode::REQUEST_TIMEOUT, + HDAccountBalanceRpcError::Transport(_) + | HDAccountBalanceRpcError::WalletStorageError(_) + | HDAccountBalanceRpcError::RpcInvalidResponse(_) + | HDAccountBalanceRpcError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +impl From for HDAccountBalanceRpcError { + fn from(e: CoinFindError) -> Self { + match e { + CoinFindError::NoSuchCoin { coin } => HDAccountBalanceRpcError::NoSuchCoin { coin }, + } + } +} + +impl From for HDAccountBalanceRpcError { + fn from(e: UnexpectedDerivationMethod) -> Self { + match e { + UnexpectedDerivationMethod::HDWalletUnavailable => HDAccountBalanceRpcError::CoinIsActivatedNotWithHDWallet, + unexpected_error => HDAccountBalanceRpcError::Internal(unexpected_error.to_string()), + } + } +} + +impl From for HDAccountBalanceRpcError { + fn from(e: BalanceError) -> Self { + match e { + BalanceError::Transport(transport) => HDAccountBalanceRpcError::Transport(transport), + BalanceError::InvalidResponse(rpc) => HDAccountBalanceRpcError::RpcInvalidResponse(rpc), + BalanceError::UnexpectedDerivationMethod(der_method) => HDAccountBalanceRpcError::from(der_method), + BalanceError::WalletStorageError(e) => HDAccountBalanceRpcError::Internal(e), + BalanceError::Internal(internal) => HDAccountBalanceRpcError::Internal(internal), + } + } +} + +impl From for HDAccountBalanceRpcError { + fn from(e: InvalidBip44ChainError) -> Self { HDAccountBalanceRpcError::InvalidBip44Chain { chain: e.chain } } +} + +impl From for HDAccountBalanceRpcError { + fn from(e: AddressDerivingError) -> Self { + match e { + AddressDerivingError::Bip32Error(bip32) => { + HDAccountBalanceRpcError::ErrorDerivingAddress(bip32.to_string()) + }, + } + } +} + +impl From for HDAccountBalanceRpcError { + fn from(e: RpcTaskError) -> Self { + let error = e.to_string(); + match e { + RpcTaskError::Canceled => HDAccountBalanceRpcError::Internal("Canceled".to_owned()), + RpcTaskError::Timeout(timeout) => HDAccountBalanceRpcError::Timeout(timeout), + RpcTaskError::NoSuchTask(_) | RpcTaskError::UnexpectedTaskStatus { .. } => { + HDAccountBalanceRpcError::Internal(error) + }, + RpcTaskError::Internal(internal) => HDAccountBalanceRpcError::Internal(internal), + } + } +} diff --git a/mm2src/coins/init_create_account.rs b/mm2src/coins/rpc_command/init_create_account.rs similarity index 100% rename from mm2src/coins/init_create_account.rs rename to mm2src/coins/rpc_command/init_create_account.rs diff --git a/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs b/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs new file mode 100644 index 0000000000..e26a9a95a7 --- /dev/null +++ b/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs @@ -0,0 +1,152 @@ +use crate::coin_balance::HDAddressBalance; +use crate::rpc_command::hd_account_balance_rpc_error::HDAccountBalanceRpcError; +use crate::{lp_coinfind_or_err, CoinsContext, MmCoinEnum}; +use async_trait::async_trait; +use common::mm_ctx::MmArc; +use common::mm_error::prelude::*; +use crypto::RpcDerivationPath; +use rpc_task::rpc_common::{InitRpcTaskResponse, RpcTaskStatusError, RpcTaskStatusRequest}; +use rpc_task::{RpcTask, RpcTaskHandle, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, RpcTaskTypes}; + +pub type ScanAddressesTaskManager = RpcTaskManager; +pub type ScanAddressesTaskManagerShared = RpcTaskManagerShared; +pub type ScanAddressesTaskHandle = RpcTaskHandle; +pub type ScanAddressesRpcTaskStatus = RpcTaskStatus< + ScanAddressesResponse, + HDAccountBalanceRpcError, + ScanAddressesInProgressStatus, + ScanAddressesAwaitingStatus, +>; + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct ScanAddressesResponse { + pub account_index: u32, + pub derivation_path: RpcDerivationPath, + pub new_addresses: Vec, +} + +#[derive(Deserialize)] +pub struct ScanAddressesRequest { + coin: String, + #[serde(flatten)] + params: ScanAddressesParams, +} + +#[derive(Deserialize)] +pub struct ScanAddressesParams { + pub account_index: u32, + pub gap_limit: Option, +} + +#[derive(Clone, Serialize)] +pub enum ScanAddressesInProgressStatus { + InProgress, +} + +/// We can't use `std::convert::Infallible` as [`RpcTaskTypes::UserAction`] because it doesn't implement `Serialize`. +/// Use `!` when it's stable. +#[derive(Clone, Serialize)] +pub enum ScanAddressesUserAction {} + +/// We can't use `std::convert::Infallible` as [`RpcTaskTypes::AwaitingStatus`] because it doesn't implement `Serialize`. +/// Use `!` when it's stable. +#[derive(Clone, Serialize)] +pub enum ScanAddressesAwaitingStatus {} + +#[async_trait] +pub trait InitScanAddressesRpcOps { + async fn init_scan_for_new_addresses_rpc( + &self, + params: ScanAddressesParams, + ) -> MmResult; +} + +pub struct InitScanAddressesTask { + req: ScanAddressesRequest, + coin: MmCoinEnum, +} + +impl RpcTaskTypes for InitScanAddressesTask { + type Item = ScanAddressesResponse; + type Error = HDAccountBalanceRpcError; + type InProgressStatus = ScanAddressesInProgressStatus; + type AwaitingStatus = ScanAddressesAwaitingStatus; + type UserAction = ScanAddressesUserAction; +} + +#[async_trait] +impl RpcTask for InitScanAddressesTask { + fn initial_status(&self) -> Self::InProgressStatus { ScanAddressesInProgressStatus::InProgress } + + async fn run(self, _task_handle: &ScanAddressesTaskHandle) -> Result> { + match self.coin { + MmCoinEnum::UtxoCoin(utxo) => utxo.init_scan_for_new_addresses_rpc(self.req.params).await, + MmCoinEnum::QtumCoin(qtum) => qtum.init_scan_for_new_addresses_rpc(self.req.params).await, + _ => MmError::err(HDAccountBalanceRpcError::CoinIsActivatedNotWithHDWallet), + } + } +} + +pub async fn init_scan_for_new_addresses( + ctx: MmArc, + req: ScanAddressesRequest, +) -> MmResult { + let coin = lp_coinfind_or_err(&ctx, &req.coin).await?; + let coins_ctx = CoinsContext::from_ctx(&ctx).map_to_mm(HDAccountBalanceRpcError::Internal)?; + let task = InitScanAddressesTask { req, coin }; + let task_id = ScanAddressesTaskManager::spawn_rpc_task(&coins_ctx.scan_addresses_manager, task)?; + Ok(InitRpcTaskResponse { task_id }) +} + +pub async fn init_scan_for_new_addresses_status( + ctx: MmArc, + req: RpcTaskStatusRequest, +) -> MmResult { + let coins_ctx = CoinsContext::from_ctx(&ctx).map_to_mm(RpcTaskStatusError::Internal)?; + let mut task_manager = coins_ctx + .scan_addresses_manager + .lock() + .map_to_mm(|e| RpcTaskStatusError::Internal(e.to_string()))?; + task_manager + .task_status(req.task_id, req.forget_if_finished) + .or_mm_err(|| RpcTaskStatusError::NoSuchTask(req.task_id)) +} + +pub mod common_impl { + use super::*; + use crate::coin_balance::HDWalletBalanceOps; + use crate::hd_wallet::{HDAccountOps, HDWalletCoinOps, HDWalletOps}; + use crate::CoinWithDerivationMethod; + use std::fmt; + use std::ops::DerefMut; + + pub async fn scan_for_new_addresses_rpc( + coin: &Coin, + params: ScanAddressesParams, + ) -> MmResult + where + Coin: CoinWithDerivationMethod::HDWallet> + HDWalletBalanceOps + Sync, + ::Address: fmt::Display, + { + let hd_wallet = coin.derivation_method().hd_wallet_or_err()?; + + let account_id = params.account_index; + let mut hd_account = hd_wallet + .get_account_mut(account_id) + .await + .or_mm_err(|| HDAccountBalanceRpcError::CoinIsActivatedNotWithHDWallet)?; + let account_derivation_path = hd_account.account_derivation_path(); + let address_scanner = coin.produce_hd_address_scanner().await?; + let gap_limit = params.gap_limit.unwrap_or_else(|| hd_wallet.gap_limit()); + + let new_addresses = coin + .scan_for_new_addresses(hd_wallet, hd_account.deref_mut(), &address_scanner, gap_limit) + .await?; + + Ok(ScanAddressesResponse { + account_index: account_id, + derivation_path: RpcDerivationPath(account_derivation_path), + new_addresses, + }) + } +} diff --git a/mm2src/coins/init_withdraw.rs b/mm2src/coins/rpc_command/init_withdraw.rs similarity index 100% rename from mm2src/coins/init_withdraw.rs rename to mm2src/coins/rpc_command/init_withdraw.rs diff --git a/mm2src/coins/rpc_command/mod.rs b/mm2src/coins/rpc_command/mod.rs new file mode 100644 index 0000000000..7a98d3fc2e --- /dev/null +++ b/mm2src/coins/rpc_command/mod.rs @@ -0,0 +1,5 @@ +pub mod account_balance; +pub mod hd_account_balance_rpc_error; +pub mod init_create_account; +pub mod init_scan_for_new_addresses; +pub mod init_withdraw; diff --git a/mm2src/coins/utxo/qtum.rs b/mm2src/coins/utxo/qtum.rs index 62a215434c..f236b4a277 100644 --- a/mm2src/coins/utxo/qtum.rs +++ b/mm2src/coins/utxo/qtum.rs @@ -1,15 +1,17 @@ use super::*; -use crate::coin_balance::{self, AccountBalanceParams, CheckHDAccountBalanceParams, CheckHDAccountBalanceResponse, - EnableCoinBalanceError, HDAccountBalance, HDAccountBalanceResponse, - HDAccountBalanceRpcError, HDAddressBalance, HDWalletBalance, HDWalletBalanceOps, - HDWalletBalanceRpcOps}; +use crate::coin_balance::{self, EnableCoinBalanceError, HDAccountBalance, HDAddressBalance, HDWalletBalance, + HDWalletBalanceOps}; use crate::hd_pubkey::{ExtractExtendedPubkey, HDExtractPubkeyError, HDXPubExtractor}; use crate::hd_wallet::{self, AccountUpdatingError, AddressDerivingError, GetNewHDAddressParams, GetNewHDAddressResponse, HDAccountMut, HDWalletRpcError, HDWalletRpcOps, NewAccountCreatingError}; use crate::hd_wallet_storage::HDWalletCoinWithStorageOps; -use crate::init_create_account::{self, CreateNewAccountParams, InitCreateHDAccountRpcOps}; -use crate::init_withdraw::{InitWithdrawCoin, WithdrawTaskHandle}; +use crate::rpc_command::account_balance::{self, AccountBalanceParams, AccountBalanceRpcOps, HDAccountBalanceResponse}; +use crate::rpc_command::hd_account_balance_rpc_error::HDAccountBalanceRpcError; +use crate::rpc_command::init_create_account::{self, CreateNewAccountParams, InitCreateHDAccountRpcOps}; +use crate::rpc_command::init_scan_for_new_addresses::{self, InitScanAddressesRpcOps, ScanAddressesParams, + ScanAddressesResponse}; +use crate::rpc_command::init_withdraw::{InitWithdrawCoin, WithdrawTaskHandle}; use crate::utxo::utxo_builder::{MergeUtxoArcOps, UtxoCoinBuildError, UtxoCoinBuilder, UtxoCoinBuilderCommonOps, UtxoFieldsWithHardwareWalletBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::{eth, CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, DelegationError, DelegationFut, @@ -981,19 +983,22 @@ impl HDWalletRpcOps for QtumCoin { } #[async_trait] -impl HDWalletBalanceRpcOps for QtumCoin { +impl AccountBalanceRpcOps for QtumCoin { async fn account_balance_rpc( &self, params: AccountBalanceParams, ) -> MmResult { - coin_balance::common_impl::account_balance_rpc(self, params).await + account_balance::common_impl::account_balance_rpc(self, params).await } +} - async fn scan_for_new_addresses_rpc( +#[async_trait] +impl InitScanAddressesRpcOps for QtumCoin { + async fn init_scan_for_new_addresses_rpc( &self, - params: CheckHDAccountBalanceParams, - ) -> MmResult { - coin_balance::common_impl::scan_for_new_addresses_rpc(self, params).await + params: ScanAddressesParams, + ) -> MmResult { + init_scan_for_new_addresses::common_impl::scan_for_new_addresses_rpc(self, params).await } } diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index ab88e812b3..3888e0e1ca 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -4,7 +4,7 @@ use crate::hd_pubkey::{ExtractExtendedPubkey, HDExtractPubkeyError, HDXPubExtrac use crate::hd_wallet::{AccountUpdatingError, AddressDerivingError, HDAccountMut, HDAccountsMap, NewAccountCreatingError}; use crate::hd_wallet_storage::{HDWalletCoinWithStorageOps, HDWalletStorageResult}; -use crate::init_withdraw::WithdrawTaskHandle; +use crate::rpc_command::init_withdraw::WithdrawTaskHandle; use crate::utxo::rpc_clients::{electrum_script_hash, BlockHashOrHeight, UnspentInfo, UnspentMap, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcResult}; use crate::utxo::utxo_withdraw::{InitUtxoWithdraw, StandardUtxoWithdraw, UtxoWithdraw}; diff --git a/mm2src/coins/utxo/utxo_standard.rs b/mm2src/coins/utxo/utxo_standard.rs index 7e2a2fa17a..02bc4a1578 100644 --- a/mm2src/coins/utxo/utxo_standard.rs +++ b/mm2src/coins/utxo/utxo_standard.rs @@ -1,15 +1,17 @@ use super::*; -use crate::coin_balance::{self, AccountBalanceParams, CheckHDAccountBalanceParams, CheckHDAccountBalanceResponse, - EnableCoinBalanceError, HDAccountBalance, HDAccountBalanceResponse, - HDAccountBalanceRpcError, HDAddressBalance, HDWalletBalance, HDWalletBalanceOps, - HDWalletBalanceRpcOps}; +use crate::coin_balance::{self, EnableCoinBalanceError, HDAccountBalance, HDAddressBalance, HDWalletBalance, + HDWalletBalanceOps}; use crate::hd_pubkey::{ExtractExtendedPubkey, HDExtractPubkeyError, HDXPubExtractor}; use crate::hd_wallet::{self, AccountUpdatingError, AddressDerivingError, GetNewHDAddressParams, GetNewHDAddressResponse, HDAccountMut, HDWalletRpcError, HDWalletRpcOps, NewAccountCreatingError}; use crate::hd_wallet_storage::HDWalletCoinWithStorageOps; -use crate::init_create_account::{self, CreateNewAccountParams, InitCreateHDAccountRpcOps}; -use crate::init_withdraw::{InitWithdrawCoin, WithdrawTaskHandle}; +use crate::rpc_command::account_balance::{self, AccountBalanceParams, AccountBalanceRpcOps, HDAccountBalanceResponse}; +use crate::rpc_command::hd_account_balance_rpc_error::HDAccountBalanceRpcError; +use crate::rpc_command::init_create_account::{self, CreateNewAccountParams, InitCreateHDAccountRpcOps}; +use crate::rpc_command::init_scan_for_new_addresses::{self, InitScanAddressesRpcOps, ScanAddressesParams, + ScanAddressesResponse}; +use crate::rpc_command::init_withdraw::{InitWithdrawCoin, WithdrawTaskHandle}; use crate::utxo::utxo_builder::{UtxoArcBuilder, UtxoCoinBuilder}; use crate::{CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, GetWithdrawSenderAddress, NegotiateSwapContractAddrErr, PrivKeyBuildPolicy, SwapOps, TradePreimageValue, ValidateAddressResult, @@ -756,19 +758,22 @@ impl HDWalletCoinWithStorageOps for UtxoStandardCoin { } #[async_trait] -impl HDWalletBalanceRpcOps for UtxoStandardCoin { +impl AccountBalanceRpcOps for UtxoStandardCoin { async fn account_balance_rpc( &self, params: AccountBalanceParams, ) -> MmResult { - coin_balance::common_impl::account_balance_rpc(self, params).await + account_balance::common_impl::account_balance_rpc(self, params).await } +} - async fn scan_for_new_addresses_rpc( +#[async_trait] +impl InitScanAddressesRpcOps for UtxoStandardCoin { + async fn init_scan_for_new_addresses_rpc( &self, - params: CheckHDAccountBalanceParams, - ) -> MmResult { - coin_balance::common_impl::scan_for_new_addresses_rpc(self, params).await + params: ScanAddressesParams, + ) -> MmResult { + init_scan_for_new_addresses::common_impl::scan_for_new_addresses_rpc(self, params).await } } diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index db9f48a86c..fa079bafac 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -1,8 +1,10 @@ use super::*; -use crate::coin_balance::{AccountBalanceParams, CheckHDAccountBalanceParams, CheckHDAccountBalanceResponse, - HDAccountBalanceResponse, HDAddressBalance, HDWalletBalanceRpcOps}; +use crate::coin_balance::HDAddressBalance; use crate::hd_wallet::HDAccountsMap; use crate::hd_wallet_storage::{HDWalletMockStorage, HDWalletStorageInternalOps}; +use crate::rpc_command::account_balance::{AccountBalanceParams, AccountBalanceRpcOps, HDAccountBalanceResponse}; +use crate::rpc_command::init_scan_for_new_addresses::{InitScanAddressesRpcOps, ScanAddressesParams, + ScanAddressesResponse}; use crate::utxo::qtum::{qtum_coin_with_priv_key, QtumCoin, QtumDelegationOps, QtumDelegationRequest}; use crate::utxo::rpc_clients::{BlockHashOrHeight, ElectrumBalance, ElectrumClient, ElectrumClientImpl, GetAddressInfoRes, ListSinceBlockRes, ListTransactionsItem, NativeClient, @@ -3705,12 +3707,12 @@ fn test_scan_for_new_addresses() { NEW_INTERNAL_ADDRESSES_NUMBER = 4; } - let params = CheckHDAccountBalanceParams { + let params = ScanAddressesParams { account_index: 0, gap_limit: Some(3), }; - let actual = block_on(coin.scan_for_new_addresses_rpc(params)).expect("!account_balance_rpc"); - let expected = CheckHDAccountBalanceResponse { + let actual = block_on(coin.init_scan_for_new_addresses_rpc(params)).expect("!account_balance_rpc"); + let expected = ScanAddressesResponse { account_index: 0, derivation_path: DerivationPath::from_str("m/44'/141'/0'").unwrap().into(), new_addresses: get_balances!( @@ -3730,12 +3732,12 @@ fn test_scan_for_new_addresses() { NEW_INTERNAL_ADDRESSES_NUMBER = 2; } - let params = CheckHDAccountBalanceParams { + let params = ScanAddressesParams { account_index: 1, gap_limit: None, }; - let actual = block_on(coin.scan_for_new_addresses_rpc(params)).expect("!account_balance_rpc"); - let expected = CheckHDAccountBalanceResponse { + let actual = block_on(coin.init_scan_for_new_addresses_rpc(params)).expect("!account_balance_rpc"); + let expected = ScanAddressesResponse { account_index: 1, derivation_path: DerivationPath::from_str("m/44'/141'/1'").unwrap().into(), new_addresses: get_balances!( diff --git a/mm2src/coins/utxo/utxo_withdraw.rs b/mm2src/coins/utxo/utxo_withdraw.rs index 154a39a68b..0cc788245c 100644 --- a/mm2src/coins/utxo/utxo_withdraw.rs +++ b/mm2src/coins/utxo/utxo_withdraw.rs @@ -1,4 +1,4 @@ -use crate::init_withdraw::{WithdrawAwaitingStatus, WithdrawInProgressStatus, WithdrawTaskHandle}; +use crate::rpc_command::init_withdraw::{WithdrawAwaitingStatus, WithdrawInProgressStatus, WithdrawTaskHandle}; use crate::utxo::utxo_common::{big_decimal_from_sat, UtxoTxBuilder}; use crate::utxo::{output_script, sat_from_big_decimal, ActualTxFee, Address, FeePolicy, PrivKeyPolicy, UtxoAddressFormat, UtxoCoinFields, UtxoCommonOps, UtxoFeeDetails, UtxoTx, UTXO_LOCK}; diff --git a/mm2src/lp_swap/saved_swap.rs b/mm2src/lp_swap/saved_swap.rs index 6d610bbe16..0f525fe2cc 100644 --- a/mm2src/lp_swap/saved_swap.rs +++ b/mm2src/lp_swap/saved_swap.rs @@ -225,7 +225,7 @@ mod native_impl { async fn save_to_db(&self, ctx: &MmArc) -> SavedSwapResult<()> { let path = my_swap_file_path(ctx, self.uuid()); - write_json(self, &path,USE_TMP_FILE).await?; + write_json(self, &path, USE_TMP_FILE).await?; Ok(()) } @@ -234,11 +234,11 @@ mod native_impl { match self { SavedSwap::Maker(maker) => { let path = stats_maker_swap_file_path(ctx, &maker.uuid); - write_json(self, &path,USE_TMP_FILE).await?; + write_json(self, &path, USE_TMP_FILE).await?; }, SavedSwap::Taker(taker) => { let path = stats_taker_swap_file_path(ctx, &taker.uuid); - write_json(self, &path,USE_TMP_FILE).await?; + write_json(self, &path, USE_TMP_FILE).await?; }, } Ok(()) diff --git a/mm2src/rpc/dispatcher/dispatcher.rs b/mm2src/rpc/dispatcher/dispatcher.rs index d6b9575556..db81ef2456 100644 --- a/mm2src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/rpc/dispatcher/dispatcher.rs @@ -6,12 +6,13 @@ use crate::{mm2::lp_stats::{add_node_to_version_stat, remove_node_from_version_s stop_version_stat_collection, update_version_stat_collection}, mm2::lp_swap::{recreate_swap_data, trade_preimage_rpc}, mm2::rpc::get_public_key::get_public_key}; -use coins::coin_balance::{account_balance, scan_for_new_addresses}; use coins::hd_wallet::get_new_address; -use coins::init_create_account::{init_create_new_account, init_create_new_account_status, - init_create_new_account_user_action}; -use coins::init_withdraw::{init_withdraw, withdraw_status, withdraw_user_action}; use coins::my_tx_history_v2::my_tx_history_v2_rpc; +use coins::rpc_command::account_balance::account_balance; +use coins::rpc_command::init_create_account::{init_create_new_account, init_create_new_account_status, + init_create_new_account_user_action}; +use coins::rpc_command::init_scan_for_new_addresses::{init_scan_for_new_addresses, init_scan_for_new_addresses_status}; +use coins::rpc_command::init_withdraw::{init_withdraw, withdraw_status, withdraw_user_action}; use coins::utxo::bch::BchCoin; use coins::utxo::qtum::QtumCoin; use coins::utxo::slp::SlpToken; @@ -143,7 +144,8 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, recreate_swap_data).await, "remove_delegation" => handle_mmrpc(ctx, request, remove_delegation).await, "remove_node_from_version_stat" => handle_mmrpc(ctx, request, remove_node_from_version_stat).await, - "scan_for_new_addresses" => handle_mmrpc(ctx, request, scan_for_new_addresses).await, + "init_scan_for_new_addresses" => handle_mmrpc(ctx, request, init_scan_for_new_addresses).await, + "init_scan_for_new_addresses_status" => handle_mmrpc(ctx, request, init_scan_for_new_addresses_status).await, "start_simple_market_maker_bot" => handle_mmrpc(ctx, request, start_simple_market_maker_bot).await, "start_version_stat_collection" => handle_mmrpc(ctx, request, start_version_stat_collection).await, "stop_simple_market_maker_bot" => handle_mmrpc(ctx, request, stop_simple_market_maker_bot).await, From ff8400b59edb473be69a4f3cd6b6b77f3d2c0665 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Thu, 14 Apr 2022 15:23:17 +0700 Subject: [PATCH 08/25] Fix WASM build, ZCoin warnings --- mm2src/coins/utxo/utxo_common.rs | 2 +- mm2src/coins/z_coin.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 3888e0e1ca..eca1b1307c 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2931,7 +2931,7 @@ pub async fn cache_transactions_if_possible(coin: &UtxoCoinFields, txs: &HashMap /// Save the given `txs` transactions in a TX cache. /// It takes `txs` as the Hash map to avoid duplicating same transactions. #[cfg(target_arch = "wasm32")] -pub async fn cache_transactions_if_possible(coin: &UtxoCoinFields, txs: &HashMap) { Ok(()) } +pub async fn cache_transactions_if_possible(_coin: &UtxoCoinFields, _txs: &HashMap) {} /// Swap contract address is not used by standard UTXO coins. pub fn swap_contract_address() -> Option { None } diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index ad4e79081c..0f7de18d67 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -4,10 +4,10 @@ use crate::utxo::utxo_builder::{UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPriv UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::{big_decimal_from_sat_unsigned, payment_script}; use crate::utxo::{sat_from_big_decimal, utxo_common, ActualTxFee, AdditionalTxData, Address, BroadcastTxErr, - FeePolicy, HistoryUtxoTx, HistoryUtxoTxMap, ListUtxoOps, MatureUnspentList, MatureUnspentMap, - RecentlySpentOutPoints, RecentlySpentOutPointsGuard, UtxoActivationParams, UtxoAddressFormat, - UtxoArc, UtxoCoinFields, UtxoCommonOps, UtxoFeeDetails, UtxoTxBroadcastOps, UtxoTxGenerationOps, - UtxoWeak, VerboseTransactionFrom}; + FeePolicy, HistoryUtxoTx, HistoryUtxoTxMap, ListUtxoOps, MatureUnspentMap, + RecentlySpentOutPointsGuard, UtxoActivationParams, UtxoAddressFormat, UtxoArc, UtxoCoinFields, + UtxoCommonOps, UtxoFeeDetails, UtxoTxBroadcastOps, UtxoTxGenerationOps, UtxoWeak, + VerboseTransactionFrom}; use crate::{BalanceFut, CoinBalance, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MarketCoinOps, MmCoin, NegotiateSwapContractAddrErr, NumConversError, SwapOps, TradeFee, TradePreimageFut, TradePreimageResult, TradePreimageValue, TransactionDetails, TransactionEnum, TransactionFut, TxFeeDetails, @@ -24,7 +24,7 @@ use common::mm_number::{BigDecimal, MmNumber}; use common::privkey::key_pair_from_secret; use common::{log, now_ms}; use futures::compat::Future01CompatExt; -use futures::lock::{Mutex as AsyncMutex, MutexGuard as AsyncMutexGuard}; +use futures::lock::Mutex as AsyncMutex; use futures::{FutureExt, TryFutureExt}; use futures01::Future; use keys::hash::H256; From 36e7e4823d40fc89dfa569d6a1d115d8f7b8e2bf Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Sun, 17 Apr 2022 20:53:11 +0700 Subject: [PATCH 09/25] Minor changes, cover some functions with comments --- Cargo.lock | 1 - mm2src/coins/hd_pubkey.rs | 2 +- mm2src/coins/qrc20.rs | 6 +-- mm2src/coins/utxo.rs | 11 ++--- mm2src/coins/utxo/bch.rs | 8 ++-- mm2src/coins/utxo/qtum.rs | 6 +-- mm2src/coins/utxo/rpc_clients.rs | 33 +++++++++++---- mm2src/coins/utxo/tx_cache.rs | 4 +- mm2src/coins/utxo/utxo_common.rs | 28 +++++++------ mm2src/coins/utxo/utxo_standard.rs | 6 +-- mm2src/coins/z_coin.rs | 6 +-- .../src/utxo_activation/common_impl.rs | 2 +- mm2src/common/custom_iter.rs | 14 +++---- mm2src/common/jsonrpc_client.rs | 42 +++++++++++++++++-- mm2src/crypto/Cargo.toml | 1 - 15 files changed, 111 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fb5428efda..d33d51f0e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1501,7 +1501,6 @@ checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" name = "crypto" version = "0.1.0" dependencies = [ - "async-std", "async-trait", "bip32", "bitcrypto", diff --git a/mm2src/coins/hd_pubkey.rs b/mm2src/coins/hd_pubkey.rs index 1fa88ef6b8..a35ee69565 100644 --- a/mm2src/coins/hd_pubkey.rs +++ b/mm2src/coins/hd_pubkey.rs @@ -155,7 +155,7 @@ where } /// Constructs an Xpub extractor without checking if the MarketMaker is initialized with a hardware wallet. - pub async fn new_unchecked( + pub fn new_unchecked( ctx: &MmArc, task_handle: &'task RpcTaskHandle, statuses: HwConnectStatuses, diff --git a/mm2src/coins/qrc20.rs b/mm2src/coins/qrc20.rs index 922fd067d2..2cbef85478 100644 --- a/mm2src/coins/qrc20.rs +++ b/mm2src/coins/qrc20.rs @@ -587,10 +587,10 @@ impl ListUtxoOps for Qrc20Coin { utxo_common::get_unspent_ordered_map(self, addresses).await } - async fn get_all_unspent_ordered_map<'a>( - &'a self, + async fn get_all_unspent_ordered_map( + &self, addresses: Vec
, - ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard)> { + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { utxo_common::get_all_unspent_ordered_map(self, addresses).await } diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index c60b8cd3ea..11a3be3464 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -740,6 +740,7 @@ impl UtxoAddressScanner { } } +/// Contains lists of mature and immature UTXOs. #[derive(Debug, Default)] pub struct MatureUnspentList { mature: Vec, @@ -761,7 +762,7 @@ impl MatureUnspentList { } } - pub fn into_mature(self) -> Vec { self.mature } + pub fn only_mature(self) -> Vec { self.mature } pub fn to_coin_balance(&self, decimals: u8) -> CoinBalance { let fold = |acc: BigDecimal, x: &UnspentInfo| acc + big_decimal_from_sat_unsigned(x.value, decimals); @@ -824,7 +825,7 @@ pub trait UtxoCommonOps: keypair: &KeyPair, ) -> Result; - /// Try to load verbose transaction from cache or try to request it from Rpc client. + /// Loads verbose transactions from cache or requests it using RPC client. fn get_verbose_transactions_from_cache_or_rpc( &self, tx_ids: HashSet, @@ -883,10 +884,10 @@ pub trait ListUtxoOps { /// /// The function doesn't check if the unspents are mature or immature. /// Consider using [`ListUtxoOps::get_unspent_ordered_list`] or [`ListUtxoOps::get_unspent_ordered_map`] instead. - async fn get_all_unspent_ordered_map<'a>( - &'a self, + async fn get_all_unspent_ordered_map( + &self, addresses: Vec
, - ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard)>; + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)>; /// Returns available mature and immature unspents in ascending order for every given `addresses` /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). diff --git a/mm2src/coins/utxo/bch.rs b/mm2src/coins/utxo/bch.rs index 2fecdfcf88..02a5bdf2db 100644 --- a/mm2src/coins/utxo/bch.rs +++ b/mm2src/coins/utxo/bch.rs @@ -326,7 +326,7 @@ impl BchCoin { addresses: Vec
, ) -> UtxoRpcResult<(BchUnspentsMap, RecentlySpentOutPointsGuard<'_>)> { let (all_unspents, recently_spent) = utxo_common::get_unspent_ordered_map(self, addresses).await?; - // Get an iterator of futures: `Iterator>` + // Get an iterator of futures: `Future>` let fut_it = all_unspents.into_iter().map(|(address, unspents)| { self.utxos_into_bch_unspents(unspents) .map(move |res| -> UtxoRpcResult<_> { @@ -756,10 +756,10 @@ impl ListUtxoOps for BchCoin { Ok((unspents, recently_spent)) } - async fn get_all_unspent_ordered_map<'a>( - &'a self, + async fn get_all_unspent_ordered_map( + &self, addresses: Vec
, - ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard)> { + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { utxo_common::get_all_unspent_ordered_map(self, addresses).await } diff --git a/mm2src/coins/utxo/qtum.rs b/mm2src/coins/utxo/qtum.rs index 0e62b86f2a..2613769499 100644 --- a/mm2src/coins/utxo/qtum.rs +++ b/mm2src/coins/utxo/qtum.rs @@ -334,10 +334,10 @@ impl ListUtxoOps for QtumCoin { utxo_common::get_unspent_ordered_map(self, addresses).await } - async fn get_all_unspent_ordered_map<'a>( - &'a self, + async fn get_all_unspent_ordered_map( + &self, addresses: Vec
, - ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard)> { + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { utxo_common::get_all_unspent_ordered_map(self, addresses).await } diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index 77241581c9..ab891554d3 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -7,7 +7,7 @@ use async_trait::async_trait; use bigdecimal::BigDecimal; use chain::{BlockHeader, BlockHeaderBits, BlockHeaderNonce, OutPoint, Transaction as UtxoTx}; use common::custom_futures::{select_ok_sequential, FutureTimerExt}; -use common::custom_iter::IntoGroupMapResult; +use common::custom_iter::TryIntoGroupMap; use common::executor::{spawn, Timer}; use common::jsonrpc_client::{BatchRequest, JsonRpcBatchResponse, JsonRpcClient, JsonRpcError, JsonRpcErrorType, JsonRpcId, JsonRpcMultiClient, JsonRpcRemoteAddr, JsonRpcRequest, JsonRpcRequestEnum, @@ -26,6 +26,7 @@ use futures01::future::select_ok; use futures01::sync::{mpsc, oneshot}; use futures01::{Future, Sink, Stream}; use http::Uri; +use itertools::Itertools; use keys::hash::H256; use keys::{Address, Type as ScriptType}; #[cfg(test)] use mocktopus::macros::*; @@ -35,7 +36,7 @@ use serialization::{deserialize, serialize, serialize_with_flags, CoinVariant, C SERIALIZE_TRANSACTION_WITNESS}; use sha2::{Digest, Sha256}; use std::collections::hash_map::Entry; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::fmt; use std::io; use std::net::{SocketAddr, ToSocketAddrs}; @@ -262,27 +263,38 @@ impl From for UtxoRpcError { /// Common operations that both types of UTXO clients have but implement them differently #[async_trait] pub trait UtxoRpcClientOps: fmt::Debug + Send + Sync + 'static { + /// Returns available unspents for the given `address`. fn list_unspent(&self, address: &Address, decimals: u8) -> UtxoRpcFut>; + /// Returns available unspents for every given `addresses`. fn list_unspent_group(&self, addresses: Vec
, decimals: u8) -> UtxoRpcFut; + /// Submits the given `tx` transaction to blockchain network. fn send_transaction(&self, tx: &UtxoTx) -> UtxoRpcFut; + /// Submits the raw `tx` transaction (serialized, hex-encoded) to blockchain network. fn send_raw_transaction(&self, tx: BytesJson) -> UtxoRpcFut; + /// Returns raw transaction (serialized, hex-encoded) by the given `txid`. fn get_transaction_bytes(&self, txid: &H256Json) -> UtxoRpcFut; + /// Returns verbose transaction by the given `txid`. fn get_verbose_transaction(&self, txid: &H256Json) -> UtxoRpcFut; + /// Returns verbose transactions in the same order they were requested. fn get_verbose_transactions(&self, tx_ids: &[H256Json]) -> UtxoRpcFut>; + /// Returns the height of the most-work fully-validated chain. fn get_block_count(&self) -> UtxoRpcFut; + /// Requests balance of the given `address`. fn display_balance(&self, address: Address, decimals: u8) -> RpcRes; + /// Requests balances of the given `addresses`. + /// The pairs `(Address, BigDecimal)` are guaranteed to be in the same order in which they were requested. fn display_balances(&self, addresses: Vec
, decimals: u8) -> UtxoRpcFut>; - /// returns fee estimation per KByte in satoshis + /// Returns fee estimation per KByte in satoshis. fn estimate_fee_sat( &self, decimals: u8, @@ -291,8 +303,10 @@ pub trait UtxoRpcClientOps: fmt::Debug + Send + Sync + 'static { n_blocks: u32, ) -> UtxoRpcFut; + /// Returns the minimum fee a low-priority transaction must pay in order to be accepted to the daemon’s memory pool. fn get_relay_fee(&self) -> RpcRes; + /// Tries to find a transaction that spends the specified `vout` output of the `tx_hash` transaction. fn find_output_spend( &self, tx_hash: H256, @@ -309,6 +323,7 @@ pub trait UtxoRpcClientOps: fmt::Debug + Send + Sync + 'static { coin_variant: CoinVariant, ) -> UtxoRpcFut; + /// Returns block time in seconds since epoch (Jan 1 1970 GMT). async fn get_block_timestamp(&self, height: u64) -> Result>; } @@ -682,7 +697,7 @@ impl UtxoRpcClientOps for NativeClient { Ok((orig_address, unspent_info)) }) // Collect `(Address, UnspentInfo)` items into `HashMap>` grouped by the addresses. - .into_group_map_result() + .try_into_group_map() }); Box::new(fut) } @@ -954,7 +969,7 @@ impl NativeClientImpl { } /// https://developer.bitcoin.org/reference/rpc/getrawtransaction.html - /// Always returns verbose transactions in a batch. + /// Always returns verbose transactions in the same order they were requested. fn get_raw_transaction_verbose_batch(&self, tx_ids: &[H256Json]) -> RpcRes> { let verbose = 1; Box::new( @@ -1733,11 +1748,9 @@ impl ElectrumClient { unspents .into_iter() .map(|hash_unspents| { - let mut set = HashSet::with_capacity(hash_unspents.len()); let hash_unspents: Vec<_> = hash_unspents .into_iter() - // Don't use `Itertools::unique_by` because it seems to be inefficient. - .filter(|unspent| set.insert((unspent.tx_hash, unspent.tx_pos))) + .unique_by(|unspent| (unspent.tx_hash, unspent.tx_pos)) .collect(); hash_unspents }) @@ -1762,6 +1775,8 @@ impl ElectrumClient { Box::new(fut.boxed().compat()) } + /// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-gethistory + /// Requests balances in a batch and returns them in the same order they were requested. pub fn scripthash_get_balances(&self, hashes: I) -> RpcRes> where I: IntoIterator, @@ -1991,6 +2006,8 @@ impl UtxoRpcClientOps for ElectrumClient { .map(move |results| { results .into_iter() + // `scripthash_get_balances` returns `ElectrumBalance` elements in the same order in which they were requested. + // So we can zip `addresses` and the balances into one iterator. .zip(addresses) .map(|(result, address)| { let balance_sat = BigInt::from(result.confirmed) + BigInt::from(result.unconfirmed); diff --git a/mm2src/coins/utxo/tx_cache.rs b/mm2src/coins/utxo/tx_cache.rs index 194fd10588..a709c618a7 100644 --- a/mm2src/coins/utxo/tx_cache.rs +++ b/mm2src/coins/utxo/tx_cache.rs @@ -51,7 +51,7 @@ where futures::future::join_all(it).await.into_iter().collect() } -/// Upload transactions to cache concurrently. +/// Uploads transactions to cache concurrently. /// Note: this function locks the `TX_CACHE_LOCK` mutex and takes `txs` as the Hash map /// to avoid reading and writing the same files at the same time. pub async fn cache_transactions_concurrently(tx_cache_path: &Path, txs: &HashMap) { @@ -71,7 +71,7 @@ async fn load_transaction_from_cache(tx_cache_path: &Path, txid: H256Json) -> Tx read_json(&path).await.mm_err(TxCacheError::from) } -/// Upload transaction to cache. +/// Uploads transaction to cache. async fn cache_transaction(tx_cache_path: &Path, tx: &RpcTransaction) -> TxCacheResult<()> { const USE_TMP_FILE: bool = true; diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 85ece47613..873bc40df1 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -383,6 +383,7 @@ pub async fn load_hd_accounts_from_storage( } } +/// Requests balance of the given `address`. pub async fn address_balance(coin: &T, address: &Address) -> BalanceResult where T: UtxoCommonOps + MarketCoinOps, @@ -409,6 +410,8 @@ where }) } +/// Requests balances of the given `addresses`. +/// The pairs `(Address, CoinBalance)` are guaranteed to be in the same order in which they were requested. pub async fn addresses_balances(coin: &T, addresses: Vec
) -> BalanceResult> where T: UtxoCommonOps + MarketCoinOps, @@ -2756,7 +2759,7 @@ pub async fn get_mature_unspent_ordered_map( addresses: Vec
, ) -> UtxoRpcResult<(MatureUnspentMap, RecentlySpentOutPointsGuard<'_>)> { let (unspents_map, recently_spent) = coin.get_all_unspent_ordered_map(addresses).await?; - // Get an iterator of futures: `Iterator>` + // Get an iterator of futures: `Future>` let fut_it = unspents_map.into_iter().map(|(address, unspents)| { identify_mature_unspents(coin, unspents).map(|res| -> UtxoRpcResult<(Address, MatureUnspentList)> { let mature_unspents = res?; @@ -2768,8 +2771,7 @@ pub async fn get_mature_unspent_ordered_map( Ok((result_map, recently_spent)) } -/// Separates the given `unspents` outputs into mature and immature. -/// Identifies mature and immature unspent outputs. +/// Splits the given `unspents` outputs into mature and immature. pub async fn identify_mature_unspents(coin: &T, unspents: Vec) -> UtxoRpcResult where T: UtxoCommonOps, @@ -2777,7 +2779,7 @@ where /// Returns `true` if the given transaction has a known non-zero height. fn can_tx_be_cached(tx: &RpcTransaction) -> bool { tx.height > Some(0) } - /// Calculates an actual confirmations number of the given `tx` transaction loaded from cache. + /// Calculates actual confirmations number of the given `tx` transaction loaded from cache. fn calc_actual_cached_tx_confirmations(tx: &RpcTransaction, block_count: u64) -> UtxoRpcResult { let tx_height = tx.height.or_mm_err(|| { UtxoRpcError::Internal(format!(r#"Warning, height of cached "{:?}" tx is unknown"#, tx.txid)) @@ -2868,7 +2870,7 @@ pub fn is_unspent_mature(mature_confirmations: u32, output: &RpcTransaction) -> } /// [`UtxoCommonOps::get_verbose_transactions_from_cache_or_rpc`] implementation. -/// Loads a verbose `RpcTransaction` transaction from cache or requests it using RPC client. +/// Loads verbose transactions from cache or requests it using RPC client. #[cfg(not(target_arch = "wasm32"))] pub async fn get_verbose_transactions_from_cache_or_rpc( coin: &UtxoCoinFields, @@ -2879,7 +2881,7 @@ pub async fn get_verbose_transactions_from_cache_or_rpc( match coin.tx_cache_directory { Some(ref tx_cache_path) => { - tx_cache::load_transactions_from_cache_concurrently(tx_cache_path, tx_ids.iter().cloned()) + tx_cache::load_transactions_from_cache_concurrently(tx_cache_path, tx_ids.into_iter()) .await .into_iter() .for_each(|(txid, res)| match res { @@ -2897,7 +2899,7 @@ pub async fn get_verbose_transactions_from_cache_or_rpc( }); }, // the coin doesn't support TX local cache, don't try to load from cache - None => to_request = tx_ids.iter().copied().collect(), + None => to_request = tx_ids.into_iter().collect(), } result_map.extend( @@ -2912,13 +2914,13 @@ pub async fn get_verbose_transactions_from_cache_or_rpc( } /// [`UtxoCommonOps::get_verbose_transactions_from_cache_or_rpc`] implementation. -/// Loads a verbose `RpcTransaction` transaction from cache or requests it using RPC client. +/// Loads verbose transactions from cache or requests it using RPC client. #[cfg(target_arch = "wasm32")] pub async fn get_verbose_transactions_from_cache_or_rpc( coin: &UtxoCoinFields, tx_ids: HashSet, ) -> UtxoRpcResult> { - let tx_ids: Vec<_> = tx_ids.iter().copied().collect(); + let tx_ids: Vec<_> = tx_ids.into_iter().collect(); let txs = coin .rpc_client .get_verbose_transactions(&tx_ids) @@ -2930,7 +2932,7 @@ pub async fn get_verbose_transactions_from_cache_or_rpc( Ok(txs) } -/// Save the given `txs` transactions in a TX cache. +/// Saves the given `txs` transactions in a TX cache. /// The function takes `txs` as the Hash map to avoid duplicating same transactions. #[cfg(not(target_arch = "wasm32"))] pub async fn cache_transactions_if_possible(coin: &UtxoCoinFields, txs: &HashMap) { @@ -2939,7 +2941,7 @@ pub async fn cache_transactions_if_possible(coin: &UtxoCoinFields, txs: &HashMap } } -/// Save the given `txs` transactions in a TX cache. +/// Saves the given `txs` transactions in a TX cache. /// It takes `txs` as the Hash map to avoid duplicating same transactions. #[cfg(target_arch = "wasm32")] pub async fn cache_transactions_if_possible(_coin: &UtxoCoinFields, _txs: &HashMap) {} @@ -3303,7 +3305,7 @@ pub async fn get_unspent_ordered_map( .map(|(mature_unspents_map, recently_spent)| { let unspents_map = mature_unspents_map .into_iter() - .map(|(address, unspents)| (address, unspents.into_mature())) + .map(|(address, unspents)| (address, unspents.only_mature())) .collect(); (unspents_map, recently_spent) }) @@ -3333,7 +3335,7 @@ pub async fn get_all_unspent_ordered_map( .into_iter() // dedup just in case we add duplicates of same unspent out .unique_by(|unspent| unspent.outpoint.clone()) - .sorted_by(|a, b| { + .sorted_unstable_by(|a, b| { if a.value < b.value { Ordering::Less } else { diff --git a/mm2src/coins/utxo/utxo_standard.rs b/mm2src/coins/utxo/utxo_standard.rs index 440132ec72..3218ffd21e 100644 --- a/mm2src/coins/utxo/utxo_standard.rs +++ b/mm2src/coins/utxo/utxo_standard.rs @@ -106,10 +106,10 @@ impl ListUtxoOps for UtxoStandardCoin { utxo_common::get_unspent_ordered_map(self, addresses).await } - async fn get_all_unspent_ordered_map<'a>( - &'a self, + async fn get_all_unspent_ordered_map( + &self, addresses: Vec
, - ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard)> { + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { utxo_common::get_all_unspent_ordered_map(self, addresses).await } diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index eda244d371..da47269e5a 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -1339,10 +1339,10 @@ impl ListUtxoOps for ZCoin { utxo_common::get_unspent_ordered_map(self, addresses).await } - async fn get_all_unspent_ordered_map<'a>( - &'a self, + async fn get_all_unspent_ordered_map( + &self, addresses: Vec
, - ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard)> { + ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { utxo_common::get_all_unspent_ordered_map(self, addresses).await } diff --git a/mm2src/coins_activation/src/utxo_activation/common_impl.rs b/mm2src/coins_activation/src/utxo_activation/common_impl.rs index 8914d508c0..6cde33d662 100644 --- a/mm2src/coins_activation/src/utxo_activation/common_impl.rs +++ b/mm2src/coins_activation/src/utxo_activation/common_impl.rs @@ -40,7 +40,7 @@ where // Construct an Xpub extractor without checking if the MarketMaker supports HD wallet ops. // [`EnableCoinBalanceOps::enable_coin_balance`] won't just use `xpub_extractor` // if the coin has been initialized with an Iguana priv key. - let xpub_extractor = RpcTaskXPubExtractor::new_unchecked(ctx, task_handle, xpub_extractor_rpc_statuses()).await; + let xpub_extractor = RpcTaskXPubExtractor::new_unchecked(ctx, task_handle, xpub_extractor_rpc_statuses()); task_handle.update_in_progress_status(UtxoStandardInProgressStatus::RequestingWalletBalance)?; let wallet_balance = coin .enable_coin_balance(&xpub_extractor, activation_params.scan_policy) diff --git a/mm2src/common/custom_iter.rs b/mm2src/common/custom_iter.rs index a944def952..d87661e89f 100644 --- a/mm2src/common/custom_iter.rs +++ b/mm2src/common/custom_iter.rs @@ -1,10 +1,10 @@ use std::collections::HashMap; use std::hash::Hash; -pub trait IntoGroupMapResult { +pub trait TryIntoGroupMap { /// An iterator method that unwraps the given `Result<(Key, Value), Err>` items yielded by the input iterator /// and collects `(Key, Value)` tuple pairs into a `HashMap` of keys mapped to `Vec`s of values until an `Err` error is encountered. - fn into_group_map_result(self) -> Result>, Err> + fn try_into_group_map(self) -> Result>, Err> where Self: Iterator> + Sized, K: Hash + Eq, @@ -21,14 +21,14 @@ pub trait IntoGroupMapResult { } } -impl IntoGroupMapResult for T {} +impl TryIntoGroupMap for T {} pub trait TryUnzip where Self: Iterator> + Sized, { /// An iterator method that unwraps the given `Result<(A, B), Err>` items yielded by the input iterator - /// and collects `(A, B)` tuple pairs into the pair of `(FromA, FromB)` containers until an `Err` error is encountered. + /// and collects `(A, B)` tuple pairs into the pair of `(FromA, FromB)` containers until a `E` error is encountered. fn try_unzip(self) -> Result<(FromA, FromB), E> where FromA: Default + Extend, @@ -47,16 +47,16 @@ where impl TryUnzip for T where T: Iterator> {} #[test] -fn test_into_group_map_result() { +fn test_try_into_group_map() { let actual: Result<_, &'static str> = vec![Ok(("foo", 1)), Ok(("bar", 2)), Ok(("foo", 3))] .into_iter() - .into_group_map_result(); + .try_into_group_map(); let expected: HashMap<_, _> = vec![("foo", vec![1, 3]), ("bar", vec![2])].into_iter().collect(); assert_eq!(actual, Ok(expected)); let err = vec![Ok(("foo", 1)), Ok(("bar", 2)), Err("Error"), Ok(("foo", 3))] .into_iter() - .into_group_map_result() + .try_into_group_map() .unwrap_err(); assert_eq!(err, "Error"); } diff --git a/mm2src/common/jsonrpc_client.rs b/mm2src/common/jsonrpc_client.rs index 5d5927edd8..7f60aae032 100644 --- a/mm2src/common/jsonrpc_client.rs +++ b/mm2src/common/jsonrpc_client.rs @@ -102,6 +102,7 @@ pub enum JsonRpcId { Batch(BTreeSet), } +/// Serializable RPC request that is either single or batch. #[derive(Clone, Deserialize, Serialize)] #[serde(untagged)] pub enum JsonRpcRequestEnum { @@ -110,10 +111,12 @@ pub enum JsonRpcRequestEnum { } impl JsonRpcRequestEnum { + /// Creates [`JsonRpcRequestEnum::Batch`] from the given `requests`. pub fn new_batch(requests: Vec) -> JsonRpcRequestEnum { JsonRpcRequestEnum::Batch(JsonRpcBatchRequest(requests)) } + /// Returns a `JsonRpcId` identifier of the request. pub fn rpc_id(&self) -> JsonRpcId { match self { JsonRpcRequestEnum::Single(single) => single.rpc_id(), @@ -131,7 +134,7 @@ impl fmt::Debug for JsonRpcRequestEnum { } } -/// Serializable RPC request +/// Serializable RPC single request. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct JsonRpcRequest { pub jsonrpc: String, @@ -142,8 +145,10 @@ pub struct JsonRpcRequest { } impl JsonRpcRequest { + // Returns [`JsonRpcRequest::id`]. pub fn get_id(&self) -> &str { &self.id } + /// Returns a `JsonRpcId` identifier of the request. pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Single(self.id.clone()) } } @@ -151,19 +156,22 @@ impl From for JsonRpcRequestEnum { fn from(single: JsonRpcRequest) -> Self { JsonRpcRequestEnum::Single(single) } } -/// Serializable RPC request +/// Serializable RPC batch request. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct JsonRpcBatchRequest(Vec); impl JsonRpcBatchRequest { + /// Returns a `JsonRpcId` identifier of the request. pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Batch(self.orig_sequence_ids().collect()) } + /// Returns the number of the requests in the batch. pub fn len(&self) -> usize { self.0.len() } + /// Whether the batch is empty. pub fn is_empty(&self) -> bool { self.0.is_empty() } /// Returns original sequence of identifiers. - /// The method is used to process batch response in the same order in which the requests were sent. + /// The method is used to process batch responses in the same order in which the requests were sent. fn orig_sequence_ids(&self) -> impl Iterator + '_ { self.0.iter().map(|req| req.id.clone()) } } @@ -171,6 +179,7 @@ impl From for JsonRpcRequestEnum { fn from(batch: JsonRpcBatchRequest) -> Self { JsonRpcRequestEnum::Batch(batch) } } +/// Deserializable RPC response that is either single or batch. #[derive(Clone, Debug, Deserialize)] #[serde(untagged)] pub enum JsonRpcResponseEnum { @@ -179,6 +188,7 @@ pub enum JsonRpcResponseEnum { } impl JsonRpcResponseEnum { + /// Returns a `JsonRpcId` identifier of the response. pub fn rpc_id(&self) -> JsonRpcId { match self { JsonRpcResponseEnum::Single(single) => single.rpc_id(), @@ -187,6 +197,7 @@ impl JsonRpcResponseEnum { } } +/// Deserializable RPC single response. #[derive(Clone, Debug, Deserialize)] pub struct JsonRpcResponse { #[serde(default)] @@ -200,17 +211,22 @@ pub struct JsonRpcResponse { } impl JsonRpcResponse { + /// Returns a `JsonRpcId` identifier of the response. pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Single(self.id.clone()) } } +/// Deserializable RPC batch response. #[derive(Clone, Debug, Deserialize)] pub struct JsonRpcBatchResponse(Vec); impl JsonRpcBatchResponse { + /// Returns a `JsonRpcId` identifier of the response. pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Batch(self.0.iter().map(|res| res.id.clone()).collect()) } + /// Returns the number of the requests in the batch. pub fn len(&self) -> usize { self.0.len() } + /// Whether the batch is empty. pub fn is_empty(&self) -> bool { self.0.is_empty() } } @@ -249,19 +265,26 @@ pub enum JsonRpcErrorType { } impl JsonRpcErrorType { - pub fn is_transport(&self) -> bool { matches!(*self, JsonRpcErrorType::Transport(_)) } + /// Whether the error type is [`JsonRpcErrorType::Transport`]. + pub fn is_transport(&self) -> bool { matches!(self, JsonRpcErrorType::Transport(_)) } } pub trait JsonRpcClient { + /// Returns a stringified version of the JSON-RPC protocol. fn version(&self) -> &'static str; + /// Returns a stringified identifier of the next request. fn next_id(&self) -> String; /// Get info that is used in particular to supplement the error info fn client_info(&self) -> String; + /// Sends the given `request` to the remote. + /// Returns either an address `JsonRpcRemoteAddr` of the responder and the `JsonRpcResponseEnum` response, + /// or a stringified error. fn transport(&self, request: JsonRpcRequestEnum) -> JsonRpcResponseFut; + /// Sends the given single `request` to the remote and tries to decode the response into `T`. fn send_request(&self, request: JsonRpcRequest) -> RpcRes { let client_info = self.client_info(); Box::new( @@ -270,6 +293,7 @@ pub trait JsonRpcClient { ) } + /// Sends the given batch `request` to the remote and tries to decode the batch response into `Vec`. /// Responses are guaranteed to be in the same order in which they were requested. fn send_batch_request(&self, request: JsonRpcBatchRequest) -> RpcRes> { try_fu!(self.validate_batch_request(&request)); @@ -295,8 +319,12 @@ pub trait JsonRpcClient { /// The trait is used when the rpc client instance has more than one remote endpoints. pub trait JsonRpcMultiClient: JsonRpcClient { + /// Sends the given `request` to the specified `to_addr` remote. + /// Returns either an address `JsonRpcRemoteAddr` of the responder and the `JsonRpcResponseEnum` response, + /// or a stringified error. fn transport_exact(&self, to_addr: String, request: JsonRpcRequestEnum) -> JsonRpcResponseFut; + /// Sends the given single `request` to the specified `to_addr` remote and tries to decode the response into `T`. fn send_request_to( &self, to_addr: &str, @@ -310,6 +338,8 @@ pub trait JsonRpcMultiClient: JsonRpcClient { } } +/// Checks if the given `result` is success and contains `JsonRpcResponse`. +/// Tries to decode the batch response into `T`. fn process_transport_single_result( result: Result<(JsonRpcRemoteAddr, JsonRpcResponseEnum), String>, client_info: String, @@ -337,6 +367,8 @@ fn process_transport_single_result( } } +/// Checks if the given `result` is success and contains `JsonRpcBatchResponse`. +/// Tries to decode the batch response into `Vec` in the same order in which they were requested. fn process_transport_batch_result( result: Result<(JsonRpcRemoteAddr, JsonRpcResponseEnum), String>, client_info: String, @@ -404,6 +436,8 @@ fn process_transport_batch_result( Ok(result) } +/// Tries to decode the given single `response` into `T` if it doesn't contain an error, +/// otherwise returns `JsonRpcError`. fn process_single_response( client_info: String, remote_addr: JsonRpcRemoteAddr, diff --git a/mm2src/crypto/Cargo.toml b/mm2src/crypto/Cargo.toml index 95f1103be0..eb3a682812 100644 --- a/mm2src/crypto/Cargo.toml +++ b/mm2src/crypto/Cargo.toml @@ -6,7 +6,6 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -async-std = { version = "1.5", features = ["unstable"] } async-trait = "0.1" bip32 = { version = "0.2.2", default-features = false, features = ["alloc", "secp256k1-ffi"] } bitcrypto = { path = "../mm2_bitcoin/crypto" } From f7f2dc27ed59e5d52905ab3d92bc93066f1f3576 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Mon, 25 Apr 2022 10:40:08 +0300 Subject: [PATCH 10/25] Fix PR issues * Do not inherit `UtxoCommonOps` from `ListUtxoOps` (due to `ZCoin` support) * Refactor some of `UtxoRpcClientOps`, `utxo_common` methods by getting rid of nesting * Add the `common::custom_iter::CollectInto` trait * Add `ListUtxoOps::get_mature_unspent_ordered_list` --- mm2src/coins/qrc20.rs | 11 +- mm2src/coins/utxo.rs | 54 +++++-- mm2src/coins/utxo/bch.rs | 19 +-- mm2src/coins/utxo/qtum.rs | 7 - mm2src/coins/utxo/rpc_clients.rs | 146 +++++++++--------- .../utxo/utxo_builder/utxo_arc_builder.rs | 8 +- mm2src/coins/utxo/utxo_common.rs | 120 ++++++++------ mm2src/coins/utxo/utxo_standard.rs | 7 - mm2src/coins/utxo/utxo_withdraw.rs | 8 +- mm2src/coins/z_coin.rs | 12 +- mm2src/common/custom_iter.rs | 30 ++++ 11 files changed, 234 insertions(+), 188 deletions(-) diff --git a/mm2src/coins/qrc20.rs b/mm2src/coins/qrc20.rs index 2cbef85478..a9bd7929e0 100644 --- a/mm2src/coins/qrc20.rs +++ b/mm2src/coins/qrc20.rs @@ -2,8 +2,8 @@ use crate::eth::{self, u256_to_big_decimal, wei_from_big_decimal, TryToAddress}; use crate::qrc20::rpc_clients::{LogEntry, Qrc20ElectrumOps, Qrc20NativeOps, Qrc20RpcOps, TopicFilter, TxReceipt, ViewContractCallType}; use crate::utxo::qtum::QtumBasedCoin; -use crate::utxo::rpc_clients::{ElectrumClient, NativeClient, UnspentInfo, UnspentMap, UtxoRpcClientEnum, - UtxoRpcClientOps, UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; +use crate::utxo::rpc_clients::{ElectrumClient, NativeClient, UnspentMap, UtxoRpcClientEnum, UtxoRpcClientOps, + UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuildResult, UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPrivKeyBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::{self, big_decimal_from_sat, check_all_inputs_signed_by_pub, UtxoTxBuilder}; @@ -573,13 +573,6 @@ impl UtxoTxGenerationOps for Qrc20Coin { #[async_trait] #[cfg_attr(test, mockable)] impl ListUtxoOps for Qrc20Coin { - async fn get_unspent_ordered_list( - &self, - address: &Address, - ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { - utxo_common::get_unspent_ordered_list(self, address).await - } - async fn get_unspent_ordered_map( &self, addresses: Vec
, diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 11a3be3464..b716614cf3 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -776,7 +776,7 @@ impl MatureUnspentList { #[async_trait] #[cfg_attr(test, mockable)] pub trait UtxoCommonOps: - AsRef + UtxoTxGenerationOps + UtxoTxBroadcastOps + ListUtxoOps + Clone + Send + Sync + 'static + AsRef + UtxoTxGenerationOps + UtxoTxBroadcastOps + Clone + Send + Sync + 'static { async fn get_htlc_spend_fee(&self, tx_size: u64) -> UtxoRpcResult; @@ -860,13 +860,18 @@ pub trait UtxoCommonOps: #[async_trait] #[cfg_attr(test, mockable)] pub trait ListUtxoOps { - /// Returns available unspents in ascending order + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). + /// Returns available unspents in ascending order + /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). /// The function uses either [`ListUtxoOps::get_all_unspent_ordered_map`] or [`ListUtxoOps::get_mature_unspent_ordered_map`] /// depending on the coin configuration. async fn get_unspent_ordered_list( &self, address: &Address, - ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)>; + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + let (unspent_map, recently_spent) = self.get_unspent_ordered_map(vec![address.clone()]).await?; + let unspent_list = try_get_unspent_list(address, unspent_map, "get_unspent_ordered_map")?; + Ok((unspent_list, recently_spent)) + } /// Returns available unspents in ascending order for every given `addresses` /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). @@ -883,19 +888,35 @@ pub trait ListUtxoOps { /// # Important /// /// The function doesn't check if the unspents are mature or immature. - /// Consider using [`ListUtxoOps::get_unspent_ordered_list`] or [`ListUtxoOps::get_unspent_ordered_map`] instead. + /// Consider using [`ListUtxoOps::get_unspent_ordered_map`] instead. async fn get_all_unspent_ordered_map( &self, addresses: Vec
, ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)>; + /// Returns available mature and immature unspents in ascending order + /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). + /// + /// # Important + /// + /// The function may request extra data using RPC to check each unspent output whether it's mature or not. + /// It may be overhead in some cases, so consider using [`ListUtxoOps::get_unspent_ordered_list`] instead. + async fn get_mature_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(MatureUnspentList, RecentlySpentOutPointsGuard<'_>)> { + let (unspent_map, recently_spent) = self.get_mature_unspent_ordered_map(vec![address.clone()]).await?; + let unspent_list = try_get_unspent_list(address, unspent_map, "get_mature_unspent_ordered_map")?; + Ok((unspent_list, recently_spent)) + } + /// Returns available mature and immature unspents in ascending order for every given `addresses` /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). /// /// # Important /// - /// The function may request an extra data using RPC to check each unspent output whether it's mature or not. - /// It may be overhead in some cases, so consider using [`ListUtxoOps::get_unspent_ordered_list`] or [`ListUtxoOps::get_unspent_ordered_map`] instead. + /// The function may request extra data using RPC to check each unspent output whether it's mature or not. + /// It may be overhead in some cases, so consider using [`ListUtxoOps::get_unspent_ordered_map`] instead. async fn get_mature_unspent_ordered_map( &self, addresses: Vec
, @@ -1148,9 +1169,9 @@ impl RpcTransportEventHandler for ElectrumProtoVerifier { #[derive(Clone, Debug, Deserialize, Serialize)] pub struct UtxoMergeParams { pub merge_at: usize, - #[serde(default = "ten_f64")] + #[serde(default = "common::ten_f64")] pub check_every: f64, - #[serde(default = "one_hundred")] + #[serde(default = "common::one_hundred")] pub max_merge_at_once: usize, } @@ -1530,7 +1551,7 @@ pub fn sat_from_big_decimal(amount: &BigDecimal, decimals: u8) -> NumConversResu async fn send_outputs_from_my_address_impl(coin: T, outputs: Vec) -> Result where - T: UtxoCommonOps, + T: UtxoCommonOps + ListUtxoOps, { let my_address = try_s!(coin.as_ref().derivation_method.iguana_or_err()); let (unspents, recently_sent_txs) = try_s!(coin.get_unspent_ordered_list(my_address).await); @@ -1639,6 +1660,17 @@ pub fn address_by_conf_and_pubkey_str( address.display_address() } +pub(crate) fn try_get_unspent_list( + address: &Address, + mut unspent_map: HashMap, + unspent_map_returned_by: &str, +) -> UtxoRpcResult { + unspent_map.remove(address).or_mm_err(|| { + let error = format!("{:?} should have returned '{}'", unspent_map_returned_by, address); + UtxoRpcError::Internal(error) + }) +} + fn parse_hex_encoded_u32(hex_encoded: &str) -> Result> { let hex_encoded = hex_encoded.strip_prefix("0x").unwrap_or(hex_encoded); let bytes = hex::decode(hex_encoded).map_to_mm(|e| e.to_string())?; @@ -1649,10 +1681,6 @@ fn parse_hex_encoded_u32(hex_encoded: &str) -> Result> { Ok(u32::from_be_bytes(be_bytes)) } -fn ten_f64() -> f64 { 10. } - -fn one_hundred() -> usize { 100 } - #[test] fn test_parse_hex_encoded_u32() { assert_eq!(parse_hex_encoded_u32("0x892f2085"), Ok(2301567109)); diff --git a/mm2src/coins/utxo/bch.rs b/mm2src/coins/utxo/bch.rs index 02a5bdf2db..58ddf035fb 100644 --- a/mm2src/coins/utxo/bch.rs +++ b/mm2src/coins/utxo/bch.rs @@ -20,7 +20,7 @@ use serde_json::{self as json, Value as Json}; use serialization::{deserialize, CoinVariant}; use std::sync::MutexGuard; -pub type BchUnspentsMap = HashMap; +pub type BchUnspentMap = HashMap; #[derive(Clone, Debug, Deserialize, Serialize)] pub struct BchActivationRequest { @@ -314,17 +314,16 @@ impl BchCoin { &self, address: &Address, ) -> UtxoRpcResult<(BchUnspents, RecentlySpentOutPointsGuard<'_>)> { - let (all_unspents, recently_spent) = utxo_common::get_unspent_ordered_list(self, address).await?; - let result = self.utxos_into_bch_unspents(all_unspents).await?; - - Ok((result, recently_spent)) + let (bch_unspent_map, recently_spent) = self.bch_unspents_map_for_spend(vec![address.clone()]).await?; + let bch_unspents = try_get_unspent_list(address, bch_unspent_map, "bch_unspents_map_for_spend")?; + Ok((bch_unspents, recently_spent)) } /// Locks recently spent cache to safely return UTXOs for spending pub async fn bch_unspents_map_for_spend( &self, addresses: Vec
, - ) -> UtxoRpcResult<(BchUnspentsMap, RecentlySpentOutPointsGuard<'_>)> { + ) -> UtxoRpcResult<(BchUnspentMap, RecentlySpentOutPointsGuard<'_>)> { let (all_unspents, recently_spent) = utxo_common::get_unspent_ordered_map(self, addresses).await?; // Get an iterator of futures: `Future>` let fut_it = all_unspents.into_iter().map(|(address, unspents)| { @@ -736,14 +735,6 @@ impl UtxoTxGenerationOps for BchCoin { #[async_trait] #[cfg_attr(test, mockable)] impl ListUtxoOps for BchCoin { - async fn get_unspent_ordered_list( - &self, - address: &Address, - ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { - let (bch_unspents, recently_spent) = self.bch_unspents_for_spend(address).await?; - Ok((bch_unspents.standard, recently_spent)) - } - async fn get_unspent_ordered_map( &self, addresses: Vec
, diff --git a/mm2src/coins/utxo/qtum.rs b/mm2src/coins/utxo/qtum.rs index 2613769499..35d1a864f7 100644 --- a/mm2src/coins/utxo/qtum.rs +++ b/mm2src/coins/utxo/qtum.rs @@ -320,13 +320,6 @@ impl UtxoTxGenerationOps for QtumCoin { #[async_trait] #[cfg_attr(test, mockable)] impl ListUtxoOps for QtumCoin { - async fn get_unspent_ordered_list( - &self, - address: &Address, - ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { - utxo_common::get_unspent_ordered_list(self, address).await - } - async fn get_unspent_ordered_map( &self, addresses: Vec
, diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index ab891554d3..4c2c6d6cdd 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -7,7 +7,7 @@ use async_trait::async_trait; use bigdecimal::BigDecimal; use chain::{BlockHeader, BlockHeaderBits, BlockHeaderNonce, OutPoint, Transaction as UtxoTx}; use common::custom_futures::{select_ok_sequential, FutureTimerExt}; -use common::custom_iter::TryIntoGroupMap; +use common::custom_iter::{CollectInto, TryIntoGroupMap}; use common::executor::{spawn, Timer}; use common::jsonrpc_client::{BatchRequest, JsonRpcBatchResponse, JsonRpcClient, JsonRpcError, JsonRpcErrorType, JsonRpcId, JsonRpcMultiClient, JsonRpcRemoteAddr, JsonRpcRequest, JsonRpcRequestEnum, @@ -747,26 +747,31 @@ impl UtxoRpcClientOps for NativeClient { } fn display_balances(&self, addresses: Vec
, decimals: u8) -> UtxoRpcFut> { - Box::new( - self.list_unspent_group(addresses.clone(), decimals) - .map(move |mut unspent_map| { - addresses - .into_iter() - .map(|address| { - // If `balances` doesn't contain `address`, there are no unspents related to the address. - // Consider the balance of that address equal to 0. - let balance = unspent_map - .remove(&address) - .unwrap_or_default() - .into_iter() - .fold(BigDecimal::from(0), |sum, unspent| { - sum + big_decimal_from_sat_unsigned(unspent.value, decimals) - }); - (address, balance) - }) - .collect() - }), - ) + fn address_balance_from_unspent_map(address: &Address, unspent_map: &UnspentMap, decimals: u8) -> BigDecimal { + let unspents = match unspent_map.get(address) { + Some(unspents) => unspents, + // If `balances` doesn't contain `address`, there are no unspents related to the address. + // Consider the balance of that address equal to 0. + None => return BigDecimal::from(0), + }; + unspents.iter().fold(BigDecimal::from(0), |sum, unspent| { + sum + big_decimal_from_sat_unsigned(unspent.value, decimals) + }) + } + + let this = self.clone(); + let fut = async move { + let unspent_map = this.list_unspent_group(addresses.clone(), decimals).compat().await?; + let balances = addresses + .into_iter() + .map(|address| { + let balance = address_balance_from_unspent_map(&address, &unspent_map, decimals); + (address, balance) + }) + .collect(); + Ok(balances) + }; + Box::new(fut.boxed().compat()) } fn estimate_fee_sat( @@ -1271,6 +1276,13 @@ pub struct ElectrumBalance { pub(crate) unconfirmed: i128, } +impl ElectrumBalance { + pub fn to_big_decimal(&self, decimals: u8) -> BigDecimal { + let balance_sat = BigInt::from(self.confirmed) + BigInt::from(self.unconfirmed); + BigDecimal::from(balance_sat) / BigDecimal::from(10u64.pow(decimals as u32)) + } +} + fn sha_256(input: &[u8]) -> Vec { let mut sha = Sha256::new(); sha.input(input); @@ -1904,33 +1916,22 @@ impl UtxoRpcClientOps for ElectrumClient { hex::encode(script_hash) }) .collect(); - let fut = self - .scripthash_list_unspent_batch(script_hashes) - .map_to_mm_fut(UtxoRpcError::from) - .map(move |unspents| { - addresses - .into_iter() - // `scripthash_list_unspent_batch` returns `ScriptHashUnspents` elements in the same order in which they were requested. - // So we can zip `addresses` and `unspents` into one iterator. - .zip(unspents) - // Map `(Address, Vec)` pairs into `(Address, Vec)`. - .map(|(address, electrum_unspents)| { - let unspents: Vec<_> = electrum_unspents - .into_iter() - .map(|electrum_unspent| UnspentInfo { - outpoint: OutPoint { - hash: electrum_unspent.tx_hash.reversed().into(), - index: electrum_unspent.tx_pos, - }, - value: electrum_unspent.value, - height: electrum_unspent.height, - }) - .collect(); - (address, unspents) - }) - .collect() - }); - Box::new(fut) + + let this = self.clone(); + let fut = async move { + let unspents = this.scripthash_list_unspent_batch(script_hashes).compat().await?; + + let unspent_map = addresses + .into_iter() + // `scripthash_list_unspent_batch` returns `ScriptHashUnspents` elements in the same order in which they were requested. + // So we can zip `addresses` and `unspents` into one iterator. + .zip(unspents) + // Map `(Address, Vec)` pairs into `(Address, Vec)`. + .map(|(address, electrum_unspents)| (address, electrum_unspents.collect_into())) + .collect(); + Ok(unspent_map) + }; + Box::new(fut.boxed().compat()) } fn send_transaction(&self, tx: &UtxoTx) -> UtxoRpcFut { @@ -1990,35 +1991,32 @@ impl UtxoRpcClientOps for ElectrumClient { fn display_balance(&self, address: Address, decimals: u8) -> RpcRes { let hash = electrum_script_hash(&output_script(&address, ScriptType::P2PKH)); let hash_str = hex::encode(hash); - Box::new(self.scripthash_get_balance(&hash_str).map(move |result| { - let balance_sat = BigInt::from(result.confirmed) + BigInt::from(result.unconfirmed); - BigDecimal::from(balance_sat) / BigDecimal::from(10u64.pow(decimals as u32)) - })) + Box::new( + self.scripthash_get_balance(&hash_str) + .map(move |electrum_balance| electrum_balance.to_big_decimal(decimals)), + ) } fn display_balances(&self, addresses: Vec
, decimals: u8) -> UtxoRpcFut> { - let hashes = addresses.iter().map(|address| { - let hash = electrum_script_hash(&output_script(address, ScriptType::P2PKH)); - hex::encode(hash) - }); - Box::new( - self.scripthash_get_balances(hashes) - .map(move |results| { - results - .into_iter() - // `scripthash_get_balances` returns `ElectrumBalance` elements in the same order in which they were requested. - // So we can zip `addresses` and the balances into one iterator. - .zip(addresses) - .map(|(result, address)| { - let balance_sat = BigInt::from(result.confirmed) + BigInt::from(result.unconfirmed); - let balance_dec = - BigDecimal::from(balance_sat) / BigDecimal::from(10u64.pow(decimals as u32)); - (address, balance_dec) - }) - .collect() - }) - .map_to_mm_fut(UtxoRpcError::from), - ) + let this = self.clone(); + let fut = async move { + let hashes = addresses.iter().map(|address| { + let hash = electrum_script_hash(&output_script(address, ScriptType::P2PKH)); + hex::encode(hash) + }); + + let electrum_balances = this.scripthash_get_balances(hashes).compat().await?; + let balances = electrum_balances + .into_iter() + // `scripthash_get_balances` returns `ElectrumBalance` elements in the same order in which they were requested. + // So we can zip `addresses` and the balances into one iterator. + .zip(addresses) + .map(|(electrum_balance, address)| (address, electrum_balance.to_big_decimal(decimals))) + .collect(); + Ok(balances) + }; + + Box::new(fut.boxed().compat()) } fn estimate_fee_sat( diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index 9e70474b98..4ac1a1eddd 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -2,7 +2,7 @@ use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuilder, UtxoCoinBuilderCommonOps, UtxoFieldsWithHardwareWalletBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::{block_header_utxo_loop, merge_utxo_loop}; -use crate::utxo::{UtxoArc, UtxoCommonOps, UtxoWeak}; +use crate::utxo::{ListUtxoOps, UtxoArc, UtxoCommonOps, UtxoWeak}; use crate::{PrivKeyBuildPolicy, UtxoActivationParams}; use async_trait::async_trait; use common::executor::spawn; @@ -75,7 +75,7 @@ impl<'a, F, T> UtxoFieldsWithHardwareWalletBuilder for UtxoArcBuilder<'a, F, T> impl<'a, F, T> UtxoCoinBuilder for UtxoArcBuilder<'a, F, T> where F: Fn(UtxoArc) -> T + Clone + Send + Sync + 'static, - T: UtxoCommonOps, + T: UtxoCommonOps + ListUtxoOps, { type ResultCoin = T; type Error = UtxoCoinBuildError; @@ -103,7 +103,7 @@ where impl<'a, F, T> MergeUtxoArcOps for UtxoArcBuilder<'a, F, T> where F: Fn(UtxoArc) -> T + Send + Sync + 'static, - T: UtxoCommonOps, + T: UtxoCommonOps + ListUtxoOps, { } @@ -114,7 +114,7 @@ where { } -pub trait MergeUtxoArcOps: UtxoCoinBuilderCommonOps { +pub trait MergeUtxoArcOps: UtxoCoinBuilderCommonOps { fn spawn_merge_utxo_loop_if_required(&self, weak: UtxoWeak, constructor: F) where F: Fn(UtxoArc) -> T + Send + Sync + 'static, diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 873bc40df1..ca3f6366cf 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -7,6 +7,7 @@ use crate::hd_wallet_storage::{HDWalletCoinWithStorageOps, HDWalletStorageResult use crate::rpc_command::init_withdraw::WithdrawTaskHandle; use crate::utxo::rpc_clients::{electrum_script_hash, BlockHashOrHeight, UnspentInfo, UnspentMap, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcResult}; +use crate::utxo::tx_cache::TxCacheResult; use crate::utxo::utxo_withdraw::{InitUtxoWithdraw, StandardUtxoWithdraw, UtxoWithdraw}; use crate::{CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, GetWithdrawSenderAddress, HDAddressId, RawTransactionError, RawTransactionRequest, RawTransactionRes, TradePreimageValue, TxFeeDetails, @@ -386,14 +387,10 @@ pub async fn load_hd_accounts_from_storage( /// Requests balance of the given `address`. pub async fn address_balance(coin: &T, address: &Address) -> BalanceResult where - T: UtxoCommonOps + MarketCoinOps, + T: UtxoCommonOps + ListUtxoOps + MarketCoinOps, { if coin.as_ref().check_utxo_maturity { - let (mut unspents_map, _) = coin.get_mature_unspent_ordered_map(vec![address.clone()]).await?; - let unspents = unspents_map.remove(address).or_mm_err(|| { - let error = format!("'get_mature_unspent_ordered_map' should have returned '{}'", address); - BalanceError::Internal(error) - })?; + let (unspents, _) = coin.get_mature_unspent_ordered_list(address).await?; return Ok(unspents.to_coin_balance(coin.as_ref().decimals)); } @@ -414,7 +411,7 @@ where /// The pairs `(Address, CoinBalance)` are guaranteed to be in the same order in which they were requested. pub async fn addresses_balances(coin: &T, addresses: Vec
) -> BalanceResult> where - T: UtxoCommonOps + MarketCoinOps, + T: UtxoCommonOps + ListUtxoOps + MarketCoinOps, { if coin.as_ref().check_utxo_maturity { let (unspents_map, _) = coin.get_mature_unspent_ordered_map(addresses.clone()).await?; @@ -588,7 +585,10 @@ pub async fn get_current_mtp(coin: &UtxoCoinFields, coin_variant: CoinVariant) - .await } -pub fn send_outputs_from_my_address(coin: T, outputs: Vec) -> TransactionFut { +pub fn send_outputs_from_my_address(coin: T, outputs: Vec) -> TransactionFut +where + T: UtxoCommonOps + ListUtxoOps, +{ let fut = send_outputs_from_my_address_impl(coin, outputs); Box::new(fut.boxed().compat().map(|tx| tx.into())) } @@ -1032,7 +1032,10 @@ pub async fn p2sh_spending_tx( }) } -pub fn send_taker_fee(coin: T, fee_pub_key: &[u8], amount: BigDecimal) -> TransactionFut { +pub fn send_taker_fee(coin: T, fee_pub_key: &[u8], amount: BigDecimal) -> TransactionFut +where + T: UtxoCommonOps + ListUtxoOps, +{ let address = try_fus!(address_from_raw_pubkey( fee_pub_key, coin.as_ref().conf.pub_addr_prefix, @@ -1049,14 +1052,17 @@ pub fn send_taker_fee(coin: T, fee_pub_key: &[u8], amount: Big send_outputs_from_my_address(coin, vec![output]) } -pub fn send_maker_payment( +pub fn send_maker_payment( coin: T, time_lock: u32, maker_pub: &[u8], taker_pub: &[u8], secret_hash: &[u8], amount: BigDecimal, -) -> TransactionFut { +) -> TransactionFut +where + T: UtxoCommonOps + ListUtxoOps, +{ let SwapPaymentOutputsResult { payment_address, outputs, @@ -1083,14 +1089,17 @@ pub fn send_maker_payment( Box::new(send_fut) } -pub fn send_taker_payment( +pub fn send_taker_payment( coin: T, time_lock: u32, taker_pub: &[u8], maker_pub: &[u8], secret_hash: &[u8], amount: BigDecimal, -) -> TransactionFut { +) -> TransactionFut +where + T: UtxoCommonOps + ListUtxoOps, +{ let SwapPaymentOutputsResult { payment_address, outputs, @@ -1666,7 +1675,7 @@ pub fn my_address(coin: &T) -> Result { pub fn my_balance(coin: T) -> BalanceFut where - T: UtxoCommonOps + MarketCoinOps, + T: UtxoCommonOps + ListUtxoOps + MarketCoinOps, { let my_address = try_f!(coin .as_ref() @@ -1800,7 +1809,7 @@ pub async fn get_raw_transaction(coin: &UtxoCoinFields, req: RawTransactionReque pub async fn withdraw(coin: T, req: WithdrawRequest) -> WithdrawResult where - T: UtxoCommonOps + MarketCoinOps, + T: UtxoCommonOps + ListUtxoOps + MarketCoinOps, { StandardUtxoWithdraw::new(coin, req)?.build().await } @@ -1813,6 +1822,7 @@ pub async fn init_withdraw( ) -> WithdrawResult where T: UtxoCommonOps + + ListUtxoOps + UtxoSignerOps + CoinWithDerivationMethod + GetWithdrawSenderAddress
, @@ -2559,13 +2569,16 @@ pub fn get_trade_fee(coin: T) -> Box get_sender_trade_fee(TradePreimageValue::Exact(10000))`. /// So we should always return a fee as if a transaction includes the change output. -pub async fn preimage_trade_fee_required_to_send_outputs( +pub async fn preimage_trade_fee_required_to_send_outputs( coin: &T, outputs: Vec, fee_policy: FeePolicy, gas_fee: Option, stage: &FeeApproxStage, -) -> TradePreimageResult { +) -> TradePreimageResult +where + T: UtxoCommonOps + ListUtxoOps, +{ let ticker = coin.as_ref().conf.ticker.clone(); let decimals = coin.as_ref().decimals; let tx_fee = coin.get_tx_fee().await?; @@ -2754,10 +2767,13 @@ pub fn is_coin_protocol_supported(coin: &T, info: &Option( +pub async fn get_mature_unspent_ordered_map( coin: &T, addresses: Vec
, -) -> UtxoRpcResult<(MatureUnspentMap, RecentlySpentOutPointsGuard<'_>)> { +) -> UtxoRpcResult<(MatureUnspentMap, RecentlySpentOutPointsGuard<'_>)> +where + T: UtxoCommonOps + ListUtxoOps, +{ let (unspents_map, recently_spent) = coin.get_all_unspent_ordered_map(addresses).await?; // Get an iterator of futures: `Future>` let fut_it = unspents_map.into_iter().map(|(address, unspents)| { @@ -2876,6 +2892,33 @@ pub async fn get_verbose_transactions_from_cache_or_rpc( coin: &UtxoCoinFields, tx_ids: HashSet, ) -> UtxoRpcResult> { + /// Determines whether the transaction is needed to be requested through RPC or not. + /// Puts the inner `RpcTransaction` transaction into `result_map` if it has been loaded successfully, + /// otherwise puts `txid` into `to_request`. + fn on_cached_transaction_result( + result_map: &mut HashMap, + to_request: &mut Vec, + txid: H256Json, + res: TxCacheResult>, + ) { + match res { + Ok(Some(tx)) => { + result_map.insert(txid, VerboseTransactionFrom::Cache(tx)); + }, + // txid not found + Ok(None) => { + to_request.push(txid); + }, + Err(err) => { + error!( + "Error loading the {:?} transaction: {:?}. Trying to request tx using RPC client", + err, txid + ); + to_request.push(txid); + }, + } + } + let mut result_map = HashMap::with_capacity(tx_ids.len()); let mut to_request = Vec::with_capacity(tx_ids.len()); @@ -2884,19 +2927,7 @@ pub async fn get_verbose_transactions_from_cache_or_rpc( tx_cache::load_transactions_from_cache_concurrently(tx_cache_path, tx_ids.into_iter()) .await .into_iter() - .for_each(|(txid, res)| match res { - Ok(Some(tx)) => { - result_map.insert(txid, VerboseTransactionFrom::Cache(tx)); - }, - // txid not found - Ok(None) => { - to_request.push(txid); - }, - Err(err) => { - log!("Error " [err] " loading the " [txid] " transaction. Trying request tx using Rpc client"); - to_request.push(txid); - }, - }); + .for_each(|(txid, res)| on_cached_transaction_result(&mut result_map, &mut to_request, txid, res)); }, // the coin doesn't support TX local cache, don't try to load from cache None => to_request = tx_ids.into_iter().collect(), @@ -3279,25 +3310,16 @@ pub fn dex_fee_script(uuid: [u8; 16], time_lock: u32, watcher_pub: &Public, send .into_script() } -pub async fn get_unspent_ordered_list<'a, T: ListUtxoOps>( - coin: &'a T, - address: &Address, -) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'a>)> { - let (mut unspent_map, recently_spent) = coin.get_unspent_ordered_map(vec![address.clone()]).await?; - let unspents = unspent_map.remove(address).or_mm_err(|| { - let error = format!("'get_unspent_ordered_map' should have returned '{}'", address); - UtxoRpcError::Internal(error) - })?; - Ok((unspents, recently_spent)) -} - /// [`ListUtxoOps::get_unspent_ordered_map`] implementation. /// Returns available unspents in ascending order + `RecentlySpentOutPoints` MutexGuard for further interaction /// (e.g. to add new transaction to it). -pub async fn get_unspent_ordered_map( +pub async fn get_unspent_ordered_map( coin: &T, addresses: Vec
, -) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { +) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> +where + T: UtxoCommonOps + ListUtxoOps, +{ if coin.as_ref().check_utxo_maturity { coin.get_mature_unspent_ordered_map(addresses) .await @@ -3516,13 +3538,15 @@ pub async fn block_header_utxo_loop(weak: UtxoWeak, constructo } } -pub async fn merge_utxo_loop( +pub async fn merge_utxo_loop( weak: UtxoWeak, merge_at: usize, check_every: f64, max_merge_at_once: usize, constructor: impl Fn(UtxoArc) -> T, -) { +) where + T: UtxoCommonOps + ListUtxoOps, +{ loop { Timer::sleep(check_every).await; diff --git a/mm2src/coins/utxo/utxo_standard.rs b/mm2src/coins/utxo/utxo_standard.rs index 3218ffd21e..370b3ec773 100644 --- a/mm2src/coins/utxo/utxo_standard.rs +++ b/mm2src/coins/utxo/utxo_standard.rs @@ -92,13 +92,6 @@ impl UtxoTxGenerationOps for UtxoStandardCoin { #[async_trait] #[cfg_attr(test, mockable)] impl ListUtxoOps for UtxoStandardCoin { - async fn get_unspent_ordered_list( - &self, - address: &Address, - ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { - utxo_common::get_unspent_ordered_list(self, address).await - } - async fn get_unspent_ordered_map( &self, addresses: Vec
, diff --git a/mm2src/coins/utxo/utxo_withdraw.rs b/mm2src/coins/utxo/utxo_withdraw.rs index 0cc788245c..8987b3693b 100644 --- a/mm2src/coins/utxo/utxo_withdraw.rs +++ b/mm2src/coins/utxo/utxo_withdraw.rs @@ -1,6 +1,6 @@ use crate::rpc_command::init_withdraw::{WithdrawAwaitingStatus, WithdrawInProgressStatus, WithdrawTaskHandle}; use crate::utxo::utxo_common::{big_decimal_from_sat, UtxoTxBuilder}; -use crate::utxo::{output_script, sat_from_big_decimal, ActualTxFee, Address, FeePolicy, PrivKeyPolicy, +use crate::utxo::{output_script, sat_from_big_decimal, ActualTxFee, Address, FeePolicy, ListUtxoOps, PrivKeyPolicy, UtxoAddressFormat, UtxoCoinFields, UtxoCommonOps, UtxoFeeDetails, UtxoTx, UTXO_LOCK}; use crate::{CoinWithDerivationMethod, GetWithdrawSenderAddress, MarketCoinOps, TransactionDetails, WithdrawError, WithdrawFee, WithdrawRequest, WithdrawResult}; @@ -100,7 +100,7 @@ impl From for WithdrawError { pub trait UtxoWithdraw where Self: Sized + Sync, - Coin: UtxoCommonOps, + Coin: UtxoCommonOps + ListUtxoOps, { fn coin(&self) -> &Coin; @@ -246,7 +246,7 @@ pub struct InitUtxoWithdraw<'a, Coin> { #[async_trait] impl<'a, Coin> UtxoWithdraw for InitUtxoWithdraw<'a, Coin> where - Coin: UtxoCommonOps + UtxoSignerOps, + Coin: UtxoCommonOps + ListUtxoOps + UtxoSignerOps, { fn coin(&self) -> &Coin { &self.coin } @@ -404,7 +404,7 @@ pub struct StandardUtxoWithdraw { #[async_trait] impl UtxoWithdraw for StandardUtxoWithdraw where - Coin: UtxoCommonOps, + Coin: UtxoCommonOps + ListUtxoOps, { fn coin(&self) -> &Coin { &self.coin } diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index da47269e5a..a7f2b15a5b 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -1,4 +1,4 @@ -use crate::utxo::rpc_clients::{UnspentInfo, UnspentMap, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcError, UtxoRpcFut, +use crate::utxo::rpc_clients::{UnspentMap, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; use crate::utxo::utxo_builder::{UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPrivKeyBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; @@ -1322,16 +1322,12 @@ impl UtxoTxBroadcastOps for ZCoin { } } +/// Please note `ZCoin` is not assumed to work with transparent UTXOs. +/// Remove implementation of the `ListUtxoOps` trait for `ZCoin` +/// when [`ZCoin::preimage_trade_fee_required_to_send_outputs`] is refactored. #[async_trait] #[cfg_attr(test, mockable)] impl ListUtxoOps for ZCoin { - async fn get_unspent_ordered_list( - &self, - address: &Address, - ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { - utxo_common::get_unspent_ordered_list(self, address).await - } - async fn get_unspent_ordered_map( &self, addresses: Vec
, diff --git a/mm2src/common/custom_iter.rs b/mm2src/common/custom_iter.rs index d87661e89f..0b7f62e203 100644 --- a/mm2src/common/custom_iter.rs +++ b/mm2src/common/custom_iter.rs @@ -1,5 +1,28 @@ use std::collections::HashMap; use std::hash::Hash; +use std::iter::FromIterator; + +pub trait CollectInto { + /// Collects `FromB` from an `IntoIterator` given the fact that `A: Into`. + /// + /// # Usage + /// + /// ```rust + /// let actual: Vec = vec!["foo", "bar"].collect_into(); + /// let expected = vec!["foo".to_owned(), "bar".to_owned()]; + /// assert_eq!(actual, expected); + /// ``` + fn collect_into(self) -> FromB + where + Self: IntoIterator + Sized, + A: Into, + FromB: FromIterator, + { + self.into_iter().map(A::into).collect() + } +} + +impl CollectInto for T {} pub trait TryIntoGroupMap { /// An iterator method that unwraps the given `Result<(Key, Value), Err>` items yielded by the input iterator @@ -46,6 +69,13 @@ where impl TryUnzip for T where T: Iterator> {} +#[test] +fn test_collect_into() { + let actual: Vec = vec!["foo", "bar"].collect_into(); + let expected = vec!["foo".to_owned(), "bar".to_owned()]; + assert_eq!(actual, expected); +} + #[test] fn test_try_into_group_map() { let actual: Result<_, &'static str> = vec![Ok(("foo", 1)), Ok(("bar", 2)), Ok(("foo", 3))] From 987cc1e320b49d2df8cc553a358f2b725205dd1b Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Mon, 25 Apr 2022 11:54:02 +0300 Subject: [PATCH 11/25] Minor change * Reorder RPCs alphabetically --- mm2src/rpc/dispatcher/dispatcher.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm2src/rpc/dispatcher/dispatcher.rs b/mm2src/rpc/dispatcher/dispatcher.rs index f75616de60..c0505ee8d6 100644 --- a/mm2src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/rpc/dispatcher/dispatcher.rs @@ -132,6 +132,8 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, init_standalone_coin::).await, "init_qtum_status" => handle_mmrpc(ctx, request, init_standalone_coin_status::).await, "init_qtum_user_action" => handle_mmrpc(ctx, request, init_standalone_coin_user_action::).await, + "init_scan_for_new_addresses" => handle_mmrpc(ctx, request, init_scan_for_new_addresses).await, + "init_scan_for_new_addresses_status" => handle_mmrpc(ctx, request, init_scan_for_new_addresses_status).await, "init_trezor" => handle_mmrpc(ctx, request, init_trezor).await, "init_trezor_status" => handle_mmrpc(ctx, request, init_trezor_status).await, "init_trezor_user_action" => handle_mmrpc(ctx, request, init_trezor_user_action).await, @@ -145,14 +147,12 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, recreate_swap_data).await, "remove_delegation" => handle_mmrpc(ctx, request, remove_delegation).await, "remove_node_from_version_stat" => handle_mmrpc(ctx, request, remove_node_from_version_stat).await, - "init_scan_for_new_addresses" => handle_mmrpc(ctx, request, init_scan_for_new_addresses).await, - "init_scan_for_new_addresses_status" => handle_mmrpc(ctx, request, init_scan_for_new_addresses_status).await, "start_simple_market_maker_bot" => handle_mmrpc(ctx, request, start_simple_market_maker_bot).await, "start_version_stat_collection" => handle_mmrpc(ctx, request, start_version_stat_collection).await, "stop_simple_market_maker_bot" => handle_mmrpc(ctx, request, stop_simple_market_maker_bot).await, "stop_version_stat_collection" => handle_mmrpc(ctx, request, stop_version_stat_collection).await, - "update_version_stat_collection" => handle_mmrpc(ctx, request, update_version_stat_collection).await, "trade_preimage" => handle_mmrpc(ctx, request, trade_preimage_rpc).await, + "update_version_stat_collection" => handle_mmrpc(ctx, request, update_version_stat_collection).await, "withdraw" => handle_mmrpc(ctx, request, withdraw).await, "withdraw_status" => handle_mmrpc(ctx, request, withdraw_status).await, "withdraw_user_action" => handle_mmrpc(ctx, request, withdraw_user_action).await, From 013e214ac66b4ff96f1aa65bcb55e24c76f155e0 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Tue, 26 Apr 2022 22:45:12 +0300 Subject: [PATCH 12/25] Fix PR issues * Fix `from_id` paging option for `account_balance` RPC * Fix WASM build --- mm2src/coins/qrc20/qrc20_tests.rs | 1 + mm2src/coins/rpc_command/account_balance.rs | 2 +- mm2src/coins/utxo/utxo_common.rs | 3 +-- mm2src/coins/utxo/utxo_tests.rs | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mm2src/coins/qrc20/qrc20_tests.rs b/mm2src/coins/qrc20/qrc20_tests.rs index 29fbaeec8c..3d3ee9269b 100644 --- a/mm2src/coins/qrc20/qrc20_tests.rs +++ b/mm2src/coins/qrc20/qrc20_tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::utxo::rpc_clients::UnspentInfo; use crate::TxFeeDetails; use bigdecimal::Zero; use chain::OutPoint; diff --git a/mm2src/coins/rpc_command/account_balance.rs b/mm2src/coins/rpc_command/account_balance.rs index 1eecd810c0..80b1aa2fe9 100644 --- a/mm2src/coins/rpc_command/account_balance.rs +++ b/mm2src/coins/rpc_command/account_balance.rs @@ -84,7 +84,7 @@ pub mod common_impl { let total_addresses_number = hd_account.known_addresses_number(params.chain)?; let from_address_id = match params.paging_options { - PagingOptionsEnum::FromId(from_address_id) => from_address_id, + PagingOptionsEnum::FromId(from_address_id) => from_address_id + 1, PagingOptionsEnum::PageNumber(page_number) => ((page_number.get() - 1) * params.limit) as u32, }; let to_address_id = std::cmp::min(from_address_id + params.limit as u32, total_addresses_number); diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index ca3f6366cf..ef4bf6e444 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -7,7 +7,6 @@ use crate::hd_wallet_storage::{HDWalletCoinWithStorageOps, HDWalletStorageResult use crate::rpc_command::init_withdraw::WithdrawTaskHandle; use crate::utxo::rpc_clients::{electrum_script_hash, BlockHashOrHeight, UnspentInfo, UnspentMap, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcResult}; -use crate::utxo::tx_cache::TxCacheResult; use crate::utxo::utxo_withdraw::{InitUtxoWithdraw, StandardUtxoWithdraw, UtxoWithdraw}; use crate::{CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, GetWithdrawSenderAddress, HDAddressId, RawTransactionError, RawTransactionRequest, RawTransactionRes, TradePreimageValue, TxFeeDetails, @@ -2899,7 +2898,7 @@ pub async fn get_verbose_transactions_from_cache_or_rpc( result_map: &mut HashMap, to_request: &mut Vec, txid: H256Json, - res: TxCacheResult>, + res: tx_cache::TxCacheResult>, ) { match res { Ok(Some(tx)) => { diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index fa079bafac..c2c1911697 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -3473,13 +3473,13 @@ fn test_account_balance_rpc() { }; assert_eq!(actual, expected); - // Request a balance of Account#0, internal addresses, starting from idx=1 + // Request a balance of Account#0, internal addresses, where idx > 0 let params = AccountBalanceParams { account_index: 0, chain: Bip44Chain::Internal, limit: 3, - paging_options: PagingOptionsEnum::FromId(1), + paging_options: PagingOptionsEnum::FromId(0), }; let actual = block_on(coin.account_balance_rpc(params)).expect("!account_balance_rpc"); let expected = HDAccountBalanceResponse { @@ -3491,7 +3491,7 @@ fn test_account_balance_rpc() { skipped: 1, total: 3, total_pages: 1, - paging_options: PagingOptionsEnum::FromId(1), + paging_options: PagingOptionsEnum::FromId(0), }; assert_eq!(actual, expected); @@ -3539,13 +3539,13 @@ fn test_account_balance_rpc() { }; assert_eq!(actual, expected); - // Request a balance of Account#1, external addresses, starting from idx=1 (out of bound) + // Request a balance of Account#1, external addresses, where idx > 0 (out of bound) let params = AccountBalanceParams { account_index: 1, chain: Bip44Chain::Internal, limit: 3, - paging_options: PagingOptionsEnum::FromId(1), + paging_options: PagingOptionsEnum::FromId(0), }; let actual = block_on(coin.account_balance_rpc(params)).expect("!account_balance_rpc"); let expected = HDAccountBalanceResponse { @@ -3557,7 +3557,7 @@ fn test_account_balance_rpc() { skipped: 1, total: 1, total_pages: 1, - paging_options: PagingOptionsEnum::FromId(1), + paging_options: PagingOptionsEnum::FromId(0), }; assert_eq!(actual, expected); } From abe381d1c2e7ec71acd3393d2a013f783798ea2c Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Thu, 28 Apr 2022 19:42:23 +0300 Subject: [PATCH 13/25] Fix PR issues * Divide `ListUtxoOps` into `GetUtxoListOps` and `GetUtxoMapOps` * Implement `GetUtxoMapOps` for `UtxoStandardCoin` and `QtumCoin` only --- mm2src/coins/lightning.rs | 2 +- mm2src/coins/qrc20.rs | 36 +++--- mm2src/coins/utxo.rs | 73 +++++------ mm2src/coins/utxo/bch.rs | 63 +++------ mm2src/coins/utxo/qtum.rs | 27 +++- mm2src/coins/utxo/qtum_delegation.rs | 2 +- mm2src/coins/utxo/slp.rs | 2 +- .../utxo/utxo_builder/utxo_arc_builder.rs | 8 +- mm2src/coins/utxo/utxo_common.rs | 121 +++++++++++++----- mm2src/coins/utxo/utxo_standard.rs | 27 +++- mm2src/coins/utxo/utxo_tests.rs | 110 +++++++--------- mm2src/coins/utxo/utxo_withdraw.rs | 8 +- mm2src/coins/z_coin.rs | 32 ++--- mm2src/docker_tests.rs | 2 +- 14 files changed, 287 insertions(+), 226 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 2f9e0aed14..a862195182 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -1,7 +1,7 @@ use super::{lp_coinfind_or_err, MmCoinEnum}; use crate::utxo::rpc_clients::UtxoRpcClientEnum; use crate::utxo::utxo_common::{big_decimal_from_sat_unsigned, UtxoTxBuilder}; -use crate::utxo::{sat_from_big_decimal, BlockchainNetwork, FeePolicy, ListUtxoOps, UtxoTxGenerationOps}; +use crate::utxo::{sat_from_big_decimal, BlockchainNetwork, FeePolicy, GetUtxoListOps, UtxoTxGenerationOps}; use crate::{BalanceFut, CoinBalance, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MarketCoinOps, MmCoin, NegotiateSwapContractAddrErr, RawTransactionFut, RawTransactionRequest, SwapOps, TradeFee, TradePreimageFut, TradePreimageResult, TradePreimageValue, TransactionEnum, TransactionFut, diff --git a/mm2src/coins/qrc20.rs b/mm2src/coins/qrc20.rs index a9bd7929e0..6b446984fd 100644 --- a/mm2src/coins/qrc20.rs +++ b/mm2src/coins/qrc20.rs @@ -2,15 +2,15 @@ use crate::eth::{self, u256_to_big_decimal, wei_from_big_decimal, TryToAddress}; use crate::qrc20::rpc_clients::{LogEntry, Qrc20ElectrumOps, Qrc20NativeOps, Qrc20RpcOps, TopicFilter, TxReceipt, ViewContractCallType}; use crate::utxo::qtum::QtumBasedCoin; -use crate::utxo::rpc_clients::{ElectrumClient, NativeClient, UnspentMap, UtxoRpcClientEnum, UtxoRpcClientOps, +use crate::utxo::rpc_clients::{ElectrumClient, NativeClient, UnspentInfo, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuildResult, UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPrivKeyBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::{self, big_decimal_from_sat, check_all_inputs_signed_by_pub, UtxoTxBuilder}; -use crate::utxo::{qtum, ActualTxFee, AdditionalTxData, BroadcastTxErr, FeePolicy, GenerateTxError, HistoryUtxoTx, - HistoryUtxoTxMap, ListUtxoOps, MatureUnspentMap, RecentlySpentOutPointsGuard, UtxoActivationParams, - UtxoAddressFormat, UtxoCoinFields, UtxoCommonOps, UtxoFromLegacyReqErr, UtxoTx, UtxoTxBroadcastOps, - UtxoTxGenerationOps, VerboseTransactionFrom, UTXO_LOCK}; +use crate::utxo::{qtum, ActualTxFee, AdditionalTxData, BroadcastTxErr, FeePolicy, GenerateTxError, GetUtxoListOps, + HistoryUtxoTx, HistoryUtxoTxMap, MatureUnspentList, RecentlySpentOutPointsGuard, + UtxoActivationParams, UtxoAddressFormat, UtxoCoinFields, UtxoCommonOps, UtxoFromLegacyReqErr, + UtxoTx, UtxoTxBroadcastOps, UtxoTxGenerationOps, VerboseTransactionFrom, UTXO_LOCK}; use crate::{BalanceError, BalanceFut, CoinBalance, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MarketCoinOps, MmCoin, NegotiateSwapContractAddrErr, PrivKeyNotAllowed, RawTransactionFut, RawTransactionRequest, SwapOps, TradeFee, TradePreimageError, TradePreimageFut, TradePreimageResult, TradePreimageValue, @@ -572,26 +572,26 @@ impl UtxoTxGenerationOps for Qrc20Coin { #[async_trait] #[cfg_attr(test, mockable)] -impl ListUtxoOps for Qrc20Coin { - async fn get_unspent_ordered_map( +impl GetUtxoListOps for Qrc20Coin { + async fn get_unspent_ordered_list( &self, - addresses: Vec
, - ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { - utxo_common::get_unspent_ordered_map(self, addresses).await + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_unspent_ordered_list(self, address).await } - async fn get_all_unspent_ordered_map( + async fn get_all_unspent_ordered_list( &self, - addresses: Vec
, - ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { - utxo_common::get_all_unspent_ordered_map(self, addresses).await + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_all_unspent_ordered_list(self, address).await } - async fn get_mature_unspent_ordered_map( + async fn get_mature_unspent_ordered_list( &self, - addresses: Vec
, - ) -> UtxoRpcResult<(MatureUnspentMap, RecentlySpentOutPointsGuard<'_>)> { - utxo_common::get_mature_unspent_ordered_map(self, addresses).await + address: &Address, + ) -> UtxoRpcResult<(MatureUnspentList, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_mature_unspent_ordered_list(self, address).await } } diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index b716614cf3..913dbcd3e2 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -859,23 +859,47 @@ pub trait UtxoCommonOps: #[async_trait] #[cfg_attr(test, mockable)] -pub trait ListUtxoOps { +pub trait GetUtxoListOps { /// Returns available unspents in ascending order /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). - /// The function uses either [`ListUtxoOps::get_all_unspent_ordered_map`] or [`ListUtxoOps::get_mature_unspent_ordered_map`] + /// The function uses either [`GetUtxoListOps::get_all_unspent_ordered_list`] or [`GetUtxoListOps::get_mature_unspent_ordered_list`] /// depending on the coin configuration. async fn get_unspent_ordered_list( &self, address: &Address, - ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { - let (unspent_map, recently_spent) = self.get_unspent_ordered_map(vec![address.clone()]).await?; - let unspent_list = try_get_unspent_list(address, unspent_map, "get_unspent_ordered_map")?; - Ok((unspent_list, recently_spent)) - } + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)>; + + /// Returns available unspents in ascending order + /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). + /// + /// # Important + /// + /// The function doesn't check if the unspents are mature or immature. + /// Consider using [`GetUtxoListOps::get_unspent_ordered_list`] instead. + async fn get_all_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)>; + + /// Returns available mature and immature unspents in ascending order + /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). + /// + /// # Important + /// + /// The function may request extra data using RPC to check each unspent output whether it's mature or not. + /// It may be overhead in some cases, so consider using [`GetUtxoListOps::get_unspent_ordered_list`] instead. + async fn get_mature_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(MatureUnspentList, RecentlySpentOutPointsGuard<'_>)>; +} +#[async_trait] +#[cfg_attr(test, mockable)] +pub trait GetUtxoMapOps { /// Returns available unspents in ascending order for every given `addresses` /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). - /// The function uses either [`ListUtxoOps::get_all_unspent_ordered_map`] or [`ListUtxoOps::get_mature_unspent_ordered_map`] + /// The function uses either [`GetUtxoMapOps::get_all_unspent_ordered_map`] or [`GetUtxoMapOps::get_mature_unspent_ordered_map`] /// depending on the coin configuration. async fn get_unspent_ordered_map( &self, @@ -888,35 +912,19 @@ pub trait ListUtxoOps { /// # Important /// /// The function doesn't check if the unspents are mature or immature. - /// Consider using [`ListUtxoOps::get_unspent_ordered_map`] instead. + /// Consider using [`GetUtxoMapOps::get_unspent_ordered_map`] instead. async fn get_all_unspent_ordered_map( &self, addresses: Vec
, ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)>; - /// Returns available mature and immature unspents in ascending order - /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). - /// - /// # Important - /// - /// The function may request extra data using RPC to check each unspent output whether it's mature or not. - /// It may be overhead in some cases, so consider using [`ListUtxoOps::get_unspent_ordered_list`] instead. - async fn get_mature_unspent_ordered_list( - &self, - address: &Address, - ) -> UtxoRpcResult<(MatureUnspentList, RecentlySpentOutPointsGuard<'_>)> { - let (unspent_map, recently_spent) = self.get_mature_unspent_ordered_map(vec![address.clone()]).await?; - let unspent_list = try_get_unspent_list(address, unspent_map, "get_mature_unspent_ordered_map")?; - Ok((unspent_list, recently_spent)) - } - /// Returns available mature and immature unspents in ascending order for every given `addresses` /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). /// /// # Important /// /// The function may request extra data using RPC to check each unspent output whether it's mature or not. - /// It may be overhead in some cases, so consider using [`ListUtxoOps::get_unspent_ordered_map`] instead. + /// It may be overhead in some cases, so consider using [`GetUtxoMapOps::get_unspent_ordered_map`] instead. async fn get_mature_unspent_ordered_map( &self, addresses: Vec
, @@ -1551,7 +1559,7 @@ pub fn sat_from_big_decimal(amount: &BigDecimal, decimals: u8) -> NumConversResu async fn send_outputs_from_my_address_impl(coin: T, outputs: Vec) -> Result where - T: UtxoCommonOps + ListUtxoOps, + T: UtxoCommonOps + GetUtxoListOps, { let my_address = try_s!(coin.as_ref().derivation_method.iguana_or_err()); let (unspents, recently_sent_txs) = try_s!(coin.get_unspent_ordered_list(my_address).await); @@ -1660,17 +1668,6 @@ pub fn address_by_conf_and_pubkey_str( address.display_address() } -pub(crate) fn try_get_unspent_list( - address: &Address, - mut unspent_map: HashMap, - unspent_map_returned_by: &str, -) -> UtxoRpcResult { - unspent_map.remove(address).or_mm_err(|| { - let error = format!("{:?} should have returned '{}'", unspent_map_returned_by, address); - UtxoRpcError::Internal(error) - }) -} - fn parse_hex_encoded_u32(hex_encoded: &str) -> Result> { let hex_encoded = hex_encoded.strip_prefix("0x").unwrap_or(hex_encoded); let bytes = hex::decode(hex_encoded).map_to_mm(|e| e.to_string())?; diff --git a/mm2src/coins/utxo/bch.rs b/mm2src/coins/utxo/bch.rs index 58ddf035fb..eab88b6d31 100644 --- a/mm2src/coins/utxo/bch.rs +++ b/mm2src/coins/utxo/bch.rs @@ -314,28 +314,10 @@ impl BchCoin { &self, address: &Address, ) -> UtxoRpcResult<(BchUnspents, RecentlySpentOutPointsGuard<'_>)> { - let (bch_unspent_map, recently_spent) = self.bch_unspents_map_for_spend(vec![address.clone()]).await?; - let bch_unspents = try_get_unspent_list(address, bch_unspent_map, "bch_unspents_map_for_spend")?; - Ok((bch_unspents, recently_spent)) - } + let (all_unspents, recently_spent) = utxo_common::get_unspent_ordered_list(self, address).await?; + let result = self.utxos_into_bch_unspents(all_unspents).await?; - /// Locks recently spent cache to safely return UTXOs for spending - pub async fn bch_unspents_map_for_spend( - &self, - addresses: Vec
, - ) -> UtxoRpcResult<(BchUnspentMap, RecentlySpentOutPointsGuard<'_>)> { - let (all_unspents, recently_spent) = utxo_common::get_unspent_ordered_map(self, addresses).await?; - // Get an iterator of futures: `Future>` - let fut_it = all_unspents.into_iter().map(|(address, unspents)| { - self.utxos_into_bch_unspents(unspents) - .map(move |res| -> UtxoRpcResult<_> { - let bch_unspents = res?; - Ok((address, bch_unspents)) - }) - }); - // Poll the `fut_it` futures concurrently. - let bch_unspents_map = futures::future::try_join_all(fut_it).await?.into_iter().collect(); - Ok((bch_unspents_map, recently_spent)) + Ok((result, recently_spent)) } pub async fn get_token_utxos_for_spend( @@ -734,37 +716,28 @@ impl UtxoTxGenerationOps for BchCoin { #[async_trait] #[cfg_attr(test, mockable)] -impl ListUtxoOps for BchCoin { - async fn get_unspent_ordered_map( +impl GetUtxoListOps for BchCoin { + async fn get_unspent_ordered_list( &self, - addresses: Vec
, - ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { - let (bch_unspents_map, recently_spent) = self.bch_unspents_map_for_spend(addresses).await?; - let unspents = bch_unspents_map - .into_iter() - .map(|(address, bch_unspents)| (address, bch_unspents.standard)) - .collect(); - Ok((unspents, recently_spent)) + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + let (bch_unspents, recently_spent) = self.bch_unspents_for_spend(address).await?; + Ok((bch_unspents.standard, recently_spent)) } - async fn get_all_unspent_ordered_map( + async fn get_all_unspent_ordered_list( &self, - addresses: Vec
, - ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { - utxo_common::get_all_unspent_ordered_map(self, addresses).await + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_all_unspent_ordered_list(self, address).await } - async fn get_mature_unspent_ordered_map( + async fn get_mature_unspent_ordered_list( &self, - addresses: Vec
, - ) -> UtxoRpcResult<(MatureUnspentMap, RecentlySpentOutPointsGuard<'_>)> { - let (unspents_map, recently_spent) = utxo_common::get_all_unspent_ordered_map(self, addresses).await?; - // Convert `UnspentMap` into `MatureUnspentMap`. - let mature_unspents_map = unspents_map - .into_iter() - .map(|(address, unspents)| (address, MatureUnspentList::new_mature(unspents))) - .collect(); - Ok((mature_unspents_map, recently_spent)) + address: &Address, + ) -> UtxoRpcResult<(MatureUnspentList, RecentlySpentOutPointsGuard<'_>)> { + let (unspents, recently_spent) = utxo_common::get_all_unspent_ordered_list(self, address).await?; + Ok((MatureUnspentList::new_mature(unspents), recently_spent)) } } diff --git a/mm2src/coins/utxo/qtum.rs b/mm2src/coins/utxo/qtum.rs index 35d1a864f7..815596992a 100644 --- a/mm2src/coins/utxo/qtum.rs +++ b/mm2src/coins/utxo/qtum.rs @@ -319,7 +319,32 @@ impl UtxoTxGenerationOps for QtumCoin { #[async_trait] #[cfg_attr(test, mockable)] -impl ListUtxoOps for QtumCoin { +impl GetUtxoListOps for QtumCoin { + async fn get_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_unspent_ordered_list(self, address).await + } + + async fn get_all_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_all_unspent_ordered_list(self, address).await + } + + async fn get_mature_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(MatureUnspentList, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_mature_unspent_ordered_list(self, address).await + } +} + +#[async_trait] +#[cfg_attr(test, mockable)] +impl GetUtxoMapOps for QtumCoin { async fn get_unspent_ordered_map( &self, addresses: Vec
, diff --git a/mm2src/coins/utxo/qtum_delegation.rs b/mm2src/coins/utxo/qtum_delegation.rs index 49fae734d3..6a8d844ca2 100644 --- a/mm2src/coins/utxo/qtum_delegation.rs +++ b/mm2src/coins/utxo/qtum_delegation.rs @@ -5,7 +5,7 @@ use crate::qrc20::{contract_addr_into_rpc_format, ContractCallOutput, GenerateQr use crate::utxo::qtum::{QtumBasedCoin, QtumCoin, QtumDelegationOps, QtumDelegationRequest, QtumStakingInfosDetails}; use crate::utxo::rpc_clients::UtxoRpcClientEnum; use crate::utxo::utxo_common::{big_decimal_from_sat_unsigned, UtxoTxBuilder}; -use crate::utxo::{qtum, utxo_common, Address, ListUtxoOps, UtxoCommonOps}; +use crate::utxo::{qtum, utxo_common, Address, GetUtxoListOps, UtxoCommonOps}; use crate::utxo::{PrivKeyNotAllowed, UTXO_LOCK}; use crate::{DelegationError, DelegationFut, DelegationResult, MarketCoinOps, StakingInfos, StakingInfosError, StakingInfosFut, StakingInfosResult, TransactionDetails, TransactionType}; diff --git a/mm2src/coins/utxo/slp.rs b/mm2src/coins/utxo/slp.rs index 9b3a613476..acd251c0ec 100644 --- a/mm2src/coins/utxo/slp.rs +++ b/mm2src/coins/utxo/slp.rs @@ -1779,7 +1779,7 @@ pub fn slp_addr_from_pubkey_str(pubkey: &str, prefix: &str) -> Result UtxoFieldsWithHardwareWalletBuilder for UtxoArcBuilder<'a, F, T> impl<'a, F, T> UtxoCoinBuilder for UtxoArcBuilder<'a, F, T> where F: Fn(UtxoArc) -> T + Clone + Send + Sync + 'static, - T: UtxoCommonOps + ListUtxoOps, + T: UtxoCommonOps + GetUtxoListOps, { type ResultCoin = T; type Error = UtxoCoinBuildError; @@ -103,7 +103,7 @@ where impl<'a, F, T> MergeUtxoArcOps for UtxoArcBuilder<'a, F, T> where F: Fn(UtxoArc) -> T + Send + Sync + 'static, - T: UtxoCommonOps + ListUtxoOps, + T: UtxoCommonOps + GetUtxoListOps, { } @@ -114,7 +114,7 @@ where { } -pub trait MergeUtxoArcOps: UtxoCoinBuilderCommonOps { +pub trait MergeUtxoArcOps: UtxoCoinBuilderCommonOps { fn spawn_merge_utxo_loop_if_required(&self, weak: UtxoWeak, constructor: F) where F: Fn(UtxoArc) -> T + Send + Sync + 'static, diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index ef4bf6e444..51d09e5ca9 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -386,7 +386,7 @@ pub async fn load_hd_accounts_from_storage( /// Requests balance of the given `address`. pub async fn address_balance(coin: &T, address: &Address) -> BalanceResult where - T: UtxoCommonOps + ListUtxoOps + MarketCoinOps, + T: UtxoCommonOps + GetUtxoListOps + MarketCoinOps, { if coin.as_ref().check_utxo_maturity { let (unspents, _) = coin.get_mature_unspent_ordered_list(address).await?; @@ -410,7 +410,7 @@ where /// The pairs `(Address, CoinBalance)` are guaranteed to be in the same order in which they were requested. pub async fn addresses_balances(coin: &T, addresses: Vec
) -> BalanceResult> where - T: UtxoCommonOps + ListUtxoOps + MarketCoinOps, + T: UtxoCommonOps + GetUtxoMapOps + MarketCoinOps, { if coin.as_ref().check_utxo_maturity { let (unspents_map, _) = coin.get_mature_unspent_ordered_map(addresses.clone()).await?; @@ -586,7 +586,7 @@ pub async fn get_current_mtp(coin: &UtxoCoinFields, coin_variant: CoinVariant) - pub fn send_outputs_from_my_address(coin: T, outputs: Vec) -> TransactionFut where - T: UtxoCommonOps + ListUtxoOps, + T: UtxoCommonOps + GetUtxoListOps, { let fut = send_outputs_from_my_address_impl(coin, outputs); Box::new(fut.boxed().compat().map(|tx| tx.into())) @@ -1033,7 +1033,7 @@ pub async fn p2sh_spending_tx( pub fn send_taker_fee(coin: T, fee_pub_key: &[u8], amount: BigDecimal) -> TransactionFut where - T: UtxoCommonOps + ListUtxoOps, + T: UtxoCommonOps + GetUtxoListOps, { let address = try_fus!(address_from_raw_pubkey( fee_pub_key, @@ -1060,7 +1060,7 @@ pub fn send_maker_payment( amount: BigDecimal, ) -> TransactionFut where - T: UtxoCommonOps + ListUtxoOps, + T: UtxoCommonOps + GetUtxoListOps, { let SwapPaymentOutputsResult { payment_address, @@ -1097,7 +1097,7 @@ pub fn send_taker_payment( amount: BigDecimal, ) -> TransactionFut where - T: UtxoCommonOps + ListUtxoOps, + T: UtxoCommonOps + GetUtxoListOps, { let SwapPaymentOutputsResult { payment_address, @@ -1674,7 +1674,7 @@ pub fn my_address(coin: &T) -> Result { pub fn my_balance(coin: T) -> BalanceFut where - T: UtxoCommonOps + ListUtxoOps + MarketCoinOps, + T: UtxoCommonOps + GetUtxoListOps + MarketCoinOps, { let my_address = try_f!(coin .as_ref() @@ -1808,7 +1808,7 @@ pub async fn get_raw_transaction(coin: &UtxoCoinFields, req: RawTransactionReque pub async fn withdraw(coin: T, req: WithdrawRequest) -> WithdrawResult where - T: UtxoCommonOps + ListUtxoOps + MarketCoinOps, + T: UtxoCommonOps + GetUtxoListOps + MarketCoinOps, { StandardUtxoWithdraw::new(coin, req)?.build().await } @@ -1821,7 +1821,7 @@ pub async fn init_withdraw( ) -> WithdrawResult where T: UtxoCommonOps - + ListUtxoOps + + GetUtxoListOps + UtxoSignerOps + CoinWithDerivationMethod + GetWithdrawSenderAddress
, @@ -2576,7 +2576,7 @@ pub async fn preimage_trade_fee_required_to_send_outputs( stage: &FeeApproxStage, ) -> TradePreimageResult where - T: UtxoCommonOps + ListUtxoOps, + T: UtxoCommonOps + GetUtxoListOps, { let ticker = coin.as_ref().conf.ticker.clone(); let decimals = coin.as_ref().decimals; @@ -2763,7 +2763,22 @@ pub fn is_coin_protocol_supported(coin: &T, info: &Option( + coin: &'a T, + address: &Address, +) -> UtxoRpcResult<(MatureUnspentList, RecentlySpentOutPointsGuard<'a>)> +where + T: UtxoCommonOps + GetUtxoListOps, +{ + let (unspents, recently_spent) = coin.get_all_unspent_ordered_list(address).await?; + let mature_unspents = identify_mature_unspents(coin, unspents).await?; + Ok((mature_unspents, recently_spent)) +} + +/// [`GetUtxoMapOps::get_mature_unspent_ordered_map`] implementation. /// Returns available mature and immature unspents in ascending order for every given `addresses` /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). pub async fn get_mature_unspent_ordered_map( @@ -2771,7 +2786,7 @@ pub async fn get_mature_unspent_ordered_map( addresses: Vec
, ) -> UtxoRpcResult<(MatureUnspentMap, RecentlySpentOutPointsGuard<'_>)> where - T: UtxoCommonOps + ListUtxoOps, + T: UtxoCommonOps + GetUtxoMapOps, { let (unspents_map, recently_spent) = coin.get_all_unspent_ordered_map(addresses).await?; // Get an iterator of futures: `Future>` @@ -3309,7 +3324,27 @@ pub fn dex_fee_script(uuid: [u8; 16], time_lock: u32, watcher_pub: &Public, send .into_script() } -/// [`ListUtxoOps::get_unspent_ordered_map`] implementation. +/// [`GetUtxoListOps::get_unspent_ordered_list`] implementation. +/// Returns available unspents in ascending order +/// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). +pub async fn get_unspent_ordered_list<'a, T>( + coin: &'a T, + address: &Address, +) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'a>)> +where + T: UtxoCommonOps + GetUtxoListOps, +{ + if coin.as_ref().check_utxo_maturity { + coin.get_mature_unspent_ordered_list(address) + .await + // Convert `MatureUnspentList` into `Vec` by discarding immature unspents. + .map(|(mature_unspents, recently_spent)| (mature_unspents.only_mature(), recently_spent)) + } else { + coin.get_all_unspent_ordered_list(address).await + } +} + +/// [`GetUtxoMapOps::get_unspent_ordered_map`] implementation. /// Returns available unspents in ascending order + `RecentlySpentOutPoints` MutexGuard for further interaction /// (e.g. to add new transaction to it). pub async fn get_unspent_ordered_map( @@ -3317,7 +3352,7 @@ pub async fn get_unspent_ordered_map( addresses: Vec
, ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> where - T: UtxoCommonOps + ListUtxoOps, + T: UtxoCommonOps + GetUtxoMapOps, { if coin.as_ref().check_utxo_maturity { coin.get_mature_unspent_ordered_map(addresses) @@ -3335,7 +3370,27 @@ where } } -/// [`ListUtxoOps::get_all_unspent_ordered_map`] implementation. +/// [`GetUtxoListOps::get_all_unspent_ordered_list`] implementation. +/// Returns available mature and immature unspents in ascending +/// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). +pub async fn get_all_unspent_ordered_list<'a, T: UtxoCommonOps>( + coin: &'a T, + address: &Address, +) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'a>)> { + let decimals = coin.as_ref().decimals; + let unspents = coin + .as_ref() + .rpc_client + .list_unspent(address, decimals) + .compat() + .await?; + let recently_spent = coin.as_ref().recently_spent_outpoints.lock().await; + let unordered_unspents = recently_spent.replace_spent_outputs_with_cache(unspents.into_iter().collect()); + let ordered_unspents = sort_dedup_unspents(unordered_unspents); + Ok((ordered_unspents, recently_spent)) +} + +/// [`GetUtxoMapOps::get_all_unspent_ordered_map`] implementation. /// Returns available mature and immature unspents in ascending order for every given `addresses` /// + `RecentlySpentOutPoints` MutexGuard for further interaction (e.g. to add new transaction to it). pub async fn get_all_unspent_ordered_map( @@ -3351,19 +3406,8 @@ pub async fn get_all_unspent_ordered_map( .await?; let recently_spent = coin.as_ref().recently_spent_outpoints.lock().await; for (_address, unspents) in unspents_map.iter_mut() { - *unspents = recently_spent - .replace_spent_outputs_with_cache(unspents.iter().cloned().collect()) - .into_iter() - // dedup just in case we add duplicates of same unspent out - .unique_by(|unspent| unspent.outpoint.clone()) - .sorted_unstable_by(|a, b| { - if a.value < b.value { - Ordering::Less - } else { - Ordering::Greater - } - }) - .collect(); + let unordered_unspents = recently_spent.replace_spent_outputs_with_cache(unspents.iter().cloned().collect()); + *unspents = sort_dedup_unspents(unordered_unspents); } Ok((unspents_map, recently_spent)) } @@ -3544,7 +3588,7 @@ pub async fn merge_utxo_loop( max_merge_at_once: usize, constructor: impl Fn(UtxoArc) -> T, ) where - T: UtxoCommonOps + ListUtxoOps, + T: UtxoCommonOps + GetUtxoListOps, { loop { Timer::sleep(check_every).await; @@ -3712,6 +3756,25 @@ where } } +/// Sorts and deduplicates the given `unspents` in ascending order. +fn sort_dedup_unspents(unspents: I) -> Vec +where + I: IntoIterator, +{ + unspents + .into_iter() + // dedup just in case we add duplicates of same unspent out + .unique_by(|unspent| unspent.outpoint.clone()) + .sorted_unstable_by(|a, b| { + if a.value < b.value { + Ordering::Less + } else { + Ordering::Greater + } + }) + .collect() +} + #[test] fn test_increase_by_percent() { assert_eq!(increase_by_percent(4300, 1.), 4343); diff --git a/mm2src/coins/utxo/utxo_standard.rs b/mm2src/coins/utxo/utxo_standard.rs index 370b3ec773..469dd4dca3 100644 --- a/mm2src/coins/utxo/utxo_standard.rs +++ b/mm2src/coins/utxo/utxo_standard.rs @@ -91,7 +91,32 @@ impl UtxoTxGenerationOps for UtxoStandardCoin { #[async_trait] #[cfg_attr(test, mockable)] -impl ListUtxoOps for UtxoStandardCoin { +impl GetUtxoListOps for UtxoStandardCoin { + async fn get_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_unspent_ordered_list(self, address).await + } + + async fn get_all_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_all_unspent_ordered_list(self, address).await + } + + async fn get_mature_unspent_ordered_list( + &self, + address: &Address, + ) -> UtxoRpcResult<(MatureUnspentList, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_mature_unspent_ordered_list(self, address).await + } +} + +#[async_trait] +#[cfg_attr(test, mockable)] +impl GetUtxoMapOps for UtxoStandardCoin { async fn get_unspent_ordered_map( &self, addresses: Vec
, diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index c2c1911697..33a9f06969 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -25,7 +25,6 @@ use common::{block_on, now_ms, OrdRange, PagingOptionsEnum, DEX_FEE_ADDR_RAW_PUB use crypto::{Bip44Chain, RpcDerivationPath}; use futures::future::join_all; use futures::TryFutureExt; -use itertools::Itertools; use mocktopus::mocking::*; use rpc::v1::types::H256 as H256Json; use serialization::{deserialize, CoinVariant}; @@ -1676,8 +1675,7 @@ fn test_qtum_remove_delegation() { #[test] fn test_qtum_my_balance() { - QtumCoin::get_mature_unspent_ordered_map.mock_safe(move |coin, addresses| { - let address = addresses.into_iter().exactly_one().unwrap(); + QtumCoin::get_mature_unspent_ordered_list.mock_safe(move |coin, _address| { let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); // spendable balance (66.0) let mature = vec![ @@ -1707,8 +1705,10 @@ fn test_qtum_my_balance() { value: 200000000, height: Default::default(), }]; - let unspents_map: MatureUnspentMap = iter::once((address, MatureUnspentList { mature, immature })).collect(); - MockResult::Return(Box::pin(futures::future::ok((unspents_map, cache)))) + MockResult::Return(Box::pin(futures::future::ok(( + MatureUnspentList { mature, immature }, + cache, + )))) }); let conf = json!({"coin":"tQTUM","rpcport":13889,"pubtype":120,"p2shtype":110}); @@ -1740,9 +1740,9 @@ fn test_qtum_my_balance_with_check_utxo_maturity_false() { ElectrumClient::display_balance.mock_safe(move |_, _, _| { MockResult::Return(Box::new(futures01::future::ok(BigDecimal::from(DISPLAY_BALANCE)))) }); - QtumCoin::get_all_unspent_ordered_map.mock_safe(move |_, _| { + QtumCoin::get_all_unspent_ordered_list.mock_safe(move |_, _| { panic!( - "'QtumCoin::get_all_unspent_ordered_map' is not expected to be called when `check_utxo_maturity` is false" + "'QtumCoin::get_all_unspent_ordered_list' is not expected to be called when `check_utxo_maturity` is false" ) }); @@ -1786,8 +1786,7 @@ fn test_get_mature_unspent_ordered_map_from_cache_impl( verbose.height = cached_height; // prepare mocks - ElectrumClient::list_unspent_group.mock_safe(move |_, addresses, _| { - let address = addresses.into_iter().exactly_one().unwrap(); + ElectrumClient::list_unspent.mock_safe(move |_, _, _| { let unspents = vec![UnspentInfo { outpoint: OutPoint { hash: H256::from_reversed_str(TX_HASH), @@ -1796,8 +1795,7 @@ fn test_get_mature_unspent_ordered_map_from_cache_impl( value: 1000000000, height: unspent_height, }]; - let unspents_map: UnspentMap = iter::once((address, unspents)).collect(); - MockResult::Return(Box::new(futures01::future::ok(unspents_map))) + MockResult::Return(Box::new(futures01::future::ok(unspents))) }); ElectrumClient::get_block_count .mock_safe(move |_| MockResult::Return(Box::new(futures01::future::ok(block_count)))); @@ -1817,13 +1815,13 @@ fn test_get_mature_unspent_ordered_map_from_cache_impl( // run test let coin = utxo_coin_for_test(UtxoRpcClientEnum::Electrum(client), None, false); - let (unspents_map, _) = - block_on(coin.get_mature_unspent_ordered_map(vec![Address::from("R9o9xTocqr6CeEDGDH6mEYpwLoMz6jNjMW")])) + let (unspents, _) = + block_on(coin.get_mature_unspent_ordered_list(&Address::from("R9o9xTocqr6CeEDGDH6mEYpwLoMz6jNjMW"))) .expect("Expected an empty unspent list"); // unspents should be empty because `is_unspent_mature()` always returns false - let (_address, unspents) = unspents_map.into_iter().exactly_one().unwrap(); - assert!(unspents.mature.is_empty()); assert!(unsafe { IS_UNSPENT_MATURE_CALLED == true }); + assert!(unspents.mature.is_empty()); + assert_eq!(unspents.immature.len(), 1); } #[test] @@ -1982,11 +1980,8 @@ fn test_native_client_unspents_filtered_using_tx_cache_single_tx_in_cache() { tx.hash(), tx.outputs.clone(), ); - NativeClient::list_unspent_group.mock_safe(move |_, addresses, _| { - let address = addresses.into_iter().exactly_one().unwrap(); - let unspents_map: UnspentMap = iter::once((address, spent_by_tx.clone())).collect(); - MockResult::Return(Box::new(futures01::future::ok(unspents_map))) - }); + NativeClient::list_unspent + .mock_safe(move |_, _, _| MockResult::Return(Box::new(futures01::future::ok(spent_by_tx.clone())))); let (unspents_ordered, _) = block_on(coin.get_unspent_ordered_list(&address)).unwrap(); // output 2 is change so it must be returned @@ -2079,11 +2074,8 @@ fn test_native_client_unspents_filtered_using_tx_cache_single_several_chained_tx unspents_to_return.extend(spent_by_tx_1); unspents_to_return.extend(spent_by_tx_2); - NativeClient::list_unspent_group.mock_safe(move |_, addresses, _| { - let address = addresses.into_iter().exactly_one().unwrap(); - let unspents_map: UnspentMap = iter::once((address, unspents_to_return.clone())).collect(); - MockResult::Return(Box::new(futures01::future::ok(unspents_map))) - }); + NativeClient::list_unspent + .mock_safe(move |_, _, _| MockResult::Return(Box::new(futures01::future::ok(unspents_to_return.clone())))); let (unspents_ordered, _) = block_on(coin.get_unspent_ordered_list(&address)).unwrap(); @@ -3143,14 +3135,10 @@ fn test_withdraw_to_p2wpkh() { fn test_utxo_standard_with_check_utxo_maturity_true() { static mut LIST_MATURE_UNSPENT_ORDERED_CALLED: bool = false; - UtxoStandardCoin::get_mature_unspent_ordered_map.mock_safe(|coin, addresses| { + UtxoStandardCoin::get_mature_unspent_ordered_list.mock_safe(|coin, _| { unsafe { LIST_MATURE_UNSPENT_ORDERED_CALLED = true }; - - let address = addresses.into_iter().exactly_one().unwrap(); - let unspents_map: MatureUnspentMap = iter::once((address, MatureUnspentList::default())).collect(); - let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); - MockResult::Return(Box::pin(futures::future::ok((unspents_map, cache)))) + MockResult::Return(Box::pin(futures::future::ok((MatureUnspentList::default(), cache)))) }); let conf = json!({"coin":"RICK","asset":"RICK","rpcport":25435,"txversion":4,"overwintered":1,"mm2":1,"protocol":{"type":"UTXO"}}); @@ -3173,7 +3161,7 @@ fn test_utxo_standard_with_check_utxo_maturity_true() { .unwrap(); let address = Address::from("R9o9xTocqr6CeEDGDH6mEYpwLoMz6jNjMW"); - // Don't use `block_on` here because it's used within a mock of [`UtxoStandardCoin::get_mature_unspent_ordered_map`]. + // Don't use `block_on` here because it's used within a mock of [`GetUtxoListOps::get_mature_unspent_ordered_list`]. coin.get_unspent_ordered_list(&address).compat().wait().unwrap(); assert!(unsafe { LIST_MATURE_UNSPENT_ORDERED_CALLED }); } @@ -3182,20 +3170,17 @@ fn test_utxo_standard_with_check_utxo_maturity_true() { /// https://github.com/KomodoPlatform/atomicDEX-API/issues/1181 #[test] fn test_utxo_standard_without_check_utxo_maturity() { - static mut GET_ALL_UNSPENT_ORDERED_MAP_CALLED: bool = false; - - UtxoStandardCoin::get_all_unspent_ordered_map.mock_safe(|coin, addresses| { - unsafe { GET_ALL_UNSPENT_ORDERED_MAP_CALLED = true }; - - let address = addresses.into_iter().exactly_one().unwrap(); - let unspents_map: HashMap<_, _> = iter::once((address, Vec::new())).collect(); + static mut GET_ALL_UNSPENT_ORDERED_LIST_CALLED: bool = false; + UtxoStandardCoin::get_all_unspent_ordered_list.mock_safe(|coin, _| { + unsafe { GET_ALL_UNSPENT_ORDERED_LIST_CALLED = true }; let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); - MockResult::Return(Box::pin(futures::future::ok((unspents_map, cache)))) + let unspents = Vec::new(); + MockResult::Return(Box::pin(futures::future::ok((unspents, cache)))) }); - UtxoStandardCoin::get_mature_unspent_ordered_map.mock_safe(|_, _| { - panic!("'UtxoStandardCoin::get_mature_unspent_ordered_map' is not expected to be called when `check_utxo_maturity` is not set") + UtxoStandardCoin::get_mature_unspent_ordered_list.mock_safe(|_, _| { + panic!("'UtxoStandardCoin::get_mature_unspent_ordered_list' is not expected to be called when `check_utxo_maturity` is not set") }); let conf = json!({"coin":"RICK","asset":"RICK","rpcport":25435,"txversion":4,"overwintered":1,"mm2":1,"protocol":{"type":"UTXO"}}); @@ -3217,25 +3202,21 @@ fn test_utxo_standard_without_check_utxo_maturity() { .unwrap(); let address = Address::from("R9o9xTocqr6CeEDGDH6mEYpwLoMz6jNjMW"); - // Don't use `block_on` here because it's used within a mock of [`UtxoStandardCoin::get_mature_unspent_ordered_map`]. + // Don't use `block_on` here because it's used within a mock of [`UtxoStandardCoin::get_all_unspent_ordered_list`]. coin.get_unspent_ordered_list(&address).compat().wait().unwrap(); - assert!(unsafe { GET_ALL_UNSPENT_ORDERED_MAP_CALLED }); + assert!(unsafe { GET_ALL_UNSPENT_ORDERED_LIST_CALLED }); } /// `QtumCoin` has to check UTXO maturity if `check_utxo_maturity` is not set. /// https://github.com/KomodoPlatform/atomicDEX-API/issues/1181 #[test] fn test_qtum_without_check_utxo_maturity() { - static mut GET_MATURE_UNSPENT_ORDERED_MAP_CALLED: bool = false; - - QtumCoin::get_mature_unspent_ordered_map.mock_safe(|coin, addresses| { - unsafe { GET_MATURE_UNSPENT_ORDERED_MAP_CALLED = true }; - - let address = addresses.into_iter().exactly_one().unwrap(); - let unspents_map: HashMap<_, _> = iter::once((address, MatureUnspentList::default())).collect(); + static mut GET_MATURE_UNSPENT_ORDERED_LIST_CALLED: bool = false; + QtumCoin::get_mature_unspent_ordered_list.mock_safe(|coin, _| { + unsafe { GET_MATURE_UNSPENT_ORDERED_LIST_CALLED = true }; let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); - MockResult::Return(Box::pin(futures::future::ok((unspents_map, cache)))) + MockResult::Return(Box::pin(futures::future::ok((MatureUnspentList::default(), cache)))) }); let conf = json!({"coin":"tQTUM","rpcport":13889,"pubtype":120,"p2shtype":110}); @@ -3254,29 +3235,26 @@ fn test_qtum_without_check_utxo_maturity() { let coin = block_on(qtum_coin_with_priv_key(&ctx, "QTUM", &conf, ¶ms, &[1u8; 32])).unwrap(); let address = Address::from("qcyBHeSct7Wr4mAw18iuQ1zW5mMFYmtmBE"); - // Don't use `block_on` here because it's used within a mock of [`QtumCoin::get_mature_unspent_ordered_map`]. + // Don't use `block_on` here because it's used within a mock of [`QtumCoin::get_mature_unspent_ordered_list`]. coin.get_unspent_ordered_list(&address).compat().wait().unwrap(); - assert!(unsafe { GET_MATURE_UNSPENT_ORDERED_MAP_CALLED }); + assert!(unsafe { GET_MATURE_UNSPENT_ORDERED_LIST_CALLED }); } /// `QtumCoin` hasn't to check UTXO maturity if `check_utxo_maturity` is `false`. /// https://github.com/KomodoPlatform/atomicDEX-API/issues/1181 #[test] fn test_qtum_with_check_utxo_maturity_false() { - static mut GET_ALL_UNSPENT_ORDERED_MAP_CALLED: bool = false; - - QtumCoin::get_all_unspent_ordered_map.mock_safe(|coin, addresses| { - unsafe { GET_ALL_UNSPENT_ORDERED_MAP_CALLED = true }; - - let address = addresses.into_iter().exactly_one().unwrap(); - let unspents_map: HashMap<_, _> = iter::once((address, Vec::new())).collect(); + static mut GET_ALL_UNSPENT_ORDERED_LIST_CALLED: bool = false; + QtumCoin::get_all_unspent_ordered_list.mock_safe(|coin, _address| { + unsafe { GET_ALL_UNSPENT_ORDERED_LIST_CALLED = true }; let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); - MockResult::Return(Box::pin(futures::future::ok((unspents_map, cache)))) + let unspents = Vec::new(); + MockResult::Return(Box::pin(futures::future::ok((unspents, cache)))) }); - QtumCoin::get_mature_unspent_ordered_map.mock_safe(|_, _| { + QtumCoin::get_mature_unspent_ordered_list.mock_safe(|_, _| { panic!( - "'QtumCoin::get_mature_unspent_ordered_map' is not expected to be called when `check_utxo_maturity` is false" + "'QtumCoin::get_mature_unspent_ordered_list' is not expected to be called when `check_utxo_maturity` is false" ) }); @@ -3297,9 +3275,9 @@ fn test_qtum_with_check_utxo_maturity_false() { let coin = block_on(qtum_coin_with_priv_key(&ctx, "QTUM", &conf, ¶ms, &[1u8; 32])).unwrap(); let address = Address::from("qcyBHeSct7Wr4mAw18iuQ1zW5mMFYmtmBE"); - // Don't use `block_on` here because it's used within a mock of [`QtumCoin::get_all_unspent_ordered_map`]. + // Don't use `block_on` here because it's used within a mock of [`QtumCoin::get_all_unspent_ordered_list`]. coin.get_unspent_ordered_list(&address).compat().wait().unwrap(); - assert!(unsafe { GET_ALL_UNSPENT_ORDERED_MAP_CALLED }); + assert!(unsafe { GET_ALL_UNSPENT_ORDERED_LIST_CALLED }); } #[test] diff --git a/mm2src/coins/utxo/utxo_withdraw.rs b/mm2src/coins/utxo/utxo_withdraw.rs index 8987b3693b..edd06c8d7a 100644 --- a/mm2src/coins/utxo/utxo_withdraw.rs +++ b/mm2src/coins/utxo/utxo_withdraw.rs @@ -1,6 +1,6 @@ use crate::rpc_command::init_withdraw::{WithdrawAwaitingStatus, WithdrawInProgressStatus, WithdrawTaskHandle}; use crate::utxo::utxo_common::{big_decimal_from_sat, UtxoTxBuilder}; -use crate::utxo::{output_script, sat_from_big_decimal, ActualTxFee, Address, FeePolicy, ListUtxoOps, PrivKeyPolicy, +use crate::utxo::{output_script, sat_from_big_decimal, ActualTxFee, Address, FeePolicy, GetUtxoListOps, PrivKeyPolicy, UtxoAddressFormat, UtxoCoinFields, UtxoCommonOps, UtxoFeeDetails, UtxoTx, UTXO_LOCK}; use crate::{CoinWithDerivationMethod, GetWithdrawSenderAddress, MarketCoinOps, TransactionDetails, WithdrawError, WithdrawFee, WithdrawRequest, WithdrawResult}; @@ -100,7 +100,7 @@ impl From for WithdrawError { pub trait UtxoWithdraw where Self: Sized + Sync, - Coin: UtxoCommonOps + ListUtxoOps, + Coin: UtxoCommonOps + GetUtxoListOps, { fn coin(&self) -> &Coin; @@ -246,7 +246,7 @@ pub struct InitUtxoWithdraw<'a, Coin> { #[async_trait] impl<'a, Coin> UtxoWithdraw for InitUtxoWithdraw<'a, Coin> where - Coin: UtxoCommonOps + ListUtxoOps + UtxoSignerOps, + Coin: UtxoCommonOps + GetUtxoListOps + UtxoSignerOps, { fn coin(&self) -> &Coin { &self.coin } @@ -404,7 +404,7 @@ pub struct StandardUtxoWithdraw { #[async_trait] impl UtxoWithdraw for StandardUtxoWithdraw where - Coin: UtxoCommonOps + ListUtxoOps, + Coin: UtxoCommonOps + GetUtxoListOps, { fn coin(&self) -> &Coin { &self.coin } diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index a7f2b15a5b..fa6dac777c 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -1,10 +1,10 @@ -use crate::utxo::rpc_clients::{UnspentMap, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcError, UtxoRpcFut, +use crate::utxo::rpc_clients::{UnspentInfo, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; use crate::utxo::utxo_builder::{UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPrivKeyBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::{big_decimal_from_sat_unsigned, payment_script}; use crate::utxo::{sat_from_big_decimal, utxo_common, ActualTxFee, AdditionalTxData, Address, BroadcastTxErr, - FeePolicy, HistoryUtxoTx, HistoryUtxoTxMap, ListUtxoOps, MatureUnspentMap, + FeePolicy, GetUtxoListOps, HistoryUtxoTx, HistoryUtxoTxMap, MatureUnspentList, RecentlySpentOutPointsGuard, UtxoActivationParams, UtxoAddressFormat, UtxoArc, UtxoCoinFields, UtxoCommonOps, UtxoFeeDetails, UtxoTxBroadcastOps, UtxoTxGenerationOps, UtxoWeak, VerboseTransactionFrom}; @@ -1323,30 +1323,30 @@ impl UtxoTxBroadcastOps for ZCoin { } /// Please note `ZCoin` is not assumed to work with transparent UTXOs. -/// Remove implementation of the `ListUtxoOps` trait for `ZCoin` +/// Remove implementation of the `GetUtxoListOps` trait for `ZCoin` /// when [`ZCoin::preimage_trade_fee_required_to_send_outputs`] is refactored. #[async_trait] #[cfg_attr(test, mockable)] -impl ListUtxoOps for ZCoin { - async fn get_unspent_ordered_map( +impl GetUtxoListOps for ZCoin { + async fn get_unspent_ordered_list( &self, - addresses: Vec
, - ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { - utxo_common::get_unspent_ordered_map(self, addresses).await + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_unspent_ordered_list(self, address).await } - async fn get_all_unspent_ordered_map( + async fn get_all_unspent_ordered_list( &self, - addresses: Vec
, - ) -> UtxoRpcResult<(UnspentMap, RecentlySpentOutPointsGuard<'_>)> { - utxo_common::get_all_unspent_ordered_map(self, addresses).await + address: &Address, + ) -> UtxoRpcResult<(Vec, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_all_unspent_ordered_list(self, address).await } - async fn get_mature_unspent_ordered_map( + async fn get_mature_unspent_ordered_list( &self, - addresses: Vec
, - ) -> UtxoRpcResult<(MatureUnspentMap, RecentlySpentOutPointsGuard<'_>)> { - utxo_common::get_mature_unspent_ordered_map(self, addresses).await + address: &Address, + ) -> UtxoRpcResult<(MatureUnspentList, RecentlySpentOutPointsGuard<'_>)> { + utxo_common::get_mature_unspent_ordered_list(self, address).await } } diff --git a/mm2src/docker_tests.rs b/mm2src/docker_tests.rs index fec053b380..61a245c1e6 100644 --- a/mm2src/docker_tests.rs +++ b/mm2src/docker_tests.rs @@ -112,7 +112,7 @@ mod docker_tests { use coins::utxo::slp::{slp_genesis_output, SlpOutput}; use coins::utxo::utxo_common::send_outputs_from_my_address; use coins::utxo::utxo_standard::{utxo_standard_coin_with_priv_key, UtxoStandardCoin}; - use coins::utxo::{dhash160, ListUtxoOps, UtxoActivationParams, UtxoCommonOps}; + use coins::utxo::{dhash160, GetUtxoListOps, UtxoActivationParams, UtxoCommonOps}; use coins::{CoinProtocol, FoundSwapTxSpend, MarketCoinOps, MmCoin, SwapOps, Transaction, TransactionEnum, WithdrawRequest}; use common::for_tests::{check_my_swap_status_amounts, enable_electrum}; From 73e9dafbe7c9ce9c597f1e088d511790f3c43c58 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Thu, 28 Apr 2022 20:06:09 +0300 Subject: [PATCH 14/25] Fix tests compilation error --- mm2src/coins/z_coin.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index fa6dac777c..29997f4075 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -30,6 +30,7 @@ use futures::{FutureExt, TryFutureExt}; use futures01::Future; use keys::hash::H256; use keys::{KeyPair, Public}; +#[cfg(test)] use mocktopus::macros::*; use primitives::bytes::Bytes; use rpc::v1::types::{Bytes as BytesJson, ToTxHash, Transaction as RpcTransaction, H256 as H256Json}; use rusqlite::types::Type; From b85298414ff5735c5c504c5d6278e95394aaa6cd Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Sun, 1 May 2022 13:16:09 +0700 Subject: [PATCH 15/25] Add `UtxoVerboseCache` to avoid locking global mutex --- mm2src/coins/qrc20.rs | 7 + mm2src/coins/utxo.rs | 6 +- mm2src/coins/utxo/tx_cache.rs | 125 ++++++++++++------ .../utxo/utxo_builder/utxo_coin_builder.rs | 16 ++- mm2src/coins/utxo/utxo_common.rs | 11 +- mm2src/coins/utxo/utxo_tests.rs | 2 +- 6 files changed, 113 insertions(+), 54 deletions(-) diff --git a/mm2src/coins/qrc20.rs b/mm2src/coins/qrc20.rs index 6b446984fd..6fec696748 100644 --- a/mm2src/coins/qrc20.rs +++ b/mm2src/coins/qrc20.rs @@ -4,6 +4,7 @@ use crate::qrc20::rpc_clients::{LogEntry, Qrc20ElectrumOps, Qrc20NativeOps, Qrc2 use crate::utxo::qtum::QtumBasedCoin; use crate::utxo::rpc_clients::{ElectrumClient, NativeClient, UnspentInfo, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; +use crate::utxo::tx_cache::UtxoVerboseCache; use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuildResult, UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPrivKeyBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::{self, big_decimal_from_sat, check_all_inputs_signed_by_pub, UtxoTxBuilder}; @@ -241,6 +242,12 @@ impl<'a> UtxoCoinBuilderCommonOps for Qrc20CoinBuilder<'a> { } true } + + /// Override [`UtxoCoinBuilderCommonOps::tx_cache`] to initialize TX cache with the platform ticker. + fn tx_cache(&self) -> UtxoVerboseCache { + let tx_cache_path = self.ctx().dbdir().join("TX_CACHE"); + UtxoVerboseCache::new(self.platform.clone(), tx_cache_path) + } } #[async_trait] diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 913dbcd3e2..74dce3dbc0 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -98,8 +98,10 @@ use super::{big_decimal_from_sat_unsigned, BalanceError, BalanceFut, BalanceResu use crate::coin_balance::{EnableCoinScanPolicy, HDAddressBalanceScanner}; use crate::hd_wallet::{HDAccountOps, HDAccountsMutex, HDAddress, HDWalletCoinOps, HDWalletOps, InvalidBip44ChainError}; use crate::hd_wallet_storage::{HDAccountStorageItem, HDWalletCoinStorage, HDWalletStorageError, HDWalletStorageResult}; +use crate::utxo::tx_cache::UtxoVerboseCache; use crate::utxo::utxo_block_header_storage::BlockHeaderStorageError; use utxo_block_header_storage::BlockHeaderStorage; + #[cfg(not(target_arch = "wasm32"))] pub mod tx_cache; #[cfg(target_arch = "wasm32")] pub mod utxo_indexedb_block_header_storage; @@ -522,8 +524,8 @@ pub struct UtxoCoinFields { /// Either an Iguana address or an info about last derived account/address. pub derivation_method: DerivationMethod, pub history_sync_state: Mutex, - /// Path to the TX cache directory - pub tx_cache_directory: Option, + /// The cache of verbose transactions. + pub tx_cache: Option, pub block_headers_storage: Option, /// The cache of recently send transactions used to track the spent UTXOs and replace them with new outputs /// The daemon needs some time to update the listunspent list for address which makes it return already spent UTXOs diff --git a/mm2src/coins/utxo/tx_cache.rs b/mm2src/coins/utxo/tx_cache.rs index a709c618a7..ecf527988c 100644 --- a/mm2src/coins/utxo/tx_cache.rs +++ b/mm2src/coins/utxo/tx_cache.rs @@ -4,14 +4,17 @@ use common::mm_error::prelude::*; use derive_more::Display; use futures::lock::Mutex as AsyncMutex; use futures::FutureExt; +use parking_lot::Mutex as PaMutex; use rpc::v1::types::{Transaction as RpcTransaction, H256 as H256Json}; +use std::collections::hash_map::RawEntryMut; use std::collections::HashMap; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; +use std::sync::Arc; pub type TxCacheResult = MmResult; lazy_static! { - static ref TX_CACHE_LOCK: AsyncMutex<()> = AsyncMutex::new(()); + static ref TX_CACHE_LOCK: TxCacheLock = TxCacheLock::default(); } #[derive(Debug, Display)] @@ -33,52 +36,90 @@ impl From for TxCacheError { } } -/// Tries to load transactions from cache concurrently. -/// Note 1: tx.confirmations can be out-of-date. -/// Note 2: this function locks the `TX_CACHE_LOCK` mutex to avoid reading and writing the same files at the same time. -pub async fn load_transactions_from_cache_concurrently( - tx_cache_path: &Path, - tx_ids: I, -) -> HashMap>> -where - I: IntoIterator, -{ - let _ = TX_CACHE_LOCK.lock().await; - - let it = tx_ids - .into_iter() - .map(|txid| load_transaction_from_cache(tx_cache_path, txid).map(move |res| (txid, res))); - futures::future::join_all(it).await.into_iter().collect() +#[derive(Default)] +struct TxCacheLock { + /// The collection of `Ticker -> Mutex` pairs. + mutexes: PaMutex>>>, } -/// Uploads transactions to cache concurrently. -/// Note: this function locks the `TX_CACHE_LOCK` mutex and takes `txs` as the Hash map -/// to avoid reading and writing the same files at the same time. -pub async fn cache_transactions_concurrently(tx_cache_path: &Path, txs: &HashMap) { - let _ = TX_CACHE_LOCK.lock().await; - - let it = txs.iter().map(|(_txid, tx)| cache_transaction(tx_cache_path, tx)); - futures::future::join_all(it) - .await - .into_iter() - .for_each(|tx| tx.error_log()); +impl TxCacheLock { + /// Get the mutex corresponding to the specified `ticker`. + pub fn mutex_by_ticker(&self, ticker: &str) -> Arc> { + let mut locks = self.mutexes.lock(); + + match locks.raw_entry_mut().from_key(ticker) { + RawEntryMut::Occupied(mutex) => mutex.get().clone(), + RawEntryMut::Vacant(vacant_mutex) => { + let (_key, mutex) = vacant_mutex.insert(ticker.to_owned(), Arc::new(AsyncMutex::new(()))); + mutex.clone() + }, + } + } } -/// Tries to load transaction from cache. -/// Note: tx.confirmations can be out-of-date. -async fn load_transaction_from_cache(tx_cache_path: &Path, txid: H256Json) -> TxCacheResult> { - let path = cached_transaction_path(tx_cache_path, &txid); - read_json(&path).await.mm_err(TxCacheError::from) +/// The cache instance that assigned to a specified coin. +/// +/// Please note [`UtxoVerboseCache::ticker`] may not equal to [`Coin::ticker`]. +/// In particular, `QRC20` tokens have the same transactions as `Qtum` coin, +/// so [`Qrc20Coin::platform_ticker`] is used as [`UtxoVerboseCache::ticker`]. +#[derive(Debug)] +pub struct UtxoVerboseCache { + ticker: String, + tx_cache_path: PathBuf, } -/// Uploads transaction to cache. -async fn cache_transaction(tx_cache_path: &Path, tx: &RpcTransaction) -> TxCacheResult<()> { - const USE_TMP_FILE: bool = true; +impl UtxoVerboseCache { + pub fn new(ticker: String, tx_cache_path: PathBuf) -> UtxoVerboseCache { + UtxoVerboseCache { ticker, tx_cache_path } + } + + /// Tries to load transactions from cache concurrently. + /// Note 1: `tx.confirmations` can be out-of-date. + /// Note 2: this function locks the `TX_CACHE_LOCK` mutex to avoid reading and writing the same files at the same time. + pub async fn load_transactions_from_cache_concurrently( + &self, + tx_ids: I, + ) -> HashMap>> + where + I: IntoIterator, + { + let mutex = TX_CACHE_LOCK.mutex_by_ticker(&self.ticker); + let _lock = mutex.lock().await; - let path = cached_transaction_path(tx_cache_path, &tx.txid); - write_json(tx, &path, USE_TMP_FILE).await.mm_err(TxCacheError::from) -} + let it = tx_ids + .into_iter() + .map(|txid| self.load_transaction_from_cache(txid).map(move |res| (txid, res))); + futures::future::join_all(it).await.into_iter().collect() + } + + /// Uploads transactions to cache concurrently. + /// Note: this function locks the `TX_CACHE_LOCK` mutex and takes `txs` as the Hash map + /// to avoid reading and writing the same files at the same time. + pub async fn cache_transactions_concurrently(&self, txs: &HashMap) { + let mutex = TX_CACHE_LOCK.mutex_by_ticker(&self.ticker); + let _lock = mutex.lock().await; + + let it = txs.iter().map(|(_txid, tx)| self.cache_transaction(tx)); + futures::future::join_all(it) + .await + .into_iter() + .for_each(|tx| tx.error_log()); + } + + /// Tries to load transaction from cache. + /// Note: `tx.confirmations` can be out-of-date. + async fn load_transaction_from_cache(&self, txid: H256Json) -> TxCacheResult> { + let path = self.cached_transaction_path(&txid); + read_json(&path).await.mm_err(TxCacheError::from) + } + + /// Uploads transaction to cache. + async fn cache_transaction(&self, tx: &RpcTransaction) -> TxCacheResult<()> { + const USE_TMP_FILE: bool = true; + + let path = self.cached_transaction_path(&tx.txid); + write_json(tx, &path, USE_TMP_FILE).await.mm_err(TxCacheError::from) + } -fn cached_transaction_path(tx_cache_path: &Path, txid: &H256Json) -> PathBuf { - tx_cache_path.join(format!("{:?}", txid)) + fn cached_transaction_path(&self, txid: &H256Json) -> PathBuf { self.tx_cache_path.join(format!("{:?}", txid)) } } diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index a0fa074b15..4ea586b931 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -2,6 +2,7 @@ use crate::hd_wallet::{HDAccountsMap, HDAccountsMutex}; use crate::hd_wallet_storage::{HDWalletCoinStorage, HDWalletStorageError}; use crate::utxo::rpc_clients::{ElectrumClient, ElectrumClientImpl, ElectrumRpcRequest, EstimateFeeMethod, UtxoRpcClientEnum}; +use crate::utxo::tx_cache::UtxoVerboseCache; use crate::utxo::utxo_block_header_storage::{BlockHeaderStorage, InitBlockHeaderStorageOps}; use crate::utxo::utxo_builder::utxo_conf_builder::{UtxoConfBuilder, UtxoConfError, UtxoConfResult}; use crate::utxo::{output_script, utxo_common, ElectrumBuilderArgs, ElectrumProtoVerifier, RecentlySpentOutPoints, @@ -158,9 +159,10 @@ pub trait UtxoFieldsWithIguanaPrivKeyBuilder: UtxoCoinBuilderCommonOps { let dust_amount = self.dust_amount(); let initial_history_state = self.initial_history_state(); - let tx_cache_directory = Some(self.ctx().dbdir().join("TX_CACHE")); let tx_hash_algo = self.tx_hash_algo(); let check_utxo_maturity = self.check_utxo_maturity(); + // TX cache can be not initialized within tests only. + let tx_cache = Some(self.tx_cache()); let block_headers_storage = self.block_headers_storage()?; let coin = UtxoCoinFields { @@ -171,7 +173,7 @@ pub trait UtxoFieldsWithIguanaPrivKeyBuilder: UtxoCoinBuilderCommonOps { priv_key_policy, derivation_method, history_sync_state: Mutex::new(initial_history_state), - tx_cache_directory, + tx_cache, block_headers_storage, recently_spent_outpoints: AsyncMutex::new(RecentlySpentOutPoints::new(my_script_pubkey)), tx_fee, @@ -221,9 +223,10 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { let dust_amount = self.dust_amount(); let initial_history_state = self.initial_history_state(); - let tx_cache_directory = Some(self.ctx().dbdir().join("TX_CACHE")); let tx_hash_algo = self.tx_hash_algo(); let check_utxo_maturity = self.check_utxo_maturity(); + // TX cache can be not initialized within tests only. + let tx_cache = Some(self.tx_cache()); let block_headers_storage = self.block_headers_storage()?; let coin = UtxoCoinFields { @@ -235,7 +238,7 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { derivation_method: DerivationMethod::HDWallet(hd_wallet), history_sync_state: Mutex::new(initial_history_state), block_headers_storage, - tx_cache_directory, + tx_cache, recently_spent_outpoints, tx_fee, tx_hash_algo, @@ -560,6 +563,11 @@ pub trait UtxoCoinBuilderCommonOps { fn check_utxo_maturity(&self) -> bool { self.activation_params().check_utxo_maturity.unwrap_or_default() } fn is_hw_coin(&self, conf: &UtxoCoinConf) -> bool { conf.trezor_coin.is_some() } + + fn tx_cache(&self) -> UtxoVerboseCache { + let tx_cache_path = self.ctx().dbdir().join("TX_CACHE"); + UtxoVerboseCache::new(self.ticker().to_owned(), tx_cache_path) + } } /// Attempts to parse native daemon conf file and return rpcport, rpcuser and rpcpassword diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 0f5b9e583c..4d1abad0db 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2936,9 +2936,10 @@ pub async fn get_verbose_transactions_from_cache_or_rpc( let mut result_map = HashMap::with_capacity(tx_ids.len()); let mut to_request = Vec::with_capacity(tx_ids.len()); - match coin.tx_cache_directory { - Some(ref tx_cache_path) => { - tx_cache::load_transactions_from_cache_concurrently(tx_cache_path, tx_ids.into_iter()) + match coin.tx_cache { + Some(ref tx_cache) => { + tx_cache + .load_transactions_from_cache_concurrently(tx_ids.into_iter()) .await .into_iter() .for_each(|(txid, res)| on_cached_transaction_result(&mut result_map, &mut to_request, txid, res)); @@ -2981,8 +2982,8 @@ pub async fn get_verbose_transactions_from_cache_or_rpc( /// The function takes `txs` as the Hash map to avoid duplicating same transactions. #[cfg(not(target_arch = "wasm32"))] pub async fn cache_transactions_if_possible(coin: &UtxoCoinFields, txs: &HashMap) { - if let Some(ref tx_cache_path) = coin.tx_cache_directory { - tx_cache::cache_transactions_concurrently(tx_cache_path, txs).await + if let Some(ref tx_cache) = coin.tx_cache { + tx_cache.cache_transactions_concurrently(txs).await } } diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 33a9f06969..e84f801d88 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -150,7 +150,7 @@ fn utxo_coin_fields_for_test( priv_key_policy, derivation_method, history_sync_state: Mutex::new(HistorySyncState::NotEnabled), - tx_cache_directory: None, + tx_cache: None, block_headers_storage: None, recently_spent_outpoints: AsyncMutex::new(RecentlySpentOutPoints::new(my_script_pubkey)), tx_hash_algo: TxHashAlgo::DSHA256, From b0be26eac27136d3c47ff84d5b41e2dccb8afa87 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Sun, 1 May 2022 16:23:52 +0700 Subject: [PATCH 16/25] Add `DummyVerboseCache` that is used within tests * Rename `UtxoVerboseCache` to `FsVerboseCache` * Add `WasmVerboseCache` that is an alias to `DummyVerboseCache` --- mm2src/coins/qrc20.rs | 9 ++- mm2src/coins/utxo.rs | 10 ++-- mm2src/coins/utxo/tx_cache/dummy_tx_cache.rs | 20 +++++++ .../{tx_cache.rs => tx_cache/fs_tx_cache.rs} | 45 +++++--------- mm2src/coins/utxo/tx_cache/mod.rs | 49 ++++++++++++++++ .../utxo/utxo_builder/utxo_coin_builder.rs | 18 +++--- mm2src/coins/utxo/utxo_common.rs | 58 ++++--------------- mm2src/coins/utxo/utxo_tests.rs | 4 +- 8 files changed, 119 insertions(+), 94 deletions(-) create mode 100644 mm2src/coins/utxo/tx_cache/dummy_tx_cache.rs rename mm2src/coins/utxo/{tx_cache.rs => tx_cache/fs_tx_cache.rs} (75%) create mode 100644 mm2src/coins/utxo/tx_cache/mod.rs diff --git a/mm2src/coins/qrc20.rs b/mm2src/coins/qrc20.rs index 6fec696748..191f70fded 100644 --- a/mm2src/coins/qrc20.rs +++ b/mm2src/coins/qrc20.rs @@ -4,7 +4,8 @@ use crate::qrc20::rpc_clients::{LogEntry, Qrc20ElectrumOps, Qrc20NativeOps, Qrc2 use crate::utxo::qtum::QtumBasedCoin; use crate::utxo::rpc_clients::{ElectrumClient, NativeClient, UnspentInfo, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; -use crate::utxo::tx_cache::UtxoVerboseCache; +#[cfg(not(target_arch = "wasm32"))] +use crate::utxo::tx_cache::{UtxoVerboseCacheOps, UtxoVerboseCacheShared}; use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuildResult, UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPrivKeyBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::{self, big_decimal_from_sat, check_all_inputs_signed_by_pub, UtxoTxBuilder}; @@ -244,9 +245,11 @@ impl<'a> UtxoCoinBuilderCommonOps for Qrc20CoinBuilder<'a> { } /// Override [`UtxoCoinBuilderCommonOps::tx_cache`] to initialize TX cache with the platform ticker. - fn tx_cache(&self) -> UtxoVerboseCache { + /// Please note the method is overridden for Native mode only. + #[cfg(not(target_arch = "wasm32"))] + fn tx_cache(&self) -> UtxoVerboseCacheShared { let tx_cache_path = self.ctx().dbdir().join("TX_CACHE"); - UtxoVerboseCache::new(self.platform.clone(), tx_cache_path) + crate::utxo::tx_cache::fs_tx_cache::FsVerboseCache::new(self.platform.clone(), tx_cache_path).into_shared() } } diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 74dce3dbc0..fc59d5e384 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -75,8 +75,8 @@ use std::convert::TryInto; use std::hash::Hash; use std::num::NonZeroU64; use std::ops::Deref; -#[cfg(not(target_arch = "wasm32"))] use std::path::Path; -use std::path::PathBuf; +#[cfg(not(target_arch = "wasm32"))] +use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::atomic::{AtomicBool, AtomicU64}; use std::sync::{Arc, Mutex, Weak}; @@ -98,11 +98,11 @@ use super::{big_decimal_from_sat_unsigned, BalanceError, BalanceFut, BalanceResu use crate::coin_balance::{EnableCoinScanPolicy, HDAddressBalanceScanner}; use crate::hd_wallet::{HDAccountOps, HDAccountsMutex, HDAddress, HDWalletCoinOps, HDWalletOps, InvalidBip44ChainError}; use crate::hd_wallet_storage::{HDAccountStorageItem, HDWalletCoinStorage, HDWalletStorageError, HDWalletStorageResult}; -use crate::utxo::tx_cache::UtxoVerboseCache; +use crate::utxo::tx_cache::UtxoVerboseCacheShared; use crate::utxo::utxo_block_header_storage::BlockHeaderStorageError; use utxo_block_header_storage::BlockHeaderStorage; -#[cfg(not(target_arch = "wasm32"))] pub mod tx_cache; +pub mod tx_cache; #[cfg(target_arch = "wasm32")] pub mod utxo_indexedb_block_header_storage; #[cfg(not(target_arch = "wasm32"))] @@ -525,7 +525,7 @@ pub struct UtxoCoinFields { pub derivation_method: DerivationMethod, pub history_sync_state: Mutex, /// The cache of verbose transactions. - pub tx_cache: Option, + pub tx_cache: UtxoVerboseCacheShared, pub block_headers_storage: Option, /// The cache of recently send transactions used to track the spent UTXOs and replace them with new outputs /// The daemon needs some time to update the listunspent list for address which makes it return already spent UTXOs diff --git a/mm2src/coins/utxo/tx_cache/dummy_tx_cache.rs b/mm2src/coins/utxo/tx_cache/dummy_tx_cache.rs new file mode 100644 index 0000000000..e18244ca39 --- /dev/null +++ b/mm2src/coins/utxo/tx_cache/dummy_tx_cache.rs @@ -0,0 +1,20 @@ +use crate::utxo::tx_cache::{TxCacheResult, UtxoVerboseCacheOps}; +use async_trait::async_trait; +use rpc::v1::types::{Transaction as RpcTransaction, H256 as H256Json}; +use std::collections::{HashMap, HashSet}; + +/// The dummy TX cache. +#[derive(Debug, Default)] +pub struct DummyVerboseCache; + +#[async_trait] +impl UtxoVerboseCacheOps for DummyVerboseCache { + async fn load_transactions_from_cache_concurrently( + &self, + tx_ids: HashSet, + ) -> HashMap>> { + tx_ids.into_iter().map(|txid| (txid, Ok(None))).collect() + } + + async fn cache_transactions_concurrently(&self, _txs: &HashMap) {} +} diff --git a/mm2src/coins/utxo/tx_cache.rs b/mm2src/coins/utxo/tx_cache/fs_tx_cache.rs similarity index 75% rename from mm2src/coins/utxo/tx_cache.rs rename to mm2src/coins/utxo/tx_cache/fs_tx_cache.rs index ecf527988c..7b54172685 100644 --- a/mm2src/coins/utxo/tx_cache.rs +++ b/mm2src/coins/utxo/tx_cache/fs_tx_cache.rs @@ -1,30 +1,21 @@ +use crate::utxo::tx_cache::{TxCacheError, TxCacheResult, UtxoVerboseCacheOps}; +use async_trait::async_trait; use common::fs::{read_json, write_json, FsJsonError}; use common::log::LogOnError; use common::mm_error::prelude::*; -use derive_more::Display; use futures::lock::Mutex as AsyncMutex; use futures::FutureExt; use parking_lot::Mutex as PaMutex; use rpc::v1::types::{Transaction as RpcTransaction, H256 as H256Json}; use std::collections::hash_map::RawEntryMut; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use std::sync::Arc; -pub type TxCacheResult = MmResult; - lazy_static! { static ref TX_CACHE_LOCK: TxCacheLock = TxCacheLock::default(); } -#[derive(Debug, Display)] -pub enum TxCacheError { - ErrorLoading(String), - ErrorSaving(String), - ErrorDeserializing(String), - ErrorSerializing(String), -} - impl From for TxCacheError { fn from(e: FsJsonError) -> Self { match e { @@ -63,26 +54,17 @@ impl TxCacheLock { /// In particular, `QRC20` tokens have the same transactions as `Qtum` coin, /// so [`Qrc20Coin::platform_ticker`] is used as [`UtxoVerboseCache::ticker`]. #[derive(Debug)] -pub struct UtxoVerboseCache { +pub struct FsVerboseCache { ticker: String, tx_cache_path: PathBuf, } -impl UtxoVerboseCache { - pub fn new(ticker: String, tx_cache_path: PathBuf) -> UtxoVerboseCache { - UtxoVerboseCache { ticker, tx_cache_path } - } - - /// Tries to load transactions from cache concurrently. - /// Note 1: `tx.confirmations` can be out-of-date. - /// Note 2: this function locks the `TX_CACHE_LOCK` mutex to avoid reading and writing the same files at the same time. - pub async fn load_transactions_from_cache_concurrently( +#[async_trait] +impl UtxoVerboseCacheOps for FsVerboseCache { + async fn load_transactions_from_cache_concurrently( &self, - tx_ids: I, - ) -> HashMap>> - where - I: IntoIterator, - { + tx_ids: HashSet, + ) -> HashMap>> { let mutex = TX_CACHE_LOCK.mutex_by_ticker(&self.ticker); let _lock = mutex.lock().await; @@ -92,10 +74,7 @@ impl UtxoVerboseCache { futures::future::join_all(it).await.into_iter().collect() } - /// Uploads transactions to cache concurrently. - /// Note: this function locks the `TX_CACHE_LOCK` mutex and takes `txs` as the Hash map - /// to avoid reading and writing the same files at the same time. - pub async fn cache_transactions_concurrently(&self, txs: &HashMap) { + async fn cache_transactions_concurrently(&self, txs: &HashMap) { let mutex = TX_CACHE_LOCK.mutex_by_ticker(&self.ticker); let _lock = mutex.lock().await; @@ -105,6 +84,10 @@ impl UtxoVerboseCache { .into_iter() .for_each(|tx| tx.error_log()); } +} + +impl FsVerboseCache { + pub fn new(ticker: String, tx_cache_path: PathBuf) -> FsVerboseCache { FsVerboseCache { ticker, tx_cache_path } } /// Tries to load transaction from cache. /// Note: `tx.confirmations` can be out-of-date. diff --git a/mm2src/coins/utxo/tx_cache/mod.rs b/mm2src/coins/utxo/tx_cache/mod.rs new file mode 100644 index 0000000000..2b79cc5761 --- /dev/null +++ b/mm2src/coins/utxo/tx_cache/mod.rs @@ -0,0 +1,49 @@ +use async_trait::async_trait; +use common::mm_error::prelude::*; +use derive_more::Display; +use rpc::v1::types::{Transaction as RpcTransaction, H256 as H256Json}; +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::sync::Arc; + +pub mod dummy_tx_cache; +#[cfg(not(target_arch = "wasm32"))] pub mod fs_tx_cache; + +#[cfg(target_arch = "wasm32")] +pub mod wasm_tx_cache { + pub type WasmVerboseCache = crate::utxo::tx_cache::dummy_tx_cache::DummyVerboseCache; +} + +pub type TxCacheResult = MmResult; +pub type UtxoVerboseCacheShared = Arc; + +#[derive(Debug, Display)] +pub enum TxCacheError { + ErrorLoading(String), + ErrorSaving(String), + ErrorDeserializing(String), + ErrorSerializing(String), +} + +#[async_trait] +pub trait UtxoVerboseCacheOps: fmt::Debug { + fn into_shared(self) -> UtxoVerboseCacheShared + where + Self: Sized + Send + Sync + 'static, + { + Arc::new(self) + } + + /// Tries to load transactions from cache concurrently. + /// Note 1: `tx.confirmations` can be out-of-date. + /// Note 2: this function locks the `TX_CACHE_LOCK` mutex to avoid reading and writing the same files at the same time. + async fn load_transactions_from_cache_concurrently( + &self, + tx_ids: HashSet, + ) -> HashMap>>; + + /// Uploads transactions to cache concurrently. + /// Note: this function locks the `TX_CACHE_LOCK` mutex and takes `txs` as the Hash map + /// to avoid reading and writing the same files at the same time. + async fn cache_transactions_concurrently(&self, txs: &HashMap); +} diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 4ea586b931..3f7141dd61 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -2,7 +2,7 @@ use crate::hd_wallet::{HDAccountsMap, HDAccountsMutex}; use crate::hd_wallet_storage::{HDWalletCoinStorage, HDWalletStorageError}; use crate::utxo::rpc_clients::{ElectrumClient, ElectrumClientImpl, ElectrumRpcRequest, EstimateFeeMethod, UtxoRpcClientEnum}; -use crate::utxo::tx_cache::UtxoVerboseCache; +use crate::utxo::tx_cache::{UtxoVerboseCacheOps, UtxoVerboseCacheShared}; use crate::utxo::utxo_block_header_storage::{BlockHeaderStorage, InitBlockHeaderStorageOps}; use crate::utxo::utxo_builder::utxo_conf_builder::{UtxoConfBuilder, UtxoConfError, UtxoConfResult}; use crate::utxo::{output_script, utxo_common, ElectrumBuilderArgs, ElectrumProtoVerifier, RecentlySpentOutPoints, @@ -161,8 +161,7 @@ pub trait UtxoFieldsWithIguanaPrivKeyBuilder: UtxoCoinBuilderCommonOps { let initial_history_state = self.initial_history_state(); let tx_hash_algo = self.tx_hash_algo(); let check_utxo_maturity = self.check_utxo_maturity(); - // TX cache can be not initialized within tests only. - let tx_cache = Some(self.tx_cache()); + let tx_cache = self.tx_cache(); let block_headers_storage = self.block_headers_storage()?; let coin = UtxoCoinFields { @@ -225,8 +224,7 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { let initial_history_state = self.initial_history_state(); let tx_hash_algo = self.tx_hash_algo(); let check_utxo_maturity = self.check_utxo_maturity(); - // TX cache can be not initialized within tests only. - let tx_cache = Some(self.tx_cache()); + let tx_cache = self.tx_cache(); let block_headers_storage = self.block_headers_storage()?; let coin = UtxoCoinFields { @@ -564,9 +562,15 @@ pub trait UtxoCoinBuilderCommonOps { fn is_hw_coin(&self, conf: &UtxoCoinConf) -> bool { conf.trezor_coin.is_some() } - fn tx_cache(&self) -> UtxoVerboseCache { + #[cfg(target_arch = "wasm32")] + fn tx_cache(&self) -> UtxoVerboseCacheShared { + crate::utxo::tx_cache::wasm_tx_cache::WasmVerboseCache::default().into_shared() + } + + #[cfg(not(target_arch = "wasm32"))] + fn tx_cache(&self) -> UtxoVerboseCacheShared { let tx_cache_path = self.ctx().dbdir().join("TX_CACHE"); - UtxoVerboseCache::new(self.ticker().to_owned(), tx_cache_path) + crate::utxo::tx_cache::fs_tx_cache::FsVerboseCache::new(self.ticker().to_owned(), tx_cache_path).into_shared() } } diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 4d1abad0db..94f7663962 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -7,6 +7,7 @@ use crate::hd_wallet_storage::{HDWalletCoinWithStorageOps, HDWalletStorageResult use crate::rpc_command::init_withdraw::WithdrawTaskHandle; use crate::utxo::rpc_clients::{electrum_script_hash, BlockHashOrHeight, UnspentInfo, UnspentMap, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcResult}; +use crate::utxo::tx_cache::TxCacheResult; use crate::utxo::utxo_withdraw::{InitUtxoWithdraw, StandardUtxoWithdraw, UtxoWithdraw}; use crate::{CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, GetWithdrawSenderAddress, HDAddressId, RawTransactionError, RawTransactionRequest, RawTransactionRes, TradePreimageValue, TxFeeDetails, @@ -2890,7 +2891,10 @@ where } } - cache_transactions_if_possible(coin.as_ref(), &txs_to_cache).await; + coin.as_ref() + .tx_cache + .cache_transactions_concurrently(&txs_to_cache) + .await; Ok(result) } @@ -2901,7 +2905,6 @@ pub fn is_unspent_mature(mature_confirmations: u32, output: &RpcTransaction) -> /// [`UtxoCommonOps::get_verbose_transactions_from_cache_or_rpc`] implementation. /// Loads verbose transactions from cache or requests it using RPC client. -#[cfg(not(target_arch = "wasm32"))] pub async fn get_verbose_transactions_from_cache_or_rpc( coin: &UtxoCoinFields, tx_ids: HashSet, @@ -2913,7 +2916,7 @@ pub async fn get_verbose_transactions_from_cache_or_rpc( result_map: &mut HashMap, to_request: &mut Vec, txid: H256Json, - res: tx_cache::TxCacheResult>, + res: TxCacheResult>, ) { match res { Ok(Some(tx)) => { @@ -2936,17 +2939,11 @@ pub async fn get_verbose_transactions_from_cache_or_rpc( let mut result_map = HashMap::with_capacity(tx_ids.len()); let mut to_request = Vec::with_capacity(tx_ids.len()); - match coin.tx_cache { - Some(ref tx_cache) => { - tx_cache - .load_transactions_from_cache_concurrently(tx_ids.into_iter()) - .await - .into_iter() - .for_each(|(txid, res)| on_cached_transaction_result(&mut result_map, &mut to_request, txid, res)); - }, - // the coin doesn't support TX local cache, don't try to load from cache - None => to_request = tx_ids.into_iter().collect(), - } + coin.tx_cache + .load_transactions_from_cache_concurrently(tx_ids) + .await + .into_iter() + .for_each(|(txid, res)| on_cached_transaction_result(&mut result_map, &mut to_request, txid, res)); result_map.extend( coin.rpc_client @@ -2959,39 +2956,6 @@ pub async fn get_verbose_transactions_from_cache_or_rpc( Ok(result_map) } -/// [`UtxoCommonOps::get_verbose_transactions_from_cache_or_rpc`] implementation. -/// Loads verbose transactions from cache or requests it using RPC client. -#[cfg(target_arch = "wasm32")] -pub async fn get_verbose_transactions_from_cache_or_rpc( - coin: &UtxoCoinFields, - tx_ids: HashSet, -) -> UtxoRpcResult> { - let tx_ids: Vec<_> = tx_ids.into_iter().collect(); - let txs = coin - .rpc_client - .get_verbose_transactions(&tx_ids) - .compat() - .await? - .into_iter() - .map(|tx| (tx.txid, VerboseTransactionFrom::Rpc(tx))) - .collect(); - Ok(txs) -} - -/// Saves the given `txs` transactions in a TX cache. -/// The function takes `txs` as the Hash map to avoid duplicating same transactions. -#[cfg(not(target_arch = "wasm32"))] -pub async fn cache_transactions_if_possible(coin: &UtxoCoinFields, txs: &HashMap) { - if let Some(ref tx_cache) = coin.tx_cache { - tx_cache.cache_transactions_concurrently(txs).await - } -} - -/// Saves the given `txs` transactions in a TX cache. -/// It takes `txs` as the Hash map to avoid duplicating same transactions. -#[cfg(target_arch = "wasm32")] -pub async fn cache_transactions_if_possible(_coin: &UtxoCoinFields, _txs: &HashMap) {} - /// Swap contract address is not used by standard UTXO coins. pub fn swap_contract_address() -> Option { None } diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index e84f801d88..07c5ba201b 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -10,6 +10,8 @@ use crate::utxo::rpc_clients::{BlockHashOrHeight, ElectrumBalance, ElectrumClien GetAddressInfoRes, ListSinceBlockRes, ListTransactionsItem, NativeClient, NativeClientImpl, NativeUnspent, NetworkInfo, UtxoRpcClientOps, ValidateAddressRes, VerboseBlock}; +use crate::utxo::tx_cache::dummy_tx_cache::DummyVerboseCache; +use crate::utxo::tx_cache::UtxoVerboseCacheOps; use crate::utxo::utxo_builder::{UtxoArcBuilder, UtxoCoinBuilderCommonOps}; use crate::utxo::utxo_common::UtxoTxBuilder; use crate::utxo::utxo_common_tests; @@ -150,7 +152,7 @@ fn utxo_coin_fields_for_test( priv_key_policy, derivation_method, history_sync_state: Mutex::new(HistorySyncState::NotEnabled), - tx_cache: None, + tx_cache: DummyVerboseCache::default().into_shared(), block_headers_storage: None, recently_spent_outpoints: AsyncMutex::new(RecentlySpentOutPoints::new(my_script_pubkey)), tx_hash_algo: TxHashAlgo::DSHA256, From 64d2cc38837da342f70c9261eb15a3b33b41fab6 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Sun, 1 May 2022 18:38:44 +0700 Subject: [PATCH 17/25] Minor changes --- mm2src/coins/utxo/tx_cache/fs_tx_cache.rs | 1 + mm2src/coins/utxo/tx_cache/mod.rs | 5 +- mm2src/common/custom_iter.rs | 65 ++++++++++++----------- 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/mm2src/coins/utxo/tx_cache/fs_tx_cache.rs b/mm2src/coins/utxo/tx_cache/fs_tx_cache.rs index 7b54172685..98d02fdc75 100644 --- a/mm2src/coins/utxo/tx_cache/fs_tx_cache.rs +++ b/mm2src/coins/utxo/tx_cache/fs_tx_cache.rs @@ -27,6 +27,7 @@ impl From for TxCacheError { } } +/// The cache lock is used to avoid reading and writing the same files at the same time. #[derive(Default)] struct TxCacheLock { /// The collection of `Ticker -> Mutex` pairs. diff --git a/mm2src/coins/utxo/tx_cache/mod.rs b/mm2src/coins/utxo/tx_cache/mod.rs index 2b79cc5761..da556510e4 100644 --- a/mm2src/coins/utxo/tx_cache/mod.rs +++ b/mm2src/coins/utxo/tx_cache/mod.rs @@ -35,15 +35,12 @@ pub trait UtxoVerboseCacheOps: fmt::Debug { } /// Tries to load transactions from cache concurrently. - /// Note 1: `tx.confirmations` can be out-of-date. - /// Note 2: this function locks the `TX_CACHE_LOCK` mutex to avoid reading and writing the same files at the same time. + /// Please note `tx.confirmations` can be out-of-date. async fn load_transactions_from_cache_concurrently( &self, tx_ids: HashSet, ) -> HashMap>>; /// Uploads transactions to cache concurrently. - /// Note: this function locks the `TX_CACHE_LOCK` mutex and takes `txs` as the Hash map - /// to avoid reading and writing the same files at the same time. async fn cache_transactions_concurrently(&self, txs: &HashMap); } diff --git a/mm2src/common/custom_iter.rs b/mm2src/common/custom_iter.rs index 0b7f62e203..b38cd628a9 100644 --- a/mm2src/common/custom_iter.rs +++ b/mm2src/common/custom_iter.rs @@ -69,38 +69,43 @@ where impl TryUnzip for T where T: Iterator> {} -#[test] -fn test_collect_into() { - let actual: Vec = vec!["foo", "bar"].collect_into(); - let expected = vec!["foo".to_owned(), "bar".to_owned()]; - assert_eq!(actual, expected); -} +#[cfg(test)] +mod tests { + use super::*; -#[test] -fn test_try_into_group_map() { - let actual: Result<_, &'static str> = vec![Ok(("foo", 1)), Ok(("bar", 2)), Ok(("foo", 3))] - .into_iter() - .try_into_group_map(); - let expected: HashMap<_, _> = vec![("foo", vec![1, 3]), ("bar", vec![2])].into_iter().collect(); - assert_eq!(actual, Ok(expected)); + #[test] + fn test_collect_into() { + let actual: Vec = vec!["foo", "bar"].collect_into(); + let expected = vec!["foo".to_owned(), "bar".to_owned()]; + assert_eq!(actual, expected); + } - let err = vec![Ok(("foo", 1)), Ok(("bar", 2)), Err("Error"), Ok(("foo", 3))] - .into_iter() - .try_into_group_map() - .unwrap_err(); - assert_eq!(err, "Error"); -} + #[test] + fn test_try_into_group_map() { + let actual: Result<_, &'static str> = vec![Ok(("foo", 1)), Ok(("bar", 2)), Ok(("foo", 3))] + .into_iter() + .try_into_group_map(); + let expected: HashMap<_, _> = vec![("foo", vec![1, 3]), ("bar", vec![2])].into_iter().collect(); + assert_eq!(actual, Ok(expected)); -#[test] -fn test_try_unzip() { - let actual: Result<(Vec<_>, Vec<_>), &'static str> = vec![Ok(("foo", 1)), Ok(("bar", 2)), Ok(("foo", 3))] - .into_iter() - .try_unzip(); - assert_eq!(actual, Ok((vec!["foo", "bar", "foo"], vec![1, 2, 3]))); + let err = vec![Ok(("foo", 1)), Ok(("bar", 2)), Err("Error"), Ok(("foo", 3))] + .into_iter() + .try_into_group_map() + .unwrap_err(); + assert_eq!(err, "Error"); + } + + #[test] + fn test_try_unzip() { + let actual: Result<(Vec<_>, Vec<_>), &'static str> = vec![Ok(("foo", 1)), Ok(("bar", 2)), Ok(("foo", 3))] + .into_iter() + .try_unzip(); + assert_eq!(actual, Ok((vec!["foo", "bar", "foo"], vec![1, 2, 3]))); - let err = vec![Ok(("foo", 1)), Ok(("bar", 2)), Err("Error"), Ok(("foo", 3))] - .into_iter() - .try_unzip::, Vec<_>>() - .unwrap_err(); - assert_eq!(err, "Error"); + let err = vec![Ok(("foo", 1)), Ok(("bar", 2)), Err("Error"), Ok(("foo", 3))] + .into_iter() + .try_unzip::, Vec<_>>() + .unwrap_err(); + assert_eq!(err, "Error"); + } } From d9004d60cc072495586351d8631710a8f2b11cdd Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Mon, 2 May 2022 18:19:14 +0700 Subject: [PATCH 18/25] Fix PR issues * Refactor `BchRequest` trait into `JsonRpcBatchClient` * Move `send_batch_request` from `JsonRpcClient` to `JsonRpcBatchClient` * Refactor `BchCoin::utxos_into_bch_unspents` --- mm2src/coins/utxo/bch.rs | 21 +++++------ mm2src/coins/utxo/rpc_clients.rs | 64 +++++++++++++++----------------- mm2src/common/jsonrpc_client.rs | 44 +++++++++------------- 3 files changed, 57 insertions(+), 72 deletions(-) diff --git a/mm2src/coins/utxo/bch.rs b/mm2src/coins/utxo/bch.rs index eab88b6d31..cbbad5213e 100644 --- a/mm2src/coins/utxo/bch.rs +++ b/mm2src/coins/utxo/bch.rs @@ -186,33 +186,32 @@ impl BchCoin { pub fn bchd_urls(&self) -> &[String] { &self.bchd_urls } async fn utxos_into_bch_unspents(&self, utxos: Vec) -> UtxoRpcResult { + let mut result = BchUnspents::default(); + let mut temporary_undetermined = Vec::new(); + let to_verbose: HashSet = utxos - .iter() + .into_iter() .filter_map(|unspent| { if unspent.outpoint.index == 0 { // Zero output is reserved for OP_RETURN of specific protocols // so if we get it we can safely consider this as standard BCH UTXO. // There is no need to request verbose transaction for such UTXO. + result.add_standard(unspent); None } else { - Some(unspent.outpoint.hash.reversed().into()) + let hash = unspent.outpoint.hash.reversed().into(); + temporary_undetermined.push(unspent); + Some(hash) } }) .collect(); + let verbose_txs = self .get_verbose_transactions_from_cache_or_rpc(to_verbose) .compat() .await?; - let mut result = BchUnspents::default(); - for unspent in utxos { - if unspent.outpoint.index == 0 { - // Zero output is reserved for OP_RETURN of specific protocols - // so if we get it we can safely consider this as standard BCH UTXO. - result.add_standard(unspent); - continue; - } - + for unspent in temporary_undetermined { let prev_tx_hash = unspent.outpoint.hash.reversed().into(); let prev_tx_bytes = verbose_txs .get(&prev_tx_hash) diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index 4c2c6d6cdd..7296431fb8 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -9,7 +9,7 @@ use chain::{BlockHeader, BlockHeaderBits, BlockHeaderNonce, OutPoint, Transactio use common::custom_futures::{select_ok_sequential, FutureTimerExt}; use common::custom_iter::{CollectInto, TryIntoGroupMap}; use common::executor::{spawn, Timer}; -use common::jsonrpc_client::{BatchRequest, JsonRpcBatchResponse, JsonRpcClient, JsonRpcError, JsonRpcErrorType, +use common::jsonrpc_client::{JsonRpcBatchClient, JsonRpcBatchResponse, JsonRpcClient, JsonRpcError, JsonRpcErrorType, JsonRpcId, JsonRpcMultiClient, JsonRpcRemoteAddr, JsonRpcRequest, JsonRpcRequestEnum, JsonRpcResponse, JsonRpcResponseEnum, JsonRpcResponseFut, RpcRes}; use common::log::{error, info, warn}; @@ -636,6 +636,8 @@ impl JsonRpcClient for NativeClientImpl { } } +impl JsonRpcBatchClient for NativeClientImpl {} + // if mockable is placed before async_trait there is `munmap_chunk(): invalid pointer` error on async fn mocking attempt #[async_trait] #[cfg_attr(test, mockable)] @@ -977,12 +979,10 @@ impl NativeClientImpl { /// Always returns verbose transactions in the same order they were requested. fn get_raw_transaction_verbose_batch(&self, tx_ids: &[H256Json]) -> RpcRes> { let verbose = 1; - Box::new( - tx_ids - .iter() - .map(|txid| rpc_req!(self, "getrawtransaction", txid, verbose)) - .batch_rpc(self), - ) + let requests = tx_ids + .iter() + .map(|txid| rpc_req!(self, "getrawtransaction", txid, verbose)); + self.batch_rpc(requests) } /// https://developer.bitcoin.org/reference/rpc/getrawtransaction.html @@ -1699,6 +1699,8 @@ impl JsonRpcClient for ElectrumClient { } } +impl JsonRpcBatchClient for ElectrumClient {} + impl JsonRpcMultiClient for ElectrumClient { fn transport_exact(&self, to_addr: String, request: JsonRpcRequestEnum) -> JsonRpcResponseFut { Box::new(electrum_request_to(self.clone(), request, to_addr).boxed().compat()) @@ -1751,24 +1753,21 @@ impl ElectrumClient { /// We should remove them to build valid transactions. /// Please note the function returns `ScriptHashUnspents` elements in the same order in which they were requested. pub fn scripthash_list_unspent_batch(&self, hashes: Vec) -> RpcRes> { - Box::new( - hashes - .iter() - .map(|hash| rpc_req!(self, "blockchain.scripthash.listunspent", hash)) - .batch_rpc(self) - .map(move |unspents: Vec| { - unspents + let requests = hashes + .iter() + .map(|hash| rpc_req!(self, "blockchain.scripthash.listunspent", hash)); + Box::new(self.batch_rpc(requests).map(move |unspents: Vec| { + unspents + .into_iter() + .map(|hash_unspents| { + let hash_unspents: Vec<_> = hash_unspents .into_iter() - .map(|hash_unspents| { - let hash_unspents: Vec<_> = hash_unspents - .into_iter() - .unique_by(|unspent| (unspent.tx_hash, unspent.tx_pos)) - .collect(); - hash_unspents - }) - .collect() - }), - ) + .unique_by(|unspent| (unspent.tx_hash, unspent.tx_pos)) + .collect(); + hash_unspents + }) + .collect() + })) } /// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-history @@ -1793,10 +1792,10 @@ impl ElectrumClient { where I: IntoIterator, { - hashes + let requests = hashes .into_iter() - .map(|hash| rpc_req!(self, "blockchain.scripthash.get_balance", &hash)) - .batch_rpc(self) + .map(|hash| rpc_req!(self, "blockchain.scripthash.get_balance", &hash)); + self.batch_rpc(requests) } /// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#blockchain-headers-subscribe @@ -1971,13 +1970,10 @@ impl UtxoRpcClientOps for ElectrumClient { /// Returns verbose transactions in a batch. fn get_verbose_transactions(&self, tx_ids: &[H256Json]) -> UtxoRpcFut> { let verbose = true; - Box::new( - tx_ids - .iter() - .map(|txid| rpc_req!(self, "blockchain.transaction.get", txid, verbose)) - .batch_rpc(self) - .map_to_mm_fut(UtxoRpcError::from), - ) + let requests = tx_ids + .iter() + .map(|txid| rpc_req!(self, "blockchain.transaction.get", txid, verbose)); + Box::new(self.batch_rpc(requests).map_to_mm_fut(UtxoRpcError::from)) } fn get_block_count(&self) -> UtxoRpcFut { diff --git a/mm2src/common/jsonrpc_client.rs b/mm2src/common/jsonrpc_client.rs index 7f60aae032..4d87d5db97 100644 --- a/mm2src/common/jsonrpc_client.rs +++ b/mm2src/common/jsonrpc_client.rs @@ -51,33 +51,6 @@ pub type JsonRpcResponseFut = Box + Send + 'static>; pub type RpcRes = Box + Send + 'static>; -pub trait BatchRequest { - /// Sends the RPC batch request. - /// Responses are guaranteed to be in the same order in which they were requested. - fn batch_rpc(self, rpc_client: &Rpc) -> RpcRes> - where - Rpc: JsonRpcClient, - T: DeserializeOwned + Send + 'static; -} - -impl BatchRequest for I -where - I: Iterator, -{ - fn batch_rpc(self, rpc_client: &Rpc) -> RpcRes> - where - Rpc: JsonRpcClient, - T: DeserializeOwned + Send + 'static, - { - let requests: Vec<_> = self.collect(); - if requests.is_empty() { - // Return an empty vector of results. - return Box::new(futures01::future::ok(Vec::new())); - } - rpc_client.send_batch_request(JsonRpcBatchRequest(requests)) - } -} - /// Address of server from which an Rpc response was received #[derive(Clone, Default)] pub struct JsonRpcRemoteAddr(pub String); @@ -292,6 +265,23 @@ pub trait JsonRpcClient { .then(move |result| process_transport_single_result(result, client_info, request)), ) } +} + +pub trait JsonRpcBatchClient: JsonRpcClient { + /// Sends the RPC batch request. + /// Responses are guaranteed to be in the same order in which they were requested. + fn batch_rpc(&self, batch_requests: I) -> RpcRes> + where + I: IntoIterator, + T: DeserializeOwned + Send + 'static, + { + let requests: Vec<_> = batch_requests.into_iter().collect(); + if requests.is_empty() { + // Return an empty vector of results. + return Box::new(futures01::future::ok(Vec::new())); + } + self.send_batch_request(JsonRpcBatchRequest(requests)) + } /// Sends the given batch `request` to the remote and tries to decode the batch response into `Vec`. /// Responses are guaranteed to be in the same order in which they were requested. From 7224b85e39a2406b5b9e8cb9a8e69cd59fda514f Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Tue, 3 May 2022 11:42:01 +0700 Subject: [PATCH 19/25] Fix PR issues * Move `address_balance_from_unspent_map` outside the `NativeClient::display_balance` function --- mm2src/coins/utxo/rpc_clients.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index 7296431fb8..2ca123927f 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -749,18 +749,6 @@ impl UtxoRpcClientOps for NativeClient { } fn display_balances(&self, addresses: Vec
, decimals: u8) -> UtxoRpcFut> { - fn address_balance_from_unspent_map(address: &Address, unspent_map: &UnspentMap, decimals: u8) -> BigDecimal { - let unspents = match unspent_map.get(address) { - Some(unspents) => unspents, - // If `balances` doesn't contain `address`, there are no unspents related to the address. - // Consider the balance of that address equal to 0. - None => return BigDecimal::from(0), - }; - unspents.iter().fold(BigDecimal::from(0), |sum, unspent| { - sum + big_decimal_from_sat_unsigned(unspent.value, decimals) - }) - } - let this = self.clone(); let fut = async move { let unspent_map = this.list_unspent_group(addresses.clone(), decimals).compat().await?; @@ -2581,3 +2569,15 @@ fn electrum_request( .map_err(|e| ERRL!("{}", e)); Box::new(send_fut) } + +fn address_balance_from_unspent_map(address: &Address, unspent_map: &UnspentMap, decimals: u8) -> BigDecimal { + let unspents = match unspent_map.get(address) { + Some(unspents) => unspents, + // If `balances` doesn't contain `address`, there are no unspents related to the address. + // Consider the balance of that address equal to 0. + None => return BigDecimal::from(0), + }; + unspents.iter().fold(BigDecimal::from(0), |sum, unspent| { + sum + big_decimal_from_sat_unsigned(unspent.value, decimals) + }) +} From 537a3f21abb5b8d89952b11c83fd20adbcebc0a2 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Wed, 4 May 2022 17:10:04 +0700 Subject: [PATCH 20/25] Fix PR issues * Avoid adding unnecessary variables * Add `inline` attribute where it's needed --- mm2src/coins/qrc20.rs | 5 ++-- mm2src/coins/rpc_command/account_balance.rs | 9 ++++---- .../hd_account_balance_rpc_error.rs | 3 +-- .../init_scan_for_new_addresses.rs | 1 + mm2src/coins/utxo.rs | 5 ++++ mm2src/coins/utxo/rpc_clients.rs | 10 +++++--- mm2src/coins/utxo/tx_cache/fs_tx_cache.rs | 2 ++ mm2src/coins/utxo/tx_cache/mod.rs | 1 + .../utxo/utxo_builder/utxo_coin_builder.rs | 23 +++++++++++++++++-- mm2src/common/custom_iter.rs | 1 + mm2src/common/jsonrpc_client.rs | 13 +++++++++++ 11 files changed, 59 insertions(+), 14 deletions(-) diff --git a/mm2src/coins/qrc20.rs b/mm2src/coins/qrc20.rs index 191f70fded..413f871e02 100644 --- a/mm2src/coins/qrc20.rs +++ b/mm2src/coins/qrc20.rs @@ -246,10 +246,11 @@ impl<'a> UtxoCoinBuilderCommonOps for Qrc20CoinBuilder<'a> { /// Override [`UtxoCoinBuilderCommonOps::tx_cache`] to initialize TX cache with the platform ticker. /// Please note the method is overridden for Native mode only. + #[inline(always)] #[cfg(not(target_arch = "wasm32"))] fn tx_cache(&self) -> UtxoVerboseCacheShared { - let tx_cache_path = self.ctx().dbdir().join("TX_CACHE"); - crate::utxo::tx_cache::fs_tx_cache::FsVerboseCache::new(self.platform.clone(), tx_cache_path).into_shared() + crate::utxo::tx_cache::fs_tx_cache::FsVerboseCache::new(self.platform.clone(), self.tx_cache_path()) + .into_shared() } } diff --git a/mm2src/coins/rpc_command/account_balance.rs b/mm2src/coins/rpc_command/account_balance.rs index 80b1aa2fe9..cf5fe837df 100644 --- a/mm2src/coins/rpc_command/account_balance.rs +++ b/mm2src/coins/rpc_command/account_balance.rs @@ -51,8 +51,7 @@ pub async fn account_balance( ctx: MmArc, req: HDAccountBalanceRequest, ) -> MmResult { - let coin = lp_coinfind_or_err(&ctx, &req.coin).await?; - match coin { + match lp_coinfind_or_err(&ctx, &req.coin).await? { MmCoinEnum::UtxoCoin(utxo) => utxo.account_balance_rpc(req.params).await, MmCoinEnum::QtumCoin(qtum) => qtum.account_balance_rpc(req.params).await, _ => MmError::err(HDAccountBalanceRpcError::CoinIsActivatedNotWithHDWallet), @@ -73,11 +72,11 @@ pub mod common_impl { Coin: HDWalletBalanceOps + CoinWithDerivationMethod::HDWallet> + Sync, ::Address: fmt::Display + Clone, { - let hd_wallet = coin.derivation_method().hd_wallet_or_err()?; - let account_id = params.account_index; let chain = params.chain; - let hd_account = hd_wallet + let hd_account = coin + .derivation_method() + .hd_wallet_or_err()? .get_account(account_id) .await .or_mm_err(|| HDAccountBalanceRpcError::UnknownAccount { account_id })?; diff --git a/mm2src/coins/rpc_command/hd_account_balance_rpc_error.rs b/mm2src/coins/rpc_command/hd_account_balance_rpc_error.rs index e47191293e..851d2b42d8 100644 --- a/mm2src/coins/rpc_command/hd_account_balance_rpc_error.rs +++ b/mm2src/coins/rpc_command/hd_account_balance_rpc_error.rs @@ -94,12 +94,11 @@ impl From for HDAccountBalanceRpcError { impl From for HDAccountBalanceRpcError { fn from(e: RpcTaskError) -> Self { - let error = e.to_string(); match e { RpcTaskError::Canceled => HDAccountBalanceRpcError::Internal("Canceled".to_owned()), RpcTaskError::Timeout(timeout) => HDAccountBalanceRpcError::Timeout(timeout), RpcTaskError::NoSuchTask(_) | RpcTaskError::UnexpectedTaskStatus { .. } => { - HDAccountBalanceRpcError::Internal(error) + HDAccountBalanceRpcError::Internal(e.to_string()) }, RpcTaskError::Internal(internal) => HDAccountBalanceRpcError::Internal(internal), } diff --git a/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs b/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs index e26a9a95a7..52a77fdbe1 100644 --- a/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs +++ b/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs @@ -76,6 +76,7 @@ impl RpcTaskTypes for InitScanAddressesTask { #[async_trait] impl RpcTask for InitScanAddressesTask { + #[inline(always)] fn initial_status(&self) -> Self::InProgressStatus { ScanAddressesInProgressStatus::InProgress } async fn run(self, _task_handle: &ScanAddressesTaskHandle) -> Result> { diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index fc59d5e384..9503dde4f7 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -750,6 +750,7 @@ pub struct MatureUnspentList { } impl MatureUnspentList { + #[inline(always)] pub fn with_capacity(capacity: usize) -> MatureUnspentList { MatureUnspentList { mature: Vec::with_capacity(capacity), @@ -757,6 +758,7 @@ impl MatureUnspentList { } } + #[inline(always)] pub fn new_mature(mature: Vec) -> MatureUnspentList { MatureUnspentList { mature, @@ -764,8 +766,10 @@ impl MatureUnspentList { } } + #[inline(always)] pub fn only_mature(self) -> Vec { self.mature } + #[inline(always)] pub fn to_coin_balance(&self, decimals: u8) -> CoinBalance { let fold = |acc: BigDecimal, x: &UnspentInfo| acc + big_decimal_from_sat_unsigned(x.value, decimals); CoinBalance { @@ -1067,6 +1071,7 @@ pub enum VerboseTransactionFrom { } impl VerboseTransactionFrom { + #[inline(always)] fn to_inner(&self) -> &RpcTransaction { match self { VerboseTransactionFrom::Rpc(tx) | VerboseTransactionFrom::Cache(tx) => tx, diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index 2ca123927f..dd8cea4edb 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -1174,12 +1174,14 @@ impl ElectrumBlockHeaderV12 { } } + #[inline(always)] pub fn as_hex(&self) -> String { let block_header = self.as_block_header(); let serialized = serialize(&block_header); hex::encode(serialized) } + #[inline(always)] pub fn hash(&self) -> H256Json { let block_header = self.as_block_header(); BlockHeader::hash(&block_header).into() @@ -1265,18 +1267,21 @@ pub struct ElectrumBalance { } impl ElectrumBalance { + #[inline(always)] pub fn to_big_decimal(&self, decimals: u8) -> BigDecimal { let balance_sat = BigInt::from(self.confirmed) + BigInt::from(self.unconfirmed); BigDecimal::from(balance_sat) / BigDecimal::from(10u64.pow(decimals as u32)) } } +#[inline(always)] fn sha_256(input: &[u8]) -> Vec { let mut sha = Sha256::new(); sha.input(input); sha.result().to_vec() } +#[inline(always)] pub fn electrum_script_hash(script: &[u8]) -> Vec { let mut result = sha_256(script); result.reverse(); @@ -1748,11 +1753,10 @@ impl ElectrumClient { unspents .into_iter() .map(|hash_unspents| { - let hash_unspents: Vec<_> = hash_unspents + hash_unspents .into_iter() .unique_by(|unspent| (unspent.tx_hash, unspent.tx_pos)) - .collect(); - hash_unspents + .collect::>() }) .collect() })) diff --git a/mm2src/coins/utxo/tx_cache/fs_tx_cache.rs b/mm2src/coins/utxo/tx_cache/fs_tx_cache.rs index 98d02fdc75..d9072fbc18 100644 --- a/mm2src/coins/utxo/tx_cache/fs_tx_cache.rs +++ b/mm2src/coins/utxo/tx_cache/fs_tx_cache.rs @@ -88,6 +88,7 @@ impl UtxoVerboseCacheOps for FsVerboseCache { } impl FsVerboseCache { + #[inline(always)] pub fn new(ticker: String, tx_cache_path: PathBuf) -> FsVerboseCache { FsVerboseCache { ticker, tx_cache_path } } /// Tries to load transaction from cache. @@ -105,5 +106,6 @@ impl FsVerboseCache { write_json(tx, &path, USE_TMP_FILE).await.mm_err(TxCacheError::from) } + #[inline(always)] fn cached_transaction_path(&self, txid: &H256Json) -> PathBuf { self.tx_cache_path.join(format!("{:?}", txid)) } } diff --git a/mm2src/coins/utxo/tx_cache/mod.rs b/mm2src/coins/utxo/tx_cache/mod.rs index da556510e4..a64200598d 100644 --- a/mm2src/coins/utxo/tx_cache/mod.rs +++ b/mm2src/coins/utxo/tx_cache/mod.rs @@ -27,6 +27,7 @@ pub enum TxCacheError { #[async_trait] pub trait UtxoVerboseCacheOps: fmt::Debug { + #[inline(always)] fn into_shared(self) -> UtxoVerboseCacheShared where Self: Sized + Send + Sync + 'static, diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 3f7141dd61..9322d5a616 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -255,6 +255,7 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { .mm_err(UtxoCoinBuildError::from) } + #[inline(always)] fn derivation_path(&self) -> UtxoConfResult { if self.conf()["derivation_path"].is_null() { return MmError::err(UtxoConfError::DerivationPathIsNotSet); @@ -263,10 +264,13 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { .map_to_mm(|e| UtxoConfError::ErrorDeserializingDerivationPath(e.to_string())) } + #[inline(always)] fn gap_limit(&self) -> u32 { self.activation_params().gap_limit.unwrap_or(DEFAULT_GAP_LIMIT) } + #[inline(always)] fn supports_trezor(&self, conf: &UtxoCoinConf) -> bool { conf.trezor_coin.is_some() } + #[inline(always)] fn check_if_trezor_is_initialized(&self) -> UtxoCoinBuildResult<()> { let crypto_ctx = CryptoCtx::from_ctx(self.ctx())?; let hw_ctx = crypto_ctx @@ -288,6 +292,7 @@ pub trait UtxoCoinBuilderCommonOps { fn ticker(&self) -> &str; + #[inline(always)] fn block_headers_storage(&self) -> UtxoCoinBuildResult> { let params: Option<_> = json::from_value(self.conf()["block_header_params"].clone()) .map_to_mm(|e| UtxoConfError::InvalidBlockHeaderParams(e.to_string()))?; @@ -337,6 +342,7 @@ pub trait UtxoCoinBuilderCommonOps { Ok(address_format) } + #[inline(always)] fn pub_addr_prefix(&self) -> u8 { let pubtype = self.conf()["pubtype"] .as_u64() @@ -344,14 +350,17 @@ pub trait UtxoCoinBuilderCommonOps { pubtype as u8 } + #[inline(always)] fn p2sh_address_prefix(&self) -> u8 { self.conf()["p2shtype"] .as_u64() .unwrap_or(if self.ticker() == "BTC" { 5 } else { 85 }) as u8 } + #[inline(always)] fn dust_amount(&self) -> u64 { json::from_value(self.conf()["dust"].clone()).unwrap_or(UTXO_DUST_AMOUNT) } + #[inline(always)] fn network(&self) -> UtxoCoinBuildResult { let conf = self.conf(); if !conf["network"].is_null() { @@ -361,6 +370,7 @@ pub trait UtxoCoinBuilderCommonOps { Ok(BlockchainNetwork::Mainnet) } + #[inline(always)] async fn decimals(&self, _rpc_client: &UtxoRpcClientEnum) -> UtxoCoinBuildResult { Ok(self.conf()["decimals"].as_u64().unwrap_or(8) as u8) } @@ -384,6 +394,7 @@ pub trait UtxoCoinBuilderCommonOps { Ok(tx_fee) } + #[inline(always)] fn initial_history_state(&self) -> HistorySyncState { if self.activation_params().tx_history { HistorySyncState::NotStarted @@ -550,6 +561,7 @@ pub trait UtxoCoinBuilderCommonOps { } } + #[inline(always)] fn tx_hash_algo(&self) -> TxHashAlgo { if self.ticker() == "GRS" { TxHashAlgo::SHA256 @@ -558,20 +570,27 @@ pub trait UtxoCoinBuilderCommonOps { } } + #[inline(always)] fn check_utxo_maturity(&self) -> bool { self.activation_params().check_utxo_maturity.unwrap_or_default() } + #[inline(always)] fn is_hw_coin(&self, conf: &UtxoCoinConf) -> bool { conf.trezor_coin.is_some() } + #[inline(always)] #[cfg(target_arch = "wasm32")] fn tx_cache(&self) -> UtxoVerboseCacheShared { crate::utxo::tx_cache::wasm_tx_cache::WasmVerboseCache::default().into_shared() } + #[inline(always)] #[cfg(not(target_arch = "wasm32"))] fn tx_cache(&self) -> UtxoVerboseCacheShared { - let tx_cache_path = self.ctx().dbdir().join("TX_CACHE"); - crate::utxo::tx_cache::fs_tx_cache::FsVerboseCache::new(self.ticker().to_owned(), tx_cache_path).into_shared() + crate::utxo::tx_cache::fs_tx_cache::FsVerboseCache::new(self.ticker().to_owned(), self.tx_cache_path()) + .into_shared() } + + #[inline(always)] + fn tx_cache_path(&self) -> PathBuf { self.ctx().dbdir().join("TX_CACHE") } } /// Attempts to parse native daemon conf file and return rpcport, rpcuser and rpcpassword diff --git a/mm2src/common/custom_iter.rs b/mm2src/common/custom_iter.rs index b38cd628a9..9761d9fb4d 100644 --- a/mm2src/common/custom_iter.rs +++ b/mm2src/common/custom_iter.rs @@ -12,6 +12,7 @@ pub trait CollectInto { /// let expected = vec!["foo".to_owned(), "bar".to_owned()]; /// assert_eq!(actual, expected); /// ``` + #[inline(always)] fn collect_into(self) -> FromB where Self: IntoIterator + Sized, diff --git a/mm2src/common/jsonrpc_client.rs b/mm2src/common/jsonrpc_client.rs index 4d87d5db97..cd1dff6105 100644 --- a/mm2src/common/jsonrpc_client.rs +++ b/mm2src/common/jsonrpc_client.rs @@ -85,11 +85,13 @@ pub enum JsonRpcRequestEnum { impl JsonRpcRequestEnum { /// Creates [`JsonRpcRequestEnum::Batch`] from the given `requests`. + #[inline(always)] pub fn new_batch(requests: Vec) -> JsonRpcRequestEnum { JsonRpcRequestEnum::Batch(JsonRpcBatchRequest(requests)) } /// Returns a `JsonRpcId` identifier of the request. + #[inline(always)] pub fn rpc_id(&self) -> JsonRpcId { match self { JsonRpcRequestEnum::Single(single) => single.rpc_id(), @@ -119,9 +121,11 @@ pub struct JsonRpcRequest { impl JsonRpcRequest { // Returns [`JsonRpcRequest::id`]. + #[inline(always)] pub fn get_id(&self) -> &str { &self.id } /// Returns a `JsonRpcId` identifier of the request. + #[inline(always)] pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Single(self.id.clone()) } } @@ -135,16 +139,20 @@ pub struct JsonRpcBatchRequest(Vec); impl JsonRpcBatchRequest { /// Returns a `JsonRpcId` identifier of the request. + #[inline(always)] pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Batch(self.orig_sequence_ids().collect()) } /// Returns the number of the requests in the batch. + #[inline(always)] pub fn len(&self) -> usize { self.0.len() } /// Whether the batch is empty. + #[inline(always)] pub fn is_empty(&self) -> bool { self.0.is_empty() } /// Returns original sequence of identifiers. /// The method is used to process batch responses in the same order in which the requests were sent. + #[inline(always)] fn orig_sequence_ids(&self) -> impl Iterator + '_ { self.0.iter().map(|req| req.id.clone()) } } @@ -162,6 +170,7 @@ pub enum JsonRpcResponseEnum { impl JsonRpcResponseEnum { /// Returns a `JsonRpcId` identifier of the response. + #[inline(always)] pub fn rpc_id(&self) -> JsonRpcId { match self { JsonRpcResponseEnum::Single(single) => single.rpc_id(), @@ -185,6 +194,7 @@ pub struct JsonRpcResponse { impl JsonRpcResponse { /// Returns a `JsonRpcId` identifier of the response. + #[inline(always)] pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Single(self.id.clone()) } } @@ -197,9 +207,11 @@ impl JsonRpcBatchResponse { pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Batch(self.0.iter().map(|res| res.id.clone()).collect()) } /// Returns the number of the requests in the batch. + #[inline(always)] pub fn len(&self) -> usize { self.0.len() } /// Whether the batch is empty. + #[inline(always)] pub fn is_empty(&self) -> bool { self.0.is_empty() } } @@ -239,6 +251,7 @@ pub enum JsonRpcErrorType { impl JsonRpcErrorType { /// Whether the error type is [`JsonRpcErrorType::Transport`]. + #[inline(always)] pub fn is_transport(&self) -> bool { matches!(self, JsonRpcErrorType::Transport(_)) } } From 0aaefda0671429b5f16c320928da9808a0105d60 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Wed, 4 May 2022 17:42:26 +0700 Subject: [PATCH 21/25] Minor changes within `utxo_tests` --- .../utxo/utxo_builder/utxo_coin_builder.rs | 1 + mm2src/coins/utxo/utxo_common_tests.rs | 12 +++++--- mm2src/coins/utxo/utxo_tests.rs | 28 ++++++++++++------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 9322d5a616..35a0bbeccf 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -590,6 +590,7 @@ pub trait UtxoCoinBuilderCommonOps { } #[inline(always)] + #[cfg(not(target_arch = "wasm32"))] fn tx_cache_path(&self) -> PathBuf { self.ctx().dbdir().join("TX_CACHE") } } diff --git a/mm2src/coins/utxo/utxo_common_tests.rs b/mm2src/coins/utxo/utxo_common_tests.rs index 5614dda8ea..db334a705d 100644 --- a/mm2src/coins/utxo/utxo_common_tests.rs +++ b/mm2src/coins/utxo/utxo_common_tests.rs @@ -3,16 +3,20 @@ use crate::utxo::rpc_clients::{ElectrumClient, UtxoRpcClientOps}; use common::jsonrpc_client::JsonRpcErrorType; pub async fn test_electrum_display_balances(rpc_client: &ElectrumClient) { + let addresses = vec![ + "RG278CfeNPFtNztFZQir8cgdWexVhViYVy".into(), + "RYPz6Lr4muj4gcFzpMdv3ks1NCGn3mkDPN".into(), + "RJeDDtDRtKUoL8BCKdH7TNCHqUKr7kQRsi".into(), + "RQHn9VPHBqNjYwyKfJbZCiaxVrWPKGQjeF".into(), + ]; + let actual = rpc_client.display_balances(addresses, 8).compat().await.unwrap(); + let expected: Vec<(Address, BigDecimal)> = vec![ ("RG278CfeNPFtNztFZQir8cgdWexVhViYVy".into(), BigDecimal::from(5.77699)), ("RYPz6Lr4muj4gcFzpMdv3ks1NCGn3mkDPN".into(), BigDecimal::from(0)), ("RJeDDtDRtKUoL8BCKdH7TNCHqUKr7kQRsi".into(), BigDecimal::from(0.77699)), ("RQHn9VPHBqNjYwyKfJbZCiaxVrWPKGQjeF".into(), BigDecimal::from(16.55398)), ]; - - let addresses = expected.iter().map(|(address, _)| address.clone()).collect(); - let actual = rpc_client.display_balances(addresses, 8).compat().await.unwrap(); - assert_eq!(actual, expected); let invalid_hashes = vec![ diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 07c5ba201b..0ada75cc16 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -3135,10 +3135,11 @@ fn test_withdraw_to_p2wpkh() { /// https://github.com/KomodoPlatform/atomicDEX-API/issues/1181 #[test] fn test_utxo_standard_with_check_utxo_maturity_true() { - static mut LIST_MATURE_UNSPENT_ORDERED_CALLED: bool = false; + /// Whether [`UtxoStandardCoin::get_mature_unspent_ordered_list`] is called or not. + static mut GET_MATURE_UNSPENT_ORDERED_LIST_CALLED: bool = false; UtxoStandardCoin::get_mature_unspent_ordered_list.mock_safe(|coin, _| { - unsafe { LIST_MATURE_UNSPENT_ORDERED_CALLED = true }; + unsafe { GET_MATURE_UNSPENT_ORDERED_LIST_CALLED = true }; let cache = block_on(coin.as_ref().recently_spent_outpoints.lock()); MockResult::Return(Box::pin(futures::future::ok((MatureUnspentList::default(), cache)))) }); @@ -3165,13 +3166,14 @@ fn test_utxo_standard_with_check_utxo_maturity_true() { let address = Address::from("R9o9xTocqr6CeEDGDH6mEYpwLoMz6jNjMW"); // Don't use `block_on` here because it's used within a mock of [`GetUtxoListOps::get_mature_unspent_ordered_list`]. coin.get_unspent_ordered_list(&address).compat().wait().unwrap(); - assert!(unsafe { LIST_MATURE_UNSPENT_ORDERED_CALLED }); + assert!(unsafe { GET_MATURE_UNSPENT_ORDERED_LIST_CALLED }); } /// `UtxoStandardCoin` hasn't to check UTXO maturity if `check_utxo_maturity` is not set. /// https://github.com/KomodoPlatform/atomicDEX-API/issues/1181 #[test] fn test_utxo_standard_without_check_utxo_maturity() { + /// Whether [`UtxoStandardCoin::get_all_unspent_ordered_list`] is called or not. static mut GET_ALL_UNSPENT_ORDERED_LIST_CALLED: bool = false; UtxoStandardCoin::get_all_unspent_ordered_list.mock_safe(|coin, _| { @@ -3213,6 +3215,7 @@ fn test_utxo_standard_without_check_utxo_maturity() { /// https://github.com/KomodoPlatform/atomicDEX-API/issues/1181 #[test] fn test_qtum_without_check_utxo_maturity() { + /// Whether [`QtumCoin::get_mature_unspent_ordered_list`] is called or not. static mut GET_MATURE_UNSPENT_ORDERED_LIST_CALLED: bool = false; QtumCoin::get_mature_unspent_ordered_list.mock_safe(|coin, _| { @@ -3246,6 +3249,7 @@ fn test_qtum_without_check_utxo_maturity() { /// https://github.com/KomodoPlatform/atomicDEX-API/issues/1181 #[test] fn test_qtum_with_check_utxo_maturity_false() { + /// Whether [`QtumCoin::get_all_unspent_ordered_list`] is called or not. static mut GET_ALL_UNSPENT_ORDERED_LIST_CALLED: bool = false; QtumCoin::get_all_unspent_ordered_list.mock_safe(|coin, _address| { @@ -3790,18 +3794,22 @@ fn test_native_display_balances() { let rpc_client = native_client_for_test(); - let expected: Vec<(Address, BigDecimal)> = vec![ - ("RG278CfeNPFtNztFZQir8cgdWexVhViYVy".into(), BigDecimal::from(5.77699)), - ("RYPz6Lr4muj4gcFzpMdv3ks1NCGn3mkDPN".into(), BigDecimal::from(0)), - ("RJeDDtDRtKUoL8BCKdH7TNCHqUKr7kQRsi".into(), BigDecimal::from(0.77699)), - ("RQHn9VPHBqNjYwyKfJbZCiaxVrWPKGQjeF".into(), BigDecimal::from(0.99998)), + let addresses = vec![ + "RG278CfeNPFtNztFZQir8cgdWexVhViYVy".into(), + "RYPz6Lr4muj4gcFzpMdv3ks1NCGn3mkDPN".into(), + "RJeDDtDRtKUoL8BCKdH7TNCHqUKr7kQRsi".into(), + "RQHn9VPHBqNjYwyKfJbZCiaxVrWPKGQjeF".into(), ]; - - let addresses = expected.iter().map(|(address, _)| address.clone()).collect(); let actual = rpc_client .display_balances(addresses, TEST_COIN_DECIMALS) .wait() .unwrap(); + let expected: Vec<(Address, BigDecimal)> = vec![ + ("RG278CfeNPFtNztFZQir8cgdWexVhViYVy".into(), BigDecimal::from(5.77699)), + ("RYPz6Lr4muj4gcFzpMdv3ks1NCGn3mkDPN".into(), BigDecimal::from(0)), + ("RJeDDtDRtKUoL8BCKdH7TNCHqUKr7kQRsi".into(), BigDecimal::from(0.77699)), + ("RQHn9VPHBqNjYwyKfJbZCiaxVrWPKGQjeF".into(), BigDecimal::from(0.99998)), + ]; assert_eq!(actual, expected); } From 7ac3a6ef2cd3086dbf9c01199b1a86961ee16355 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Wed, 4 May 2022 17:49:38 +0700 Subject: [PATCH 22/25] Minor changes within `accoutn_balance_rpc` --- mm2src/coins/rpc_command/account_balance.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/rpc_command/account_balance.rs b/mm2src/coins/rpc_command/account_balance.rs index cf5fe837df..e13a926abd 100644 --- a/mm2src/coins/rpc_command/account_balance.rs +++ b/mm2src/coins/rpc_command/account_balance.rs @@ -73,7 +73,6 @@ pub mod common_impl { ::Address: fmt::Display + Clone, { let account_id = params.account_index; - let chain = params.chain; let hd_account = coin .derivation_method() .hd_wallet_or_err()? @@ -89,14 +88,14 @@ pub mod common_impl { let to_address_id = std::cmp::min(from_address_id + params.limit as u32, total_addresses_number); let addresses = coin - .known_addresses_balances_with_ids(&hd_account, chain, from_address_id..to_address_id) + .known_addresses_balances_with_ids(&hd_account, params.chain, from_address_id..to_address_id) .await?; let page_balance = addresses.iter().fold(CoinBalance::default(), |total, addr_balance| { total + addr_balance.balance.clone() }); let result = HDAccountBalanceResponse { - account_index: params.account_index, + account_index: account_id, derivation_path: RpcDerivationPath(hd_account.account_derivation_path()), addresses, page_balance, From 5a70b25f77dd3cf300b17a578d896788bc9a1130 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Fri, 6 May 2022 16:40:50 +0700 Subject: [PATCH 23/25] Avoid using #[inline(alaways)] where it's not necessary --- mm2src/coins/qrc20.rs | 2 +- .../init_scan_for_new_addresses.rs | 2 +- mm2src/coins/utxo.rs | 10 +++--- mm2src/coins/utxo/rpc_clients.rs | 10 +++--- mm2src/coins/utxo/tx_cache/fs_tx_cache.rs | 4 +-- mm2src/coins/utxo/tx_cache/mod.rs | 2 +- .../utxo/utxo_builder/utxo_coin_builder.rs | 34 +++++++++---------- mm2src/common/custom_iter.rs | 2 +- mm2src/common/jsonrpc_client.rs | 26 +++++++------- 9 files changed, 46 insertions(+), 46 deletions(-) diff --git a/mm2src/coins/qrc20.rs b/mm2src/coins/qrc20.rs index 02351fec3f..dd640bf9fb 100644 --- a/mm2src/coins/qrc20.rs +++ b/mm2src/coins/qrc20.rs @@ -246,7 +246,7 @@ impl<'a> UtxoCoinBuilderCommonOps for Qrc20CoinBuilder<'a> { /// Override [`UtxoCoinBuilderCommonOps::tx_cache`] to initialize TX cache with the platform ticker. /// Please note the method is overridden for Native mode only. - #[inline(always)] + #[inline] #[cfg(not(target_arch = "wasm32"))] fn tx_cache(&self) -> UtxoVerboseCacheShared { crate::utxo::tx_cache::fs_tx_cache::FsVerboseCache::new(self.platform.clone(), self.tx_cache_path()) diff --git a/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs b/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs index 52a77fdbe1..34ffcc7b90 100644 --- a/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs +++ b/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs @@ -76,7 +76,7 @@ impl RpcTaskTypes for InitScanAddressesTask { #[async_trait] impl RpcTask for InitScanAddressesTask { - #[inline(always)] + #[inline] fn initial_status(&self) -> Self::InProgressStatus { ScanAddressesInProgressStatus::InProgress } async fn run(self, _task_handle: &ScanAddressesTaskHandle) -> Result> { diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 484312b345..370e063773 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -753,7 +753,7 @@ pub struct MatureUnspentList { } impl MatureUnspentList { - #[inline(always)] + #[inline] pub fn with_capacity(capacity: usize) -> MatureUnspentList { MatureUnspentList { mature: Vec::with_capacity(capacity), @@ -761,7 +761,7 @@ impl MatureUnspentList { } } - #[inline(always)] + #[inline] pub fn new_mature(mature: Vec) -> MatureUnspentList { MatureUnspentList { mature, @@ -769,10 +769,10 @@ impl MatureUnspentList { } } - #[inline(always)] + #[inline] pub fn only_mature(self) -> Vec { self.mature } - #[inline(always)] + #[inline] pub fn to_coin_balance(&self, decimals: u8) -> CoinBalance { let fold = |acc: BigDecimal, x: &UnspentInfo| acc + big_decimal_from_sat_unsigned(x.value, decimals); CoinBalance { @@ -1074,7 +1074,7 @@ pub enum VerboseTransactionFrom { } impl VerboseTransactionFrom { - #[inline(always)] + #[inline] fn to_inner(&self) -> &RpcTransaction { match self { VerboseTransactionFrom::Rpc(tx) | VerboseTransactionFrom::Cache(tx) => tx, diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index 4bcdbf198d..c291e28965 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -1182,14 +1182,14 @@ impl ElectrumBlockHeaderV12 { } } - #[inline(always)] + #[inline] pub fn as_hex(&self) -> String { let block_header = self.as_block_header(); let serialized = serialize(&block_header); hex::encode(serialized) } - #[inline(always)] + #[inline] pub fn hash(&self) -> H256Json { let block_header = self.as_block_header(); BlockHeader::hash(&block_header).into() @@ -1275,21 +1275,21 @@ pub struct ElectrumBalance { } impl ElectrumBalance { - #[inline(always)] + #[inline] pub fn to_big_decimal(&self, decimals: u8) -> BigDecimal { let balance_sat = BigInt::from(self.confirmed) + BigInt::from(self.unconfirmed); BigDecimal::from(balance_sat) / BigDecimal::from(10u64.pow(decimals as u32)) } } -#[inline(always)] +#[inline] fn sha_256(input: &[u8]) -> Vec { let mut sha = Sha256::new(); sha.input(input); sha.result().to_vec() } -#[inline(always)] +#[inline] pub fn electrum_script_hash(script: &[u8]) -> Vec { let mut result = sha_256(script); result.reverse(); diff --git a/mm2src/coins/utxo/tx_cache/fs_tx_cache.rs b/mm2src/coins/utxo/tx_cache/fs_tx_cache.rs index d9072fbc18..6cee3e5ee9 100644 --- a/mm2src/coins/utxo/tx_cache/fs_tx_cache.rs +++ b/mm2src/coins/utxo/tx_cache/fs_tx_cache.rs @@ -88,7 +88,7 @@ impl UtxoVerboseCacheOps for FsVerboseCache { } impl FsVerboseCache { - #[inline(always)] + #[inline] pub fn new(ticker: String, tx_cache_path: PathBuf) -> FsVerboseCache { FsVerboseCache { ticker, tx_cache_path } } /// Tries to load transaction from cache. @@ -106,6 +106,6 @@ impl FsVerboseCache { write_json(tx, &path, USE_TMP_FILE).await.mm_err(TxCacheError::from) } - #[inline(always)] + #[inline] fn cached_transaction_path(&self, txid: &H256Json) -> PathBuf { self.tx_cache_path.join(format!("{:?}", txid)) } } diff --git a/mm2src/coins/utxo/tx_cache/mod.rs b/mm2src/coins/utxo/tx_cache/mod.rs index a64200598d..98c33e5868 100644 --- a/mm2src/coins/utxo/tx_cache/mod.rs +++ b/mm2src/coins/utxo/tx_cache/mod.rs @@ -27,7 +27,7 @@ pub enum TxCacheError { #[async_trait] pub trait UtxoVerboseCacheOps: fmt::Debug { - #[inline(always)] + #[inline] fn into_shared(self) -> UtxoVerboseCacheShared where Self: Sized + Send + Sync + 'static, diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 35a0bbeccf..46ae1989f0 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -255,7 +255,7 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { .mm_err(UtxoCoinBuildError::from) } - #[inline(always)] + #[inline] fn derivation_path(&self) -> UtxoConfResult { if self.conf()["derivation_path"].is_null() { return MmError::err(UtxoConfError::DerivationPathIsNotSet); @@ -264,13 +264,13 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { .map_to_mm(|e| UtxoConfError::ErrorDeserializingDerivationPath(e.to_string())) } - #[inline(always)] + #[inline] fn gap_limit(&self) -> u32 { self.activation_params().gap_limit.unwrap_or(DEFAULT_GAP_LIMIT) } - #[inline(always)] + #[inline] fn supports_trezor(&self, conf: &UtxoCoinConf) -> bool { conf.trezor_coin.is_some() } - #[inline(always)] + #[inline] fn check_if_trezor_is_initialized(&self) -> UtxoCoinBuildResult<()> { let crypto_ctx = CryptoCtx::from_ctx(self.ctx())?; let hw_ctx = crypto_ctx @@ -292,7 +292,7 @@ pub trait UtxoCoinBuilderCommonOps { fn ticker(&self) -> &str; - #[inline(always)] + #[inline] fn block_headers_storage(&self) -> UtxoCoinBuildResult> { let params: Option<_> = json::from_value(self.conf()["block_header_params"].clone()) .map_to_mm(|e| UtxoConfError::InvalidBlockHeaderParams(e.to_string()))?; @@ -342,7 +342,7 @@ pub trait UtxoCoinBuilderCommonOps { Ok(address_format) } - #[inline(always)] + #[inline] fn pub_addr_prefix(&self) -> u8 { let pubtype = self.conf()["pubtype"] .as_u64() @@ -350,17 +350,17 @@ pub trait UtxoCoinBuilderCommonOps { pubtype as u8 } - #[inline(always)] + #[inline] fn p2sh_address_prefix(&self) -> u8 { self.conf()["p2shtype"] .as_u64() .unwrap_or(if self.ticker() == "BTC" { 5 } else { 85 }) as u8 } - #[inline(always)] + #[inline] fn dust_amount(&self) -> u64 { json::from_value(self.conf()["dust"].clone()).unwrap_or(UTXO_DUST_AMOUNT) } - #[inline(always)] + #[inline] fn network(&self) -> UtxoCoinBuildResult { let conf = self.conf(); if !conf["network"].is_null() { @@ -370,7 +370,7 @@ pub trait UtxoCoinBuilderCommonOps { Ok(BlockchainNetwork::Mainnet) } - #[inline(always)] + #[inline] async fn decimals(&self, _rpc_client: &UtxoRpcClientEnum) -> UtxoCoinBuildResult { Ok(self.conf()["decimals"].as_u64().unwrap_or(8) as u8) } @@ -394,7 +394,7 @@ pub trait UtxoCoinBuilderCommonOps { Ok(tx_fee) } - #[inline(always)] + #[inline] fn initial_history_state(&self) -> HistorySyncState { if self.activation_params().tx_history { HistorySyncState::NotStarted @@ -561,7 +561,7 @@ pub trait UtxoCoinBuilderCommonOps { } } - #[inline(always)] + #[inline] fn tx_hash_algo(&self) -> TxHashAlgo { if self.ticker() == "GRS" { TxHashAlgo::SHA256 @@ -570,26 +570,26 @@ pub trait UtxoCoinBuilderCommonOps { } } - #[inline(always)] + #[inline] fn check_utxo_maturity(&self) -> bool { self.activation_params().check_utxo_maturity.unwrap_or_default() } - #[inline(always)] + #[inline] fn is_hw_coin(&self, conf: &UtxoCoinConf) -> bool { conf.trezor_coin.is_some() } - #[inline(always)] + #[inline] #[cfg(target_arch = "wasm32")] fn tx_cache(&self) -> UtxoVerboseCacheShared { crate::utxo::tx_cache::wasm_tx_cache::WasmVerboseCache::default().into_shared() } - #[inline(always)] + #[inline] #[cfg(not(target_arch = "wasm32"))] fn tx_cache(&self) -> UtxoVerboseCacheShared { crate::utxo::tx_cache::fs_tx_cache::FsVerboseCache::new(self.ticker().to_owned(), self.tx_cache_path()) .into_shared() } - #[inline(always)] + #[inline] #[cfg(not(target_arch = "wasm32"))] fn tx_cache_path(&self) -> PathBuf { self.ctx().dbdir().join("TX_CACHE") } } diff --git a/mm2src/common/custom_iter.rs b/mm2src/common/custom_iter.rs index 9761d9fb4d..4a26d4b336 100644 --- a/mm2src/common/custom_iter.rs +++ b/mm2src/common/custom_iter.rs @@ -12,7 +12,7 @@ pub trait CollectInto { /// let expected = vec!["foo".to_owned(), "bar".to_owned()]; /// assert_eq!(actual, expected); /// ``` - #[inline(always)] + #[inline] fn collect_into(self) -> FromB where Self: IntoIterator + Sized, diff --git a/mm2src/common/jsonrpc_client.rs b/mm2src/common/jsonrpc_client.rs index cd1dff6105..63f2ea0a55 100644 --- a/mm2src/common/jsonrpc_client.rs +++ b/mm2src/common/jsonrpc_client.rs @@ -85,13 +85,13 @@ pub enum JsonRpcRequestEnum { impl JsonRpcRequestEnum { /// Creates [`JsonRpcRequestEnum::Batch`] from the given `requests`. - #[inline(always)] + #[inline] pub fn new_batch(requests: Vec) -> JsonRpcRequestEnum { JsonRpcRequestEnum::Batch(JsonRpcBatchRequest(requests)) } /// Returns a `JsonRpcId` identifier of the request. - #[inline(always)] + #[inline] pub fn rpc_id(&self) -> JsonRpcId { match self { JsonRpcRequestEnum::Single(single) => single.rpc_id(), @@ -121,11 +121,11 @@ pub struct JsonRpcRequest { impl JsonRpcRequest { // Returns [`JsonRpcRequest::id`]. - #[inline(always)] + #[inline] pub fn get_id(&self) -> &str { &self.id } /// Returns a `JsonRpcId` identifier of the request. - #[inline(always)] + #[inline] pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Single(self.id.clone()) } } @@ -139,20 +139,20 @@ pub struct JsonRpcBatchRequest(Vec); impl JsonRpcBatchRequest { /// Returns a `JsonRpcId` identifier of the request. - #[inline(always)] + #[inline] pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Batch(self.orig_sequence_ids().collect()) } /// Returns the number of the requests in the batch. - #[inline(always)] + #[inline] pub fn len(&self) -> usize { self.0.len() } /// Whether the batch is empty. - #[inline(always)] + #[inline] pub fn is_empty(&self) -> bool { self.0.is_empty() } /// Returns original sequence of identifiers. /// The method is used to process batch responses in the same order in which the requests were sent. - #[inline(always)] + #[inline] fn orig_sequence_ids(&self) -> impl Iterator + '_ { self.0.iter().map(|req| req.id.clone()) } } @@ -170,7 +170,7 @@ pub enum JsonRpcResponseEnum { impl JsonRpcResponseEnum { /// Returns a `JsonRpcId` identifier of the response. - #[inline(always)] + #[inline] pub fn rpc_id(&self) -> JsonRpcId { match self { JsonRpcResponseEnum::Single(single) => single.rpc_id(), @@ -194,7 +194,7 @@ pub struct JsonRpcResponse { impl JsonRpcResponse { /// Returns a `JsonRpcId` identifier of the response. - #[inline(always)] + #[inline] pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Single(self.id.clone()) } } @@ -207,11 +207,11 @@ impl JsonRpcBatchResponse { pub fn rpc_id(&self) -> JsonRpcId { JsonRpcId::Batch(self.0.iter().map(|res| res.id.clone()).collect()) } /// Returns the number of the requests in the batch. - #[inline(always)] + #[inline] pub fn len(&self) -> usize { self.0.len() } /// Whether the batch is empty. - #[inline(always)] + #[inline] pub fn is_empty(&self) -> bool { self.0.is_empty() } } @@ -251,7 +251,7 @@ pub enum JsonRpcErrorType { impl JsonRpcErrorType { /// Whether the error type is [`JsonRpcErrorType::Transport`]. - #[inline(always)] + #[inline] pub fn is_transport(&self) -> bool { matches!(self, JsonRpcErrorType::Transport(_)) } } From 996f590c1ed579a06ac1c8d49c550040db551459 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Tue, 10 May 2022 10:38:55 +0700 Subject: [PATCH 24/25] Minor changes * Refactor `utxo_common::addresses_balances` --- mm2src/coins/utxo/utxo_common.rs | 34 +++++++++++++++----------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index b584c32b4b..731590f21b 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -419,7 +419,7 @@ where { if coin.as_ref().check_utxo_maturity { let (unspents_map, _) = coin.get_mature_unspent_ordered_map(addresses.clone()).await?; - return addresses + addresses .into_iter() .map(|address| { let unspents = unspents_map.get(&address).or_mm_err(|| { @@ -429,24 +429,22 @@ where let balance = unspents.to_coin_balance(coin.as_ref().decimals); Ok((address, balance)) }) - .collect(); + .collect() + } else { + Ok(coin + .as_ref() + .rpc_client + .display_balances(addresses.clone(), coin.as_ref().decimals) + .compat() + .await? + .into_iter() + .map(|(address, spendable)| { + let unspendable = BigDecimal::from(0); + let balance = CoinBalance { spendable, unspendable }; + (address, balance) + }) + .collect()) } - - let balances = coin - .as_ref() - .rpc_client - .display_balances(addresses.clone(), coin.as_ref().decimals) - .compat() - .await? - .into_iter() - .map(|(address, spendable)| { - let unspendable = BigDecimal::from(0); - let balance = CoinBalance { spendable, unspendable }; - (address, balance) - }) - .collect(); - - Ok(balances) } pub fn derivation_method(coin: &UtxoCoinFields) -> &DerivationMethod { &coin.derivation_method } From 798f14d1a39319c0b5e80e44449bcb2809088c64 Mon Sep 17 00:00:00 2001 From: Sergey Boyko Date: Tue, 10 May 2022 14:12:56 +0700 Subject: [PATCH 25/25] Minor changes * Make `OutPoint` copiable --- mm2src/coins/utxo.rs | 2 +- mm2src/coins/utxo/utxo_common.rs | 4 ++-- mm2src/coins/utxo_signer/src/sign_common.rs | 8 ++++---- mm2src/mm2_bitcoin/chain/src/transaction.rs | 2 +- mm2src/mm2_bitcoin/script/src/sign.rs | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 4c027f816b..6c4e7900c3 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -1608,7 +1608,7 @@ where .inputs .iter() .map(|input| UnspentInfo { - outpoint: input.previous_output.clone(), + outpoint: input.previous_output, value: input.amount, height: None, }) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 731590f21b..9bb7ef7554 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -832,7 +832,7 @@ impl<'a, T: AsRef + UtxoTxGenerationOps> UtxoTxBuilder<'a, T> { for utxo in self.available_inputs.clone() { self.tx.inputs.push(UnsignedTransactionInput { - previous_output: utxo.outpoint.clone(), + previous_output: utxo.outpoint, sequence: SEQUENCE_FINAL, amount: utxo.value, witness: vec![], @@ -3857,7 +3857,7 @@ where unspents .into_iter() // dedup just in case we add duplicates of same unspent out - .unique_by(|unspent| unspent.outpoint.clone()) + .unique_by(|unspent| unspent.outpoint) .sorted_unstable_by(|a, b| { if a.value < b.value { Ordering::Less diff --git a/mm2src/coins/utxo_signer/src/sign_common.rs b/mm2src/coins/utxo_signer/src/sign_common.rs index fff677f8fa..723367c7ea 100644 --- a/mm2src/coins/utxo_signer/src/sign_common.rs +++ b/mm2src/coins/utxo_signer/src/sign_common.rs @@ -36,7 +36,7 @@ pub(crate) fn p2pk_spend_with_signature( let script_sig = script_sig(signature, fork_id); TransactionInput { - previous_output: unsigned_input.previous_output.clone(), + previous_output: unsigned_input.previous_output, script_sig: Builder::default().push_bytes(&script_sig).into_bytes(), sequence: unsigned_input.sequence, script_witness: vec![], @@ -52,7 +52,7 @@ pub(crate) fn p2pkh_spend_with_signature( let script_sig = script_sig_with_pub(public_key, fork_id, signature); TransactionInput { - previous_output: unsigned_input.previous_output.clone(), + previous_output: unsigned_input.previous_output, script_sig, sequence: unsigned_input.sequence, script_witness: vec![], @@ -77,7 +77,7 @@ pub(crate) fn p2sh_spend_with_signature( resulting_script.extend_from_slice(&redeem_part); TransactionInput { - previous_output: unsigned_input.previous_output.clone(), + previous_output: unsigned_input.previous_output, script_sig: resulting_script, sequence: unsigned_input.sequence, script_witness: vec![], @@ -93,7 +93,7 @@ pub(crate) fn p2wpkh_spend_with_signature( let script_sig = script_sig(signature, fork_id); TransactionInput { - previous_output: unsigned_input.previous_output.clone(), + previous_output: unsigned_input.previous_output, script_sig: Bytes::from(Vec::new()), sequence: unsigned_input.sequence, script_witness: vec![script_sig, Bytes::from(public_key.to_vec())], diff --git a/mm2src/mm2_bitcoin/chain/src/transaction.rs b/mm2src/mm2_bitcoin/chain/src/transaction.rs index 91b90a3897..7219c9707d 100644 --- a/mm2src/mm2_bitcoin/chain/src/transaction.rs +++ b/mm2src/mm2_bitcoin/chain/src/transaction.rs @@ -21,7 +21,7 @@ const WITNESS_FLAG: u8 = 1; /// Maximum supported list size (inputs, outputs, etc.) const MAX_LIST_SIZE: usize = 8192; -#[derive(Debug, PartialEq, Eq, Hash, Clone, Default, Serializable, Deserializable)] +#[derive(Clone, Copy, Debug, Default, Deserializable, Eq, Hash, PartialEq, Serializable)] pub struct OutPoint { pub hash: H256, pub index: u32, diff --git a/mm2src/mm2_bitcoin/script/src/sign.rs b/mm2src/mm2_bitcoin/script/src/sign.rs index 3231d8cacb..8a654576ad 100644 --- a/mm2src/mm2_bitcoin/script/src/sign.rs +++ b/mm2src/mm2_bitcoin/script/src/sign.rs @@ -268,7 +268,7 @@ impl TransactionInputSigner { let unsigned_input = &self.inputs[input_index]; TransactionInput { - previous_output: unsigned_input.previous_output.clone(), + previous_output: unsigned_input.previous_output, sequence: unsigned_input.sequence, script_sig: script_sig.to_bytes(), script_witness: vec![], @@ -301,7 +301,7 @@ impl TransactionInputSigner { let inputs = if sighash.anyone_can_pay { let input = &self.inputs[input_index]; vec![TransactionInput { - previous_output: input.previous_output.clone(), + previous_output: input.previous_output, script_sig: script_pubkey.to_bytes(), sequence: input.sequence, script_witness: vec![], @@ -311,7 +311,7 @@ impl TransactionInputSigner { .iter() .enumerate() .map(|(n, input)| TransactionInput { - previous_output: input.previous_output.clone(), + previous_output: input.previous_output, script_sig: if n == input_index { script_pubkey.to_bytes() } else {