From ac57077d29ee05d13c7495989b5c08f1d3adc8b9 Mon Sep 17 00:00:00 2001 From: borngraced <51881311+borngraced@users.noreply.github.com> Date: Wed, 27 Jul 2022 09:25:29 -0400 Subject: [PATCH 001/249] [r2r] get_current_mtp rpc impl (#1340) * implemented get_current_mtp rpc && test * fmt fix * fix wasm * reitration * better error namings * review fixes * adjusted get mtp test * remove test_get_current_mtp from wasm target * Improved unit test with Mm2TestConf --- mm2src/coins/rpc_command/get_current_mtp.rs | 72 +++++++++++++++++++ mm2src/coins/rpc_command/mod.rs | 1 + mm2src/mm2_main/src/mm2_tests.rs | 39 ++++++++++ .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 2 + 4 files changed, 114 insertions(+) create mode 100644 mm2src/coins/rpc_command/get_current_mtp.rs diff --git a/mm2src/coins/rpc_command/get_current_mtp.rs b/mm2src/coins/rpc_command/get_current_mtp.rs new file mode 100644 index 0000000000..46fa0f3034 --- /dev/null +++ b/mm2src/coins/rpc_command/get_current_mtp.rs @@ -0,0 +1,72 @@ +use common::{HttpStatusCode, StatusCode}; +use derive_more::Display; +use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::prelude::MmError; + +use crate::{lp_coinfind_or_err, + utxo::{rpc_clients::UtxoRpcError, UtxoCommonOps}, + CoinFindError, MmCoinEnum}; + +pub type GetCurrentMtpRpcResult = Result>; + +#[derive(Deserialize)] +pub struct GetCurrentMtpRequest { + coin: String, +} + +#[derive(Serialize)] +pub struct GetCurrentMtpResponse { + mtp: u32, +} + +#[derive(Serialize, Display, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum GetCurrentMtpError { + NoSuchCoin(String), + #[display(fmt = "Requested coin: {}; is not supported for this action.", _0)] + NotSupportedCoin(String), + RpcError(String), +} + +impl HttpStatusCode for GetCurrentMtpError { + fn status_code(&self) -> StatusCode { + match self { + GetCurrentMtpError::NoSuchCoin(_) => StatusCode::PRECONDITION_REQUIRED, + GetCurrentMtpError::NotSupportedCoin(_) => StatusCode::BAD_REQUEST, + GetCurrentMtpError::RpcError(_) => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +impl From for GetCurrentMtpError { + fn from(err: UtxoRpcError) -> Self { Self::RpcError(err.to_string()) } +} + +impl From for GetCurrentMtpError { + fn from(err: CoinFindError) -> Self { Self::NoSuchCoin(err.to_string()) } +} + +pub async fn get_current_mtp_rpc( + ctx: MmArc, + req: GetCurrentMtpRequest, +) -> GetCurrentMtpRpcResult { + match lp_coinfind_or_err(&ctx, &req.coin).await? { + MmCoinEnum::UtxoCoin(utxo) => Ok(GetCurrentMtpResponse { + mtp: utxo.get_current_mtp().await?, + }), + MmCoinEnum::QtumCoin(qtum) => Ok(GetCurrentMtpResponse { + mtp: qtum.get_current_mtp().await?, + }), + MmCoinEnum::Qrc20Coin(qrc) => Ok(GetCurrentMtpResponse { + mtp: qrc.get_current_mtp().await?, + }), + #[cfg(not(target_arch = "wasm32"))] + MmCoinEnum::ZCoin(zcoin) => Ok(GetCurrentMtpResponse { + mtp: zcoin.get_current_mtp().await?, + }), + MmCoinEnum::Bch(bch) => Ok(GetCurrentMtpResponse { + mtp: bch.get_current_mtp().await?, + }), + _ => Err(MmError::new(GetCurrentMtpError::NotSupportedCoin(req.coin))), + } +} diff --git a/mm2src/coins/rpc_command/mod.rs b/mm2src/coins/rpc_command/mod.rs index 7a98d3fc2e..af78a96d70 100644 --- a/mm2src/coins/rpc_command/mod.rs +++ b/mm2src/coins/rpc_command/mod.rs @@ -1,4 +1,5 @@ pub mod account_balance; +pub mod get_current_mtp; pub mod hd_account_balance_rpc_error; pub mod init_create_account; pub mod init_scan_for_new_addresses; diff --git a/mm2src/mm2_main/src/mm2_tests.rs b/mm2src/mm2_main/src/mm2_tests.rs index 39553e51d9..c7f37cf10d 100644 --- a/mm2src/mm2_main/src/mm2_tests.rs +++ b/mm2src/mm2_main/src/mm2_tests.rs @@ -6790,6 +6790,45 @@ fn test_mm2_db_migration() { .unwrap(); } +#[test] +#[cfg(not(target_arch = "wasm32"))] +fn test_get_current_mtp() { + use mm2_test_helpers::for_tests::Mm2TestConf; + + // KMD coin config used for this test + let coins = json!([ + {"coin":"KMD","txversion":4,"overwintered":1,"txfee":10000,"protocol":{"type":"UTXO"}}, + ]); + let passphrase = "cMhHM3PMpMrChygR4bLF7QsTdenhWpFrrmf2UezBG3eeFsz41rtL"; + + let conf = Mm2TestConf::seednode(&passphrase, &coins); + let mm = MarketMakerIt::start(conf.conf, conf.rpc_password, conf.local).unwrap(); + let (_dump_log, _dump_dashboard) = mm.mm_dump(); + + let electrum = block_on(enable_electrum(&mm, "KMD", false, &[ + "electrum1.cipig.net:10001", + "electrum2.cipig.net:10001", + "electrum3.cipig.net:10001", + ])); + log!("{:?}", electrum); + + let rc = block_on(mm.rpc(&json!({ + "userpass": mm.userpass, + "mmrpc": "2.0", + "method": "get_current_mtp", + "params": { + "coin": "KMD", + }, + }))) + .unwrap(); + + // Test if request is successful before proceeding. + assert_eq!(true, rc.0.is_success()); + let mtp_result: Json = json::from_str(&rc.1).unwrap(); + // Test if mtp returns a u32 Number. + assert_eq!(true, mtp_result["result"]["mtp"].is_number()); +} + #[test] #[cfg(not(target_arch = "wasm32"))] fn test_get_public_key() { diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index 0629389beb..e33e16b3a4 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -10,6 +10,7 @@ use crate::{mm2::lp_stats::{add_node_to_version_stat, remove_node_from_version_s use coins::hd_wallet::get_new_address; use coins::my_tx_history_v2::my_tx_history_v2_rpc; use coins::rpc_command::account_balance::account_balance; +use coins::rpc_command::get_current_mtp::get_current_mtp_rpc; use coins::rpc_command::init_create_account::{init_create_new_account, init_create_new_account_status, init_create_new_account_user_action}; use coins::rpc_command::init_scan_for_new_addresses::{init_scan_for_new_addresses, init_scan_for_new_addresses_status}; @@ -125,6 +126,7 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, best_orders_rpc_v2).await, "enable_bch_with_tokens" => handle_mmrpc(ctx, request, enable_platform_coin_with_tokens::).await, "enable_slp" => handle_mmrpc(ctx, request, enable_token::).await, + "get_current_mtp" => handle_mmrpc(ctx, request, get_current_mtp_rpc).await, "get_new_address" => handle_mmrpc(ctx, request, get_new_address).await, "get_public_key" => handle_mmrpc(ctx, request, get_public_key).await, "get_public_key_hash" => handle_mmrpc(ctx, request, get_public_key_hash).await, From 09a91e0cd4a80657c1b4a4428a61e7145474cec7 Mon Sep 17 00:00:00 2001 From: shamardy <39480341+shamardy@users.noreply.github.com> Date: Thu, 28 Jul 2022 12:32:34 +0200 Subject: [PATCH 002/249] Lightning batch transactions confirmations, spv (wip), refactors #1045 (#1339) * remove ok_or_retry_after_sleep_sync! macro * process_txs_confirmations independent from update_best_block * use get_tx_height in get_confirmed_registered_txs to prepare for batch requests * validate_spv_proof refactor wip, spv for lightning wip * fix test_spv_proof * remove find_watched_output_spend_with_header * impl some features of rust-lightning v0.0.106 wip, import code from lightning-persister v0.0.106 and lightning-background-processor v0.0.106 wip * add inbound channels details to sql db * remove lightning-persister and lightning-background-processor crates from our codebase and use v0.0.106 crates instead * remove ok_or_continue macro, use join_all to check for transactions confirmations in batches * small fixes after testing * fix fmt * get confirmed transaction info from spv proof fn * move get_tx_if_onchain, get_tx_height to rpc_clients * spv proof refactoring wip * fix fmt * spv proof refactor * break ln_storage.rs to multiple files, break up persister struct into db struct and filesystem struct * fixes for persisting monitors to back up dir * fix clippy for windows * fix some todos * btc difficulty calculations for spv validation wip * continued: btc difficulty calculations for spv validation wip, added more test cases * continued: btc difficulty calculations for spv validation wip, work.rs, storage.rs * fix wasm, tests * spv difficulty testnet * fix some todos * fixes after merge * add events abort handlers * Review fixes wip * Review fixes wip, log db error in open_channel, workTestVectors.json * Review fixes, save claiming tx to db after successful broadcasting * Review fixes wip, use bool without casting, use i64 instead of u64 casting for db operations * Review fixes wip, #[cfg(target_family = "windows")], refactors * Review fixes, get tx height by block hash --- Cargo.lock | 44 +- Cargo.toml | 2 - mm2src/coins/Cargo.toml | 4 +- mm2src/coins/lightning.rs | 155 +- mm2src/coins/lightning/ln_conf.rs | 9 + .../src/storage.rs => lightning/ln_db.rs} | 123 +- mm2src/coins/lightning/ln_errors.rs | 57 +- mm2src/coins/lightning/ln_events.rs | 285 ++- .../lightning/ln_filesystem_persister.rs | 399 ++++ mm2src/coins/lightning/ln_p2p.rs | 1 - mm2src/coins/lightning/ln_platform.rs | 416 ++-- mm2src/coins/lightning/ln_sql.rs | 1435 +++++++++++ mm2src/coins/lightning/ln_storage.rs | 33 + mm2src/coins/lightning/ln_utils.rs | 96 +- .../lightning_background_processor/Cargo.toml | 20 - .../lightning_background_processor/src/lib.rs | 950 -------- mm2src/coins/lightning_persister/Cargo.toml | 32 - mm2src/coins/lightning_persister/src/lib.rs | 2097 ----------------- mm2src/coins/lightning_persister/src/util.rs | 196 -- mm2src/coins/lp_coins.rs | 32 - mm2src/coins/utxo.rs | 68 +- mm2src/coins/utxo/qtum.rs | 19 +- mm2src/coins/utxo/rpc_clients.rs | 131 +- mm2src/coins/utxo/slp.rs | 40 +- mm2src/coins/utxo/spv.rs | 91 + .../coins/utxo/utxo_block_header_storage.rs | 132 +- mm2src/coins/utxo/utxo_builder/mod.rs | 2 +- .../utxo/utxo_builder/utxo_arc_builder.rs | 64 +- .../utxo/utxo_builder/utxo_coin_builder.rs | 36 +- .../utxo/utxo_builder/utxo_conf_builder.rs | 1 - mm2src/coins/utxo/utxo_common.rs | 229 +- .../utxo_indexedb_block_header_storage.rs | 39 +- .../utxo/utxo_sql_block_header_storage.rs | 322 ++- mm2src/coins/utxo/utxo_tests.rs | 18 +- mm2src/coins/utxo/utxo_wasm_tests.rs | 2 +- mm2src/coins/z_coin.rs | 2 + mm2src/coins/z_coin/z_rpc.rs | 4 +- mm2src/db_common/src/sqlite.rs | 2 +- mm2src/mm2_bitcoin/chain/src/block_header.rs | 63 + mm2src/mm2_bitcoin/chain/src/raw_block.rs | 2 +- mm2src/mm2_bitcoin/chain/src/transaction.rs | 57 +- mm2src/mm2_bitcoin/primitives/Cargo.toml | 3 +- mm2src/mm2_bitcoin/primitives/src/hash.rs | 4 + mm2src/mm2_bitcoin/primitives/src/lib.rs | 1 + .../rpc/src/v1/types/get_block_response.rs | 6 +- mm2src/mm2_bitcoin/spv_validation/Cargo.toml | 6 + .../src/for_tests/workTestVectors.json | 30 + .../spv_validation/src/helpers_validation.rs | 52 +- mm2src/mm2_bitcoin/spv_validation/src/lib.rs | 8 + .../spv_validation/src/spv_proof.rs | 4 +- .../mm2_bitcoin/spv_validation/src/storage.rs | 82 + mm2src/mm2_bitcoin/spv_validation/src/work.rs | 337 +++ 52 files changed, 3699 insertions(+), 4544 deletions(-) rename mm2src/coins/{lightning_persister/src/storage.rs => lightning/ln_db.rs} (68%) create mode 100644 mm2src/coins/lightning/ln_filesystem_persister.rs create mode 100644 mm2src/coins/lightning/ln_sql.rs create mode 100644 mm2src/coins/lightning/ln_storage.rs delete mode 100644 mm2src/coins/lightning_background_processor/Cargo.toml delete mode 100644 mm2src/coins/lightning_background_processor/src/lib.rs delete mode 100644 mm2src/coins/lightning_persister/Cargo.toml delete mode 100644 mm2src/coins/lightning_persister/src/lib.rs delete mode 100644 mm2src/coins/lightning_persister/src/util.rs create mode 100644 mm2src/coins/utxo/spv.rs create mode 100644 mm2src/mm2_bitcoin/spv_validation/src/for_tests/workTestVectors.json create mode 100644 mm2src/mm2_bitcoin/spv_validation/src/storage.rs create mode 100644 mm2src/mm2_bitcoin/spv_validation/src/work.rs diff --git a/Cargo.lock b/Cargo.lock index ec11587fa3..ca6200024e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -505,7 +505,6 @@ checksum = "9a41df6ad9642c5c15ae312dd3d074de38fd3eb7cc87ad4ce10f90292a83fe4d" dependencies = [ "bech32", "bitcoin_hashes", - "bitcoinconsensus", "secp256k1", ] @@ -515,16 +514,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "006cc91e1a1d99819bc5b8214be3555c1f0611b169f527a1fdc54ed1f2b745b0" -[[package]] -name = "bitcoinconsensus" -version = "0.19.0-3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a8aa43b5cd02f856cb126a9af819e77b8910fdd74dd1407be649f2f5fe3a1b5" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "bitcrypto" version = "0.1.0" @@ -2128,7 +2117,7 @@ dependencies = [ "impl-rlp", "impl-serde", "primitive-types", - "uint 0.9.1", + "uint 0.9.3", ] [[package]] @@ -3729,19 +3718,16 @@ version = "0.0.106" source = "git+https://github.com/shamardy/rust-lightning?branch=0.0.106#af4a89c08c22d0110d386df0e288b2f825aaebbc" dependencies = [ "bitcoin", - "hex 0.4.2", - "regex", "secp256k1", ] [[package]] name = "lightning-background-processor" version = "0.0.106" +source = "git+https://github.com/shamardy/rust-lightning?branch=0.0.106#af4a89c08c22d0110d386df0e288b2f825aaebbc" dependencies = [ "bitcoin", - "db_common", "lightning", - "lightning-invoice", "lightning-persister", ] @@ -3770,21 +3756,11 @@ dependencies = [ [[package]] name = "lightning-persister" version = "0.0.106" +source = "git+https://github.com/shamardy/rust-lightning?branch=0.0.106#af4a89c08c22d0110d386df0e288b2f825aaebbc" dependencies = [ - "async-trait", "bitcoin", - "common", - "db_common", - "derive_more", - "hex 0.4.2", "libc", "lightning", - "mm2_io", - "parking_lot 0.12.0", - "rand 0.7.3", - "secp256k1", - "serde", - "serde_json", "winapi", ] @@ -5014,16 +4990,17 @@ dependencies = [ "impl-rlp", "impl-serde", "scale-info", - "uint 0.9.1", + "uint 0.9.3", ] [[package]] name = "primitives" version = "0.1.0" dependencies = [ + "bitcoin_hashes", "byteorder 1.4.3", "rustc-hex 2.1.0", - "uint 0.9.1", + "uint 0.9.3", ] [[package]] @@ -7242,7 +7219,12 @@ dependencies = [ name = "spv_validation" version = "0.1.0" dependencies = [ + "async-trait", "chain", + "common", + "derive_more", + "keys", + "lazy_static", "primitives", "ripemd160", "rustc-hex 2.1.0", @@ -8148,9 +8130,9 @@ dependencies = [ [[package]] name = "uint" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6470ab50f482bde894a037a57064480a246dbfdd5960bd65a44824693f08da5f" +checksum = "12f03af7ccf01dd611cc450a0d10dbc9b745770d096473e2faf0ca6e2d66d1e0" dependencies = [ "byteorder 1.4.3", "crunchy 0.2.2", diff --git a/Cargo.toml b/Cargo.toml index 4e27bed80d..e224d8eafb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,6 @@ [workspace] members = [ "mm2src/coins", - "mm2src/coins/lightning_persister", - "mm2src/coins/lightning_background_processor", "mm2src/coins/utxo_signer", "mm2src/coins_activation", "mm2src/common/shared_ref_counter", diff --git a/mm2src/coins/Cargo.toml b/mm2src/coins/Cargo.toml index d541a1153b..617df41d6f 100644 --- a/mm2src/coins/Cargo.toml +++ b/mm2src/coins/Cargo.toml @@ -50,7 +50,6 @@ keys = { path = "../mm2_bitcoin/keys" } lazy_static = "1.4" libc = "0.2" lightning = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } -lightning-background-processor = { path = "lightning_background_processor" } lightning-invoice = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } metrics = "0.12" mm2_core = { path = "../mm2_core" } @@ -99,7 +98,8 @@ web-sys = { version = "0.3.55", features = ["console", "Headers", "Request", "Re [target.'cfg(not(target_arch = "wasm32"))'.dependencies] dirs = { version = "1" } -lightning-persister = { path = "lightning_persister" } +lightning-background-processor = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } +lightning-persister = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } lightning-net-tokio = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } rust-ini = { version = "0.13" } rustls = { version = "0.20", features = ["dangerous_configuration"] } diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 08e2acbb00..d796d791d7 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -1,12 +1,18 @@ pub mod ln_conf; +mod ln_db; pub mod ln_errors; mod ln_events; +mod ln_filesystem_persister; mod ln_p2p; mod ln_platform; mod ln_serialization; +mod ln_sql; +mod ln_storage; mod ln_utils; use super::{lp_coinfind_or_err, DerivationMethod, MmCoinEnum}; +use crate::lightning::ln_events::init_events_abort_handlers; +use crate::lightning::ln_sql::SqliteLightningDB; use crate::utxo::rpc_clients::UtxoRpcClientEnum; use crate::utxo::utxo_common::{big_decimal_from_sat_unsigned, UtxoTxBuilder}; use crate::utxo::{sat_from_big_decimal, BlockchainNetwork, FeePolicy, GetUtxoListOps, UtxoTxGenerationOps}; @@ -23,7 +29,7 @@ use bitcrypto::dhash256; use bitcrypto::ChecksumType; use chain::TransactionOutput; use common::executor::spawn; -use common::log::{LogOnError, LogState}; +use common::log::{error, LogOnError, LogState}; use common::{async_blocking, calc_total_pages, log, now_ms, ten, PagingOptionsEnum}; use futures::{FutureExt, TryFutureExt}; use futures01::Future; @@ -39,11 +45,9 @@ use lightning_background_processor::BackgroundProcessor; use lightning_invoice::payment; use lightning_invoice::utils::{create_invoice_from_channelmanager, DefaultRouter}; use lightning_invoice::{Invoice, InvoiceDescription}; -use lightning_persister::storage::{ClosedChannelsFilter, DbStorage, FileSystemStorage, HTLCStatus, - NodesAddressesMapShared, PaymentInfo, PaymentType, PaymentsFilter, Scorer, - SqlChannelDetails}; -use lightning_persister::LightningPersister; use ln_conf::{ChannelOptions, LightningCoinConf, LightningProtocolConf, PlatformCoinConfirmations}; +use ln_db::{ClosedChannelsFilter, DBChannelDetails, DBPaymentInfo, DBPaymentsFilter, HTLCStatus, LightningDB, + PaymentType}; use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelError, CloseChannelResult, ConnectToNodeError, ConnectToNodeResult, EnableLightningError, EnableLightningResult, GenerateInvoiceError, GenerateInvoiceResult, GetChannelDetailsError, GetChannelDetailsResult, @@ -51,9 +55,11 @@ use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelErr ListPaymentsError, ListPaymentsResult, OpenChannelError, OpenChannelResult, SendPaymentError, SendPaymentResult}; use ln_events::LightningEventHandler; +use ln_filesystem_persister::{LightningFilesystemPersister, LightningPersisterShared}; use ln_p2p::{connect_to_node, ConnectToNodeRes, PeerManager}; use ln_platform::{h256_json_from_txid, Platform}; use ln_serialization::{InvoiceForRPC, NodeAddress, PublicKeyForRPC}; +use ln_storage::{LightningStorage, NodesAddressesMapShared, Scorer}; use ln_utils::{ChainMonitor, ChannelManager}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; @@ -81,7 +87,7 @@ pub struct LightningCoin { pub conf: LightningCoinConf, /// The lightning node peer manager that takes care of connecting to peers, etc.. pub peer_manager: Arc, - /// The lightning node background processor that takes care of tasks that need to happen periodically + /// The lightning node background processor that takes care of tasks that need to happen periodically. pub background_processor: Arc, /// The lightning node channel manager which keeps track of the number of open channels and sends messages to the appropriate /// channel, also tracks HTLC preimages and forwards onion packets appropriately. @@ -93,7 +99,9 @@ pub struct LightningCoin { /// The lightning node invoice payer. pub invoice_payer: Arc>>, /// The lightning node persister that takes care of writing/reading data from storage. - pub persister: Arc, + pub persister: LightningPersisterShared, + /// The lightning node db struct that takes care of reading/writing data from/to db. + pub db: SqliteLightningDB, /// The mutex storing the addresses of the nodes that the lightning node has open channels with, /// these addresses are used for reconnecting. pub open_channels_nodes: NodesAddressesMapShared, @@ -125,7 +133,7 @@ impl LightningCoin { }) } - fn pay_invoice(&self, invoice: Invoice) -> SendPaymentResult { + fn pay_invoice(&self, invoice: Invoice) -> SendPaymentResult { self.invoice_payer .pay_invoice(&invoice) .map_to_mm(|e| SendPaymentError::PaymentError(format!("{:?}", e)))?; @@ -138,17 +146,17 @@ impl LightningCoin { InvoiceDescription::Hash(h) => hex::encode(h.0.into_inner()), }; let payment_secret = Some(*invoice.payment_secret()); - Ok(PaymentInfo { + Ok(DBPaymentInfo { payment_hash, payment_type, description, preimage: None, secret: payment_secret, - amt_msat: invoice.amount_milli_satoshis(), + amt_msat: invoice.amount_milli_satoshis().map(|a| a as i64), fee_paid_msat: None, status: HTLCStatus::Pending, - created_at: now_ms() / 1000, - last_updated: now_ms() / 1000, + created_at: (now_ms() / 1000) as i64, + last_updated: (now_ms() / 1000) as i64, }) } @@ -157,7 +165,7 @@ impl LightningCoin { destination: PublicKey, amount_msat: u64, final_cltv_expiry_delta: u32, - ) -> SendPaymentResult { + ) -> SendPaymentResult { if final_cltv_expiry_delta < MIN_FINAL_CLTV_EXPIRY { return MmError::err(SendPaymentError::CLTVExpiryError( final_cltv_expiry_delta, @@ -171,17 +179,17 @@ impl LightningCoin { let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0).into_inner()); let payment_type = PaymentType::OutboundPayment { destination }; - Ok(PaymentInfo { + Ok(DBPaymentInfo { payment_hash, payment_type, description: "".into(), preimage: Some(payment_preimage), secret: None, - amt_msat: Some(amount_msat), + amt_msat: Some(amount_msat as i64), fee_paid_msat: None, status: HTLCStatus::Pending, - created_at: now_ms() / 1000, - last_updated: now_ms() / 1000, + created_at: (now_ms() / 1000) as i64, + last_updated: (now_ms() / 1000) as i64, }) } @@ -621,28 +629,29 @@ pub async fn start_lightning( let logger = ctx.log.0.clone(); // Initialize Persister - let persister = ln_utils::init_persister(ctx, platform.clone(), conf.ticker.clone(), params.backup_path).await?; + let persister = ln_utils::init_persister(ctx, conf.ticker.clone(), params.backup_path).await?; // Initialize the KeysManager let keys_manager = ln_utils::init_keys_manager(ctx)?; // Initialize the NetGraphMsgHandler. This is used for providing routes to send payments over let network_graph = Arc::new(persister.get_network_graph(protocol_conf.network.into()).await?); - spawn(ln_utils::persist_network_graph_loop( - persister.clone(), - network_graph.clone(), - )); + let network_gossip = Arc::new(NetGraphMsgHandler::new( network_graph.clone(), None::>, logger.clone(), )); + // Initialize DB + let db = ln_utils::init_db(ctx, conf.ticker.clone()).await?; + // Initialize the ChannelManager let (chain_monitor, channel_manager) = ln_utils::init_channel_manager( platform.clone(), logger.clone(), persister.clone(), + db.clone(), keys_manager.clone(), conf.clone().into(), ) @@ -661,13 +670,15 @@ pub async fn start_lightning( ) .await?; + let events_abort_handlers = init_events_abort_handlers(platform.clone(), db.clone()).await?; + // Initialize the event handler let event_handler = Arc::new(ln_events::LightningEventHandler::new( - // It's safe to use unwrap here for now until implementing Native Client for Lightning platform.clone(), channel_manager.clone(), keys_manager.clone(), - persister.clone(), + db.clone(), + events_abort_handlers, )); // Initialize routing Scorer @@ -675,6 +686,12 @@ pub async fn start_lightning( spawn(ln_utils::persist_scorer_loop(persister.clone(), scorer.clone())); // Create InvoicePayer + // random_seed_bytes are additional random seed to improve privacy by adding a random CLTV expiry offset to each path's final hop. + // This helps obscure the intended recipient from adversarial intermediate hops. The seed is also used to randomize candidate paths during route selection. + // TODO: random_seed_bytes should be taken in consideration when implementing swaps because they change the payment lock-time. + // https://github.com/lightningdevkit/rust-lightning/issues/158 + // https://github.com/lightningdevkit/rust-lightning/pull/1286 + // https://github.com/lightningdevkit/rust-lightning/pull/1359 let router = DefaultRouter::new(network_graph, logger.clone(), keys_manager.get_secure_random_bytes()); let invoice_payer = Arc::new(InvoicePayer::new( channel_manager.clone(), @@ -685,17 +702,12 @@ pub async fn start_lightning( payment::RetryAttempts(params.payment_retries.unwrap_or(5)), )); - // Persist ChannelManager - // Note: if the ChannelManager is not persisted properly to disk, there is risk of channels force closing the next time LN starts up - let channel_manager_persister = persister.clone(); - let persist_channel_manager_callback = - move |node: &ChannelManager| channel_manager_persister.persist_manager(&*node); - // Start Background Processing. Runs tasks periodically in the background to keep LN node operational. // InvoicePayer will act as our event handler as it handles some of the payments related events before // delegating it to LightningEventHandler. + // note: background_processor stops automatically when dropped since BackgroundProcessor implements the Drop trait. let background_processor = Arc::new(BackgroundProcessor::start( - persist_channel_manager_callback, + persister.clone(), invoice_payer.clone(), chain_monitor.clone(), channel_manager.clone(), @@ -731,6 +743,7 @@ pub async fn start_lightning( keys_manager, invoice_payer, persister, + db, open_channels_nodes, }) } @@ -863,7 +876,7 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes user_config.own_channel_config.our_htlc_minimum_msat = min; } - let rpc_channel_id = ln_coin.persister.get_last_channel_rpc_id().await? as u64 + 1; + let rpc_channel_id = ln_coin.db.get_last_channel_rpc_id().await? as u64 + 1; let temp_channel_id = async_blocking(move || { channel_manager @@ -877,7 +890,7 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes unsigned_funding_txs.insert(rpc_channel_id, unsigned); } - let pending_channel_details = SqlChannelDetails::new( + let pending_channel_details = DBChannelDetails::new( rpc_channel_id, temp_channel_id, node_pubkey, @@ -892,7 +905,9 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes .save_nodes_addresses(ln_coin.open_channels_nodes) .await?; - ln_coin.persister.add_channel_to_db(pending_channel_details).await?; + if let Err(e) = ln_coin.db.add_channel_to_db(pending_channel_details).await { + error!("Unable to add new outbound channel {} to db: {}", rpc_channel_id, e); + } Ok(OpenChannelResponse { rpc_channel_id, @@ -1080,7 +1095,7 @@ pub struct ListClosedChannelsRequest { #[derive(Serialize)] pub struct ListClosedChannelsResponse { - closed_channels: Vec, + closed_channels: Vec, limit: usize, skipped: usize, total: usize, @@ -1098,7 +1113,7 @@ pub async fn list_closed_channels_by_filter( _ => return MmError::err(ListChannelsError::UnsupportedCoin(coin.ticker().to_string())), }; let closed_channels_res = ln_coin - .persister + .db .get_closed_channels_by_filter(req.filter, req.paging_options.clone(), req.limit) .await?; @@ -1122,7 +1137,7 @@ pub struct GetChannelDetailsRequest { #[serde(tag = "status", content = "details")] pub enum GetChannelDetailsResponse { Open(ChannelDetailsForRPC), - Closed(SqlChannelDetails), + Closed(DBChannelDetails), } pub async fn get_channel_details( @@ -1143,7 +1158,7 @@ pub async fn get_channel_details( Some(details) => GetChannelDetailsResponse::Open(details.into()), None => GetChannelDetailsResponse::Closed( ln_coin - .persister + .db .get_channel_from_db(req.rpc_channel_id) .await? .ok_or(GetChannelDetailsError::NoSuchChannel(req.rpc_channel_id))?, @@ -1194,19 +1209,19 @@ pub async fn generate_invoice( req.description.clone(), )?; let payment_hash = invoice.payment_hash().into_inner(); - let payment_info = PaymentInfo { + let payment_info = DBPaymentInfo { payment_hash: PaymentHash(payment_hash), payment_type: PaymentType::InboundPayment, description: req.description, preimage: None, secret: Some(*invoice.payment_secret()), - amt_msat: req.amount_in_msat, + amt_msat: req.amount_in_msat.map(|a| a as i64), fee_paid_msat: None, status: HTLCStatus::Pending, - created_at: now_ms() / 1000, - last_updated: now_ms() / 1000, + created_at: (now_ms() / 1000) as i64, + last_updated: (now_ms() / 1000) as i64, }; - ln_coin.persister.add_or_update_payment_in_db(payment_info).await?; + ln_coin.db.add_or_update_payment_in_db(payment_info).await?; Ok(GenerateInvoiceResponse { payment_hash: payment_hash.into(), invoice: invoice.into(), @@ -1265,10 +1280,7 @@ pub async fn send_payment(ctx: MmArc, req: SendPaymentReq) -> SendPaymentResult< expiry, } => ln_coin.keysend(destination.into(), amount_in_msat, expiry)?, }; - ln_coin - .persister - .add_or_update_payment_in_db(payment_info.clone()) - .await?; + ln_coin.db.add_or_update_payment_in_db(payment_info.clone()).await?; Ok(SendPaymentResponse { payment_hash: payment_info.payment_hash.0.into(), }) @@ -1287,18 +1299,27 @@ pub struct PaymentsFilterForRPC { pub to_timestamp: Option, } -impl From for PaymentsFilter { +impl From for DBPaymentsFilter { fn from(filter: PaymentsFilterForRPC) -> Self { - PaymentsFilter { - payment_type: filter.payment_type.map(From::from), + let (is_outbound, destination) = if let Some(payment_type) = filter.payment_type { + match payment_type { + PaymentTypeForRPC::OutboundPayment { destination } => (Some(true), Some(destination.0.to_string())), + PaymentTypeForRPC::InboundPayment => (Some(false), None), + } + } else { + (None, None) + }; + DBPaymentsFilter { + is_outbound, + destination, description: filter.description, - status: filter.status, - from_amount_msat: filter.from_amount_msat, - to_amount_msat: filter.to_amount_msat, - from_fee_paid_msat: filter.from_fee_paid_msat, - to_fee_paid_msat: filter.to_fee_paid_msat, - from_timestamp: filter.from_timestamp, - to_timestamp: filter.to_timestamp, + status: filter.status.map(|s| s.to_string()), + from_amount_msat: filter.from_amount_msat.map(|a| a as i64), + to_amount_msat: filter.to_amount_msat.map(|a| a as i64), + from_fee_paid_msat: filter.from_fee_paid_msat.map(|f| f as i64), + to_fee_paid_msat: filter.to_fee_paid_msat.map(|f| f as i64), + from_timestamp: filter.from_timestamp.map(|f| f as i64), + to_timestamp: filter.to_timestamp.map(|f| f as i64), } } } @@ -1350,16 +1371,16 @@ pub struct PaymentInfoForRPC { payment_type: PaymentTypeForRPC, description: String, #[serde(skip_serializing_if = "Option::is_none")] - amount_in_msat: Option, + amount_in_msat: Option, #[serde(skip_serializing_if = "Option::is_none")] - fee_paid_msat: Option, + fee_paid_msat: Option, status: HTLCStatus, - created_at: u64, - last_updated: u64, + created_at: i64, + last_updated: i64, } -impl From for PaymentInfoForRPC { - fn from(info: PaymentInfo) -> Self { +impl From for PaymentInfoForRPC { + fn from(info: DBPaymentInfo) -> Self { PaymentInfoForRPC { payment_hash: info.payment_hash.0.into(), payment_type: info.payment_type.into(), @@ -1390,7 +1411,7 @@ pub async fn list_payments_by_filter(ctx: MmArc, req: ListPaymentsReq) -> ListPa _ => return MmError::err(ListPaymentsError::UnsupportedCoin(coin.ticker().to_string())), }; let get_payments_res = ln_coin - .persister + .db .get_payments_by_filter( req.filter.map(From::from), req.paging_options.clone().map(|h| PaymentHash(h.0)), @@ -1429,11 +1450,7 @@ pub async fn get_payment_details( _ => return MmError::err(GetPaymentDetailsError::UnsupportedCoin(coin.ticker().to_string())), }; - if let Some(payment_info) = ln_coin - .persister - .get_payment_from_db(PaymentHash(req.payment_hash.0)) - .await? - { + if let Some(payment_info) = ln_coin.db.get_payment_from_db(PaymentHash(req.payment_hash.0)).await? { return Ok(GetPaymentDetailsResponse { payment_details: payment_info.into(), }); diff --git a/mm2src/coins/lightning/ln_conf.rs b/mm2src/coins/lightning/ln_conf.rs index f69a9bca10..b40234d32d 100644 --- a/mm2src/coins/lightning/ln_conf.rs +++ b/mm2src/coins/lightning/ln_conf.rs @@ -124,6 +124,11 @@ pub struct OurChannelsConfig { /// The smallest value HTLC we will accept to process. The channel gets closed any time /// our counterparty misbehaves by sending us an HTLC with a value smaller than this. pub our_htlc_minimum_msat: Option, + /// If set, we attempt to negotiate the `scid_privacy` (referred to as `scid_alias` in the + /// BOLTs) option for outbound private channels. This provides better privacy by not including + /// our real on-chain channel UTXO in each invoice and requiring that our counterparty only + /// relay HTLCs to us using the channel's SCID alias. + pub negotiate_scid_privacy: Option, } impl From for ChannelHandshakeConfig { @@ -142,6 +147,10 @@ impl From for ChannelHandshakeConfig { channel_handshake_config.our_htlc_minimum_msat = min; } + if let Some(scid_privacy) = config.negotiate_scid_privacy { + channel_handshake_config.negotiate_scid_privacy = scid_privacy + } + channel_handshake_config } } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning/ln_db.rs similarity index 68% rename from mm2src/coins/lightning_persister/src/storage.rs rename to mm2src/coins/lightning/ln_db.rs index c0b9ac7e9d..b47a72361a 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning/ln_db.rs @@ -1,53 +1,21 @@ use async_trait::async_trait; -use bitcoin::Network; use common::{now_ms, PagingOptionsEnum}; use db_common::sqlite::rusqlite::types::FromSqlError; use derive_more::Display; use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; -use lightning::routing::network_graph::NetworkGraph; -use lightning::routing::scoring::ProbabilisticScorer; -use parking_lot::Mutex as PaMutex; use secp256k1::PublicKey; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::net::SocketAddr; use std::str::FromStr; -use std::sync::{Arc, Mutex}; - -pub type NodesAddressesMap = HashMap; -pub type NodesAddressesMapShared = Arc>; -pub type Scorer = ProbabilisticScorer>; -#[async_trait] -pub trait FileSystemStorage { - type Error; - - /// Initializes dirs/collection/tables in storage for a specified coin - async fn init_fs(&self) -> Result<(), Self::Error>; - - async fn is_fs_initialized(&self) -> Result; - - async fn get_nodes_addresses(&self) -> Result, Self::Error>; - - async fn save_nodes_addresses(&self, nodes_addresses: NodesAddressesMapShared) -> Result<(), Self::Error>; - - async fn get_network_graph(&self, network: Network) -> Result; - - async fn save_network_graph(&self, network_graph: Arc) -> Result<(), Self::Error>; - - async fn get_scorer(&self, network_graph: Arc) -> Result; - - async fn save_scorer(&self, scorer: Arc>) -> Result<(), Self::Error>; -} #[derive(Clone, Debug, PartialEq, Serialize)] -pub struct SqlChannelDetails { - pub rpc_id: u64, +pub struct DBChannelDetails { + pub rpc_id: i64, pub channel_id: String, pub counterparty_node_id: String, #[serde(skip_serializing_if = "Option::is_none")] pub funding_tx: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub funding_value: Option, + pub funding_value: Option, #[serde(skip_serializing_if = "Option::is_none")] pub closing_tx: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -57,16 +25,16 @@ pub struct SqlChannelDetails { #[serde(skip_serializing_if = "Option::is_none")] pub claimed_balance: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub funding_generated_in_block: Option, + pub funding_generated_in_block: Option, pub is_outbound: bool, pub is_public: bool, pub is_closed: bool, - pub created_at: u64, + pub created_at: i64, #[serde(skip_serializing_if = "Option::is_none")] - pub closed_at: Option, + pub closed_at: Option, } -impl SqlChannelDetails { +impl DBChannelDetails { #[inline] pub fn new( rpc_id: u64, @@ -75,8 +43,8 @@ impl SqlChannelDetails { is_outbound: bool, is_public: bool, ) -> Self { - SqlChannelDetails { - rpc_id, + DBChannelDetails { + rpc_id: rpc_id as i64, channel_id: hex::encode(channel_id), counterparty_node_id: counterparty_node_id.to_string(), funding_tx: None, @@ -89,7 +57,7 @@ impl SqlChannelDetails { is_outbound, is_public, is_closed: false, - created_at: now_ms() / 1000, + created_at: (now_ms() / 1000) as i64, closed_at: None, } } @@ -112,8 +80,8 @@ pub struct ClosedChannelsFilter { pub channel_id: Option, pub counterparty_node_id: Option, pub funding_tx: Option, - pub from_funding_value: Option, - pub to_funding_value: Option, + pub from_funding_value: Option, + pub to_funding_value: Option, pub closing_tx: Option, pub closure_reason: Option, pub claiming_tx: Option, @@ -124,7 +92,7 @@ pub struct ClosedChannelsFilter { } pub struct GetClosedChannelsResult { - pub channels: Vec, + pub channels: Vec, pub skipped: usize, pub total: usize, } @@ -157,40 +125,41 @@ pub enum PaymentType { } #[derive(Clone, Debug, PartialEq)] -pub struct PaymentInfo { +pub struct DBPaymentInfo { pub payment_hash: PaymentHash, pub payment_type: PaymentType, pub description: String, pub preimage: Option, pub secret: Option, - pub amt_msat: Option, - pub fee_paid_msat: Option, + pub amt_msat: Option, + pub fee_paid_msat: Option, pub status: HTLCStatus, - pub created_at: u64, - pub last_updated: u64, + pub created_at: i64, + pub last_updated: i64, } #[derive(Clone)] -pub struct PaymentsFilter { - pub payment_type: Option, +pub struct DBPaymentsFilter { + pub is_outbound: Option, + pub destination: Option, pub description: Option, - pub status: Option, - pub from_amount_msat: Option, - pub to_amount_msat: Option, - pub from_fee_paid_msat: Option, - pub to_fee_paid_msat: Option, - pub from_timestamp: Option, - pub to_timestamp: Option, + pub status: Option, + pub from_amount_msat: Option, + pub to_amount_msat: Option, + pub from_fee_paid_msat: Option, + pub to_fee_paid_msat: Option, + pub from_timestamp: Option, + pub to_timestamp: Option, } pub struct GetPaymentsResult { - pub payments: Vec, + pub payments: Vec, pub skipped: usize, pub total: usize, } #[async_trait] -pub trait DbStorage { +pub trait LightningDB { type Error; /// Initializes tables in DB. @@ -204,35 +173,37 @@ pub trait DbStorage { /// Inserts a new channel record in the DB. The record's data is completed using add_funding_tx_to_db, /// add_closing_tx_to_db, add_claiming_tx_to_db when this information is available. - async fn add_channel_to_db(&self, details: SqlChannelDetails) -> Result<(), Self::Error>; + async fn add_channel_to_db(&self, details: DBChannelDetails) -> Result<(), Self::Error>; /// Updates a channel's DB record with the channel's funding transaction information. async fn add_funding_tx_to_db( &self, - rpc_id: u64, + rpc_id: i64, funding_tx: String, - funding_value: u64, - funding_generated_in_block: u64, + funding_value: i64, + funding_generated_in_block: i64, ) -> Result<(), Self::Error>; /// Updates funding_tx_block_height value for a channel in the DB. Should be used to update the block height of /// the funding tx when the transaction is confirmed on-chain. - async fn update_funding_tx_block_height(&self, funding_tx: String, block_height: u64) -> Result<(), Self::Error>; + async fn update_funding_tx_block_height(&self, funding_tx: String, block_height: i64) -> Result<(), Self::Error>; /// Updates the is_closed value for a channel in the DB to 1. async fn update_channel_to_closed( &self, - rpc_id: u64, + rpc_id: i64, closure_reason: String, - close_at: u64, + close_at: i64, ) -> Result<(), Self::Error>; - /// Gets the list of closed channels records in the DB with no closing tx hashs saved yet. Can be used to check if - /// the closing tx hash needs to be fetched from the chain and saved to DB when initializing the persister. - async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error>; + /// Gets the list of closed channels records in the DB that have funding tx hashes saved with no closing + /// tx hashes saved yet. + /// Can be used to check if the closing tx hash needs to be fetched from the chain and saved to DB + /// when initializing the persister. + async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error>; /// Updates a channel's DB record with the channel's closing transaction hash. - async fn add_closing_tx_to_db(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error>; + async fn add_closing_tx_to_db(&self, rpc_id: i64, closing_tx: String) -> Result<(), Self::Error>; /// Updates a channel's DB record with information about the transaction responsible for claiming the channel's /// closing balance back to the user's address. @@ -244,7 +215,7 @@ pub trait DbStorage { ) -> Result<(), Self::Error>; /// Gets a channel record from DB by the channel's rpc_id. - async fn get_channel_from_db(&self, rpc_id: u64) -> Result, Self::Error>; + async fn get_channel_from_db(&self, rpc_id: u64) -> Result, Self::Error>; /// Gets the list of closed channels that match the provided filter criteria. The number of requested records is /// specified by the limit parameter, the starting record to list from is specified by the paging parameter. The @@ -257,17 +228,17 @@ pub trait DbStorage { ) -> Result; /// Inserts or updates a new payment record in the DB. - async fn add_or_update_payment_in_db(&self, info: PaymentInfo) -> Result<(), Self::Error>; + async fn add_or_update_payment_in_db(&self, info: DBPaymentInfo) -> Result<(), Self::Error>; /// Gets a payment's record from DB by the payment's hash. - async fn get_payment_from_db(&self, hash: PaymentHash) -> Result, Self::Error>; + async fn get_payment_from_db(&self, hash: PaymentHash) -> Result, Self::Error>; /// Gets the list of payments that match the provided filter criteria. The number of requested records is specified /// by the limit parameter, the starting record to list from is specified by the paging parameter. The total number /// of matched records along with the number of skipped records are also returned in the result. async fn get_payments_by_filter( &self, - filter: Option, + filter: Option, paging: PagingOptionsEnum, limit: usize, ) -> Result; diff --git a/mm2src/coins/lightning/ln_errors.rs b/mm2src/coins/lightning/ln_errors.rs index 72b581f647..e70f555d8c 100644 --- a/mm2src/coins/lightning/ln_errors.rs +++ b/mm2src/coins/lightning/ln_errors.rs @@ -1,8 +1,6 @@ use crate::utxo::rpc_clients::UtxoRpcError; use crate::utxo::GenerateTxError; use crate::{BalanceError, CoinFindError, NumConversError, PrivKeyNotAllowed, UnexpectedDerivationMethod}; -use bitcoin::consensus::encode; -use common::jsonrpc_client::JsonRpcError; use common::HttpStatusCode; use db_common::sqlite::rusqlite::Error as SqlError; use derive_more::Display; @@ -10,6 +8,7 @@ use http::StatusCode; use lightning_invoice::SignOrCreationError; use mm2_err_handle::prelude::*; use rpc::v1::types::H256 as H256Json; +use std::num::TryFromIntError; use utxo_signer::with_key_pair::UtxoSignWithKeyPairError; pub type EnableLightningResult = Result>; @@ -42,8 +41,6 @@ pub enum EnableLightningError { InvalidPath(String), #[display(fmt = "System time error {}", _0)] SystemTimeError(String), - #[display(fmt = "Hash error {}", _0)] - HashError(String), #[display(fmt = "RPC error {}", _0)] RpcError(String), #[display(fmt = "DB error {}", _0)] @@ -60,7 +57,6 @@ impl HttpStatusCode for EnableLightningError { | EnableLightningError::InvalidPath(_) | EnableLightningError::SystemTimeError(_) | EnableLightningError::IOError(_) - | EnableLightningError::HashError(_) | EnableLightningError::ConnectToNodeError(_) | EnableLightningError::InvalidConfiguration(_) | EnableLightningError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, @@ -493,7 +489,7 @@ impl From for ClaimableBalancesError { } } -#[derive(Display)] +#[derive(Display, PartialEq)] pub enum SaveChannelClosingError { #[display(fmt = "DB error: {}", _0)] DbError(String), @@ -507,55 +503,14 @@ pub enum SaveChannelClosingError { FundingTxParseError(String), #[display(fmt = "Error while waiting for the funding transaction to be spent: {}", _0)] WaitForFundingTxSpendError(String), + #[display(fmt = "Error while converting types: {}", _0)] + ConversionError(TryFromIntError), } impl From for SaveChannelClosingError { fn from(err: SqlError) -> SaveChannelClosingError { SaveChannelClosingError::DbError(err.to_string()) } } -#[derive(Debug)] -#[allow(clippy::large_enum_variant)] -pub enum GetTxError { - Rpc(UtxoRpcError), - TxDeserialization(encode::Error), -} - -impl From for GetTxError { - fn from(err: UtxoRpcError) -> GetTxError { GetTxError::Rpc(err) } -} - -impl From for GetTxError { - fn from(err: encode::Error) -> GetTxError { GetTxError::TxDeserialization(err) } -} - -#[derive(Debug)] -#[allow(clippy::large_enum_variant)] -pub enum GetHeaderError { - Rpc(JsonRpcError), - HeaderDeserialization(encode::Error), -} - -impl From for GetHeaderError { - fn from(err: JsonRpcError) -> GetHeaderError { GetHeaderError::Rpc(err) } -} - -impl From for GetHeaderError { - fn from(err: encode::Error) -> GetHeaderError { GetHeaderError::HeaderDeserialization(err) } -} - -#[derive(Debug)] -#[allow(clippy::large_enum_variant)] -pub enum FindWatchedOutputSpendError { - HashNotHeight, - DeserializationErr(encode::Error), - RpcError(String), - GetHeaderError(GetHeaderError), -} - -impl From for FindWatchedOutputSpendError { - fn from(err: JsonRpcError) -> Self { FindWatchedOutputSpendError::RpcError(err.to_string()) } -} - -impl From for FindWatchedOutputSpendError { - fn from(err: encode::Error) -> Self { FindWatchedOutputSpendError::DeserializationErr(err) } +impl From for SaveChannelClosingError { + fn from(err: TryFromIntError) -> SaveChannelClosingError { SaveChannelClosingError::ConversionError(err) } } diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 3f898e4fa3..59af6a94ee 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -1,14 +1,18 @@ use super::*; +use crate::lightning::ln_db::{DBChannelDetails, HTLCStatus, LightningDB, PaymentType}; use crate::lightning::ln_errors::{SaveChannelClosingError, SaveChannelClosingResult}; +use crate::lightning::ln_sql::SqliteLightningDB; use bitcoin::blockdata::script::Script; use bitcoin::blockdata::transaction::Transaction; +use bitcoin::consensus::encode::serialize_hex; use common::executor::{spawn, Timer}; use common::log::{error, info}; -use common::now_ms; +use common::{now_ms, spawn_abortable, AbortOnDropHandle}; use core::time::Duration; -use lightning::chain::chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator}; +use lightning::chain::chaininterface::{ConfirmationTarget, FeeEstimator}; use lightning::chain::keysinterface::SpendableOutputDescriptor; use lightning::util::events::{Event, EventHandler, PaymentPurpose}; +use parking_lot::Mutex as PaMutex; use rand::Rng; use script::{Builder, SignatureVersion}; use secp256k1::Secp256k1; @@ -22,7 +26,8 @@ pub struct LightningEventHandler { platform: Arc, channel_manager: Arc, keys_manager: Arc, - persister: Arc, + db: SqliteLightningDB, + abort_handlers: Arc>>, } impl EventHandler for LightningEventHandler { @@ -114,22 +119,41 @@ impl EventHandler for LightningEventHandler { funding_satoshis, push_msat, channel_type: _, - } => { - info!( - "Handling OpenChannelRequest from node: {} with funding value: {} and starting balance: {}", - counterparty_node_id, - funding_satoshis, - push_msat, - ); - if self.channel_manager.accept_inbound_channel(temporary_channel_id, 0).is_ok() { - // Todo: once the rust-lightning PR for user_channel_id in accept_inbound_channel is released - // use user_channel_id to get the funding tx here once the funding tx is available. - } - }, + } => self.handle_open_channel_request(*temporary_channel_id, *counterparty_node_id, *funding_satoshis, *push_msat), } } } +pub async fn init_events_abort_handlers( + platform: Arc, + db: SqliteLightningDB, +) -> EnableLightningResult>>> { + let abort_handlers = Arc::new(PaMutex::new(Vec::new())); + let closed_channels_without_closing_tx = db.get_closed_channels_with_no_closing_tx().await?; + for channel_details in closed_channels_without_closing_tx { + let platform = platform.clone(); + let db = db.clone(); + let user_channel_id = channel_details.rpc_id; + let abort_handler = spawn_abortable(async move { + if let Ok(closing_tx_hash) = platform + .get_channel_closing_tx(channel_details) + .await + .error_log_passthrough() + { + if let Err(e) = db.add_closing_tx_to_db(user_channel_id, closing_tx_hash).await { + log::error!( + "Unable to update channel {} closing details in DB: {}", + user_channel_id, + e + ); + } + } + }); + abort_handlers.lock().push(abort_handler); + } + Ok(abort_handlers) +} + // Generates the raw funding transaction with one output equal to the channel value. fn sign_funding_transaction( user_channel_id: u64, @@ -167,39 +191,55 @@ fn sign_funding_transaction( } async fn save_channel_closing_details( - persister: Arc, + db: SqliteLightningDB, platform: Arc, user_channel_id: u64, reason: String, ) -> SaveChannelClosingResult<()> { - persister - .update_channel_to_closed(user_channel_id, reason, now_ms() / 1000) + db.update_channel_to_closed(user_channel_id as i64, reason, (now_ms() / 1000) as i64) .await?; - let channel_details = persister + let channel_details = db .get_channel_from_db(user_channel_id) .await? .ok_or_else(|| MmError::new(SaveChannelClosingError::ChannelNotFound(user_channel_id)))?; let closing_tx_hash = platform.get_channel_closing_tx(channel_details).await?; - persister.add_closing_tx_to_db(user_channel_id, closing_tx_hash).await?; + db.add_closing_tx_to_db(user_channel_id as i64, closing_tx_hash).await?; Ok(()) } +async fn add_claiming_tx_to_db_loop( + db: SqliteLightningDB, + closing_txid: String, + claiming_txid: String, + claimed_balance: f64, +) { + while let Err(e) = db + .add_claiming_tx_to_db(closing_txid.clone(), claiming_txid.clone(), claimed_balance) + .await + { + error!("error {}", e); + Timer::sleep(TRY_LOOP_INTERVAL).await; + } +} + impl LightningEventHandler { pub fn new( platform: Arc, channel_manager: Arc, keys_manager: Arc, - persister: Arc, + db: SqliteLightningDB, + abort_handlers: Arc>>, ) -> Self { LightningEventHandler { platform, channel_manager, keys_manager, - persister, + db, + abort_handlers, } } @@ -235,18 +275,17 @@ impl LightningEventHandler { return; } let platform = self.platform.clone(); - let persister = self.persister.clone(); + let db = self.db.clone(); spawn(async move { let best_block_height = platform.best_block_height(); - persister - .add_funding_tx_to_db( - user_channel_id, - funding_txid.to_string(), - channel_value_satoshis, - best_block_height, - ) - .await - .error_log(); + db.add_funding_tx_to_db( + user_channel_id as i64, + funding_txid.to_string(), + channel_value_satoshis as i64, + best_block_height as i64, + ) + .await + .error_log(); }); } @@ -276,38 +315,34 @@ impl LightningEventHandler { }, false => HTLCStatus::Failed, }; - let persister = self.persister.clone(); + let db = self.db.clone(); match purpose { PaymentPurpose::InvoicePayment { .. } => spawn(async move { - if let Ok(Some(mut payment_info)) = persister - .get_payment_from_db(payment_hash) - .await - .error_log_passthrough() - { + if let Ok(Some(mut payment_info)) = db.get_payment_from_db(payment_hash).await.error_log_passthrough() { payment_info.preimage = Some(payment_preimage); payment_info.status = HTLCStatus::Succeeded; - payment_info.amt_msat = Some(amt); - payment_info.last_updated = now_ms() / 1000; - if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { + payment_info.amt_msat = Some(amt as i64); + payment_info.last_updated = (now_ms() / 1000) as i64; + if let Err(e) = db.add_or_update_payment_in_db(payment_info).await { error!("Unable to update payment information in DB: {}", e); } } }), PaymentPurpose::SpontaneousPayment(_) => { - let payment_info = PaymentInfo { + let payment_info = DBPaymentInfo { payment_hash, payment_type: PaymentType::InboundPayment, description: "".into(), preimage: Some(payment_preimage), secret: payment_secret, - amt_msat: Some(amt), + amt_msat: Some(amt as i64), fee_paid_msat: None, status, - created_at: now_ms() / 1000, - last_updated: now_ms() / 1000, + created_at: (now_ms() / 1000) as i64, + last_updated: (now_ms() / 1000) as i64, }; spawn(async move { - if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { + if let Err(e) = db.add_or_update_payment_in_db(payment_info).await { error!("Unable to update payment information in DB: {}", e); } }); @@ -325,19 +360,15 @@ impl LightningEventHandler { "Handling PaymentSent event for payment_hash: {}", hex::encode(payment_hash.0) ); - let persister = self.persister.clone(); + let db = self.db.clone(); spawn(async move { - if let Ok(Some(mut payment_info)) = persister - .get_payment_from_db(payment_hash) - .await - .error_log_passthrough() - { + if let Ok(Some(mut payment_info)) = db.get_payment_from_db(payment_hash).await.error_log_passthrough() { payment_info.preimage = Some(payment_preimage); payment_info.status = HTLCStatus::Succeeded; - payment_info.fee_paid_msat = fee_paid_msat; - payment_info.last_updated = now_ms() / 1000; + payment_info.fee_paid_msat = fee_paid_msat.map(|f| f as i64); + payment_info.last_updated = (now_ms() / 1000) as i64; let amt_msat = payment_info.amt_msat; - if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { + if let Err(e) = db.add_or_update_payment_in_db(payment_info).await { error!("Unable to update payment information in DB: {}", e); } info!( @@ -355,21 +386,20 @@ impl LightningEventHandler { hex::encode(channel_id), reason ); - let persister = self.persister.clone(); + let db = self.db.clone(); let platform = self.platform.clone(); - // Todo: Handle inbound channels closure case after updating to latest version of rust-lightning - // as it has a new OpenChannelRequest event where we can give an inbound channel a user_channel_id - // other than 0 in sql - if user_channel_id != 0 { - spawn(async move { - if let Err(e) = save_channel_closing_details(persister, platform, user_channel_id, reason).await { + let abort_handler = spawn_abortable(async move { + if let Err(e) = save_channel_closing_details(db, platform, user_channel_id, reason).await { + // This is the case when a channel is closed before funding is broadcasted due to the counterparty disconnecting or other incompatibility issue. + if e != SaveChannelClosingError::FundingTxNull.into() { error!( "Unable to update channel {} closing details in DB: {}", user_channel_id, e ); } - }); - } + } + }); + self.abort_handlers.lock().push(abort_handler); } fn handle_payment_failed(&self, payment_hash: PaymentHash) { @@ -377,16 +407,12 @@ impl LightningEventHandler { "Handling PaymentFailed event for payment_hash: {}", hex::encode(payment_hash.0) ); - let persister = self.persister.clone(); + let db = self.db.clone(); spawn(async move { - if let Ok(Some(mut payment_info)) = persister - .get_payment_from_db(payment_hash) - .await - .error_log_passthrough() - { + if let Ok(Some(mut payment_info)) = db.get_payment_from_db(payment_hash).await.error_log_passthrough() { payment_info.status = HTLCStatus::Failed; - payment_info.last_updated = now_ms() / 1000; - if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { + payment_info.last_updated = (now_ms() / 1000) as i64; + if let Err(e) = db.add_or_update_payment_in_db(payment_info).await { error!("Unable to update payment information in DB: {}", e); } } @@ -418,7 +444,7 @@ impl LightningEventHandler { let change_destination_script = Builder::build_witness_script(&my_address.hash).to_bytes().take().into(); let feerate_sat_per_1000_weight = self.platform.get_est_sat_per_1000_weight(ConfirmationTarget::Normal); let output_descriptors = &outputs.iter().collect::>(); - let spending_tx = match self.keys_manager.spend_spendable_outputs( + let claiming_tx = match self.keys_manager.spend_spendable_outputs( output_descriptors, Vec::new(), change_destination_script, @@ -432,12 +458,23 @@ impl LightningEventHandler { }, }; + let claiming_txid = claiming_tx.txid(); + let tx_hex = serialize_hex(&claiming_tx); + if let Err(e) = tokio::task::block_in_place(move || self.platform.coin.send_raw_tx(&tx_hex).wait()) { + // TODO: broadcast transaction through p2p network in this case + error!( + "Broadcasting of the claiming transaction {} failed: {}", + claiming_txid, e + ); + return; + } + let claiming_tx_inputs_value = outputs.iter().fold(0, |sum, output| match output { SpendableOutputDescriptor::StaticOutput { output, .. } => sum + output.value, SpendableOutputDescriptor::DelayedPaymentOutput(descriptor) => sum + descriptor.output.value, SpendableOutputDescriptor::StaticPaymentOutput(descriptor) => sum + descriptor.output.value, }); - let claiming_tx_outputs_value = spending_tx.output.iter().fold(0, |sum, txout| sum + txout.value); + let claiming_tx_outputs_value = claiming_tx.output.iter().fold(0, |sum, txout| sum + txout.value); if claiming_tx_inputs_value < claiming_tx_outputs_value { error!( "Claiming transaction input value {} can't be less than outputs value {}!", @@ -460,22 +497,90 @@ impl LightningEventHandler { (descriptor.outpoint.txid.to_string(), descriptor.output.value) }, }; - let claiming_txid = spending_tx.txid().to_string(); - let persister = self.persister.clone(); - spawn(async move { - ok_or_retry_after_sleep!( - persister - .add_claiming_tx_to_db( - closing_txid.clone(), - claiming_txid.clone(), - (claimed_balance as f64) - claiming_tx_fee_per_channel, - ) - .await, - TRY_LOOP_INTERVAL - ); - }); - - self.platform.broadcast_transaction(&spending_tx); + let db = self.db.clone(); + + // This doesn't need to be respawned on restart unlike add_closing_tx_to_db since Event::SpendableOutputs will be re-fired on restart + // if the spending_tx is not broadcasted. + let abort_handler = spawn_abortable(add_claiming_tx_to_db_loop( + db, + closing_txid, + claiming_txid.to_string(), + (claimed_balance as f64) - claiming_tx_fee_per_channel, + )); + self.abort_handlers.lock().push(abort_handler); } } + + fn handle_open_channel_request( + &self, + temporary_channel_id: [u8; 32], + counterparty_node_id: PublicKey, + funding_satoshis: u64, + push_msat: u64, + ) { + info!( + "Handling OpenChannelRequest from node: {} with funding value: {} and starting balance: {}", + counterparty_node_id, funding_satoshis, push_msat, + ); + + let db = self.db.clone(); + let channel_manager = self.channel_manager.clone(); + let platform = self.platform.clone(); + spawn(async move { + if let Ok(last_channel_rpc_id) = db.get_last_channel_rpc_id().await.error_log_passthrough() { + let user_channel_id = last_channel_rpc_id as u64 + 1; + if channel_manager + .accept_inbound_channel(&temporary_channel_id, user_channel_id) + .is_ok() + { + let is_public = match channel_manager + .list_channels() + .into_iter() + .find(|chan| chan.user_channel_id == user_channel_id) + { + Some(details) => details.is_public, + None => { + error!( + "Inbound channel {} details should be found by list_channels!", + user_channel_id + ); + return; + }, + }; + + let pending_channel_details = DBChannelDetails::new( + user_channel_id, + temporary_channel_id, + counterparty_node_id, + false, + is_public, + ); + if let Err(e) = db.add_channel_to_db(pending_channel_details).await { + error!("Unable to add new inbound channel {} to db: {}", user_channel_id, e); + } + + while let Some(details) = channel_manager + .list_channels() + .into_iter() + .find(|chan| chan.user_channel_id == user_channel_id) + { + if let Some(funding_tx) = details.funding_txo { + let best_block_height = platform.best_block_height(); + db.add_funding_tx_to_db( + user_channel_id as i64, + funding_tx.txid.to_string(), + funding_satoshis as i64, + best_block_height as i64, + ) + .await + .error_log(); + break; + } + + Timer::sleep(TRY_LOOP_INTERVAL).await; + } + } + } + }); + } } diff --git a/mm2src/coins/lightning/ln_filesystem_persister.rs b/mm2src/coins/lightning/ln_filesystem_persister.rs new file mode 100644 index 0000000000..7182f51eb3 --- /dev/null +++ b/mm2src/coins/lightning/ln_filesystem_persister.rs @@ -0,0 +1,399 @@ +use crate::lightning::ln_platform::Platform; +use crate::lightning::ln_storage::{LightningStorage, NodesAddressesMap, NodesAddressesMapShared, Scorer}; +use crate::lightning::ln_utils::{ChainMonitor, ChannelManager}; +use async_trait::async_trait; +use bitcoin::blockdata::constants::genesis_block; +use bitcoin::Network; +use bitcoin_hashes::hex::ToHex; +use common::async_blocking; +use common::log::LogState; +use lightning::chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate}; +use lightning::chain::keysinterface::{InMemorySigner, KeysManager, Sign}; +use lightning::chain::transaction::OutPoint; +use lightning::chain::{chainmonitor, ChannelMonitorUpdateErr}; +use lightning::routing::network_graph::NetworkGraph; +use lightning::routing::scoring::ProbabilisticScoringParameters; +use lightning::util::ser::{Readable, ReadableArgs, Writeable}; +use lightning_background_processor::Persister; +use lightning_persister::FilesystemPersister; +use mm2_io::fs::check_dir_operations; +use secp256k1::PublicKey; +use std::collections::HashMap; +use std::fs; +use std::io::{BufReader, BufWriter}; +use std::net::SocketAddr; +use std::ops::Deref; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; + +#[cfg(target_family = "unix")] use std::os::unix::io::AsRawFd; + +#[cfg(target_family = "windows")] +use {std::ffi::OsStr, std::os::windows::ffi::OsStrExt}; + +pub struct LightningFilesystemPersister { + main_path: PathBuf, + backup_path: Option, + channels_persister: FilesystemPersister, +} + +impl LightningFilesystemPersister { + /// Initialize a new LightningPersister and set the path to the individual channels' + /// files. + #[inline] + pub fn new(main_path: PathBuf, backup_path: Option) -> Self { + Self { + main_path: main_path.clone(), + backup_path, + channels_persister: FilesystemPersister::new(main_path.display().to_string()), + } + } + + /// Get the directory which was provided when this persister was initialized. + #[inline] + pub fn main_path(&self) -> PathBuf { self.main_path.clone() } + + /// Get the backup directory which was provided when this persister was initialized. + #[inline] + pub fn backup_path(&self) -> Option { self.backup_path.clone() } + + /// Get the channels_persister which was initialized when this persister was initialized. + #[inline] + pub fn channels_persister(&self) -> &FilesystemPersister { &self.channels_persister } + + pub fn monitor_backup_path(&self) -> Option { + if let Some(mut backup_path) = self.backup_path() { + backup_path.push("monitors"); + return Some(backup_path); + } + None + } + + pub fn nodes_addresses_path(&self) -> PathBuf { + let mut path = self.main_path(); + path.push("channel_nodes_data"); + path + } + + pub fn nodes_addresses_backup_path(&self) -> Option { + if let Some(mut backup_path) = self.backup_path() { + backup_path.push("channel_nodes_data"); + return Some(backup_path); + } + None + } + + pub fn network_graph_path(&self) -> PathBuf { + let mut path = self.main_path(); + path.push("network_graph"); + path + } + + pub fn scorer_path(&self) -> PathBuf { + let mut path = self.main_path(); + path.push("scorer"); + path + } + + pub fn manager_path(&self) -> PathBuf { + let mut path = self.main_path(); + path.push("manager"); + path + } +} + +#[derive(Clone)] +pub struct LightningPersisterShared(pub Arc); + +impl Deref for LightningPersisterShared { + type Target = LightningFilesystemPersister; + fn deref(&self) -> &LightningFilesystemPersister { self.0.deref() } +} + +impl Persister, Arc, Arc, Arc, Arc> + for LightningPersisterShared +{ + fn persist_manager(&self, channel_manager: &ChannelManager) -> Result<(), std::io::Error> { + FilesystemPersister::persist_manager(self.0.main_path().display().to_string(), channel_manager)?; + if let Some(backup_path) = self.0.backup_path() { + let file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(backup_path)?; + channel_manager.write(&mut BufWriter::new(file))?; + } + Ok(()) + } + + fn persist_graph(&self, network_graph: &NetworkGraph) -> Result<(), std::io::Error> { + if FilesystemPersister::persist_network_graph(self.0.main_path().display().to_string(), network_graph).is_err() + { + // Persistence errors here are non-fatal as we can just fetch the routing graph + // again later, but they may indicate a disk error which could be fatal elsewhere. + eprintln!("Warning: Failed to persist network graph, check your disk and permissions"); + } + + Ok(()) + } +} + +#[cfg(target_family = "windows")] +macro_rules! call { + ($e: expr) => { + if $e != 0 { + return Ok(()); + } else { + return Err(std::io::Error::last_os_error()); + } + }; +} + +#[cfg(target_family = "windows")] +fn path_to_windows_str>(path: T) -> Vec { + path.as_ref().encode_wide().chain(Some(0)).collect() +} + +fn write_monitor_to_file( + mut path: PathBuf, + filename: String, + monitor: &ChannelMonitor, +) -> std::io::Result<()> { + // Do a crazy dance with lots of fsync()s to be overly cautious here... + // We never want to end up in a state where we've lost the old data, or end up using the + // old data on power loss after we've returned. + // The way to atomically write a file on Unix platforms is: + // open(tmpname), write(tmpfile), fsync(tmpfile), close(tmpfile), rename(), fsync(dir) + path.push(filename); + let filename_with_path = path.display().to_string(); + let tmp_filename = format!("{}.tmp", filename_with_path); + + { + let mut f = fs::File::create(&tmp_filename)?; + monitor.write(&mut f)?; + f.sync_all()?; + } + // Fsync the parent directory on Unix. + #[cfg(target_family = "unix")] + { + fs::rename(&tmp_filename, &filename_with_path)?; + let path = Path::new(&filename_with_path).parent().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("can't find parent dir for {}", filename_with_path), + ) + })?; + let dir_file = fs::OpenOptions::new().read(true).open(path)?; + unsafe { + libc::fsync(dir_file.as_raw_fd()); + } + } + #[cfg(target_family = "windows")] + { + let src = PathBuf::from(tmp_filename); + let dst = PathBuf::from(filename_with_path.clone()); + if Path::new(&filename_with_path).exists() { + unsafe { + winapi::um::winbase::ReplaceFileW( + path_to_windows_str(dst).as_ptr(), + path_to_windows_str(src).as_ptr(), + std::ptr::null(), + winapi::um::winbase::REPLACEFILE_IGNORE_MERGE_ERRORS, + std::ptr::null_mut() as *mut winapi::ctypes::c_void, + std::ptr::null_mut() as *mut winapi::ctypes::c_void, + ) + }; + } else { + call!(unsafe { + winapi::um::winbase::MoveFileExW( + path_to_windows_str(src).as_ptr(), + path_to_windows_str(dst).as_ptr(), + winapi::um::winbase::MOVEFILE_WRITE_THROUGH | winapi::um::winbase::MOVEFILE_REPLACE_EXISTING, + ) + }); + } + } + Ok(()) +} + +impl chainmonitor::Persist for LightningFilesystemPersister { + fn persist_new_channel( + &self, + funding_txo: OutPoint, + monitor: &ChannelMonitor, + update_id: chainmonitor::MonitorUpdateId, + ) -> Result<(), ChannelMonitorUpdateErr> { + self.channels_persister + .persist_new_channel(funding_txo, monitor, update_id)?; + if let Some(backup_path) = self.monitor_backup_path() { + let filename = format!("{}_{}", funding_txo.txid.to_hex(), funding_txo.index); + write_monitor_to_file(backup_path, filename, monitor) + .map_err(|_| ChannelMonitorUpdateErr::PermanentFailure)?; + } + Ok(()) + } + + fn update_persisted_channel( + &self, + funding_txo: OutPoint, + update: &Option, + monitor: &ChannelMonitor, + update_id: chainmonitor::MonitorUpdateId, + ) -> Result<(), ChannelMonitorUpdateErr> { + self.channels_persister + .update_persisted_channel(funding_txo, update, monitor, update_id)?; + if let Some(backup_path) = self.monitor_backup_path() { + let filename = format!("{}_{}", funding_txo.txid.to_hex(), funding_txo.index); + write_monitor_to_file(backup_path, filename, monitor) + .map_err(|_| ChannelMonitorUpdateErr::PermanentFailure)?; + } + Ok(()) + } +} + +#[async_trait] +impl LightningStorage for LightningFilesystemPersister { + type Error = std::io::Error; + + async fn init_fs(&self) -> Result<(), Self::Error> { + let path = self.main_path(); + let backup_path = self.backup_path(); + async_blocking(move || { + fs::create_dir_all(path.clone())?; + if let Some(path) = backup_path { + fs::create_dir_all(path.clone())?; + check_dir_operations(&path)?; + } + check_dir_operations(&path) + }) + .await + } + + async fn is_fs_initialized(&self) -> Result { + let dir_path = self.main_path(); + let backup_dir_path = self.backup_path(); + async_blocking(move || { + if !dir_path.exists() || backup_dir_path.as_ref().map(|path| !path.exists()).unwrap_or(false) { + Ok(false) + } else if !dir_path.is_dir() { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("{} is not a directory", dir_path.display()), + )) + } else if backup_dir_path.as_ref().map(|path| !path.is_dir()).unwrap_or(false) { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Backup path is not a directory", + )) + } else { + let check_backup_ops = if let Some(backup_path) = backup_dir_path { + check_dir_operations(&backup_path).is_ok() + } else { + true + }; + check_dir_operations(&dir_path).map(|_| check_backup_ops) + } + }) + .await + } + + async fn get_nodes_addresses(&self) -> Result { + let path = self.nodes_addresses_path(); + if !path.exists() { + return Ok(HashMap::new()); + } + async_blocking(move || { + let file = fs::File::open(path)?; + let reader = BufReader::new(file); + let nodes_addresses: HashMap = + serde_json::from_reader(reader).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + nodes_addresses + .iter() + .map(|(pubkey_str, addr)| { + let pubkey = PublicKey::from_str(pubkey_str) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + Ok((pubkey, *addr)) + }) + .collect() + }) + .await + } + + async fn save_nodes_addresses(&self, nodes_addresses: NodesAddressesMapShared) -> Result<(), Self::Error> { + let path = self.nodes_addresses_path(); + let backup_path = self.nodes_addresses_backup_path(); + async_blocking(move || { + let nodes_addresses: HashMap = nodes_addresses + .lock() + .iter() + .map(|(pubkey, addr)| (pubkey.to_string(), *addr)) + .collect(); + + let file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path)?; + serde_json::to_writer(file, &nodes_addresses) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + + if let Some(path) = backup_path { + let file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path)?; + serde_json::to_writer(file, &nodes_addresses) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + } + + Ok(()) + }) + .await + } + + async fn get_network_graph(&self, network: Network) -> Result { + let path = self.network_graph_path(); + if !path.exists() { + return Ok(NetworkGraph::new(genesis_block(network).header.block_hash())); + } + async_blocking(move || { + let file = fs::File::open(path)?; + common::log::info!("Reading the saved lightning network graph from file, this can take some time!"); + NetworkGraph::read(&mut BufReader::new(file)) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) + }) + .await + } + + async fn get_scorer(&self, network_graph: Arc) -> Result { + let path = self.scorer_path(); + if !path.exists() { + return Ok(Scorer::new(ProbabilisticScoringParameters::default(), network_graph)); + } + async_blocking(move || { + let file = fs::File::open(path)?; + Scorer::read( + &mut BufReader::new(file), + (ProbabilisticScoringParameters::default(), network_graph), + ) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) + }) + .await + } + + async fn save_scorer(&self, scorer: Arc>) -> Result<(), Self::Error> { + let path = self.scorer_path(); + async_blocking(move || { + let scorer = scorer.lock().unwrap(); + let file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path)?; + scorer.write(&mut BufWriter::new(file)) + }) + .await + } +} diff --git a/mm2src/coins/lightning/ln_p2p.rs b/mm2src/coins/lightning/ln_p2p.rs index 00bd5cdd5b..cb4ab8c14a 100644 --- a/mm2src/coins/lightning/ln_p2p.rs +++ b/mm2src/coins/lightning/ln_p2p.rs @@ -7,7 +7,6 @@ use lightning::ln::msgs::NetAddress; use lightning::ln::peer_handler::{IgnoringMessageHandler, MessageHandler, SimpleArcPeerManager}; use lightning::routing::network_graph::{NetGraphMsgHandler, NetworkGraph}; use lightning_net_tokio::SocketDescriptor; -use lightning_persister::storage::NodesAddressesMapShared; use mm2_net::ip_addr::fetch_external_ip; use rand::RngCore; use secp256k1::SecretKey; diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index dfc3f24554..38535624a3 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -1,10 +1,8 @@ use super::*; -use crate::lightning::ln_errors::{FindWatchedOutputSpendError, GetHeaderError, GetTxError, SaveChannelClosingError, - SaveChannelClosingResult}; -use crate::utxo::rpc_clients::{electrum_script_hash, BestBlock as RpcBestBlock, BlockHashOrHeight, - ElectrumBlockHeader, ElectrumClient, ElectrumNonce, EstimateFeeMethod, - UtxoRpcClientEnum, UtxoRpcError}; -use crate::utxo::utxo_common; +use crate::lightning::ln_errors::{SaveChannelClosingError, SaveChannelClosingResult}; +use crate::utxo::rpc_clients::{BestBlock as RpcBestBlock, BlockHashOrHeight, ElectrumBlockHeader, ElectrumClient, + ElectrumNonce, EstimateFeeMethod, UtxoRpcClientEnum}; +use crate::utxo::spv::{ConfirmedTransactionInfo, SimplePaymentVerification}; use crate::utxo::utxo_standard::UtxoStandardCoin; use crate::{MarketCoinOps, MmCoin}; use bitcoin::blockdata::block::BlockHeader; @@ -14,15 +12,16 @@ use bitcoin::consensus::encode::{deserialize, serialize_hex}; use bitcoin::hash_types::{BlockHash, TxMerkleNode, Txid}; use bitcoin_hashes::{sha256d, Hash}; use common::executor::{spawn, Timer}; -use common::jsonrpc_client::JsonRpcErrorType; use common::log::{debug, error, info}; use futures::compat::Future01CompatExt; +use futures::future::join_all; use keys::hash::H256; use lightning::chain::{chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator}, Confirm, Filter, WatchedOutput}; -use rpc::v1::types::H256 as H256Json; +use rpc::v1::types::{Bytes as BytesJson, H256 as H256Json}; +use spv_validation::spv_proof::TRY_SPV_PROOF_INTERVAL; use std::cmp; -use std::convert::TryFrom; +use std::convert::{TryFrom, TryInto}; use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; const CHECK_FOR_NEW_BEST_BLOCK_INTERVAL: f64 = 60.; @@ -32,55 +31,8 @@ const TRY_LOOP_INTERVAL: f64 = 60.; #[inline] pub fn h256_json_from_txid(txid: Txid) -> H256Json { H256Json::from(txid.as_hash().into_inner()).reversed() } -struct TxWithBlockInfo { - tx: Transaction, - block_header: BlockHeader, - block_height: u64, -} - -async fn get_block_header(electrum_client: &ElectrumClient, height: u64) -> Result { - Ok(deserialize( - &electrum_client.blockchain_block_header(height).compat().await?, - )?) -} - -async fn find_watched_output_spend_with_header( - electrum_client: &ElectrumClient, - output: &WatchedOutput, -) -> Result, FindWatchedOutputSpendError> { - // from_block parameter is not used in find_output_spend for electrum clients - let utxo_client: UtxoRpcClientEnum = electrum_client.clone().into(); - let tx_hash = H256::from(output.outpoint.txid.as_hash().into_inner()); - let output_spend = match utxo_client - .find_output_spend( - tx_hash, - output.script_pubkey.as_ref(), - output.outpoint.index.into(), - BlockHashOrHeight::Hash(Default::default()), - ) - .compat() - .await - .map_err(FindWatchedOutputSpendError::RpcError)? - { - Some(output) => output, - None => return Ok(None), - }; - - let height = match output_spend.spent_in_block { - BlockHashOrHeight::Height(h) => h, - _ => return Err(FindWatchedOutputSpendError::HashNotHeight), - }; - let block_header = get_block_header(electrum_client, height as u64) - .await - .map_err(FindWatchedOutputSpendError::GetHeaderError)?; - let spending_tx = Transaction::try_from(output_spend.spending_tx)?; - - Ok(Some(TxWithBlockInfo { - tx: spending_tx, - block_header, - block_height: height as u64, - })) -} +#[inline] +pub fn h256_from_txid(txid: Txid) -> H256 { H256::from(txid.as_hash().into_inner()) } pub async fn get_best_header(best_header_listener: &ElectrumClient) -> EnableLightningResult { best_header_listener @@ -104,20 +56,8 @@ pub async fn update_best_block( return; }, }; - let prev_blockhash = match sha256d::Hash::from_slice(&h.prev_block_hash.0) { - Ok(h) => h, - Err(e) => { - error!("Error while parsing previous block hash for lightning node: {}", e); - return; - }, - }; - let merkle_root = match sha256d::Hash::from_slice(&h.merkle_root.0) { - Ok(h) => h, - Err(e) => { - error!("Error while parsing merkle root for lightning node: {}", e); - return; - }, - }; + let prev_blockhash = sha256d::Hash::from_inner(h.prev_block_hash.0); + let merkle_root = sha256d::Hash::from_inner(h.merkle_root.0); ( BlockHeader { version: h.version as i32, @@ -148,7 +88,7 @@ pub async fn update_best_block( pub async fn ln_best_block_update_loop( platform: Arc, - persister: Arc, + db: SqliteLightningDB, chain_monitor: Arc, channel_manager: Arc, best_header_listener: ElectrumClient, @@ -156,15 +96,18 @@ pub async fn ln_best_block_update_loop( ) { let mut current_best_block = best_block; loop { + // Transactions confirmations check can be done at every CHECK_FOR_NEW_BEST_BLOCK_INTERVAL instead of at every new block + // in case a transaction confirmation fails due to electrums being down. This way there will be no need to wait for a new + // block to confirm such transaction and causing delays. + platform + .process_txs_confirmations(&best_header_listener, &db, &chain_monitor, &channel_manager) + .await; let best_header = ok_or_continue_after_sleep!(get_best_header(&best_header_listener).await, TRY_LOOP_INTERVAL); if current_best_block != best_header.clone().into() { platform.update_best_block_height(best_header.block_height()); platform .process_txs_unconfirmations(&chain_monitor, &channel_manager) .await; - platform - .process_txs_confirmations(&best_header_listener, &persister, &chain_monitor, &channel_manager) - .await; current_best_block = best_header.clone().into(); update_best_block(&chain_monitor, &channel_manager, best_header).await; } @@ -172,22 +115,15 @@ pub async fn ln_best_block_update_loop( } } -struct ConfirmedTransactionInfo { - txid: Txid, - header: BlockHeader, - index: usize, - transaction: Transaction, - height: u32, -} - -impl ConfirmedTransactionInfo { - fn new(txid: Txid, header: BlockHeader, index: usize, transaction: Transaction, height: u32) -> Self { - ConfirmedTransactionInfo { - txid, - header, - index, - transaction, - height, +async fn get_funding_tx_bytes_loop(rpc_client: &UtxoRpcClientEnum, tx_hash: H256Json) -> BytesJson { + loop { + match rpc_client.get_transaction_bytes(&tx_hash).compat().await { + Ok(res) => break res, + Err(e) => { + error!("error {}", e); + Timer::sleep(TRY_LOOP_INTERVAL).await; + continue; + }, } } } @@ -202,7 +138,7 @@ pub struct Platform { /// estimate_fee_sat fails. pub default_fees_and_confirmations: PlatformCoinConfirmations, /// This cache stores the transactions that the LN node has interest in. - pub registered_txs: PaMutex>>, + pub registered_txs: PaMutex>, /// This cache stores the outputs that the LN node has interest in. pub registered_outputs: PaMutex>, /// This cache stores transactions to be broadcasted once the other node accepts the channel @@ -221,7 +157,7 @@ impl Platform { network, best_block_height: AtomicU64::new(0), default_fees_and_confirmations, - registered_txs: PaMutex::new(HashMap::new()), + registered_txs: PaMutex::new(HashSet::new()), registered_outputs: PaMutex::new(Vec::new()), unsigned_funding_txs: PaMutex::new(HashMap::new()), } @@ -238,12 +174,9 @@ impl Platform { #[inline] pub fn best_block_height(&self) -> u64 { self.best_block_height.load(AtomicOrdering::Relaxed) } - pub fn add_tx(&self, txid: Txid, script_pubkey: Script) { + pub fn add_tx(&self, txid: Txid) { let mut registered_txs = self.registered_txs.lock(); - registered_txs - .entry(txid) - .or_insert_with(HashSet::new) - .insert(script_pubkey); + registered_txs.insert(txid); } pub fn add_output(&self, output: WatchedOutput) { @@ -251,36 +184,12 @@ impl Platform { registered_outputs.push(output); } - async fn get_tx_if_onchain(&self, txid: Txid) -> Result, GetTxError> { - let txid = h256_json_from_txid(txid); - match self - .rpc_client() - .get_transaction_bytes(&txid) - .compat() - .await - .map_err(|e| e.into_inner()) - { - Ok(bytes) => Ok(Some(deserialize(&bytes.into_vec())?)), - Err(err) => { - if let UtxoRpcError::ResponseParseError(ref json_err) = err { - if let JsonRpcErrorType::Response(_, json) = &json_err.error { - if let Some(message) = json["message"].as_str() { - if message.contains(utxo_common::NO_TX_ERROR_CODE) { - return Ok(None); - } - } - } - } - Err(err.into()) - }, - } - } - async fn process_tx_for_unconfirmation(&self, txid: Txid, monitor: &T) where T: Confirm, { - match self.get_tx_if_onchain(txid).await { + let rpc_txid = h256_json_from_txid(txid); + match self.rpc_client().get_tx_if_onchain(&rpc_txid).await { Ok(Some(_)) => {}, Ok(None) => { info!( @@ -288,6 +197,10 @@ impl Platform { txid, ); monitor.transaction_unconfirmed(&txid); + // If a transaction is unconfirmed due to a block reorganization; LDK will rebroadcast it. + // In this case, this transaction needs to be added again to the registered transactions + // to start watching for it on the chain again. + self.add_tx(txid); }, Err(e) => error!( "Error while trying to check if the transaction {} is discarded or not :{:?}", @@ -312,48 +225,50 @@ impl Platform { async fn get_confirmed_registered_txs(&self, client: &ElectrumClient) -> Vec { let registered_txs = self.registered_txs.lock().clone(); - let mut confirmed_registered_txs = Vec::new(); - for (txid, scripts) in registered_txs { - if let Some(transaction) = - ok_or_continue_after_sleep!(self.get_tx_if_onchain(txid).await, TRY_LOOP_INTERVAL) - { - for (_, vout) in transaction.output.iter().enumerate() { - if scripts.contains(&vout.script_pubkey) { - let script_hash = hex::encode(electrum_script_hash(vout.script_pubkey.as_ref())); - let history = ok_or_retry_after_sleep!( - client.scripthash_get_history(&script_hash).compat().await, - TRY_LOOP_INTERVAL - ); - for item in history { - let rpc_txid = h256_json_from_txid(txid); - if item.tx_hash == rpc_txid && item.height > 0 { - let height = item.height as u64; - let header = - ok_or_retry_after_sleep!(get_block_header(client, height).await, TRY_LOOP_INTERVAL); - let index = ok_or_retry_after_sleep!( - client - .blockchain_transaction_get_merkle(rpc_txid, height) - .compat() - .await, - TRY_LOOP_INTERVAL - ) - .pos; - let confirmed_transaction_info = ConfirmedTransactionInfo::new( - txid, - header, - index, - transaction.clone(), - height as u32, - ); - confirmed_registered_txs.push(confirmed_transaction_info); - self.registered_txs.lock().remove(&txid); - } - } - } - } - } - } - confirmed_registered_txs + + let on_chain_txs_futs = registered_txs + .into_iter() + .map(|txid| async move { + let rpc_txid = h256_json_from_txid(txid); + self.rpc_client().get_tx_if_onchain(&rpc_txid).await + }) + .collect::>(); + let on_chain_txs = join_all(on_chain_txs_futs) + .await + .into_iter() + .filter_map(|maybe_tx| match maybe_tx { + Ok(maybe_tx) => maybe_tx, + Err(e) => { + error!( + "Error while trying to figure if transaction is on-chain or not: {:?}", + e + ); + None + }, + }); + + let confirmed_transactions_futs = on_chain_txs + .map(|transaction| async move { + client + .validate_spv_proof(&transaction, (now_ms() / 1000) + TRY_SPV_PROOF_INTERVAL) + .await + }) + .collect::>(); + join_all(confirmed_transactions_futs) + .await + .into_iter() + .filter_map(|confirmed_transaction| match confirmed_transaction { + Ok(confirmed_tx) => { + let txid = Txid::from_hash(confirmed_tx.tx.hash().reversed().to_sha256d()); + self.registered_txs.lock().remove(&txid); + Some(confirmed_tx) + }, + Err(e) => { + error!("Error verifying transaction: {:?}", e); + None + }, + }) + .collect() } async fn append_spent_registered_output_txs( @@ -361,47 +276,76 @@ impl Platform { transactions_to_confirm: &mut Vec, client: &ElectrumClient, ) { - let mut outputs_to_remove = Vec::new(); let registered_outputs = self.registered_outputs.lock().clone(); - for output in registered_outputs { - if let Some(tx_info) = ok_or_continue_after_sleep!( - find_watched_output_spend_with_header(client, &output).await, - TRY_LOOP_INTERVAL - ) { - if !transactions_to_confirm - .iter() - .any(|info| info.txid == tx_info.tx.txid()) - { - let rpc_txid = h256_json_from_txid(tx_info.tx.txid()); - let index = ok_or_retry_after_sleep!( - client - .blockchain_transaction_get_merkle(rpc_txid, tx_info.block_height) - .compat() - .await, - TRY_LOOP_INTERVAL + + let spent_outputs_info_fut = registered_outputs + .into_iter() + .map(|output| async move { + self.rpc_client() + .find_output_spend( + h256_from_txid(output.outpoint.txid), + output.script_pubkey.as_ref(), + output.outpoint.index.into(), + BlockHashOrHeight::Hash(Default::default()), ) - .pos; - let confirmed_transaction_info = ConfirmedTransactionInfo::new( - tx_info.tx.txid(), - tx_info.block_header, - index, - tx_info.tx, - tx_info.block_height as u32, - ); - transactions_to_confirm.push(confirmed_transaction_info); - } - outputs_to_remove.push(output); - } - } - self.registered_outputs - .lock() - .retain(|output| !outputs_to_remove.contains(output)); + .compat() + .await + }) + .collect::>(); + let mut spent_outputs_info = join_all(spent_outputs_info_fut) + .await + .into_iter() + .filter_map(|maybe_spent| match maybe_spent { + Ok(maybe_spent) => maybe_spent, + Err(e) => { + error!("Error while trying to figure if output is spent or not: {:?}", e); + None + }, + }) + .collect::>(); + spent_outputs_info.retain(|output| { + !transactions_to_confirm + .iter() + .any(|info| info.tx.hash() == output.spending_tx.hash()) + }); + + let confirmed_transactions_futs = spent_outputs_info + .into_iter() + .map(|output| async move { + client + .validate_spv_proof(&output.spending_tx, (now_ms() / 1000) + TRY_SPV_PROOF_INTERVAL) + .await + }) + .collect::>(); + let mut confirmed_transaction_info = join_all(confirmed_transactions_futs) + .await + .into_iter() + .filter_map(|confirmed_transaction| match confirmed_transaction { + Ok(confirmed_tx) => { + self.registered_outputs.lock().retain(|output| { + !confirmed_tx + .tx + .clone() + .inputs + .into_iter() + .any(|txin| txin.previous_output.hash == h256_from_txid(output.outpoint.txid)) + }); + Some(confirmed_tx) + }, + Err(e) => { + error!("Error verifying transaction: {:?}", e); + None + }, + }) + .collect(); + + transactions_to_confirm.append(&mut confirmed_transaction_info); } pub async fn process_txs_confirmations( &self, client: &ElectrumClient, - persister: &LightningPersister, + db: &SqliteLightningDB, chain_monitor: &ChainMonitor, channel_manager: &ChannelManager, ) { @@ -412,10 +356,10 @@ impl Platform { transactions_to_confirm.sort_by(|a, b| (a.height, a.index).cmp(&(b.height, b.index))); for confirmed_transaction_info in transactions_to_confirm { - let best_block_height = self.best_block_height(); - if let Err(e) = persister + let best_block_height = self.best_block_height() as i64; + if let Err(e) = db .update_funding_tx_block_height( - confirmed_transaction_info.transaction.txid().to_string(), + confirmed_transaction_info.tx.hash().reversed().to_string(), best_block_height, ) .await @@ -423,25 +367,25 @@ impl Platform { error!("Unable to update the funding tx block height in DB: {}", e); } channel_manager.transactions_confirmed( - &confirmed_transaction_info.header, + &confirmed_transaction_info.header.clone().into(), &[( - confirmed_transaction_info.index, - &confirmed_transaction_info.transaction, + confirmed_transaction_info.index as usize, + &confirmed_transaction_info.tx.clone().into(), )], - confirmed_transaction_info.height, + confirmed_transaction_info.height as u32, ); chain_monitor.transactions_confirmed( - &confirmed_transaction_info.header, + &confirmed_transaction_info.header.into(), &[( - confirmed_transaction_info.index, - &confirmed_transaction_info.transaction, + confirmed_transaction_info.index as usize, + &confirmed_transaction_info.tx.into(), )], - confirmed_transaction_info.height, + confirmed_transaction_info.height as u32, ); } } - pub async fn get_channel_closing_tx(&self, channel_details: SqlChannelDetails) -> SaveChannelClosingResult { + pub async fn get_channel_closing_tx(&self, channel_details: DBChannelDetails) -> SaveChannelClosingResult { let from_block = channel_details .funding_generated_in_block .ok_or_else(|| MmError::new(SaveChannelClosingError::BlockHeightNull))?; @@ -453,17 +397,14 @@ impl Platform { let tx_hash = H256Json::from_str(&tx_id).map_to_mm(|e| SaveChannelClosingError::FundingTxParseError(e.to_string()))?; - let funding_tx_bytes = ok_or_retry_after_sleep!( - self.rpc_client().get_transaction_bytes(&tx_hash).compat().await, - TRY_LOOP_INTERVAL - ); + let funding_tx_bytes = get_funding_tx_bytes_loop(self.rpc_client(), tx_hash).await; let closing_tx = self .coin .wait_for_tx_spend( &funding_tx_bytes.into_vec(), (now_ms() / 1000) + 3600, - from_block, + from_block.try_into()?, &None, ) .compat() @@ -521,6 +462,7 @@ impl BroadcasterInterface for Platform { spawn(async move { match fut.compat().await { Ok(id) => info!("Transaction broadcasted successfully: {:?} ", id), + // TODO: broadcast transaction through p2p network in case of error Err(e) => error!("Broadcast transaction {} failed: {}", txid, e), } }); @@ -530,12 +472,11 @@ impl BroadcasterInterface for Platform { impl Filter for Platform { // Watches for this transaction on-chain #[inline] - fn register_tx(&self, txid: &Txid, script_pubkey: &Script) { self.add_tx(*txid, script_pubkey.clone()); } + fn register_tx(&self, txid: &Txid, _script_pubkey: &Script) { self.add_tx(*txid); } // Watches for any transactions that spend this output on-chain fn register_output(&self, output: WatchedOutput) -> Option<(usize, Transaction)> { self.add_output(output.clone()); - let block_hash = match output.block_hash { Some(h) => H256Json::from(h.as_hash().into_inner()), None => return None, @@ -545,29 +486,20 @@ impl Filter for Platform { // the filter interface which includes register_output and register_tx should be used for electrum clients only, // this is the reason for initializing the filter as an option in the start_lightning function as it will be None // when implementing lightning for native clients - let output_spend_info = tokio::task::block_in_place(move || { - let delay = TRY_LOOP_INTERVAL as u64; - ok_or_retry_after_sleep_sync!( - self.rpc_client() - .find_output_spend( - H256::from(output.outpoint.txid.as_hash().into_inner()), - output.script_pubkey.as_ref(), - output.outpoint.index.into(), - BlockHashOrHeight::Hash(block_hash), - ) - .wait(), - delay - ) - }); + let output_spend_fut = self.rpc_client().find_output_spend( + h256_from_txid(output.outpoint.txid), + output.script_pubkey.as_ref(), + output.outpoint.index.into(), + BlockHashOrHeight::Hash(block_hash), + ); + let maybe_output_spend_res = + tokio::task::block_in_place(move || output_spend_fut.wait()).error_log_passthrough(); - if let Some(info) = output_spend_info { - match Transaction::try_from(info.spending_tx) { - Ok(tx) => Some((info.input_index, tx)), - Err(e) => { - error!("Can't convert transaction error: {}", e.to_string()); - return None; - }, - }; + if let Ok(Some(spent_output_info)) = maybe_output_spend_res { + match Transaction::try_from(spent_output_info.spending_tx) { + Ok(spending_tx) => return Some((spent_output_info.input_index, spending_tx)), + Err(e) => error!("Can't convert transaction error: {}", e.to_string()), + } } None diff --git a/mm2src/coins/lightning/ln_sql.rs b/mm2src/coins/lightning/ln_sql.rs new file mode 100644 index 0000000000..2cbeb98116 --- /dev/null +++ b/mm2src/coins/lightning/ln_sql.rs @@ -0,0 +1,1435 @@ +use crate::lightning::ln_db::{ChannelType, ChannelVisibility, ClosedChannelsFilter, DBChannelDetails, DBPaymentInfo, + DBPaymentsFilter, GetClosedChannelsResult, GetPaymentsResult, HTLCStatus, LightningDB, + PaymentType}; +use async_trait::async_trait; +use common::{async_blocking, PagingOptionsEnum}; +use db_common::sqlite::rusqlite::{Error as SqlError, Row, ToSql, NO_PARAMS}; +use db_common::sqlite::sql_builder::SqlBuilder; +use db_common::sqlite::{h256_option_slice_from_row, h256_slice_from_row, offset_by_id, query_single_row, + sql_text_conversion_err, string_from_row, validate_table_name, SqlNamedParams, + SqliteConnShared, CHECK_TABLE_EXISTS_SQL}; +use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; +use secp256k1::PublicKey; +use std::convert::TryInto; +use std::str::FromStr; + +fn channels_history_table(ticker: &str) -> String { ticker.to_owned() + "_channels_history" } + +fn payments_history_table(ticker: &str) -> String { ticker.to_owned() + "_payments_history" } + +fn create_channels_history_table_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "CREATE TABLE IF NOT EXISTS {} ( + id INTEGER NOT NULL PRIMARY KEY, + rpc_id INTEGER NOT NULL UNIQUE, + channel_id VARCHAR(255) NOT NULL, + counterparty_node_id VARCHAR(255) NOT NULL, + funding_tx VARCHAR(255), + funding_value INTEGER, + funding_generated_in_block Integer, + closing_tx VARCHAR(255), + closure_reason TEXT, + claiming_tx VARCHAR(255), + claimed_balance REAL, + is_outbound INTEGER NOT NULL, + is_public INTEGER NOT NULL, + is_closed INTEGER NOT NULL, + created_at INTEGER NOT NULL, + closed_at INTEGER + );", + table_name + ); + + Ok(sql) +} + +fn create_payments_history_table_sql(for_coin: &str) -> Result { + let table_name = payments_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "CREATE TABLE IF NOT EXISTS {} ( + id INTEGER NOT NULL PRIMARY KEY, + payment_hash VARCHAR(255) NOT NULL UNIQUE, + destination VARCHAR(255), + description VARCHAR(641) NOT NULL, + preimage VARCHAR(255), + secret VARCHAR(255), + amount_msat INTEGER, + fee_paid_msat INTEGER, + is_outbound INTEGER NOT NULL, + status VARCHAR(255) NOT NULL, + created_at INTEGER NOT NULL, + last_updated INTEGER NOT NULL + );", + table_name + ); + + Ok(sql) +} + +fn insert_channel_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "INSERT INTO {} ( + rpc_id, + channel_id, + counterparty_node_id, + is_outbound, + is_public, + is_closed, + created_at + ) VALUES ( + ?1, ?2, ?3, ?4, ?5, ?6, ?7 + );", + table_name + ); + + Ok(sql) +} + +fn upsert_payment_sql(for_coin: &str) -> Result { + let table_name = payments_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "INSERT OR REPLACE INTO {} ( + payment_hash, + destination, + description, + preimage, + secret, + amount_msat, + fee_paid_msat, + is_outbound, + status, + created_at, + last_updated + ) VALUES ( + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 + );", + table_name + ); + + Ok(sql) +} + +fn select_channel_by_rpc_id_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "SELECT + rpc_id, + channel_id, + counterparty_node_id, + funding_tx, + funding_value, + funding_generated_in_block, + closing_tx, + closure_reason, + claiming_tx, + claimed_balance, + is_outbound, + is_public, + is_closed, + created_at, + closed_at + FROM + {} + WHERE + rpc_id=?1", + table_name + ); + + Ok(sql) +} + +fn select_payment_by_hash_sql(for_coin: &str) -> Result { + let table_name = payments_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "SELECT + payment_hash, + destination, + description, + preimage, + secret, + amount_msat, + fee_paid_msat, + status, + is_outbound, + created_at, + last_updated + FROM + {} + WHERE + payment_hash=?1;", + table_name + ); + + Ok(sql) +} + +fn channel_details_from_row(row: &Row<'_>) -> Result { + let channel_details = DBChannelDetails { + rpc_id: row.get(0)?, + channel_id: row.get(1)?, + counterparty_node_id: row.get(2)?, + funding_tx: row.get(3)?, + funding_value: row.get(4)?, + funding_generated_in_block: row.get(5)?, + closing_tx: row.get(6)?, + closure_reason: row.get(7)?, + claiming_tx: row.get(8)?, + claimed_balance: row.get(9)?, + is_outbound: row.get(10)?, + is_public: row.get(11)?, + is_closed: row.get(12)?, + created_at: row.get(13)?, + closed_at: row.get(14)?, + }; + Ok(channel_details) +} + +fn payment_info_from_row(row: &Row<'_>) -> Result { + let is_outbound = row.get::<_, bool>(8)?; + let payment_type = if is_outbound { + PaymentType::OutboundPayment { + destination: PublicKey::from_str(&row.get::<_, String>(1)?).map_err(|e| sql_text_conversion_err(1, e))?, + } + } else { + PaymentType::InboundPayment + }; + + let payment_info = DBPaymentInfo { + payment_hash: PaymentHash(h256_slice_from_row::(row, 0)?), + payment_type, + description: row.get(2)?, + preimage: h256_option_slice_from_row::(row, 3)?.map(PaymentPreimage), + secret: h256_option_slice_from_row::(row, 4)?.map(PaymentSecret), + amt_msat: row.get(5)?, + fee_paid_msat: row.get(6)?, + status: HTLCStatus::from_str(&row.get::<_, String>(7)?)?, + created_at: row.get(9)?, + last_updated: row.get(10)?, + }; + Ok(payment_info) +} + +fn get_last_channel_rpc_id_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!("SELECT IFNULL(MAX(rpc_id), 0) FROM {};", table_name); + + Ok(sql) +} + +fn update_funding_tx_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "UPDATE {} SET + funding_tx = ?1, + funding_value = ?2, + funding_generated_in_block = ?3 + WHERE + rpc_id = ?4;", + table_name + ); + + Ok(sql) +} + +fn update_funding_tx_block_height_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "UPDATE {} SET funding_generated_in_block = ?1 WHERE funding_tx = ?2;", + table_name + ); + + Ok(sql) +} + +fn update_channel_to_closed_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "UPDATE {} SET closure_reason = ?1, is_closed = ?2, closed_at = ?3 WHERE rpc_id = ?4;", + table_name + ); + + Ok(sql) +} + +fn update_closing_tx_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!("UPDATE {} SET closing_tx = ?1 WHERE rpc_id = ?2;", table_name); + + Ok(sql) +} + +fn get_channels_builder_preimage(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let mut sql_builder = SqlBuilder::select_from(table_name); + sql_builder.and_where("is_closed = 1"); + Ok(sql_builder) +} + +fn add_fields_to_get_channels_sql_builder(sql_builder: &mut SqlBuilder) { + sql_builder + .field("rpc_id") + .field("channel_id") + .field("counterparty_node_id") + .field("funding_tx") + .field("funding_value") + .field("funding_generated_in_block") + .field("closing_tx") + .field("closure_reason") + .field("claiming_tx") + .field("claimed_balance") + .field("is_outbound") + .field("is_public") + .field("is_closed") + .field("created_at") + .field("closed_at"); +} + +fn finalize_get_channels_sql_builder(sql_builder: &mut SqlBuilder, offset: usize, limit: usize) { + sql_builder.offset(offset); + sql_builder.limit(limit); + sql_builder.order_desc("closed_at"); +} + +fn apply_get_channels_filter<'a>( + builder: &mut SqlBuilder, + params: &mut SqlNamedParams<'a>, + filter: &'a ClosedChannelsFilter, +) { + if let Some(channel_id) = &filter.channel_id { + builder.and_where("channel_id = :channel_id"); + params.push((":channel_id", channel_id)); + } + + if let Some(counterparty_node_id) = &filter.counterparty_node_id { + builder.and_where("counterparty_node_id = :counterparty_node_id"); + params.push((":counterparty_node_id", counterparty_node_id)); + } + + if let Some(funding_tx) = &filter.funding_tx { + builder.and_where("funding_tx = :funding_tx"); + params.push((":funding_tx", funding_tx)); + } + + if let Some(from_funding_value) = &filter.from_funding_value { + builder.and_where("funding_value >= :from_funding_value"); + params.push((":from_funding_value", from_funding_value)); + } + + if let Some(to_funding_value) = &filter.to_funding_value { + builder.and_where("funding_value <= :to_funding_value"); + params.push((":to_funding_value", to_funding_value)); + } + + if let Some(closing_tx) = &filter.closing_tx { + builder.and_where("closing_tx = :closing_tx"); + params.push((":closing_tx", closing_tx)); + } + + if let Some(closure_reason) = &filter.closure_reason { + builder.and_where(format!("closure_reason LIKE '%{}%'", closure_reason)); + } + + if let Some(claiming_tx) = &filter.claiming_tx { + builder.and_where("claiming_tx = :claiming_tx"); + params.push((":claiming_tx", claiming_tx)); + } + + if let Some(from_claimed_balance) = &filter.from_claimed_balance { + builder.and_where("claimed_balance >= :from_claimed_balance"); + params.push((":from_claimed_balance", from_claimed_balance)); + } + + if let Some(to_claimed_balance) = &filter.to_claimed_balance { + builder.and_where("claimed_balance <= :to_claimed_balance"); + params.push((":to_claimed_balance", to_claimed_balance)); + } + + if let Some(channel_type) = &filter.channel_type { + let is_outbound = match channel_type { + ChannelType::Outbound => &true, + ChannelType::Inbound => &false, + }; + + builder.and_where("is_outbound = :is_outbound"); + params.push((":is_outbound", is_outbound)); + } + + if let Some(channel_visibility) = &filter.channel_visibility { + let is_public = match channel_visibility { + ChannelVisibility::Public => &true, + ChannelVisibility::Private => &false, + }; + + builder.and_where("is_public = :is_public"); + params.push((":is_public", is_public)); + } +} + +fn get_payments_builder_preimage(for_coin: &str) -> Result { + let table_name = payments_history_table(for_coin); + validate_table_name(&table_name)?; + + Ok(SqlBuilder::select_from(table_name)) +} + +fn finalize_get_payments_sql_builder(sql_builder: &mut SqlBuilder, offset: usize, limit: usize) { + sql_builder + .field("payment_hash") + .field("destination") + .field("description") + .field("preimage") + .field("secret") + .field("amount_msat") + .field("fee_paid_msat") + .field("status") + .field("is_outbound") + .field("created_at") + .field("last_updated"); + sql_builder.offset(offset); + sql_builder.limit(limit); + sql_builder.order_desc("last_updated"); +} + +fn apply_get_payments_filter<'a>( + builder: &mut SqlBuilder, + params: &mut SqlNamedParams<'a>, + filter: &'a DBPaymentsFilter, +) { + if let Some(dest) = &filter.destination { + builder.and_where("destination = :dest"); + params.push((":dest", dest)); + } + + if let Some(outbound) = &filter.is_outbound { + builder.and_where("is_outbound = :is_outbound"); + params.push((":is_outbound", outbound)); + } + + if let Some(description) = &filter.description { + builder.and_where(format!("description LIKE '%{}%'", description)); + } + + if let Some(status) = &filter.status { + builder.and_where("status = :status"); + params.push((":status", status)); + } + + if let Some(from_amount) = &filter.from_amount_msat { + builder.and_where("amount_msat >= :from_amount"); + params.push((":from_amount", from_amount)); + } + + if let Some(to_amount) = &filter.to_amount_msat { + builder.and_where("amount_msat <= :to_amount"); + params.push((":to_amount", to_amount)); + } + + if let Some(from_fee) = &filter.from_fee_paid_msat { + builder.and_where("fee_paid_msat >= :from_fee"); + params.push((":from_fee", from_fee)); + } + + if let Some(to_fee) = &filter.to_fee_paid_msat { + builder.and_where("fee_paid_msat <= :to_fee"); + params.push((":to_fee", to_fee)); + } + + if let Some(from_time) = &filter.from_timestamp { + builder.and_where("created_at >= :from_time"); + params.push((":from_time", from_time)); + } + + if let Some(to_time) = &filter.to_timestamp { + builder.and_where("created_at <= :to_time"); + params.push((":to_time", to_time)); + } +} + +fn update_claiming_tx_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "UPDATE {} SET claiming_tx = ?1, claimed_balance = ?2 WHERE closing_tx = ?3;", + table_name + ); + + Ok(sql) +} + +#[derive(Clone)] +pub struct SqliteLightningDB { + db_ticker: String, + sqlite_connection: SqliteConnShared, +} + +impl SqliteLightningDB { + pub fn new(ticker: String, sqlite_connection: SqliteConnShared) -> Self { + Self { + db_ticker: ticker.replace('-', "_"), + sqlite_connection, + } + } +} + +#[async_trait] +impl LightningDB for SqliteLightningDB { + type Error = SqlError; + + async fn init_db(&self) -> Result<(), Self::Error> { + let sqlite_connection = self.sqlite_connection.clone(); + + let sql_channels_history = create_channels_history_table_sql(self.db_ticker.as_str())?; + let sql_payments_history = create_payments_history_table_sql(self.db_ticker.as_str())?; + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + conn.execute(&sql_channels_history, NO_PARAMS).map(|_| ())?; + conn.execute(&sql_payments_history, NO_PARAMS).map(|_| ())?; + Ok(()) + }) + .await + } + + async fn is_db_initialized(&self) -> Result { + let channels_history_table = channels_history_table(self.db_ticker.as_str()); + validate_table_name(&channels_history_table)?; + let payments_history_table = payments_history_table(self.db_ticker.as_str()); + validate_table_name(&payments_history_table)?; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + let channels_history_initialized = + query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [channels_history_table], string_from_row)?; + let payments_history_initialized = + query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [payments_history_table], string_from_row)?; + Ok(channels_history_initialized.is_some() && payments_history_initialized.is_some()) + }) + .await + } + + async fn get_last_channel_rpc_id(&self) -> Result { + let sql = get_last_channel_rpc_id_sql(self.db_ticker.as_str())?; + let sqlite_connection = self.sqlite_connection.clone(); + + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + let count: u32 = conn.query_row(&sql, NO_PARAMS, |r| r.get(0))?; + Ok(count) + }) + .await + } + + async fn add_channel_to_db(&self, details: DBChannelDetails) -> Result<(), Self::Error> { + let for_coin = self.db_ticker.clone(); + let rpc_id = details.rpc_id as i64; + let created_at = details.created_at as i64; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [ + &rpc_id as &dyn ToSql, + &details.channel_id as &dyn ToSql, + &details.counterparty_node_id as &dyn ToSql, + &details.is_outbound as &dyn ToSql, + &details.is_public as &dyn ToSql, + &details.is_closed as &dyn ToSql, + &created_at as &dyn ToSql, + ]; + sql_transaction.execute(&insert_channel_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn add_funding_tx_to_db( + &self, + rpc_id: i64, + funding_tx: String, + funding_value: i64, + funding_generated_in_block: i64, + ) -> Result<(), Self::Error> { + let for_coin = self.db_ticker.clone(); + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [ + &funding_tx as &dyn ToSql, + &funding_value as &dyn ToSql, + &funding_generated_in_block as &dyn ToSql, + &rpc_id as &dyn ToSql, + ]; + sql_transaction.execute(&update_funding_tx_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn update_funding_tx_block_height(&self, funding_tx: String, block_height: i64) -> Result<(), Self::Error> { + let for_coin = self.db_ticker.clone(); + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [&block_height as &dyn ToSql, &funding_tx as &dyn ToSql]; + sql_transaction.execute(&update_funding_tx_block_height_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn update_channel_to_closed( + &self, + rpc_id: i64, + closure_reason: String, + closed_at: i64, + ) -> Result<(), Self::Error> { + let for_coin = self.db_ticker.clone(); + let is_closed = true; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [ + &closure_reason as &dyn ToSql, + &is_closed as &dyn ToSql, + &closed_at as &dyn ToSql, + &rpc_id as &dyn ToSql, + ]; + sql_transaction.execute(&update_channel_to_closed_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error> { + let mut builder = get_channels_builder_preimage(self.db_ticker.as_str())?; + builder.and_where("funding_tx IS NOT NULL"); + builder.and_where("closing_tx IS NULL"); + add_fields_to_get_channels_sql_builder(&mut builder); + let sql = builder.sql().expect("valid sql"); + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + + let mut stmt = conn.prepare(&sql)?; + let result = stmt + .query_map_named(&[], channel_details_from_row)? + .collect::>()?; + Ok(result) + }) + .await + } + + async fn add_closing_tx_to_db(&self, rpc_id: i64, closing_tx: String) -> Result<(), Self::Error> { + let for_coin = self.db_ticker.clone(); + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [&closing_tx as &dyn ToSql, &rpc_id as &dyn ToSql]; + sql_transaction.execute(&update_closing_tx_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn add_claiming_tx_to_db( + &self, + closing_tx: String, + claiming_tx: String, + claimed_balance: f64, + ) -> Result<(), Self::Error> { + let for_coin = self.db_ticker.clone(); + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [ + &claiming_tx as &dyn ToSql, + &claimed_balance as &dyn ToSql, + &closing_tx as &dyn ToSql, + ]; + sql_transaction.execute(&update_claiming_tx_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn get_channel_from_db(&self, rpc_id: u64) -> Result, Self::Error> { + let params = [rpc_id.to_string()]; + let sql = select_channel_by_rpc_id_sql(self.db_ticker.as_str())?; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + query_single_row(&conn, &sql, params, channel_details_from_row) + }) + .await + } + + async fn get_closed_channels_by_filter( + &self, + filter: Option, + paging: PagingOptionsEnum, + limit: usize, + ) -> Result { + let mut sql_builder = get_channels_builder_preimage(self.db_ticker.as_str())?; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + + let mut total_builder = sql_builder.clone(); + total_builder.count("id"); + let total_sql = total_builder.sql().expect("valid sql"); + let total: isize = conn.query_row(&total_sql, NO_PARAMS, |row| row.get(0))?; + let total = total.try_into().expect("count should be always above zero"); + + let offset = match paging { + PagingOptionsEnum::PageNumber(page) => (page.get() - 1) * limit, + PagingOptionsEnum::FromId(rpc_id) => { + let params = [rpc_id as u32]; + let maybe_offset = + offset_by_id(&conn, &sql_builder, params, "rpc_id", "closed_at DESC", "rpc_id = ?1")?; + match maybe_offset { + Some(offset) => offset, + None => { + return Ok(GetClosedChannelsResult { + channels: vec![], + skipped: 0, + total, + }) + }, + } + }, + }; + + let mut params = vec![]; + if let Some(f) = &filter { + apply_get_channels_filter(&mut sql_builder, &mut params, f); + } + add_fields_to_get_channels_sql_builder(&mut sql_builder); + finalize_get_channels_sql_builder(&mut sql_builder, offset, limit); + + let sql = sql_builder.sql().expect("valid sql"); + let mut stmt = conn.prepare(&sql)?; + let channels = stmt + .query_map_named(params.as_slice(), channel_details_from_row)? + .collect::>()?; + let result = GetClosedChannelsResult { + channels, + skipped: offset, + total, + }; + Ok(result) + }) + .await + } + + async fn add_or_update_payment_in_db(&self, info: DBPaymentInfo) -> Result<(), Self::Error> { + let for_coin = self.db_ticker.clone(); + let payment_hash = hex::encode(info.payment_hash.0); + let (is_outbound, destination) = match info.payment_type { + PaymentType::OutboundPayment { destination } => (true, Some(destination.to_string())), + PaymentType::InboundPayment => (false, None), + }; + let preimage = info.preimage.map(|p| hex::encode(p.0)); + let secret = info.secret.map(|s| hex::encode(s.0)); + let status = info.status.to_string(); + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let params = [ + &payment_hash as &dyn ToSql, + &destination as &dyn ToSql, + &info.description as &dyn ToSql, + &preimage as &dyn ToSql, + &secret as &dyn ToSql, + &info.amt_msat as &dyn ToSql, + &info.fee_paid_msat as &dyn ToSql, + &is_outbound as &dyn ToSql, + &status as &dyn ToSql, + &info.created_at as &dyn ToSql, + &info.last_updated as &dyn ToSql, + ]; + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + sql_transaction.execute(&upsert_payment_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn get_payment_from_db(&self, hash: PaymentHash) -> Result, Self::Error> { + let params = [hex::encode(hash.0)]; + let sql = select_payment_by_hash_sql(self.db_ticker.as_str())?; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + query_single_row(&conn, &sql, params, payment_info_from_row) + }) + .await + } + + async fn get_payments_by_filter( + &self, + filter: Option, + paging: PagingOptionsEnum, + limit: usize, + ) -> Result { + let mut sql_builder = get_payments_builder_preimage(self.db_ticker.as_str())?; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + + let mut total_builder = sql_builder.clone(); + total_builder.count("id"); + let total_sql = total_builder.sql().expect("valid sql"); + let total: isize = conn.query_row(&total_sql, NO_PARAMS, |row| row.get(0))?; + let total = total.try_into().expect("count should be always above zero"); + + let offset = match paging { + PagingOptionsEnum::PageNumber(page) => (page.get() - 1) * limit, + PagingOptionsEnum::FromId(hash) => { + let hash_str = hex::encode(hash.0); + let params = [&hash_str]; + let maybe_offset = offset_by_id( + &conn, + &sql_builder, + params, + "payment_hash", + "last_updated DESC", + "payment_hash = ?1", + )?; + match maybe_offset { + Some(offset) => offset, + None => { + return Ok(GetPaymentsResult { + payments: vec![], + skipped: 0, + total, + }) + }, + } + }, + }; + + let mut params = vec![]; + if let Some(f) = &filter { + apply_get_payments_filter(&mut sql_builder, &mut params, f); + } + let params_as_trait: Vec<_> = params.iter().map(|(key, value)| (*key, value as &dyn ToSql)).collect(); + finalize_get_payments_sql_builder(&mut sql_builder, offset, limit); + + let sql = sql_builder.sql().expect("valid sql"); + let mut stmt = conn.prepare(&sql)?; + let payments = stmt + .query_map_named(params_as_trait.as_slice(), payment_info_from_row)? + .collect::>()?; + let result = GetPaymentsResult { + payments, + skipped: offset, + total, + }; + Ok(result) + }) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lightning::ln_db::DBChannelDetails; + use common::{block_on, now_ms}; + use db_common::sqlite::rusqlite::Connection; + use rand::distributions::Alphanumeric; + use rand::{Rng, RngCore}; + use secp256k1::{Secp256k1, SecretKey}; + use std::num::NonZeroUsize; + use std::sync::{Arc, Mutex}; + + fn generate_random_channels(num: u64) -> Vec { + let mut rng = rand::thread_rng(); + let mut channels = vec![]; + let s = Secp256k1::new(); + let mut bytes = [0; 32]; + for i in 0..num { + let details = DBChannelDetails { + rpc_id: (i + 1) as i64, + channel_id: { + rng.fill_bytes(&mut bytes); + hex::encode(bytes) + }, + counterparty_node_id: { + rng.fill_bytes(&mut bytes); + let secret = SecretKey::from_slice(&bytes).unwrap(); + let pubkey = PublicKey::from_secret_key(&s, &secret); + pubkey.to_string() + }, + funding_tx: { + rng.fill_bytes(&mut bytes); + Some(hex::encode(bytes)) + }, + funding_value: Some(rng.gen::()), + closing_tx: { + rng.fill_bytes(&mut bytes); + Some(hex::encode(bytes)) + }, + closure_reason: { + Some( + rng.sample_iter(&Alphanumeric) + .take(30) + .map(char::from) + .collect::(), + ) + }, + claiming_tx: { + rng.fill_bytes(&mut bytes); + Some(hex::encode(bytes)) + }, + claimed_balance: Some(rng.gen::()), + funding_generated_in_block: Some(rng.gen::()), + is_outbound: rng.gen::(), + is_public: rng.gen::(), + is_closed: rand::random(), + created_at: rng.gen::(), + closed_at: Some(rng.gen::()), + }; + channels.push(details); + } + channels + } + + fn generate_random_payments(num: u64) -> Vec { + let mut rng = rand::thread_rng(); + let mut payments = vec![]; + let s = Secp256k1::new(); + let mut bytes = [0; 32]; + for _ in 0..num { + let payment_type = if rng.gen::() { + rng.fill_bytes(&mut bytes); + let secret = SecretKey::from_slice(&bytes).unwrap(); + PaymentType::OutboundPayment { + destination: PublicKey::from_secret_key(&s, &secret), + } + } else { + PaymentType::InboundPayment + }; + let status_rng: u8 = rng.gen(); + let status = if status_rng % 3 == 0 { + HTLCStatus::Succeeded + } else if status_rng % 3 == 1 { + HTLCStatus::Pending + } else { + HTLCStatus::Failed + }; + let description: String = rng.sample_iter(&Alphanumeric).take(30).map(char::from).collect(); + let info = DBPaymentInfo { + payment_hash: { + rng.fill_bytes(&mut bytes); + PaymentHash(bytes) + }, + payment_type, + description, + preimage: { + rng.fill_bytes(&mut bytes); + Some(PaymentPreimage(bytes)) + }, + secret: { + rng.fill_bytes(&mut bytes); + Some(PaymentSecret(bytes)) + }, + amt_msat: Some(rng.gen::()), + fee_paid_msat: Some(rng.gen::()), + status, + created_at: rng.gen::(), + last_updated: rng.gen::(), + }; + payments.push(info); + } + payments + } + + #[test] + fn test_init_sql_collection() { + let db = SqliteLightningDB::new( + "init_sql_collection".into(), + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + let initialized = block_on(db.is_db_initialized()).unwrap(); + assert!(!initialized); + + block_on(db.init_db()).unwrap(); + // repetitive init must not fail + block_on(db.init_db()).unwrap(); + + let initialized = block_on(db.is_db_initialized()).unwrap(); + assert!(initialized); + } + + #[test] + fn test_add_get_channel_sql() { + let db = SqliteLightningDB::new( + "add_get_channel".into(), + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + + block_on(db.init_db()).unwrap(); + + let last_channel_rpc_id = block_on(db.get_last_channel_rpc_id()).unwrap(); + assert_eq!(last_channel_rpc_id, 0); + + let channel = block_on(db.get_channel_from_db(1)).unwrap(); + assert!(channel.is_none()); + + let mut expected_channel_details = DBChannelDetails::new( + 1, + [0; 32], + PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), + true, + true, + ); + block_on(db.add_channel_to_db(expected_channel_details.clone())).unwrap(); + let last_channel_rpc_id = block_on(db.get_last_channel_rpc_id()).unwrap(); + assert_eq!(last_channel_rpc_id, 1); + + let actual_channel_details = block_on(db.get_channel_from_db(1)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); + + // must fail because we are adding channel with the same rpc_id + block_on(db.add_channel_to_db(expected_channel_details.clone())).unwrap_err(); + assert_eq!(last_channel_rpc_id, 1); + + expected_channel_details.rpc_id = 2; + block_on(db.add_channel_to_db(expected_channel_details.clone())).unwrap(); + let last_channel_rpc_id = block_on(db.get_last_channel_rpc_id()).unwrap(); + assert_eq!(last_channel_rpc_id, 2); + + block_on(db.add_funding_tx_to_db( + 2, + "9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into(), + 3000, + 50000, + )) + .unwrap(); + expected_channel_details.funding_tx = + Some("9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into()); + expected_channel_details.funding_value = Some(3000); + expected_channel_details.funding_generated_in_block = Some(50000); + + let actual_channel_details = block_on(db.get_channel_from_db(2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); + + block_on(db.update_funding_tx_block_height( + "9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into(), + 50001, + )) + .unwrap(); + expected_channel_details.funding_generated_in_block = Some(50001); + + let actual_channel_details = block_on(db.get_channel_from_db(2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); + + let current_time = (now_ms() / 1000) as i64; + block_on(db.update_channel_to_closed(2, "the channel was cooperatively closed".into(), current_time)).unwrap(); + expected_channel_details.closure_reason = Some("the channel was cooperatively closed".into()); + expected_channel_details.is_closed = true; + expected_channel_details.closed_at = Some(current_time); + + let actual_channel_details = block_on(db.get_channel_from_db(2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); + + let closed_channels = + block_on(db.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 10)).unwrap(); + assert_eq!(closed_channels.channels.len(), 1); + assert_eq!(expected_channel_details, closed_channels.channels[0]); + + block_on(db.update_channel_to_closed( + 1, + "the channel was cooperatively closed".into(), + (now_ms() / 1000) as i64, + )) + .unwrap(); + let closed_channels = + block_on(db.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 10)).unwrap(); + assert_eq!(closed_channels.channels.len(), 2); + + let actual_channels = block_on(db.get_closed_channels_with_no_closing_tx()).unwrap(); + assert_eq!(actual_channels.len(), 1); + + block_on(db.add_closing_tx_to_db( + 2, + "5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into(), + )) + .unwrap(); + expected_channel_details.closing_tx = + Some("5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into()); + + let actual_channels = block_on(db.get_closed_channels_with_no_closing_tx()).unwrap(); + assert!(actual_channels.is_empty()); + + let actual_channel_details = block_on(db.get_channel_from_db(2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); + + block_on(db.add_claiming_tx_to_db( + "5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into(), + "97f061634a4a7b0b0c2b95648f86b1c39b95e0cf5073f07725b7143c095b612a".into(), + 2000.333333, + )) + .unwrap(); + expected_channel_details.claiming_tx = + Some("97f061634a4a7b0b0c2b95648f86b1c39b95e0cf5073f07725b7143c095b612a".into()); + expected_channel_details.claimed_balance = Some(2000.333333); + + let actual_channel_details = block_on(db.get_channel_from_db(2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); + } + + #[test] + fn test_add_get_payment_sql() { + let db = SqliteLightningDB::new( + "add_get_payment".into(), + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + + block_on(db.init_db()).unwrap(); + + let payment = block_on(db.get_payment_from_db(PaymentHash([0; 32]))).unwrap(); + assert!(payment.is_none()); + + let mut expected_payment_info = DBPaymentInfo { + payment_hash: PaymentHash([0; 32]), + payment_type: PaymentType::InboundPayment, + description: "test payment".into(), + preimage: Some(PaymentPreimage([2; 32])), + secret: Some(PaymentSecret([3; 32])), + amt_msat: Some(2000), + fee_paid_msat: Some(100), + status: HTLCStatus::Failed, + created_at: (now_ms() / 1000) as i64, + last_updated: (now_ms() / 1000) as i64, + }; + block_on(db.add_or_update_payment_in_db(expected_payment_info.clone())).unwrap(); + + let actual_payment_info = block_on(db.get_payment_from_db(PaymentHash([0; 32]))).unwrap().unwrap(); + assert_eq!(expected_payment_info, actual_payment_info); + + expected_payment_info.payment_hash = PaymentHash([1; 32]); + expected_payment_info.payment_type = PaymentType::OutboundPayment { + destination: PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9") + .unwrap(), + }; + expected_payment_info.secret = None; + expected_payment_info.amt_msat = None; + expected_payment_info.status = HTLCStatus::Succeeded; + expected_payment_info.last_updated = (now_ms() / 1000) as i64; + block_on(db.add_or_update_payment_in_db(expected_payment_info.clone())).unwrap(); + + let actual_payment_info = block_on(db.get_payment_from_db(PaymentHash([1; 32]))).unwrap().unwrap(); + assert_eq!(expected_payment_info, actual_payment_info); + } + + #[test] + fn test_get_payments_by_filter() { + let db = SqliteLightningDB::new( + "test_get_payments_by_filter".into(), + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + + block_on(db.init_db()).unwrap(); + + let mut payments = generate_random_payments(100); + + for payment in payments.clone() { + block_on(db.add_or_update_payment_in_db(payment)).unwrap(); + } + + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); + let limit = 4; + + let result = block_on(db.get_payments_by_filter(None, paging, limit)).unwrap(); + + payments.sort_by(|a, b| b.last_updated.cmp(&a.last_updated)); + let expected_payments = &payments[..4].to_vec(); + let actual_payments = &result.payments; + + assert_eq!(0, result.skipped); + assert_eq!(100, result.total); + assert_eq!(expected_payments, actual_payments); + + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(2).unwrap()); + let limit = 5; + + let result = block_on(db.get_payments_by_filter(None, paging, limit)).unwrap(); + + let expected_payments = &payments[5..10].to_vec(); + let actual_payments = &result.payments; + + assert_eq!(5, result.skipped); + assert_eq!(100, result.total); + assert_eq!(expected_payments, actual_payments); + + let from_payment_hash = payments[20].payment_hash; + let paging = PagingOptionsEnum::FromId(from_payment_hash); + let limit = 3; + + let result = block_on(db.get_payments_by_filter(None, paging, limit)).unwrap(); + + let expected_payments = &payments[21..24].to_vec(); + let actual_payments = &result.payments; + + assert_eq!(expected_payments, actual_payments); + + let mut filter = DBPaymentsFilter { + is_outbound: Some(false), + destination: None, + description: None, + status: None, + from_amount_msat: None, + to_amount_msat: None, + from_fee_paid_msat: None, + to_fee_paid_msat: None, + from_timestamp: None, + to_timestamp: None, + }; + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); + let limit = 10; + + let result = block_on(db.get_payments_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); + let expected_payments_vec: Vec = payments + .iter() + .map(|p| p.clone()) + .filter(|p| p.payment_type == PaymentType::InboundPayment) + .collect(); + let expected_payments = if expected_payments_vec.len() > 10 { + expected_payments_vec[..10].to_vec() + } else { + expected_payments_vec.clone() + }; + let actual_payments = result.payments; + + assert_eq!(expected_payments, actual_payments); + + filter.status = Some(HTLCStatus::Succeeded.to_string()); + let result = block_on(db.get_payments_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); + let expected_payments_vec: Vec = expected_payments_vec + .iter() + .map(|p| p.clone()) + .filter(|p| p.status == HTLCStatus::Succeeded) + .collect(); + let expected_payments = if expected_payments_vec.len() > 10 { + expected_payments_vec[..10].to_vec() + } else { + expected_payments_vec + }; + let actual_payments = result.payments; + + assert_eq!(expected_payments, actual_payments); + + let description = &payments[42].description; + let substr = &description[5..10]; + filter.is_outbound = None; + filter.destination = None; + filter.status = None; + filter.description = Some(substr.to_string()); + let result = block_on(db.get_payments_by_filter(Some(filter), paging, limit)).unwrap(); + let expected_payments_vec: Vec = payments + .iter() + .map(|p| p.clone()) + .filter(|p| p.description.contains(&substr)) + .collect(); + let expected_payments = if expected_payments_vec.len() > 10 { + expected_payments_vec[..10].to_vec() + } else { + expected_payments_vec.clone() + }; + let actual_payments = result.payments; + + assert_eq!(expected_payments, actual_payments); + } + + #[test] + fn test_get_channels_by_filter() { + let db = SqliteLightningDB::new( + "test_get_channels_by_filter".into(), + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + + block_on(db.init_db()).unwrap(); + + let channels = generate_random_channels(100); + + for channel in channels { + block_on(db.add_channel_to_db(channel.clone())).unwrap(); + block_on(db.add_funding_tx_to_db( + channel.rpc_id, + channel.funding_tx.unwrap(), + channel.funding_value.unwrap(), + channel.funding_generated_in_block.unwrap(), + )) + .unwrap(); + block_on(db.update_channel_to_closed(channel.rpc_id, channel.closure_reason.unwrap(), 1655806080)).unwrap(); + block_on(db.add_closing_tx_to_db(channel.rpc_id, channel.closing_tx.clone().unwrap())).unwrap(); + block_on(db.add_claiming_tx_to_db( + channel.closing_tx.unwrap(), + channel.claiming_tx.unwrap(), + channel.claimed_balance.unwrap(), + )) + .unwrap(); + } + + // get all channels from SQL since updated_at changed from channels generated by generate_random_channels + let channels = block_on(db.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 100)) + .unwrap() + .channels; + assert_eq!(100, channels.len()); + + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); + let limit = 4; + + let result = block_on(db.get_closed_channels_by_filter(None, paging, limit)).unwrap(); + + let expected_channels = &channels[..4].to_vec(); + let actual_channels = &result.channels; + + assert_eq!(0, result.skipped); + assert_eq!(100, result.total); + assert_eq!(expected_channels, actual_channels); + + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(2).unwrap()); + let limit = 5; + + let result = block_on(db.get_closed_channels_by_filter(None, paging, limit)).unwrap(); + + let expected_channels = &channels[5..10].to_vec(); + let actual_channels = &result.channels; + + assert_eq!(5, result.skipped); + assert_eq!(100, result.total); + assert_eq!(expected_channels, actual_channels); + + let from_rpc_id = 20; + let paging = PagingOptionsEnum::FromId(from_rpc_id); + let limit = 3; + + let result = block_on(db.get_closed_channels_by_filter(None, paging, limit)).unwrap(); + + let expected_channels = channels[20..23].to_vec(); + let actual_channels = result.channels; + + assert_eq!(expected_channels, actual_channels); + + let mut filter = ClosedChannelsFilter { + channel_id: None, + counterparty_node_id: None, + funding_tx: None, + from_funding_value: None, + to_funding_value: None, + closing_tx: None, + closure_reason: None, + claiming_tx: None, + from_claimed_balance: None, + to_claimed_balance: None, + channel_type: Some(ChannelType::Outbound), + channel_visibility: None, + }; + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); + let limit = 10; + + let result = block_on(db.get_closed_channels_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); + let expected_channels_vec: Vec = channels + .iter() + .map(|chan| chan.clone()) + .filter(|chan| chan.is_outbound) + .collect(); + let expected_channels = if expected_channels_vec.len() > 10 { + expected_channels_vec[..10].to_vec() + } else { + expected_channels_vec.clone() + }; + let actual_channels = result.channels; + + assert_eq!(expected_channels, actual_channels); + + filter.channel_visibility = Some(ChannelVisibility::Public); + let result = block_on(db.get_closed_channels_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); + let expected_channels_vec: Vec = expected_channels_vec + .iter() + .map(|chan| chan.clone()) + .filter(|chan| chan.is_public) + .collect(); + let expected_channels = if expected_channels_vec.len() > 10 { + expected_channels_vec[..10].to_vec() + } else { + expected_channels_vec + }; + let actual_channels = result.channels; + + assert_eq!(expected_channels, actual_channels); + + let channel_id = channels[42].channel_id.clone(); + filter.channel_type = None; + filter.channel_visibility = None; + filter.channel_id = Some(channel_id.clone()); + let result = block_on(db.get_closed_channels_by_filter(Some(filter), paging, limit)).unwrap(); + let expected_channels_vec: Vec = channels + .iter() + .map(|chan| chan.clone()) + .filter(|chan| chan.channel_id == channel_id) + .collect(); + let expected_channels = if expected_channels_vec.len() > 10 { + expected_channels_vec[..10].to_vec() + } else { + expected_channels_vec.clone() + }; + let actual_channels = result.channels; + + assert_eq!(expected_channels, actual_channels); + } +} diff --git a/mm2src/coins/lightning/ln_storage.rs b/mm2src/coins/lightning/ln_storage.rs new file mode 100644 index 0000000000..bd44fdc0e1 --- /dev/null +++ b/mm2src/coins/lightning/ln_storage.rs @@ -0,0 +1,33 @@ +use async_trait::async_trait; +use bitcoin::Network; +use lightning::routing::network_graph::NetworkGraph; +use lightning::routing::scoring::ProbabilisticScorer; +use parking_lot::Mutex as PaMutex; +use secp256k1::PublicKey; +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; + +pub type NodesAddressesMap = HashMap; +pub type NodesAddressesMapShared = Arc>; +pub type Scorer = ProbabilisticScorer>; + +#[async_trait] +pub trait LightningStorage { + type Error; + + /// Initializes dirs/collection/tables in storage for a specified coin + async fn init_fs(&self) -> Result<(), Self::Error>; + + async fn is_fs_initialized(&self) -> Result; + + async fn get_nodes_addresses(&self) -> Result, Self::Error>; + + async fn save_nodes_addresses(&self, nodes_addresses: NodesAddressesMapShared) -> Result<(), Self::Error>; + + async fn get_network_graph(&self, network: Network) -> Result; + + async fn get_scorer(&self, network_graph: Arc) -> Result; + + async fn save_scorer(&self, scorer: Arc>) -> Result<(), Self::Error>; +} diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 05e4fd4f63..30b3e3c2de 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -1,5 +1,9 @@ use super::*; +use crate::lightning::ln_db::LightningDB; +use crate::lightning::ln_filesystem_persister::LightningPersisterShared; use crate::lightning::ln_platform::{get_best_header, ln_best_block_update_loop, update_best_block}; +use crate::lightning::ln_sql::SqliteLightningDB; +use crate::lightning::ln_storage::{LightningStorage, NodesAddressesMap, Scorer}; use crate::utxo::rpc_clients::BestBlock as RpcBestBlock; use bitcoin::hash_types::BlockHash; use bitcoin_hashes::{sha256d, Hash}; @@ -10,18 +14,14 @@ use lightning::chain::keysinterface::{InMemorySigner, KeysManager}; use lightning::chain::{chainmonitor, BestBlock, Watch}; use lightning::ln::channelmanager; use lightning::ln::channelmanager::{ChainParameters, ChannelManagerReadArgs, SimpleArcChannelManager}; -use lightning::routing::network_graph::NetworkGraph; use lightning::util::config::UserConfig; use lightning::util::ser::ReadableArgs; -use lightning_persister::storage::{DbStorage, FileSystemStorage, NodesAddressesMap, Scorer}; -use lightning_persister::LightningPersister; use mm2_core::mm_ctx::MmArc; use std::fs::File; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::SystemTime; -const NETWORK_GRAPH_PERSIST_INTERVAL: u64 = 600; const SCORER_PERSIST_INTERVAL: u64 = 600; pub type ChainMonitor = chainmonitor::ChainMonitor< @@ -30,7 +30,7 @@ pub type ChainMonitor = chainmonitor::ChainMonitor< Arc, Arc, Arc, - Arc, + LightningPersisterShared, >; pub type ChannelManager = SimpleArcChannelManager; @@ -50,54 +50,39 @@ fn ln_data_backup_dir(ctx: &MmArc, path: Option, ticker: &str) -> Option pub async fn init_persister( ctx: &MmArc, - platform: Arc, ticker: String, backup_path: Option, -) -> EnableLightningResult> { +) -> EnableLightningResult { let ln_data_dir = ln_data_dir(ctx, &ticker); let ln_data_backup_dir = ln_data_backup_dir(ctx, backup_path, &ticker); - let persister = Arc::new(LightningPersister::new( - ticker.replace('-', "_"), + let persister = LightningPersisterShared(Arc::new(LightningFilesystemPersister::new( ln_data_dir, ln_data_backup_dir, + ))); + + let is_initialized = persister.is_fs_initialized().await?; + if !is_initialized { + persister.init_fs().await?; + } + + Ok(persister) +} + +pub async fn init_db(ctx: &MmArc, ticker: String) -> EnableLightningResult { + let db = SqliteLightningDB::new( + ticker, ctx.sqlite_connection .ok_or(MmError::new(EnableLightningError::DbError( "sqlite_connection is not initialized".into(), )))? .clone(), - )); - let is_initialized = persister.is_fs_initialized().await?; - if !is_initialized { - persister.init_fs().await?; - } - let is_db_initialized = persister.is_db_initialized().await?; - if !is_db_initialized { - persister.init_db().await?; - } + ); - let closed_channels_without_closing_tx = persister.get_closed_channels_with_no_closing_tx().await?; - for channel_details in closed_channels_without_closing_tx { - let platform = platform.clone(); - let persister = persister.clone(); - let user_channel_id = channel_details.rpc_id; - spawn(async move { - if let Ok(closing_tx_hash) = platform - .get_channel_closing_tx(channel_details) - .await - .error_log_passthrough() - { - if let Err(e) = persister.add_closing_tx_to_db(user_channel_id, closing_tx_hash).await { - log::error!( - "Unable to update channel {} closing details in DB: {}", - user_channel_id, - e - ); - } - } - }); + if !db.is_db_initialized().await? { + db.init_db().await?; } - Ok(persister) + Ok(db) } pub fn init_keys_manager(ctx: &MmArc) -> EnableLightningResult> { @@ -113,7 +98,8 @@ pub fn init_keys_manager(ctx: &MmArc) -> EnableLightningResult> pub async fn init_channel_manager( platform: Arc, logger: Arc, - persister: Arc, + persister: LightningPersisterShared, + db: SqliteLightningDB, keys_manager: Arc, user_config: UserConfig, ) -> EnableLightningResult<(Arc, Arc)> { @@ -135,6 +121,7 @@ pub async fn init_channel_manager( // Read ChannelMonitor state from disk, important for lightning node is restarting and has at least 1 channel let mut channelmonitors = persister + .channels_persister() .read_channelmonitors(keys_manager.clone()) .map_to_mm(|e| EnableLightningError::IOError(e.to_string()))?; @@ -155,9 +142,7 @@ pub async fn init_channel_manager( let best_header = get_best_header(&rpc_client).await?; platform.update_best_block_height(best_header.block_height()); let best_block = RpcBestBlock::from(best_header.clone()); - let best_block_hash = BlockHash::from_hash( - sha256d::Hash::from_slice(&best_block.hash.0).map_to_mm(|e| EnableLightningError::HashError(e.to_string()))?, - ); + let best_block_hash = BlockHash::from_hash(sha256d::Hash::from_inner(best_block.hash.0)); let (channel_manager_blockhash, channel_manager) = { if let Ok(mut f) = File::open(persister.manager_path()) { let mut channel_monitor_mut_references = Vec::new(); @@ -198,13 +183,13 @@ pub async fn init_channel_manager( let channel_manager: Arc = Arc::new(channel_manager); // Sync ChannelMonitors and ChannelManager to chain tip if the node is restarting and has open channels + platform + .process_txs_confirmations(&rpc_client, &db, &chain_monitor, &channel_manager) + .await; if channel_manager_blockhash != best_block_hash { platform .process_txs_unconfirmations(&chain_monitor, &channel_manager) .await; - platform - .process_txs_confirmations(&rpc_client, &persister, &chain_monitor, &channel_manager) - .await; update_best_block(&chain_monitor, &channel_manager, best_header).await; } @@ -218,9 +203,8 @@ pub async fn init_channel_manager( // Update best block whenever there's a new chain tip or a block has been newly disconnected spawn(ln_best_block_update_loop( - // It's safe to use unwrap here for now until implementing Native Client for Lightning platform, - persister.clone(), + db, chain_monitor.clone(), channel_manager.clone(), rpc_client.clone(), @@ -230,19 +214,7 @@ pub async fn init_channel_manager( Ok((chain_monitor, channel_manager)) } -pub async fn persist_network_graph_loop(persister: Arc, network_graph: Arc) { - loop { - if let Err(e) = persister.save_network_graph(network_graph.clone()).await { - log::warn!( - "Failed to persist network graph error: {}, please check disk space and permissions", - e - ); - } - Timer::sleep(NETWORK_GRAPH_PERSIST_INTERVAL as f64).await; - } -} - -pub async fn persist_scorer_loop(persister: Arc, scorer: Arc>) { +pub async fn persist_scorer_loop(persister: LightningPersisterShared, scorer: Arc>) { loop { if let Err(e) = persister.save_scorer(scorer.clone()).await { log::warn!( @@ -255,7 +227,7 @@ pub async fn persist_scorer_loop(persister: Arc, scorer: Arc } pub async fn get_open_channels_nodes_addresses( - persister: Arc, + persister: LightningPersisterShared, channel_manager: Arc, ) -> EnableLightningResult { let channels = channel_manager.list_channels(); diff --git a/mm2src/coins/lightning_background_processor/Cargo.toml b/mm2src/coins/lightning_background_processor/Cargo.toml deleted file mode 100644 index 5710dcfc2c..0000000000 --- a/mm2src/coins/lightning_background_processor/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "lightning-background-processor" -version = "0.0.106" -authors = ["Valentine Wallace "] -license = "MIT OR Apache-2.0" -repository = "http://github.com/lightningdevkit/rust-lightning" -description = """ -Utilities to perform required background tasks for Rust Lightning. -""" -edition = "2018" - -[dependencies] -bitcoin = "0.27.1" -lightning = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106", features = ["std"] } - -[dev-dependencies] -db_common = { path = "../../db_common" } -lightning = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106", features = ["_test_utils"] } -lightning-invoice = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } -lightning-persister = { version = "0.0.106", path = "../lightning_persister" } diff --git a/mm2src/coins/lightning_background_processor/src/lib.rs b/mm2src/coins/lightning_background_processor/src/lib.rs deleted file mode 100644 index 4ca2fe9ad4..0000000000 --- a/mm2src/coins/lightning_background_processor/src/lib.rs +++ /dev/null @@ -1,950 +0,0 @@ -//! Utilities that take care of tasks that (1) need to happen periodically to keep Rust-Lightning -//! running properly, and (2) either can or should be run in the background. See docs for -//! [`BackgroundProcessor`] for more details on the nitty-gritty. - -#[macro_use] extern crate lightning; - -use lightning::chain; -use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator}; -use lightning::chain::chainmonitor::{ChainMonitor, Persist}; -use lightning::chain::keysinterface::{KeysInterface, Sign}; -use lightning::ln::channelmanager::ChannelManager; -use lightning::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler}; -use lightning::ln::peer_handler::{CustomMessageHandler, PeerManager, SocketDescriptor}; -use lightning::routing::network_graph::{NetGraphMsgHandler, NetworkGraph}; -use lightning::util::events::{Event, EventHandler, EventsProvider}; -use lightning::util::logger::Logger; -use std::ops::Deref; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::thread; -use std::thread::JoinHandle; -use std::time::{Duration, Instant}; - -/// `BackgroundProcessor` takes care of tasks that (1) need to happen periodically to keep -/// Rust-Lightning running properly, and (2) either can or should be run in the background. Its -/// responsibilities are: -/// * Processing [`Event`]s with a user-provided [`EventHandler`]. -/// * Monitoring whether the [`ChannelManager`] needs to be re-persisted to disk, and if so, -/// writing it to disk/backups by invoking the callback given to it at startup. -/// [`ChannelManager`] persistence should be done in the background. -/// * Calling [`ChannelManager::timer_tick_occurred`] and [`PeerManager::timer_tick_occurred`] -/// at the appropriate intervals. -/// * Calling [`NetworkGraph::remove_stale_channels`] (if a [`NetGraphMsgHandler`] is provided to -/// [`BackgroundProcessor::start`]). -/// -/// It will also call [`PeerManager::process_events`] periodically though this shouldn't be relied -/// upon as doing so may result in high latency. -/// -/// # Note -/// -/// If [`ChannelManager`] persistence fails and the persisted manager becomes out-of-date, then -/// there is a risk of channels force-closing on startup when the manager realizes it's outdated. -/// However, as long as [`ChannelMonitor`] backups are sound, no funds besides those used for -/// unilateral chain closure fees are at risk. -/// -/// [`ChannelMonitor`]: lightning::chain::channelmonitor::ChannelMonitor -/// [`Event`]: lightning::util::events::Event -#[must_use = "BackgroundProcessor will immediately stop on drop. It should be stored until shutdown."] -pub struct BackgroundProcessor { - stop_thread: Arc, - thread_handle: Option>>, -} - -#[cfg(not(test))] -const FRESHNESS_TIMER: u64 = 60; -#[cfg(test)] -const FRESHNESS_TIMER: u64 = 1; - -#[cfg(all(not(test), not(debug_assertions)))] -const PING_TIMER: u64 = 10; -/// Signature operations take a lot longer without compiler optimisations. -/// Increasing the ping timer allows for this but slower devices will be disconnected if the -/// timeout is reached. -#[cfg(all(not(test), debug_assertions))] -const PING_TIMER: u64 = 30; -#[cfg(test)] -const PING_TIMER: u64 = 1; - -/// Prune the network graph of stale entries hourly. -const NETWORK_PRUNE_TIMER: u64 = 60 * 60; - -/// Trait which handles persisting a [`ChannelManager`] to disk. -/// -/// [`ChannelManager`]: lightning::ln::channelmanager::ChannelManager -pub trait ChannelManagerPersister -where - M::Target: 'static + chain::Watch, - T::Target: 'static + BroadcasterInterface, - K::Target: 'static + KeysInterface, - F::Target: 'static + FeeEstimator, - L::Target: 'static + Logger, -{ - /// Persist the given [`ChannelManager`] to disk, returning an error if persistence failed - /// (which will cause the [`BackgroundProcessor`] which called this method to exit. - /// - /// [`ChannelManager`]: lightning::ln::channelmanager::ChannelManager - fn persist_manager(&self, channel_manager: &ChannelManager) -> Result<(), std::io::Error>; -} - -impl ChannelManagerPersister - for Fun -where - M::Target: 'static + chain::Watch, - T::Target: 'static + BroadcasterInterface, - K::Target: 'static + KeysInterface, - F::Target: 'static + FeeEstimator, - L::Target: 'static + Logger, - Fun: Fn(&ChannelManager) -> Result<(), std::io::Error>, -{ - fn persist_manager(&self, channel_manager: &ChannelManager) -> Result<(), std::io::Error> { - self(channel_manager) - } -} - -/// Decorates an [`EventHandler`] with common functionality provided by standard [`EventHandler`]s. -struct DecoratingEventHandler< - E: EventHandler, - N: Deref>, - G: Deref, - A: Deref, - L: Deref, -> where - A::Target: chain::Access, - L::Target: Logger, -{ - event_handler: E, - net_graph_msg_handler: Option, -} - -impl< - E: EventHandler, - N: Deref>, - G: Deref, - A: Deref, - L: Deref, - > EventHandler for DecoratingEventHandler -where - A::Target: chain::Access, - L::Target: Logger, -{ - fn handle_event(&self, event: &Event) { - if let Some(event_handler) = &self.net_graph_msg_handler { - event_handler.handle_event(event); - } - self.event_handler.handle_event(event); - } -} - -impl BackgroundProcessor { - /// Start a background thread that takes care of responsibilities enumerated in the [top-level - /// documentation]. - /// - /// The thread runs indefinitely unless the object is dropped, [`stop`] is called, or - /// `persist_manager` returns an error. In case of an error, the error is retrieved by calling - /// either [`join`] or [`stop`]. - /// - /// # Data Persistence - /// - /// `persist_manager` is responsible for writing out the [`ChannelManager`] to disk, and/or - /// uploading to one or more backup services. See [`ChannelManager::write`] for writing out a - /// [`ChannelManager`]. See [`LightningPersister::persist_manager`] for Rust-Lightning's - /// provided implementation. - /// - /// Typically, users should either implement [`ChannelManagerPersister`] to never return an - /// error or call [`join`] and handle any error that may arise. For the latter case, - /// `BackgroundProcessor` must be restarted by calling `start` again after handling the error. - /// - /// # Event Handling - /// - /// `event_handler` is responsible for handling events that users should be notified of (e.g., - /// payment failed). [`BackgroundProcessor`] may decorate the given [`EventHandler`] with common - /// functionality implemented by other handlers. - /// * [`NetGraphMsgHandler`] if given will update the [`NetworkGraph`] based on payment failures. - /// - /// [top-level documentation]: BackgroundProcessor - /// [`join`]: Self::join - /// [`stop`]: Self::stop - /// [`ChannelManager`]: lightning::ln::channelmanager::ChannelManager - /// [`ChannelManager::write`]: lightning::ln::channelmanager::ChannelManager#impl-Writeable - /// [`LightningPersister::persist_manager`]: lightning_persister::LightningPersister::persist_manager - /// [`NetworkGraph`]: lightning::routing::network_graph::NetworkGraph - pub fn start< - Signer: 'static + Sign, - CA: 'static + Deref + Send + Sync, - CF: 'static + Deref + Send + Sync, - CW: 'static + Deref + Send + Sync, - T: 'static + Deref + Send + Sync, - K: 'static + Deref + Send + Sync, - F: 'static + Deref + Send + Sync, - G: 'static + Deref + Send + Sync, - L: 'static + Deref + Send + Sync, - P: 'static + Deref + Send + Sync, - Descriptor: 'static + SocketDescriptor + Send + Sync, - CMH: 'static + Deref + Send + Sync, - RMH: 'static + Deref + Send + Sync, - EH: 'static + EventHandler + Send, - CMP: 'static + Send + ChannelManagerPersister, - M: 'static + Deref> + Send + Sync, - CM: 'static + Deref> + Send + Sync, - NG: 'static + Deref> + Send + Sync, - UMH: 'static + Deref + Send + Sync, - PM: 'static + Deref> + Send + Sync, - >( - persister: CMP, - event_handler: EH, - chain_monitor: M, - channel_manager: CM, - net_graph_msg_handler: Option, - peer_manager: PM, - logger: L, - ) -> Self - where - CA::Target: 'static + chain::Access, - CF::Target: 'static + chain::Filter, - CW::Target: 'static + chain::Watch, - T::Target: 'static + BroadcasterInterface, - K::Target: 'static + KeysInterface, - F::Target: 'static + FeeEstimator, - L::Target: 'static + Logger, - P::Target: 'static + Persist, - CMH::Target: 'static + ChannelMessageHandler, - RMH::Target: 'static + RoutingMessageHandler, - UMH::Target: 'static + CustomMessageHandler, - { - let stop_thread = Arc::new(AtomicBool::new(false)); - let stop_thread_clone = stop_thread.clone(); - let handle = thread::spawn(move || -> Result<(), std::io::Error> { - let event_handler = DecoratingEventHandler { - event_handler, - net_graph_msg_handler: net_graph_msg_handler.as_deref(), - }; - - log_trace!(logger, "Calling ChannelManager's timer_tick_occurred on startup"); - channel_manager.timer_tick_occurred(); - - let mut last_freshness_call = Instant::now(); - let mut last_ping_call = Instant::now(); - let mut last_prune_call = Instant::now(); - let mut have_pruned = false; - - loop { - peer_manager.process_events(); // Note that this may block on ChannelManager's locking - channel_manager.process_pending_events(&event_handler); - chain_monitor.process_pending_events(&event_handler); - - // We wait up to 100ms, but track how long it takes to detect being put to sleep, - // see `await_start`'s use below. - let await_start = Instant::now(); - let updates_available = channel_manager.await_persistable_update_timeout(Duration::from_millis(100)); - let await_time = await_start.elapsed(); - - if updates_available { - log_trace!(logger, "Persisting ChannelManager..."); - persister.persist_manager(&*channel_manager)?; - log_trace!(logger, "Done persisting ChannelManager."); - } - // Exit the loop if the background processor was requested to stop. - if stop_thread.load(Ordering::Acquire) { - log_trace!(logger, "Terminating background processor."); - break; - } - if last_freshness_call.elapsed().as_secs() > FRESHNESS_TIMER { - log_trace!(logger, "Calling ChannelManager's timer_tick_occurred"); - channel_manager.timer_tick_occurred(); - last_freshness_call = Instant::now(); - } - if await_time > Duration::from_secs(1) { - // On various platforms, we may be starved of CPU cycles for several reasons. - // E.g. on iOS, if we've been in the background, we will be entirely paused. - // Similarly, if we're on a desktop platform and the device has been asleep, we - // may not get any cycles. - // We detect this by checking if our max-100ms-sleep, above, ran longer than a - // full second, at which point we assume sockets may have been killed (they - // appear to be at least on some platforms, even if it has only been a second). - // Note that we have to take care to not get here just because user event - // processing was slow at the top of the loop. For example, the sample client - // may call Bitcoin Core RPCs during event handling, which very often takes - // more than a handful of seconds to complete, and shouldn't disconnect all our - // peers. - log_trace!(logger, "100ms sleep took more than a second, disconnecting peers."); - peer_manager.disconnect_all_peers(); - last_ping_call = Instant::now(); - } else if last_ping_call.elapsed().as_secs() > PING_TIMER { - log_trace!(logger, "Calling PeerManager's timer_tick_occurred"); - peer_manager.timer_tick_occurred(); - last_ping_call = Instant::now(); - } - - // Note that we want to run a graph prune once not long after startup before - // falling back to our usual hourly prunes. This avoids short-lived clients never - // pruning their network graph. We run once 60 seconds after startup before - // continuing our normal cadence. - if last_prune_call.elapsed().as_secs() > if have_pruned { NETWORK_PRUNE_TIMER } else { 60 } { - if let Some(ref handler) = net_graph_msg_handler { - log_trace!(logger, "Pruning network graph of stale entries"); - handler.network_graph().remove_stale_channels(); - last_prune_call = Instant::now(); - have_pruned = true; - } - } - } - // After we exit, ensure we persist the ChannelManager one final time - this avoids - // some races where users quit while channel updates were in-flight, with - // ChannelMonitor update(s) persisted without a corresponding ChannelManager update. - persister.persist_manager(&*channel_manager) - }); - Self { - stop_thread: stop_thread_clone, - thread_handle: Some(handle), - } - } - - /// Join `BackgroundProcessor`'s thread, returning any error that occurred while persisting - /// [`ChannelManager`]. - /// - /// # Panics - /// - /// This function panics if the background thread has panicked such as while persisting or - /// handling events. - /// - /// [`ChannelManager`]: lightning::ln::channelmanager::ChannelManager - pub fn join(mut self) -> Result<(), std::io::Error> { - assert!(self.thread_handle.is_some()); - self.join_thread() - } - - /// Stop `BackgroundProcessor`'s thread, returning any error that occurred while persisting - /// [`ChannelManager`]. - /// - /// # Panics - /// - /// This function panics if the background thread has panicked such as while persisting or - /// handling events. - /// - /// [`ChannelManager`]: lightning::ln::channelmanager::ChannelManager - pub fn stop(mut self) -> Result<(), std::io::Error> { - assert!(self.thread_handle.is_some()); - self.stop_and_join_thread() - } - - fn stop_and_join_thread(&mut self) -> Result<(), std::io::Error> { - self.stop_thread.store(true, Ordering::Release); - self.join_thread() - } - - fn join_thread(&mut self) -> Result<(), std::io::Error> { - match self.thread_handle.take() { - Some(handle) => handle.join().unwrap(), - None => Ok(()), - } - } -} - -impl Drop for BackgroundProcessor { - fn drop(&mut self) { self.stop_and_join_thread().unwrap(); } -} - -#[cfg(test)] -mod tests { - use super::{BackgroundProcessor, FRESHNESS_TIMER}; - use bitcoin::blockdata::block::BlockHeader; - use bitcoin::blockdata::constants::genesis_block; - use bitcoin::blockdata::transaction::{Transaction, TxOut}; - use bitcoin::network::constants::Network; - use db_common::sqlite::rusqlite::Connection; - use lightning::chain::channelmonitor::ANTI_REORG_DELAY; - use lightning::chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager, Recipient}; - use lightning::chain::transaction::OutPoint; - use lightning::chain::{chainmonitor, BestBlock, Confirm}; - use lightning::get_event_msg; - use lightning::ln::channelmanager::{ChainParameters, ChannelManager, SimpleArcChannelManager, BREAKDOWN_TIMEOUT}; - use lightning::ln::features::InitFeatures; - use lightning::ln::msgs::{ChannelMessageHandler, Init}; - use lightning::ln::peer_handler::{IgnoringMessageHandler, MessageHandler, PeerManager, SocketDescriptor}; - use lightning::routing::network_graph::{NetGraphMsgHandler, NetworkGraph}; - use lightning::util::config::UserConfig; - use lightning::util::events::{Event, MessageSendEvent, MessageSendEventsProvider}; - use lightning::util::ser::Writeable; - use lightning::util::test_utils; - use lightning_invoice::payment::{InvoicePayer, RetryAttempts}; - use lightning_invoice::utils::DefaultRouter; - use lightning_persister::LightningPersister; - use std::fs; - use std::path::PathBuf; - use std::sync::{Arc, Mutex}; - use std::time::Duration; - - const EVENT_DEADLINE: u64 = 5 * FRESHNESS_TIMER; - - #[derive(Clone, Eq, Hash, PartialEq)] - struct TestDescriptor {} - impl SocketDescriptor for TestDescriptor { - fn send_data(&mut self, _data: &[u8], _resume_read: bool) -> usize { 0 } - - fn disconnect_socket(&mut self) {} - } - - type ChainMonitor = chainmonitor::ChainMonitor< - InMemorySigner, - Arc, - Arc, - Arc, - Arc, - Arc, - >; - - struct Node { - node: Arc< - SimpleArcChannelManager< - ChainMonitor, - test_utils::TestBroadcaster, - test_utils::TestFeeEstimator, - test_utils::TestLogger, - >, - >, - net_graph_msg_handler: Option< - Arc, Arc, Arc>>, - >, - peer_manager: Arc< - PeerManager< - TestDescriptor, - Arc, - Arc, - Arc, - IgnoringMessageHandler, - >, - >, - chain_monitor: Arc, - persister: Arc, - tx_broadcaster: Arc, - network_graph: Arc, - logger: Arc, - best_block: BestBlock, - } - - impl Drop for Node { - fn drop(&mut self) { - let data_dir = self.persister.main_path(); - match fs::remove_dir_all(data_dir.clone()) { - Err(e) => println!( - "Failed to remove test persister directory {}: {}", - data_dir.to_str().unwrap(), - e - ), - _ => {}, - } - } - } - - fn get_full_filepath(filepath: String, filename: String) -> String { - let mut path = PathBuf::from(filepath); - path.push(filename); - path.to_str().unwrap().to_string() - } - - fn create_nodes(num_nodes: usize, persist_dir: String) -> Vec { - let mut nodes = Vec::new(); - for i in 0..num_nodes { - let tx_broadcaster = Arc::new(test_utils::TestBroadcaster { - txn_broadcasted: Mutex::new(Vec::new()), - blocks: Arc::new(Mutex::new(Vec::new())), - }); - let fee_estimator = Arc::new(test_utils::TestFeeEstimator { - sat_per_kw: Mutex::new(253), - }); - let chain_source = Arc::new(test_utils::TestChainSource::new(Network::Testnet)); - let logger = Arc::new(test_utils::TestLogger::with_id(format!("node {}", i))); - let persister = Arc::new(LightningPersister::new( - format!("node_{}_ticker", i), - PathBuf::from(format!("{}_persister_{}", persist_dir, i)), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - )); - let seed = [i as u8; 32]; - let network = Network::Testnet; - let genesis_block = genesis_block(network); - let now = Duration::from_secs(genesis_block.header.time as u64); - let keys_manager = Arc::new(KeysManager::new(&seed, now.as_secs(), now.subsec_nanos())); - let chain_monitor = Arc::new(chainmonitor::ChainMonitor::new( - Some(chain_source.clone()), - tx_broadcaster.clone(), - logger.clone(), - fee_estimator.clone(), - persister.clone(), - )); - let best_block = BestBlock::from_genesis(network); - let params = ChainParameters { network, best_block }; - let manager = Arc::new(ChannelManager::new( - fee_estimator.clone(), - chain_monitor.clone(), - tx_broadcaster.clone(), - logger.clone(), - keys_manager.clone(), - UserConfig::default(), - params, - )); - let network_graph = Arc::new(NetworkGraph::new(genesis_block.header.block_hash())); - let net_graph_msg_handler = Some(Arc::new(NetGraphMsgHandler::new( - network_graph.clone(), - Some(chain_source.clone()), - logger.clone(), - ))); - let msg_handler = MessageHandler { - chan_handler: Arc::new(test_utils::TestChannelMessageHandler::new()), - route_handler: Arc::new(test_utils::TestRoutingMessageHandler::new()), - }; - let peer_manager = Arc::new(PeerManager::new( - msg_handler, - keys_manager.get_node_secret(Recipient::Node).unwrap(), - &seed, - logger.clone(), - IgnoringMessageHandler {}, - )); - let node = Node { - node: manager, - net_graph_msg_handler, - peer_manager, - chain_monitor, - persister, - tx_broadcaster, - network_graph, - logger, - best_block, - }; - nodes.push(node); - } - - for i in 0..num_nodes { - for j in (i + 1)..num_nodes { - nodes[i].node.peer_connected(&nodes[j].node.get_our_node_id(), &Init { - features: InitFeatures::known(), - remote_network_address: None, - }); - nodes[j].node.peer_connected(&nodes[i].node.get_our_node_id(), &Init { - features: InitFeatures::known(), - remote_network_address: None, - }); - } - } - - nodes - } - - macro_rules! open_channel { - ($node_a: expr, $node_b: expr, $channel_value: expr) => {{ - begin_open_channel!($node_a, $node_b, $channel_value); - let events = $node_a.node.get_and_clear_pending_events(); - assert_eq!(events.len(), 1); - let (temporary_channel_id, tx) = handle_funding_generation_ready!(&events[0], $channel_value); - end_open_channel!($node_a, $node_b, temporary_channel_id, tx); - tx - }}; - } - - macro_rules! begin_open_channel { - ($node_a: expr, $node_b: expr, $channel_value: expr) => {{ - $node_a - .node - .create_channel($node_b.node.get_our_node_id(), $channel_value, 100, 42, None) - .unwrap(); - $node_b.node.handle_open_channel( - &$node_a.node.get_our_node_id(), - InitFeatures::known(), - &get_event_msg!( - $node_a, - MessageSendEvent::SendOpenChannel, - $node_b.node.get_our_node_id() - ), - ); - $node_a.node.handle_accept_channel( - &$node_b.node.get_our_node_id(), - InitFeatures::known(), - &get_event_msg!( - $node_b, - MessageSendEvent::SendAcceptChannel, - $node_a.node.get_our_node_id() - ), - ); - }}; - } - - macro_rules! handle_funding_generation_ready { - ($event: expr, $channel_value: expr) => {{ - match $event { - &Event::FundingGenerationReady { - temporary_channel_id, - channel_value_satoshis, - ref output_script, - user_channel_id, - } => { - assert_eq!(channel_value_satoshis, $channel_value); - assert_eq!(user_channel_id, 42); - - let tx = Transaction { - version: 1 as i32, - lock_time: 0, - input: Vec::new(), - output: vec![TxOut { - value: channel_value_satoshis, - script_pubkey: output_script.clone(), - }], - }; - (temporary_channel_id, tx) - }, - _ => panic!("Unexpected event"), - } - }}; - } - - macro_rules! end_open_channel { - ($node_a: expr, $node_b: expr, $temporary_channel_id: expr, $tx: expr) => {{ - $node_a - .node - .funding_transaction_generated(&$temporary_channel_id, $tx.clone()) - .unwrap(); - $node_b.node.handle_funding_created( - &$node_a.node.get_our_node_id(), - &get_event_msg!( - $node_a, - MessageSendEvent::SendFundingCreated, - $node_b.node.get_our_node_id() - ), - ); - $node_a.node.handle_funding_signed( - &$node_b.node.get_our_node_id(), - &get_event_msg!( - $node_b, - MessageSendEvent::SendFundingSigned, - $node_a.node.get_our_node_id() - ), - ); - }}; - } - - fn confirm_transaction_depth(node: &mut Node, tx: &Transaction, depth: u32) { - for i in 1..=depth { - let prev_blockhash = node.best_block.block_hash(); - let height = node.best_block.height() + 1; - let header = BlockHeader { - version: 0x20000000, - prev_blockhash, - merkle_root: Default::default(), - time: height, - bits: 42, - nonce: 42, - }; - let txdata = vec![(0, tx)]; - node.best_block = BestBlock::new(header.block_hash(), height); - match i { - 1 => { - node.node.transactions_confirmed(&header, &txdata, height); - node.chain_monitor.transactions_confirmed(&header, &txdata, height); - }, - x if x == depth => { - node.node.best_block_updated(&header, height); - node.chain_monitor.best_block_updated(&header, height); - }, - _ => {}, - } - } - } - fn confirm_transaction(node: &mut Node, tx: &Transaction) { confirm_transaction_depth(node, tx, ANTI_REORG_DELAY); } - - #[test] - fn test_background_processor() { - // Test that when a new channel is created, the ChannelManager needs to be re-persisted with - // updates. Also test that when new updates are available, the manager signals that it needs - // re-persistence and is successfully re-persisted. - let nodes = create_nodes(2, "test_background_processor".to_string()); - - // Go through the channel creation process so that each node has something to persist. Since - // open_channel consumes events, it must complete before starting BackgroundProcessor to - // avoid a race with processing events. - let tx = open_channel!(nodes[0], nodes[1], 100000); - - // Initiate the background processors to watch each node. - let node_0_persister = nodes[0].persister.clone(); - let persister = move |node: &ChannelManager< - InMemorySigner, - Arc, - Arc, - Arc, - Arc, - Arc, - >| node_0_persister.persist_manager(node); - let event_handler = |_: &_| {}; - let bg_processor = BackgroundProcessor::start( - persister, - event_handler, - nodes[0].chain_monitor.clone(), - nodes[0].node.clone(), - nodes[0].net_graph_msg_handler.clone(), - nodes[0].peer_manager.clone(), - nodes[0].logger.clone(), - ); - - macro_rules! check_persisted_data { - ($node: expr, $filepath: expr, $expected_bytes: expr) => { - loop { - $expected_bytes.clear(); - match $node.write(&mut $expected_bytes) { - Ok(()) => match std::fs::read($filepath) { - Ok(bytes) => { - if bytes == $expected_bytes { - break; - } else { - continue; - } - }, - Err(_) => continue, - }, - Err(e) => panic!("Unexpected error: {}", e), - } - } - }; - } - - // Check that the initial channel manager data is persisted as expected. - let filepath = get_full_filepath( - "test_background_processor_persister_0".to_string(), - "manager".to_string(), - ); - let mut expected_bytes = Vec::new(); - check_persisted_data!(nodes[0].node, filepath.clone(), expected_bytes); - loop { - if !nodes[0].node.get_persistence_condvar_value() { - break; - } - } - - // Force-close the channel. - nodes[0] - .node - .force_close_channel( - &OutPoint { - txid: tx.txid(), - index: 0, - } - .to_channel_id(), - ) - .unwrap(); - - // Check that the force-close updates are persisted. - let mut expected_bytes = Vec::new(); - check_persisted_data!(nodes[0].node, filepath.clone(), expected_bytes); - loop { - if !nodes[0].node.get_persistence_condvar_value() { - break; - } - } - - assert!(bg_processor.stop().is_ok()); - } - - #[test] - fn test_timer_tick_called() { - // Test that ChannelManager's and PeerManager's `timer_tick_occurred` is called every - // `FRESHNESS_TIMER`. - let nodes = create_nodes(1, "test_timer_tick_called".to_string()); - let node_0_persister = nodes[0].persister.clone(); - let persister = move |node: &ChannelManager< - InMemorySigner, - Arc, - Arc, - Arc, - Arc, - Arc, - >| node_0_persister.persist_manager(node); - let event_handler = |_: &_| {}; - let bg_processor = BackgroundProcessor::start( - persister, - event_handler, - nodes[0].chain_monitor.clone(), - nodes[0].node.clone(), - nodes[0].net_graph_msg_handler.clone(), - nodes[0].peer_manager.clone(), - nodes[0].logger.clone(), - ); - loop { - let log_entries = nodes[0].logger.lines.lock().unwrap(); - let desired_log = "Calling ChannelManager's timer_tick_occurred".to_string(); - let second_desired_log = "Calling PeerManager's timer_tick_occurred".to_string(); - if log_entries - .get(&("lightning_background_processor".to_string(), desired_log)) - .is_some() - && log_entries - .get(&("lightning_background_processor".to_string(), second_desired_log)) - .is_some() - { - break; - } - } - - assert!(bg_processor.stop().is_ok()); - } - - #[test] - fn test_persist_error() { - // Test that if we encounter an error during manager persistence, the thread panics. - let nodes = create_nodes(2, "test_persist_error".to_string()); - open_channel!(nodes[0], nodes[1], 100000); - - let persister = |_: &_| Err(std::io::Error::new(std::io::ErrorKind::Other, "test")); - let event_handler = |_: &_| {}; - let bg_processor = BackgroundProcessor::start( - persister, - event_handler, - nodes[0].chain_monitor.clone(), - nodes[0].node.clone(), - nodes[0].net_graph_msg_handler.clone(), - nodes[0].peer_manager.clone(), - nodes[0].logger.clone(), - ); - match bg_processor.join() { - Ok(_) => panic!("Expected error persisting manager"), - Err(e) => { - assert_eq!(e.kind(), std::io::ErrorKind::Other); - assert_eq!(e.get_ref().unwrap().to_string(), "test"); - }, - } - } - - #[test] - fn test_background_event_handling() { - let mut nodes = create_nodes(2, "test_background_event_handling".to_string()); - let channel_value = 100000; - let node_0_persister = nodes[0].persister.clone(); - let persister = move |node: &_| node_0_persister.persist_manager(node); - - // Set up a background event handler for FundingGenerationReady events. - let (sender, receiver) = std::sync::mpsc::sync_channel(1); - let event_handler = move |event: &Event| { - sender - .send(handle_funding_generation_ready!(event, channel_value)) - .unwrap(); - }; - let bg_processor = BackgroundProcessor::start( - persister.clone(), - event_handler, - nodes[0].chain_monitor.clone(), - nodes[0].node.clone(), - nodes[0].net_graph_msg_handler.clone(), - nodes[0].peer_manager.clone(), - nodes[0].logger.clone(), - ); - - // Open a channel and check that the FundingGenerationReady event was handled. - begin_open_channel!(nodes[0], nodes[1], channel_value); - let (temporary_channel_id, funding_tx) = receiver - .recv_timeout(Duration::from_secs(EVENT_DEADLINE)) - .expect("FundingGenerationReady not handled within deadline"); - end_open_channel!(nodes[0], nodes[1], temporary_channel_id, funding_tx); - - // Confirm the funding transaction. - confirm_transaction(&mut nodes[0], &funding_tx); - let as_funding = get_event_msg!( - nodes[0], - MessageSendEvent::SendFundingLocked, - nodes[1].node.get_our_node_id() - ); - confirm_transaction(&mut nodes[1], &funding_tx); - let bs_funding = get_event_msg!( - nodes[1], - MessageSendEvent::SendFundingLocked, - nodes[0].node.get_our_node_id() - ); - nodes[0] - .node - .handle_funding_locked(&nodes[1].node.get_our_node_id(), &bs_funding); - let _as_channel_update = get_event_msg!( - nodes[0], - MessageSendEvent::SendChannelUpdate, - nodes[1].node.get_our_node_id() - ); - nodes[1] - .node - .handle_funding_locked(&nodes[0].node.get_our_node_id(), &as_funding); - let _bs_channel_update = get_event_msg!( - nodes[1], - MessageSendEvent::SendChannelUpdate, - nodes[0].node.get_our_node_id() - ); - - assert!(bg_processor.stop().is_ok()); - - // Set up a background event handler for SpendableOutputs events. - let (sender, receiver) = std::sync::mpsc::sync_channel(1); - let event_handler = move |event: &Event| sender.send(event.clone()).unwrap(); - let bg_processor = BackgroundProcessor::start( - persister, - event_handler, - nodes[0].chain_monitor.clone(), - nodes[0].node.clone(), - nodes[0].net_graph_msg_handler.clone(), - nodes[0].peer_manager.clone(), - nodes[0].logger.clone(), - ); - - // Force close the channel and check that the SpendableOutputs event was handled. - nodes[0] - .node - .force_close_channel(&nodes[0].node.list_channels()[0].channel_id) - .unwrap(); - let commitment_tx = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().pop().unwrap(); - confirm_transaction_depth(&mut nodes[0], &commitment_tx, BREAKDOWN_TIMEOUT as u32); - let event = receiver - .recv_timeout(Duration::from_secs(EVENT_DEADLINE)) - .expect("SpendableOutputs not handled within deadline"); - match event { - Event::SpendableOutputs { .. } => {}, - Event::ChannelClosed { .. } => {}, - _ => panic!("Unexpected event: {:?}", event), - } - - assert!(bg_processor.stop().is_ok()); - } - - #[test] - fn test_invoice_payer() { - let keys_manager = test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet); - let random_seed_bytes = keys_manager.get_secure_random_bytes(); - let nodes = create_nodes(2, "test_invoice_payer".to_string()); - - // Initiate the background processors to watch each node. - let node_0_persister = nodes[0].persister.clone(); - let persister = move |node: &ChannelManager< - InMemorySigner, - Arc, - Arc, - Arc, - Arc, - Arc, - >| node_0_persister.persist_manager(node); - let router = DefaultRouter::new( - Arc::clone(&nodes[0].network_graph), - Arc::clone(&nodes[0].logger), - random_seed_bytes, - ); - let scorer = Arc::new(Mutex::new(test_utils::TestScorer::with_penalty(0))); - let invoice_payer = Arc::new(InvoicePayer::new( - Arc::clone(&nodes[0].node), - router, - scorer, - Arc::clone(&nodes[0].logger), - |_: &_| {}, - RetryAttempts(2), - )); - let event_handler = Arc::clone(&invoice_payer); - let bg_processor = BackgroundProcessor::start( - persister, - event_handler, - nodes[0].chain_monitor.clone(), - nodes[0].node.clone(), - nodes[0].net_graph_msg_handler.clone(), - nodes[0].peer_manager.clone(), - nodes[0].logger.clone(), - ); - assert!(bg_processor.stop().is_ok()); - } -} diff --git a/mm2src/coins/lightning_persister/Cargo.toml b/mm2src/coins/lightning_persister/Cargo.toml deleted file mode 100644 index 32b5d7eb1d..0000000000 --- a/mm2src/coins/lightning_persister/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -[package] -name = "lightning-persister" -version = "0.0.106" -edition = "2018" -authors = ["Valentine Wallace", "Matt Corallo"] -license = "MIT OR Apache-2.0" -repository = "https://github.com/lightningdevkit/rust-lightning/" -description = """ -Utilities to manage Rust-Lightning channel data persistence and retrieval. -""" - -[dependencies] -async-trait = "0.1" -bitcoin = "0.27.1" -common = { path = "../../common" } -mm2_io = { path = "../../mm2_io" } -db_common = { path = "../../db_common" } -derive_more = "0.99" -hex = "0.4.2" -lightning = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } -libc = "0.2" -parking_lot = { version = "0.12.0", features = ["nightly"] } -secp256k1 = { version = "0.20" } -serde = "1.0" -serde_json = "1.0" - -[target.'cfg(windows)'.dependencies] -winapi = { version = "0.3", features = ["winbase"] } - -[dev-dependencies] -lightning = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106", features = ["_test_utils"] } -rand = { version = "0.7", features = ["std", "small_rng"] } \ No newline at end of file diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs deleted file mode 100644 index 303205c26f..0000000000 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ /dev/null @@ -1,2097 +0,0 @@ -//! Utilities that handle persisting Rust-Lightning data to disk via standard filesystem APIs. - -#![feature(io_error_more)] - -pub mod storage; -mod util; - -extern crate async_trait; -extern crate bitcoin; -extern crate common; -extern crate libc; -extern crate lightning; -extern crate secp256k1; -extern crate serde_json; - -use crate::storage::{ChannelType, ChannelVisibility, ClosedChannelsFilter, DbStorage, FileSystemStorage, - GetClosedChannelsResult, GetPaymentsResult, HTLCStatus, NodesAddressesMap, - NodesAddressesMapShared, PaymentInfo, PaymentType, PaymentsFilter, Scorer, SqlChannelDetails}; -use crate::util::DiskWriteable; -use async_trait::async_trait; -use bitcoin::blockdata::constants::genesis_block; -use bitcoin::hash_types::{BlockHash, Txid}; -use bitcoin::hashes::hex::{FromHex, ToHex}; -use bitcoin::Network; -use common::{async_blocking, PagingOptionsEnum}; -use db_common::sqlite::rusqlite::{Error as SqlError, Row, ToSql, NO_PARAMS}; -use db_common::sqlite::sql_builder::SqlBuilder; -use db_common::sqlite::{h256_option_slice_from_row, h256_slice_from_row, offset_by_id, query_single_row, - sql_text_conversion_err, string_from_row, validate_table_name, SqliteConnShared, - CHECK_TABLE_EXISTS_SQL}; -use lightning::chain; -use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator}; -use lightning::chain::chainmonitor; -use lightning::chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate}; -use lightning::chain::keysinterface::{KeysInterface, Sign}; -use lightning::chain::transaction::OutPoint; -use lightning::ln::channelmanager::ChannelManager; -use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; -use lightning::routing::network_graph::NetworkGraph; -use lightning::routing::scoring::ProbabilisticScoringParameters; -use lightning::util::logger::Logger; -use lightning::util::ser::{Readable, ReadableArgs, Writeable}; -use mm2_io::fs::check_dir_operations; -use secp256k1::PublicKey; -use std::collections::HashMap; -use std::convert::TryInto; -use std::fs; -use std::io::{BufReader, BufWriter, Cursor, Error}; -use std::net::SocketAddr; -use std::ops::Deref; -use std::path::{Path, PathBuf}; -use std::str::FromStr; -use std::sync::{Arc, Mutex}; - -/// LightningPersister persists channel data on disk, where each channel's -/// data is stored in a file named after its funding outpoint. -/// It is also used to persist payments and channels history to sqlite database. -/// -/// Warning: this module does the best it can with calls to persist data, but it -/// can only guarantee that the data is passed to the drive. It is up to the -/// drive manufacturers to do the actual persistence properly, which they often -/// don't (especially on consumer-grade hardware). Therefore, it is up to the -/// user to validate their entire storage stack, to ensure the writes are -/// persistent. -/// Corollary: especially when dealing with larger amounts of money, it is best -/// practice to have multiple channel data backups and not rely only on one -/// LightningPersister. - -pub struct LightningPersister { - storage_ticker: String, - main_path: PathBuf, - backup_path: Option, - sqlite_connection: SqliteConnShared, -} - -impl DiskWriteable for ChannelMonitor { - fn write_to_file(&self, writer: &mut fs::File) -> Result<(), Error> { self.write(writer) } -} - -impl DiskWriteable - for ChannelManager -where - M::Target: chain::Watch, - T::Target: BroadcasterInterface, - K::Target: KeysInterface, - F::Target: FeeEstimator, - L::Target: Logger, -{ - fn write_to_file(&self, writer: &mut fs::File) -> Result<(), std::io::Error> { self.write(writer) } -} - -fn channels_history_table(ticker: &str) -> String { ticker.to_owned() + "_channels_history" } - -fn payments_history_table(ticker: &str) -> String { ticker.to_owned() + "_payments_history" } - -fn create_channels_history_table_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "CREATE TABLE IF NOT EXISTS {} ( - id INTEGER NOT NULL PRIMARY KEY, - rpc_id INTEGER NOT NULL UNIQUE, - channel_id VARCHAR(255) NOT NULL, - counterparty_node_id VARCHAR(255) NOT NULL, - funding_tx VARCHAR(255), - funding_value INTEGER, - funding_generated_in_block Integer, - closing_tx VARCHAR(255), - closure_reason TEXT, - claiming_tx VARCHAR(255), - claimed_balance REAL, - is_outbound INTEGER NOT NULL, - is_public INTEGER NOT NULL, - is_closed INTEGER NOT NULL, - created_at INTEGER NOT NULL, - closed_at INTEGER - );", - table_name - ); - - Ok(sql) -} - -fn create_payments_history_table_sql(for_coin: &str) -> Result { - let table_name = payments_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "CREATE TABLE IF NOT EXISTS {} ( - id INTEGER NOT NULL PRIMARY KEY, - payment_hash VARCHAR(255) NOT NULL UNIQUE, - destination VARCHAR(255), - description VARCHAR(641) NOT NULL, - preimage VARCHAR(255), - secret VARCHAR(255), - amount_msat INTEGER, - fee_paid_msat INTEGER, - is_outbound INTEGER NOT NULL, - status VARCHAR(255) NOT NULL, - created_at INTEGER NOT NULL, - last_updated INTEGER NOT NULL - );", - table_name - ); - - Ok(sql) -} - -fn insert_channel_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "INSERT INTO {} ( - rpc_id, - channel_id, - counterparty_node_id, - is_outbound, - is_public, - is_closed, - created_at - ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7 - );", - table_name - ); - - Ok(sql) -} - -fn upsert_payment_sql(for_coin: &str) -> Result { - let table_name = payments_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "INSERT OR REPLACE INTO {} ( - payment_hash, - destination, - description, - preimage, - secret, - amount_msat, - fee_paid_msat, - is_outbound, - status, - created_at, - last_updated - ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 - );", - table_name - ); - - Ok(sql) -} - -fn select_channel_by_rpc_id_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "SELECT - rpc_id, - channel_id, - counterparty_node_id, - funding_tx, - funding_value, - funding_generated_in_block, - closing_tx, - closure_reason, - claiming_tx, - claimed_balance, - is_outbound, - is_public, - is_closed, - created_at, - closed_at - FROM - {} - WHERE - rpc_id=?1", - table_name - ); - - Ok(sql) -} - -fn select_payment_by_hash_sql(for_coin: &str) -> Result { - let table_name = payments_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "SELECT - payment_hash, - destination, - description, - preimage, - secret, - amount_msat, - fee_paid_msat, - status, - is_outbound, - created_at, - last_updated - FROM - {} - WHERE - payment_hash=?1;", - table_name - ); - - Ok(sql) -} - -fn channel_details_from_row(row: &Row<'_>) -> Result { - let channel_details = SqlChannelDetails { - rpc_id: row.get::<_, u32>(0)? as u64, - channel_id: row.get(1)?, - counterparty_node_id: row.get(2)?, - funding_tx: row.get(3)?, - funding_value: row.get::<_, Option>(4)?.map(|v| v as u64), - funding_generated_in_block: row.get::<_, Option>(5)?.map(|v| v as u64), - closing_tx: row.get(6)?, - closure_reason: row.get(7)?, - claiming_tx: row.get(8)?, - claimed_balance: row.get::<_, Option>(9)?, - is_outbound: row.get(10)?, - is_public: row.get(11)?, - is_closed: row.get(12)?, - created_at: row.get::<_, u32>(13)? as u64, - closed_at: row.get::<_, Option>(14)?.map(|t| t as u64), - }; - Ok(channel_details) -} - -fn payment_info_from_row(row: &Row<'_>) -> Result { - let is_outbound = row.get::<_, bool>(8)?; - let payment_type = if is_outbound { - PaymentType::OutboundPayment { - destination: PublicKey::from_str(&row.get::<_, String>(1)?).map_err(|e| sql_text_conversion_err(1, e))?, - } - } else { - PaymentType::InboundPayment - }; - - let payment_info = PaymentInfo { - payment_hash: PaymentHash(h256_slice_from_row::(row, 0)?), - payment_type, - description: row.get(2)?, - preimage: h256_option_slice_from_row::(row, 3)?.map(PaymentPreimage), - secret: h256_option_slice_from_row::(row, 4)?.map(PaymentSecret), - amt_msat: row.get::<_, Option>(5)?.map(|v| v as u64), - fee_paid_msat: row.get::<_, Option>(6)?.map(|v| v as u64), - status: HTLCStatus::from_str(&row.get::<_, String>(7)?)?, - created_at: row.get::<_, u32>(9)? as u64, - last_updated: row.get::<_, u32>(10)? as u64, - }; - Ok(payment_info) -} - -fn get_last_channel_rpc_id_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!("SELECT IFNULL(MAX(rpc_id), 0) FROM {};", table_name); - - Ok(sql) -} - -fn update_funding_tx_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "UPDATE {} SET - funding_tx = ?1, - funding_value = ?2, - funding_generated_in_block = ?3 - WHERE - rpc_id = ?4;", - table_name - ); - - Ok(sql) -} - -fn update_funding_tx_block_height_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "UPDATE {} SET funding_generated_in_block = ?1 WHERE funding_tx = ?2;", - table_name - ); - - Ok(sql) -} - -fn update_channel_to_closed_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "UPDATE {} SET closure_reason = ?1, is_closed = ?2, closed_at = ?3 WHERE rpc_id = ?4;", - table_name - ); - - Ok(sql) -} - -fn update_closing_tx_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!("UPDATE {} SET closing_tx = ?1 WHERE rpc_id = ?2;", table_name); - - Ok(sql) -} - -fn get_channels_builder_preimage(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let mut sql_builder = SqlBuilder::select_from(table_name); - sql_builder.and_where("is_closed = 1"); - Ok(sql_builder) -} - -fn add_fields_to_get_channels_sql_builder(sql_builder: &mut SqlBuilder) { - sql_builder - .field("rpc_id") - .field("channel_id") - .field("counterparty_node_id") - .field("funding_tx") - .field("funding_value") - .field("funding_generated_in_block") - .field("closing_tx") - .field("closure_reason") - .field("claiming_tx") - .field("claimed_balance") - .field("is_outbound") - .field("is_public") - .field("is_closed") - .field("created_at") - .field("closed_at"); -} - -fn finalize_get_channels_sql_builder(sql_builder: &mut SqlBuilder, offset: usize, limit: usize) { - sql_builder.offset(offset); - sql_builder.limit(limit); - sql_builder.order_desc("closed_at"); -} - -fn apply_get_channels_filter(builder: &mut SqlBuilder, params: &mut Vec<(&str, String)>, filter: ClosedChannelsFilter) { - if let Some(channel_id) = filter.channel_id { - builder.and_where("channel_id = :channel_id"); - params.push((":channel_id", channel_id)); - } - - if let Some(counterparty_node_id) = filter.counterparty_node_id { - builder.and_where("counterparty_node_id = :counterparty_node_id"); - params.push((":counterparty_node_id", counterparty_node_id)); - } - - if let Some(funding_tx) = filter.funding_tx { - builder.and_where("funding_tx = :funding_tx"); - params.push((":funding_tx", funding_tx)); - } - - if let Some(from_funding_value) = filter.from_funding_value { - builder.and_where("funding_value >= :from_funding_value"); - params.push((":from_funding_value", from_funding_value.to_string())); - } - - if let Some(to_funding_value) = filter.to_funding_value { - builder.and_where("funding_value <= :to_funding_value"); - params.push((":to_funding_value", to_funding_value.to_string())); - } - - if let Some(closing_tx) = filter.closing_tx { - builder.and_where("closing_tx = :closing_tx"); - params.push((":closing_tx", closing_tx)); - } - - if let Some(closure_reason) = filter.closure_reason { - builder.and_where(format!("closure_reason LIKE '%{}%'", closure_reason)); - } - - if let Some(claiming_tx) = filter.claiming_tx { - builder.and_where("claiming_tx = :claiming_tx"); - params.push((":claiming_tx", claiming_tx)); - } - - if let Some(from_claimed_balance) = filter.from_claimed_balance { - builder.and_where("claimed_balance >= :from_claimed_balance"); - params.push((":from_claimed_balance", from_claimed_balance.to_string())); - } - - if let Some(to_claimed_balance) = filter.to_claimed_balance { - builder.and_where("claimed_balance <= :to_claimed_balance"); - params.push((":to_claimed_balance", to_claimed_balance.to_string())); - } - - if let Some(channel_type) = filter.channel_type { - let is_outbound = match channel_type { - ChannelType::Outbound => true as i32, - ChannelType::Inbound => false as i32, - }; - - builder.and_where("is_outbound = :is_outbound"); - params.push((":is_outbound", is_outbound.to_string())); - } - - if let Some(channel_visibility) = filter.channel_visibility { - let is_public = match channel_visibility { - ChannelVisibility::Public => true as i32, - ChannelVisibility::Private => false as i32, - }; - - builder.and_where("is_public = :is_public"); - params.push((":is_public", is_public.to_string())); - } -} - -fn get_payments_builder_preimage(for_coin: &str) -> Result { - let table_name = payments_history_table(for_coin); - validate_table_name(&table_name)?; - - Ok(SqlBuilder::select_from(table_name)) -} - -fn finalize_get_payments_sql_builder(sql_builder: &mut SqlBuilder, offset: usize, limit: usize) { - sql_builder - .field("payment_hash") - .field("destination") - .field("description") - .field("preimage") - .field("secret") - .field("amount_msat") - .field("fee_paid_msat") - .field("status") - .field("is_outbound") - .field("created_at") - .field("last_updated"); - sql_builder.offset(offset); - sql_builder.limit(limit); - sql_builder.order_desc("last_updated"); -} - -fn apply_get_payments_filter(builder: &mut SqlBuilder, params: &mut Vec<(&str, String)>, filter: PaymentsFilter) { - if let Some(payment_type) = filter.payment_type { - let (is_outbound, destination) = match payment_type { - PaymentType::OutboundPayment { destination } => (true as i32, Some(destination.to_string())), - PaymentType::InboundPayment => (false as i32, None), - }; - if let Some(dest) = destination { - builder.and_where("destination = :dest"); - params.push((":dest", dest)); - } - - builder.and_where("is_outbound = :is_outbound"); - params.push((":is_outbound", is_outbound.to_string())); - } - - if let Some(description) = filter.description { - builder.and_where(format!("description LIKE '%{}%'", description)); - } - - if let Some(status) = filter.status { - builder.and_where("status = :status"); - params.push((":status", status.to_string())); - } - - if let Some(from_amount) = filter.from_amount_msat { - builder.and_where("amount_msat >= :from_amount"); - params.push((":from_amount", from_amount.to_string())); - } - - if let Some(to_amount) = filter.to_amount_msat { - builder.and_where("amount_msat <= :to_amount"); - params.push((":to_amount", to_amount.to_string())); - } - - if let Some(from_fee) = filter.from_fee_paid_msat { - builder.and_where("fee_paid_msat >= :from_fee"); - params.push((":from_fee", from_fee.to_string())); - } - - if let Some(to_fee) = filter.to_fee_paid_msat { - builder.and_where("fee_paid_msat <= :to_fee"); - params.push((":to_fee", to_fee.to_string())); - } - - if let Some(from_time) = filter.from_timestamp { - builder.and_where("created_at >= :from_time"); - params.push((":from_time", from_time.to_string())); - } - - if let Some(to_time) = filter.to_timestamp { - builder.and_where("created_at <= :to_time"); - params.push((":to_time", to_time.to_string())); - } -} - -fn update_claiming_tx_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "UPDATE {} SET claiming_tx = ?1, claimed_balance = ?2 WHERE closing_tx = ?3;", - table_name - ); - - Ok(sql) -} - -impl LightningPersister { - /// Initialize a new LightningPersister and set the path to the individual channels' - /// files. - pub fn new( - storage_ticker: String, - main_path: PathBuf, - backup_path: Option, - sqlite_connection: SqliteConnShared, - ) -> Self { - Self { - storage_ticker, - main_path, - backup_path, - sqlite_connection, - } - } - - /// Get the directory which was provided when this persister was initialized. - pub fn main_path(&self) -> PathBuf { self.main_path.clone() } - - /// Get the backup directory which was provided when this persister was initialized. - pub fn backup_path(&self) -> Option { self.backup_path.clone() } - - pub(crate) fn monitor_path(&self) -> PathBuf { - let mut path = self.main_path(); - path.push("monitors"); - path - } - - pub(crate) fn monitor_backup_path(&self) -> Option { - if let Some(mut backup_path) = self.backup_path() { - backup_path.push("monitors"); - return Some(backup_path); - } - None - } - - pub(crate) fn nodes_addresses_path(&self) -> PathBuf { - let mut path = self.main_path(); - path.push("channel_nodes_data"); - path - } - - pub(crate) fn nodes_addresses_backup_path(&self) -> Option { - if let Some(mut backup_path) = self.backup_path() { - backup_path.push("channel_nodes_data"); - return Some(backup_path); - } - None - } - - pub(crate) fn network_graph_path(&self) -> PathBuf { - let mut path = self.main_path(); - path.push("network_graph"); - path - } - - pub(crate) fn scorer_path(&self) -> PathBuf { - let mut path = self.main_path(); - path.push("scorer"); - path - } - - pub fn manager_path(&self) -> PathBuf { - let mut path = self.main_path(); - path.push("manager"); - path - } - - /// Writes the provided `ChannelManager` to the path provided at `LightningPersister` - /// initialization, within a file called "manager". - pub fn persist_manager( - &self, - manager: &ChannelManager, - ) -> Result<(), std::io::Error> - where - M::Target: chain::Watch, - T::Target: BroadcasterInterface, - K::Target: KeysInterface, - F::Target: FeeEstimator, - L::Target: Logger, - { - let path = self.main_path(); - util::write_to_file(path, "manager".to_string(), manager)?; - if let Some(backup_path) = self.backup_path() { - util::write_to_file(backup_path, "manager".to_string(), manager)?; - } - Ok(()) - } - - /// Read `ChannelMonitor`s from disk. - pub fn read_channelmonitors( - &self, - keys_manager: K, - ) -> Result)>, std::io::Error> - where - K::Target: KeysInterface + Sized, - { - let path = self.monitor_path(); - if !Path::new(&path).exists() { - return Ok(Vec::new()); - } - let mut res = Vec::new(); - for file_option in fs::read_dir(path).unwrap() { - let file = file_option.unwrap(); - let owned_file_name = file.file_name(); - let filename = owned_file_name.to_str(); - if filename.is_none() || !filename.unwrap().is_ascii() || filename.unwrap().len() < 65 { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "Invalid ChannelMonitor file name", - )); - } - if filename.unwrap().ends_with(".tmp") { - // If we were in the middle of committing an new update and crashed, it should be - // safe to ignore the update - we should never have returned to the caller and - // irrevocably committed to the new state in any way. - continue; - } - - let txid = Txid::from_hex(filename.unwrap().split_at(64).0); - if txid.is_err() { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "Invalid tx ID in filename", - )); - } - - let index = filename.unwrap().split_at(65).1.parse::(); - if index.is_err() { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "Invalid tx index in filename", - )); - } - - let contents = fs::read(&file.path())?; - let mut buffer = Cursor::new(&contents); - match <(BlockHash, ChannelMonitor)>::read(&mut buffer, &*keys_manager) { - Ok((blockhash, channel_monitor)) => { - if channel_monitor.get_funding_txo().0.txid != txid.unwrap() - || channel_monitor.get_funding_txo().0.index != index.unwrap() - { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "ChannelMonitor was stored in the wrong file", - )); - } - res.push((blockhash, channel_monitor)); - }, - Err(e) => { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("Failed to deserialize ChannelMonitor: {}", e), - )) - }, - } - } - Ok(res) - } -} - -impl chainmonitor::Persist for LightningPersister { - // TODO: We really need a way for the persister to inform the user that its time to crash/shut - // down once these start returning failure. - // A PermanentFailure implies we need to shut down since we're force-closing channels without - // even broadcasting! - - fn persist_new_channel( - &self, - funding_txo: OutPoint, - monitor: &ChannelMonitor, - _update_id: chainmonitor::MonitorUpdateId, - ) -> Result<(), chain::ChannelMonitorUpdateErr> { - let filename = format!("{}_{}", funding_txo.txid.to_hex(), funding_txo.index); - util::write_to_file(self.monitor_path(), filename.clone(), monitor) - .map_err(|_| chain::ChannelMonitorUpdateErr::PermanentFailure)?; - if let Some(backup_path) = self.monitor_backup_path() { - util::write_to_file(backup_path, filename, monitor) - .map_err(|_| chain::ChannelMonitorUpdateErr::PermanentFailure)?; - } - Ok(()) - } - - fn update_persisted_channel( - &self, - funding_txo: OutPoint, - _update: &Option, - monitor: &ChannelMonitor, - _update_id: chainmonitor::MonitorUpdateId, - ) -> Result<(), chain::ChannelMonitorUpdateErr> { - let filename = format!("{}_{}", funding_txo.txid.to_hex(), funding_txo.index); - util::write_to_file(self.monitor_path(), filename.clone(), monitor) - .map_err(|_| chain::ChannelMonitorUpdateErr::PermanentFailure)?; - if let Some(backup_path) = self.monitor_backup_path() { - util::write_to_file(backup_path, filename, monitor) - .map_err(|_| chain::ChannelMonitorUpdateErr::PermanentFailure)?; - } - Ok(()) - } -} - -#[async_trait] -impl FileSystemStorage for LightningPersister { - type Error = std::io::Error; - - async fn init_fs(&self) -> Result<(), Self::Error> { - let path = self.main_path(); - let backup_path = self.backup_path(); - async_blocking(move || { - fs::create_dir_all(path.clone())?; - if let Some(path) = backup_path { - fs::create_dir_all(path.clone())?; - check_dir_operations(&path)?; - } - check_dir_operations(&path) - }) - .await - } - - async fn is_fs_initialized(&self) -> Result { - let dir_path = self.main_path(); - let backup_dir_path = self.backup_path(); - async_blocking(move || { - if !dir_path.exists() || backup_dir_path.as_ref().map(|path| !path.exists()).unwrap_or(false) { - Ok(false) - } else if !dir_path.is_dir() { - Err(std::io::Error::new( - std::io::ErrorKind::NotADirectory, - format!("{} is not a directory", dir_path.display()), - )) - } else if backup_dir_path.as_ref().map(|path| !path.is_dir()).unwrap_or(false) { - Err(std::io::Error::new( - std::io::ErrorKind::NotADirectory, - "Backup path is not a directory", - )) - } else { - let check_backup_ops = if let Some(backup_path) = backup_dir_path { - check_dir_operations(&backup_path).is_ok() - } else { - true - }; - check_dir_operations(&dir_path).map(|_| check_backup_ops) - } - }) - .await - } - - async fn get_nodes_addresses(&self) -> Result { - let path = self.nodes_addresses_path(); - if !path.exists() { - return Ok(HashMap::new()); - } - async_blocking(move || { - let file = fs::File::open(path)?; - let reader = BufReader::new(file); - let nodes_addresses: HashMap = - serde_json::from_reader(reader).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - nodes_addresses - .iter() - .map(|(pubkey_str, addr)| { - let pubkey = PublicKey::from_str(pubkey_str) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - Ok((pubkey, *addr)) - }) - .collect() - }) - .await - } - - async fn save_nodes_addresses(&self, nodes_addresses: NodesAddressesMapShared) -> Result<(), Self::Error> { - let path = self.nodes_addresses_path(); - let backup_path = self.nodes_addresses_backup_path(); - async_blocking(move || { - let nodes_addresses: HashMap = nodes_addresses - .lock() - .iter() - .map(|(pubkey, addr)| (pubkey.to_string(), *addr)) - .collect(); - - let file = fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(path)?; - serde_json::to_writer(file, &nodes_addresses) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - - if let Some(path) = backup_path { - let file = fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(path)?; - serde_json::to_writer(file, &nodes_addresses) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - } - - Ok(()) - }) - .await - } - - async fn get_network_graph(&self, network: Network) -> Result { - let path = self.network_graph_path(); - if !path.exists() { - return Ok(NetworkGraph::new(genesis_block(network).header.block_hash())); - } - async_blocking(move || { - let file = fs::File::open(path)?; - common::log::info!("Reading the saved lightning network graph from file, this can take some time!"); - NetworkGraph::read(&mut BufReader::new(file)) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) - }) - .await - } - - async fn save_network_graph(&self, network_graph: Arc) -> Result<(), Self::Error> { - let path = self.network_graph_path(); - async_blocking(move || { - let file = fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(path)?; - network_graph.write(&mut BufWriter::new(file)) - }) - .await - } - - async fn get_scorer(&self, network_graph: Arc) -> Result { - let path = self.scorer_path(); - if !path.exists() { - return Ok(Scorer::new(ProbabilisticScoringParameters::default(), network_graph)); - } - async_blocking(move || { - let file = fs::File::open(path)?; - Scorer::read( - &mut BufReader::new(file), - (ProbabilisticScoringParameters::default(), network_graph), - ) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) - }) - .await - } - - async fn save_scorer(&self, scorer: Arc>) -> Result<(), Self::Error> { - let path = self.scorer_path(); - async_blocking(move || { - let scorer = scorer.lock().unwrap(); - let file = fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(path)?; - scorer.write(&mut BufWriter::new(file)) - }) - .await - } -} - -#[async_trait] -impl DbStorage for LightningPersister { - type Error = SqlError; - - async fn init_db(&self) -> Result<(), Self::Error> { - let sqlite_connection = self.sqlite_connection.clone(); - let sql_channels_history = create_channels_history_table_sql(self.storage_ticker.as_str())?; - let sql_payments_history = create_payments_history_table_sql(self.storage_ticker.as_str())?; - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - conn.execute(&sql_channels_history, NO_PARAMS).map(|_| ())?; - conn.execute(&sql_payments_history, NO_PARAMS).map(|_| ())?; - Ok(()) - }) - .await - } - - async fn is_db_initialized(&self) -> Result { - let channels_history_table = channels_history_table(self.storage_ticker.as_str()); - validate_table_name(&channels_history_table)?; - let payments_history_table = payments_history_table(self.storage_ticker.as_str()); - validate_table_name(&payments_history_table)?; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - let channels_history_initialized = - query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [channels_history_table], string_from_row)?; - let payments_history_initialized = - query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [payments_history_table], string_from_row)?; - Ok(channels_history_initialized.is_some() && payments_history_initialized.is_some()) - }) - .await - } - - async fn get_last_channel_rpc_id(&self) -> Result { - let sql = get_last_channel_rpc_id_sql(self.storage_ticker.as_str())?; - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - let count: u32 = conn.query_row(&sql, NO_PARAMS, |r| r.get(0))?; - Ok(count) - }) - .await - } - - async fn add_channel_to_db(&self, details: SqlChannelDetails) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let rpc_id = details.rpc_id.to_string(); - let channel_id = details.channel_id; - let counterparty_node_id = details.counterparty_node_id; - let is_outbound = (details.is_outbound as i32).to_string(); - let is_public = (details.is_public as i32).to_string(); - let is_closed = (details.is_closed as i32).to_string(); - let created_at = (details.created_at as u32).to_string(); - - let params = [ - rpc_id, - channel_id, - counterparty_node_id, - is_outbound, - is_public, - is_closed, - created_at, - ]; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - sql_transaction.execute(&insert_channel_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn add_funding_tx_to_db( - &self, - rpc_id: u64, - funding_tx: String, - funding_value: u64, - funding_generated_in_block: u64, - ) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let funding_value = funding_value.to_string(); - let funding_generated_in_block = funding_generated_in_block.to_string(); - let rpc_id = rpc_id.to_string(); - - let params = [funding_tx, funding_value, funding_generated_in_block, rpc_id]; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - sql_transaction.execute(&update_funding_tx_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn update_funding_tx_block_height(&self, funding_tx: String, block_height: u64) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let generated_in_block = block_height as u32; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - let params = [&generated_in_block as &dyn ToSql, &funding_tx as &dyn ToSql]; - sql_transaction.execute(&update_funding_tx_block_height_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn update_channel_to_closed( - &self, - rpc_id: u64, - closure_reason: String, - closed_at: u64, - ) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let is_closed = "1".to_string(); - let rpc_id = rpc_id.to_string(); - - let params = [closure_reason, is_closed, closed_at.to_string(), rpc_id]; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - sql_transaction.execute(&update_channel_to_closed_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error> { - let mut builder = get_channels_builder_preimage(self.storage_ticker.as_str())?; - builder.and_where("closing_tx IS NULL"); - add_fields_to_get_channels_sql_builder(&mut builder); - let sql = builder.sql().expect("valid sql"); - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - - let mut stmt = conn.prepare(&sql)?; - let result = stmt - .query_map_named(&[], channel_details_from_row)? - .collect::>()?; - Ok(result) - }) - .await - } - - async fn add_closing_tx_to_db(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let rpc_id = rpc_id.to_string(); - - let params = [closing_tx, rpc_id]; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - sql_transaction.execute(&update_closing_tx_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn add_claiming_tx_to_db( - &self, - closing_tx: String, - claiming_tx: String, - claimed_balance: f64, - ) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let claimed_balance = claimed_balance.to_string(); - - let params = [claiming_tx, claimed_balance, closing_tx]; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - sql_transaction.execute(&update_claiming_tx_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn get_channel_from_db(&self, rpc_id: u64) -> Result, Self::Error> { - let params = [rpc_id.to_string()]; - let sql = select_channel_by_rpc_id_sql(self.storage_ticker.as_str())?; - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - query_single_row(&conn, &sql, params, channel_details_from_row) - }) - .await - } - - async fn get_closed_channels_by_filter( - &self, - filter: Option, - paging: PagingOptionsEnum, - limit: usize, - ) -> Result { - let mut sql_builder = get_channels_builder_preimage(self.storage_ticker.as_str())?; - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - - let mut total_builder = sql_builder.clone(); - total_builder.count("id"); - let total_sql = total_builder.sql().expect("valid sql"); - let total: isize = conn.query_row(&total_sql, NO_PARAMS, |row| row.get(0))?; - let total = total.try_into().expect("count should be always above zero"); - - let offset = match paging { - PagingOptionsEnum::PageNumber(page) => (page.get() - 1) * limit, - PagingOptionsEnum::FromId(rpc_id) => { - let params = [rpc_id as u32]; - let maybe_offset = - offset_by_id(&conn, &sql_builder, params, "rpc_id", "closed_at DESC", "rpc_id = ?1")?; - match maybe_offset { - Some(offset) => offset, - None => { - return Ok(GetClosedChannelsResult { - channels: vec![], - skipped: 0, - total, - }) - }, - } - }, - }; - - let mut params = vec![]; - if let Some(f) = filter { - apply_get_channels_filter(&mut sql_builder, &mut params, f); - } - let params_as_trait: Vec<_> = params.iter().map(|(key, value)| (*key, value as &dyn ToSql)).collect(); - add_fields_to_get_channels_sql_builder(&mut sql_builder); - finalize_get_channels_sql_builder(&mut sql_builder, offset, limit); - - let sql = sql_builder.sql().expect("valid sql"); - let mut stmt = conn.prepare(&sql)?; - let channels = stmt - .query_map_named(params_as_trait.as_slice(), channel_details_from_row)? - .collect::>()?; - let result = GetClosedChannelsResult { - channels, - skipped: offset, - total, - }; - Ok(result) - }) - .await - } - - async fn add_or_update_payment_in_db(&self, info: PaymentInfo) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let payment_hash = hex::encode(info.payment_hash.0); - let (is_outbound, destination) = match info.payment_type { - PaymentType::OutboundPayment { destination } => (true as i32, Some(destination.to_string())), - PaymentType::InboundPayment => (false as i32, None), - }; - let description = info.description; - let preimage = info.preimage.map(|p| hex::encode(p.0)); - let secret = info.secret.map(|s| hex::encode(s.0)); - let amount_msat = info.amt_msat.map(|a| a as u32); - let fee_paid_msat = info.fee_paid_msat.map(|f| f as u32); - let status = info.status.to_string(); - let created_at = info.created_at as u32; - let last_updated = info.last_updated as u32; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let params = [ - &payment_hash as &dyn ToSql, - &destination as &dyn ToSql, - &description as &dyn ToSql, - &preimage as &dyn ToSql, - &secret as &dyn ToSql, - &amount_msat as &dyn ToSql, - &fee_paid_msat as &dyn ToSql, - &is_outbound as &dyn ToSql, - &status as &dyn ToSql, - &created_at as &dyn ToSql, - &last_updated as &dyn ToSql, - ]; - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - sql_transaction.execute(&upsert_payment_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn get_payment_from_db(&self, hash: PaymentHash) -> Result, Self::Error> { - let params = [hex::encode(hash.0)]; - let sql = select_payment_by_hash_sql(self.storage_ticker.as_str())?; - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - query_single_row(&conn, &sql, params, payment_info_from_row) - }) - .await - } - - async fn get_payments_by_filter( - &self, - filter: Option, - paging: PagingOptionsEnum, - limit: usize, - ) -> Result { - let mut sql_builder = get_payments_builder_preimage(self.storage_ticker.as_str())?; - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - - let mut total_builder = sql_builder.clone(); - total_builder.count("id"); - let total_sql = total_builder.sql().expect("valid sql"); - let total: isize = conn.query_row(&total_sql, NO_PARAMS, |row| row.get(0))?; - let total = total.try_into().expect("count should be always above zero"); - - let offset = match paging { - PagingOptionsEnum::PageNumber(page) => (page.get() - 1) * limit, - PagingOptionsEnum::FromId(hash) => { - let hash_str = hex::encode(hash.0); - let params = [&hash_str]; - let maybe_offset = offset_by_id( - &conn, - &sql_builder, - params, - "payment_hash", - "last_updated DESC", - "payment_hash = ?1", - )?; - match maybe_offset { - Some(offset) => offset, - None => { - return Ok(GetPaymentsResult { - payments: vec![], - skipped: 0, - total, - }) - }, - } - }, - }; - - let mut params = vec![]; - if let Some(f) = filter { - apply_get_payments_filter(&mut sql_builder, &mut params, f); - } - let params_as_trait: Vec<_> = params.iter().map(|(key, value)| (*key, value as &dyn ToSql)).collect(); - finalize_get_payments_sql_builder(&mut sql_builder, offset, limit); - - let sql = sql_builder.sql().expect("valid sql"); - let mut stmt = conn.prepare(&sql)?; - let payments = stmt - .query_map_named(params_as_trait.as_slice(), payment_info_from_row)? - .collect::>()?; - let result = GetPaymentsResult { - payments, - skipped: offset, - total, - }; - Ok(result) - }) - .await - } -} - -#[cfg(test)] -mod tests { - use super::*; - extern crate bitcoin; - extern crate lightning; - use bitcoin::blockdata::block::{Block, BlockHeader}; - use bitcoin::hashes::hex::FromHex; - use bitcoin::Txid; - use common::{block_on, now_ms}; - use db_common::sqlite::rusqlite::Connection; - use lightning::chain::chainmonitor::Persist; - use lightning::chain::transaction::OutPoint; - use lightning::chain::ChannelMonitorUpdateErr; - use lightning::ln::features::InitFeatures; - use lightning::ln::functional_test_utils::*; - use lightning::util::events::{ClosureReason, MessageSendEventsProvider}; - use lightning::util::test_utils; - use lightning::{check_added_monitors, check_closed_broadcast, check_closed_event}; - use rand::distributions::Alphanumeric; - use rand::{Rng, RngCore}; - use secp256k1::{Secp256k1, SecretKey}; - use std::fs; - use std::num::NonZeroUsize; - use std::path::PathBuf; - use std::sync::{Arc, Mutex}; - - impl Drop for LightningPersister { - fn drop(&mut self) { - // We test for invalid directory names, so it's OK if directory removal - // fails. - match fs::remove_dir_all(&self.main_path) { - Err(e) => println!("Failed to remove test persister directory: {}", e), - _ => {}, - } - } - } - - fn generate_random_channels(num: u64) -> Vec { - let mut rng = rand::thread_rng(); - let mut channels = vec![]; - let s = Secp256k1::new(); - let mut bytes = [0; 32]; - for i in 0..num { - let details = SqlChannelDetails { - rpc_id: i + 1, - channel_id: { - rng.fill_bytes(&mut bytes); - hex::encode(bytes) - }, - counterparty_node_id: { - rng.fill_bytes(&mut bytes); - let secret = SecretKey::from_slice(&bytes).unwrap(); - let pubkey = PublicKey::from_secret_key(&s, &secret); - pubkey.to_string() - }, - funding_tx: { - rng.fill_bytes(&mut bytes); - Some(hex::encode(bytes)) - }, - funding_value: Some(rng.gen::() as u64), - closing_tx: { - rng.fill_bytes(&mut bytes); - Some(hex::encode(bytes)) - }, - closure_reason: { - Some( - rng.sample_iter(&Alphanumeric) - .take(30) - .map(char::from) - .collect::(), - ) - }, - claiming_tx: { - rng.fill_bytes(&mut bytes); - Some(hex::encode(bytes)) - }, - claimed_balance: Some(rng.gen::()), - funding_generated_in_block: Some(rng.gen::() as u64), - is_outbound: rand::random(), - is_public: rand::random(), - is_closed: rand::random(), - created_at: rng.gen::() as u64, - closed_at: Some(rng.gen::() as u64), - }; - channels.push(details); - } - channels - } - - fn generate_random_payments(num: u64) -> Vec { - let mut rng = rand::thread_rng(); - let mut payments = vec![]; - let s = Secp256k1::new(); - let mut bytes = [0; 32]; - for _ in 0..num { - let payment_type = if let 0 = rng.gen::() % 2 { - PaymentType::InboundPayment - } else { - rng.fill_bytes(&mut bytes); - let secret = SecretKey::from_slice(&bytes).unwrap(); - PaymentType::OutboundPayment { - destination: PublicKey::from_secret_key(&s, &secret), - } - }; - let status_rng: u8 = rng.gen(); - let status = if status_rng % 3 == 0 { - HTLCStatus::Succeeded - } else if status_rng % 3 == 1 { - HTLCStatus::Pending - } else { - HTLCStatus::Failed - }; - let description: String = rng.sample_iter(&Alphanumeric).take(30).map(char::from).collect(); - let info = PaymentInfo { - payment_hash: { - rng.fill_bytes(&mut bytes); - PaymentHash(bytes) - }, - payment_type, - description, - preimage: { - rng.fill_bytes(&mut bytes); - Some(PaymentPreimage(bytes)) - }, - secret: { - rng.fill_bytes(&mut bytes); - Some(PaymentSecret(bytes)) - }, - amt_msat: Some(rng.gen::() as u64), - fee_paid_msat: Some(rng.gen::() as u64), - status, - created_at: rng.gen::() as u64, - last_updated: rng.gen::() as u64, - }; - payments.push(info); - } - payments - } - - // Integration-test the LightningPersister. Test relaying a few payments - // and check that the persisted data is updated the appropriate number of - // times. - #[test] - fn test_filesystem_persister() { - // Create the nodes, giving them LightningPersisters for data persisters. - let persister_0 = LightningPersister::new( - "test_filesystem_persister_0".into(), - PathBuf::from("test_filesystem_persister_0"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - let persister_1 = LightningPersister::new( - "test_filesystem_persister_1".into(), - PathBuf::from("test_filesystem_persister_1"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - let chanmon_cfgs = create_chanmon_cfgs(2); - let mut node_cfgs = create_node_cfgs(2, &chanmon_cfgs); - let chain_mon_0 = test_utils::TestChainMonitor::new( - Some(&chanmon_cfgs[0].chain_source), - &chanmon_cfgs[0].tx_broadcaster, - &chanmon_cfgs[0].logger, - &chanmon_cfgs[0].fee_estimator, - &persister_0, - &node_cfgs[0].keys_manager, - ); - let chain_mon_1 = test_utils::TestChainMonitor::new( - Some(&chanmon_cfgs[1].chain_source), - &chanmon_cfgs[1].tx_broadcaster, - &chanmon_cfgs[1].logger, - &chanmon_cfgs[1].fee_estimator, - &persister_1, - &node_cfgs[1].keys_manager, - ); - node_cfgs[0].chain_monitor = chain_mon_0; - node_cfgs[1].chain_monitor = chain_mon_1; - let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); - let nodes = create_network(2, &node_cfgs, &node_chanmgrs); - - // Check that the persisted channel data is empty before any channels are - // open. - let mut persisted_chan_data_0 = persister_0.read_channelmonitors(nodes[0].keys_manager).unwrap(); - assert_eq!(persisted_chan_data_0.len(), 0); - let mut persisted_chan_data_1 = persister_1.read_channelmonitors(nodes[1].keys_manager).unwrap(); - assert_eq!(persisted_chan_data_1.len(), 0); - - // Helper to make sure the channel is on the expected update ID. - macro_rules! check_persisted_data { - ($expected_update_id: expr) => { - persisted_chan_data_0 = persister_0.read_channelmonitors(nodes[0].keys_manager).unwrap(); - assert_eq!(persisted_chan_data_0.len(), 1); - for (_, mon) in persisted_chan_data_0.iter() { - assert_eq!(mon.get_latest_update_id(), $expected_update_id); - } - persisted_chan_data_1 = persister_1.read_channelmonitors(nodes[1].keys_manager).unwrap(); - assert_eq!(persisted_chan_data_1.len(), 1); - for (_, mon) in persisted_chan_data_1.iter() { - assert_eq!(mon.get_latest_update_id(), $expected_update_id); - } - }; - } - - // Create some initial channel and check that a channel was persisted. - let _ = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()); - check_persisted_data!(0); - - // Send a few payments and make sure the monitors are updated to the latest. - send_payment(&nodes[0], &vec![&nodes[1]][..], 8000000); - check_persisted_data!(5); - send_payment(&nodes[1], &vec![&nodes[0]][..], 4000000); - check_persisted_data!(10); - - // Force close because cooperative close doesn't result in any persisted - // updates. - nodes[0] - .node - .force_close_channel(&nodes[0].node.list_channels()[0].channel_id) - .unwrap(); - check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed); - check_closed_broadcast!(nodes[0], true); - check_added_monitors!(nodes[0], 1); - - let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap(); - assert_eq!(node_txn.len(), 1); - - let header = BlockHeader { - version: 0x20000000, - prev_blockhash: nodes[0].best_block_hash(), - merkle_root: Default::default(), - time: 42, - bits: 42, - nonce: 42, - }; - connect_block(&nodes[1], &Block { - header, - txdata: vec![node_txn[0].clone(), node_txn[0].clone()], - }); - check_closed_broadcast!(nodes[1], true); - check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed); - check_added_monitors!(nodes[1], 1); - - // Make sure everything is persisted as expected after close. - check_persisted_data!(11); - } - - // Test that if the persister's path to channel data is read-only, writing a - // monitor to it results in the persister returning a PermanentFailure. - // Windows ignores the read-only flag for folders, so this test is Unix-only. - #[cfg(not(target_os = "windows"))] - #[test] - fn test_readonly_dir_perm_failure() { - let persister = LightningPersister::new( - "test_readonly_dir_perm_failure".into(), - PathBuf::from("test_readonly_dir_perm_failure"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - fs::create_dir_all(&persister.main_path).unwrap(); - - // Set up a dummy channel and force close. This will produce a monitor - // that we can then use to test persistence. - let chanmon_cfgs = create_chanmon_cfgs(2); - let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); - let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); - let nodes = create_network(2, &node_cfgs, &node_chanmgrs); - let chan = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()); - nodes[1].node.force_close_channel(&chan.2).unwrap(); - check_closed_event!(nodes[1], 1, ClosureReason::HolderForceClosed); - let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap(); - let update_map = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap(); - let update_id = update_map.get(&added_monitors[0].0.to_channel_id()).unwrap(); - - // Set the persister's directory to read-only, which should result in - // returning a permanent failure when we then attempt to persist a - // channel update. - let path = &persister.main_path; - let mut perms = fs::metadata(path).unwrap().permissions(); - perms.set_readonly(true); - fs::set_permissions(path, perms).unwrap(); - - let test_txo = OutPoint { - txid: Txid::from_hex("8984484a580b825b9972d7adb15050b3ab624ccd731946b3eeddb92f4e7ef6be").unwrap(), - index: 0, - }; - match persister.persist_new_channel(test_txo, &added_monitors[0].1, update_id.2) { - Err(ChannelMonitorUpdateErr::PermanentFailure) => {}, - _ => panic!("unexpected result from persisting new channel"), - } - - nodes[1].node.get_and_clear_pending_msg_events(); - added_monitors.clear(); - } - - // Test that if a persister's directory name is invalid, monitor persistence - // will fail. - #[cfg(target_os = "windows")] - #[test] - fn test_fail_on_open() { - // Set up a dummy channel and force close. This will produce a monitor - // that we can then use to test persistence. - let chanmon_cfgs = create_chanmon_cfgs(2); - let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); - let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); - let nodes = create_network(2, &node_cfgs, &node_chanmgrs); - let chan = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()); - nodes[1].node.force_close_channel(&chan.2).unwrap(); - check_closed_event!(nodes[1], 1, ClosureReason::HolderForceClosed); - let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap(); - let update_map = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap(); - let update_id = update_map.get(&added_monitors[0].0.to_channel_id()).unwrap(); - - // Create the persister with an invalid directory name and test that the - // channel fails to open because the directories fail to be created. There - // don't seem to be invalid filename characters on Unix that Rust doesn't - // handle, hence why the test is Windows-only. - let persister = LightningPersister::new( - "test_fail_on_open".into(), - PathBuf::from(":<>/"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - - let test_txo = OutPoint { - txid: Txid::from_hex("8984484a580b825b9972d7adb15050b3ab624ccd731946b3eeddb92f4e7ef6be").unwrap(), - index: 0, - }; - match persister.persist_new_channel(test_txo, &added_monitors[0].1, update_id.2) { - Err(ChannelMonitorUpdateErr::PermanentFailure) => {}, - _ => panic!("unexpected result from persisting new channel"), - } - - nodes[1].node.get_and_clear_pending_msg_events(); - added_monitors.clear(); - } - - #[test] - fn test_init_sql_collection() { - let persister = LightningPersister::new( - "init_sql_collection".into(), - PathBuf::from("test_filesystem_persister"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - let initialized = block_on(persister.is_db_initialized()).unwrap(); - assert!(!initialized); - - block_on(persister.init_db()).unwrap(); - // repetitive init must not fail - block_on(persister.init_db()).unwrap(); - - let initialized = block_on(persister.is_db_initialized()).unwrap(); - assert!(initialized); - } - - #[test] - fn test_add_get_channel_sql() { - let persister = LightningPersister::new( - "add_get_channel".into(), - PathBuf::from("test_filesystem_persister"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - - block_on(persister.init_db()).unwrap(); - - let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id()).unwrap(); - assert_eq!(last_channel_rpc_id, 0); - - let channel = block_on(persister.get_channel_from_db(1)).unwrap(); - assert!(channel.is_none()); - - let mut expected_channel_details = SqlChannelDetails::new( - 1, - [0; 32], - PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), - true, - true, - ); - block_on(persister.add_channel_to_db(expected_channel_details.clone())).unwrap(); - let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id()).unwrap(); - assert_eq!(last_channel_rpc_id, 1); - - let actual_channel_details = block_on(persister.get_channel_from_db(1)).unwrap().unwrap(); - assert_eq!(expected_channel_details, actual_channel_details); - - // must fail because we are adding channel with the same rpc_id - block_on(persister.add_channel_to_db(expected_channel_details.clone())).unwrap_err(); - assert_eq!(last_channel_rpc_id, 1); - - expected_channel_details.rpc_id = 2; - block_on(persister.add_channel_to_db(expected_channel_details.clone())).unwrap(); - let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id()).unwrap(); - assert_eq!(last_channel_rpc_id, 2); - - block_on(persister.add_funding_tx_to_db( - 2, - "9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into(), - 3000, - 50000, - )) - .unwrap(); - expected_channel_details.funding_tx = - Some("9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into()); - expected_channel_details.funding_value = Some(3000); - expected_channel_details.funding_generated_in_block = Some(50000); - - let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); - assert_eq!(expected_channel_details, actual_channel_details); - - block_on(persister.update_funding_tx_block_height( - "9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into(), - 50001, - )) - .unwrap(); - expected_channel_details.funding_generated_in_block = Some(50001); - - let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); - assert_eq!(expected_channel_details, actual_channel_details); - - let current_time = now_ms() / 1000; - block_on(persister.update_channel_to_closed(2, "the channel was cooperatively closed".into(), current_time)) - .unwrap(); - expected_channel_details.closure_reason = Some("the channel was cooperatively closed".into()); - expected_channel_details.is_closed = true; - expected_channel_details.closed_at = Some(current_time); - - let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); - assert_eq!(expected_channel_details, actual_channel_details); - - let actual_channels = block_on(persister.get_closed_channels_with_no_closing_tx()).unwrap(); - assert_eq!(actual_channels.len(), 1); - - let closed_channels = - block_on(persister.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 10)).unwrap(); - assert_eq!(closed_channels.channels.len(), 1); - assert_eq!(expected_channel_details, closed_channels.channels[0]); - - block_on(persister.update_channel_to_closed(1, "the channel was cooperatively closed".into(), now_ms() / 1000)) - .unwrap(); - let closed_channels = - block_on(persister.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 10)).unwrap(); - assert_eq!(closed_channels.channels.len(), 2); - - let actual_channels = block_on(persister.get_closed_channels_with_no_closing_tx()).unwrap(); - assert_eq!(actual_channels.len(), 2); - - block_on(persister.add_closing_tx_to_db( - 2, - "5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into(), - )) - .unwrap(); - expected_channel_details.closing_tx = - Some("5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into()); - - let actual_channels = block_on(persister.get_closed_channels_with_no_closing_tx()).unwrap(); - assert_eq!(actual_channels.len(), 1); - - let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); - assert_eq!(expected_channel_details, actual_channel_details); - - block_on(persister.add_claiming_tx_to_db( - "5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into(), - "97f061634a4a7b0b0c2b95648f86b1c39b95e0cf5073f07725b7143c095b612a".into(), - 2000.333333, - )) - .unwrap(); - expected_channel_details.claiming_tx = - Some("97f061634a4a7b0b0c2b95648f86b1c39b95e0cf5073f07725b7143c095b612a".into()); - expected_channel_details.claimed_balance = Some(2000.333333); - - let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); - assert_eq!(expected_channel_details, actual_channel_details); - } - - #[test] - fn test_add_get_payment_sql() { - let persister = LightningPersister::new( - "add_get_payment".into(), - PathBuf::from("test_filesystem_persister"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - - block_on(persister.init_db()).unwrap(); - - let payment = block_on(persister.get_payment_from_db(PaymentHash([0; 32]))).unwrap(); - assert!(payment.is_none()); - - let mut expected_payment_info = PaymentInfo { - payment_hash: PaymentHash([0; 32]), - payment_type: PaymentType::InboundPayment, - description: "test payment".into(), - preimage: Some(PaymentPreimage([2; 32])), - secret: Some(PaymentSecret([3; 32])), - amt_msat: Some(2000), - fee_paid_msat: Some(100), - status: HTLCStatus::Failed, - created_at: now_ms() / 1000, - last_updated: now_ms() / 1000, - }; - block_on(persister.add_or_update_payment_in_db(expected_payment_info.clone())).unwrap(); - - let actual_payment_info = block_on(persister.get_payment_from_db(PaymentHash([0; 32]))) - .unwrap() - .unwrap(); - assert_eq!(expected_payment_info, actual_payment_info); - - expected_payment_info.payment_hash = PaymentHash([1; 32]); - expected_payment_info.payment_type = PaymentType::OutboundPayment { - destination: PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9") - .unwrap(), - }; - expected_payment_info.secret = None; - expected_payment_info.amt_msat = None; - expected_payment_info.status = HTLCStatus::Succeeded; - expected_payment_info.last_updated = now_ms() / 1000; - block_on(persister.add_or_update_payment_in_db(expected_payment_info.clone())).unwrap(); - - let actual_payment_info = block_on(persister.get_payment_from_db(PaymentHash([1; 32]))) - .unwrap() - .unwrap(); - assert_eq!(expected_payment_info, actual_payment_info); - } - - #[test] - fn test_get_payments_by_filter() { - let persister = LightningPersister::new( - "test_get_payments_by_filter".into(), - PathBuf::from("test_filesystem_persister"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - - block_on(persister.init_db()).unwrap(); - - let mut payments = generate_random_payments(100); - - for payment in payments.clone() { - block_on(persister.add_or_update_payment_in_db(payment)).unwrap(); - } - - let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); - let limit = 4; - - let result = block_on(persister.get_payments_by_filter(None, paging, limit)).unwrap(); - - payments.sort_by(|a, b| b.last_updated.cmp(&a.last_updated)); - let expected_payments = &payments[..4].to_vec(); - let actual_payments = &result.payments; - - assert_eq!(0, result.skipped); - assert_eq!(100, result.total); - assert_eq!(expected_payments, actual_payments); - - let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(2).unwrap()); - let limit = 5; - - let result = block_on(persister.get_payments_by_filter(None, paging, limit)).unwrap(); - - let expected_payments = &payments[5..10].to_vec(); - let actual_payments = &result.payments; - - assert_eq!(5, result.skipped); - assert_eq!(100, result.total); - assert_eq!(expected_payments, actual_payments); - - let from_payment_hash = payments[20].payment_hash; - let paging = PagingOptionsEnum::FromId(from_payment_hash); - let limit = 3; - - let result = block_on(persister.get_payments_by_filter(None, paging, limit)).unwrap(); - - let expected_payments = &payments[21..24].to_vec(); - let actual_payments = &result.payments; - - assert_eq!(expected_payments, actual_payments); - - let mut filter = PaymentsFilter { - payment_type: Some(PaymentType::InboundPayment), - description: None, - status: None, - from_amount_msat: None, - to_amount_msat: None, - from_fee_paid_msat: None, - to_fee_paid_msat: None, - from_timestamp: None, - to_timestamp: None, - }; - let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); - let limit = 10; - - let result = block_on(persister.get_payments_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); - let expected_payments_vec: Vec = payments - .iter() - .map(|p| p.clone()) - .filter(|p| p.payment_type == PaymentType::InboundPayment) - .collect(); - let expected_payments = if expected_payments_vec.len() > 10 { - expected_payments_vec[..10].to_vec() - } else { - expected_payments_vec.clone() - }; - let actual_payments = result.payments; - - assert_eq!(expected_payments, actual_payments); - - filter.status = Some(HTLCStatus::Succeeded); - let result = block_on(persister.get_payments_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); - let expected_payments_vec: Vec = expected_payments_vec - .iter() - .map(|p| p.clone()) - .filter(|p| p.status == HTLCStatus::Succeeded) - .collect(); - let expected_payments = if expected_payments_vec.len() > 10 { - expected_payments_vec[..10].to_vec() - } else { - expected_payments_vec - }; - let actual_payments = result.payments; - - assert_eq!(expected_payments, actual_payments); - - let description = &payments[42].description; - let substr = &description[5..10]; - filter.payment_type = None; - filter.status = None; - filter.description = Some(substr.to_string()); - let result = block_on(persister.get_payments_by_filter(Some(filter), paging, limit)).unwrap(); - let expected_payments_vec: Vec = payments - .iter() - .map(|p| p.clone()) - .filter(|p| p.description.contains(&substr)) - .collect(); - let expected_payments = if expected_payments_vec.len() > 10 { - expected_payments_vec[..10].to_vec() - } else { - expected_payments_vec.clone() - }; - let actual_payments = result.payments; - - assert_eq!(expected_payments, actual_payments); - } - - #[test] - fn test_get_channels_by_filter() { - let persister = LightningPersister::new( - "test_get_channels_by_filter".into(), - PathBuf::from("test_filesystem_persister"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - - block_on(persister.init_db()).unwrap(); - - let channels = generate_random_channels(100); - - for channel in channels { - block_on(persister.add_channel_to_db(channel.clone())).unwrap(); - block_on(persister.add_funding_tx_to_db( - channel.rpc_id, - channel.funding_tx.unwrap(), - channel.funding_value.unwrap(), - channel.funding_generated_in_block.unwrap(), - )) - .unwrap(); - block_on(persister.update_channel_to_closed(channel.rpc_id, channel.closure_reason.unwrap(), 1655806080)) - .unwrap(); - block_on(persister.add_closing_tx_to_db(channel.rpc_id, channel.closing_tx.clone().unwrap())).unwrap(); - block_on(persister.add_claiming_tx_to_db( - channel.closing_tx.unwrap(), - channel.claiming_tx.unwrap(), - channel.claimed_balance.unwrap(), - )) - .unwrap(); - } - - // get all channels from SQL since updated_at changed from channels generated by generate_random_channels - let channels = block_on(persister.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 100)) - .unwrap() - .channels; - assert_eq!(100, channels.len()); - - let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); - let limit = 4; - - let result = block_on(persister.get_closed_channels_by_filter(None, paging, limit)).unwrap(); - - let expected_channels = &channels[..4].to_vec(); - let actual_channels = &result.channels; - - assert_eq!(0, result.skipped); - assert_eq!(100, result.total); - assert_eq!(expected_channels, actual_channels); - - let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(2).unwrap()); - let limit = 5; - - let result = block_on(persister.get_closed_channels_by_filter(None, paging, limit)).unwrap(); - - let expected_channels = &channels[5..10].to_vec(); - let actual_channels = &result.channels; - - assert_eq!(5, result.skipped); - assert_eq!(100, result.total); - assert_eq!(expected_channels, actual_channels); - - let from_rpc_id = 20; - let paging = PagingOptionsEnum::FromId(from_rpc_id); - let limit = 3; - - let result = block_on(persister.get_closed_channels_by_filter(None, paging, limit)).unwrap(); - - let expected_channels = channels[20..23].to_vec(); - let actual_channels = result.channels; - - assert_eq!(expected_channels, actual_channels); - - let mut filter = ClosedChannelsFilter { - channel_id: None, - counterparty_node_id: None, - funding_tx: None, - from_funding_value: None, - to_funding_value: None, - closing_tx: None, - closure_reason: None, - claiming_tx: None, - from_claimed_balance: None, - to_claimed_balance: None, - channel_type: Some(ChannelType::Outbound), - channel_visibility: None, - }; - let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); - let limit = 10; - - let result = - block_on(persister.get_closed_channels_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); - let expected_channels_vec: Vec = channels - .iter() - .map(|chan| chan.clone()) - .filter(|chan| chan.is_outbound) - .collect(); - let expected_channels = if expected_channels_vec.len() > 10 { - expected_channels_vec[..10].to_vec() - } else { - expected_channels_vec.clone() - }; - let actual_channels = result.channels; - - assert_eq!(expected_channels, actual_channels); - - filter.channel_visibility = Some(ChannelVisibility::Public); - let result = - block_on(persister.get_closed_channels_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); - let expected_channels_vec: Vec = expected_channels_vec - .iter() - .map(|chan| chan.clone()) - .filter(|chan| chan.is_public) - .collect(); - let expected_channels = if expected_channels_vec.len() > 10 { - expected_channels_vec[..10].to_vec() - } else { - expected_channels_vec - }; - let actual_channels = result.channels; - - assert_eq!(expected_channels, actual_channels); - - let channel_id = channels[42].channel_id.clone(); - filter.channel_type = None; - filter.channel_visibility = None; - filter.channel_id = Some(channel_id.clone()); - let result = block_on(persister.get_closed_channels_by_filter(Some(filter), paging, limit)).unwrap(); - let expected_channels_vec: Vec = channels - .iter() - .map(|chan| chan.clone()) - .filter(|chan| chan.channel_id == channel_id) - .collect(); - let expected_channels = if expected_channels_vec.len() > 10 { - expected_channels_vec[..10].to_vec() - } else { - expected_channels_vec.clone() - }; - let actual_channels = result.channels; - - assert_eq!(expected_channels, actual_channels); - } -} diff --git a/mm2src/coins/lightning_persister/src/util.rs b/mm2src/coins/lightning_persister/src/util.rs deleted file mode 100644 index ac5bc99de5..0000000000 --- a/mm2src/coins/lightning_persister/src/util.rs +++ /dev/null @@ -1,196 +0,0 @@ -#[cfg(target_os = "windows")] extern crate winapi; - -use std::fs; -use std::path::{Path, PathBuf}; - -#[cfg(not(target_os = "windows"))] -use std::os::unix::io::AsRawFd; - -#[cfg(target_os = "windows")] -use {std::ffi::OsStr, std::os::windows::ffi::OsStrExt}; - -pub(crate) trait DiskWriteable { - fn write_to_file(&self, writer: &mut fs::File) -> Result<(), std::io::Error>; -} - -pub(crate) fn get_full_filepath(mut filepath: PathBuf, filename: String) -> String { - filepath.push(filename); - filepath.to_str().unwrap().to_string() -} - -#[cfg(target_os = "windows")] -macro_rules! call { - ($e: expr) => { - if $e != 0 { - return Ok(()); - } else { - return Err(std::io::Error::last_os_error()); - } - }; -} - -#[cfg(target_os = "windows")] -fn path_to_windows_str>(path: T) -> Vec { - path.as_ref().encode_wide().chain(Some(0)).collect() -} - -#[allow(bare_trait_objects)] -pub(crate) fn write_to_file(path: PathBuf, filename: String, data: &D) -> std::io::Result<()> { - fs::create_dir_all(path.clone())?; - // Do a crazy dance with lots of fsync()s to be overly cautious here... - // We never want to end up in a state where we've lost the old data, or end up using the - // old data on power loss after we've returned. - // The way to atomically write a file on Unix platforms is: - // open(tmpname), write(tmpfile), fsync(tmpfile), close(tmpfile), rename(), fsync(dir) - let filename_with_path = get_full_filepath(path, filename); - let tmp_filename = format!("{}.tmp", filename_with_path); - - { - // Note that going by rust-lang/rust@d602a6b, on MacOS it is only safe to use - // rust stdlib 1.36 or higher. - let mut f = fs::File::create(&tmp_filename)?; - data.write_to_file(&mut f)?; - f.sync_all()?; - } - // Fsync the parent directory on Unix. - #[cfg(not(target_os = "windows"))] - { - fs::rename(&tmp_filename, &filename_with_path)?; - let path = Path::new(&filename_with_path).parent().unwrap(); - let dir_file = fs::OpenOptions::new().read(true).open(path)?; - unsafe { - libc::fsync(dir_file.as_raw_fd()); - } - } - #[cfg(target_os = "windows")] - { - let src = PathBuf::from(tmp_filename); - let dst = PathBuf::from(filename_with_path.clone()); - if Path::new(&filename_with_path).exists() { - unsafe { - winapi::um::winbase::ReplaceFileW( - path_to_windows_str(dst).as_ptr(), - path_to_windows_str(src).as_ptr(), - std::ptr::null(), - winapi::um::winbase::REPLACEFILE_IGNORE_MERGE_ERRORS, - std::ptr::null_mut() as *mut winapi::ctypes::c_void, - std::ptr::null_mut() as *mut winapi::ctypes::c_void, - ) - }; - } else { - call!(unsafe { - winapi::um::winbase::MoveFileExW( - path_to_windows_str(src).as_ptr(), - path_to_windows_str(dst).as_ptr(), - winapi::um::winbase::MOVEFILE_WRITE_THROUGH | winapi::um::winbase::MOVEFILE_REPLACE_EXISTING, - ) - }); - } - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::{get_full_filepath, write_to_file, DiskWriteable}; - use std::fs; - use std::io; - use std::io::Write; - use std::path::PathBuf; - - struct TestWriteable {} - impl DiskWriteable for TestWriteable { - fn write_to_file(&self, writer: &mut fs::File) -> Result<(), io::Error> { writer.write_all(&[42; 1]) } - } - - // Test that if the persister's path to channel data is read-only, writing - // data to it fails. Windows ignores the read-only flag for folders, so this - // test is Unix-only. - #[cfg(not(target_os = "windows"))] - #[test] - fn test_readonly_dir() { - let test_writeable = TestWriteable {}; - let filename = "test_readonly_dir_persister_filename".to_string(); - let path = "test_readonly_dir_persister_dir"; - fs::create_dir_all(path.to_string()).unwrap(); - let mut perms = fs::metadata(path.to_string()).unwrap().permissions(); - perms.set_readonly(true); - fs::set_permissions(path.to_string(), perms).unwrap(); - match write_to_file(PathBuf::from(path.to_string()), filename, &test_writeable) { - Err(e) => assert_eq!(e.kind(), io::ErrorKind::PermissionDenied), - _ => panic!("Unexpected error message"), - } - let mut perms = fs::metadata(path.to_string()).unwrap().permissions(); - perms.set_readonly(false); - fs::set_permissions(path.to_string(), perms).unwrap(); - fs::remove_dir_all(path).unwrap(); - } - - // Test failure to rename in the process of atomically creating a channel - // monitor's file. We induce this failure by making the `tmp` file a - // directory. - // Explanation: given "from" = the file being renamed, "to" = the destination - // file that already exists: Unix should fail because if "from" is a file, - // then "to" is also required to be a file. - // TODO: ideally try to make this work on Windows again - #[cfg(not(target_os = "windows"))] - #[test] - fn test_rename_failure() { - let test_writeable = TestWriteable {}; - let filename = "test_rename_failure_filename"; - let path = PathBuf::from("test_rename_failure_dir"); - // Create the channel data file and make it a directory. - fs::create_dir_all(get_full_filepath(path.clone(), filename.to_string())).unwrap(); - match write_to_file(path.clone(), filename.to_string(), &test_writeable) { - Err(e) => assert_eq!(e.raw_os_error(), Some(libc::EISDIR)), - _ => panic!("Unexpected Ok(())"), - } - fs::remove_dir_all(path).unwrap(); - } - - #[test] - fn test_diskwriteable_failure() { - struct FailingWriteable {} - impl DiskWriteable for FailingWriteable { - fn write_to_file(&self, _writer: &mut fs::File) -> Result<(), std::io::Error> { - Err(std::io::Error::new(std::io::ErrorKind::Other, "expected failure")) - } - } - - let filename = "test_diskwriteable_failure"; - let path = PathBuf::from("test_diskwriteable_failure_dir"); - let test_writeable = FailingWriteable {}; - match write_to_file(path.clone(), filename.to_string(), &test_writeable) { - Err(e) => { - assert_eq!(e.kind(), std::io::ErrorKind::Other); - assert_eq!(e.get_ref().unwrap().to_string(), "expected failure"); - }, - _ => panic!("unexpected result"), - } - fs::remove_dir_all(path).unwrap(); - } - - // Test failure to create the temporary file in the persistence process. - // We induce this failure by having the temp file already exist and be a - // directory. - #[test] - fn test_tmp_file_creation_failure() { - let test_writeable = TestWriteable {}; - let filename = "test_tmp_file_creation_failure_filename".to_string(); - let path = PathBuf::from("test_tmp_file_creation_failure_dir"); - - // Create the tmp file and make it a directory. - let tmp_path = get_full_filepath(path.clone(), format!("{}.tmp", filename.clone())); - fs::create_dir_all(tmp_path).unwrap(); - match write_to_file(path.clone(), filename, &test_writeable) { - Err(e) => { - #[cfg(not(target_os = "windows"))] - assert_eq!(e.raw_os_error(), Some(libc::EISDIR)); - #[cfg(target_os = "windows")] - assert_eq!(e.kind(), io::ErrorKind::PermissionDenied); - }, - _ => panic!("Unexpected error message"), - } - fs::remove_dir_all(path).unwrap(); - } -} diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 4fed0635b9..0f1e3e4085 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -178,38 +178,6 @@ macro_rules! ok_or_continue_after_sleep { }; } -#[cfg(not(target_arch = "wasm32"))] -macro_rules! ok_or_retry_after_sleep { - ($e:expr, $delay: ident) => { - loop { - match $e { - Ok(res) => break res, - Err(e) => { - error!("error {:?}", e); - Timer::sleep($delay).await; - continue; - }, - } - } - }; -} - -#[cfg(not(target_arch = "wasm32"))] -macro_rules! ok_or_retry_after_sleep_sync { - ($e:expr, $delay: ident) => { - loop { - match $e { - Ok(res) => break res, - Err(e) => { - error!("error {:?}", e); - std::thread::sleep(core::time::Duration::from_secs($delay)); - continue; - }, - } - } - }; -} - pub mod coin_balance; #[doc(hidden)] #[cfg(test)] diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 6e30d5468f..45815465b0 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -31,6 +31,7 @@ mod bchd_pb; pub mod qtum; pub mod rpc_clients; pub mod slp; +pub mod spv; pub mod utxo_block_header_storage; pub mod utxo_builder; pub mod utxo_common; @@ -69,8 +70,9 @@ use primitives::hash::{H256, H264}; use rpc::v1::types::{Bytes as BytesJson, Transaction as RpcTransaction, H256 as H256Json}; use script::{Builder, Script, SignatureVersion, TransactionInputSigner}; use serde_json::{self as json, Value as Json}; -use serialization::{serialize, serialize_with_flags, SERIALIZE_TRANSACTION_WITNESS}; +use serialization::{serialize, serialize_with_flags, Error as SerError, SERIALIZE_TRANSACTION_WITNESS}; use spv_validation::helpers_validation::SPVError; +use spv_validation::storage::BlockHeaderStorageError; use std::array::TryFromSliceError; use std::collections::{HashMap, HashSet}; use std::convert::TryInto; @@ -90,6 +92,7 @@ use utxo_signer::{TxProvider, TxProviderError, UtxoSignTxError, UtxoSignTxResult use self::rpc_clients::{electrum_script_hash, ElectrumClient, ElectrumRpcRequest, EstimateFeeMethod, EstimateFeeMode, NativeClient, UnspentInfo, UnspentMap, UtxoRpcClientEnum, UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; +use self::utxo_block_header_storage::BlockHeaderVerificationParams; use super::{big_decimal_from_sat_unsigned, BalanceError, BalanceFut, BalanceResult, CoinBalance, CoinsContext, DerivationMethod, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, KmdRewardsDetails, MarketCoinOps, MmCoin, NumConversError, NumConversResult, PrivKeyActivationPolicy, PrivKeyNotAllowed, PrivKeyPolicy, @@ -101,9 +104,7 @@ use crate::coin_balance::{EnableCoinScanPolicy, HDAddressBalanceScanner}; use crate::hd_wallet::{HDAccountOps, HDAccountsMutex, HDAddress, HDWalletCoinOps, HDWalletOps, InvalidBip44ChainError}; use crate::hd_wallet_storage::{HDAccountStorageItem, HDWalletCoinStorage, HDWalletStorageError, HDWalletStorageResult}; use crate::utxo::tx_cache::UtxoVerboseCacheShared; -use crate::utxo::utxo_block_header_storage::BlockHeaderStorageError; use crate::TransactionErr; -use utxo_block_header_storage::BlockHeaderStorage; pub mod tx_cache; #[cfg(target_arch = "wasm32")] @@ -530,7 +531,6 @@ pub struct UtxoCoinFields { pub history_sync_state: Mutex, /// The cache of verbose transactions. pub tx_cache: UtxoVerboseCacheShared, - pub block_headers_storage: Option, /// The cache of recently send transactions used to track the spent UTXOs and replace them with new outputs /// The daemon needs some time to update the listunspent list for address which makes it return already spent UTXOs /// This cache helps to prevent UTXO reuse in such cases @@ -566,27 +566,51 @@ impl From for WithdrawError { fn from(e: UnsupportedAddr) -> Self { WithdrawError::InvalidAddress(e.to_string()) } } +#[derive(Debug)] +#[allow(clippy::large_enum_variant)] +pub enum GetTxError { + Rpc(UtxoRpcError), + TxDeserialization(SerError), +} + +impl From for GetTxError { + fn from(err: UtxoRpcError) -> GetTxError { GetTxError::Rpc(err) } +} + +impl From for GetTxError { + fn from(err: SerError) -> GetTxError { GetTxError::TxDeserialization(err) } +} + #[derive(Debug)] pub enum GetTxHeightError { - HeightNotFound, + HeightNotFound(String), } impl From for SPVError { fn from(e: GetTxHeightError) -> Self { match e { - GetTxHeightError::HeightNotFound => SPVError::InvalidHeight, + GetTxHeightError::HeightNotFound(e) => SPVError::InvalidHeight(e), } } } -#[derive(Debug)] +impl From for GetTxHeightError { + fn from(e: UtxoRpcError) -> Self { GetTxHeightError::HeightNotFound(e.to_string()) } +} + +#[derive(Debug, Display)] pub enum GetBlockHeaderError { + #[display(fmt = "Block header storage error: {}", _0)] StorageError(BlockHeaderStorageError), + #[display(fmt = "RPC error: {}", _0)] RpcError(JsonRpcError), + #[display(fmt = "Serialization error: {}", _0)] SerializationError(serialization::Error), + #[display(fmt = "Invalid response: {}", _0)] InvalidResponse(String), + #[display(fmt = "Error validating headers: {}", _0)] SPVError(SPVError), - NativeNotSupported(String), + #[display(fmt = "Internal error: {}", _0)] Internal(String), } @@ -604,10 +628,6 @@ impl From for GetBlockHeaderError { } } -impl From for GetBlockHeaderError { - fn from(e: SPVError) -> Self { GetBlockHeaderError::SPVError(e) } -} - impl From for GetBlockHeaderError { fn from(err: serialization::Error) -> Self { GetBlockHeaderError::SerializationError(err) } } @@ -616,6 +636,10 @@ impl From for GetBlockHeaderError { fn from(err: BlockHeaderStorageError) -> Self { GetBlockHeaderError::StorageError(err) } } +impl From for SPVError { + fn from(e: GetBlockHeaderError) -> Self { SPVError::UnableToGetHeader(e.to_string()) } +} + impl UtxoCoinFields { pub fn transaction_preimage(&self) -> TransactionInputSigner { let lock_time = if self.conf.ticker == "KMD" { @@ -1191,14 +1215,6 @@ pub struct UtxoMergeParams { pub max_merge_at_once: usize, } -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct UtxoBlockHeaderVerificationParams { - pub difficulty_check: bool, - pub constant_difficulty: bool, - pub blocks_limit_to_check: NonZeroU64, - pub check_every: f64, -} - #[derive(Clone, Debug, Deserialize, Serialize)] pub struct UtxoActivationParams { pub mode: UtxoRpcMode, @@ -1239,7 +1255,12 @@ impl UtxoActivationParams { Some("electrum") => { let servers = json::from_value(req["servers"].clone()).map_to_mm(UtxoFromLegacyReqErr::InvalidElectrumServers)?; - UtxoRpcMode::Electrum { servers } + let block_header_params = json::from_value(req["block_header_params"].clone()) + .map_to_mm(UtxoFromLegacyReqErr::InvalidBlockHeaderVerificationParams)?; + UtxoRpcMode::Electrum { + servers, + block_header_params, + } }, _ => return MmError::err(UtxoFromLegacyReqErr::UnexpectedMethod), }; @@ -1281,7 +1302,10 @@ impl UtxoActivationParams { #[serde(tag = "rpc", content = "rpc_data")] pub enum UtxoRpcMode { Native, - Electrum { servers: Vec }, + Electrum { + servers: Vec, + block_header_params: Option, + }, } #[derive(Debug)] diff --git a/mm2src/coins/utxo/qtum.rs b/mm2src/coins/utxo/qtum.rs index cd9ba229af..70adae0cc2 100644 --- a/mm2src/coins/utxo/qtum.rs +++ b/mm2src/coins/utxo/qtum.rs @@ -12,8 +12,9 @@ use crate::rpc_command::init_create_account::{self, CreateNewAccountParams, Init use crate::rpc_command::init_scan_for_new_addresses::{self, InitScanAddressesRpcOps, ScanAddressesParams, ScanAddressesResponse}; use crate::rpc_command::init_withdraw::{InitWithdrawCoin, WithdrawTaskHandle}; -use crate::utxo::utxo_builder::{MergeUtxoArcOps, UtxoCoinBuildError, UtxoCoinBuilder, UtxoCoinBuilderCommonOps, - UtxoFieldsWithHardwareWalletBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; +use crate::utxo::utxo_builder::{BlockHeaderUtxoArcOps, MergeUtxoArcOps, UtxoCoinBuildError, UtxoCoinBuilder, + UtxoCoinBuilderCommonOps, UtxoFieldsWithHardwareWalletBuilder, + UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::{eth, CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, DelegationError, DelegationFut, GetWithdrawSenderAddress, NegotiateSwapContractAddrErr, PrivKeyBuildPolicy, SearchForSwapTxSpendInput, SignatureResult, StakingInfosFut, SwapOps, TradePreimageValue, TransactionFut, UnexpectedDerivationMethod, @@ -216,17 +217,29 @@ impl<'a> UtxoCoinBuilder for QtumCoinBuilder<'a> { async fn build(self) -> MmResult { let utxo = self.build_utxo_fields().await?; + let rpc_client = utxo.rpc_client.clone(); let utxo_arc = UtxoArc::new(utxo); let utxo_weak = utxo_arc.downgrade(); let result_coin = QtumCoin::from(utxo_arc); - self.spawn_merge_utxo_loop_if_required(utxo_weak, QtumCoin::from); + if let Some(abort_handler) = self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), QtumCoin::from) { + self.ctx.abort_handlers.lock().unwrap().push(abort_handler); + } + + if let Some(abort_handler) = + self.spawn_block_header_utxo_loop_if_required(utxo_weak, &rpc_client, QtumCoin::from) + { + self.ctx.abort_handlers.lock().unwrap().push(abort_handler); + } + Ok(result_coin) } } impl<'a> MergeUtxoArcOps for QtumCoinBuilder<'a> {} +impl<'a> BlockHeaderUtxoArcOps for QtumCoinBuilder<'a> {} + impl<'a> QtumCoinBuilder<'a> { pub fn new( ctx: &'a MmArc, diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index dac1d3712f..41a09e6272 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -1,7 +1,8 @@ #![cfg_attr(target_arch = "wasm32", allow(unused_macros))] #![cfg_attr(target_arch = "wasm32", allow(dead_code))] -use crate::utxo::{output_script, sat_from_big_decimal}; +use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; +use crate::utxo::{output_script, sat_from_big_decimal, GetBlockHeaderError, GetTxError, GetTxHeightError}; use crate::{big_decimal_from_sat_unsigned, NumConversError, RpcTransportEventHandler, RpcTransportEventHandlerShared}; use async_trait::async_trait; use chain::{BlockHeader, BlockHeaderBits, BlockHeaderNonce, OutPoint, Transaction as UtxoTx}; @@ -34,8 +35,11 @@ use serde_json::{self as json, Value as Json}; use serialization::{coin_variant_by_ticker, deserialize, serialize, serialize_with_flags, CoinVariant, CompactInteger, Reader, SERIALIZE_TRANSACTION_WITNESS}; use sha2::{Digest, Sha256}; +use spv_validation::helpers_validation::{validate_headers, SPVError}; +use spv_validation::storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::convert::TryInto; use std::fmt; use std::io; use std::net::{SocketAddr, ToSocketAddrs}; @@ -63,6 +67,8 @@ cfg_native! { use webpki_roots::TLS_SERVER_ROOTS; } +pub const NO_TX_ERROR_CODE: &str = "'code': -5"; + pub type AddressesByLabelResult = HashMap; pub type JsonRpcPendingRequestsShared = Arc>; pub type JsonRpcPendingRequests = HashMap>; @@ -338,6 +344,28 @@ pub trait UtxoRpcClientOps: fmt::Debug + Send + Sync + 'static { /// Returns block time in seconds since epoch (Jan 1 1970 GMT). async fn get_block_timestamp(&self, height: u64) -> Result>; + + /// Returns verbose transaction by the given `txid` if it's on-chain or None if it's not. + async fn get_tx_if_onchain(&self, tx_hash: &H256Json) -> Result, MmError> { + match self + .get_transaction_bytes(tx_hash) + .compat() + .await + .map_err(|e| e.into_inner()) + { + Ok(bytes) => Ok(Some(deserialize(bytes.as_slice())?)), + Err(err) => { + if let UtxoRpcError::ResponseParseError(ref json_err) = err { + if let JsonRpcErrorType::Response(_, json) = &json_err.error { + if json["message"].as_str().unwrap_or_default().contains(NO_TX_ERROR_CODE) { + return Ok(None); + } + } + } + Err(err.into()) + }, + } + } } #[derive(Clone, Deserialize, Debug)] @@ -1539,6 +1567,7 @@ pub struct ElectrumClientImpl { protocol_version: OrdRange, get_balance_concurrent_map: ConcurrentRequestMap, list_unspent_concurrent_map: ConcurrentRequestMap>, + block_headers_storage: Option, } async fn electrum_request_multi( @@ -1679,6 +1708,9 @@ impl ElectrumClientImpl { /// Get available protocol versions. pub fn protocol_version(&self) -> &OrdRange { &self.protocol_version } + + /// Get block headers storage. + pub fn block_headers_storage(&self) -> &Option { &self.block_headers_storage } } #[derive(Clone, Debug)] @@ -1886,6 +1918,93 @@ impl ElectrumClient { pub fn blockchain_transaction_get_merkle(&self, txid: H256Json, height: u64) -> RpcRes { rpc_func!(self, "blockchain.transaction.get_merkle", txid, height) } + + async fn get_tx_height(&self, tx: &UtxoTx) -> Result> { + for output in tx.outputs.clone() { + let script_pubkey_str = hex::encode(electrum_script_hash(&output.script_pubkey)); + if let Ok(history) = self.scripthash_get_history(script_pubkey_str.as_str()).compat().await { + if let Some(item) = history + .into_iter() + .find(|item| item.tx_hash.reversed() == H256Json(*tx.hash()) && item.height > 0) + { + return Ok(item.height as u64); + } + } + } + MmError::err(GetTxHeightError::HeightNotFound( + "Couldn't find height through electrum!".into(), + )) + } + + async fn tx_height_from_storage_or_rpc(&self, tx: &UtxoTx) -> Result> { + if let Some(storage) = &self.block_headers_storage { + let ticker = self.coin_name(); + let tx_hash = tx.hash().reversed(); + let blockhash = self.get_verbose_transaction(&tx_hash.into()).compat().await?.blockhash; + if let Ok(Some(height)) = storage.get_block_height_by_hash(ticker, blockhash.into()).await { + if let Ok(height) = height.try_into() { + return Ok(height); + } + } + } + + self.get_tx_height(tx).await + } + + async fn valid_block_header_from_storage(&self, height: u64) -> Result> { + let storage = match &self.block_headers_storage { + Some(storage) => storage, + None => { + return MmError::err(GetBlockHeaderError::StorageError(BlockHeaderStorageError::Internal( + "block_headers_storage is not initialized".to_owned(), + ))) + }, + }; + let ticker = self.coin_name(); + match storage.get_block_header(ticker, height).await? { + None => { + let bytes = self.blockchain_block_header(height).compat().await?; + let header: BlockHeader = deserialize(bytes.0.as_slice())?; + let params = &storage.params; + let blocks_limit = params.blocks_limit_to_check; + let (headers_registry, headers) = self.retrieve_last_headers(blocks_limit, height).compat().await?; + match validate_headers(headers, params.difficulty_check, params.constant_difficulty) { + Ok(_) => { + storage.add_block_headers_to_storage(ticker, headers_registry).await?; + Ok(header) + }, + Err(err) => MmError::err(GetBlockHeaderError::SPVError(err)), + } + }, + Some(header) => Ok(header), + } + } + + async fn block_header_from_storage_or_rpc(&self, height: u64) -> Result> { + match &self.block_headers_storage { + Some(_) => self.valid_block_header_from_storage(height).await, + None => Ok(deserialize( + self.blockchain_block_header(height).compat().await?.as_slice(), + )?), + } + } + + pub async fn get_merkle_and_header( + &self, + tx: &UtxoTx, + ) -> Result<(TxMerkleBranch, BlockHeader, u64), MmError> { + let height = self.tx_height_from_storage_or_rpc(tx).await?; + + let merkle_branch = self + .blockchain_transaction_get_merkle(tx.hash().reversed().into(), height) + .compat() + .await + .map_to_mm(|e| SPVError::UnableToGetMerkle(e.to_string()))?; + + let header = self.block_header_from_storage_or_rpc(height).await?; + + Ok((merkle_branch, header, height)) + } } // if mockable is placed before async_trait there is `munmap_chunk(): invalid pointer` error on async fn mocking attempt @@ -2117,7 +2236,11 @@ impl UtxoRpcClientOps for ElectrumClient { #[cfg_attr(test, mockable)] impl ElectrumClientImpl { - pub fn new(coin_ticker: String, event_handlers: Vec) -> ElectrumClientImpl { + pub fn new( + coin_ticker: String, + event_handlers: Vec, + block_headers_storage: Option, + ) -> ElectrumClientImpl { let protocol_version = OrdRange::new(1.2, 1.4).unwrap(); ElectrumClientImpl { coin_ticker, @@ -2127,6 +2250,7 @@ impl ElectrumClientImpl { protocol_version, get_balance_concurrent_map: ConcurrentRequestMap::new(), list_unspent_concurrent_map: ConcurrentRequestMap::new(), + block_headers_storage, } } @@ -2135,10 +2259,11 @@ impl ElectrumClientImpl { coin_ticker: String, event_handlers: Vec, protocol_version: OrdRange, + block_headers_storage: Option, ) -> ElectrumClientImpl { ElectrumClientImpl { protocol_version, - ..ElectrumClientImpl::new(coin_ticker, event_handlers) + ..ElectrumClientImpl::new(coin_ticker, event_handlers, block_headers_storage) } } } diff --git a/mm2src/coins/utxo/slp.rs b/mm2src/coins/utxo/slp.rs index 9ed15d4843..6af6a0ea13 100644 --- a/mm2src/coins/utxo/slp.rs +++ b/mm2src/coins/utxo/slp.rs @@ -41,7 +41,7 @@ use rpc::v1::types::{Bytes as BytesJson, ToTxHash, H256 as H256Json}; use script::bytes::Bytes; use script::{Builder as ScriptBuilder, Opcode, Script, TransactionInputSigner}; use serde_json::Value as Json; -use serialization::{deserialize, serialize, Deserializable, Error, Reader}; +use serialization::{deserialize, serialize, Deserializable, Error as SerError, Reader}; use serialization_derive::Deserializable; use std::convert::TryInto; use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; @@ -102,7 +102,7 @@ struct SlpTxPreimage { enum ValidateHtlcError { TxLackOfOutputs, #[display(fmt = "TxParseError: {:?}", _0)] - TxParseError(Error), + TxParseError(SerError), #[display(fmt = "OpReturnParseError: {:?}", _0)] OpReturnParseError(ParseSlpScriptError), InvalidSlpDetails, @@ -186,7 +186,7 @@ impl From for SpendP2SHError { pub enum SpendHtlcError { TxLackOfOutputs, #[display(fmt = "DeserializationErr: {:?}", _0)] - DeserializationErr(Error), + DeserializationErr(SerError), #[display(fmt = "PubkeyParseError: {:?}", _0)] PubkeyParseErr(keys::Error), InvalidSlpDetails, @@ -206,8 +206,8 @@ impl From for SpendHtlcError { fn from(err: NumConversError) -> SpendHtlcError { SpendHtlcError::NumConversionErr(err) } } -impl From for SpendHtlcError { - fn from(err: Error) -> SpendHtlcError { SpendHtlcError::DeserializationErr(err) } +impl From for SpendHtlcError { + fn from(err: SerError) -> SpendHtlcError { SpendHtlcError::DeserializationErr(err) } } impl From for SpendHtlcError { @@ -815,7 +815,7 @@ impl SlpTransaction { } impl Deserializable for SlpTransaction { - fn deserialize(reader: &mut Reader) -> Result + fn deserialize(reader: &mut Reader) -> Result where Self: Sized, T: std::io::Read, @@ -831,7 +831,7 @@ impl Deserializable for SlpTransaction { } else { let mut url = vec![0; maybe_push_op_code as usize]; reader.read_slice(&mut url)?; - String::from_utf8(url).map_err(|e| Error::Custom(e.to_string()))? + String::from_utf8(url).map_err(|e| SerError::Custom(e.to_string()))? }; let maybe_push_op_code: u8 = reader.read()?; @@ -852,7 +852,7 @@ impl Deserializable for SlpTransaction { }; let bytes: Vec = reader.read_list()?; if bytes.len() != 8 { - return Err(Error::Custom(format!("Expected 8 bytes, got {}", bytes.len()))); + return Err(SerError::Custom(format!("Expected 8 bytes, got {}", bytes.len()))); } let initial_token_mint_quantity = u64::from_be_bytes(bytes.try_into().expect("length is 8 bytes")); @@ -869,7 +869,10 @@ impl Deserializable for SlpTransaction { SLP_MINT => { let maybe_id: Vec = reader.read_list()?; if maybe_id.len() != 32 { - return Err(Error::Custom(format!("Unexpected token id length {}", maybe_id.len()))); + return Err(SerError::Custom(format!( + "Unexpected token id length {}", + maybe_id.len() + ))); } let maybe_push_op_code: u8 = reader.read()?; @@ -882,7 +885,7 @@ impl Deserializable for SlpTransaction { let bytes: Vec = reader.read_list()?; if bytes.len() != 8 { - return Err(Error::Custom(format!("Expected 8 bytes, got {}", bytes.len()))); + return Err(SerError::Custom(format!("Expected 8 bytes, got {}", bytes.len()))); } let additional_token_quantity = u64::from_be_bytes(bytes.try_into().expect("length is 8 bytes")); @@ -895,7 +898,10 @@ impl Deserializable for SlpTransaction { SLP_SEND => { let maybe_id: Vec = reader.read_list()?; if maybe_id.len() != 32 { - return Err(Error::Custom(format!("Unexpected token id length {}", maybe_id.len()))); + return Err(SerError::Custom(format!( + "Unexpected token id length {}", + maybe_id.len() + ))); } let token_id = H256::from(maybe_id.as_slice()); @@ -903,21 +909,21 @@ impl Deserializable for SlpTransaction { while !reader.is_finished() { let bytes: Vec = reader.read_list()?; if bytes.len() != 8 { - return Err(Error::Custom(format!("Expected 8 bytes, got {}", bytes.len()))); + return Err(SerError::Custom(format!("Expected 8 bytes, got {}", bytes.len()))); } let amount = u64::from_be_bytes(bytes.try_into().expect("length is 8 bytes")); amounts.push(amount) } if amounts.len() > 19 { - return Err(Error::Custom(format!( + return Err(SerError::Custom(format!( "Expected at most 19 token amounts, got {}", amounts.len() ))); } Ok(SlpTransaction::Send { token_id, amounts }) }, - _ => Err(Error::Custom(format!( + _ => Err(SerError::Custom(format!( "Unsupported transaction type {}", transaction_type ))), @@ -940,11 +946,11 @@ pub enum ParseSlpScriptError { #[display(fmt = "UnexpectedTokenType: {:?}", _0)] UnexpectedTokenType(Vec), #[display(fmt = "DeserializeFailed: {:?}", _0)] - DeserializeFailed(Error), + DeserializeFailed(SerError), } -impl From for ParseSlpScriptError { - fn from(err: Error) -> ParseSlpScriptError { ParseSlpScriptError::DeserializeFailed(err) } +impl From for ParseSlpScriptError { + fn from(err: SerError) -> ParseSlpScriptError { ParseSlpScriptError::DeserializeFailed(err) } } pub fn parse_slp_script(script: &[u8]) -> Result> { diff --git a/mm2src/coins/utxo/spv.rs b/mm2src/coins/utxo/spv.rs new file mode 100644 index 0000000000..6d85d0afbe --- /dev/null +++ b/mm2src/coins/utxo/spv.rs @@ -0,0 +1,91 @@ +use crate::utxo::rpc_clients::ElectrumClient; +use async_trait::async_trait; +use chain::{BlockHeader, RawBlockHeader, Transaction as UtxoTx}; +use common::executor::Timer; +use common::log::error; +use common::now_ms; +use keys::hash::H256; +use mm2_err_handle::prelude::*; +use serialization::serialize_list; +use spv_validation::helpers_validation::SPVError; +use spv_validation::spv_proof::{SPVProof, TRY_SPV_PROOF_INTERVAL}; + +pub struct ConfirmedTransactionInfo { + pub tx: UtxoTx, + pub header: BlockHeader, + pub index: u64, + pub height: u64, +} + +#[async_trait] +pub trait SimplePaymentVerification { + async fn validate_spv_proof( + &self, + tx: &UtxoTx, + try_spv_proof_until: u64, + ) -> Result>; +} + +#[async_trait] +impl SimplePaymentVerification for ElectrumClient { + async fn validate_spv_proof( + &self, + tx: &UtxoTx, + try_spv_proof_until: u64, + ) -> Result> { + if tx.outputs.is_empty() { + return MmError::err(SPVError::InvalidVout); + } + + let (merkle_branch, header, height) = loop { + if now_ms() / 1000 > try_spv_proof_until { + error!( + "Waited too long until {} for transaction {:?} to validate spv proof", + try_spv_proof_until, + tx.hash().reversed(), + ); + return MmError::err(SPVError::Timeout); + } + + match self.get_merkle_and_header(tx).await { + Ok(res) => break res, + Err(e) => { + error!( + "Failed spv proof validation for transaction {} with error: {:?}, retrying in {} seconds.", + tx.hash().reversed(), + e, + TRY_SPV_PROOF_INTERVAL, + ); + + Timer::sleep(TRY_SPV_PROOF_INTERVAL as f64).await; + }, + } + }; + + let raw_header = RawBlockHeader::new(header.raw().take())?; + let intermediate_nodes: Vec = merkle_branch + .merkle + .into_iter() + .map(|hash| hash.reversed().into()) + .collect(); + + let proof = SPVProof { + tx_id: tx.hash(), + vin: serialize_list(&tx.inputs).take(), + vout: serialize_list(&tx.outputs).take(), + index: merkle_branch.pos as u64, + confirming_header: header.clone(), + raw_header, + intermediate_nodes, + }; + + proof.validate().map_err(MmError::new)?; + + Ok(ConfirmedTransactionInfo { + tx: tx.clone(), + header, + index: proof.index, + height, + }) + } +} diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index 925a0c80c3..32f53dd89f 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -1,40 +1,29 @@ -use crate::utxo::rpc_clients::ElectrumBlockHeader; #[cfg(target_arch = "wasm32")] use crate::utxo::utxo_indexedb_block_header_storage::IndexedDBBlockHeadersStorage; #[cfg(not(target_arch = "wasm32"))] use crate::utxo::utxo_sql_block_header_storage::SqliteBlockHeadersStorage; -use crate::utxo::UtxoBlockHeaderVerificationParams; use async_trait::async_trait; use chain::BlockHeader; -use derive_more::Display; use mm2_core::mm_ctx::MmArc; -use mm2_err_handle::prelude::*; +use primitives::hash::H256; +use spv_validation::storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; use std::collections::HashMap; use std::fmt::{Debug, Formatter}; - -#[derive(Debug, Display)] -pub enum BlockHeaderStorageError { - #[display(fmt = "Can't add to the storage for {} - reason: {}", ticker, reason)] - AddToStorageError { ticker: String, reason: String }, - #[display(fmt = "Can't get from the storage for {} - reason: {}", ticker, reason)] - GetFromStorageError { ticker: String, reason: String }, - #[display( - fmt = "Can't retrieve the table from the storage for {} - reason: {}", - ticker, - reason - )] - CantRetrieveTableError { ticker: String, reason: String }, - #[display(fmt = "Can't query from the storage - query: {} - reason: {}", query, reason)] - QueryError { query: String, reason: String }, - #[display(fmt = "Can't init from the storage - ticker: {} - reason: {}", ticker, reason)] - InitializationError { ticker: String, reason: String }, - #[display(fmt = "Can't decode/deserialize from storage for {} - reason: {}", ticker, reason)] - DecodeError { ticker: String, reason: String }, +use std::num::NonZeroU64; + +/// SPV headers verification parameters +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct BlockHeaderVerificationParams { + pub difficulty_check: bool, + pub constant_difficulty: bool, + // This should to be equal to or greater than the number of blocks needed before the chain is safe from reorganization (e.g. 6 blocks for BTC) + pub blocks_limit_to_check: NonZeroU64, + pub check_every: f64, } pub struct BlockHeaderStorage { pub inner: Box, - pub params: UtxoBlockHeaderVerificationParams, + pub params: BlockHeaderVerificationParams, } impl Debug for BlockHeaderStorage { @@ -42,63 +31,29 @@ impl Debug for BlockHeaderStorage { } pub trait InitBlockHeaderStorageOps: Send + Sync + 'static { - fn new_from_ctx(ctx: MmArc, params: UtxoBlockHeaderVerificationParams) -> Option + fn new_from_ctx( + ctx: MmArc, + params: BlockHeaderVerificationParams, + ) -> Result where Self: Sized; } -#[async_trait] -pub trait BlockHeaderStorageOps: Send + Sync + 'static { - /// Initializes collection/tables in storage for a specified coin - async fn init(&self, for_coin: &str) -> Result<(), MmError>; - - async fn is_initialized_for(&self, for_coin: &str) -> Result>; - - // Adds multiple block headers to the selected coin's header storage - // Should store it as `TICKER_HEIGHT=hex_string` - // use this function for headers that comes from `blockchain_headers_subscribe` - async fn add_electrum_block_headers_to_storage( - &self, - for_coin: &str, - headers: Vec, - ) -> Result<(), MmError>; - - // Adds multiple block headers to the selected coin's header storage - // Should store it as `TICKER_HEIGHT=hex_string` - // use this function for headers that comes from `blockchain_block_headers` - async fn add_block_headers_to_storage( - &self, - for_coin: &str, - headers: HashMap, - ) -> Result<(), MmError>; - - /// Gets the block header by height from the selected coin's storage as BlockHeader - async fn get_block_header( - &self, - for_coin: &str, - height: u64, - ) -> Result, MmError>; - - /// Gets the block header by height from the selected coin's storage as hex - async fn get_block_header_raw( - &self, - for_coin: &str, - height: u64, - ) -> Result, MmError>; -} - impl InitBlockHeaderStorageOps for BlockHeaderStorage { #[cfg(not(target_arch = "wasm32"))] - fn new_from_ctx(ctx: MmArc, params: UtxoBlockHeaderVerificationParams) -> Option { - ctx.sqlite_connection.as_option().map(|connection| BlockHeaderStorage { - inner: Box::new(SqliteBlockHeadersStorage(connection.clone())), + fn new_from_ctx(ctx: MmArc, params: BlockHeaderVerificationParams) -> Result { + let sqlite_connection = ctx.sqlite_connection.ok_or(BlockHeaderStorageError::Internal( + "sqlite_connection is not initialized".to_owned(), + ))?; + Ok(BlockHeaderStorage { + inner: Box::new(SqliteBlockHeadersStorage(sqlite_connection.clone())), params, }) } #[cfg(target_arch = "wasm32")] - fn new_from_ctx(_ctx: MmArc, params: UtxoBlockHeaderVerificationParams) -> Option { - Some(BlockHeaderStorage { + fn new_from_ctx(_ctx: MmArc, params: BlockHeaderVerificationParams) -> Result { + Ok(BlockHeaderStorage { inner: Box::new(IndexedDBBlockHeadersStorage {}), params, }) @@ -107,29 +62,17 @@ impl InitBlockHeaderStorageOps for BlockHeaderStorage { #[async_trait] impl BlockHeaderStorageOps for BlockHeaderStorage { - async fn init(&self, for_coin: &str) -> Result<(), MmError> { - self.inner.init(for_coin).await - } + async fn init(&self, for_coin: &str) -> Result<(), BlockHeaderStorageError> { self.inner.init(for_coin).await } - async fn is_initialized_for(&self, for_coin: &str) -> Result> { + async fn is_initialized_for(&self, for_coin: &str) -> Result { self.inner.is_initialized_for(for_coin).await } - async fn add_electrum_block_headers_to_storage( - &self, - for_coin: &str, - headers: Vec, - ) -> Result<(), MmError> { - self.inner - .add_electrum_block_headers_to_storage(for_coin, headers) - .await - } - async fn add_block_headers_to_storage( &self, for_coin: &str, headers: HashMap, - ) -> Result<(), MmError> { + ) -> Result<(), BlockHeaderStorageError> { self.inner.add_block_headers_to_storage(for_coin, headers).await } @@ -137,7 +80,7 @@ impl BlockHeaderStorageOps for BlockHeaderStorage { &self, for_coin: &str, height: u64, - ) -> Result, MmError> { + ) -> Result, BlockHeaderStorageError> { self.inner.get_block_header(for_coin, height).await } @@ -145,7 +88,22 @@ impl BlockHeaderStorageOps for BlockHeaderStorage { &self, for_coin: &str, height: u64, - ) -> Result, MmError> { + ) -> Result, BlockHeaderStorageError> { self.inner.get_block_header_raw(for_coin, height).await } + + async fn get_last_block_header_with_non_max_bits( + &self, + for_coin: &str, + ) -> Result, BlockHeaderStorageError> { + self.inner.get_last_block_header_with_non_max_bits(for_coin).await + } + + async fn get_block_height_by_hash( + &self, + for_coin: &str, + hash: H256, + ) -> Result, BlockHeaderStorageError> { + self.inner.get_block_height_by_hash(for_coin, hash).await + } } diff --git a/mm2src/coins/utxo/utxo_builder/mod.rs b/mm2src/coins/utxo/utxo_builder/mod.rs index 9c1cf135d0..cd48444513 100644 --- a/mm2src/coins/utxo/utxo_builder/mod.rs +++ b/mm2src/coins/utxo/utxo_builder/mod.rs @@ -2,7 +2,7 @@ mod utxo_arc_builder; mod utxo_coin_builder; mod utxo_conf_builder; -pub use utxo_arc_builder::{MergeUtxoArcOps, UtxoArcBuilder}; +pub use utxo_arc_builder::{BlockHeaderUtxoArcOps, MergeUtxoArcOps, UtxoArcBuilder}; pub use utxo_coin_builder::{UtxoCoinBuildError, UtxoCoinBuildResult, UtxoCoinBuilder, UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPrivKeyBuilder, UtxoFieldsWithHardwareWalletBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index d4964b12c4..2a90bc036f 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -1,4 +1,4 @@ -use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; +use crate::utxo::rpc_clients::UtxoRpcClientEnum; use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuilder, UtxoCoinBuilderCommonOps, UtxoFieldsWithHardwareWalletBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::{block_header_utxo_loop, merge_utxo_loop}; @@ -84,16 +84,19 @@ where async fn build(self) -> MmResult { let utxo = self.build_utxo_fields().await?; + let rpc_client = utxo.rpc_client.clone(); let utxo_arc = UtxoArc::new(utxo); let utxo_weak = utxo_arc.downgrade(); let result_coin = (self.constructor)(utxo_arc); - self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()); - if let Some(abort_handler) = self.spawn_block_header_utxo_loop_if_required( - utxo_weak, - &result_coin.as_ref().block_headers_storage, - self.constructor.clone(), - ) { + if let Some(abort_handler) = self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()) + { + self.ctx.abort_handlers.lock().unwrap().push(abort_handler); + } + + if let Some(abort_handler) = + self.spawn_block_header_utxo_loop_if_required(utxo_weak, &rpc_client, self.constructor.clone()) + { self.ctx.abort_handlers.lock().unwrap().push(abort_handler); } Ok(result_coin) @@ -115,21 +118,28 @@ where } pub trait MergeUtxoArcOps: UtxoCoinBuilderCommonOps { - fn spawn_merge_utxo_loop_if_required(&self, weak: UtxoWeak, constructor: F) + fn spawn_merge_utxo_loop_if_required(&self, weak: UtxoWeak, constructor: F) -> Option where F: Fn(UtxoArc) -> T + Send + Sync + 'static, { if let Some(ref merge_params) = self.activation_params().utxo_merge_params { - let fut = merge_utxo_loop( + let (fut, abort_handle) = abortable(merge_utxo_loop( weak, merge_params.merge_at, merge_params.check_every, merge_params.max_merge_at_once, constructor, - ); - info!("Starting UTXO merge loop for coin {}", self.ticker()); - spawn(fut); + )); + let ticker = self.ticker().to_owned(); + info!("Starting UTXO merge loop for coin {}", ticker); + spawn(async move { + if let Err(e) = fut.await { + info!("spawn_merge_utxo_loop_if_required stopped for {}, reason {}", ticker, e); + } + }); + return Some(abort_handle); } + None } } @@ -137,26 +147,28 @@ pub trait BlockHeaderUtxoArcOps: UtxoCoinBuilderCommonOps { fn spawn_block_header_utxo_loop_if_required( &self, weak: UtxoWeak, - maybe_storage: &Option, + rpc_client: &UtxoRpcClientEnum, constructor: F, ) -> Option where F: Fn(UtxoArc) -> T + Send + Sync + 'static, T: UtxoCommonOps, { - if maybe_storage.is_some() { - let ticker = self.ticker().to_owned(); - let (fut, abort_handle) = abortable(block_header_utxo_loop(weak, constructor)); - info!("Starting UTXO block header loop for coin {}", ticker); - spawn(async move { - if let Err(e) = fut.await { - info!( - "spawn_block_header_utxo_loop_if_required stopped for {}, reason {}", - ticker, e - ); - } - }); - return Some(abort_handle); + if let UtxoRpcClientEnum::Electrum(electrum) = rpc_client { + if electrum.block_headers_storage().is_some() { + let ticker = self.ticker().to_owned(); + let (fut, abort_handle) = abortable(block_header_utxo_loop(weak, constructor)); + info!("Starting UTXO block header loop for coin {}", ticker); + spawn(async move { + if let Err(e) = fut.await { + info!( + "spawn_block_header_utxo_loop_if_required stopped for {}, reason {}", + ticker, e + ); + } + }); + return Some(abort_handle); + } } None } diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index dac910db93..faced1c8f1 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -3,7 +3,8 @@ use crate::hd_wallet_storage::{HDWalletCoinStorage, HDWalletStorageError}; use crate::utxo::rpc_clients::{ElectrumClient, ElectrumClientImpl, ElectrumRpcRequest, EstimateFeeMethod, UtxoRpcClientEnum}; use crate::utxo::tx_cache::{UtxoVerboseCacheOps, UtxoVerboseCacheShared}; -use crate::utxo::utxo_block_header_storage::{BlockHeaderStorage, InitBlockHeaderStorageOps}; +use crate::utxo::utxo_block_header_storage::{BlockHeaderStorage, BlockHeaderVerificationParams, + InitBlockHeaderStorageOps}; use crate::utxo::utxo_builder::utxo_conf_builder::{UtxoConfBuilder, UtxoConfError, UtxoConfResult}; use crate::utxo::{output_script, utxo_common, ElectrumBuilderArgs, ElectrumProtoVerifier, RecentlySpentOutPoints, TxFee, UtxoCoinConf, UtxoCoinFields, UtxoHDAccount, UtxoHDWallet, UtxoRpcMode, DEFAULT_GAP_LIMIT, @@ -163,7 +164,6 @@ pub trait UtxoFieldsWithIguanaPrivKeyBuilder: UtxoCoinBuilderCommonOps { let tx_hash_algo = self.tx_hash_algo(); let check_utxo_maturity = self.check_utxo_maturity(); let tx_cache = self.tx_cache(); - let block_headers_storage = self.block_headers_storage()?; let coin = UtxoCoinFields { conf, @@ -174,7 +174,6 @@ pub trait UtxoFieldsWithIguanaPrivKeyBuilder: UtxoCoinBuilderCommonOps { derivation_method, history_sync_state: Mutex::new(initial_history_state), tx_cache, - block_headers_storage, recently_spent_outpoints: AsyncMutex::new(RecentlySpentOutPoints::new(my_script_pubkey)), tx_fee, tx_hash_algo, @@ -226,7 +225,6 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { let tx_hash_algo = self.tx_hash_algo(); let check_utxo_maturity = self.check_utxo_maturity(); let tx_cache = self.tx_cache(); - let block_headers_storage = self.block_headers_storage()?; let coin = UtxoCoinFields { conf, @@ -236,7 +234,6 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { priv_key_policy: PrivKeyPolicy::Trezor, derivation_method: DerivationMethod::HDWallet(hd_wallet), history_sync_state: Mutex::new(initial_history_state), - block_headers_storage, tx_cache, recently_spent_outpoints, tx_fee, @@ -289,15 +286,6 @@ pub trait UtxoCoinBuilderCommonOps { fn ticker(&self) -> &str; - fn block_headers_storage(&self) -> UtxoCoinBuildResult> { - let params: Option<_> = json::from_value(self.conf()["block_header_params"].clone()) - .map_to_mm(|e| UtxoConfError::InvalidBlockHeaderParams(e.to_string()))?; - match params { - None => Ok(None), - Some(params) => Ok(BlockHeaderStorage::new_from_ctx(self.ctx().clone(), params)), - } - } - fn address_format(&self) -> UtxoCoinBuildResult { let format_from_req = self.activation_params().address_format.clone(); let format_from_conf = json::from_value::>(self.conf()["address_format"].clone()) @@ -406,8 +394,13 @@ pub trait UtxoCoinBuilderCommonOps { Ok(UtxoRpcClientEnum::Native(native)) } }, - UtxoRpcMode::Electrum { servers } => { - let electrum = self.electrum_client(ElectrumBuilderArgs::default(), servers).await?; + UtxoRpcMode::Electrum { + servers, + block_header_params, + } => { + let electrum = self + .electrum_client(ElectrumBuilderArgs::default(), servers, block_header_params) + .await?; Ok(UtxoRpcClientEnum::Electrum(electrum)) }, } @@ -417,6 +410,7 @@ pub trait UtxoCoinBuilderCommonOps { &self, args: ElectrumBuilderArgs, mut servers: Vec, + block_header_params: Option, ) -> UtxoCoinBuildResult { let (on_connect_tx, on_connect_rx) = mpsc::unbounded(); let ticker = self.ticker().to_owned(); @@ -432,9 +426,17 @@ pub trait UtxoCoinBuilderCommonOps { event_handlers.push(ElectrumProtoVerifier { on_connect_tx }.into_shared()); } + let block_headers_storage = match block_header_params { + Some(params) => Some( + BlockHeaderStorage::new_from_ctx(self.ctx().clone(), params) + .map_to_mm(|e| UtxoCoinBuildError::Internal(e.to_string()))?, + ), + None => None, + }; + let mut rng = small_rng(); servers.as_mut_slice().shuffle(&mut rng); - let client = ElectrumClientImpl::new(ticker, event_handlers); + let client = ElectrumClientImpl::new(ticker, event_handlers, block_headers_storage); for server in servers.iter() { match client.add_server(server).await { Ok(_) => (), diff --git a/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs index 3cc89efe7b..db1512b38d 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs @@ -40,7 +40,6 @@ pub enum UtxoConfError { InvalidConsensusBranchId(String), InvalidVersionGroupId(String), InvalidAddressFormat(String), - InvalidBlockHeaderParams(String), InvalidDecimals(String), } diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index a67c9136c4..583ed7f73b 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -1,4 +1,3 @@ -use super::rpc_clients::TxMerkleBranch; use super::*; use crate::coin_balance::{AddressBalanceStatus, HDAddressBalance, HDWalletBalanceOps}; use crate::hd_pubkey::{ExtractExtendedPubkey, HDExtractPubkeyError, HDXPubExtractor}; @@ -8,6 +7,7 @@ use crate::hd_wallet_storage::{HDWalletCoinWithStorageOps, HDWalletStorageResult use crate::rpc_command::init_withdraw::WithdrawTaskHandle; use crate::utxo::rpc_clients::{electrum_script_hash, BlockHashOrHeight, UnspentInfo, UnspentMap, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcResult}; +use crate::utxo::spv::SimplePaymentVerification; use crate::utxo::tx_cache::TxCacheResult; use crate::utxo::utxo_withdraw::{InitUtxoWithdraw, StandardUtxoWithdraw, UtxoWithdraw}; use crate::{CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, GetWithdrawSenderAddress, HDAddressId, @@ -18,7 +18,7 @@ use crate::{CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, GetWithdrawSen use bitcrypto::dhash256; pub use bitcrypto::{dhash160, sha256, ChecksumType}; use chain::constants::SEQUENCE_FINAL; -use chain::{BlockHeader, OutPoint, RawBlockHeader, TransactionOutput}; +use chain::{OutPoint, TransactionOutput}; use common::executor::Timer; use common::jsonrpc_client::JsonRpcErrorType; use common::log::{debug, error, info, warn}; @@ -40,16 +40,14 @@ use rpc::v1::types::{Bytes as BytesJson, ToTxHash, TransactionInputEnum, H256 as use script::{Builder, Opcode, Script, ScriptAddress, TransactionInputSigner, UnsignedTransactionInput}; use secp256k1::{PublicKey, Signature}; use serde_json::{self as json}; -use serialization::{deserialize, serialize, serialize_list, serialize_with_flags, CoinVariant, CompactInteger, - Serializable, Stream, SERIALIZE_TRANSACTION_WITNESS}; +use serialization::{deserialize, serialize, serialize_with_flags, CoinVariant, CompactInteger, Serializable, Stream, + SERIALIZE_TRANSACTION_WITNESS}; use spv_validation::helpers_validation::validate_headers; -use spv_validation::helpers_validation::SPVError; -use spv_validation::spv_proof::{SPVProof, TRY_SPV_PROOF_INTERVAL}; +use spv_validation::storage::BlockHeaderStorageOps; use std::cmp::Ordering; use std::collections::hash_map::{Entry, HashMap}; use std::str::FromStr; use std::sync::atomic::Ordering as AtomicOrdering; -use utxo_block_header_storage::BlockHeaderStorageOps; use utxo_signer::with_key_pair::p2sh_spend; use utxo_signer::UtxoSignerOps; @@ -59,7 +57,6 @@ pub const DEFAULT_FEE_VOUT: usize = 0; pub const DEFAULT_SWAP_TX_SPEND_SIZE: u64 = 305; pub const DEFAULT_SWAP_VOUT: usize = 0; const MIN_BTC_TRADING_VOL: &str = "0.00777"; -pub const NO_TX_ERROR_CODE: &str = "'code': -5"; macro_rules! true_or { ($cond: expr, $etype: expr) => { @@ -3078,130 +3075,6 @@ pub fn address_from_pubkey( } } -pub async fn validate_spv_proof( - coin: T, - tx: UtxoTx, - try_spv_proof_until: u64, -) -> Result<(), MmError> { - let client = match &coin.as_ref().rpc_client { - UtxoRpcClientEnum::Native(_) => return Ok(()), - UtxoRpcClientEnum::Electrum(electrum_client) => electrum_client, - }; - if tx.outputs.is_empty() { - return MmError::err(SPVError::InvalidVout); - } - - let (merkle_branch, block_header) = spv_proof_retry_pool(&coin, client, &tx, try_spv_proof_until).await?; - let raw_header = RawBlockHeader::new(block_header.raw().take())?; - let intermediate_nodes: Vec = merkle_branch - .merkle - .into_iter() - .map(|hash| hash.reversed().into()) - .collect(); - - let proof = SPVProof { - tx_id: tx.hash(), - vin: serialize_list(&tx.inputs).take(), - vout: serialize_list(&tx.outputs).take(), - index: merkle_branch.pos as u64, - confirming_header: block_header, - raw_header, - intermediate_nodes, - }; - - proof.validate().map_err(MmError::new) -} - -async fn spv_proof_retry_pool( - coin: &T, - client: &ElectrumClient, - tx: &UtxoTx, - try_spv_proof_until: u64, -) -> Result<(TxMerkleBranch, BlockHeader), MmError> { - let mut height: Option = None; - let mut merkle_branch: Option = None; - - loop { - if now_ms() / 1000 > try_spv_proof_until { - error!( - "Waited too long until {} for transaction {:?} to validate spv proof", - try_spv_proof_until, - tx.hash(), - ); - return Err(SPVError::Timeout.into()); - } - - if height.is_none() { - match get_tx_height(tx, client).await { - Ok(h) => height = Some(h), - Err(e) => { - debug!("`get_tx_height` returned an error {:?}", e); - error!("{:?} for tx {:?}", SPVError::InvalidHeight, tx); - }, - } - } - - if height.is_some() && merkle_branch.is_none() { - match client - .blockchain_transaction_get_merkle(tx.hash().reversed().into(), height.unwrap()) - .compat() - .await - { - Ok(m) => merkle_branch = Some(m), - Err(e) => { - debug!("`blockchain_transaction_get_merkle` returned an error {:?}", e); - error!( - "{:?} by tx: {:?}, height: {}", - SPVError::UnableToGetMerkle, - H256Json::from(tx.hash().reversed()), - height.unwrap() - ); - }, - } - } - - if height.is_some() && merkle_branch.is_some() { - match block_header_from_storage_or_rpc(&coin, height.unwrap(), &coin.as_ref().block_headers_storage, client) - .await - { - Ok(block_header) => { - return Ok((merkle_branch.unwrap(), block_header)); - }, - Err(e) => { - debug!("`block_header_from_storage_or_rpc` returned an error {:?}", e); - error!( - "{:?}, Received header likely not compatible with header format in mm2", - SPVError::UnableToGetHeader - ); - }, - } - } - - error!( - "Failed spv proof validation for transaction {:?}, retrying in {} seconds.", - tx.hash(), - TRY_SPV_PROOF_INTERVAL, - ); - - Timer::sleep(TRY_SPV_PROOF_INTERVAL as f64).await; - } -} - -pub async fn get_tx_height(tx: &UtxoTx, client: &ElectrumClient) -> Result> { - for output in tx.outputs.clone() { - let script_pubkey_str = hex::encode(electrum_script_hash(&output.script_pubkey)); - if let Ok(history) = client.scripthash_get_history(script_pubkey_str.as_str()).compat().await { - if let Some(item) = history - .into_iter() - .find(|item| item.tx_hash.reversed() == H256Json(*tx.hash()) && item.height > 0) - { - return Ok(item.height as u64); - } - } - } - MmError::err(GetTxHeightError::HeightNotFound) -} - #[allow(clippy::too_many_arguments)] #[cfg_attr(test, mockable)] pub fn validate_payment( @@ -3268,16 +3141,16 @@ pub fn validate_payment( ); } - if !coin.as_ref().conf.enable_spv_proof { - return Ok(()); + if let UtxoRpcClientEnum::Electrum(client) = &coin.as_ref().rpc_client { + if coin.as_ref().conf.enable_spv_proof && confirmations != 0 { + client + .validate_spv_proof(&tx, try_spv_proof_until) + .await + .map_err(|e| format!("{:?}", e))?; + } } - return match confirmations { - 0 => Ok(()), - _ => validate_spv_proof(coin, tx, try_spv_proof_until) - .await - .map_err(|e| format!("{:?}", e)), - }; + return Ok(()); } }; Box::new(fut.boxed().compat()) @@ -3561,61 +3434,6 @@ fn increase_by_percent(num: u64, percent: f64) -> u64 { num + (percent.round() as u64) } -pub async fn valid_block_header_from_storage( - coin: &T, - height: u64, - storage: &BlockHeaderStorage, - client: &ElectrumClient, -) -> Result> -where - T: AsRef, -{ - match storage - .get_block_header(coin.as_ref().conf.ticker.as_str(), height) - .await? - { - None => { - let bytes = client.blockchain_block_header(height).compat().await?; - let header: BlockHeader = deserialize(bytes.0.as_slice())?; - let params = &storage.params; - let blocks_limit = params.blocks_limit_to_check; - let (headers_registry, headers) = client.retrieve_last_headers(blocks_limit, height).compat().await?; - match spv_validation::helpers_validation::validate_headers( - headers, - params.difficulty_check, - params.constant_difficulty, - ) { - Ok(_) => { - storage - .add_block_headers_to_storage(coin.as_ref().conf.ticker.as_str(), headers_registry) - .await?; - Ok(header) - }, - Err(err) => MmError::err(GetBlockHeaderError::SPVError(err)), - } - }, - Some(header) => Ok(header), - } -} - -#[inline] -pub async fn block_header_from_storage_or_rpc( - coin: &T, - height: u64, - storage: &Option, - client: &ElectrumClient, -) -> Result> -where - T: AsRef, -{ - match storage { - Some(ref storage) => valid_block_header_from_storage(&coin, height, storage, client).await, - None => Ok(deserialize( - client.blockchain_block_header(height).compat().await?.as_slice(), - )?), - } -} - pub async fn block_header_utxo_loop(weak: UtxoWeak, constructor: impl Fn(UtxoArc) -> T) { { let coin = match weak.upgrade() { @@ -3623,9 +3441,12 @@ pub async fn block_header_utxo_loop(weak: UtxoWeak, constructo None => return, }; let ticker = coin.as_ref().conf.ticker.as_str(); - let storage = match &coin.as_ref().block_headers_storage { - None => return, - Some(storage) => storage, + let storage = match &coin.as_ref().rpc_client { + UtxoRpcClientEnum::Native(_) => return, + UtxoRpcClientEnum::Electrum(e) => match e.block_headers_storage() { + None => return, + Some(storage) => storage, + }, }; match storage.is_initialized_for(ticker).await { Ok(true) => info!("Block Header Storage already initialized for {}", ticker), @@ -3644,8 +3465,12 @@ pub async fn block_header_utxo_loop(weak: UtxoWeak, constructo } while let Some(arc) = weak.upgrade() { let coin = constructor(arc); - let storage = match &coin.as_ref().block_headers_storage { - None => break, + let client = match &coin.as_ref().rpc_client { + UtxoRpcClientEnum::Native(_) => break, + UtxoRpcClientEnum::Electrum(client) => client, + }; + let storage = match client.block_headers_storage() { + None => return, Some(storage) => storage, }; let params = storage.params.clone(); @@ -3657,10 +3482,6 @@ pub async fn block_header_utxo_loop(weak: UtxoWeak, constructo ); let height = ok_or_continue_after_sleep!(coin.as_ref().rpc_client.get_block_count().compat().await, check_every); - let client = match &coin.as_ref().rpc_client { - UtxoRpcClientEnum::Native(_) => break, - UtxoRpcClientEnum::Electrum(client) => client, - }; let (block_registry, block_headers) = ok_or_continue_after_sleep!( client .retrieve_last_headers(blocks_limit_to_check, height) diff --git a/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs b/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs index d00d4f261f..e0e98658a4 100644 --- a/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs @@ -1,9 +1,7 @@ -use crate::utxo::rpc_clients::ElectrumBlockHeader; -use crate::utxo::utxo_block_header_storage::BlockHeaderStorageError; -use crate::utxo::utxo_block_header_storage::BlockHeaderStorageOps; use async_trait::async_trait; use chain::BlockHeader; -use mm2_err_handle::prelude::*; +use primitives::hash::H256; +use spv_validation::storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; use std::collections::HashMap; #[derive(Debug)] @@ -11,23 +9,15 @@ pub struct IndexedDBBlockHeadersStorage {} #[async_trait] impl BlockHeaderStorageOps for IndexedDBBlockHeadersStorage { - async fn init(&self, _for_coin: &str) -> Result<(), MmError> { Ok(()) } + async fn init(&self, _for_coin: &str) -> Result<(), BlockHeaderStorageError> { Ok(()) } - async fn is_initialized_for(&self, _for_coin: &str) -> Result> { Ok(true) } - - async fn add_electrum_block_headers_to_storage( - &self, - _for_coin: &str, - _headers: Vec, - ) -> Result<(), MmError> { - Ok(()) - } + async fn is_initialized_for(&self, _for_coin: &str) -> Result { Ok(true) } async fn add_block_headers_to_storage( &self, _for_coin: &str, _headers: HashMap, - ) -> Result<(), MmError> { + ) -> Result<(), BlockHeaderStorageError> { Ok(()) } @@ -35,7 +25,7 @@ impl BlockHeaderStorageOps for IndexedDBBlockHeadersStorage { &self, _for_coin: &str, _height: u64, - ) -> Result, MmError> { + ) -> Result, BlockHeaderStorageError> { Ok(None) } @@ -43,7 +33,22 @@ impl BlockHeaderStorageOps for IndexedDBBlockHeadersStorage { &self, _for_coin: &str, _height: u64, - ) -> Result, MmError> { + ) -> Result, BlockHeaderStorageError> { + Ok(None) + } + + async fn get_last_block_header_with_non_max_bits( + &self, + _for_coin: &str, + ) -> Result, BlockHeaderStorageError> { + Ok(None) + } + + async fn get_block_height_by_hash( + &self, + for_coin: &str, + hash: H256, + ) -> Result, BlockHeaderStorageError> { Ok(None) } } diff --git a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs index d11b794e54..c73d0c40de 100644 --- a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs @@ -1,5 +1,3 @@ -use crate::utxo::rpc_clients::ElectrumBlockHeader; -use crate::utxo::utxo_block_header_storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; use async_trait::async_trait; use chain::BlockHeader; use common::async_blocking; @@ -8,28 +6,32 @@ use db_common::{sqlite::rusqlite::Error as SqlError, sqlite::string_from_row, sqlite::validate_table_name, sqlite::CHECK_TABLE_EXISTS_SQL}; -use mm2_err_handle::prelude::*; -use serialization::deserialize; +use primitives::hash::H256; +use spv_validation::storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; +use spv_validation::work::MAX_BITS_BTC; use std::collections::HashMap; +use std::convert::TryInto; use std::sync::{Arc, Mutex}; fn block_headers_cache_table(ticker: &str) -> String { ticker.to_owned() + "_block_headers_cache" } -fn get_table_name_and_validate(for_coin: &str) -> Result> { +fn get_table_name_and_validate(for_coin: &str) -> Result { let table_name = block_headers_cache_table(for_coin); validate_table_name(&table_name).map_err(|e| BlockHeaderStorageError::CantRetrieveTableError { - ticker: for_coin.to_string(), + coin: for_coin.to_string(), reason: e.to_string(), })?; Ok(table_name) } -fn create_block_header_cache_table_sql(for_coin: &str) -> Result> { +fn create_block_header_cache_table_sql(for_coin: &str) -> Result { let table_name = get_table_name_and_validate(for_coin)?; let sql = format!( "CREATE TABLE IF NOT EXISTS {} ( block_height INTEGER NOT NULL UNIQUE, - hex TEXT NOT NULL + hex TEXT NOT NULL, + block_bits INTEGER NOT NULL, + block_hash VARCHAR(255) NOT NULL UNIQUE );", table_name ); @@ -37,23 +39,40 @@ fn create_block_header_cache_table_sql(for_coin: &str) -> Result Result> { +fn insert_block_header_in_cache_sql(for_coin: &str) -> Result { let table_name = get_table_name_and_validate(for_coin)?; // Always update the block headers with new values just in case a chain reorganization occurs. let sql = format!( - "INSERT OR REPLACE INTO {} (block_height, hex) VALUES (?1, ?2);", + "INSERT OR REPLACE INTO {} (block_height, hex, block_bits, block_hash) VALUES (?1, ?2, ?3, ?4);", table_name ); Ok(sql) } -fn get_block_header_by_height(for_coin: &str) -> Result> { +fn get_block_header_by_height(for_coin: &str) -> Result { let table_name = get_table_name_and_validate(for_coin)?; let sql = format!("SELECT hex FROM {} WHERE block_height=?1;", table_name); Ok(sql) } +fn get_last_block_header_with_non_max_bits_sql(for_coin: &str) -> Result { + let table_name = get_table_name_and_validate(for_coin)?; + let sql = format!( + "SELECT hex FROM {} WHERE block_bits<>{} ORDER BY block_height DESC LIMIT 1;", + table_name, MAX_BITS_BTC + ); + + Ok(sql) +} + +fn get_block_height_by_hash(for_coin: &str) -> Result { + let table_name = get_table_name_and_validate(for_coin)?; + let sql = format!("SELECT block_height FROM {} WHERE block_hash=?1;", table_name); + + Ok(sql) +} + #[derive(Clone, Debug)] pub struct SqliteBlockHeadersStorage(pub Arc>); @@ -62,89 +81,29 @@ fn query_single_row( query: &str, params: P, map_fn: F, -) -> Result, MmError> +) -> Result, BlockHeaderStorageError> where P: IntoIterator, P::Item: ToSql, F: FnOnce(&Row<'_>) -> Result, { - db_common::sqlite::query_single_row(conn, query, params, map_fn).map_err(|e| { - MmError::new(BlockHeaderStorageError::QueryError { - query: query.to_string(), - reason: e.to_string(), - }) + db_common::sqlite::query_single_row(conn, query, params, map_fn).map_err(|e| BlockHeaderStorageError::QueryError { + query: query.to_string(), + reason: e.to_string(), }) } -struct SqlBlockHeader { - block_height: String, - block_hex: String, -} - -impl From for SqlBlockHeader { - fn from(header: ElectrumBlockHeader) -> Self { - match header { - ElectrumBlockHeader::V12(h) => { - let block_hex = h.as_hex(); - let block_height = h.block_height.to_string(); - SqlBlockHeader { - block_height, - block_hex, - } - }, - ElectrumBlockHeader::V14(h) => { - let block_hex = format!("{:02x}", h.hex); - let block_height = h.height.to_string(); - SqlBlockHeader { - block_height, - block_hex, - } - }, - } - } -} -async fn common_headers_insert( - for_coin: &str, - storage: SqliteBlockHeadersStorage, - headers: Vec, -) -> Result<(), MmError> { - let for_coin = for_coin.to_owned(); - let mut conn = storage.0.lock().unwrap(); - let sql_transaction = conn - .transaction() - .map_err(|e| BlockHeaderStorageError::AddToStorageError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - for header in headers { - let block_cache_params = [&header.block_height, &header.block_hex]; - sql_transaction - .execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params) - .map_err(|e| BlockHeaderStorageError::AddToStorageError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - } - sql_transaction - .commit() - .map_err(|e| BlockHeaderStorageError::AddToStorageError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - Ok(()) -} - #[async_trait] impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { - async fn init(&self, for_coin: &str) -> Result<(), MmError> { + async fn init(&self, for_coin: &str) -> Result<(), BlockHeaderStorageError> { let selfi = self.clone(); let sql_cache = create_block_header_cache_table_sql(for_coin)?; - let ticker = for_coin.to_owned(); + let coin = for_coin.to_owned(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); conn.execute(&sql_cache, NO_PARAMS).map(|_| ()).map_err(|e| { BlockHeaderStorageError::InitializationError { - ticker, + coin, reason: e.to_string(), } })?; @@ -153,7 +112,7 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { .await } - async fn is_initialized_for(&self, for_coin: &str) -> Result> { + async fn is_initialized_for(&self, for_coin: &str) -> Result { let block_headers_cache_table = get_table_name_and_validate(for_coin)?; let selfi = self.clone(); async_blocking(move || { @@ -169,45 +128,64 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { .await } - async fn add_electrum_block_headers_to_storage( - &self, - for_coin: &str, - headers: Vec, - ) -> Result<(), MmError> { - let headers_for_sql = headers.into_iter().map(Into::into).collect(); - common_headers_insert(for_coin, self.clone(), headers_for_sql).await - } - async fn add_block_headers_to_storage( &self, for_coin: &str, headers: HashMap, - ) -> Result<(), MmError> { - let headers_for_sql = headers - .into_iter() - .map(|(height, header)| SqlBlockHeader { - block_height: height.to_string(), - block_hex: hex::encode(header.raw()), - }) - .collect(); - common_headers_insert(for_coin, self.clone(), headers_for_sql).await + ) -> Result<(), BlockHeaderStorageError> { + let for_coin = for_coin.to_owned(); + let selfi = self.clone(); + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let sql_transaction = conn + .transaction() + .map_err(|e| BlockHeaderStorageError::AddToStorageError { + coin: for_coin.to_string(), + reason: e.to_string(), + })?; + + for (height, header) in headers { + let height = height as i64; + let hash = header.hash().reversed().to_string(); + let raw_header = hex::encode(header.raw()); + let bits: u32 = header.bits.into(); + let block_cache_params = [ + &height as &dyn ToSql, + &raw_header as &dyn ToSql, + &bits as &dyn ToSql, + &hash as &dyn ToSql, + ]; + sql_transaction + .execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params) + .map_err(|e| BlockHeaderStorageError::AddToStorageError { + coin: for_coin.to_string(), + reason: e.to_string(), + })?; + } + sql_transaction + .commit() + .map_err(|e| BlockHeaderStorageError::AddToStorageError { + coin: for_coin.to_string(), + reason: e.to_string(), + })?; + Ok(()) + }) + .await } async fn get_block_header( &self, for_coin: &str, height: u64, - ) -> Result, MmError> { + ) -> Result, BlockHeaderStorageError> { if let Some(header_raw) = self.get_block_header_raw(for_coin, height).await? { - let header_bytes = hex::decode(header_raw).map_err(|e| BlockHeaderStorageError::DecodeError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; let header: BlockHeader = - deserialize(header_bytes.as_slice()).map_err(|e| BlockHeaderStorageError::DecodeError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; + header_raw + .try_into() + .map_err(|e: serialization::Error| BlockHeaderStorageError::DecodeError { + coin: for_coin.to_string(), + reason: e.to_string(), + })?; return Ok(Some(header)); } Ok(None) @@ -217,8 +195,8 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { &self, for_coin: &str, height: u64, - ) -> Result, MmError> { - let params = [height.to_string()]; + ) -> Result, BlockHeaderStorageError> { + let params = [height as i64]; let sql = get_block_header_by_height(for_coin)?; let selfi = self.clone(); @@ -227,11 +205,59 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { query_single_row(&conn, &sql, params, string_from_row) }) .await - .map_err(|e| { - MmError::new(BlockHeaderStorageError::GetFromStorageError { - ticker: for_coin.to_string(), - reason: e.into_inner().to_string(), - }) + .map_err(|e| BlockHeaderStorageError::GetFromStorageError { + coin: for_coin.to_string(), + reason: e.to_string(), + }) + } + + async fn get_last_block_header_with_non_max_bits( + &self, + for_coin: &str, + ) -> Result, BlockHeaderStorageError> { + let sql = get_last_block_header_with_non_max_bits_sql(for_coin)?; + let selfi = self.clone(); + + let maybe_header_raw = async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + query_single_row(&conn, &sql, NO_PARAMS, string_from_row) + }) + .await + .map_err(|e| BlockHeaderStorageError::GetFromStorageError { + coin: for_coin.to_string(), + reason: e.to_string(), + })?; + + if let Some(header_raw) = maybe_header_raw { + let header: BlockHeader = + header_raw + .try_into() + .map_err(|e: serialization::Error| BlockHeaderStorageError::DecodeError { + coin: for_coin.to_string(), + reason: e.to_string(), + })?; + return Ok(Some(header)); + } + Ok(None) + } + + async fn get_block_height_by_hash( + &self, + for_coin: &str, + hash: H256, + ) -> Result, BlockHeaderStorageError> { + let params = [hash.to_string()]; + let sql = get_block_height_by_hash(for_coin)?; + let selfi = self.clone(); + + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + query_single_row(&conn, &sql, params, |row| row.get(0)) + }) + .await + .map_err(|e| BlockHeaderStorageError::GetFromStorageError { + coin: for_coin.to_string(), + reason: e.to_string(), }) } } @@ -254,7 +280,7 @@ impl SqliteBlockHeadersStorage { #[cfg(test)] mod sql_block_headers_storage_tests { use super::*; - use crate::utxo::rpc_clients::ElectrumBlockHeaderV14; + use chain::BlockHeaderBits; use common::block_on; use primitives::hash::H256; @@ -283,12 +309,10 @@ mod sql_block_headers_storage_tests { let initialized = block_on(storage.is_initialized_for(for_coin)).unwrap(); assert!(initialized); - let block_header = ElectrumBlockHeaderV14 { - height: 520481, - hex: "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".into(), - }.into(); - let headers = vec![ElectrumBlockHeader::V14(block_header)]; - block_on(storage.add_electrum_block_headers_to_storage(for_coin, headers)).unwrap(); + let mut headers = HashMap::with_capacity(1); + let block_header: BlockHeader = "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".into(); + headers.insert(520481, block_header); + block_on(storage.add_block_headers_to_storage(for_coin, headers)).unwrap(); assert!(!storage.is_table_empty(&table)); } @@ -302,12 +326,11 @@ mod sql_block_headers_storage_tests { let initialized = block_on(storage.is_initialized_for(for_coin)).unwrap(); assert!(initialized); - let block_header = ElectrumBlockHeaderV14 { - height: 520481, - hex: "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".into(), - }.into(); - let headers = vec![ElectrumBlockHeader::V14(block_header)]; - block_on(storage.add_electrum_block_headers_to_storage(for_coin, headers)).unwrap(); + let mut headers = HashMap::with_capacity(1); + let block_header: BlockHeader = "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".into(); + headers.insert(520481, block_header); + + block_on(storage.add_block_headers_to_storage(for_coin, headers)).unwrap(); assert!(!storage.is_table_empty(&table)); let hex = block_on(storage.get_block_header_raw(for_coin, 520481)) @@ -316,9 +339,48 @@ mod sql_block_headers_storage_tests { assert_eq!(hex, "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".to_string()); let block_header = block_on(storage.get_block_header(for_coin, 520481)).unwrap().unwrap(); - assert_eq!( - block_header.hash(), - H256::from_reversed_str("0000000000000000002e31d0714a5ab23100945ff87ba2d856cd566a3c9344ec") - ) + let block_hash: H256 = "0000000000000000002e31d0714a5ab23100945ff87ba2d856cd566a3c9344ec".into(); + assert_eq!(block_header.hash(), block_hash.reversed()); + + let height = block_on(storage.get_block_height_by_hash(for_coin, block_hash)) + .unwrap() + .unwrap(); + assert_eq!(height, 520481); + } + + #[test] + fn test_get_last_block_header_with_non_max_bits() { + let for_coin = "get"; + let storage = SqliteBlockHeadersStorage::in_memory(); + let table = block_headers_cache_table(for_coin); + block_on(storage.init(for_coin)).unwrap(); + + let initialized = block_on(storage.is_initialized_for(for_coin)).unwrap(); + assert!(initialized); + + let mut headers = HashMap::with_capacity(2); + + // This block has max difficulty + // https://live.blockcypher.com/btc-testnet/block/00000000961a9d117feb57e516e17217207a849bf6cdfce529f31d9a96053530/ + let block_header: BlockHeader = "02000000ea01a61a2d7420a1b23875e40eb5eb4ca18b378902c8e6384514ad0000000000c0c5a1ae80582b3fe319d8543307fa67befc2a734b8eddb84b1780dfdf11fa2b20e71353ffff001d00805fe0".into(); + headers.insert(201595, block_header); + + // https://live.blockcypher.com/btc-testnet/block/0000000000ad144538e6c80289378ba14cebb50ee47538b2a120742d1aa601ea/ + let expected_block_header: BlockHeader = "02000000cbed7fd98f1f06e85c47e13ff956533642056be45e7e6b532d4d768f00000000f2680982f333fcc9afa7f9a5e2a84dc54b7fe10605cd187362980b3aa882e9683be21353ab80011c813e1fc0".into(); + headers.insert(201594, expected_block_header.clone()); + + // This block has max difficulty + // https://live.blockcypher.com/btc-testnet/block/0000000000ad144538e6c80289378ba14cebb50ee47538b2a120742d1aa601ea/ + let block_header: BlockHeader = "020000001f38c8e30b30af912fbd4c3e781506713cfb43e73dff6250348e060000000000afa8f3eede276ccb4c4ee649ad9823fc181632f262848ca330733e7e7e541beb9be51353ffff001d00a63037".into(); + headers.insert(201593, block_header); + + block_on(storage.add_block_headers_to_storage(for_coin, headers)).unwrap(); + assert!(!storage.is_table_empty(&table)); + + let actual_block_header = block_on(storage.get_last_block_header_with_non_max_bits(for_coin)) + .unwrap() + .unwrap(); + assert_ne!(actual_block_header.bits, BlockHeaderBits::Compact(MAX_BITS_BTC.into())); + assert_eq!(actual_block_header, expected_block_header); } } diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 00f74b03ce..56dc7a9015 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -10,6 +10,7 @@ use crate::utxo::rpc_clients::{BlockHashOrHeight, ElectrumBalance, ElectrumClien GetAddressInfoRes, ListSinceBlockRes, ListTransactionsItem, NativeClient, NativeClientImpl, NativeUnspent, NetworkInfo, UtxoRpcClientOps, ValidateAddressRes, VerboseBlock}; +use crate::utxo::spv::SimplePaymentVerification; use crate::utxo::tx_cache::dummy_tx_cache::DummyVerboseCache; use crate::utxo::tx_cache::UtxoVerboseCacheOps; use crate::utxo::utxo_builder::{UtxoArcBuilder, UtxoCoinBuilderCommonOps}; @@ -69,7 +70,7 @@ pub fn electrum_client_for_test(servers: &[&str]) -> ElectrumClient { }; let servers = servers.into_iter().map(|s| json::from_value(s).unwrap()).collect(); - block_on(builder.electrum_client(args, servers)).unwrap() + block_on(builder.electrum_client(args, servers, None)).unwrap() } /// Returned client won't work by default, requires some mocks to be usable @@ -165,7 +166,6 @@ fn utxo_coin_fields_for_test( derivation_method, history_sync_state: Mutex::new(HistorySyncState::NotEnabled), tx_cache: DummyVerboseCache::default().into_shared(), - block_headers_storage: None, recently_spent_outpoints: AsyncMutex::new(RecentlySpentOutPoints::new(my_script_pubkey)), tx_hash_algo: TxHashAlgo::DSHA256, check_utxo_maturity: false, @@ -469,7 +469,7 @@ fn test_wait_for_payment_spend_timeout_electrum() { MockResult::Return(Box::new(futures01::future::ok(None))) }); - let client = ElectrumClientImpl::new(TEST_COIN_NAME.into(), Default::default()); + let client = ElectrumClientImpl::new(TEST_COIN_NAME.into(), Default::default(), None); let client = UtxoRpcClientEnum::Electrum(ElectrumClient(Arc::new(client))); let coin = utxo_coin_for_test(client, None, false); let transaction = hex::decode("01000000000102fff7f7881a8099afa6940d42d1e7f6362bec38171ea3edf433541db4e4ad969f00000000494830450221008b9d1dc26ba6a9cb62127b02742fa9d754cd3bebf337f7a55d114c8e5cdd30be022040529b194ba3f9281a99f2b1c0a19c0489bc22ede944ccf4ecbab4cc618ef3ed01eeffffffef51e1b804cc89d182d279655c3aa89e815b1b309fe287d9b2b55d57b90ec68a0100000000ffffffff02202cb206000000001976a9148280b37df378db99f66f85c95a783a76ac7a6d5988ac9093510d000000001976a9143bde42dbee7e4dbe6a21b2d50ce2f0167faa815988ac000247304402203609e17b84f6a7d30c80bfa610b5b4542f32a8a0d5447a12fb1366d7f01cc44a0220573a954c4518331561406f90300e8f3358f51928d43c212a8caed02de67eebee0121025476c2e83188368da1ff3e292e7acafcdb3566bb0ad253f62fc70f07aeee635711000000") @@ -956,18 +956,13 @@ fn test_utxo_lock() { #[test] fn test_spv_proof() { let client = electrum_client_for_test(RICK_ELECTRUM_ADDRS); - let coin = utxo_coin_for_test( - client.into(), - Some("spice describe gravity federal blast come thank unfair canal monkey style afraid"), - false, - ); // https://rick.explorer.dexstats.info/tx/78ea7839f6d1b0dafda2ba7e34c1d8218676a58bd1b33f03a5f76391f61b72b0 let tx_str = "0400008085202f8902bf17bf7d1daace52e08f732a6b8771743ca4b1cb765a187e72fd091a0aabfd52000000006a47304402203eaaa3c4da101240f80f9c5e9de716a22b1ec6d66080de6a0cca32011cd77223022040d9082b6242d6acf9a1a8e658779e1c655d708379862f235e8ba7b8ca4e69c6012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffffff023ca13c0e9e085dd13f481f193e8a3e8fd609020936e98b5587342d994f4d020000006b483045022100c0ba56adb8de923975052312467347d83238bd8d480ce66e8b709a7997373994022048507bcac921fdb2302fa5224ce86e41b7efc1a2e20ae63aa738dfa99b7be826012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9141ee6d4c38a3c078eab87ad1a5e4b00f21259b10d870000000000000000166a1400000000000000000000000000000000000000001b94d736000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac2d08e35e000000000000000000000000000000"; let tx: UtxoTx = tx_str.into(); - let res = block_on(utxo_common::validate_spv_proof(coin.clone(), tx, now_ms() / 1000 + 30)); - res.unwrap() + let res = block_on(client.validate_spv_proof(&tx, now_ms() / 1000 + 30)); + res.unwrap(); } #[test] @@ -1473,11 +1468,12 @@ fn test_network_info_negative_time_offset() { #[test] fn test_unavailable_electrum_proto_version() { - ElectrumClientImpl::new.mock_safe(|coin_ticker, event_handlers| { + ElectrumClientImpl::new.mock_safe(|coin_ticker, event_handlers, _| { MockResult::Return(ElectrumClientImpl::with_protocol_version( coin_ticker, event_handlers, OrdRange::new(1.8, 1.9).unwrap(), + None, )) }); diff --git a/mm2src/coins/utxo/utxo_wasm_tests.rs b/mm2src/coins/utxo/utxo_wasm_tests.rs index 4eaaf01d79..20c057c01b 100644 --- a/mm2src/coins/utxo/utxo_wasm_tests.rs +++ b/mm2src/coins/utxo/utxo_wasm_tests.rs @@ -11,7 +11,7 @@ wasm_bindgen_test_configure!(run_in_browser); const TEST_COIN_NAME: &'static str = "RICK"; pub async fn electrum_client_for_test(servers: &[&str]) -> ElectrumClient { - let client = ElectrumClientImpl::new(TEST_COIN_NAME.into(), Default::default()); + let client = ElectrumClientImpl::new(TEST_COIN_NAME.into(), Default::default(), None); for server in servers { client .add_server(&ElectrumRpcRequest { diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index f9b68bbf4b..b799158d74 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -823,6 +823,8 @@ impl<'a> ZCoinBuilder<'a> { ZcoinRpcMode::Native => UtxoRpcMode::Native, ZcoinRpcMode::Light { electrum_servers, .. } => UtxoRpcMode::Electrum { servers: electrum_servers.clone(), + // TODO: Implement spv validation for zcoin + block_header_params: None, }, }; let utxo_params = UtxoActivationParams { diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 6a752bde42..404f229405 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -1,5 +1,5 @@ use super::{z_coin_errors::*, ZcoinConsensusParams}; -use crate::utxo::utxo_common; +use crate::utxo::rpc_clients; use common::executor::Timer; use common::log::{debug, error, info}; use common::{async_blocking, spawn_abortable, AbortOnDropHandle}; @@ -367,7 +367,7 @@ impl SaplingSyncLoopHandle { Ok(_) => break, Err(e) => { error!("Error on getting tx {}", tx_id); - if e.message().contains(utxo_common::NO_TX_ERROR_CODE) { + if e.message().contains(rpc_clients::NO_TX_ERROR_CODE) { if attempts >= 3 { self.watch_for_tx = None; return; diff --git a/mm2src/db_common/src/sqlite.rs b/mm2src/db_common/src/sqlite.rs index 8cb351f1db..0b18dfa54d 100644 --- a/mm2src/db_common/src/sqlite.rs +++ b/mm2src/db_common/src/sqlite.rs @@ -31,7 +31,7 @@ pub(crate) type OwnedSqlParam = Value; pub(crate) type OwnedSqlParams = Vec; type SqlNamedParam<'a> = (&'a str, &'a dyn ToSql); -type SqlNamedParams<'a> = Vec>; +pub type SqlNamedParams<'a> = Vec>; type OwnedSqlNamedParam = (&'static str, Value); pub type OwnedSqlNamedParams = Vec; diff --git a/mm2src/mm2_bitcoin/chain/src/block_header.rs b/mm2src/mm2_bitcoin/chain/src/block_header.rs index 611977d52e..90bd7097f6 100644 --- a/mm2src/mm2_bitcoin/chain/src/block_header.rs +++ b/mm2src/mm2_bitcoin/chain/src/block_header.rs @@ -1,10 +1,13 @@ use compact::Compact; use crypto::dhash256; +use ext_bitcoin::blockdata::block::BlockHeader as ExtBlockHeader; +use ext_bitcoin::hash_types::{BlockHash as ExtBlockHash, TxMerkleNode as ExtTxMerkleNode}; use hash::H256; use hex::FromHex; use primitives::bytes::Bytes; use primitives::U256; use ser::{deserialize, serialize, Deserializable, Reader, Serializable, Stream}; +use std::convert::TryFrom; use std::io; use transaction::{deserialize_tx, TxType}; use {OutPoint, Transaction}; @@ -39,6 +42,24 @@ impl Serializable for BlockHeaderBits { } } +impl From for u32 { + fn from(bits: BlockHeaderBits) -> Self { + match bits { + BlockHeaderBits::Compact(c) => c.into(), + BlockHeaderBits::U32(n) => n, + } + } +} + +impl From for Compact { + fn from(bits: BlockHeaderBits) -> Self { + match bits { + BlockHeaderBits::Compact(c) => c, + BlockHeaderBits::U32(n) => Compact::new(n), + } + } +} + const AUX_POW_VERSION_DOGE: u32 = 6422788; const AUX_POW_VERSION_SYS: u32 = 537919744; const MTP_POW_VERSION: u32 = 0x20001000u32; @@ -317,8 +338,39 @@ impl From<&'static str> for BlockHeader { fn from(s: &'static str) -> Self { deserialize(&s.from_hex::>().unwrap() as &[u8]).unwrap() } } +impl TryFrom for BlockHeader { + type Error = ser::Error; + fn try_from(s: String) -> Result { + deserialize( + &s.from_hex::>() + .map_err(|e| Self::Error::Custom(e.to_string()))? as &[u8], + ) + } +} + +impl From for ExtBlockHeader { + fn from(header: BlockHeader) -> Self { + let prev_blockhash = ExtBlockHash::from_hash(header.previous_header_hash.to_sha256d()); + let merkle_root = ExtTxMerkleNode::from_hash(header.merkle_root_hash.to_sha256d()); + // note: H256 nonce is not supported for bitcoin, we will just set nonce to 0 in this case since this will never happen + let nonce = match header.nonce { + BlockHeaderNonce::U32(n) => n, + _ => 0, + }; + ExtBlockHeader { + version: header.version as i32, + prev_blockhash, + merkle_root, + time: header.time, + bits: header.bits.into(), + nonce, + } + } +} + #[cfg(test)] mod tests { + use super::ExtBlockHeader; use block_header::{BlockHeader, BlockHeaderBits, BlockHeaderNonce, AUX_POW_VERSION_DOGE, AUX_POW_VERSION_SYS, BIP9_NO_SOFT_FORK_BLOCK_HEADER_VERSION, KAWPOW_VERSION, MTP_POW_VERSION, PROG_POW_SWITCH_TIME}; use hex::FromHex; @@ -2387,4 +2439,15 @@ mod tests { let serialized = serialize_list(&headers); assert_eq!(serialized.take(), headers_bytes); } + + #[test] + fn test_from_blockheader_to_ext_blockheader() { + // https://live.blockcypher.com/btc/block/00000000000000000020cf2bdc6563fb25c424af588d5fb7223461e72715e4a9/ + let header: BlockHeader = "0200000066720b99e07d284bd4fe67ff8c49a5db1dd8514fcdab610000000000000000007829844f4c3a41a537b3131ca992643eaa9d093b2383e4cdc060ad7dc548118751eb505ac1910018de19b302".into(); + let ext_header = ExtBlockHeader::from(header.clone()); + assert_eq!( + header.hash().reversed().to_string(), + ext_header.block_hash().to_string() + ); + } } diff --git a/mm2src/mm2_bitcoin/chain/src/raw_block.rs b/mm2src/mm2_bitcoin/chain/src/raw_block.rs index 075d15ad6a..c50e4c1e31 100644 --- a/mm2src/mm2_bitcoin/chain/src/raw_block.rs +++ b/mm2src/mm2_bitcoin/chain/src/raw_block.rs @@ -8,7 +8,7 @@ pub const MIN_RAW_HEADER_SIZE: usize = 80_usize; /// Hex-encoded block #[derive(Default, PartialEq, Clone, Eq, Hash)] -pub struct RawBlockHeader(Bytes); +pub struct RawBlockHeader(pub Bytes); #[derive(Debug, PartialEq, Eq, Clone)] pub enum RawHeaderError { diff --git a/mm2src/mm2_bitcoin/chain/src/transaction.rs b/mm2src/mm2_bitcoin/chain/src/transaction.rs index 37d84e09db..e2585275e2 100644 --- a/mm2src/mm2_bitcoin/chain/src/transaction.rs +++ b/mm2src/mm2_bitcoin/chain/src/transaction.rs @@ -4,13 +4,12 @@ use bytes::Bytes; use constants::{LOCKTIME_THRESHOLD, SEQUENCE_FINAL}; use crypto::{dhash256, sha256}; -use ext_bitcoin::blockdata::transaction::Transaction as ExtTransaction; -use ext_bitcoin::consensus::encode::{deserialize as deserialize_ext, Error as EncodeError}; +use ext_bitcoin::blockdata::transaction::{OutPoint as ExtOutpoint, Transaction as ExtTransaction, TxIn, TxOut}; +use ext_bitcoin::hash_types::Txid; use hash::{CipherText, EncCipherText, OutCipherText, ZkProof, ZkProofSapling, H256, H512, H64}; use hex::FromHex; use ser::{deserialize, serialize, serialize_with_flags, SERIALIZE_TRANSACTION_WITNESS}; use ser::{CompactInteger, Deserializable, Error, Reader, Serializable, Stream}; -use std::convert::TryFrom; use std::io; use std::io::Read; @@ -38,6 +37,15 @@ impl OutPoint { pub fn is_null(&self) -> bool { self.hash.is_zero() && self.index == u32::MAX } } +impl From for ExtOutpoint { + fn from(outpoint: OutPoint) -> Self { + ExtOutpoint { + txid: Txid::from_hash(outpoint.hash.to_sha256d()), + vout: outpoint.index, + } + } +} + #[derive(Debug, PartialEq, Default, Clone)] pub struct TransactionInput { pub previous_output: OutPoint, @@ -61,6 +69,17 @@ impl TransactionInput { pub fn has_witness(&self) -> bool { !self.script_witness.is_empty() } } +impl From for TxIn { + fn from(txin: TransactionInput) -> Self { + TxIn { + previous_output: txin.previous_output.into(), + script_sig: txin.script_sig.take().into(), + sequence: txin.sequence, + witness: txin.script_witness.into_iter().map(|s| s.take()).collect(), + } + } +} + #[derive(Debug, PartialEq, Clone, Serializable, Deserializable)] pub struct TransactionOutput { pub value: u64, @@ -76,6 +95,15 @@ impl Default for TransactionOutput { } } +impl From for TxOut { + fn from(txout: TransactionOutput) -> Self { + TxOut { + value: txout.value, + script_pubkey: txout.script_pubkey.take().into(), + } + } +} + #[derive(Debug, PartialEq, Clone, Serializable, Deserializable)] pub struct ShieldedSpend { pub cv: H256, @@ -198,16 +226,14 @@ impl From<&'static str> for Transaction { fn from(s: &'static str) -> Self { deserialize(&s.from_hex::>().unwrap() as &[u8]).unwrap() } } -impl TryFrom for ExtTransaction { - type Error = EncodeError; - - fn try_from(tx: Transaction) -> Result { - let tx_hex = if tx.has_witness() { - serialize_with_flags(&tx, SERIALIZE_TRANSACTION_WITNESS) - } else { - serialize(&tx) - }; - deserialize_ext(&tx_hex.take()) +impl From for ExtTransaction { + fn from(tx: Transaction) -> Self { + ExtTransaction { + version: tx.version, + lock_time: tx.lock_time, + input: tx.inputs.into_iter().map(|i| i.into()).collect(), + output: tx.outputs.into_iter().map(|o| o.into()).collect(), + } } } @@ -527,7 +553,6 @@ mod tests { use hash::{H256, H512}; use hex::ToHex; use ser::{deserialize, serialize, serialize_with_flags, Serializable, SERIALIZE_TRANSACTION_WITNESS}; - use std::convert::TryFrom; use TxHashAlgo; // real transaction from block 80000 @@ -995,10 +1020,10 @@ mod tests { } #[test] - fn test_try_from_tx_to_ext_tx() { + fn test_from_tx_to_ext_tx() { // https://live.blockcypher.com/btc-testnet/tx/2be90e03abb4d5328bf7e9467ca9c571aef575837b55f1253119b87e85ccb94f/ let tx: Transaction = "010000000001016546e6d844ad0142c8049a839e8deae16c17f0a6587e36e75ff2181ed7020a800100000000ffffffff0247070800000000002200200bbfbd271853ec0a775e5455d4bb19d32818e9b5bda50655ac183fb15c9aa01625910300000000001600149a85cc05e9a722575feb770a217c73fd6145cf0102473044022002eac5d11f3800131985c14a3d1bc03dfe5e694f5731bde39b0d2b183eb7d3d702201d62e7ff2dd433260bf7a8223db400d539a2c4eccd27a5aa24d83f5ad9e9e1750121031ac6d25833a5961e2a8822b2e8b0ac1fd55d90cbbbb18a780552cbd66fc02bb35c099c61".into(); - let ext_tx = ExtTransaction::try_from(tx.clone()).unwrap(); + let ext_tx = ExtTransaction::from(tx.clone()); assert_eq!(tx.hash().reversed().to_string(), ext_tx.txid().to_string()); } } diff --git a/mm2src/mm2_bitcoin/primitives/Cargo.toml b/mm2src/mm2_bitcoin/primitives/Cargo.toml index ef679deeb8..764f02d1fa 100644 --- a/mm2src/mm2_bitcoin/primitives/Cargo.toml +++ b/mm2src/mm2_bitcoin/primitives/Cargo.toml @@ -5,5 +5,6 @@ authors = ["debris "] [dependencies] rustc-hex = "2" +bitcoin_hashes = "0.10.0" byteorder = "1.0" -uint = "0.9.1" +uint = "0.9.3" diff --git a/mm2src/mm2_bitcoin/primitives/src/hash.rs b/mm2src/mm2_bitcoin/primitives/src/hash.rs index 785653a980..e7c10aa4e8 100644 --- a/mm2src/mm2_bitcoin/primitives/src/hash.rs +++ b/mm2src/mm2_bitcoin/primitives/src/hash.rs @@ -1,5 +1,6 @@ //! Fixed-size hashes +use bitcoin_hashes::{sha256d, Hash as ExtHash}; use hex::{FromHex, FromHexError, ToHex}; use std::hash::{Hash, Hasher}; use std::{cmp, fmt, ops, str}; @@ -165,4 +166,7 @@ impl H256 { #[inline] pub fn to_reversed_str(self) -> String { self.reversed().to_string() } + + #[inline] + pub fn to_sha256d(self) -> sha256d::Hash { sha256d::Hash::from_inner(self.take()) } } diff --git a/mm2src/mm2_bitcoin/primitives/src/lib.rs b/mm2src/mm2_bitcoin/primitives/src/lib.rs index 9e07a813c7..78dc7330e7 100644 --- a/mm2src/mm2_bitcoin/primitives/src/lib.rs +++ b/mm2src/mm2_bitcoin/primitives/src/lib.rs @@ -1,6 +1,7 @@ #![allow(clippy::assign_op_pattern)] #![allow(clippy::ptr_offset_with_cast)] +extern crate bitcoin_hashes; extern crate byteorder; extern crate rustc_hex as hex; extern crate uint; diff --git a/mm2src/mm2_bitcoin/rpc/src/v1/types/get_block_response.rs b/mm2src/mm2_bitcoin/rpc/src/v1/types/get_block_response.rs index bebfd32878..ba29704208 100644 --- a/mm2src/mm2_bitcoin/rpc/src/v1/types/get_block_response.rs +++ b/mm2src/mm2_bitcoin/rpc/src/v1/types/get_block_response.rs @@ -124,7 +124,7 @@ mod tests { let block = VerboseBlock::default(); assert_eq!( serde_json::to_string(&block).unwrap(), - r#"{"hash":"0000000000000000000000000000000000000000000000000000000000000000","confirmations":0,"size":0,"strippedsize":0,"weight":0,"height":null,"version":0,"versionHex":"","merkleroot":"0000000000000000000000000000000000000000000000000000000000000000","tx":[],"time":0,"mediantime":null,"nonce":0,"bits":0,"difficulty":0.0,"chainwork":"0","previousblockhash":null,"nextblockhash":null}"# + r#"{"hash":"0000000000000000000000000000000000000000000000000000000000000000","confirmations":0,"size":0,"strippedsize":0,"weight":0,"height":null,"version":0,"versionHex":"","merkleroot":"0000000000000000000000000000000000000000000000000000000000000000","tx":[],"time":0,"mediantime":null,"nonce":0,"bits":0,"difficulty":0.0,"chainwork":"00","previousblockhash":null,"nextblockhash":null}"# ); let block = VerboseBlock { @@ -149,7 +149,7 @@ mod tests { }; assert_eq!( serde_json::to_string(&block).unwrap(), - r#"{"hash":"0100000000000000000000000000000000000000000000000000000000000000","confirmations":-1,"size":500000,"strippedsize":444444,"weight":5236235,"height":3513513,"version":1,"versionHex":"01","merkleroot":"0200000000000000000000000000000000000000000000000000000000000000","tx":["0300000000000000000000000000000000000000000000000000000000000000","0400000000000000000000000000000000000000000000000000000000000000"],"time":111,"mediantime":100,"nonce":124,"bits":13513,"difficulty":555.555,"chainwork":"3","previousblockhash":"0400000000000000000000000000000000000000000000000000000000000000","nextblockhash":"0500000000000000000000000000000000000000000000000000000000000000"}"# + r#"{"hash":"0100000000000000000000000000000000000000000000000000000000000000","confirmations":-1,"size":500000,"strippedsize":444444,"weight":5236235,"height":3513513,"version":1,"versionHex":"01","merkleroot":"0200000000000000000000000000000000000000000000000000000000000000","tx":["0300000000000000000000000000000000000000000000000000000000000000","0400000000000000000000000000000000000000000000000000000000000000"],"time":111,"mediantime":100,"nonce":124,"bits":13513,"difficulty":555.555,"chainwork":"03","previousblockhash":"0400000000000000000000000000000000000000000000000000000000000000","nextblockhash":"0500000000000000000000000000000000000000000000000000000000000000"}"# ); } @@ -197,7 +197,7 @@ mod tests { let verbose_response = GetBlockResponse::Verbose(Box::new(block)); assert_eq!( serde_json::to_string(&verbose_response).unwrap(), - r#"{"hash":"0000000000000000000000000000000000000000000000000000000000000000","confirmations":0,"size":0,"strippedsize":0,"weight":0,"height":null,"version":0,"versionHex":"","merkleroot":"0000000000000000000000000000000000000000000000000000000000000000","tx":[],"time":0,"mediantime":null,"nonce":0,"bits":0,"difficulty":0.0,"chainwork":"0","previousblockhash":null,"nextblockhash":null}"# + r#"{"hash":"0000000000000000000000000000000000000000000000000000000000000000","confirmations":0,"size":0,"strippedsize":0,"weight":0,"height":null,"version":0,"versionHex":"","merkleroot":"0000000000000000000000000000000000000000000000000000000000000000","tx":[],"time":0,"mediantime":null,"nonce":0,"bits":0,"difficulty":0.0,"chainwork":"00","previousblockhash":null,"nextblockhash":null}"# ); } } diff --git a/mm2src/mm2_bitcoin/spv_validation/Cargo.toml b/mm2src/mm2_bitcoin/spv_validation/Cargo.toml index ccf46b61ae..6d3a5e96d6 100644 --- a/mm2src/mm2_bitcoin/spv_validation/Cargo.toml +++ b/mm2src/mm2_bitcoin/spv_validation/Cargo.toml @@ -2,9 +2,13 @@ name = "spv_validation" version = "0.1.0" authors = ["Roman Sztergbaum "] +edition = "2018" [dependencies] +async-trait = "0.1" chain = {path = "../chain"} +derive_more = "0.99" +keys = {path = "../keys"} primitives = { path = "../primitives" } ripemd160 = "0.9.0" rustc-hex = "2" @@ -13,5 +17,7 @@ sha2 = "0.9" test_helpers = { path = "../test_helpers" } [dev-dependencies] +common = { path = "../../common" } +lazy_static = "1.4" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" \ No newline at end of file diff --git a/mm2src/mm2_bitcoin/spv_validation/src/for_tests/workTestVectors.json b/mm2src/mm2_bitcoin/spv_validation/src/for_tests/workTestVectors.json new file mode 100644 index 0000000000..764b633307 --- /dev/null +++ b/mm2src/mm2_bitcoin/spv_validation/src/for_tests/workTestVectors.json @@ -0,0 +1,30 @@ +{ + "BTC": [ + { + "height": 2016, + "hex": "010000006397bb6abd4fc521c0d3f6071b5650389f0b4551bc40b4e6b067306900000000ace470aecda9c8818c8fe57688cd2a772b5a57954a00df0420a7dd546b6d2c576b0e7f49ffff001d33f0192f" + }, + { + "height": 604800, + "hex": "000000208e244d2c55bc403caa5d6eaf0f922170e413eb1e02fb02000000000000000000e03b4d9df72d8db232a20bb2ff35c433a99f1467f391f75b5f62180d96f06d6aa4c4d65d3eb215179ef91633" + } + ], + "tBTC": [ + { + "height": 199584, + "hex": "0200000097f2b61897ba2bed756cca30058bcc1c2dfbb4ed0e962f47f749dc03000000006b80079a1eda8071424e294fa56849370e331c8ff7e95034576c9789c8db0fa6da551153ab80011c9bdaca25" + }, + { + "height": 201594, + "hex": "02000000cbed7fd98f1f06e85c47e13ff956533642056be45e7e6b532d4d768f00000000f2680982f333fcc9afa7f9a5e2a84dc54b7fe10605cd187362980b3aa882e9683be21353ab80011c813e1fc0" + }, + { + "height": 201595, + "hex": "02000000ea01a61a2d7420a1b23875e40eb5eb4ca18b378902c8e6384514ad0000000000c0c5a1ae80582b3fe319d8543307fa67befc2a734b8eddb84b1780dfdf11fa2b20e71353ffff001d00805fe0" + }, + { + "height": 201596, + "hex": "02000000303505969a1df329e5fccdf69b847a201772e116e557eb7f119d1a9600000000469267f52f43b8799e72f0726ba2e56432059a8ad02b84d4fff84b9476e95f7716e41353ab80011c168cb471" + } + ] +} \ No newline at end of file diff --git a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs index a91ff91289..6c3c263960 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs @@ -1,50 +1,52 @@ use chain::{BlockHeader, RawBlockHeader, RawHeaderError}; +use derive_more::Display; use primitives::hash::H256; use primitives::U256; use ripemd160::Digest; use serialization::parse_compact_int; use sha2::Sha256; -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, Display, PartialEq, Eq, Clone)] pub enum SPVError { - /// Overran a checked read on a slice + #[display(fmt = "Overran a checked read on a slice")] ReadOverrun, - /// Attempted to parse a CompactInt without enough bytes + #[display(fmt = "Attempted to parse a CompactInt without enough bytes")] BadCompactInt, - /// `extract_hash` could not identify the output type. + #[display(fmt = "`extract_hash` could not identify the output type")] MalformattedOutput, - /// Unable to get target from block header + #[display(fmt = "Unable to get target from block header")] UnableToGetTarget, - /// Unable to get block header from network or storage - UnableToGetHeader, - /// Header not exactly 80 bytes. + #[display(fmt = "Unable to get block header from network or storage: {}", _0)] + UnableToGetHeader(String), + #[display(fmt = "Header not exactly 80 bytes")] WrongLengthHeader, - /// Header chain changed difficulties unexpectedly + #[display(fmt = "Header chain changed difficulties unexpectedly")] UnexpectedDifficultyChange, - /// Header does not meet its own difficulty target. + #[display(fmt = "Header does not meet its own difficulty target")] InsufficientWork, - /// Header in chain does not correctly reference parent header. + #[display(fmt = "Header in chain does not correctly reference parent header")] InvalidChain, - /// When validating a `BitcoinHeader`, the `hash` field is not the digest - /// of the raw header. + #[display(fmt = "When validating a `BitcoinHeader`, the `hash` field is not the digest of the raw header")] WrongDigest, - /// When validating a `BitcoinHeader`, the `merkle_root` field does not - /// match the root found in the raw header. + #[display( + fmt = "When validating a `BitcoinHeader`, the `merkle_root` field does not match the root found in the raw header" + )] WrongMerkleRoot, - /// When validating a `BitcoinHeader`, the `prevhash` field does not - /// match the parent hash found in the raw header. + #[display( + fmt = "When validating a `BitcoinHeader`, the `prevhash` field does not match the parent hash found in the raw header" + )] WrongPrevHash, - /// A `vin` (transaction input vector) is malformatted. + #[display(fmt = "A `vin` (transaction input vector) is malformatted")] InvalidVin, - /// A `vout` (transaction output vector) is malformatted or empty. + #[display(fmt = "A `vout` (transaction output vector) is malformatted or empty")] InvalidVout, - /// merkle proof connecting the `tx_id_le` to the `confirming_header`. + #[display(fmt = "merkle proof connecting the `tx_id_le` to the `confirming_header`")] BadMerkleProof, - /// Unable to get merkle tree from network or storage - UnableToGetMerkle, - /// Unable to retrieve block height / block height is zero. - InvalidHeight, - /// Raises during validation loop + #[display(fmt = "Unable to get merkle tree from network or storage: {}", _0)] + UnableToGetMerkle(String), + #[display(fmt = "Unable to retrieve block height / block height is zero: {}", _0)] + InvalidHeight(String), + #[display(fmt = "Raises during validation loop")] Timeout, } diff --git a/mm2src/mm2_bitcoin/spv_validation/src/lib.rs b/mm2src/mm2_bitcoin/spv_validation/src/lib.rs index 1c30005769..d74dc35392 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/lib.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/lib.rs @@ -1,4 +1,6 @@ extern crate chain; +extern crate derive_more; +extern crate keys; extern crate primitives; extern crate ripemd160; extern crate rustc_hex as hex; @@ -12,6 +14,12 @@ pub mod helpers_validation; /// `spv_proof` Contains spv proof validation logic and data structure pub mod spv_proof; +/// `storage` Contains traits that can be implemented to provide the storage needed for spv validation +pub mod storage; + +/// `work` Contains functions that can be used to calculate proof of work difficulty, target, bits, etc... +pub mod work; + #[cfg(test)] pub(crate) mod test_utils { extern crate serde; diff --git a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs index 10df7433dd..646afc0f16 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs @@ -1,6 +1,6 @@ +use crate::helpers_validation::{merkle_prove, validate_vin, validate_vout, SPVError}; use chain::BlockHeader; use chain::RawBlockHeader; -use helpers_validation::{merkle_prove, validate_vin, validate_vout, SPVError}; use primitives::hash::H256; pub const TRY_SPV_PROOF_INTERVAL: u64 = 10; @@ -68,11 +68,11 @@ impl SPVProof { #[cfg(test)] mod spv_proof_tests { + use crate::spv_proof::SPVProof; use chain::BlockHeader; use chain::RawBlockHeader; use hex::FromHex; use serialization::deserialize; - use spv_proof::SPVProof; #[test] fn test_block_header() { diff --git a/mm2src/mm2_bitcoin/spv_validation/src/storage.rs b/mm2src/mm2_bitcoin/spv_validation/src/storage.rs new file mode 100644 index 0000000000..d516535f79 --- /dev/null +++ b/mm2src/mm2_bitcoin/spv_validation/src/storage.rs @@ -0,0 +1,82 @@ +use async_trait::async_trait; +use chain::BlockHeader; +use derive_more::Display; +use primitives::hash::H256; +use std::collections::HashMap; + +#[derive(Debug, Display)] +pub enum BlockHeaderStorageError { + #[display(fmt = "Can't add to the storage for {} - reason: {}", coin, reason)] + AddToStorageError { + coin: String, + reason: String, + }, + #[display(fmt = "Can't get from the storage for {} - reason: {}", coin, reason)] + GetFromStorageError { + coin: String, + reason: String, + }, + #[display(fmt = "Can't retrieve the table from the storage for {} - reason: {}", coin, reason)] + CantRetrieveTableError { + coin: String, + reason: String, + }, + #[display(fmt = "Can't query from the storage - query: {} - reason: {}", query, reason)] + QueryError { + query: String, + reason: String, + }, + #[display(fmt = "Can't init from the storage - coin: {} - reason: {}", coin, reason)] + InitializationError { + coin: String, + reason: String, + }, + #[display(fmt = "Can't decode/deserialize from storage for {} - reason: {}", coin, reason)] + DecodeError { + coin: String, + reason: String, + }, + Internal(String), +} + +#[async_trait] +pub trait BlockHeaderStorageOps: Send + Sync + 'static { + /// Initializes collection/tables in storage for a specified coin + async fn init(&self, for_coin: &str) -> Result<(), BlockHeaderStorageError>; + + async fn is_initialized_for(&self, for_coin: &str) -> Result; + + // Adds multiple block headers to the selected coin's header storage + // Should store it as `COIN_HEIGHT=hex_string` + // use this function for headers that comes from `blockchain_block_headers` + async fn add_block_headers_to_storage( + &self, + for_coin: &str, + headers: HashMap, + ) -> Result<(), BlockHeaderStorageError>; + + /// Gets the block header by height from the selected coin's storage as BlockHeader + async fn get_block_header( + &self, + for_coin: &str, + height: u64, + ) -> Result, BlockHeaderStorageError>; + + /// Gets the block header by height from the selected coin's storage as hex + async fn get_block_header_raw( + &self, + for_coin: &str, + height: u64, + ) -> Result, BlockHeaderStorageError>; + + async fn get_last_block_header_with_non_max_bits( + &self, + for_coin: &str, + ) -> Result, BlockHeaderStorageError>; + + async fn get_block_height_by_hash( + &self, + for_coin: &str, + hash: H256, + ) -> Result, BlockHeaderStorageError>; +} diff --git a/mm2src/mm2_bitcoin/spv_validation/src/work.rs b/mm2src/mm2_bitcoin/spv_validation/src/work.rs new file mode 100644 index 0000000000..ed5935006d --- /dev/null +++ b/mm2src/mm2_bitcoin/spv_validation/src/work.rs @@ -0,0 +1,337 @@ +use crate::storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; +use chain::{BlockHeader, BlockHeaderBits}; +use derive_more::Display; +use primitives::compact::Compact; +use primitives::U256; +use std::cmp; + +const RETARGETING_FACTOR: u32 = 4; +const TARGET_SPACING_SECONDS: u32 = 10 * 60; +const TARGET_TIMESPAN_SECONDS: u32 = 2 * 7 * 24 * 60 * 60; + +/// The Target number of blocks equals to 2 weeks or 2016 blocks +const RETARGETING_INTERVAL: u32 = TARGET_TIMESPAN_SECONDS / TARGET_SPACING_SECONDS; + +/// The upper and lower bounds for retargeting timespan +const MIN_TIMESPAN: u32 = TARGET_TIMESPAN_SECONDS / RETARGETING_FACTOR; +const MAX_TIMESPAN: u32 = TARGET_TIMESPAN_SECONDS * RETARGETING_FACTOR; + +/// The maximum value for bits corresponding to lowest difficulty of 1 +pub const MAX_BITS_BTC: u32 = 486604799; + +fn is_retarget_height(height: u32) -> bool { height % RETARGETING_INTERVAL == 0 } + +#[derive(Debug, Display)] +pub enum NextBlockBitsError { + #[display(fmt = "Block headers storage error: {}", _0)] + StorageError(BlockHeaderStorageError), + #[display(fmt = "Can't find Block header for {} with height {}", height, coin)] + NoSuchBlockHeader { + coin: String, + height: u64, + }, + #[display(fmt = "Can't find a Block header for {} with no max bits", coin)] + NoBlockHeaderWithNoMaxBits { + coin: String, + }, + Internal(String), +} + +impl From for NextBlockBitsError { + fn from(e: BlockHeaderStorageError) -> Self { NextBlockBitsError::StorageError(e) } +} + +pub enum DifficultyAlgorithm { + BitcoinMainnet, + BitcoinTestnet, +} + +pub async fn next_block_bits( + coin: &str, + current_block_timestamp: u32, + last_block_header: BlockHeader, + last_block_height: u32, + storage: &dyn BlockHeaderStorageOps, + algorithm: DifficultyAlgorithm, +) -> Result { + match algorithm { + DifficultyAlgorithm::BitcoinMainnet => { + btc_mainnet_next_block_bits(coin, last_block_header, last_block_height, storage).await + }, + DifficultyAlgorithm::BitcoinTestnet => { + btc_testnet_next_block_bits( + coin, + current_block_timestamp, + last_block_header, + last_block_height, + storage, + ) + .await + }, + } +} + +fn range_constrain(value: i64, min: i64, max: i64) -> i64 { cmp::min(cmp::max(value, min), max) } + +/// Returns constrained number of seconds since last retarget +fn retarget_timespan(retarget_timestamp: u32, last_timestamp: u32) -> u32 { + // subtract unsigned 32 bit numbers in signed 64 bit space in + // order to prevent underflow before applying the range constraint. + let timespan = last_timestamp as i64 - retarget_timestamp as i64; + range_constrain(timespan, MIN_TIMESPAN as i64, MAX_TIMESPAN as i64) as u32 +} + +async fn btc_retarget_bits( + coin: &str, + height: u32, + last_block_header: BlockHeader, + storage: &dyn BlockHeaderStorageOps, +) -> Result { + let retarget_ref = (height - RETARGETING_INTERVAL).into(); + let retarget_header = + storage + .get_block_header(coin, retarget_ref) + .await? + .ok_or(NextBlockBitsError::NoSuchBlockHeader { + coin: coin.into(), + height: retarget_ref, + })?; + // timestamp of block(height - RETARGETING_INTERVAL) + let retarget_timestamp = retarget_header.time; + // timestamp of last block + let last_timestamp = last_block_header.time; + + let retarget: Compact = last_block_header.bits.into(); + let retarget: U256 = retarget.into(); + let retarget_timespan: U256 = retarget_timespan(retarget_timestamp, last_timestamp).into(); + let retarget: U256 = retarget * retarget_timespan; + let target_timespan_seconds: U256 = TARGET_TIMESPAN_SECONDS.into(); + let retarget = retarget / target_timespan_seconds; + + let max_bits_compact: Compact = MAX_BITS_BTC.into(); + let max_bits: U256 = max_bits_compact.into(); + + if retarget > max_bits { + Ok(BlockHeaderBits::Compact(max_bits_compact)) + } else { + Ok(BlockHeaderBits::Compact(retarget.into())) + } +} + +async fn btc_mainnet_next_block_bits( + coin: &str, + last_block_header: BlockHeader, + last_block_height: u32, + storage: &dyn BlockHeaderStorageOps, +) -> Result { + if last_block_height == 0 { + return Err(NextBlockBitsError::Internal("Last block height can't be zero".into())); + } + + let height = last_block_height + 1; + let last_block_bits = last_block_header.bits.clone(); + + if is_retarget_height(height) { + btc_retarget_bits(coin, height, last_block_header, storage).await + } else { + Ok(last_block_bits) + } +} + +async fn btc_testnet_next_block_bits( + coin: &str, + current_block_timestamp: u32, + last_block_header: BlockHeader, + last_block_height: u32, + storage: &dyn BlockHeaderStorageOps, +) -> Result { + if last_block_height == 0 { + return Err(NextBlockBitsError::Internal("Last block height can't be zero".into())); + } + + let height = last_block_height + 1; + let last_block_bits = last_block_header.bits.clone(); + let max_time_gap = last_block_header.time + 2 * TARGET_SPACING_SECONDS; + let max_bits = BlockHeaderBits::Compact(MAX_BITS_BTC.into()); + + if is_retarget_height(height) { + btc_retarget_bits(coin, height, last_block_header, storage).await + } else if current_block_timestamp > max_time_gap { + Ok(max_bits) + } else if last_block_bits != max_bits { + Ok(last_block_bits.clone()) + } else { + let last_block_header_with_non_max_bits = storage + .get_last_block_header_with_non_max_bits(coin) + .await? + .ok_or(NextBlockBitsError::NoBlockHeaderWithNoMaxBits { coin: coin.into() })?; + Ok(last_block_header_with_non_max_bits.bits) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; + use async_trait::async_trait; + use common::block_on; + use lazy_static::lazy_static; + use primitives::hash::H256; + use serde::Deserialize; + use std::collections::HashMap; + + const BLOCK_HEADERS_STR: &str = include_str!("./for_tests/workTestVectors.json"); + + #[derive(Deserialize)] + struct TestRawHeader { + height: u64, + hex: String, + } + + lazy_static! { + static ref BLOCK_HEADERS_MAP: HashMap> = parse_block_headers(); + } + + fn parse_block_headers() -> HashMap> { serde_json::from_str(BLOCK_HEADERS_STR).unwrap() } + + fn get_block_headers_for_coin(coin: &str) -> HashMap { + BLOCK_HEADERS_MAP + .get(coin) + .unwrap() + .into_iter() + .map(|h| (h.height, h.hex.as_str().into())) + .collect() + } + + struct TestBlockHeadersStorage {} + + #[async_trait] + impl BlockHeaderStorageOps for TestBlockHeadersStorage { + async fn init(&self, _for_coin: &str) -> Result<(), BlockHeaderStorageError> { Ok(()) } + + async fn is_initialized_for(&self, _for_coin: &str) -> Result { Ok(true) } + + async fn add_block_headers_to_storage( + &self, + _for_coin: &str, + _headers: HashMap, + ) -> Result<(), BlockHeaderStorageError> { + Ok(()) + } + + async fn get_block_header( + &self, + for_coin: &str, + height: u64, + ) -> Result, BlockHeaderStorageError> { + Ok(get_block_headers_for_coin(for_coin).get(&height).cloned()) + } + + async fn get_block_header_raw( + &self, + _for_coin: &str, + _height: u64, + ) -> Result, BlockHeaderStorageError> { + Ok(None) + } + + async fn get_last_block_header_with_non_max_bits( + &self, + for_coin: &str, + ) -> Result, BlockHeaderStorageError> { + let mut headers = get_block_headers_for_coin(for_coin); + headers.retain(|_, h| h.bits != BlockHeaderBits::Compact(MAX_BITS_BTC.into())); + let header = headers.into_iter().max_by(|a, b| a.0.cmp(&b.0)); + Ok(header.map(|(_, h)| h)) + } + + async fn get_block_height_by_hash( + &self, + _for_coin: &str, + _hash: H256, + ) -> Result, BlockHeaderStorageError> { + Ok(None) + } + } + + #[test] + fn test_btc_mainnet_next_block_bits() { + let storage = TestBlockHeadersStorage {}; + + let last_header: BlockHeader = "000000201d758432ecd495a2177b44d3fe6c22af183461a0b9ea0d0000000000000000008283a1dfa795d9b68bd8c18601e443368265072cbf8c76bfe58de46edd303798035de95d3eb2151756fdb0e8".into(); + + let next_block_bits = block_on(btc_mainnet_next_block_bits("BTC", last_header, 606815, &storage)).unwrap(); + + assert_eq!(next_block_bits, BlockHeaderBits::Compact(387308498.into())); + + // check that bits for very early blocks that didn't change difficulty because of low hashrate is calculated correctly. + let last_header: BlockHeader = "010000000d9c8c96715756b619116cc2160937fb26c655a2f8e28e3a0aff59c0000000007676252e8434de408ea31920d986aba297bd6f7c6f20756be08748713f7c135962719449ffff001df8c1cb01".into(); + + let next_block_bits = block_on(btc_mainnet_next_block_bits("BTC", last_header, 4031, &storage)).unwrap(); + + assert_eq!(next_block_bits, BlockHeaderBits::Compact(486604799.into())); + + // check that bits stay the same when the next block is not a retarget block + // https://live.blockcypher.com/btc/block/00000000000000000002622f52b6afe70a5bb139c788e67f221ffc67a762a1e0/ + let last_header: BlockHeader = "00e0ff2f44d953fe12a047129bbc7164668c6d96f3e7a553528b02000000000000000000d0b950384cd23ab0854d1c8f23fa7a97411a6ffd92347c0a3aea4466621e4093ec09c762afa7091705dad220".into(); + + let next_block_bits = block_on(btc_mainnet_next_block_bits("BTC", last_header, 744014, &storage)).unwrap(); + + assert_eq!(next_block_bits, BlockHeaderBits::Compact(386508719.into())); + } + + #[test] + fn test_btc_testnet_next_block_bits() { + let storage = TestBlockHeadersStorage {}; + + // https://live.blockcypher.com/btc-testnet/block/000000000057db3806384e2ec1b02b2c86bd928206ff8dff98f54d616b7fa5f2/ + let current_header: BlockHeader = "02000000303505969a1df329e5fccdf69b847a201772e116e557eb7f119d1a9600000000469267f52f43b8799e72f0726ba2e56432059a8ad02b84d4fff84b9476e95f7716e41353ab80011c168cb471".into(); + // https://live.blockcypher.com/btc-testnet/block/00000000961a9d117feb57e516e17217207a849bf6cdfce529f31d9a96053530/ + let last_header: BlockHeader = "02000000ea01a61a2d7420a1b23875e40eb5eb4ca18b378902c8e6384514ad0000000000c0c5a1ae80582b3fe319d8543307fa67befc2a734b8eddb84b1780dfdf11fa2b20e71353ffff001d00805fe0".into(); + + let next_block_bits = block_on(btc_testnet_next_block_bits( + "tBTC", + current_header.time, + last_header, + 201595, + &storage, + )) + .unwrap(); + + assert_eq!(next_block_bits, BlockHeaderBits::Compact(469860523.into())); + + // https://live.blockcypher.com/btc-testnet/block/00000000961a9d117feb57e516e17217207a849bf6cdfce529f31d9a96053530/ + let current_header: BlockHeader = "02000000ea01a61a2d7420a1b23875e40eb5eb4ca18b378902c8e6384514ad0000000000c0c5a1ae80582b3fe319d8543307fa67befc2a734b8eddb84b1780dfdf11fa2b20e71353ffff001d00805fe0".into(); + // https://live.blockcypher.com/btc-testnet/block/0000000000ad144538e6c80289378ba14cebb50ee47538b2a120742d1aa601ea/ + let last_header: BlockHeader = "02000000cbed7fd98f1f06e85c47e13ff956533642056be45e7e6b532d4d768f00000000f2680982f333fcc9afa7f9a5e2a84dc54b7fe10605cd187362980b3aa882e9683be21353ab80011c813e1fc0".into(); + + let next_block_bits = block_on(btc_testnet_next_block_bits( + "tBTC", + current_header.time, + last_header, + 201594, + &storage, + )) + .unwrap(); + + assert_eq!(next_block_bits, BlockHeaderBits::Compact(486604799.into())); + + // test testnet retarget bits + + // https://live.blockcypher.com/btc-testnet/block/0000000000376bb71314321c45de3015fe958543afcbada242a3b1b072498e38/ + let current_header: BlockHeader = "02000000ee689e4dcdc3c7dac591b98e1e4dc83aae03ff9fb9d469d704a64c0100000000bfffaded2a67821eb5729b362d613747e898d08d6c83b5704646c26c13146f4c6de91353c02a601b3a817f87".into(); + // https://live.blockcypher.com/btc-testnet/block/00000000014ca604d769d4b99fff03ae3ac84d1e8eb991c5dac7c3cd4d9e68ee/ + let last_header: BlockHeader = "02000000a9dccfcf372d6ce6ae784786ea94d20ce174e093520d779348e5a9000000000077c037863a0134ac05a8c19d258d6c03c225043a08687c90813e8352a144d68035e81353ab80011ca71f3849".into(); + + let next_block_bits = block_on(btc_testnet_next_block_bits( + "tBTC", + current_header.time, + last_header, + 201599, + &storage, + )) + .unwrap(); + + assert_eq!(next_block_bits, BlockHeaderBits::Compact(459287232.into())); + } +} From a5e2ecef0f3056bd01dd7c3c5dc63d8fbda989a8 Mon Sep 17 00:00:00 2001 From: Sharon Alina <52405288+laruh@users.noreply.github.com> Date: Mon, 1 Aug 2022 15:00:21 +0700 Subject: [PATCH 003/249] target-branch added in dependabot.yml (#1424) --- .github/dependabot.yml | 136 ++--------------------------------------- 1 file changed, 6 insertions(+), 130 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2bd048103b..563b2172b0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,136 +5,12 @@ version: 2 updates: - package-ecosystem: cargo # Define the location of the package manifests - directory: "/mm2src/coins/" + directory: "/mm2src" + # Use target-branch to specify a different(from default) branch for manifest files and for pull requests. + target-branch: "dev" + # By default, Dependabot opens a maximum of five pull requests for version updates. + # Use open-pull-requests-limit to change this limit. + open-pull-requests-limit: 2 schedule: # By default, Dependabot checks for new versions on Monday at a random set time for the repository - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/common/shared_ref_counter/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/coins/lightning_persister/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/coins/lightning_background_processor/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/coins/utxo_signer/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/coins_activation/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/crypto/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/db_common/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/derives/ser_error/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/derives/ser_error_derive/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/floodsub/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/gossipsub/" - schedule: - interval: weekly - open-pull-requests-limit: 10 - - package-ecosystem: cargo - directory: "/mm2src/hw_common/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_bitcoin/crypto/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_bitcoin/chain/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_bitcoin/keys/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_bitcoin/rpc/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_bitcoin/primitives/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_bitcoin/script/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_bitcoin/serialization/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_bitcoin/serialization_derive/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_bitcoin/test_helpers/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_core/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_db/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_err_handle/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_test_helpers/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_libp2p/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_main/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_net/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_io/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/mm2_rpc/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/rpc_task/" - schedule: - interval: weekly - - package-ecosystem: cargo - directory: "/mm2src/trezor/" - schedule: interval: weekly \ No newline at end of file From 6631a20822a0a496b42b963c4ce05d58f69b7065 Mon Sep 17 00:00:00 2001 From: Onur Date: Fri, 5 Aug 2022 08:59:12 +0300 Subject: [PATCH 004/249] gui-auth & enable_v2 rpc implementation (#1335) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * implement validation strcuture Signed-off-by: Onur Özkan * add signed_message to payload if exists Signed-off-by: ozkanonur * update doc-comment of `drop_mutability` macro Signed-off-by: ozkanonur * update `fn sign_message_hash` Signed-off-by: Onur Özkan * fill `gui_auth_message_prefix` fields for tests Signed-off-by: ozkanonur * create and implement `trait GuiAuthMessages` Signed-off-by: Onur Özkan * save development state Signed-off-by: Onur Özkan * save development state Signed-off-by: Onur Özkan * save development state Signed-off-by: Onur Özkan * save development state Signed-off-by: ozkanonur * save development state Signed-off-by: Onur Özkan * save development state Signed-off-by: ozkanonur * save development state Signed-off-by: Onur Özkan * save development state Signed-off-by: Onur Özkan * save development state Signed-off-by: Onur Özkan * save development state Signed-off-by: ozkanonur * finalize `fn enable_v2` implementation Signed-off-by: ozkanonur * force UTC tz for gui-auth signed messages Signed-off-by: ozkanonur * optimize `Web3Transport` initialization Signed-off-by: ozkanonur * save development state Signed-off-by: ozkanonur * inline `Web3Transport` initialization functions Signed-off-by: ozkanonur * Update Cargo.toml * update version handling of eth coins Signed-off-by: Onur Özkan * provide better error handling Signed-off-by: ozkanonur * simplify `generate_gui_auth_signed_validation` Signed-off-by: ozkanonur * save development state Signed-off-by: ozkanonur * save development state Signed-off-by: ozkanonur * save development state Signed-off-by: Onur Özkan * save p.o.c state Signed-off-by: ozkanonur * save development state Signed-off-by: ozkanonur * exclude `activate_eth_coin` Signed-off-by: ozkanonur * finalize `eth_and_erc20_activations` functionality Signed-off-by: ozkanonur * save development state Signed-off-by: ozkanonur * save development state Signed-off-by: Onur Özkan * fix `check --tests` Signed-off-by: Onur Özkan * finish `register_token_info` fn Signed-off-by: ozkanonur * implement `erc20_token_activation` rpc Signed-off-by: ozkanonur * map errors for eth/erc20 activation Signed-off-by: ozkanonur * fix wasm build failure Signed-off-by: ozkanonur * fix fmt Signed-off-by: ozkanonur * rename `EthActivationRequest` to `EthActivationV2Request` Signed-off-by: ozkanonur * optimize implementation Signed-off-by: ozkanonur * drop mutability after updates Signed-off-by: ozkanonur * save development state Signed-off-by: ozkanonur * make `fn enable_token` less bloat Signed-off-by: ozkanonur * make `enable_tokens` less bloat Signed-off-by: ozkanonur * finalize review fixes Signed-off-by: ozkanonur * update errors Signed-off-by: ozkanonur * update visibility & privacy modifiers in `eth.rs` Signed-off-by: ozkanonur * make `get_token_balance_by_address` async fn Signed-off-by: ozkanonur * avoid using hard-coded ticker for gui-auth Signed-off-by: Onur Özkan * fix review note Signed-off-by: Onur Özkan * optimize pr stage Signed-off-by: ozkanonur * optimize `send_request` of wasm Signed-off-by: Onur Özkan * fix wasm compilation Signed-off-by: Onur Özkan * fix pr notes Signed-off-by: Onur Özkan * fix formatting Signed-off-by: Onur Özkan * update `get_public_address` to uncompress format for ethereum Signed-off-by: Onur Özkan * no need to clone Erc20TokenInfo for `get_token_balance_by_address` Signed-off-by: Onur Özkan * update `InvalidSwapContractAddr` and `InvalidFallbackSwapContract` mapping Signed-off-by: Onur Özkan * refactor `AsyncMutex` to std one for `eth20_tokens_infos` of `EthCoinImpl` Signed-off-by: ozkanonur * fix pr notes Signed-off-by: ozkanonur * fix pr reviews Signed-off-by: ozkanonur * fix pr reviews Signed-off-by: ozkanonur * create `Web3Transport::single_node` fn Signed-off-by: ozkanonur * update `start_history_background_fetching` return type Signed-off-by: ozkanonur * Revert "update `start_history_background_fetching` return type" This reverts commit 5975d6a94b70fabac9875ed00cbebc78ccca99c9. * remove `tx_history` for eth v2 activation Signed-off-by: Onur Özkan * fix Signed-off-by: Onur Özkan --- Cargo.lock | 2 + mm2src/coins/eth.rs | 181 ++++++++++--- mm2src/coins/eth/eth_tests.rs | 52 ++-- mm2src/coins/eth/eth_wasm_tests.rs | 3 +- mm2src/coins/eth/v2_activation.rs | 246 ++++++++++++++++++ mm2src/coins/eth/web3_transport.rs | 191 ++++++++++---- mm2src/coins/lp_coins.rs | 6 +- mm2src/coins/solana.rs | 3 + mm2src/coins_activation/Cargo.toml | 1 + .../src/erc20_token_activation.rs | 107 ++++++++ .../src/eth_with_token_activation.rs | 235 +++++++++++++++++ mm2src/coins_activation/src/lib.rs | 2 + .../src/platform_coin_with_tokens.rs | 11 +- mm2src/coins_activation/src/token.rs | 2 + mm2src/common/common.rs | 12 + .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 3 + mm2src/mm2_net/Cargo.toml | 1 + mm2src/mm2_net/src/transport.rs | 17 ++ 18 files changed, 977 insertions(+), 98 deletions(-) create mode 100644 mm2src/coins/eth/v2_activation.rs create mode 100644 mm2src/coins_activation/src/erc20_token_activation.rs create mode 100644 mm2src/coins_activation/src/eth_with_token_activation.rs diff --git a/Cargo.lock b/Cargo.lock index ca6200024e..4d048d3968 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1125,6 +1125,7 @@ dependencies = [ "common", "crypto", "derive_more", + "ethereum-types 0.4.2", "futures 0.3.15", "hex 0.4.2", "mm2_core", @@ -4255,6 +4256,7 @@ dependencies = [ "cfg-if 1.0.0", "common", "derive_more", + "ethkey", "futures 0.3.15", "gstuff", "http 0.2.7", diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 1a2f6dc78e..619df9b7a5 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -24,7 +24,7 @@ use async_trait::async_trait; use bitcrypto::{keccak256, sha256}; use common::executor::Timer; use common::log::{error, info, warn}; -use common::{now_ms, small_rng, DEX_FEE_ADDR_RAW_PUBKEY}; +use common::{get_utc_timestamp, now_ms, small_rng, DEX_FEE_ADDR_RAW_PUBKEY}; use crypto::privkey::key_pair_from_secret; use derive_more::Display; use ethabi::{Contract, Token}; @@ -39,7 +39,7 @@ use futures01::Future; use http::StatusCode; use mm2_core::mm_ctx::{MmArc, MmWeak}; use mm2_err_handle::prelude::*; -use mm2_net::transport::{slurp_url, SlurpError}; +use mm2_net::transport::{slurp_url, GuiAuthValidation, GuiAuthValidationGenerator, SlurpError}; use mm2_number::{BigDecimal, MmNumber}; #[cfg(test)] use mocktopus::macros::*; use rand::seq::SliceRandom; @@ -57,12 +57,12 @@ use std::sync::{Arc, Mutex}; use web3::types::{Action as TraceAction, BlockId, BlockNumber, Bytes, CallRequest, FilterBuilder, Log, Trace, TraceFilterBuilder, Transaction as Web3Transaction, TransactionId}; use web3::{self, Web3}; -use web3_transport::{EthFeeHistoryNamespace, Web3Transport}; +use web3_transport::{EthFeeHistoryNamespace, Web3Transport, Web3TransportNode}; -use super::{AsyncMutex, BalanceError, BalanceFut, CoinBalance, CoinProtocol, CoinTransportMetrics, CoinsContext, - FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MarketCoinOps, MmCoin, NegotiateSwapContractAddrErr, - NumConversError, NumConversResult, RawTransactionError, RawTransactionFut, RawTransactionRequest, - RawTransactionRes, RawTransactionResult, RpcClientType, RpcTransportEventHandler, +use super::{coin_conf, AsyncMutex, BalanceError, BalanceFut, CoinBalance, CoinProtocol, CoinTransportMetrics, + CoinsContext, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MarketCoinOps, MmCoin, + NegotiateSwapContractAddrErr, NumConversError, NumConversResult, RawTransactionError, RawTransactionFut, + RawTransactionRequest, RawTransactionRes, RawTransactionResult, RpcClientType, RpcTransportEventHandler, RpcTransportEventHandlerShared, SearchForSwapTxSpendInput, SignatureError, SignatureResult, SwapOps, TradeFee, TradePreimageError, TradePreimageFut, TradePreimageResult, TradePreimageValue, Transaction, TransactionDetails, TransactionEnum, TransactionErr, TransactionFut, UnexpectedDerivationMethod, @@ -75,6 +75,8 @@ pub use rlp; #[cfg(target_arch = "wasm32")] mod eth_wasm_tests; mod web3_transport; +#[path = "eth/v2_activation.rs"] pub mod v2_activation; + /// https://github.com/artemii235/etomic-swap/blob/master/contracts/EtomicSwap.sol /// Dev chain (195.201.0.6:8565) contract address: 0xa09ad3cd7e96586ebd05a2607ee56b56fb2db8fd /// Ropsten: https://ropsten.etherscan.io/address/0x7bc1bbdd6a0a722fc9bffc49c921b685ecb84b94 @@ -95,6 +97,10 @@ const GAS_PRICE_PERCENT: u64 = 10; const BASE_BLOCK_FEE_DIFF_PCT: u64 = 13; const DEFAULT_LOGS_BLOCK_RANGE: u64 = 1000; +const DEFAULT_REQUIRED_CONFIRMATIONS: u8 = 1; + +const ETH_DECIMALS: u8 = 18; + /// Take into account that the dynamic fee may increase by 3% during the swap. const GAS_PRICE_APPROXIMATION_PERCENT_ON_START_SWAP: u64 = 3; /// Take into account that the dynamic fee may increase at each of the following stages: @@ -107,6 +113,9 @@ const GAS_PRICE_APPROXIMATION_PERCENT_ON_ORDER_ISSUE: u64 = 5; /// - it may increase by 3% during the swap. const GAS_PRICE_APPROXIMATION_PERCENT_ON_TRADE_PREIMAGE: u64 = 7; +/// Lifetime of generated signed message for gui-auth requests +const GUI_AUTH_SIGNED_MESSAGE_LIFETIME_SEC: i64 = 90; + lazy_static! { pub static ref SWAP_CONTRACT: Contract = Contract::load(SWAP_CONTRACT_ABI.as_bytes()).unwrap(); pub static ref ERC20_CONTRACT: Contract = Contract::load(ERC20_ABI.as_bytes()).unwrap(); @@ -270,7 +279,7 @@ struct SavedErc20Events { } #[derive(Debug, PartialEq, Eq)] -enum EthCoinType { +pub enum EthCoinType { /// Ethereum itself or it's forks: ETC/others Eth, /// ERC20 token with smart contract address @@ -299,11 +308,12 @@ pub struct EthCoinImpl { required_confirmations: AtomicU64, /// Coin needs access to the context in order to reuse the logging and shutdown facilities. /// Using a weak reference by default in order to avoid circular references and leaks. - ctx: MmWeak, + pub ctx: MmWeak, chain_id: Option, /// the block range used for eth_getLogs logs_block_range: u64, nonce_lock: Arc>, + erc20_tokens_infos: Arc>>, } #[derive(Clone, Debug)] @@ -312,6 +322,12 @@ pub struct Web3Instance { is_parity: bool, } +#[derive(Clone, Debug)] +pub struct Erc20TokenInfo { + pub token_address: Address, + pub decimals: u8, +} + #[derive(Deserialize, Serialize)] #[serde(tag = "format")] pub enum EthAddressFormat { @@ -530,6 +546,25 @@ impl EthCoinImpl { pub fn address_from_str(&self, address: &str) -> Result { Ok(try_s!(valid_addr_from_str(address))) } + + pub fn erc20_token_address(&self) -> Option
{ + match self.coin_type { + EthCoinType::Erc20 { token_addr, .. } => Some(token_addr), + EthCoinType::Eth => None, + } + } + + pub fn add_erc_token_info(&self, ticker: String, info: Erc20TokenInfo) { + self.erc20_tokens_infos.lock().unwrap().insert(ticker, info); + } + + /// WARNING + /// Be very careful using this function since it returns dereferenced clone + /// of value behind the MutexGuard and makes it non-thread-safe. + pub fn get_erc_tokens_infos(&self) -> HashMap { + let guard = self.erc20_tokens_infos.lock().unwrap(); + (*guard).clone() + } } async fn get_raw_transaction_impl(coin: EthCoin, req: RawTransactionRequest) -> RawTransactionResult { @@ -1085,12 +1120,16 @@ impl MarketCoinOps for EthCoin { fn my_address(&self) -> Result { Ok(checksum_address(&format!("{:#02x}", self.my_address))) } - fn get_public_key(&self) -> Result> { unimplemented!() } + fn get_public_key(&self) -> Result> { + let uncompressed_without_prefix = hex::encode(self.key_pair.public()); + Ok(format!("04{}", uncompressed_without_prefix)) + } /// Hash message for signature using Ethereum's message signing format. /// keccak256(PREFIX_LENGTH + PREFIX + MESSAGE_LENGTH + MESSAGE) fn sign_message_hash(&self, message: &str) -> Option<[u8; 32]> { let message_prefix = self.sign_message_prefix.as_ref()?; + let mut stream = Stream::new(); let prefix_len = CompactInteger::from(message_prefix.len()); prefix_len.serialize(&mut stream); @@ -1134,7 +1173,7 @@ impl MarketCoinOps for EthCoin { fn base_coin_balance(&self) -> BalanceFut { Box::new( self.eth_balance() - .and_then(move |result| Ok(u256_to_big_decimal(result, 18)?)), + .and_then(move |result| Ok(u256_to_big_decimal(result, ETH_DECIMALS)?)), ) } @@ -1722,7 +1761,7 @@ impl EthCoin { None => None, }; - let total_amount: BigDecimal = u256_to_big_decimal(call_data.value, 18).unwrap(); + let total_amount: BigDecimal = u256_to_big_decimal(call_data.value, ETH_DECIMALS).unwrap(); let mut received_by_me = 0.into(); let mut spent_by_me = 0.into(); @@ -2464,6 +2503,41 @@ impl EthCoin { Box::new(fut.boxed().compat()) } + pub async fn get_tokens_balance_list(&self) -> Result, MmError> { + let coin = self.clone(); + let mut token_balances = HashMap::new(); + for (token_ticker, info) in self.get_erc_tokens_infos().iter() { + let balance_as_u256 = coin.get_token_balance_by_address(info.token_address).await?; + let balance_as_big_decimal = u256_to_big_decimal(balance_as_u256, info.decimals)?; + let balance = CoinBalance { + spendable: balance_as_big_decimal, + unspendable: BigDecimal::from(0), + }; + token_balances.insert(token_ticker.clone(), balance); + } + + Ok(token_balances) + } + + async fn get_token_balance_by_address(&self, token_address: Address) -> Result> { + let coin = self.clone(); + let function = ERC20_CONTRACT.function("balanceOf")?; + let data = function.encode_input(&[Token::Address(coin.my_address)])?; + let res = coin + .call_request(token_address, None, Some(data.into())) + .compat() + .await?; + let decoded = function.decode_output(&res.0)?; + + match decoded[0] { + Token::Uint(number) => Ok(number), + _ => { + let error = format!("Expected U256 as balanceOf result but got {:?}", decoded); + MmError::err(BalanceError::InvalidResponse(error)) + }, + } + } + /// Estimates how much gas is necessary to allow the contract call to complete. /// `contract_addr` can be a ERC20 token address or any other contract address. /// @@ -2937,8 +3011,8 @@ impl EthTxFeeDetails { fn new(gas: U256, gas_price: U256, coin: &str) -> NumConversResult { let total_fee = gas * gas_price; // Fees are always paid in ETH, can use 18 decimals by default - let total_fee = u256_to_big_decimal(total_fee, 18)?; - let gas_price = u256_to_big_decimal(gas_price, 18)?; + let total_fee = u256_to_big_decimal(total_fee, ETH_DECIMALS)?; + let gas_price = u256_to_big_decimal(gas_price, ETH_DECIMALS)?; Ok(EthTxFeeDetails { coin: coin.to_owned(), @@ -3020,7 +3094,7 @@ impl MmCoin for EthCoin { }; Ok(TradeFee { coin: fee_coin.into(), - amount: try_s!(u256_to_big_decimal(fee, 18)).into(), + amount: try_s!(u256_to_big_decimal(fee, ETH_DECIMALS)).into(), paid_from_trading_vol: false, }) }), @@ -3068,7 +3142,7 @@ impl MmCoin for EthCoin { }; let total_fee = gas_limit * gas_price; - let amount = u256_to_big_decimal(total_fee, 18)?; + let amount = u256_to_big_decimal(total_fee, ETH_DECIMALS)?; let fee_coin = match &self.coin_type { EthCoinType::Eth => &self.ticker, EthCoinType::Erc20 { platform, .. } => platform, @@ -3086,7 +3160,7 @@ impl MmCoin for EthCoin { let gas_price = coin.get_gas_price().compat().await?; let gas_price = increase_gas_price_by_stage(gas_price, &stage); let total_fee = gas_price * U256::from(150_000); - let amount = u256_to_big_decimal(total_fee, 18)?; + let amount = u256_to_big_decimal(total_fee, ETH_DECIMALS)?; let fee_coin = match &coin.coin_type { EthCoinType::Eth => &coin.ticker, EthCoinType::Erc20 { platform, .. } => platform, @@ -3136,7 +3210,7 @@ impl MmCoin for EthCoin { // Ideally we should determine the case when we have the insufficient balance and return `TradePreimageError::NotSufficientBalance` error. let gas_limit = self.estimate_gas(estimate_gas_req).compat().await?; let total_fee = gas_limit * gas_price; - let amount = u256_to_big_decimal(total_fee, 18)?; + let amount = u256_to_big_decimal(total_fee, ETH_DECIMALS)?; Ok(TradeFee { coin: fee_coin.into(), amount: amount.into(), @@ -3185,6 +3259,44 @@ impl TryToAddress for Option { } } +pub trait GuiAuthMessages { + fn gui_auth_sign_message_hash(message: String) -> Option<[u8; 32]>; + fn generate_gui_auth_signed_validation(generator: GuiAuthValidationGenerator) + -> SignatureResult; +} + +impl GuiAuthMessages for EthCoin { + fn gui_auth_sign_message_hash(message: String) -> Option<[u8; 32]> { + let message_prefix = "atomicDEX Auth Ethereum Signed Message:\n"; + let prefix_len = CompactInteger::from(message_prefix.len()); + + let mut stream = Stream::new(); + prefix_len.serialize(&mut stream); + stream.append_slice(message_prefix.as_bytes()); + stream.append_slice(message.len().to_string().as_bytes()); + stream.append_slice(message.as_bytes()); + + Some(keccak256(&stream.out()).take()) + } + + fn generate_gui_auth_signed_validation( + generator: GuiAuthValidationGenerator, + ) -> SignatureResult { + let timestamp_message = get_utc_timestamp() + GUI_AUTH_SIGNED_MESSAGE_LIFETIME_SEC; + + let message_hash = + EthCoin::gui_auth_sign_message_hash(timestamp_message.to_string()).ok_or(SignatureError::PrefixNotFound)?; + let signature = sign(&generator.secret, &H256::from(message_hash))?; + + Ok(GuiAuthValidation { + coin_ticker: generator.coin_ticker, + address: generator.address, + timestamp_message, + signature: format!("0x{}", signature), + }) + } +} + pub fn addr_from_raw_pubkey(pubkey: &[u8]) -> Result { let pubkey = try_s!(PublicKey::from_slice(pubkey).map_err(|e| ERRL!("{:?}", e))); let eth_public = Public::from(&pubkey.serialize_uncompressed()[1..65]); @@ -3273,7 +3385,7 @@ pub struct GasStationData { /// Using tagged representation to allow adding variants with coefficients, percentage, etc in the future. #[derive(Clone, Copy, Debug, Deserialize)] #[serde(tag = "policy", content = "additional_data")] -enum GasStationPricePolicy { +pub enum GasStationPricePolicy { /// Use mean between average and fast values, default and recommended to use on ETH mainnet due to /// gas price big spikes. MeanAverageFast, @@ -3331,7 +3443,7 @@ async fn get_token_decimals(web3: &Web3, token_addr: Address) -> Ok(decimals as u8) } -fn valid_addr_from_str(addr_str: &str) -> Result { +pub fn valid_addr_from_str(addr_str: &str) -> Result { let addr = try_s!(addr_from_str(addr_str)); if !is_valid_checksum_addr(addr_str) { return ERR!("Invalid address checksum"); @@ -3370,6 +3482,15 @@ pub async fn eth_coin_from_conf_and_request( let mut rng = small_rng(); urls.as_mut_slice().shuffle(&mut rng); + let mut nodes = vec![]; + for url in urls.iter() { + nodes.push(Web3TransportNode { + uri: try_s!(url.parse()), + gui_auth: false, + }); + } + drop_mutability!(nodes); + let swap_contract_address: Address = try_s!(json::from_value(req["swap_contract_address"].clone())); if swap_contract_address == Address::default() { return ERR!("swap_contract_address can't be zero address"); @@ -3387,16 +3508,13 @@ pub async fn eth_coin_from_conf_and_request( let mut web3_instances = vec![]; let event_handlers = rpc_event_handlers_for_eth_transport(ctx, ticker.to_string()); - for url in urls.iter() { - let transport = try_s!(Web3Transport::with_event_handlers( - vec![url.clone()], - event_handlers.clone() - )); + for node in nodes.iter() { + let transport = Web3Transport::with_event_handlers(vec![node.clone()], event_handlers.clone()); let web3 = Web3::new(transport); let version = match web3.web3().client_version().compat().await { Ok(v) => v, Err(e) => { - error!("Couldn't get client version for url {}: {}", url, e); + error!("Couldn't get client version for url {}: {}", node.uri, e); continue; }, }; @@ -3410,11 +3528,11 @@ pub async fn eth_coin_from_conf_and_request( return ERR!("Failed to get client version for all urls"); } - let transport = try_s!(Web3Transport::with_event_handlers(urls, event_handlers)); + let transport = Web3Transport::with_event_handlers(nodes, event_handlers); let web3 = Web3::new(transport); let (coin_type, decimals) = match protocol { - CoinProtocol::ETH => (EthCoinType::Eth, 18), + CoinProtocol::ETH => (EthCoinType::Eth, ETH_DECIMALS), CoinProtocol::ERC20 { platform, contract_address, @@ -3432,7 +3550,11 @@ pub async fn eth_coin_from_conf_and_request( // param from request should override the config let required_confirmations = req["required_confirmations"] .as_u64() - .unwrap_or_else(|| conf["required_confirmations"].as_u64().unwrap_or(1)) + .unwrap_or_else(|| { + conf["required_confirmations"] + .as_u64() + .unwrap_or(DEFAULT_REQUIRED_CONFIRMATIONS as u64) + }) .into(); if req["requires_notarization"].as_bool().is_some() { @@ -3480,6 +3602,7 @@ pub async fn eth_coin_from_conf_and_request( chain_id: conf["chain_id"].as_u64(), logs_block_range: conf["logs_block_range"].as_u64().unwrap_or(DEFAULT_LOGS_BLOCK_RANGE), nonce_lock, + erc20_tokens_infos: Default::default(), }; Ok(EthCoin(Arc::new(coin))) } diff --git a/mm2src/coins/eth/eth_tests.rs b/mm2src/coins/eth/eth_tests.rs index 46070326c8..b3ba38129c 100644 --- a/mm2src/coins/eth/eth_tests.rs +++ b/mm2src/coins/eth/eth_tests.rs @@ -26,7 +26,17 @@ fn eth_coin_for_test( &hex::decode("809465b17d0a4ddb3e4c69e8f23c2cabad868f51f8bed5c765ad1d6516c3306f").unwrap(), ) .unwrap(); - let transport = Web3Transport::new(urls).unwrap(); + + let mut nodes = vec![]; + for url in urls.iter() { + nodes.push(Web3TransportNode { + uri: url.parse().unwrap(), + gui_auth: false, + }); + } + drop_mutability!(nodes); + + let transport = Web3Transport::new(nodes); let web3 = Web3::new(transport); let conf = json!({ "coins":[ @@ -63,6 +73,7 @@ fn eth_coin_for_test( chain_id: None, logs_block_range: DEFAULT_LOGS_BLOCK_RANGE, nonce_lock: new_nonce_lock(), + erc20_tokens_infos: Default::default(), })); (ctx, eth_coin) } @@ -202,7 +213,7 @@ fn send_and_refund_erc20_payment() { &hex::decode("809465b17d0a4ddb3e4c69e8f23c2cabad868f51f8bed5c765ad1d6516c3306f").unwrap(), ) .unwrap(); - let transport = Web3Transport::new(vec!["http://195.201.0.6:8545".into()]).unwrap(); + let transport = Web3Transport::single_node("http://195.201.0.6:8545", false); let web3 = Web3::new(transport); let ctx = MmCtxBuilder::new().into_mm_arc(); let coin = EthCoin(Arc::new(EthCoinImpl { @@ -231,6 +242,7 @@ fn send_and_refund_erc20_payment() { chain_id: None, logs_block_range: DEFAULT_LOGS_BLOCK_RANGE, nonce_lock: new_nonce_lock(), + erc20_tokens_infos: Default::default(), })); let payment = coin @@ -272,7 +284,7 @@ fn send_and_refund_eth_payment() { &hex::decode("809465b17d0a4ddb3e4c69e8f23c2cabad868f51f8bed5c765ad1d6516c3306f").unwrap(), ) .unwrap(); - let transport = Web3Transport::new(vec!["http://195.201.0.6:8545".into()]).unwrap(); + let transport = Web3Transport::single_node("http://195.201.0.6:8545", false); let web3 = Web3::new(transport); let ctx = MmCtxBuilder::new().into_mm_arc(); let coin = EthCoin(Arc::new(EthCoinImpl { @@ -298,6 +310,7 @@ fn send_and_refund_eth_payment() { chain_id: None, logs_block_range: DEFAULT_LOGS_BLOCK_RANGE, nonce_lock: new_nonce_lock(), + erc20_tokens_infos: Default::default(), })); let payment = coin @@ -338,13 +351,11 @@ fn test_nonce_several_urls() { &hex::decode("809465b17d0a4ddb3e4c69e8f23c2cabad868f51f8bed5c765ad1d6516c3306f").unwrap(), ) .unwrap(); - let infura_transport = Web3Transport::new(vec![ - "https://ropsten.infura.io/v3/c01c1b4cf66642528547624e1d6d9d6b".into() - ]) - .unwrap(); - let linkpool_transport = Web3Transport::new(vec!["https://ropsten-rpc.linkpool.io".into()]).unwrap(); + let infura_transport = + Web3Transport::single_node("https://ropsten.infura.io/v3/c01c1b4cf66642528547624e1d6d9d6b", false); + let linkpool_transport = Web3Transport::single_node("https://ropsten-rpc.linkpool.io", false); // get nonce must succeed if some nodes are down at the moment for some reason - let failing_transport = Web3Transport::new(vec!["http://195.201.0.6:8989".into()]).unwrap(); + let failing_transport = Web3Transport::single_node("http://195.201.0.6:8989", false); let web3_infura = Web3::new(infura_transport); let web3_linkpool = Web3::new(linkpool_transport); @@ -384,6 +395,7 @@ fn test_nonce_several_urls() { chain_id: None, logs_block_range: DEFAULT_LOGS_BLOCK_RANGE, nonce_lock: new_nonce_lock(), + erc20_tokens_infos: Default::default(), })); log!("My address {:?}", coin.my_address); @@ -406,7 +418,7 @@ fn test_wait_for_payment_spend_timeout() { &hex::decode("809465b17d0a4ddb3e4c69e8f23c2cabad868f51f8bed5c765ad1d6516c3306f").unwrap(), ) .unwrap(); - let transport = Web3Transport::new(vec!["http://195.201.0.6:8555".into()]).unwrap(); + let transport = Web3Transport::single_node("http://195.201.0.6:8555", false); let web3 = Web3::new(transport); let ctx = MmCtxBuilder::new().into_mm_arc(); @@ -433,6 +445,7 @@ fn test_wait_for_payment_spend_timeout() { chain_id: None, logs_block_range: DEFAULT_LOGS_BLOCK_RANGE, nonce_lock: new_nonce_lock(), + erc20_tokens_infos: Default::default(), }; let coin = EthCoin(Arc::new(coin)); @@ -464,10 +477,7 @@ fn test_search_for_swap_tx_spend_was_spent() { &hex::decode("809465b17d0a4ddb3e4c69e8f23c2cabad868f51f8bed5c765ad1d6516c3306f").unwrap(), ) .unwrap(); - let transport = Web3Transport::new(vec![ - "https://ropsten.infura.io/v3/c01c1b4cf66642528547624e1d6d9d6b".into() - ]) - .unwrap(); + let transport = Web3Transport::single_node("https://ropsten.infura.io/v3/c01c1b4cf66642528547624e1d6d9d6b", false); let web3 = Web3::new(transport); let ctx = MmCtxBuilder::new().into_mm_arc(); @@ -495,6 +505,7 @@ fn test_search_for_swap_tx_spend_was_spent() { chain_id: None, logs_block_range: DEFAULT_LOGS_BLOCK_RANGE, nonce_lock: new_nonce_lock(), + erc20_tokens_infos: Default::default(), })); // raw transaction bytes of https://ropsten.etherscan.io/tx/0xb1c987e2ac79581bb8718267b5cb49a18274890494299239d1d0dfdb58d6d76a @@ -570,10 +581,7 @@ fn test_search_for_swap_tx_spend_was_refunded() { &hex::decode("809465b17d0a4ddb3e4c69e8f23c2cabad868f51f8bed5c765ad1d6516c3306f").unwrap(), ) .unwrap(); - let transport = Web3Transport::new(vec![ - "https://ropsten.infura.io/v3/c01c1b4cf66642528547624e1d6d9d6b".into() - ]) - .unwrap(); + let transport = Web3Transport::single_node("https://ropsten.infura.io/v3/c01c1b4cf66642528547624e1d6d9d6b", false); let web3 = Web3::new(transport); let ctx = MmCtxBuilder::new().into_mm_arc(); @@ -604,6 +612,7 @@ fn test_search_for_swap_tx_spend_was_refunded() { chain_id: None, logs_block_range: DEFAULT_LOGS_BLOCK_RANGE, nonce_lock: new_nonce_lock(), + erc20_tokens_infos: Default::default(), })); // raw transaction bytes of https://ropsten.etherscan.io/tx/0xe18bbca69dea9a4624e1f5b0b2021d5fe4c8daa03f36084a8ba011b08e5cd938 @@ -1263,7 +1272,7 @@ fn test_message_hash() { &hex::decode("809465b17d0a4ddb3e4c69e8f23c2cabad868f51f8bed5c765ad1d6516c3306f").unwrap(), ) .unwrap(); - let transport = Web3Transport::new(vec!["http://195.201.0.6:8545".into()]).unwrap(); + let transport = Web3Transport::single_node("http://195.201.0.6:8545", false); let web3 = Web3::new(transport); let ctx = MmCtxBuilder::new().into_mm_arc(); let coin = EthCoin(Arc::new(EthCoinImpl { @@ -1289,6 +1298,7 @@ fn test_message_hash() { chain_id: None, logs_block_range: DEFAULT_LOGS_BLOCK_RANGE, nonce_lock: new_nonce_lock(), + erc20_tokens_infos: Default::default(), })); let message_hash = coin.sign_message_hash("test").unwrap(); @@ -1304,7 +1314,8 @@ fn test_sign_verify_message() { &hex::decode("809465b17d0a4ddb3e4c69e8f23c2cabad868f51f8bed5c765ad1d6516c3306f").unwrap(), ) .unwrap(); - let transport = Web3Transport::new(vec!["http://195.201.0.6:8545".into()]).unwrap(); + let transport = Web3Transport::single_node("http://195.201.0.6:8545", false); + let web3 = Web3::new(transport); let ctx = MmCtxBuilder::new().into_mm_arc(); let coin = EthCoin(Arc::new(EthCoinImpl { @@ -1330,6 +1341,7 @@ fn test_sign_verify_message() { chain_id: None, logs_block_range: DEFAULT_LOGS_BLOCK_RANGE, nonce_lock: new_nonce_lock(), + erc20_tokens_infos: Default::default(), })); let message = "test"; diff --git a/mm2src/coins/eth/eth_wasm_tests.rs b/mm2src/coins/eth/eth_wasm_tests.rs index ca1f3f8362..55d02ce462 100644 --- a/mm2src/coins/eth/eth_wasm_tests.rs +++ b/mm2src/coins/eth/eth_wasm_tests.rs @@ -20,7 +20,7 @@ async fn test_send() { &hex::decode("809465b17d0a4ddb3e4c69e8f23c2cabad868f51f8bed5c765ad1d6516c3306f").unwrap(), ) .unwrap(); - let transport = Web3Transport::new(vec!["http://195.201.0.6:8565".into()]).unwrap(); + let transport = Web3Transport::single_node("http://195.201.0.6:8565", false); let web3 = Web3::new(transport); let ctx = MmCtxBuilder::new().into_mm_arc(); let coin = EthCoin(Arc::new(EthCoinImpl { @@ -46,6 +46,7 @@ async fn test_send() { chain_id: None, logs_block_range: DEFAULT_LOGS_BLOCK_RANGE, nonce_lock: new_nonce_lock(), + erc20_tokens_infos: Default::default(), })); let tx = coin .send_maker_payment( diff --git a/mm2src/coins/eth/v2_activation.rs b/mm2src/coins/eth/v2_activation.rs new file mode 100644 index 0000000000..0f9c59ceea --- /dev/null +++ b/mm2src/coins/eth/v2_activation.rs @@ -0,0 +1,246 @@ +use super::*; + +#[derive(Display, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum EthActivationV2Error { + InvalidPayload(String), + InvalidSwapContractAddr(String), + InvalidFallbackSwapContract(String), + #[display(fmt = "Platform coin {} activation failed. {}", ticker, error)] + ActivationFailed { + ticker: String, + error: String, + }, + CouldNotFetchBalance(String), + UnreachableNodes(String), + #[display(fmt = "Enable request for ETH coin must have at least 1 node")] + AtLeastOneNodeRequired, + InternalError(String), +} + +#[derive(Clone, Deserialize)] +pub struct EthActivationV2Request { + pub nodes: Vec, + pub swap_contract_address: Address, + pub fallback_swap_contract: Option
, + pub gas_station_url: Option, + pub gas_station_decimals: Option, + #[serde(default)] + pub gas_station_policy: GasStationPricePolicy, + pub mm2: Option, + pub required_confirmations: Option, +} + +#[derive(Clone, Deserialize)] +pub struct EthNode { + pub url: String, + pub gui_auth: bool, +} + +#[derive(Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum Erc20TokenActivationError { + InternalError(String), + CouldNotFetchBalance(String), +} + +#[derive(Clone, Deserialize)] +pub struct Erc20TokenActivationRequest { + pub required_confirmations: Option, +} + +pub struct Erc20Protocol { + pub platform: String, + pub token_addr: Address, +} + +#[cfg_attr(test, mockable)] +impl EthCoin { + pub async fn initialize_erc20_token( + &self, + activation_params: Erc20TokenActivationRequest, + protocol: Erc20Protocol, + ticker: String, + ) -> Result> { + // TODO + // Check if ctx is required. + // Remove it to avoid circular references if possible + let ctx = MmArc::from_weak(&self.ctx) + .ok_or_else(|| String::from("No context")) + .map_err(Erc20TokenActivationError::InternalError)?; + + let conf = coin_conf(&ctx, &ticker); + + let decimals = match conf["decimals"].as_u64() { + None | Some(0) => get_token_decimals(&self.web3, protocol.token_addr) + .await + .map_err(Erc20TokenActivationError::InternalError)?, + Some(d) => d as u8, + }; + + let required_confirmations = activation_params + .required_confirmations + .unwrap_or_else(|| conf["required_confirmations"].as_u64().unwrap_or(1)) + .into(); + + let token = EthCoinImpl { + key_pair: self.key_pair.clone(), + my_address: self.my_address, + coin_type: EthCoinType::Erc20 { + platform: protocol.platform, + token_addr: protocol.token_addr, + }, + sign_message_prefix: self.sign_message_prefix.clone(), + swap_contract_address: self.swap_contract_address, + fallback_swap_contract: self.fallback_swap_contract, + decimals, + ticker, + gas_station_url: self.gas_station_url.clone(), + gas_station_decimals: self.gas_station_decimals, + gas_station_policy: self.gas_station_policy, + web3: self.web3.clone(), + web3_instances: self.web3_instances.clone(), + history_sync_state: Mutex::new(self.history_sync_state.lock().unwrap().clone()), + ctx: self.ctx.clone(), + required_confirmations, + chain_id: self.chain_id, + logs_block_range: self.logs_block_range, + nonce_lock: self.nonce_lock.clone(), + erc20_tokens_infos: Default::default(), + }; + + Ok(EthCoin(Arc::new(token))) + } +} + +pub async fn eth_coin_from_conf_and_request_v2( + ctx: &MmArc, + ticker: &str, + conf: &Json, + req: EthActivationV2Request, + priv_key: &[u8], +) -> Result> { + if req.nodes.is_empty() { + return Err(EthActivationV2Error::AtLeastOneNodeRequired.into()); + } + + let mut rng = small_rng(); + let mut req = req; + req.nodes.as_mut_slice().shuffle(&mut rng); + drop_mutability!(req); + + let mut nodes = vec![]; + for node in req.nodes.iter() { + let uri = node + .url + .parse() + .map_err(|_| EthActivationV2Error::InvalidPayload(format!("{} could not be parsed.", node.url)))?; + + nodes.push(Web3TransportNode { + uri, + gui_auth: node.gui_auth, + }); + } + drop_mutability!(nodes); + + if req.swap_contract_address == Address::default() { + return Err(EthActivationV2Error::InvalidSwapContractAddr( + "swap_contract_address can't be zero address".to_string(), + ) + .into()); + } + + if let Some(fallback) = req.fallback_swap_contract { + if fallback == Address::default() { + return Err(EthActivationV2Error::InvalidFallbackSwapContract( + "fallback_swap_contract can't be zero address".to_string(), + ) + .into()); + } + } + + let key_pair: KeyPair = + KeyPair::from_secret_slice(priv_key).map_err(|e| EthActivationV2Error::InternalError(e.to_string()))?; + let my_address = checksum_address(&format!("{:02x}", key_pair.address())); + + let mut web3_instances = vec![]; + let event_handlers = rpc_event_handlers_for_eth_transport(ctx, ticker.to_string()); + for node in &nodes { + let mut transport = Web3Transport::with_event_handlers(vec![node.clone()], event_handlers.clone()); + transport.gui_auth_validation_generator = Some(GuiAuthValidationGenerator { + coin_ticker: ticker.to_string(), + secret: key_pair.secret().clone(), + address: my_address.clone(), + }); + drop_mutability!(transport); + + let web3 = Web3::new(transport); + let version = match web3.web3().client_version().compat().await { + Ok(v) => v, + Err(e) => { + error!("Couldn't get client version for url {}: {}", node.uri, e); + continue; + }, + }; + web3_instances.push(Web3Instance { + web3, + is_parity: version.contains("Parity") || version.contains("parity"), + }) + } + + if web3_instances.is_empty() { + return Err( + EthActivationV2Error::UnreachableNodes("Failed to get client version for all nodes".to_string()).into(), + ); + } + + let mut transport = Web3Transport::with_event_handlers(nodes, event_handlers); + transport.gui_auth_validation_generator = Some(GuiAuthValidationGenerator { + coin_ticker: ticker.to_string(), + secret: key_pair.secret().clone(), + address: my_address, + }); + drop_mutability!(transport); + + let web3 = Web3::new(transport); + + // param from request should override the config + let required_confirmations = req + .required_confirmations + .unwrap_or_else(|| { + conf["required_confirmations"] + .as_u64() + .unwrap_or(DEFAULT_REQUIRED_CONFIRMATIONS as u64) + }) + .into(); + + let sign_message_prefix: Option = json::from_value(conf["sign_message_prefix"].clone()).ok(); + + let mut map = NONCE_LOCK.lock().unwrap(); + let nonce_lock = map.entry(ticker.to_string()).or_insert_with(new_nonce_lock).clone(); + + let coin = EthCoinImpl { + key_pair: key_pair.clone(), + my_address: key_pair.address(), + coin_type: EthCoinType::Eth, + sign_message_prefix, + swap_contract_address: req.swap_contract_address, + fallback_swap_contract: req.fallback_swap_contract, + decimals: ETH_DECIMALS, + ticker: ticker.into(), + gas_station_url: req.gas_station_url, + gas_station_decimals: req.gas_station_decimals.unwrap_or(ETH_GAS_STATION_DECIMALS), + gas_station_policy: req.gas_station_policy, + web3, + web3_instances, + history_sync_state: Mutex::new(HistorySyncState::NotEnabled), + ctx: ctx.weak(), + required_confirmations, + chain_id: conf["chain_id"].as_u64(), + logs_block_range: conf["logs_block_range"].as_u64().unwrap_or(DEFAULT_LOGS_BLOCK_RANGE), + nonce_lock, + erc20_tokens_infos: Default::default(), + }; + + Ok(EthCoin(Arc::new(coin))) +} diff --git a/mm2src/coins/eth/web3_transport.rs b/mm2src/coins/eth/web3_transport.rs index 5b0b229a57..fb5ea2846b 100644 --- a/mm2src/coins/eth/web3_transport.rs +++ b/mm2src/coins/eth/web3_transport.rs @@ -1,8 +1,9 @@ -use super::{RpcTransportEventHandler, RpcTransportEventHandlerShared}; +use super::{EthCoin, GuiAuthMessages, RpcTransportEventHandler, RpcTransportEventHandlerShared, Web3RpcError}; #[cfg(not(target_arch = "wasm32"))] use futures::FutureExt; use futures::TryFutureExt; use futures01::{Future, Poll}; use jsonrpc_core::{Call, Response}; +use mm2_net::transport::{GuiAuthValidation, GuiAuthValidationGenerator}; use serde_json::Value as Json; #[cfg(not(target_arch = "wasm32"))] use std::ops::Deref; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -20,6 +21,13 @@ pub struct EthFeeHistoryNamespace { transport: T, } +#[derive(Serialize, Clone)] +pub struct AuthPayload<'a> { + #[serde(flatten)] + pub request: &'a Call, + pub signed_message: GuiAuthValidation, +} + impl Namespace for EthFeeHistoryNamespace { fn new(transport: T) -> Self where @@ -70,37 +78,55 @@ fn single_response>(response: T, rpc_url: &str) -> Resul #[derive(Clone, Debug)] pub struct Web3Transport { id: Arc, - uris: Vec, + nodes: Vec, event_handlers: Vec, + pub(crate) gui_auth_validation_generator: Option, +} + +#[derive(Clone, Debug)] +pub struct Web3TransportNode { + pub(crate) uri: http::Uri, + pub(crate) gui_auth: bool, } impl Web3Transport { #[allow(dead_code)] - pub fn new(urls: Vec) -> Result { - let mut uris = vec![]; - for url in urls.iter() { - uris.push(try_s!(url.parse())); - } - Ok(Web3Transport { + #[inline] + pub fn new(nodes: Vec) -> Self { + Web3Transport { id: Arc::new(AtomicUsize::new(0)), - uris, + nodes, event_handlers: Default::default(), - }) + gui_auth_validation_generator: None, + } } + #[inline] pub fn with_event_handlers( - urls: Vec, + nodes: Vec, event_handlers: Vec, - ) -> Result { - let mut uris = vec![]; - for url in urls.iter() { - uris.push(try_s!(url.parse())); - } - Ok(Web3Transport { + ) -> Self { + Web3Transport { id: Arc::new(AtomicUsize::new(0)), - uris, + nodes, event_handlers, - }) + gui_auth_validation_generator: None, + } + } + + #[allow(dead_code)] + pub fn single_node(url: &'static str, gui_auth: bool) -> Self { + let nodes = vec![Web3TransportNode { + uri: url.parse().unwrap(), + gui_auth, + }]; + + Web3Transport { + id: Arc::new(AtomicUsize::new(0)), + nodes, + event_handlers: Default::default(), + gui_auth_validation_generator: None, + } } } @@ -130,24 +156,74 @@ impl Transport for Web3Transport { #[cfg(not(target_arch = "wasm32"))] fn send(&self, _id: RequestId, request: Call) -> Self::Out { Box::new( - send_request(request, self.uris.clone(), self.event_handlers.clone()) - .boxed() - .compat(), + send_request( + request, + self.nodes.clone(), + self.event_handlers.clone(), + self.gui_auth_validation_generator.clone(), + ) + .boxed() + .compat(), ) } #[cfg(target_arch = "wasm32")] fn send(&self, _id: RequestId, request: Call) -> Self::Out { - let fut = send_request(request, self.uris.clone(), self.event_handlers.clone()); + let fut = send_request( + request, + self.nodes.clone(), + self.event_handlers.clone(), + self.gui_auth_validation_generator.clone(), + ); Box::new(SendFuture(Box::pin(fut).compat())) } } +/// Generates a signed message and inserts it into request +/// payload if gui_auth is activated. Returns false on errors. +fn handle_gui_auth_payload_if_activated( + gui_auth_validation_generator: &Option, + node: &Web3TransportNode, + request: &Call, +) -> Result, Web3RpcError> { + if !node.gui_auth { + return Ok(None); + } + + let generator = match gui_auth_validation_generator.clone() { + Some(gen) => gen, + None => { + return Err(Web3RpcError::Internal(format!( + "GuiAuthValidationGenerator is not provided for {:?} node", + node + ))); + }, + }; + + let signed_message = match EthCoin::generate_gui_auth_signed_validation(generator) { + Ok(t) => t, + Err(e) => { + return Err(Web3RpcError::Internal(format!( + "GuiAuth signed message generation failed for {:?} node, error: {:?}", + node, e + ))); + }, + }; + + let auth_request = AuthPayload { + request, + signed_message, + }; + + Ok(Some(to_string(&auth_request))) +} + #[cfg(not(target_arch = "wasm32"))] async fn send_request( request: Call, - uris: Vec, + nodes: Vec, event_handlers: Vec, + gui_auth_validation_generator: Option, ) -> Result { use common::executor::Timer; use common::log::warn; @@ -159,13 +235,25 @@ async fn send_request( const REQUEST_TIMEOUT_S: f64 = 60.; let mut errors = Vec::new(); - for uri in uris.iter() { - let request = to_string(&request); - event_handlers.on_outgoing_request(request.as_bytes()); - let mut req = http::Request::new(request.clone().into_bytes()); + let serialized_request = to_string(&request); + + for node in nodes.iter() { + let serialized_request = + match handle_gui_auth_payload_if_activated(&gui_auth_validation_generator, node, &request) { + Ok(Some(r)) => r, + Ok(None) => serialized_request.clone(), + Err(e) => { + errors.push(e); + continue; + }, + }; + + event_handlers.on_outgoing_request(serialized_request.as_bytes()); + + let mut req = http::Request::new(serialized_request.clone().into_bytes()); *req.method_mut() = http::Method::POST; - *req.uri_mut() = uri.clone(); + *req.uri_mut() = node.uri.clone(); req.headers_mut() .insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/json")); let timeout = Timer::sleep(REQUEST_TIMEOUT_S); @@ -174,9 +262,12 @@ async fn send_request( let res = match rc { Either::Left((r, _t)) => r, Either::Right((_t, _r)) => { - let error = ERRL!("Error requesting '{}': {}s timeout expired", uri, REQUEST_TIMEOUT_S); + let error = format!( + "Error requesting '{}': {}s timeout expired", + node.uri, REQUEST_TIMEOUT_S + ); warn!("{}", error); - errors.push(error); + errors.push(Web3RpcError::Transport(error)); continue; }, }; @@ -184,7 +275,7 @@ async fn send_request( let (status, _headers, body) = match res { Ok(r) => r, Err(err) => { - errors.push(err.to_string()); + errors.push(Web3RpcError::Transport(err.to_string())); continue; }, }; @@ -192,34 +283,46 @@ async fn send_request( event_handlers.on_incoming_response(&body); if !status.is_success() { - errors.push(ERRL!( - "Server '{}' response !200: {}, {}", - uri, + errors.push(Web3RpcError::Transport(format!( + "Server '{:?}' response !200: {}, {}", + node, status, binprint(&body, b'.') - )); + ))); continue; } - return single_response(body, &uri.to_string()); + return single_response(body, &node.uri.to_string()); } + Err(request_failed_error(&request, &errors)) } #[cfg(target_arch = "wasm32")] async fn send_request( request: Call, - uris: Vec, + nodes: Vec, event_handlers: Vec, + gui_auth_validation_generator: Option, ) -> Result { - let request_payload = to_string(&request); + let serialized_request = to_string(&request); let mut transport_errors = Vec::new(); - for uri in uris { - match send_request_once(request_payload.clone(), &uri, &event_handlers).await { + for node in nodes.iter() { + let serialized_request = + match handle_gui_auth_payload_if_activated(&gui_auth_validation_generator, node, &request) { + Ok(Some(r)) => r, + Ok(None) => serialized_request.clone(), + Err(e) => { + transport_errors.push(e); + continue; + }, + }; + + match send_request_once(serialized_request.clone(), &node.uri, &event_handlers).await { Ok(response_json) => return Ok(response_json), Err(Error(ErrorKind::Transport(e), _)) => { - transport_errors.push(e.to_string()); + transport_errors.push(Web3RpcError::Transport(e)); }, Err(e) => return Err(e), } @@ -276,8 +379,8 @@ async fn send_request_once( } } -fn request_failed_error(request: &Call, errors: &[String]) -> Error { - let errors = errors.join("; "); +fn request_failed_error(request: &Call, errors: &[Web3RpcError]) -> Error { + let errors: String = errors.iter().map(|e| format!("{:?}; ", e)).collect(); let error = format!("request {:?} failed: {}", request, errors); Error::from(ErrorKind::Transport(error)) } diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 0f1e3e4085..183ba32579 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -214,9 +214,9 @@ use hd_wallet::{HDAddress, HDAddressId}; use qrc20::Qrc20ActivationParams; use qrc20::{qrc20_coin_from_conf_and_params, Qrc20Coin, Qrc20FeeDetails}; use qtum::{Qrc20AddressError, ScriptHashTypeNotSupported}; -use rpc_command::init_create_account::{CreateAccountTaskManager, CreateAccountTaskManagerShared}; -use rpc_command::init_scan_for_new_addresses::{ScanAddressesTaskManager, ScanAddressesTaskManagerShared}; -use rpc_command::init_withdraw::{WithdrawTaskManager, WithdrawTaskManagerShared}; +use rpc_command::{init_create_account::{CreateAccountTaskManager, CreateAccountTaskManagerShared}, + init_scan_for_new_addresses::{ScanAddressesTaskManager, ScanAddressesTaskManagerShared}, + init_withdraw::{WithdrawTaskManager, WithdrawTaskManagerShared}}; use utxo::bch::{bch_coin_from_conf_and_params, BchActivationRequest, BchCoin}; use utxo::qtum::{self, qtum_coin_with_priv_key, QtumCoin}; use utxo::qtum::{QtumDelegationOps, QtumDelegationRequest, QtumStakingInfosDetails}; diff --git a/mm2src/coins/solana.rs b/mm2src/coins/solana.rs index 6bf3c7049c..c1671e7112 100644 --- a/mm2src/coins/solana.rs +++ b/mm2src/coins/solana.rs @@ -343,6 +343,9 @@ impl SolanaCoin { self.spl_tokens_infos.lock().unwrap().insert(ticker, info); } + /// WARNING + /// Be very careful using this function since it returns dereferenced clone + /// of value behind the MutexGuard and makes it non-thread-safe. pub fn get_spl_tokens_infos(&self) -> HashMap { let guard = self.spl_tokens_infos.lock().unwrap(); (*guard).clone() diff --git a/mm2src/coins_activation/Cargo.toml b/mm2src/coins_activation/Cargo.toml index e95895aa99..89f4ac538d 100644 --- a/mm2src/coins_activation/Cargo.toml +++ b/mm2src/coins_activation/Cargo.toml @@ -9,6 +9,7 @@ edition = "2018" async-trait = "0.1" coins = { path = "../coins" } common = { path = "../common" } +ethereum-types = { version = "0.4", default-features = false, features = ["std", "serialize"] } mm2_core = { path = "../mm2_core" } mm2_err_handle = { path = "../mm2_err_handle" } mm2_number = { path = "../mm2_number" } diff --git a/mm2src/coins_activation/src/erc20_token_activation.rs b/mm2src/coins_activation/src/erc20_token_activation.rs new file mode 100644 index 0000000000..1869fb046d --- /dev/null +++ b/mm2src/coins_activation/src/erc20_token_activation.rs @@ -0,0 +1,107 @@ +use crate::{prelude::{TryFromCoinProtocol, TryPlatformCoinFromMmCoinEnum}, + token::{EnableTokenError, TokenActivationOps, TokenProtocolParams}}; +use async_trait::async_trait; +use coins::{eth::{v2_activation::{Erc20Protocol, Erc20TokenActivationError, Erc20TokenActivationRequest}, + valid_addr_from_str, EthCoin}, + CoinBalance, CoinProtocol, MarketCoinOps, MmCoin, MmCoinEnum}; +use common::Future01CompatExt; +use mm2_err_handle::prelude::MmError; +use serde::Serialize; +use std::collections::HashMap; + +#[derive(Debug, Serialize)] +pub struct Erc20InitResult { + balances: HashMap, + platform_coin: String, + token_contract_address: String, + required_confirmations: u64, +} + +impl From for EnableTokenError { + fn from(err: Erc20TokenActivationError) -> Self { + match err { + Erc20TokenActivationError::InternalError(e) => EnableTokenError::Internal(e), + Erc20TokenActivationError::CouldNotFetchBalance(e) => EnableTokenError::Transport(e), + } + } +} + +impl TryPlatformCoinFromMmCoinEnum for EthCoin { + fn try_from_mm_coin(coin: MmCoinEnum) -> Option + where + Self: Sized, + { + match coin { + MmCoinEnum::EthCoin(coin) => Some(coin), + _ => None, + } + } +} + +impl TryFromCoinProtocol for Erc20Protocol { + fn try_from_coin_protocol(proto: CoinProtocol) -> Result> + where + Self: Sized, + { + match proto { + CoinProtocol::ERC20 { + platform, + contract_address, + } => { + let token_addr = valid_addr_from_str(&contract_address).map_err(|_| CoinProtocol::ERC20 { + platform: platform.clone(), + contract_address, + })?; + + Ok(Erc20Protocol { platform, token_addr }) + }, + proto => MmError::err(proto), + } + } +} + +impl TokenProtocolParams for Erc20Protocol { + fn platform_coin_ticker(&self) -> &str { &self.platform } +} + +#[async_trait] +impl TokenActivationOps for EthCoin { + type PlatformCoin = EthCoin; + type ActivationParams = Erc20TokenActivationRequest; + type ProtocolInfo = Erc20Protocol; + type ActivationResult = Erc20InitResult; + type ActivationError = Erc20TokenActivationError; + + async fn enable_token( + ticker: String, + platform_coin: Self::PlatformCoin, + activation_params: Self::ActivationParams, + protocol_conf: Self::ProtocolInfo, + ) -> Result<(Self, Self::ActivationResult), MmError> { + let token = platform_coin + .initialize_erc20_token(activation_params, protocol_conf, ticker) + .await?; + + let address = token.my_address().map_err(Erc20TokenActivationError::InternalError)?; + let token_contract_address = token + .erc20_token_address() + .ok_or_else(|| Erc20TokenActivationError::InternalError("Token contract address is missing".to_string()))?; + + let balance = token + .my_balance() + .compat() + .await + .map_err(|e| Erc20TokenActivationError::CouldNotFetchBalance(e.to_string()))?; + + let balances = HashMap::from([(address, balance)]); + + let init_result = Erc20InitResult { + balances, + platform_coin: token.platform_ticker().to_owned(), + required_confirmations: token.required_confirmations(), + token_contract_address: format!("{:#02x}", token_contract_address), + }; + + Ok((token, init_result)) + } +} diff --git a/mm2src/coins_activation/src/eth_with_token_activation.rs b/mm2src/coins_activation/src/eth_with_token_activation.rs new file mode 100644 index 0000000000..8e75488d53 --- /dev/null +++ b/mm2src/coins_activation/src/eth_with_token_activation.rs @@ -0,0 +1,235 @@ +use crate::{platform_coin_with_tokens::{EnablePlatformCoinWithTokensError, GetPlatformBalance, + InitTokensAsMmCoinsError, PlatformWithTokensActivationOps, RegisterTokenInfo, + TokenActivationParams, TokenActivationRequest, TokenAsMmCoinInitializer, + TokenInitializer, TokenOf}, + prelude::*}; +use async_trait::async_trait; +use coins::{eth::{v2_activation::{eth_coin_from_conf_and_request_v2, Erc20Protocol, Erc20TokenActivationError, + Erc20TokenActivationRequest, EthActivationV2Error, EthActivationV2Request}, + Erc20TokenInfo, EthCoin, EthCoinType}, + my_tx_history_v2::TxHistoryStorage, + CoinBalance, CoinProtocol, MarketCoinOps, MmCoin}; +use common::{mm_metrics::MetricsArc, Future01CompatExt}; +use futures::future::AbortHandle; +use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::prelude::*; +use mm2_number::BigDecimal; +use serde::{Deserialize, Serialize}; +use serde_json::Value as Json; +use std::collections::HashMap; + +impl From for EnablePlatformCoinWithTokensError { + fn from(err: EthActivationV2Error) -> Self { + match err { + EthActivationV2Error::InvalidPayload(e) + | EthActivationV2Error::InvalidSwapContractAddr(e) + | EthActivationV2Error::InvalidFallbackSwapContract(e) => { + EnablePlatformCoinWithTokensError::InvalidPayload(e) + }, + EthActivationV2Error::ActivationFailed { ticker, error } => { + EnablePlatformCoinWithTokensError::PlatformCoinCreationError { ticker, error } + }, + EthActivationV2Error::AtLeastOneNodeRequired => EnablePlatformCoinWithTokensError::AtLeastOneNodeRequired( + "Enable request for ETH coin must have at least 1 node".to_string(), + ), + EthActivationV2Error::CouldNotFetchBalance(e) | EthActivationV2Error::UnreachableNodes(e) => { + EnablePlatformCoinWithTokensError::Transport(e) + }, + EthActivationV2Error::InternalError(e) => EnablePlatformCoinWithTokensError::Internal(e), + } + } +} + +impl TryFromCoinProtocol for EthCoinType { + fn try_from_coin_protocol(proto: CoinProtocol) -> Result> + where + Self: Sized, + { + match proto { + CoinProtocol::ETH => Ok(EthCoinType::Eth), + protocol => MmError::err(protocol), + } + } +} + +pub struct Erc20Initializer { + platform_coin: EthCoin, +} + +impl From for InitTokensAsMmCoinsError { + fn from(error: Erc20TokenActivationError) -> Self { + match error { + Erc20TokenActivationError::InternalError(e) => InitTokensAsMmCoinsError::Internal(e), + Erc20TokenActivationError::CouldNotFetchBalance(e) => InitTokensAsMmCoinsError::CouldNotFetchBalance(e), + } + } +} + +#[async_trait] +impl TokenInitializer for Erc20Initializer { + type Token = EthCoin; + type TokenActivationRequest = Erc20TokenActivationRequest; + type TokenProtocol = Erc20Protocol; + type InitTokensError = Erc20TokenActivationError; + + fn tokens_requests_from_platform_request( + platform_params: &EthWithTokensActivationRequest, + ) -> Vec> { + platform_params.erc20_tokens_requests.clone() + } + + async fn enable_tokens( + &self, + activation_params: Vec>, + ) -> Result, MmError> { + let mut tokens = vec![]; + for param in activation_params { + let token: EthCoin = self + .platform_coin + .initialize_erc20_token(param.activation_request, param.protocol, param.ticker) + .await?; + tokens.push(token); + } + + Ok(tokens) + } + + fn platform_coin(&self) -> &EthCoin { &self.platform_coin } +} + +#[derive(Clone, Deserialize)] +pub struct EthWithTokensActivationRequest { + #[serde(flatten)] + platform_request: EthActivationV2Request, + erc20_tokens_requests: Vec>, +} + +impl TxHistory for EthWithTokensActivationRequest { + fn tx_history(&self) -> bool { false } +} + +impl TokenOf for EthCoin { + type PlatformCoin = EthCoin; +} + +impl RegisterTokenInfo for EthCoin { + fn register_token_info(&self, token: &EthCoin) { + self.add_erc_token_info(token.ticker().to_string(), Erc20TokenInfo { + token_address: token.erc20_token_address().unwrap(), + decimals: token.decimals(), + }); + } +} + +#[derive(Serialize)] +pub struct EthWithTokensActivationResult { + current_block: u64, + eth_addresses_infos: HashMap>, + erc20_addresses_infos: HashMap>, +} + +impl GetPlatformBalance for EthWithTokensActivationResult { + fn get_platform_balance(&self) -> BigDecimal { + self.eth_addresses_infos + .iter() + .fold(BigDecimal::from(0), |total, (_, addr_info)| { + &total + &addr_info.balances.get_total() + }) + } +} + +impl CurrentBlock for EthWithTokensActivationResult { + fn current_block(&self) -> u64 { self.current_block } +} + +#[async_trait] +impl PlatformWithTokensActivationOps for EthCoin { + type ActivationRequest = EthWithTokensActivationRequest; + type PlatformProtocolInfo = EthCoinType; + type ActivationResult = EthWithTokensActivationResult; + type ActivationError = EthActivationV2Error; + + async fn enable_platform_coin( + ctx: MmArc, + ticker: String, + platform_conf: Json, + activation_request: Self::ActivationRequest, + _protocol: Self::PlatformProtocolInfo, + priv_key: &[u8], + ) -> Result> { + let platform_coin = eth_coin_from_conf_and_request_v2( + &ctx, + &ticker, + &platform_conf, + activation_request.platform_request, + priv_key, + ) + .await?; + + Ok(platform_coin) + } + + fn token_initializers( + &self, + ) -> Vec>> { + vec![Box::new(Erc20Initializer { + platform_coin: self.clone(), + })] + } + + async fn get_activation_result(&self) -> Result> { + let my_address = self.my_address().map_err(EthActivationV2Error::InternalError)?; + let pubkey = self + .get_public_key() + .map_err(|e| EthActivationV2Error::InternalError(e.to_string()))?; + + let current_block = self + .current_block() + .compat() + .await + .map_err(EthActivationV2Error::InternalError)?; + + let eth_balance = self + .my_balance() + .compat() + .await + .map_err(|e| EthActivationV2Error::CouldNotFetchBalance(e.to_string()))?; + let token_balances = self + .get_tokens_balance_list() + .await + .map_err(|e| EthActivationV2Error::CouldNotFetchBalance(e.to_string()))?; + + let mut result = EthWithTokensActivationResult { + current_block, + eth_addresses_infos: HashMap::new(), + erc20_addresses_infos: HashMap::new(), + }; + + result + .eth_addresses_infos + .insert(my_address.to_string(), CoinAddressInfo { + derivation_method: DerivationMethod::Iguana, + pubkey: pubkey.clone(), + balances: eth_balance, + }); + + result + .erc20_addresses_infos + .insert(my_address.to_string(), CoinAddressInfo { + derivation_method: DerivationMethod::Iguana, + pubkey, + balances: token_balances, + }); + + Ok(result) + } + + fn start_history_background_fetching( + &self, + _metrics: MetricsArc, + _storage: impl TxHistoryStorage + Send + 'static, + _initial_balance: BigDecimal, + ) -> AbortHandle { + unimplemented!() + } +} diff --git a/mm2src/coins_activation/src/lib.rs b/mm2src/coins_activation/src/lib.rs index 5b6b9321da..d6af0ae421 100644 --- a/mm2src/coins_activation/src/lib.rs +++ b/mm2src/coins_activation/src/lib.rs @@ -1,5 +1,7 @@ mod bch_with_tokens_activation; mod context; +mod erc20_token_activation; +mod eth_with_token_activation; mod l2; #[cfg(not(target_arch = "wasm32"))] mod lightning_activation; mod platform_coin_with_tokens; diff --git a/mm2src/coins_activation/src/platform_coin_with_tokens.rs b/mm2src/coins_activation/src/platform_coin_with_tokens.rs index c27a440008..ecd1dac377 100644 --- a/mm2src/coins_activation/src/platform_coin_with_tokens.rs +++ b/mm2src/coins_activation/src/platform_coin_with_tokens.rs @@ -66,6 +66,8 @@ pub trait TokenAsMmCoinInitializer: Send + Sync { pub enum InitTokensAsMmCoinsError { TokenConfigIsNotFound(String), InvalidPubkey(String), + CouldNotFetchBalance(String), + Internal(String), TokenProtocolParseError { ticker: String, error: String }, UnexpectedTokenProtocol { ticker: String, protocol: CoinProtocol }, } @@ -211,6 +213,8 @@ pub enum EnablePlatformCoinWithTokensError { #[display(fmt = "Unexpected derivation method: {}", _0)] UnexpectedDerivationMethod(String), Transport(String), + AtLeastOneNodeRequired(String), + InvalidPayload(String), Internal(String), } @@ -245,7 +249,10 @@ impl From for EnablePlatformCoinWithTokensError { InitTokensAsMmCoinsError::UnexpectedTokenProtocol { ticker, protocol } => { EnablePlatformCoinWithTokensError::UnexpectedTokenProtocol { ticker, protocol } }, - InitTokensAsMmCoinsError::InvalidPubkey(e) => EnablePlatformCoinWithTokensError::Internal(e), + InitTokensAsMmCoinsError::InvalidPubkey(e) | InitTokensAsMmCoinsError::Internal(e) => { + EnablePlatformCoinWithTokensError::Internal(e) + }, + InitTokensAsMmCoinsError::CouldNotFetchBalance(e) => EnablePlatformCoinWithTokensError::Transport(e), } } } @@ -272,6 +279,8 @@ impl HttpStatusCode for EnablePlatformCoinWithTokensError { | EnablePlatformCoinWithTokensError::PlatformConfigIsNotFound(_) | EnablePlatformCoinWithTokensError::TokenConfigIsNotFound(_) | EnablePlatformCoinWithTokensError::UnexpectedPlatformProtocol { .. } + | EnablePlatformCoinWithTokensError::InvalidPayload { .. } + | EnablePlatformCoinWithTokensError::AtLeastOneNodeRequired(_) | EnablePlatformCoinWithTokensError::UnexpectedTokenProtocol { .. } => StatusCode::BAD_REQUEST, } } diff --git a/mm2src/coins_activation/src/token.rs b/mm2src/coins_activation/src/token.rs index 544c1e0b9b..9e27a9c079 100644 --- a/mm2src/coins_activation/src/token.rs +++ b/mm2src/coins_activation/src/token.rs @@ -58,6 +58,7 @@ pub enum EnableTokenError { }, #[display(fmt = "{}", _0)] UnexpectedDerivationMethod(UnexpectedDerivationMethod), + CouldNotFetchBalance(String), Transport(String), Internal(String), } @@ -156,6 +157,7 @@ impl HttpStatusCode for EnableTokenError { | EnableTokenError::UnsupportedPlatformCoin { .. } | EnableTokenError::UnexpectedDerivationMethod(_) | EnableTokenError::Transport(_) + | EnableTokenError::CouldNotFetchBalance(_) | EnableTokenError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, } } diff --git a/mm2src/common/common.rs b/mm2src/common/common.rs index 4ab31475c3..f07150ce6a 100644 --- a/mm2src/common/common.rs +++ b/mm2src/common/common.rs @@ -84,6 +84,14 @@ macro_rules! try_h { }; } +/// Drops mutability of given variable +#[macro_export] +macro_rules! drop_mutability { + ($t: ident) => { + let $t = $t; + }; +} + #[macro_use] pub mod jsonrpc_client; #[macro_use] @@ -115,6 +123,7 @@ pub mod executor; #[cfg(target_arch = "wasm32")] pub use wasm::*; use backtrace::SymbolName; +use chrono::Utc; pub use futures::compat::Future01CompatExt; use futures::future::{abortable, AbortHandle, FutureExt}; use futures01::{future, Future}; @@ -940,3 +949,6 @@ pub fn spawn_abortable(fut: impl Future03 + Send + 'static) -> Abor spawn(abortable.then(|_| async {})); AbortOnDropHandle(handle) } + +#[inline(always)] +pub fn get_utc_timestamp() -> i64 { Utc::now().timestamp() } diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index e33e16b3a4..f202c4b9f2 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -7,6 +7,7 @@ use crate::{mm2::lp_stats::{add_node_to_version_stat, remove_node_from_version_s stop_version_stat_collection, update_version_stat_collection}, mm2::lp_swap::{recreate_swap_data, trade_preimage_rpc}, mm2::rpc::lp_commands::{get_public_key, get_public_key_hash}}; +use coins::eth::EthCoin; use coins::hd_wallet::get_new_address; use coins::my_tx_history_v2::my_tx_history_v2_rpc; use coins::rpc_command::account_balance::account_balance; @@ -126,6 +127,8 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, best_orders_rpc_v2).await, "enable_bch_with_tokens" => handle_mmrpc(ctx, request, enable_platform_coin_with_tokens::).await, "enable_slp" => handle_mmrpc(ctx, request, enable_token::).await, + "enable_eth_with_tokens" => handle_mmrpc(ctx, request, enable_platform_coin_with_tokens::).await, + "enable_erc20" => handle_mmrpc(ctx, request, enable_token::).await, "get_current_mtp" => handle_mmrpc(ctx, request, get_current_mtp_rpc).await, "get_new_address" => handle_mmrpc(ctx, request, get_new_address).await, "get_public_key" => handle_mmrpc(ctx, request, get_public_key).await, diff --git a/mm2src/mm2_net/Cargo.toml b/mm2src/mm2_net/Cargo.toml index 1c08ae2282..e188bc985d 100644 --- a/mm2src/mm2_net/Cargo.toml +++ b/mm2src/mm2_net/Cargo.toml @@ -12,6 +12,7 @@ serde_json = { version = "1.0", features = ["preserve_order", "raw_value"] } bytes = "1.1" cfg-if = "1.0" common = { path = "../common" } +ethkey = { git = "https://github.com/artemii235/parity-ethereum.git" } mm2_err_handle = { path = "../mm2_err_handle" } mm2_core = { path = "../mm2_core" } derive_more = "0.99" diff --git a/mm2src/mm2_net/src/transport.rs b/mm2src/mm2_net/src/transport.rs index a76007b0f4..f829e730d5 100644 --- a/mm2src/mm2_net/src/transport.rs +++ b/mm2src/mm2_net/src/transport.rs @@ -1,4 +1,5 @@ use derive_more::Display; +use ethkey::Secret; use http::{HeaderMap, StatusCode}; use mm2_err_handle::prelude::*; use serde::{Deserialize, Serialize}; @@ -48,3 +49,19 @@ where error: e.to_string(), }) } + +#[derive(Clone, Debug)] +pub struct GuiAuthValidationGenerator { + pub coin_ticker: String, + pub secret: Secret, + pub address: String, +} + +/// gui-auth specific data-type that needed in order to perform gui-auth calls +#[derive(Serialize, Clone)] +pub struct GuiAuthValidation { + pub coin_ticker: String, + pub address: String, + pub timestamp_message: i64, + pub signature: String, +} From c86d461acb7e07af686b4ec8f2479b5a2f14dd12 Mon Sep 17 00:00:00 2001 From: Artem Vitae Date: Fri, 5 Aug 2022 13:07:18 +0700 Subject: [PATCH 005/249] Update chrono. --- Cargo.lock | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d048d3968..999eb30ad8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -940,9 +940,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.11" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2" +checksum = "6127248204b9aba09a362f6c930ef6a78f2c1b2215f8a7b398c06e1083f17af0" dependencies = [ "js-sys", "num-integer", @@ -950,6 +950,7 @@ dependencies = [ "serde", "time 0.1.43", "wasm-bindgen", + "winapi", ] [[package]] From c46543c27b29c847075310d98054ee6f183d6638 Mon Sep 17 00:00:00 2001 From: Artem Vitae Date: Fri, 5 Aug 2022 15:36:27 +0700 Subject: [PATCH 006/249] Add ignored test allowing to generate recoverable swaps #1346 (#1428) * WIP. * WIP. * WIP. * Almost done. TODO a bit refactoring. * Implementation finished. * Fix tests compilation. * Temporary ignore RUSTSEC-2022-0040. * Avoid using unimplemented!() * Do not compile lp_swap_tests for WASM. * Use can_refund_htlc in recover_funds instead of hardcoded 3700 seconds. * Fix tests. * sqlite_conn_opt instead of sqlite_initialized. Co-authored-by: Artem Vitae --- mm2src/coins/test_coin.rs | 11 +- mm2src/coins/utxo/utxo_common.rs | 4 +- mm2src/coins/utxo/utxo_tests.rs | 6 +- mm2src/mm2_bitcoin/keys/src/public.rs | 11 ++ mm2src/mm2_core/src/mm_ctx.rs | 5 + mm2src/mm2_main/src/lp_native_dex.rs | 4 +- mm2src/mm2_main/src/lp_swap.rs | 193 +++++++++++++++++++++- mm2src/mm2_main/src/lp_swap/maker_swap.rs | 64 +++++-- mm2src/mm2_main/src/lp_swap/taker_swap.rs | 62 ++++++- mm2src/mm2_test_helpers/src/for_tests.rs | 10 ++ 10 files changed, 337 insertions(+), 33 deletions(-) diff --git a/mm2src/coins/test_coin.rs b/mm2src/coins/test_coin.rs index fa8d4d1662..f240a31662 100644 --- a/mm2src/coins/test_coin.rs +++ b/mm2src/coins/test_coin.rs @@ -1,8 +1,9 @@ use super::{CoinBalance, HistorySyncState, MarketCoinOps, MmCoin, RawTransactionFut, RawTransactionRequest, SwapOps, TradeFee, TransactionEnum, TransactionFut}; -use crate::{BalanceFut, FeeApproxStage, FoundSwapTxSpend, NegotiateSwapContractAddrErr, SearchForSwapTxSpendInput, - SignatureResult, TradePreimageFut, TradePreimageResult, TradePreimageValue, UnexpectedDerivationMethod, - ValidateAddressResult, ValidatePaymentInput, VerificationResult, WithdrawFut, WithdrawRequest}; +use crate::{BalanceFut, CanRefundHtlc, FeeApproxStage, FoundSwapTxSpend, NegotiateSwapContractAddrErr, + SearchForSwapTxSpendInput, SignatureResult, TradePreimageFut, TradePreimageResult, TradePreimageValue, + UnexpectedDerivationMethod, ValidateAddressResult, ValidatePaymentInput, VerificationResult, WithdrawFut, + WithdrawRequest}; use async_trait::async_trait; use futures01::Future; use keys::KeyPair; @@ -228,6 +229,10 @@ impl SwapOps for TestCoin { } fn derive_htlc_key_pair(&self, _swap_unique_data: &[u8]) -> KeyPair { unimplemented!() } + + fn can_refund_htlc(&self, locktime: u64) -> Box + Send + '_> { + unimplemented!() + } } #[async_trait] diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 583ed7f73b..547086f00f 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3570,7 +3570,7 @@ where let now = now_ms() / 1000; if now < locktime { let to_wait = locktime - now + 1; - return Ok(CanRefundHtlc::HaveToWait(to_wait.max(3600))); + return Ok(CanRefundHtlc::HaveToWait(to_wait.min(3600))); } let mtp = coin.get_current_mtp().await?; @@ -3580,7 +3580,7 @@ where Ok(CanRefundHtlc::CanRefundNow) } else { let to_wait = (locktime - mtp + 1) as u64; - Ok(CanRefundHtlc::HaveToWait(to_wait.max(3600))) + Ok(CanRefundHtlc::HaveToWait(to_wait.min(3600))) } } diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 56dc7a9015..ccbe85377a 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -28,6 +28,7 @@ use futures::future::join_all; use futures::TryFutureExt; use mm2_core::mm_ctx::MmCtxBuilder; use mm2_number::bigdecimal::{BigDecimal, Signed}; +use mm2_test_helpers::for_tests::RICK_ELECTRUM_ADDRS; use mocktopus::mocking::*; use rpc::v1::types::H256 as H256Json; use serialization::{deserialize, CoinVariant}; @@ -39,11 +40,6 @@ use std::num::NonZeroUsize; const TEST_COIN_NAME: &'static str = "RICK"; // Made-up hrp for rick to test p2wpkh script const TEST_COIN_HRP: &'static str = "rck"; -const RICK_ELECTRUM_ADDRS: &[&'static str] = &[ - "electrum1.cipig.net:10017", - "electrum2.cipig.net:10017", - "electrum3.cipig.net:10017", -]; const TEST_COIN_DECIMALS: u8 = 8; pub fn electrum_client_for_test(servers: &[&str]) -> ElectrumClient { diff --git a/mm2src/mm2_bitcoin/keys/src/public.rs b/mm2src/mm2_bitcoin/keys/src/public.rs index 7ba0b9a27e..b57f376ee9 100644 --- a/mm2src/mm2_bitcoin/keys/src/public.rs +++ b/mm2src/mm2_bitcoin/keys/src/public.rs @@ -69,6 +69,17 @@ impl Public { }; Ok(public) } + + pub fn compressed_unprefixed(&self) -> Option<[u8; 32]> { + let mut res = [0; 32]; + match self { + Public::Compressed(pubkey) => { + res.copy_from_slice(&pubkey.as_slice()[1..33]); + Some(res) + }, + Public::Normal(_) => None, + } + } } impl ops::Deref for Public { diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index cff1011622..6278d71647 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -270,6 +270,11 @@ impl MmCtx { Ok(()) } + #[cfg(not(target_arch = "wasm32"))] + pub fn sqlite_conn_opt(&self) -> Option> { + self.sqlite_connection.as_option().map(|conn| conn.lock().unwrap()) + } + #[cfg(not(target_arch = "wasm32"))] pub fn sqlite_connection(&self) -> MutexGuard { self.sqlite_connection diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index 967f5cd413..a9e0fe72a5 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -264,7 +264,7 @@ fn default_seednodes(netid: u16) -> Vec { } #[cfg(not(target_arch = "wasm32"))] -fn fix_directories(ctx: &MmCtx) -> MmInitResult<()> { +pub fn fix_directories(ctx: &MmCtx) -> MmInitResult<()> { let dbdir = ctx.dbdir(); std::fs::create_dir_all(&dbdir).map_to_mm(|e| MmInitError::ErrorCreatingDbDir { path: dbdir.clone(), @@ -446,7 +446,7 @@ async fn kick_start(ctx: MmArc) -> MmInitResult<()> { Ok(()) } -async fn init_p2p(ctx: MmArc) -> P2PResult<()> { +pub async fn init_p2p(ctx: MmArc) -> P2PResult<()> { let i_am_seed = ctx.conf["i_am_seed"].as_bool().unwrap_or(false); let netid = ctx.netid(); diff --git a/mm2src/mm2_main/src/lp_swap.rs b/mm2src/mm2_main/src/lp_swap.rs index 66a32b9249..e82fd72d21 100644 --- a/mm2src/mm2_main/src/lp_swap.rs +++ b/mm2src/mm2_main/src/lp_swap.rs @@ -725,7 +725,9 @@ pub async fn insert_new_swap_to_db( #[cfg(not(target_arch = "wasm32"))] fn add_swap_to_db_index(ctx: &MmArc, swap: &SavedSwap) { - crate::mm2::database::stats_swaps::add_swap_to_index(&ctx.sqlite_connection(), swap) + if let Some(conn) = ctx.sqlite_conn_opt() { + crate::mm2::database::stats_swaps::add_swap_to_index(&conn, swap) + } } #[cfg(not(target_arch = "wasm32"))] @@ -1227,11 +1229,20 @@ pub async fn active_swaps_rpc(ctx: MmArc, req: Json) -> Result> Ok(try_s!(Response::builder().body(res))) } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod lp_swap_tests { - use serialization::{deserialize, serialize}; - use super::*; + use crate::mm2::lp_native_dex::{fix_directories, init_p2p}; + use coins::utxo::rpc_clients::ElectrumRpcRequest; + use coins::utxo::utxo_standard::utxo_standard_coin_with_priv_key; + use coins::utxo::{UtxoActivationParams, UtxoRpcMode}; + use coins::MarketCoinOps; + use coins::PrivKeyActivationPolicy; + use common::block_on; + use crypto::privkey::key_pair_from_seed; + use mm2_core::mm_ctx::MmCtxBuilder; + use mm2_test_helpers::for_tests::{morty_conf, rick_conf, MORTY_ELECTRUM_ADDRS, RICK_ELECTRUM_ADDRS}; + use serialization::{deserialize, serialize}; #[test] fn test_dex_fee_amount() { @@ -1523,4 +1534,178 @@ mod lp_swap_tests { assert_eq!(deserialized, v3); } + + fn utxo_activation_params(electrums: &[&str]) -> UtxoActivationParams { + UtxoActivationParams { + mode: UtxoRpcMode::Electrum { + servers: electrums + .into_iter() + .map(|url| ElectrumRpcRequest { + url: url.to_string(), + protocol: Default::default(), + disable_cert_verification: false, + }) + .collect(), + block_header_params: None, + }, + utxo_merge_params: None, + tx_history: false, + required_confirmations: Some(0), + requires_notarization: None, + address_format: None, + gap_limit: None, + scan_policy: Default::default(), + priv_key_policy: PrivKeyActivationPolicy::IguanaPrivKey, + check_utxo_maturity: None, + } + } + + #[test] + #[ignore] + fn gen_recoverable_swap() { + let maker_passphrase = std::env::var("BOB_PASSPHRASE").expect("BOB_PASSPHRASE env must be set"); + let maker_fail_at = std::env::var("MAKER_FAIL_AT").map(maker_swap::FailAt::from).ok(); + let taker_passphrase = std::env::var("ALICE_PASSPHRASE").expect("ALICE_PASSPHRASE env must be set"); + let taker_fail_at = std::env::var("TAKER_FAIL_AT").map(taker_swap::FailAt::from).ok(); + + if maker_fail_at.is_none() && taker_fail_at.is_none() { + panic!("At least one of MAKER_FAIL_AT/TAKER_FAIL_AT must be provided"); + } + + let maker_ctx_conf = json!({ + "netid": 1234, + "p2p_in_memory": true, + "p2p_in_memory_port": 777, + "i_am_seed": true, + }); + + let maker_key_pair = key_pair_from_seed(&maker_passphrase).unwrap(); + let maker_ctx = MmCtxBuilder::default() + .with_secp256k1_key_pair(maker_key_pair.clone()) + .with_conf(maker_ctx_conf) + .into_mm_arc(); + fix_directories(&maker_ctx).unwrap(); + block_on(init_p2p(maker_ctx.clone())).unwrap(); + + let rick_activation_params = utxo_activation_params(RICK_ELECTRUM_ADDRS); + let morty_activation_params = utxo_activation_params(MORTY_ELECTRUM_ADDRS); + + let rick_maker = block_on(utxo_standard_coin_with_priv_key( + &maker_ctx, + "RICK", + &rick_conf(), + &rick_activation_params, + maker_key_pair.private_ref(), + )) + .unwrap(); + + println!("Maker address {}", rick_maker.my_address().unwrap()); + + let morty_maker = block_on(utxo_standard_coin_with_priv_key( + &maker_ctx, + "MORTY", + &morty_conf(), + &morty_activation_params, + maker_key_pair.private_ref(), + )) + .unwrap(); + + let taker_key_pair = key_pair_from_seed(&taker_passphrase).unwrap(); + + let taker_ctx_conf = json!({ + "netid": 1234, + "p2p_in_memory": true, + "seednodes": vec!["/memory/777"] + }); + let taker_ctx = MmCtxBuilder::default() + .with_secp256k1_key_pair(taker_key_pair.clone()) + .with_conf(taker_ctx_conf) + .into_mm_arc(); + fix_directories(&taker_ctx).unwrap(); + block_on(init_p2p(taker_ctx.clone())).unwrap(); + + let rick_taker = block_on(utxo_standard_coin_with_priv_key( + &taker_ctx, + "RICK", + &rick_conf(), + &rick_activation_params, + taker_key_pair.private_ref(), + )) + .unwrap(); + + let morty_taker = block_on(utxo_standard_coin_with_priv_key( + &taker_ctx, + "MORTY", + &morty_conf(), + &morty_activation_params, + taker_key_pair.private_ref(), + )) + .unwrap(); + + println!("Taker address {}", rick_taker.my_address().unwrap()); + + let uuid = Uuid::new_v4(); + let maker_amount = BigDecimal::from_str("0.1").unwrap(); + let taker_amount = BigDecimal::from_str("0.1").unwrap(); + let conf_settings = SwapConfirmationsSettings { + maker_coin_confs: 0, + maker_coin_nota: false, + taker_coin_confs: 0, + taker_coin_nota: false, + }; + let payment_locktime = 30; + + let mut maker_swap = MakerSwap::new( + maker_ctx.clone(), + taker_key_pair.public().compressed_unprefixed().unwrap().into(), + maker_amount.clone(), + taker_amount.clone(), + maker_key_pair.public_slice().into(), + uuid, + None, + conf_settings, + rick_maker.into(), + morty_maker.into(), + payment_locktime, + None, + Default::default(), + ); + + maker_swap.fail_at = maker_fail_at; + + let mut taker_swap = TakerSwap::new( + taker_ctx.clone(), + maker_key_pair.public().compressed_unprefixed().unwrap().into(), + maker_amount.into(), + taker_amount.into(), + taker_key_pair.public_slice().into(), + uuid, + None, + conf_settings, + rick_taker.into(), + morty_taker.into(), + payment_locktime, + None, + ); + + taker_swap.fail_at = taker_fail_at; + + block_on(futures::future::join( + run_maker_swap(RunMakerSwapInput::StartNew(maker_swap), maker_ctx.clone()), + run_taker_swap(RunTakerSwapInput::StartNew(taker_swap), taker_ctx.clone()), + )); + + println!( + "Maker swap path {}", + std::fs::canonicalize(my_swap_file_path(&maker_ctx, &uuid)) + .unwrap() + .display() + ); + println!( + "Taker swap path {}", + std::fs::canonicalize(my_swap_file_path(&taker_ctx, &uuid)) + .unwrap() + .display() + ); + } } diff --git a/mm2src/mm2_main/src/lp_swap/maker_swap.rs b/mm2src/mm2_main/src/lp_swap/maker_swap.rs index caafef90e7..5da4e6fec6 100644 --- a/mm2src/mm2_main/src/lp_swap/maker_swap.rs +++ b/mm2src/mm2_main/src/lp_swap/maker_swap.rs @@ -175,6 +175,24 @@ pub struct MakerSwapMut { maker_payment_refund: Option, } +#[cfg(test)] +#[derive(Eq, PartialEq)] +pub(super) enum FailAt { + TakerPaymentSpend, + MakerPaymentRefund, +} + +#[cfg(test)] +impl From for FailAt { + fn from(str: String) -> Self { + match str.as_str() { + "taker_payment_spend" => FailAt::TakerPaymentSpend, + "maker_payment_refund" => FailAt::MakerPaymentRefund, + _ => panic!("Invalid MAKER_FAIL_AT value"), + } + } +} + pub struct MakerSwap { ctx: MmArc, maker_coin: MmCoinEnum, @@ -195,6 +213,8 @@ pub struct MakerSwap { /// Temporary privkey used to sign P2P messages when applicable p2p_privkey: Option, secret: H256, + #[cfg(test)] + pub(super) fail_at: Option, } impl MakerSwap { @@ -342,6 +362,8 @@ impl MakerSwap { }), ctx, secret, + #[cfg(test)] + fail_at: None, } } @@ -843,6 +865,13 @@ impl MakerSwap { } async fn spend_taker_payment(&self) -> Result<(Option, Vec), String> { + #[cfg(test)] + if self.fail_at == Some(FailAt::TakerPaymentSpend) { + return Ok((Some(MakerSwapCommand::Finish), vec![ + MakerSwapEvent::TakerPaymentSpendFailed("explicit failure".into()), + ])); + } + let duration = (self.r().data.lock_duration * 4) / 5; let timeout = self.r().data.started_at + duration; @@ -940,6 +969,13 @@ impl MakerSwap { } async fn refund_maker_payment(&self) -> Result<(Option, Vec), String> { + #[cfg(test)] + if self.fail_at == Some(FailAt::MakerPaymentRefund) { + return Ok((Some(MakerSwapCommand::Finish), vec![ + MakerSwapEvent::MakerPaymentRefundFailed("explicit failure".into()), + ])); + } + let locktime = self.r().data.maker_payment_lock; loop { match self.maker_coin.can_refund_htlc(locktime).compat().await { @@ -1218,13 +1254,14 @@ impl MakerSwap { ), Err(e) => ERR!("Error {} when trying to find maker payment spend", e), Ok(None) => { - // our payment is not spent, try to refund - info!("Trying to refund MakerPayment"); - if now_ms() / 1000 < self.r().data.maker_payment_lock + 3700 { - return ERR!( - "Too early to refund, wait until {}", - self.r().data.maker_payment_lock + 3700 - ); + let can_refund_htlc = try_s!( + self.maker_coin + .can_refund_htlc(maker_payment_lock as u64) + .compat() + .await + ); + if let CanRefundHtlc::HaveToWait(seconds_to_wait) = can_refund_htlc { + return ERR!("Too early to refund, wait until {}", now_ms() / 1000 + seconds_to_wait); } let fut = self.maker_coin.send_maker_refunds_payment( &maker_payment, @@ -2023,6 +2060,9 @@ mod maker_swap_tests { TestCoin::ticker.mock_safe(|_| MockResult::Return("ticker")); TestCoin::swap_contract_address.mock_safe(|_| MockResult::Return(None)); + TestCoin::can_refund_htlc + .mock_safe(|_, _| MockResult::Return(Box::new(futures01::future::ok(CanRefundHtlc::CanRefundNow)))); + static mut MY_PAYMENT_SENT_CALLED: bool = false; TestCoin::check_if_my_payment_sent.mock_safe(|_, _, _, _, _, _, _| { unsafe { MY_PAYMENT_SENT_CALLED = true }; @@ -2062,8 +2102,10 @@ mod maker_swap_tests { TestCoin::ticker.mock_safe(|_| MockResult::Return("ticker")); TestCoin::swap_contract_address.mock_safe(|_| MockResult::Return(None)); - static mut MAKER_REFUND_CALLED: bool = false; + TestCoin::can_refund_htlc + .mock_safe(|_, _| MockResult::Return(Box::new(futures01::future::ok(CanRefundHtlc::CanRefundNow)))); + static mut MAKER_REFUND_CALLED: bool = false; TestCoin::send_maker_refunds_payment.mock_safe(|_, _, _, _, _, _, _| { unsafe { MAKER_REFUND_CALLED = true }; MockResult::Return(Box::new(futures01::future::ok(eth_tx_for_test().into()))) @@ -2165,13 +2207,15 @@ mod maker_swap_tests { unsafe { MY_PAYMENT_SENT_CALLED = true }; MockResult::Return(Box::new(futures01::future::ok(Some(eth_tx_for_test().into())))) }); + TestCoin::can_refund_htlc + .mock_safe(|_, _| MockResult::Return(Box::new(futures01::future::ok(CanRefundHtlc::HaveToWait(1000))))); TestCoin::search_for_swap_tx_spend_my .mock_safe(|_, _| MockResult::Return(Box::pin(futures::future::ready(Ok(None))))); let maker_coin = MmCoinEnum::Test(TestCoin::default()); let taker_coin = MmCoinEnum::Test(TestCoin::default()); let (maker_swap, _) = MakerSwap::load_from_saved(ctx, maker_coin, taker_coin, maker_saved_swap).unwrap(); - maker_swap.w().data.maker_payment_lock = (now_ms() / 1000) - 3690; - assert!(block_on(maker_swap.recover_funds()).is_err()); + let error = block_on(maker_swap.recover_funds()).unwrap_err(); + assert!(error.contains("Too early to refund")); assert!(unsafe { MY_PAYMENT_SENT_CALLED }); } diff --git a/mm2src/mm2_main/src/lp_swap/taker_swap.rs b/mm2src/mm2_main/src/lp_swap/taker_swap.rs index 2c35bd5f0c..bbcfe34eba 100644 --- a/mm2src/mm2_main/src/lp_swap/taker_swap.rs +++ b/mm2src/mm2_main/src/lp_swap/taker_swap.rs @@ -469,6 +469,26 @@ pub struct TakerSwapMut { secret: H256Json, } +#[cfg(test)] +#[derive(Eq, PartialEq)] +pub(super) enum FailAt { + TakerPayment, + MakerPaymentSpend, + TakerPaymentRefund, +} + +#[cfg(test)] +impl From for FailAt { + fn from(str: String) -> Self { + match str.as_str() { + "taker_payment" => FailAt::TakerPayment, + "maker_payment_spend" => FailAt::MakerPaymentSpend, + "taker_payment_refund" => FailAt::TakerPaymentRefund, + _ => panic!("Invalid TAKER_FAIL_AT value"), + } + } +} + pub struct TakerSwap { ctx: MmArc, maker_coin: MmCoinEnum, @@ -487,6 +507,8 @@ pub struct TakerSwap { conf_settings: SwapConfirmationsSettings, payment_locktime: u64, p2p_privkey: Option, + #[cfg(test)] + pub(super) fail_at: Option, } #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] @@ -754,6 +776,8 @@ impl TakerSwap { secret: H256Json::default(), }), ctx, + #[cfg(test)] + fail_at: None, } } @@ -1155,6 +1179,13 @@ impl TakerSwap { } async fn send_taker_payment(&self) -> Result<(Option, Vec), String> { + #[cfg(test)] + if self.fail_at == Some(FailAt::TakerPayment) { + return Ok((Some(TakerSwapCommand::Finish), vec![ + TakerSwapEvent::TakerPaymentTransactionFailed("Explicit test failure".into()), + ])); + } + let timeout = self.r().data.started_at + self.r().data.lock_duration / 3; let now = now_ms() / 1000; if now > timeout { @@ -1290,6 +1321,12 @@ impl TakerSwap { } async fn spend_maker_payment(&self) -> Result<(Option, Vec), String> { + #[cfg(test)] + if self.fail_at == Some(FailAt::MakerPaymentSpend) { + return Ok((Some(TakerSwapCommand::Finish), vec![ + TakerSwapEvent::MakerPaymentSpendFailed("Explicit test failure".into()), + ])); + } let spend_fut = self.maker_coin.send_taker_spends_maker_payment( &self.r().maker_payment.clone().unwrap().tx_hex, self.maker_payment_lock.load(Ordering::Relaxed) as u32, @@ -1336,6 +1373,13 @@ impl TakerSwap { } async fn refund_taker_payment(&self) -> Result<(Option, Vec), String> { + #[cfg(test)] + if self.fail_at == Some(FailAt::TakerPaymentRefund) { + return Ok((Some(TakerSwapCommand::Finish), vec![ + TakerSwapEvent::TakerPaymentRefundFailed("Explicit test failure".into()), + ])); + } + let locktime = self.r().data.taker_payment_lock; loop { match self.taker_coin.can_refund_htlc(locktime).compat().await { @@ -1653,11 +1697,9 @@ impl TakerSwap { ), }, None => { - if now_ms() / 1000 < self.r().data.taker_payment_lock + 3700 { - return ERR!( - "Too early to refund, wait until {}", - self.r().data.taker_payment_lock + 3700 - ); + let can_refund = try_s!(self.taker_coin.can_refund_htlc(taker_payment_lock).compat().await); + if let CanRefundHtlc::HaveToWait(seconds_to_wait) = can_refund { + return ERR!("Too early to refund, wait until {}", now_ms() / 1000 + seconds_to_wait); } let fut = self.taker_coin.send_taker_refunds_payment( @@ -2141,6 +2183,8 @@ mod taker_swap_tests { TestCoin::ticker.mock_safe(|_| MockResult::Return("ticker")); TestCoin::swap_contract_address.mock_safe(|_| MockResult::Return(None)); + TestCoin::can_refund_htlc + .mock_safe(|_, _| MockResult::Return(Box::new(futures01::future::ok(CanRefundHtlc::CanRefundNow)))); static mut MY_PAYMENT_SENT_CALLED: bool = false; TestCoin::check_if_my_payment_sent.mock_safe(|_, _, _, _, _, _, _| { @@ -2234,6 +2278,8 @@ mod taker_swap_tests { TestCoin::ticker.mock_safe(|_| MockResult::Return("ticker")); TestCoin::swap_contract_address.mock_safe(|_| MockResult::Return(None)); + TestCoin::can_refund_htlc + .mock_safe(|_, _| MockResult::Return(Box::new(futures01::future::ok(CanRefundHtlc::CanRefundNow)))); static mut SEARCH_TX_SPEND_CALLED: bool = false; TestCoin::search_for_swap_tx_spend_my.mock_safe(|_, _| { @@ -2271,6 +2317,8 @@ mod taker_swap_tests { TestCoin::ticker.mock_safe(|_| MockResult::Return("ticker")); TestCoin::swap_contract_address.mock_safe(|_| MockResult::Return(None)); + TestCoin::can_refund_htlc + .mock_safe(|_, _| MockResult::Return(Box::new(futures01::future::ok(CanRefundHtlc::HaveToWait(1000))))); static mut SEARCH_TX_SPEND_CALLED: bool = false; TestCoin::search_for_swap_tx_spend_my.mock_safe(|_, _| { @@ -2280,8 +2328,8 @@ mod taker_swap_tests { let maker_coin = MmCoinEnum::Test(TestCoin::default()); let taker_coin = MmCoinEnum::Test(TestCoin::default()); let (taker_swap, _) = TakerSwap::load_from_saved(ctx, maker_coin, taker_coin, taker_saved_swap).unwrap(); - taker_swap.w().data.taker_payment_lock = (now_ms() / 1000) - 3690; - assert!(block_on(taker_swap.recover_funds()).is_err()); + let error = block_on(taker_swap.recover_funds()).unwrap_err(); + assert!(error.contains("Too early to refund")); assert!(unsafe { SEARCH_TX_SPEND_CALLED }); } diff --git a/mm2src/mm2_test_helpers/src/for_tests.rs b/mm2src/mm2_test_helpers/src/for_tests.rs index cee2c29dc2..9a7751d666 100644 --- a/mm2src/mm2_test_helpers/src/for_tests.rs +++ b/mm2src/mm2_test_helpers/src/for_tests.rs @@ -100,7 +100,17 @@ pub const TAKER_ERROR_EVENTS: [&str; 13] = [ ]; pub const RICK: &str = "RICK"; +pub const RICK_ELECTRUM_ADDRS: &[&str] = &[ + "electrum1.cipig.net:10017", + "electrum2.cipig.net:10017", + "electrum3.cipig.net:10017", +]; pub const MORTY: &str = "MORTY"; +pub const MORTY_ELECTRUM_ADDRS: &[&str] = &[ + "electrum1.cipig.net:10018", + "electrum2.cipig.net:10018", + "electrum3.cipig.net:10018", +]; pub const ZOMBIE_TICKER: &str = "ZOMBIE"; pub const ZOMBIE_ELECTRUMS: &[&str] = &["zombie.sirseven.me:10033"]; pub const ZOMBIE_LIGHTWALLETD_URLS: &[&str] = &["http://zombie.sirseven.me:443"]; From 035146401f40f3bc47d8fa65446a8c2a7010381b Mon Sep 17 00:00:00 2001 From: borngraced <51881311+borngraced@users.noreply.github.com> Date: Wed, 10 Aug 2022 06:25:05 -0400 Subject: [PATCH 007/249] [r2r] update metrics related dep && refactoring (#1312) * update metrics related dep * wasm don't mess up! * wip * wip * impl collect_json done * added collect_json test * created new mm2_metrics crate and other wip impls * update wasm * rust fmt check * wip * MetricsArc && MetricsWeak * wip * wip * removed unused dep * refactoring * removed unused deps * collect gauge from bits to u64 * gauge now export u64 * updated histogram and gauge * iteration * reiteration * reiteration * Iterations * histogram, label macro and other improvements * fixed conflicts * fix udeps, impl wasm for metrics and other iterations * code improvements * improved TryRecorder for MetricsWeak Added unit test for test_prometheus_format * renamed native to mm_metrics unit tests working correctly code improvements * fix cargo fmt * changed incompatible quanta dep --v * PR review fixes - 1 * PR Review fixes -2 * Fix WASM * PR review fixes - wip * PR review fixes * Changes to error handling and unit tests * Fixed test_collect_json test deltas * PR review fixes - 3 * edited lp_commands_legacy * PR review fix - last * Minor changes - r2r * fix cargo clippy * final pr review changes * fixed build error --- Cargo.lock | 285 +++---- Cargo.toml | 1 + deny.toml | 2 - mm2src/coins/Cargo.toml | 2 +- mm2src/coins/lp_coins.rs | 15 +- mm2src/coins/qrc20/history.rs | 12 +- mm2src/coins/utxo.rs | 2 +- mm2src/coins/utxo/bch.rs | 2 +- mm2src/coins/utxo/bch_and_slp_tx_history.rs | 2 +- mm2src/coins/utxo/qtum.rs | 2 +- mm2src/coins/utxo/utxo_common.rs | 1 - mm2src/coins/utxo/utxo_standard.rs | 2 +- mm2src/coins_activation/Cargo.toml | 1 + .../src/bch_with_tokens_activation.rs | 2 +- .../src/eth_with_token_activation.rs | 3 +- .../src/platform_coin_with_tokens.rs | 2 +- .../src/solana_with_tokens_activation.rs | 2 +- mm2src/common/Cargo.toml | 6 - mm2src/common/common.rs | 2 - mm2src/common/mm_metrics/mod.rs | 92 --- mm2src/common/mm_metrics/native.rs | 756 ------------------ mm2src/common/mm_metrics/wasm.rs | 64 -- mm2src/mm2_core/Cargo.toml | 1 + mm2src/mm2_core/src/mm_ctx.rs | 13 +- mm2src/mm2_main/Cargo.toml | 2 +- mm2src/mm2_main/src/lp_native_dex.rs | 11 +- mm2src/mm2_main/src/lp_network.rs | 10 +- mm2src/mm2_main/src/lp_ordermatch.rs | 9 +- mm2src/mm2_main/src/mm2_tests.rs | 3 +- .../src/rpc/lp_commands/lp_commands_legacy.rs | 4 +- mm2src/mm2_metrics/Cargo.toml | 27 + mm2src/mm2_metrics/src/lib.rs | 125 +++ mm2src/mm2_metrics/src/mm_metrics.rs | 554 +++++++++++++ mm2src/mm2_metrics/src/recorder.rs | 195 +++++ mm2src/mm2_test_helpers/Cargo.toml | 1 + mm2src/mm2_test_helpers/src/for_tests.rs | 2 +- 36 files changed, 1103 insertions(+), 1112 deletions(-) delete mode 100644 mm2src/common/mm_metrics/mod.rs delete mode 100644 mm2src/common/mm_metrics/native.rs delete mode 100644 mm2src/common/mm_metrics/wasm.rs create mode 100644 mm2src/mm2_metrics/Cargo.toml create mode 100644 mm2src/mm2_metrics/src/lib.rs create mode 100644 mm2src/mm2_metrics/src/mm_metrics.rs create mode 100644 mm2src/mm2_metrics/src/recorder.rs diff --git a/Cargo.lock b/Cargo.lock index 999eb30ad8..605fce9c84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -268,11 +268,11 @@ dependencies = [ [[package]] name = "atomic-shim" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d20fdac7156779a1a30d970e838195558b4810dd06aa69e7c7461bdc518edf9b" +checksum = "67cd4b51d303cf3501c301e8125df442128d3c6d7c69f71b27833d253de47e77" dependencies = [ - "crossbeam", + "crossbeam-utils 0.8.8", ] [[package]] @@ -534,15 +534,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - [[package]] name = "bitvec" version = "0.18.5" @@ -1058,11 +1049,11 @@ dependencies = [ "lightning-invoice", "lightning-net-tokio", "lightning-persister", - "metrics", "mm2_core", "mm2_db", "mm2_err_handle", "mm2_io", + "mm2_metrics", "mm2_net", "mm2_number", "mm2_test_helpers", @@ -1131,6 +1122,7 @@ dependencies = [ "hex 0.4.2", "mm2_core", "mm2_err_handle", + "mm2_metrics", "mm2_number", "rpc", "rpc_task", @@ -1149,7 +1141,6 @@ dependencies = [ "arrayref", "async-trait", "backtrace", - "base64 0.10.1", "bytes 1.1.0", "cc", "cfg-if 1.0.0", @@ -1163,7 +1154,6 @@ dependencies = [ "futures-cpupool", "getrandom 0.2.6", "gstuff", - "hdrhistogram 7.1.0", "hex 0.4.2", "http 0.2.7", "http-body 0.1.0", @@ -1176,10 +1166,6 @@ dependencies = [ "lightning", "log 0.4.14", "log4rs", - "metrics", - "metrics-core", - "metrics-runtime", - "metrics-util", "parking_lot 0.12.0", "parking_lot_core 0.6.2", "rand 0.7.3", @@ -1952,6 +1938,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "enum-as-inner" version = "0.4.0" @@ -2691,27 +2683,6 @@ dependencies = [ "hashbrown 0.9.1", ] -[[package]] -name = "hdrhistogram" -version = "6.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d331ebcdbca4acbefe5da8c3299b2e246f198a8294cc5163354e743398b89d" -dependencies = [ - "byteorder 1.4.3", - "num-traits", -] - -[[package]] -name = "hdrhistogram" -version = "7.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3c22708574c44e924720c5b3a116326c688e6d532f438c77c007ec8768644f9" -dependencies = [ - "byteorder 1.4.3", - "crossbeam-channel 0.4.4", - "num-traits", -] - [[package]] name = "heck" version = "0.4.0" @@ -3014,20 +2985,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "im" -version = "15.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" -dependencies = [ - "bitmaps", - "rand_core 0.6.3", - "rand_xoshiro", - "sized-chunks", - "typenum", - "version_check", -] - [[package]] name = "impl-codec" version = "0.6.0" @@ -3074,9 +3031,9 @@ checksum = "5a9d968042a4902e08810946fc7cd5851eb75e80301342305af755ca06cb82ce" [[package]] name = "indexmap" -version = "1.7.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" +checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a" dependencies = [ "autocfg 1.0.0", "hashbrown 0.11.2", @@ -3862,6 +3819,15 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -3937,56 +3903,62 @@ dependencies = [ [[package]] name = "metrics" -version = "0.12.1" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b70227ece8711a1aa2f99655efd795d0cff297a5b9fe39645a93aacf6ad39d" +checksum = "142c53885123b68d94108295a09d4afe1a1388ed95b54d5dacd9a454753030f2" dependencies = [ - "metrics-core", + "ahash 0.7.6", + "metrics-macros", ] [[package]] -name = "metrics-core" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c064b3a1ff41f4bf6c91185c8a0caeccf8a8a27e9d0f92cc54cf3dbec812f48" - -[[package]] -name = "metrics-observer-prometheus" -version = "0.1.4" +name = "metrics-exporter-prometheus" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bfe24ad8285ef8b239232135a65f89cc5fa4690bbfaf8907f4bef38f8b08eba" +checksum = "953cbbb6f9ba4b9304f4df79b98cdc9d14071ed93065a9fca11c00c5d9181b66" dependencies = [ - "hdrhistogram 6.3.4", - "metrics-core", + "hyper", + "indexmap", + "ipnet", + "metrics", "metrics-util", + "parking_lot 0.11.1", + "quanta", + "thiserror", + "tokio", + "tracing", ] [[package]] -name = "metrics-runtime" -version = "0.13.1" +name = "metrics-macros" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce0e4f69639ccc0c6b2f0612164f9817349eb25545ed1ffb5ef3e1e1c1d220b4" +checksum = "49e30813093f757be5cf21e50389a24dc7dbb22c49f23b7e8f51d69b508a5ffa" dependencies = [ - "arc-swap", - "atomic-shim", - "crossbeam-utils 0.7.2", - "im", - "metrics", - "metrics-core", - "metrics-observer-prometheus", - "metrics-util", - "parking_lot 0.10.2", - "quanta", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.95", ] [[package]] name = "metrics-util" -version = "0.3.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d11f8090a8886339f9468a04eeea0711e4cf27538b134014664308041307a1c5" +checksum = "fd1f4b69bef1e2b392b2d4a12902f2af90bb438ba4a66aa222d1023fa6561b50" dependencies = [ - "crossbeam-epoch 0.8.2", - "serde", + "aho-corasick", + "atomic-shim", + "crossbeam-epoch 0.9.5", + "crossbeam-utils 0.8.8", + "hashbrown 0.11.2", + "indexmap", + "metrics", + "num_cpus", + "ordered-float", + "parking_lot 0.11.1", + "quanta", + "radix_trie", + "sketches-ddsketch", ] [[package]] @@ -4095,6 +4067,7 @@ dependencies = [ "hex 0.4.2", "keys", "lazy_static", + "mm2_metrics", "mm2_rpc", "primitives", "rand 0.7.3", @@ -4202,12 +4175,12 @@ dependencies = [ "keys", "lazy_static", "libc", - "metrics", "mm2-libp2p", "mm2_core", "mm2_db", "mm2_err_handle", "mm2_io", + "mm2_metrics", "mm2_net", "mm2_number", "mm2_rpc", @@ -4248,6 +4221,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "mm2_metrics" +version = "0.1.0" +dependencies = [ + "base64 0.10.1", + "common", + "derive_more", + "futures 0.3.15", + "hyper", + "hyper-rustls", + "itertools", + "metrics", + "metrics-exporter-prometheus", + "metrics-util", + "mm2_err_handle", + "serde", + "serde_derive", + "serde_json", + "wasm-timer", +] + [[package]] name = "mm2_net" version = "0.1.0" @@ -4321,6 +4315,7 @@ dependencies = [ "lazy_static", "mm2_core", "mm2_io", + "mm2_metrics", "mm2_net", "mm2_number", "rand 0.7.3", @@ -4414,6 +4409,15 @@ dependencies = [ "unsigned-varint 0.7.1", ] +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec 1.6.1", +] + [[package]] name = "nix" version = "0.23.1" @@ -4589,6 +4593,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "ordered-float" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +dependencies = [ + "num-traits", +] + [[package]] name = "ouroboros" version = "0.13.0" @@ -4732,16 +4745,6 @@ dependencies = [ "rustc_version 0.2.3", ] -[[package]] -name = "parking_lot" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" -dependencies = [ - "lock_api 0.3.4", - "parking_lot_core 0.7.2", -] - [[package]] name = "parking_lot" version = "0.11.1" @@ -4760,7 +4763,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ "lock_api 0.4.6", - "parking_lot_core 0.9.1", + "parking_lot_core 0.9.3", ] [[package]] @@ -4791,20 +4794,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "parking_lot_core" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" -dependencies = [ - "cfg-if 0.1.10", - "cloudabi 0.0.3", - "libc", - "redox_syscall 0.1.56", - "smallvec 1.6.1", - "winapi", -] - [[package]] name = "parking_lot_core" version = "0.8.0" @@ -4822,9 +4811,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if 1.0.0", "libc", @@ -4871,9 +4860,9 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "petgraph" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b305cc4569dd4e8765bab46261f67ef5d4d11a4b6e745100ee5dad8948b46c" +checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" dependencies = [ "fixedbitset", "indexmap", @@ -5187,11 +5176,17 @@ dependencies = [ [[package]] name = "quanta" -version = "0.3.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7a1905379198075914bc93d32a5465c40474f90a078bb13439cb00c547bcc" +checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8" dependencies = [ + "crossbeam-utils 0.8.8", "libc", + "mach", + "once_cell", + "raw-cpuid", + "wasi 0.10.2+wasi-snapshot-preview1", + "web-sys", "winapi", ] @@ -5266,6 +5261,16 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.3.23" @@ -5487,12 +5492,12 @@ dependencies = [ ] [[package]] -name = "rand_xoshiro" -version = "0.6.0" +name = "raw-cpuid" +version = "10.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +checksum = "738bc47119e3eeccc7e94c4a506901aea5e7b4944ecd0829cbebf4af04ceda12" dependencies = [ - "rand_core 0.6.3", + "bitflags", ] [[package]] @@ -6332,14 +6337,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "833011ca526bd88f16778d32c699d325a9ad302fa06381cd66f7be63351d3f6d" [[package]] -name = "sized-chunks" -version = "0.6.5" +name = "sketches-ddsketch" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" -dependencies = [ - "bitmaps", - "typenum", -] +checksum = "04d2ecae5fcf33b122e2e6bd520a57ccf152d2dde3b38c71039df1a6867264ee" [[package]] name = "slab" @@ -8605,9 +8606,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -8618,33 +8619,33 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_i686_gnu" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_x86_64_gnu" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "winreg" diff --git a/Cargo.toml b/Cargo.toml index e224d8eafb..5407e4c746 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "mm2src/mm2_err_handle", "mm2src/mm2_test_helpers", "mm2src/mm2_libp2p", + "mm2src/mm2_metrics", "mm2src/mm2_main", "mm2src/mm2_net", "mm2src/mm2_number", diff --git a/deny.toml b/deny.toml index 2c6b571716..cf6377f2c7 100644 --- a/deny.toml +++ b/deny.toml @@ -48,11 +48,9 @@ notice = "warn" # A list of advisory IDs to ignore. Note that ignored advisories will still # output a note when they are encountered. -# RUSTSEC-2021-0113 is related to metrics-util crate that is not actively used for now despite being still present in deps tree # RUSTSEC-2020-0071 is related to time crate, which is used only by chrono in our deps tree, remove when https://github.com/chronotope/chrono/issues/700 is resolved # RUSTSEC-2022-0040 is related to owning-ref, which seems unmaintained. We need to find a way to get rid of it. https://github.com/KomodoPlatform/atomicDEX-API/issues/1429 ignore = [ - "RUSTSEC-2021-0113", "RUSTSEC-2020-0071", "RUSTSEC-2022-0040", #"RUSTSEC-0000-0000", diff --git a/mm2src/coins/Cargo.toml b/mm2src/coins/Cargo.toml index 617df41d6f..9a7886301b 100644 --- a/mm2src/coins/Cargo.toml +++ b/mm2src/coins/Cargo.toml @@ -51,10 +51,10 @@ lazy_static = "1.4" libc = "0.2" lightning = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } lightning-invoice = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } -metrics = "0.12" mm2_core = { path = "../mm2_core" } mm2_err_handle = { path = "../mm2_err_handle" } mm2_io = { path = "../mm2_io" } +mm2_metrics = { path = "../mm2_metrics" } mm2_net = { path = "../mm2_net" } mm2_number = { path = "../mm2_number" } mocktopus = "0.7.0" diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 183ba32579..6fb9553a27 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -28,13 +28,13 @@ #[macro_use] extern crate common; #[macro_use] extern crate gstuff; #[macro_use] extern crate lazy_static; +#[macro_use] extern crate mm2_metrics; #[macro_use] extern crate serde_derive; #[macro_use] extern crate serde_json; #[macro_use] extern crate ser_error_derive; use async_trait::async_trait; use base58::FromBase58Error; -use common::mm_metrics::MetricsWeak; use common::{calc_total_pages, now_ms, ten, HttpStatusCode}; use crypto::{Bip32Error, CryptoCtx, DerivationPath}; use derive_more::Display; @@ -46,8 +46,9 @@ use http::{Response, StatusCode}; use keys::{AddressFormat as UtxoAddressFormat, KeyPair, NetworkPrefix as CashAddrPrefix}; use mm2_core::mm_ctx::{from_ctx, MmArc}; use mm2_err_handle::prelude::*; -use mm2_number::bigdecimal::{BigDecimal, ParseBigDecimalError, Zero}; -use mm2_number::MmNumber; +use mm2_metrics::MetricsWeak; +use mm2_number::{bigdecimal::{BigDecimal, ParseBigDecimalError, Zero}, + MmNumber}; use rpc::v1::types::{Bytes as BytesJson, H256 as H256Json}; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::{self as json, Value as Json}; @@ -2179,16 +2180,16 @@ impl RpcTransportEventHandler for CoinTransportMetrics { fn on_outgoing_request(&self, data: &[u8]) { mm_counter!(self.metrics, "rpc_client.traffic.out", data.len() as u64, - "coin" => self.ticker.clone(), "client" => self.client.clone()); + "coin" => self.ticker.to_owned(), "client" => self.client.to_owned()); mm_counter!(self.metrics, "rpc_client.request.count", 1, - "coin" => self.ticker.clone(), "client" => self.client.clone()); + "coin" => self.ticker.to_owned(), "client" => self.client.to_owned()); } fn on_incoming_response(&self, data: &[u8]) { mm_counter!(self.metrics, "rpc_client.traffic.in", data.len() as u64, - "coin" => self.ticker.clone(), "client" => self.client.clone()); + "coin" => self.ticker.to_owned(), "client" => self.client.to_owned()); mm_counter!(self.metrics, "rpc_client.response.count", 1, - "coin" => self.ticker.clone(), "client" => self.client.clone()); + "coin" => self.ticker.to_owned(), "client" => self.client.to_owned()); } fn on_connected(&self, _address: String) -> Result<(), String> { diff --git a/mm2src/coins/qrc20/history.rs b/mm2src/coins/qrc20/history.rs index 967ba82ba4..0b160c1994 100644 --- a/mm2src/coins/qrc20/history.rs +++ b/mm2src/coins/qrc20/history.rs @@ -3,8 +3,8 @@ use crate::utxo::{RequestTxHistoryResult, UtxoFeeDetails}; use crate::{CoinsContext, TxFeeDetails, TxHistoryResult}; use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use common::jsonrpc_client::JsonRpcErrorType; -use common::mm_metrics::MetricsArc; use itertools::Itertools; +use mm2_metrics::MetricsArc; use script_pubkey::{extract_contract_call_from_script, extract_gas_from_script, ExtractGasEnum}; use std::collections::HashMap; use std::io::Cursor; @@ -813,7 +813,7 @@ fn is_transfer_event_log(log: &LogEntry) -> bool { mod tests { use super::*; use common::block_on; - use common::mm_metrics::{MetricType, MetricsJson, MetricsOps}; + use mm2_metrics::{MetricType, MetricsJson, MetricsOps}; use mm2_test_helpers::for_tests::find_metrics_in_json; use qrc20_tests::qrc20_coin_for_test; @@ -840,7 +840,7 @@ mod tests { 72, 172, 110, 180, 13, 123, 179, 10, 49, ]; let (ctx, coin) = qrc20_coin_for_test(&priv_key, None); - ctx.metrics.init().unwrap(); + ctx.metrics.init(); let tx_hash: H256Json = hex::decode("85ede12ccc12fb1709c4d9e403e96c0c394b0916f2f6098d41d8dfa00013fcdb") .unwrap() @@ -872,7 +872,7 @@ mod tests { 72, 172, 110, 180, 13, 123, 179, 10, 49, ]; let (ctx, coin) = qrc20_coin_for_test(&priv_key, None); - ctx.metrics.init().unwrap(); + ctx.metrics.init(); let tx_hash: H256Json = hex::decode("85ede12ccc12fb1709c4d9e403e96c0c394b0916f2f6098d41d8dfa00013fcdb") .unwrap() @@ -914,7 +914,7 @@ mod tests { 72, 172, 110, 180, 13, 123, 179, 10, 49, ]; let (ctx, coin) = qrc20_coin_for_test(&priv_key, None); - ctx.metrics.init().unwrap(); + ctx.metrics.init(); let tx_hash: H256Json = hex::decode("85ede12ccc12fb1709c4d9e403e96c0c394b0916f2f6098d41d8dfa00013fcdb") .unwrap() @@ -1008,7 +1008,7 @@ mod tests { let (ctx, coin) = qrc20_coin_for_test(&priv_key, None); let metrics = MetricsArc::new(); - metrics.init().unwrap(); + metrics.init(); let tx_hash_invalid: H256Json = hex::decode("0000000000000000000000000000000000000000000000000000000000000000") .unwrap() diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 45815465b0..526b1acb8f 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -46,7 +46,6 @@ use chain::{OutPoint, TransactionOutput, TxHashAlgo}; #[cfg(not(target_arch = "wasm32"))] use common::first_char_to_upper; use common::jsonrpc_client::JsonRpcError; -use common::mm_metrics::MetricsArc; use common::now_ms; use crypto::trezor::utxo::TrezorUtxoCoin; use crypto::{Bip32DerPathOps, Bip32Error, Bip44Chain, Bip44DerPathError, Bip44PathToAccount, Bip44PathToCoin, @@ -63,6 +62,7 @@ pub use keys::{Address, AddressFormat as UtxoAddressFormat, AddressHashEnum, Key use lightning_invoice::Currency as LightningCurrency; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; #[cfg(test)] use mocktopus::macros::*; use num_traits::ToPrimitive; diff --git a/mm2src/coins/utxo/bch.rs b/mm2src/coins/utxo/bch.rs index 1cbaea6a5e..567e1b836a 100644 --- a/mm2src/coins/utxo/bch.rs +++ b/mm2src/coins/utxo/bch.rs @@ -11,13 +11,13 @@ use crate::{BlockHeightAndTime, CanRefundHtlc, CoinBalance, CoinProtocol, Negoti SwapOps, TradePreimageValue, TransactionFut, TransactionType, TxFeeDetails, UnexpectedDerivationMethod, ValidateAddressResult, ValidatePaymentInput, VerificationResult, WithdrawFut}; use common::log::warn; -use common::mm_metrics::MetricsArc; use derive_more::Display; use futures::{FutureExt, TryFutureExt}; use itertools::Either as EitherIter; use keys::hash::H256; use keys::CashAddress; pub use keys::NetworkPrefix as CashAddrPrefix; +use mm2_metrics::MetricsArc; use mm2_number::MmNumber; use serde_json::{self as json, Value as Json}; use serialization::{deserialize, CoinVariant}; diff --git a/mm2src/coins/utxo/bch_and_slp_tx_history.rs b/mm2src/coins/utxo/bch_and_slp_tx_history.rs index 5da1212e9d..65b14e1a69 100644 --- a/mm2src/coins/utxo/bch_and_slp_tx_history.rs +++ b/mm2src/coins/utxo/bch_and_slp_tx_history.rs @@ -9,9 +9,9 @@ use crate::{BlockHeightAndTime, HistorySyncState, MarketCoinOps}; use async_trait::async_trait; use common::executor::Timer; use common::log::{error, info}; -use common::mm_metrics::MetricsArc; use common::state_machine::prelude::*; use futures::compat::Future01CompatExt; +use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; use rpc::v1::types::H256 as H256Json; use std::collections::HashMap; diff --git a/mm2src/coins/utxo/qtum.rs b/mm2src/coins/utxo/qtum.rs index 70adae0cc2..dd28557da1 100644 --- a/mm2src/coins/utxo/qtum.rs +++ b/mm2src/coins/utxo/qtum.rs @@ -19,12 +19,12 @@ use crate::{eth, CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, Delegatio GetWithdrawSenderAddress, NegotiateSwapContractAddrErr, PrivKeyBuildPolicy, SearchForSwapTxSpendInput, SignatureResult, StakingInfosFut, SwapOps, TradePreimageValue, TransactionFut, UnexpectedDerivationMethod, ValidateAddressResult, ValidatePaymentInput, VerificationResult, WithdrawFut, WithdrawSenderAddress}; -use common::mm_metrics::MetricsArc; use crypto::trezor::utxo::TrezorUtxoCoin; use crypto::Bip44Chain; use ethereum_types::H160; use futures::{FutureExt, TryFutureExt}; use keys::AddressHashEnum; +use mm2_metrics::MetricsArc; use mm2_number::MmNumber; use serde::Serialize; use serialization::CoinVariant; diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 547086f00f..f99cb247a4 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -22,7 +22,6 @@ use chain::{OutPoint, TransactionOutput}; use common::executor::Timer; use common::jsonrpc_client::JsonRpcErrorType; use common::log::{debug, error, info, warn}; -use common::mm_metrics::MetricsArc; use common::{now_ms, one_hundred, ten_f64}; use crypto::{Bip32DerPathOps, Bip44Chain, Bip44DerPathError, Bip44DerivationPath, RpcDerivationPath}; use futures::compat::Future01CompatExt; diff --git a/mm2src/coins/utxo/utxo_standard.rs b/mm2src/coins/utxo/utxo_standard.rs index b48f9bfe59..d03123fdc3 100644 --- a/mm2src/coins/utxo/utxo_standard.rs +++ b/mm2src/coins/utxo/utxo_standard.rs @@ -17,10 +17,10 @@ use crate::{CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, GetWithdrawSen NegotiateSwapContractAddrErr, PrivKeyBuildPolicy, SearchForSwapTxSpendInput, SignatureResult, SwapOps, TradePreimageValue, TransactionFut, ValidateAddressResult, ValidatePaymentInput, VerificationResult, WithdrawFut, WithdrawSenderAddress}; -use common::mm_metrics::MetricsArc; use crypto::trezor::utxo::TrezorUtxoCoin; use crypto::Bip44Chain; use futures::{FutureExt, TryFutureExt}; +use mm2_metrics::MetricsArc; use mm2_number::MmNumber; use serialization::coin_variant_by_ticker; use utxo_signer::UtxoSignerOps; diff --git a/mm2src/coins_activation/Cargo.toml b/mm2src/coins_activation/Cargo.toml index 89f4ac538d..234ff102ef 100644 --- a/mm2src/coins_activation/Cargo.toml +++ b/mm2src/coins_activation/Cargo.toml @@ -12,6 +12,7 @@ common = { path = "../common" } ethereum-types = { version = "0.4", default-features = false, features = ["std", "serialize"] } mm2_core = { path = "../mm2_core" } mm2_err_handle = { path = "../mm2_err_handle" } +mm2_metrics = { path = "../mm2_metrics" } mm2_number = { path = "../mm2_number" } crypto = { path = "../crypto" } derive_more = "0.99" diff --git a/mm2src/coins_activation/src/bch_with_tokens_activation.rs b/mm2src/coins_activation/src/bch_with_tokens_activation.rs index e0f3102b47..0ce4580e12 100644 --- a/mm2src/coins_activation/src/bch_with_tokens_activation.rs +++ b/mm2src/coins_activation/src/bch_with_tokens_activation.rs @@ -11,11 +11,11 @@ use coins::utxo::UtxoCommonOps; use coins::{CoinBalance, CoinProtocol, MarketCoinOps, MmCoin, PrivKeyNotAllowed, UnexpectedDerivationMethod}; use common::executor::spawn; use common::log::info; -use common::mm_metrics::MetricsArc; use common::Future01CompatExt; use futures::future::{abortable, AbortHandle}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; use serde_derive::{Deserialize, Serialize}; use serde_json::Value as Json; diff --git a/mm2src/coins_activation/src/eth_with_token_activation.rs b/mm2src/coins_activation/src/eth_with_token_activation.rs index 8e75488d53..33bf0b24c4 100644 --- a/mm2src/coins_activation/src/eth_with_token_activation.rs +++ b/mm2src/coins_activation/src/eth_with_token_activation.rs @@ -9,10 +9,11 @@ use coins::{eth::{v2_activation::{eth_coin_from_conf_and_request_v2, Erc20Protoc Erc20TokenInfo, EthCoin, EthCoinType}, my_tx_history_v2::TxHistoryStorage, CoinBalance, CoinProtocol, MarketCoinOps, MmCoin}; -use common::{mm_metrics::MetricsArc, Future01CompatExt}; +use common::Future01CompatExt; use futures::future::AbortHandle; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; use serde::{Deserialize, Serialize}; use serde_json::Value as Json; diff --git a/mm2src/coins_activation/src/platform_coin_with_tokens.rs b/mm2src/coins_activation/src/platform_coin_with_tokens.rs index ecd1dac377..61f58ed7cb 100644 --- a/mm2src/coins_activation/src/platform_coin_with_tokens.rs +++ b/mm2src/coins_activation/src/platform_coin_with_tokens.rs @@ -3,12 +3,12 @@ use async_trait::async_trait; use coins::my_tx_history_v2::TxHistoryStorage; use coins::tx_history_storage::{CreateTxHistoryStorageError, TxHistoryStorageBuilder}; use coins::{lp_coinfind, CoinProtocol, CoinsContext, MmCoinEnum}; -use common::mm_metrics::MetricsArc; use common::{log, HttpStatusCode, StatusCode}; use derive_more::Display; use futures::future::AbortHandle; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; use ser_error_derive::SerializeErrorType; use serde_derive::{Deserialize, Serialize}; diff --git a/mm2src/coins_activation/src/solana_with_tokens_activation.rs b/mm2src/coins_activation/src/solana_with_tokens_activation.rs index 71fdcf4027..851d5bff5d 100644 --- a/mm2src/coins_activation/src/solana_with_tokens_activation.rs +++ b/mm2src/coins_activation/src/solana_with_tokens_activation.rs @@ -10,11 +10,11 @@ use coins::my_tx_history_v2::TxHistoryStorage; use coins::solana::spl::{SplProtocolConf, SplTokenCreationError}; use coins::{solana_coin_from_conf_and_params, BalanceError, CoinBalance, CoinProtocol, MarketCoinOps, SolanaActivationParams, SolanaCoin, SplToken}; -use common::mm_metrics::MetricsArc; use common::Future01CompatExt; use futures::future::AbortHandle; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; use serde_derive::{Deserialize, Serialize}; use serde_json::Value as Json; diff --git a/mm2src/common/Cargo.toml b/mm2src/common/Cargo.toml index ddb1e0015c..0ed80ea559 100644 --- a/mm2src/common/Cargo.toml +++ b/mm2src/common/Cargo.toml @@ -16,7 +16,6 @@ track-ctx-pointer = ["shared_ref_counter/enable", "shared_ref_counter/log"] arrayref = "0.3" async-trait = "0.1" backtrace = "0.3" -base64 = "0.10.0" bytes = "1.1" cfg-if = "1.0" crossbeam = "0.7" @@ -60,17 +59,12 @@ anyhow = "1.0" chrono = "0.4" crossterm = "0.20" gstuff = { version = "0.7", features = ["crossterm", "nightly"] } -hdrhistogram = { version = "7.0", default-features = false, features = ["sync"] } hyper = { version = "0.14.11", features = ["client", "http2", "server", "tcp"] } # using webpki-tokio to avoid rejecting valid certificates # got "invalid certificate: UnknownIssuer" for https://ropsten.infura.io on iOS using default-features hyper-rustls = { version = "0.23", default-features = false, features = ["http1", "http2", "webpki-tokio"] } libc = { version = "0.2" } log4rs = { version = "1.0", default-features = false, features = ["console_appender", "pattern_encoder"] } -metrics = { version = "0.12" } -metrics-runtime = { version = "0.13", default-features = false, features = ["metrics-observer-prometheus"] } -metrics-core = { version = "0.5" } -metrics-util = { version = "0.3" } tokio = { version = "1.7", features = ["io-util", "rt-multi-thread", "net"] } [target.'cfg(windows)'.dependencies] diff --git a/mm2src/common/common.rs b/mm2src/common/common.rs index f07150ce6a..dee3b1de5a 100644 --- a/mm2src/common/common.rs +++ b/mm2src/common/common.rs @@ -96,8 +96,6 @@ macro_rules! drop_mutability { pub mod jsonrpc_client; #[macro_use] pub mod log; -#[macro_use] -pub mod mm_metrics; pub mod crash_reports; pub mod custom_futures; diff --git a/mm2src/common/mm_metrics/mod.rs b/mm2src/common/mm_metrics/mod.rs deleted file mode 100644 index 6c84d4de48..0000000000 --- a/mm2src/common/mm_metrics/mod.rs +++ /dev/null @@ -1,92 +0,0 @@ -use crate::log::LogWeak; -use serde_json::{Value as Json, Value}; -use std::collections::HashMap; -use std::sync::{Arc, Weak}; - -#[cfg(not(target_arch = "wasm32"))] mod native; -#[cfg(not(target_arch = "wasm32"))] -pub use native::{prometheus, Clock, Metrics, TrySink}; - -#[cfg(target_arch = "wasm32")] mod wasm; -#[cfg(target_arch = "wasm32")] pub use wasm::{Clock, Metrics}; - -pub trait MetricsOps { - /// If the instance was not initialized yet, create the `receiver` else return an error. - fn init(&self) -> Result<(), String>; - - /// Create new Metrics instance and spawn the metrics recording into the log, else return an error. - fn init_with_dashboard(&self, log_state: LogWeak, record_interval: f64) -> Result<(), String>; - - /// Handle for sending metric samples. - fn clock(&self) -> Result; - - /// Collect the metrics as Json. - fn collect_json(&self) -> Result; -} - -pub trait ClockOps { - fn now(&self) -> u64; -} - -#[derive(Clone, Default)] -pub struct MetricsArc(pub Arc); - -impl MetricsOps for MetricsArc { - fn init(&self) -> Result<(), String> { self.0.init() } - - fn init_with_dashboard(&self, log_state: LogWeak, record_interval: f64) -> Result<(), String> { - self.0.init_with_dashboard(log_state, record_interval) - } - - fn clock(&self) -> Result { self.0.clock() } - - fn collect_json(&self) -> Result { self.0.collect_json() } -} - -impl MetricsArc { - /// Create new `Metrics` instance - pub fn new() -> MetricsArc { MetricsArc(Arc::new(Default::default())) } - - /// Try to obtain the `Metrics` from the weak pointer. - pub fn from_weak(weak: &MetricsWeak) -> Option { weak.0.upgrade().map(MetricsArc) } - - /// Create a weak pointer from `MetricsWeak`. - pub fn weak(&self) -> MetricsWeak { MetricsWeak(Arc::downgrade(&self.0)) } -} - -#[derive(Clone, Default)] -pub struct MetricsWeak(pub Weak); - -impl MetricsWeak { - /// Create a default MmWeak without allocating any memory. - pub fn new() -> MetricsWeak { MetricsWeak::default() } - - pub fn dropped(&self) -> bool { self.0.strong_count() == 0 } -} - -#[derive(Serialize, Debug, Default, Deserialize)] -pub struct MetricsJson { - pub metrics: Vec, -} - -#[derive(Eq, Debug, Deserialize, PartialEq, Serialize)] -#[serde(rename_all = "lowercase")] -#[serde(tag = "type")] -pub enum MetricType { - Counter { - key: String, - labels: HashMap, - value: u64, - }, - Gauge { - key: String, - labels: HashMap, - value: i64, - }, - Histogram { - key: String, - labels: HashMap, - #[serde(flatten)] - quantiles: HashMap, - }, -} diff --git a/mm2src/common/mm_metrics/native.rs b/mm2src/common/mm_metrics/native.rs deleted file mode 100644 index fafa3beab0..0000000000 --- a/mm2src/common/mm_metrics/native.rs +++ /dev/null @@ -1,756 +0,0 @@ -use super::*; -use crate::executor::{spawn, Timer}; -use crate::log::error; -use gstuff::Constructible; -use hdrhistogram::Histogram; -use itertools::Itertools; -use metrics_core::{Builder, Drain, Key, Label, Observe, Observer, ScopedString}; -use metrics_runtime::{observers::PrometheusBuilder, Receiver}; -use metrics_util::{parse_quantiles, Quantile}; -use serde_json as json; -use std::collections::HashMap; -use std::fmt; -use std::slice::Iter; - -use crate::log::{LogArc, Tag}; -pub use metrics_runtime::Sink; - -/// Increment counter if an MmArc is not dropped yet and metrics system is initialized already. -#[macro_export] -macro_rules! mm_counter { - ($metrics:expr, $name:expr, $value:expr) => {{ - if let Some(mut sink) = $crate::mm_metrics::TrySink::try_sink(&$metrics) { - sink.increment_counter($name, $value); - } - }}; - ($metrics:expr, $name:expr, $value:expr, $($label_key:expr => $label_val:expr),+) => {{ - use metrics::labels; - if let Some(mut sink) = $crate::mm_metrics::TrySink::try_sink(&$metrics) { - let labels = labels!( $($label_key => $label_val),+ ); - sink.increment_counter_with_labels($name, $value, labels); - } - }}; -} - -/// Update gauge if an MmArc is not dropped yet and metrics system is initialized already. -#[macro_export] -macro_rules! mm_gauge { - ($metrics:expr, $name:expr, $value:expr) => {{ - if let Some(mut sink) = $crate::mm_metrics::TrySink::try_sink(&$metrics) { - sink.update_gauge($name, $value); - } - }}; - - ($metrics:expr, $name:expr, $value:expr, $($label_key:expr => $label_val:expr),+) => {{ - use metrics::labels; - if let Some(mut sink) = $crate::mm_metrics::TrySink::try_sink(&$metrics) { - let labels = labels!( $($label_key => $label_val),+ ); - sink.update_gauge_with_labels($name, $value, labels); - } - }}; -} - -/// Pass new timing value if an MmArc is not dropped yet and metrics system is initialized already. -#[macro_export] -macro_rules! mm_timing { - ($metrics:expr, $name:expr, $start:expr, $end:expr) => {{ - if let Some(mut sink) = $crate::mm_metrics::TrySink::try_sink(&$metrics) { - sink.record_timing($name, $start, $end); - } - }}; - - ($metrics:expr, $name:expr, $start:expr, $end:expr, $($label_key:expr => $label_val:expr),+) => {{ - use metrics::labels; - if let Some(mut sink) = $crate::mm_metrics::TrySink::try_sink(&$metrics) { - let labels = labels!( $($label_key => $label_val),+ ); - sink.record_timing_with_labels($name, $start, $end, labels); - } - }}; -} - -/// Default quantiles are "min" and "max" -const QUANTILES: &[f64] = &[0.0, 1.0]; - -pub trait TrySink { - fn try_sink(&self) -> Option; -} - -impl TrySink for MetricsArc { - fn try_sink(&self) -> Option { self.0.sink().ok() } -} - -impl TrySink for MetricsWeak { - fn try_sink(&self) -> Option { - let metrics = MetricsArc::from_weak(self)?; - metrics.0.sink().ok() - } -} - -pub struct Clock { - sink: Sink, -} - -impl From for Clock { - fn from(sink: Sink) -> Self { Clock { sink } } -} - -impl ClockOps for Clock { - fn now(&self) -> u64 { self.sink.now() } -} - -#[derive(Default)] -pub struct Metrics { - /// `Receiver` receives and collect all the metrics sent through the `sink`. - /// The `receiver` can be initialized only once time. - receiver: Constructible, -} - -impl MetricsOps for Metrics { - fn init(&self) -> Result<(), String> { - if self.receiver.is_some() { - return ERR!("metrics system is initialized already"); - } - - let receiver = try_s!(Receiver::builder().build()); - let _ = try_s!(self.receiver.pin(receiver)); - - Ok(()) - } - - fn init_with_dashboard(&self, log_state: LogWeak, record_interval: f64) -> Result<(), String> { - self.init()?; - - let controller = self.receiver.as_option().unwrap().controller(); - - let observer = TagObserver::new(QUANTILES); - let exporter = TagExporter { - log_state, - controller, - observer, - }; - - spawn(exporter.run(record_interval)); - - Ok(()) - } - - fn clock(&self) -> Result { self.sink().map_err(|e| ERRL!("{}", e)).map(Clock::from) } - - fn collect_json(&self) -> Result { - let receiver = try_s!(self.try_receiver()); - let controller = receiver.controller(); - - let mut observer = JsonObserver::new(QUANTILES); - - controller.observe(&mut observer); - - observer.into_json() - } -} - -impl Metrics { - /// Try get receiver. - fn try_receiver(&self) -> Result<&Receiver, String> { - self.receiver.ok_or("metrics system is not initialized yet".into()) - } - - fn sink(&self) -> Result { Ok(try_s!(self.try_receiver()).sink()) } - - /// Collect the metrics in Prometheus format. - pub fn collect_prometheus_format(&self) -> Result { - let receiver = try_s!(self.try_receiver()); - let controller = receiver.controller(); - - let mut observer = PrometheusBuilder::new().set_quantiles(QUANTILES).build(); - controller.observe(&mut observer); - - Ok(observer.drain()) - } -} - -type MetricName = ScopedString; - -type MetricLabels = Vec