From 94feb067ecad4e53076b29d8fc60ba6cf928ccad Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Tue, 1 Mar 2022 14:30:12 +0200 Subject: [PATCH 01/49] WIP: Channels history in SQL storage --- Cargo.lock | 2 + mm2src/coins/lightning.rs | 33 ++-- mm2src/coins/lightning/ln_connections.rs | 6 +- mm2src/coins/lightning/ln_errors.rs | 5 +- mm2src/coins/lightning/ln_utils.rs | 30 ++-- .../lightning_background_processor/src/lib.rs | 12 +- mm2src/coins/lightning_persister/Cargo.toml | 2 + mm2src/coins/lightning_persister/src/lib.rs | 163 +++++++++++++++--- .../coins/lightning_persister/src/storage.rs | 19 +- mm2src/coins/sql_tx_history_storage.rs | 30 +--- mm2src/db_common/src/sqlite.rs | 21 ++- 11 files changed, 237 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 15e3469c42..41e4f9f749 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3007,6 +3007,8 @@ dependencies = [ "async-trait", "bitcoin", "common", + "db_common", + "hex 0.4.3", "libc", "lightning", "parking_lot 0.11.1", diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 38d1b49b24..b77dad0627 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -32,8 +32,8 @@ use lightning::util::config::UserConfig; use lightning_background_processor::BackgroundProcessor; use lightning_invoice::utils::create_invoice_from_channelmanager; use lightning_invoice::Invoice; -use lightning_persister::storage::{NodesAddressesMapShared, Storage}; -use lightning_persister::FilesystemPersister; +use lightning_persister::storage::{FileSystemStorage, NodesAddressesMapShared}; +use lightning_persister::LightningPersister; use ln_conf::{ChannelOptions, LightningCoinConf, PlatformCoinConfirmations}; use ln_connections::{connect_to_node, ConnectToNodeRes}; use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelError, CloseChannelResult, @@ -140,13 +140,14 @@ pub struct LightningCoin { /// The lightning node invoice payer. pub invoice_payer: Arc>>, /// The lightning node persister that takes care of writing/reading data from storage. - pub persister: Arc, + pub persister: Arc, /// The mutex storing the inbound payments info. pub inbound_payments: PaymentsMapShared, /// The mutex storing the outbound payments info. pub outbound_payments: PaymentsMapShared, - /// The mutex storing the addresses of the nodes that are used for reconnecting. - pub nodes_addresses: NodesAddressesMapShared, + /// The mutex storing the addresses of the nodes that the lightning node has open channels with, + /// these addresses are used for reconnecting. + pub open_channel_nodes: NodesAddressesMapShared, } impl fmt::Debug for LightningCoin { @@ -537,10 +538,13 @@ pub async fn connect_to_lightning_node(ctx: MmArc, req: ConnectToNodeRequest) -> // If a node that we have an open channel with changed it's address, "connect_to_lightning_node" // can be used to reconnect to the new address while saving this new address for reconnections. if let ConnectToNodeRes::ConnectedSuccessfully(_, _) = res { - if let Entry::Occupied(mut entry) = ln_coin.nodes_addresses.lock().entry(node_pubkey) { + if let Entry::Occupied(mut entry) = ln_coin.open_channel_nodes.lock().entry(node_pubkey) { entry.insert(node_addr); } - ln_coin.persister.save_nodes_addresses(ln_coin.nodes_addresses).await?; + ln_coin + .persister + .save_nodes_addresses(ln_coin.open_channel_nodes) + .await?; } Ok(res.to_string()) @@ -654,8 +658,11 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes } // Saving node data to reconnect to it on restart - ln_coin.nodes_addresses.lock().insert(node_pubkey, node_addr); - ln_coin.persister.save_nodes_addresses(ln_coin.nodes_addresses).await?; + ln_coin.open_channel_nodes.lock().insert(node_pubkey, node_addr); + ln_coin + .persister + .save_nodes_addresses(ln_coin.open_channel_nodes) + .await?; Ok(OpenChannelResponse { temporary_channel_id: temp_channel_id.into(), @@ -784,8 +791,8 @@ pub async fn generate_invoice( MmCoinEnum::LightningCoin(c) => c, _ => return MmError::err(GenerateInvoiceError::UnsupportedCoin(coin.ticker().to_string())), }; - let nodes_addresses = ln_coin.nodes_addresses.lock().clone(); - for (node_pubkey, node_addr) in nodes_addresses { + let open_channel_nodes = ln_coin.open_channel_nodes.lock().clone(); + for (node_pubkey, node_addr) in open_channel_nodes { connect_to_node(node_pubkey, node_addr, ln_coin.peer_manager.clone()) .await .error_log_with_msg(&format!( @@ -842,8 +849,8 @@ pub async fn send_payment(ctx: MmArc, req: SendPaymentReq) -> SendPaymentResult< MmCoinEnum::LightningCoin(c) => c, _ => return MmError::err(SendPaymentError::UnsupportedCoin(coin.ticker().to_string())), }; - let nodes_addresses = ln_coin.nodes_addresses.lock().clone(); - for (node_pubkey, node_addr) in nodes_addresses { + let open_channel_nodes = ln_coin.open_channel_nodes.lock().clone(); + for (node_pubkey, node_addr) in open_channel_nodes { connect_to_node(node_pubkey, node_addr, ln_coin.peer_manager.clone()) .await .error_log_with_msg(&format!( diff --git a/mm2src/coins/lightning/ln_connections.rs b/mm2src/coins/lightning/ln_connections.rs index 1fb7d46447..da2e07f09e 100644 --- a/mm2src/coins/lightning/ln_connections.rs +++ b/mm2src/coins/lightning/ln_connections.rs @@ -83,10 +83,10 @@ pub async fn connect_to_node( )) } -pub async fn connect_to_nodes_loop(nodes_addresses: NodesAddressesMapShared, peer_manager: Arc) { +pub async fn connect_to_nodes_loop(open_channel_nodes: NodesAddressesMapShared, peer_manager: Arc) { loop { - let nodes_addresses = nodes_addresses.lock().clone(); - for (pubkey, node_addr) in nodes_addresses { + let open_channel_nodes = open_channel_nodes.lock().clone(); + for (pubkey, node_addr) in open_channel_nodes { let peer_manager = peer_manager.clone(); match connect_to_node(pubkey, node_addr, peer_manager.clone()).await { Ok(res) => { diff --git a/mm2src/coins/lightning/ln_errors.rs b/mm2src/coins/lightning/ln_errors.rs index edd45ff940..e8b36cd962 100644 --- a/mm2src/coins/lightning/ln_errors.rs +++ b/mm2src/coins/lightning/ln_errors.rs @@ -42,6 +42,8 @@ pub enum EnableLightningError { HashError(String), #[display(fmt = "RPC error {}", _0)] RpcError(String), + #[display(fmt = "SQL error {}", _0)] + SqlError(String), ConnectToNodeError(String), } @@ -56,7 +58,8 @@ impl HttpStatusCode for EnableLightningError { | EnableLightningError::IOError(_) | EnableLightningError::HashError(_) | EnableLightningError::ConnectToNodeError(_) - | EnableLightningError::InvalidConfiguration(_) => StatusCode::INTERNAL_SERVER_ERROR, + | EnableLightningError::InvalidConfiguration(_) + | EnableLightningError::SqlError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 8d30d55466..0b64f7dbe8 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -31,8 +31,8 @@ use lightning_background_processor::BackgroundProcessor; use lightning_invoice::payment; use lightning_invoice::utils::DefaultRouter; use lightning_net_tokio::SocketDescriptor; -use lightning_persister::storage::Storage; -use lightning_persister::FilesystemPersister; +use lightning_persister::storage::FileSystemStorage; +use lightning_persister::LightningPersister; use parking_lot::Mutex as PaMutex; use rand::RngCore; use rpc::v1::types::H256; @@ -57,7 +57,7 @@ pub type ChainMonitor = chainmonitor::ChainMonitor< Arc, Arc, Arc, - Arc, + Arc, >; pub type ChannelManager = SimpleArcChannelManager; @@ -171,10 +171,18 @@ pub async fn start_lightning( let ticker = conf.ticker.clone(); let ln_data_dir = ln_data_dir(ctx, &ticker); let ln_data_backup_dir = ln_data_backup_dir(ctx, params.backup_path, &ticker); - let persister = Arc::new(FilesystemPersister::new(ln_data_dir, ln_data_backup_dir)); - let is_initialized = persister.is_initialized().await?; + let persister = Arc::new(LightningPersister::new( + ln_data_dir, + ln_data_backup_dir, + ctx.sqlite_connection + .ok_or(MmError::new(EnableLightningError::SqlError( + "sqlite_connection is not initialized".into(), + )))? + .clone(), + )); + let is_initialized = persister.is_fs_initialized().await?; if !is_initialized { - persister.init().await?; + persister.init_fs().await?; } // Initialize the Filter. PlatformFields implements the Filter trait, we can use it to construct the filter. @@ -412,7 +420,7 @@ pub async fn start_lightning( ); // If node is restarting read other nodes data from disk and reconnect to channel nodes/peers if possible. - let mut nodes_addresses_map = HashMap::new(); + let mut open_channel_nodes_map = HashMap::new(); if restarting_node { let mut nodes_addresses = persister.get_nodes_addresses().await?; for (pubkey, node_addr) in nodes_addresses.drain() { @@ -422,14 +430,14 @@ pub async fn start_lightning( .map(|chan| chan.counterparty.node_id) .any(|node_id| node_id == pubkey) { - nodes_addresses_map.insert(pubkey, node_addr); + open_channel_nodes_map.insert(pubkey, node_addr); } } } - let nodes_addresses = Arc::new(PaMutex::new(nodes_addresses_map)); + let open_channel_nodes = Arc::new(PaMutex::new(open_channel_nodes_map)); if restarting_node { - spawn(connect_to_nodes_loop(nodes_addresses.clone(), peer_manager.clone())); + spawn(connect_to_nodes_loop(open_channel_nodes.clone(), peer_manager.clone())); } // Broadcast Node Announcement @@ -453,7 +461,7 @@ pub async fn start_lightning( persister, inbound_payments, outbound_payments, - nodes_addresses, + open_channel_nodes, }) } diff --git a/mm2src/coins/lightning_background_processor/src/lib.rs b/mm2src/coins/lightning_background_processor/src/lib.rs index e91956e260..65fef106e4 100644 --- a/mm2src/coins/lightning_background_processor/src/lib.rs +++ b/mm2src/coins/lightning_background_processor/src/lib.rs @@ -148,7 +148,7 @@ impl BackgroundProcessor { /// /// `persist_manager` is responsible for writing out the [`ChannelManager`] to disk, and/or /// uploading to one or more backup services. See [`ChannelManager::write`] for writing out a - /// [`ChannelManager`]. See [`FilesystemPersister::persist_manager`] for Rust-Lightning's + /// [`ChannelManager`]. See [`LightningPersister::persist_manager`] for Rust-Lightning's /// provided implementation. /// /// Typically, users should either implement [`ChannelManagerPersister`] to never return an @@ -167,7 +167,7 @@ impl BackgroundProcessor { /// [`stop`]: Self::stop /// [`ChannelManager`]: lightning::ln::channelmanager::ChannelManager /// [`ChannelManager::write`]: lightning::ln::channelmanager::ChannelManager#impl-Writeable - /// [`FilesystemPersister::persist_manager`]: lightning_persister::FilesystemPersister::persist_manager + /// [`LightningPersister::persist_manager`]: lightning_persister::LightningPersister::persist_manager /// [`NetworkGraph`]: lightning::routing::network_graph::NetworkGraph pub fn start< Signer: 'static + Sign, @@ -356,7 +356,7 @@ mod tests { use lightning::util::test_utils; use lightning_invoice::payment::{InvoicePayer, RetryAttempts}; use lightning_invoice::utils::DefaultRouter; - use lightning_persister::FilesystemPersister; + use lightning_persister::LightningPersister; use std::fs; use std::path::PathBuf; use std::sync::{Arc, Mutex}; @@ -378,7 +378,7 @@ mod tests { Arc, Arc, Arc, - Arc, + Arc, >; struct Node { @@ -403,7 +403,7 @@ mod tests { >, >, chain_monitor: Arc, - persister: Arc, + persister: Arc, tx_broadcaster: Arc, network_graph: Arc, logger: Arc, @@ -442,7 +442,7 @@ mod tests { }); let chain_source = Arc::new(test_utils::TestChainSource::new(Network::Testnet)); let logger = Arc::new(test_utils::TestLogger::with_id(format!("node {}", i))); - let persister = Arc::new(FilesystemPersister::new( + let persister = Arc::new(LightningPersister::new( PathBuf::from(format!("{}_persister_{}", persist_dir, i)), None, )); diff --git a/mm2src/coins/lightning_persister/Cargo.toml b/mm2src/coins/lightning_persister/Cargo.toml index 34bf6eac52..203ced48aa 100644 --- a/mm2src/coins/lightning_persister/Cargo.toml +++ b/mm2src/coins/lightning_persister/Cargo.toml @@ -13,6 +13,8 @@ Utilities to manage Rust-Lightning channel data persistence and retrieval. async-trait = "0.1" bitcoin = "0.27.1" common = { path = "../../common" } +db_common = { path = "../../db_common" } +hex = "0.4.2" lightning = "0.0.104" libc = "0.2" parking_lot = { version = "0.11", features = ["nightly"] } diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index e8076f0668..10e686425a 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -13,20 +13,22 @@ extern crate lightning; extern crate secp256k1; extern crate serde_json; -use crate::storage::{NodesAddressesMap, NodesAddressesMapShared, Storage}; +use crate::storage::{FileSystemStorage, NodesAddressesMap, NodesAddressesMapShared, SqlStorage}; use crate::util::DiskWriteable; use async_trait::async_trait; use bitcoin::hash_types::{BlockHash, Txid}; use bitcoin::hashes::hex::{FromHex, ToHex}; use common::async_blocking; use common::fs::check_dir_operations; +use db_common::sqlite::rusqlite::{Connection, Error as SqlError, NO_PARAMS}; +use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use lightning::chain; use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator}; use lightning::chain::chainmonitor; use lightning::chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate}; use lightning::chain::keysinterface::{KeysInterface, Sign}; use lightning::chain::transaction::OutPoint; -use lightning::ln::channelmanager::ChannelManager; +use lightning::ln::channelmanager::{ChannelDetails, ChannelManager}; use lightning::routing::network_graph::NetworkGraph; use lightning::routing::scoring::Scorer; use lightning::util::logger::Logger; @@ -41,8 +43,9 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::{Arc, Mutex}; -/// FilesystemPersister persists channel data on disk, where each channel's +/// LightningPersister persists channel data on disk, where each channel's /// data is stored in a file named after its funding outpoint. +/// It is also used to persist payments and channels history to sqlite database. /// /// Warning: this module does the best it can with calls to persist data, but it /// can only guarantee that the data is passed to the drive. It is up to the @@ -52,10 +55,12 @@ use std::sync::{Arc, Mutex}; /// persistent. /// Corollary: especially when dealing with larger amounts of money, it is best /// practice to have multiple channel data backups and not rely only on one -/// FilesystemPersister. -pub struct FilesystemPersister { +/// LightningPersister. + +pub struct LightningPersister { main_path: PathBuf, backup_path: Option, + sqlite_connection: Arc>, } impl DiskWriteable for ChannelMonitor { @@ -74,10 +79,52 @@ where fn write_to_file(&self, writer: &mut fs::File) -> Result<(), std::io::Error> { self.write(writer) } } -impl FilesystemPersister { - /// Initialize a new FilesystemPersister and set the path to the individual channels' +fn channels_history_table(ticker: &str) -> String { ticker.to_owned() + "_channels_history" } + +fn create_channels_history_table_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "CREATE TABLE IF NOT EXISTS ".to_owned() + + &table_name + + " ( + id INTEGER NOT NULL PRIMARY KEY, + channel_id VARCHAR(255) NOT NULL, + counterparty_node_id VARCHAR(255) NOT NULL, + funding_tx VARCHAR(255), + funding_tx_output_index INTEGER, + funding_tx_value_sats INTEGER NOT NULL, + closing_tx VARCHAR(255), + closing_tx_output_index INTEGER, + closing_balance_sats INTEGER, + is_outbound INTEGER NOT NULL, + is_closed INTEGER NOT NULL + );"; + + Ok(sql) +} + +fn insert_channel_in_history_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "INSERT INTO ".to_owned() + + &table_name + + " (channel_id, counterparty_node_id, funding_tx, funding_tx_output_index, funding_tx_value_sats, closing_tx, closing_tx_output_index, closing_balance_sats, is_outbound, is_closed) VALUES (?1, ?2, ?3, ?4, ?5, ?6,?7, ?8, ?9, ?10);"; + + Ok(sql) +} + +impl LightningPersister { + /// Initialize a new LightningPersister and set the path to the individual channels' /// files. - pub fn new(main_path: PathBuf, backup_path: Option) -> Self { Self { main_path, backup_path } } + pub fn new(main_path: PathBuf, backup_path: Option, sqlite_connection: Arc>) -> Self { + Self { + main_path, + backup_path, + sqlite_connection, + } + } /// Get the directory which was provided when this persister was initialized. pub fn main_path(&self) -> PathBuf { self.main_path.clone() } @@ -131,7 +178,7 @@ impl FilesystemPersister { path } - /// Writes the provided `ChannelManager` to the path provided at `FilesystemPersister` + /// Writes the provided `ChannelManager` to the path provided at `LightningPersister` /// initialization, within a file called "manager". pub fn persist_manager( &self, @@ -218,7 +265,7 @@ impl FilesystemPersister { } } -impl chainmonitor::Persist for FilesystemPersister { +impl chainmonitor::Persist for LightningPersister { // TODO: We really need a way for the persister to inform the user that its time to crash/shut // down once these start returning failure. // A PermanentFailure implies we need to shut down since we're force-closing channels without @@ -259,10 +306,10 @@ impl chainmonitor::Persist for FilesystemPer } #[async_trait] -impl Storage for FilesystemPersister { +impl FileSystemStorage for LightningPersister { type Error = std::io::Error; - async fn init(&self) -> Result<(), Self::Error> { + async fn init_fs(&self) -> Result<(), Self::Error> { let path = self.main_path(); let backup_path = self.backup_path(); async_blocking(move || { @@ -276,7 +323,7 @@ impl Storage for FilesystemPersister { .await } - async fn is_initialized(&self) -> Result { + async fn is_fs_initialized(&self) -> Result { let dir_path = self.main_path(); let backup_dir_path = self.backup_path(); async_blocking(move || { @@ -400,11 +447,85 @@ impl Storage for FilesystemPersister { } } +#[async_trait] +impl SqlStorage for LightningPersister { + type Error = SqlError; + + async fn init_sql(&self, for_coin: &str) -> Result<(), Self::Error> { + let sqlite_connection = self.sqlite_connection.clone(); + let sql_channels_history = create_channels_history_table_sql(for_coin)?; + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + conn.execute(&sql_channels_history, NO_PARAMS).map(|_| ())?; + Ok(()) + }) + .await + } + + async fn is_sql_initialized(&self, for_coin: &str) -> Result { + let channels_history_table = channels_history_table(for_coin); + validate_table_name(&channels_history_table)?; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + let channels_history_initialized = + query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [channels_history_table], string_from_row)?; + Ok(channels_history_initialized.is_some()) + }) + .await + } + + async fn add_channel_to_history(&self, for_coin: &str, channel_details: ChannelDetails) -> Result<(), Self::Error> { + let for_coin = for_coin.to_owned(); + let sqlite_connection = self.sqlite_connection.clone(); + + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + + let channel_id = hex::encode(channel_details.channel_id); + let counterparty_node_id = channel_details.counterparty.node_id.to_string(); + let funding_tx = channel_details + .funding_txo + .map(|outpoint| outpoint.txid.to_string()) + .unwrap_or_default(); + let funding_tx_output_index = channel_details + .funding_txo + .map(|outpoint| outpoint.index.to_string()) + .unwrap_or_default(); + let funding_tx_value_sats = channel_details.channel_value_satoshis.to_string(); + let closing_tx = String::default(); + let closing_tx_output_index = String::default(); + let closing_balance_sats = String::default(); + let is_outbound = (channel_details.is_outbound as i32).to_string(); + let is_closed = 0_u8.to_string(); + + let params = [ + channel_id, + counterparty_node_id, + funding_tx, + funding_tx_output_index, + funding_tx_value_sats, + closing_tx, + closing_tx_output_index, + closing_balance_sats, + is_outbound, + is_closed, + ]; + sql_transaction.execute(&insert_channel_in_history_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } +} + #[cfg(test)] mod tests { extern crate bitcoin; extern crate lightning; - use crate::FilesystemPersister; + use crate::LightningPersister; use bitcoin::blockdata::block::{Block, BlockHeader}; use bitcoin::hashes::hex::FromHex; use bitcoin::Txid; @@ -419,7 +540,7 @@ mod tests { use std::fs; use std::path::PathBuf; - impl Drop for FilesystemPersister { + impl Drop for LightningPersister { fn drop(&mut self) { // We test for invalid directory names, so it's OK if directory removal // fails. @@ -430,14 +551,14 @@ mod tests { } } - // Integration-test the FilesystemPersister. Test relaying a few payments + // Integration-test the LightningPersister. Test relaying a few payments // and check that the persisted data is updated the appropriate number of // times. #[test] fn test_filesystem_persister() { - // Create the nodes, giving them FilesystemPersisters for data persisters. - let persister_0 = FilesystemPersister::new(PathBuf::from("test_filesystem_persister_0"), None); - let persister_1 = FilesystemPersister::new(PathBuf::from("test_filesystem_persister_1"), None); + // Create the nodes, giving them LightningPersisters for data persisters. + let persister_0 = LightningPersister::new(PathBuf::from("test_filesystem_persister_0"), None); + let persister_1 = LightningPersister::new(PathBuf::from("test_filesystem_persister_1"), None); let chanmon_cfgs = create_chanmon_cfgs(2); let mut node_cfgs = create_node_cfgs(2, &chanmon_cfgs); let chain_mon_0 = test_utils::TestChainMonitor::new( @@ -533,7 +654,7 @@ mod tests { #[cfg(not(target_os = "windows"))] #[test] fn test_readonly_dir_perm_failure() { - let persister = FilesystemPersister::new(PathBuf::from("test_readonly_dir_perm_failure"), None); + let persister = LightningPersister::new(PathBuf::from("test_readonly_dir_perm_failure"), None); fs::create_dir_all(&persister.main_path).unwrap(); // Set up a dummy channel and force close. This will produce a monitor @@ -592,7 +713,7 @@ mod tests { // channel fails to open because the directories fail to be created. There // don't seem to be invalid filename characters on Unix that Rust doesn't // handle, hence why the test is Windows-only. - let persister = FilesystemPersister::new(PathBuf::from(":<>/"), None); + let persister = LightningPersister::new(PathBuf::from(":<>/"), None); let test_txo = OutPoint { txid: Txid::from_hex("8984484a580b825b9972d7adb15050b3ab624ccd731946b3eeddb92f4e7ef6be").unwrap(), diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index b0d0760857..57de01904b 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -1,4 +1,5 @@ use async_trait::async_trait; +use lightning::ln::channelmanager::ChannelDetails; use lightning::routing::network_graph::NetworkGraph; use lightning::routing::scoring::Scorer; use parking_lot::Mutex as PaMutex; @@ -11,13 +12,13 @@ pub type NodesAddressesMap = HashMap; pub type NodesAddressesMapShared = Arc>; #[async_trait] -pub trait Storage: Send + Sync + 'static { +pub trait FileSystemStorage { type Error; /// Initializes dirs/collection/tables in storage for a specified coin - async fn init(&self) -> Result<(), Self::Error>; + async fn init_fs(&self) -> Result<(), Self::Error>; - async fn is_initialized(&self) -> Result; + async fn is_fs_initialized(&self) -> Result; async fn get_nodes_addresses(&self) -> Result, Self::Error>; @@ -31,3 +32,15 @@ pub trait Storage: Send + Sync + 'static { async fn save_scorer(&self, scorer: Arc>) -> Result<(), Self::Error>; } + +#[async_trait] +pub trait SqlStorage { + type Error; + + /// Initializes dirs/collection/tables in storage for a specified coin + async fn init_sql(&self, for_coin: &str) -> Result<(), Self::Error>; + + async fn is_sql_initialized(&self, for_coin: &str) -> Result; + + async fn add_channel_to_history(&self, for_coin: &str, channel_details: ChannelDetails) -> Result<(), Self::Error>; +} diff --git a/mm2src/coins/sql_tx_history_storage.rs b/mm2src/coins/sql_tx_history_storage.rs index 71596b57a6..fe8cc7c2cd 100644 --- a/mm2src/coins/sql_tx_history_storage.rs +++ b/mm2src/coins/sql_tx_history_storage.rs @@ -5,16 +5,14 @@ use async_trait::async_trait; use common::mm_error::prelude::*; use common::{async_blocking, PagingOptionsEnum}; use db_common::sqlite::rusqlite::types::Type; -use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, ToSql, NO_PARAMS}; +use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, NO_PARAMS}; use db_common::sqlite::sql_builder::SqlBuilder; -use db_common::sqlite::{offset_by_id, validate_table_name}; +use db_common::sqlite::{offset_by_id, query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use rpc::v1::types::Bytes as BytesJson; use serde_json::{self as json}; use std::convert::TryInto; use std::sync::{Arc, Mutex}; -const CHECK_TABLE_EXISTS_SQL: &str = "SELECT name FROM sqlite_master WHERE type='table' AND name=?1;"; - fn tx_history_table(ticker: &str) -> String { ticker.to_owned() + "_tx_history" } fn tx_cache_table(ticker: &str) -> String { ticker.to_owned() + "_tx_cache" } @@ -183,28 +181,6 @@ impl SqliteTxHistoryStorage { impl TxHistoryStorageError for SqlError {} -fn query_single_row( - conn: &Connection, - query: &str, - params: P, - map_fn: F, -) -> Result, MmError> -where - P: IntoIterator, - P::Item: ToSql, - F: FnOnce(&Row<'_>) -> Result, -{ - let maybe_result = conn.query_row(query, params, map_fn); - if let Err(SqlError::QueryReturnedNoRows) = maybe_result { - return Ok(None); - } - - let result = maybe_result?; - Ok(Some(result)) -} - -fn string_from_row(row: &Row<'_>) -> Result { row.get(0) } - fn tx_details_from_row(row: &Row<'_>) -> Result { let json_string: String = row.get(0)?; json::from_str(&json_string).map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e))) @@ -325,7 +301,7 @@ impl TxHistoryStorage for SqliteTxHistoryStorage { async_blocking(move || { let conn = selfi.0.lock().unwrap(); - query_single_row(&conn, &sql, params, tx_details_from_row) + query_single_row(&conn, &sql, params, tx_details_from_row).map_err(From::from) }) .await } diff --git a/mm2src/db_common/src/sqlite.rs b/mm2src/db_common/src/sqlite.rs index e1803a841d..54957fdfdf 100644 --- a/mm2src/db_common/src/sqlite.rs +++ b/mm2src/db_common/src/sqlite.rs @@ -2,10 +2,29 @@ pub use rusqlite; pub use sql_builder; use log::debug; -use rusqlite::{Connection, Error as SqlError, Result as SqlResult, ToSql}; +use rusqlite::{Connection, Error as SqlError, Result as SqlResult, Row, ToSql}; use sql_builder::SqlBuilder; use uuid::Uuid; +pub const CHECK_TABLE_EXISTS_SQL: &str = "SELECT name FROM sqlite_master WHERE type='table' AND name=?1;"; + +pub fn string_from_row(row: &Row<'_>) -> Result { row.get(0) } + +pub fn query_single_row(conn: &Connection, query: &str, params: P, map_fn: F) -> Result, SqlError> +where + P: IntoIterator, + P::Item: ToSql, + F: FnOnce(&Row<'_>) -> Result, +{ + let maybe_result = conn.query_row(query, params, map_fn); + if let Err(SqlError::QueryReturnedNoRows) = maybe_result { + return Ok(None); + } + + let result = maybe_result?; + Ok(Some(result)) +} + pub fn validate_table_name(table_name: &str) -> SqlResult<()> { // As per https://stackoverflow.com/a/3247553, tables can't be the target of parameter substitution. // So we have to use a plain concatenation disallowing any characters in the table name that may lead to SQL injection. From 8fa6bde5ace24afd13e82dbf52f43eac0646224e Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Thu, 3 Mar 2022 10:06:41 +0200 Subject: [PATCH 02/49] review fixes: specific types, add serialization tests --- Cargo.lock | 1 + mm2src/coins/lightning.rs | 32 +++---- mm2src/coins/lightning/ln_connections.rs | 6 +- mm2src/coins/lightning/ln_serialization.rs | 90 +++++++++++++++++-- mm2src/coins/lightning/ln_utils.rs | 10 +-- .../lightning_background_processor/Cargo.toml | 1 + .../lightning_background_processor/src/lib.rs | 2 + mm2src/coins/lightning_persister/src/lib.rs | 20 ++++- mm2src/coins/lp_coins.rs | 7 +- 9 files changed, 134 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 41e4f9f749..13ff219c44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2971,6 +2971,7 @@ name = "lightning-background-processor" version = "0.0.104" dependencies = [ "bitcoin", + "db_common", "lightning", "lightning-invoice", "lightning-persister", diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index b77dad0627..f571648d11 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -147,7 +147,7 @@ pub struct LightningCoin { pub outbound_payments: PaymentsMapShared, /// The mutex storing the addresses of the nodes that the lightning node has open channels with, /// these addresses are used for reconnecting. - pub open_channel_nodes: NodesAddressesMapShared, + pub open_channels_nodes: NodesAddressesMapShared, } impl fmt::Debug for LightningCoin { @@ -538,12 +538,12 @@ pub async fn connect_to_lightning_node(ctx: MmArc, req: ConnectToNodeRequest) -> // If a node that we have an open channel with changed it's address, "connect_to_lightning_node" // can be used to reconnect to the new address while saving this new address for reconnections. if let ConnectToNodeRes::ConnectedSuccessfully(_, _) = res { - if let Entry::Occupied(mut entry) = ln_coin.open_channel_nodes.lock().entry(node_pubkey) { + if let Entry::Occupied(mut entry) = ln_coin.open_channels_nodes.lock().entry(node_pubkey) { entry.insert(node_addr); } ln_coin .persister - .save_nodes_addresses(ln_coin.open_channel_nodes) + .save_nodes_addresses(ln_coin.open_channels_nodes) .await?; } @@ -658,10 +658,10 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes } // Saving node data to reconnect to it on restart - ln_coin.open_channel_nodes.lock().insert(node_pubkey, node_addr); + ln_coin.open_channels_nodes.lock().insert(node_pubkey, node_addr); ln_coin .persister - .save_nodes_addresses(ln_coin.open_channel_nodes) + .save_nodes_addresses(ln_coin.open_channels_nodes) .await?; Ok(OpenChannelResponse { @@ -677,9 +677,9 @@ pub struct ListChannelsRequest { #[derive(Serialize)] pub struct ChannelDetailsForRPC { - pub channel_id: String, - pub counterparty_node_id: String, - pub funding_tx: Option, + pub channel_id: H256Json, + pub counterparty_node_id: PublicKeyForRPC, + pub funding_tx: Option, pub funding_tx_output_index: Option, pub funding_tx_value_sats: u64, /// True if the channel was initiated (and thus funded) by us. @@ -700,9 +700,11 @@ pub struct ChannelDetailsForRPC { impl From for ChannelDetailsForRPC { fn from(details: ChannelDetails) -> ChannelDetailsForRPC { ChannelDetailsForRPC { - channel_id: hex::encode(details.channel_id), - counterparty_node_id: details.counterparty.node_id.to_string(), - funding_tx: details.funding_txo.map(|tx| tx.txid.to_string()), + channel_id: details.channel_id.into(), + counterparty_node_id: PublicKeyForRPC(details.counterparty.node_id), + funding_tx: details + .funding_txo + .map(|tx| H256Json::from(tx.txid.as_hash().into_inner()).reversed()), funding_tx_output_index: details.funding_txo.map(|tx| tx.index), funding_tx_value_sats: details.channel_value_satoshis, is_outbound: details.is_outbound, @@ -791,8 +793,8 @@ pub async fn generate_invoice( MmCoinEnum::LightningCoin(c) => c, _ => return MmError::err(GenerateInvoiceError::UnsupportedCoin(coin.ticker().to_string())), }; - let open_channel_nodes = ln_coin.open_channel_nodes.lock().clone(); - for (node_pubkey, node_addr) in open_channel_nodes { + let open_channels_nodes = ln_coin.open_channels_nodes.lock().clone(); + for (node_pubkey, node_addr) in open_channels_nodes { connect_to_node(node_pubkey, node_addr, ln_coin.peer_manager.clone()) .await .error_log_with_msg(&format!( @@ -849,8 +851,8 @@ pub async fn send_payment(ctx: MmArc, req: SendPaymentReq) -> SendPaymentResult< MmCoinEnum::LightningCoin(c) => c, _ => return MmError::err(SendPaymentError::UnsupportedCoin(coin.ticker().to_string())), }; - let open_channel_nodes = ln_coin.open_channel_nodes.lock().clone(); - for (node_pubkey, node_addr) in open_channel_nodes { + let open_channels_nodes = ln_coin.open_channels_nodes.lock().clone(); + for (node_pubkey, node_addr) in open_channels_nodes { connect_to_node(node_pubkey, node_addr, ln_coin.peer_manager.clone()) .await .error_log_with_msg(&format!( diff --git a/mm2src/coins/lightning/ln_connections.rs b/mm2src/coins/lightning/ln_connections.rs index da2e07f09e..cc725d082b 100644 --- a/mm2src/coins/lightning/ln_connections.rs +++ b/mm2src/coins/lightning/ln_connections.rs @@ -83,10 +83,10 @@ pub async fn connect_to_node( )) } -pub async fn connect_to_nodes_loop(open_channel_nodes: NodesAddressesMapShared, peer_manager: Arc) { +pub async fn connect_to_nodes_loop(open_channels_nodes: NodesAddressesMapShared, peer_manager: Arc) { loop { - let open_channel_nodes = open_channel_nodes.lock().clone(); - for (pubkey, node_addr) in open_channel_nodes { + let open_channels_nodes = open_channels_nodes.lock().clone(); + for (pubkey, node_addr) in open_channels_nodes { let peer_manager = peer_manager.clone(); match connect_to_node(pubkey, node_addr, peer_manager.clone()).await { Ok(res) => { diff --git a/mm2src/coins/lightning/ln_serialization.rs b/mm2src/coins/lightning/ln_serialization.rs index 12d43b4299..82f0a700c7 100644 --- a/mm2src/coins/lightning/ln_serialization.rs +++ b/mm2src/coins/lightning/ln_serialization.rs @@ -47,6 +47,7 @@ impl<'de> de::Deserialize<'de> for InvoiceForRPC { } // TODO: support connection to onion addresses +#[derive(Debug, PartialEq)] pub struct NodeAddress { pub pubkey: PublicKey, pub addr: SocketAddr, @@ -101,7 +102,7 @@ impl<'de> de::Deserialize<'de> for NodeAddress { } #[derive(Clone, Debug, PartialEq)] -pub struct PublicKeyForRPC(PublicKey); +pub struct PublicKeyForRPC(pub PublicKey); impl From for PublicKey { fn from(p: PublicKeyForRPC) -> Self { p.0 } @@ -109,16 +110,93 @@ impl From for PublicKey { impl Serialize for PublicKeyForRPC { fn serialize(&self, serializer: S) -> Result { - serializer.serialize_bytes(&self.0.serialize()) + serializer.serialize_str(&self.0.to_string()) } } impl<'de> de::Deserialize<'de> for PublicKeyForRPC { fn deserialize>(deserializer: D) -> Result { - let slice: &[u8] = de::Deserialize::deserialize(deserializer)?; - let pubkey = - PublicKey::from_slice(slice).map_err(|e| de::Error::custom(format!("Error {} parsing pubkey", e)))?; + struct PublicKeyForRPCVisitor; + + impl<'de> de::Visitor<'de> for PublicKeyForRPCVisitor { + type Value = PublicKeyForRPC; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { write!(formatter, "a public key") } + + fn visit_str(self, v: &str) -> Result { + let pubkey = PublicKey::from_str(v).map_err(|e| { + let err = format!("Could not parse public key from str {}, err {}", v, e); + de::Error::custom(err) + })?; + Ok(PublicKeyForRPC(pubkey)) + } + } + + deserializer.deserialize_str(PublicKeyForRPCVisitor) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json as json; + + #[test] + fn test_invoice_for_rpc_serialize() { + let invoice_for_rpc = InvoiceForRPC(str::parse::("lntb20u1p3zqmvrpp52hej7trefx6y633aujj6nltjs8cf7lzyp78tfn5y5wpa3udk5tvqdp8xys9xcmpd3sjqsmgd9czq3njv9c8qatrvd5kumcxqrrsscqp79qy9qsqsp5ccy2qgmptg8dthxsjvw2c43uyvqkg6cqey3jpks4xf0tv7xfrqrq3xfnuffau2h2k8defphv2xsktzn2qj5n2l8d9l9zx64fg6jcmdg9kmpevneyyhfnzrpspqdrky8u7l4c6qdnquh8lnevswwrtcd9ypcq89ga09").unwrap()); + let expected = r#""lntb20u1p3zqmvrpp52hej7trefx6y633aujj6nltjs8cf7lzyp78tfn5y5wpa3udk5tvqdp8xys9xcmpd3sjqsmgd9czq3njv9c8qatrvd5kumcxqrrsscqp79qy9qsqsp5ccy2qgmptg8dthxsjvw2c43uyvqkg6cqey3jpks4xf0tv7xfrqrq3xfnuffau2h2k8defphv2xsktzn2qj5n2l8d9l9zx64fg6jcmdg9kmpevneyyhfnzrpspqdrky8u7l4c6qdnquh8lnevswwrtcd9ypcq89ga09""#; + let actual = json::to_string(&invoice_for_rpc).unwrap(); + assert_eq!(expected, actual); + } + + #[test] + fn test_invoice_for_rpc_deserialize() { + let invoice_for_rpc = r#""lntb20u1p3zqmvrpp52hej7trefx6y633aujj6nltjs8cf7lzyp78tfn5y5wpa3udk5tvqdp8xys9xcmpd3sjqsmgd9czq3njv9c8qatrvd5kumcxqrrsscqp79qy9qsqsp5ccy2qgmptg8dthxsjvw2c43uyvqkg6cqey3jpks4xf0tv7xfrqrq3xfnuffau2h2k8defphv2xsktzn2qj5n2l8d9l9zx64fg6jcmdg9kmpevneyyhfnzrpspqdrky8u7l4c6qdnquh8lnevswwrtcd9ypcq89ga09""#; + let expected = InvoiceForRPC(str::parse::("lntb20u1p3zqmvrpp52hej7trefx6y633aujj6nltjs8cf7lzyp78tfn5y5wpa3udk5tvqdp8xys9xcmpd3sjqsmgd9czq3njv9c8qatrvd5kumcxqrrsscqp79qy9qsqsp5ccy2qgmptg8dthxsjvw2c43uyvqkg6cqey3jpks4xf0tv7xfrqrq3xfnuffau2h2k8defphv2xsktzn2qj5n2l8d9l9zx64fg6jcmdg9kmpevneyyhfnzrpspqdrky8u7l4c6qdnquh8lnevswwrtcd9ypcq89ga09").unwrap()); + let actual = json::from_str(invoice_for_rpc).unwrap(); + assert_eq!(expected, actual); + } + + #[test] + fn test_node_address_serialize() { + let node_address = NodeAddress { + pubkey: PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), + addr: SocketAddr::new("203.132.94.196".parse().unwrap(), 9735), + }; + let expected = r#""038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9@203.132.94.196:9735""#; + let actual = json::to_string(&node_address).unwrap(); + assert_eq!(expected, actual); + } + + #[test] + fn test_node_address_deserialize() { + let node_address = + r#""038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9@203.132.94.196:9735""#; + let expected = NodeAddress { + pubkey: PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), + addr: SocketAddr::new("203.132.94.196".parse().unwrap(), 9735), + }; + let actual: NodeAddress = json::from_str(node_address).unwrap(); + assert_eq!(expected, actual); + } + + #[test] + fn test_public_key_for_rpc_serialize() { + let public_key_for_rpc = PublicKeyForRPC( + PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), + ); + let expected = r#""038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9""#; + let actual = json::to_string(&public_key_for_rpc).unwrap(); + assert_eq!(expected, actual); + } - Ok(PublicKeyForRPC(pubkey)) + #[test] + fn test_public_key_for_rpc_deserialize() { + let public_key_for_rpc = r#""038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9""#; + let expected = PublicKeyForRPC( + PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), + ); + let actual = json::from_str(public_key_for_rpc).unwrap(); + assert_eq!(expected, actual); } } diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 0b64f7dbe8..ca6610c023 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -420,7 +420,7 @@ pub async fn start_lightning( ); // If node is restarting read other nodes data from disk and reconnect to channel nodes/peers if possible. - let mut open_channel_nodes_map = HashMap::new(); + let mut open_channels_nodes_map = HashMap::new(); if restarting_node { let mut nodes_addresses = persister.get_nodes_addresses().await?; for (pubkey, node_addr) in nodes_addresses.drain() { @@ -430,14 +430,14 @@ pub async fn start_lightning( .map(|chan| chan.counterparty.node_id) .any(|node_id| node_id == pubkey) { - open_channel_nodes_map.insert(pubkey, node_addr); + open_channels_nodes_map.insert(pubkey, node_addr); } } } - let open_channel_nodes = Arc::new(PaMutex::new(open_channel_nodes_map)); + let open_channels_nodes = Arc::new(PaMutex::new(open_channels_nodes_map)); if restarting_node { - spawn(connect_to_nodes_loop(open_channel_nodes.clone(), peer_manager.clone())); + spawn(connect_to_nodes_loop(open_channels_nodes.clone(), peer_manager.clone())); } // Broadcast Node Announcement @@ -461,7 +461,7 @@ pub async fn start_lightning( persister, inbound_payments, outbound_payments, - open_channel_nodes, + open_channels_nodes, }) } diff --git a/mm2src/coins/lightning_background_processor/Cargo.toml b/mm2src/coins/lightning_background_processor/Cargo.toml index e9e85b5f77..3a535e56ec 100644 --- a/mm2src/coins/lightning_background_processor/Cargo.toml +++ b/mm2src/coins/lightning_background_processor/Cargo.toml @@ -14,6 +14,7 @@ bitcoin = "0.27.1" lightning = { version = "0.0.104", features = ["std"] } [dev-dependencies] +db_common = { path = "../../db_common" } lightning = { version = "0.0.104", features = ["_test_utils"] } lightning-invoice = "0.12.0" lightning-persister = { version = "0.0.104", path = "../lightning_persister" } diff --git a/mm2src/coins/lightning_background_processor/src/lib.rs b/mm2src/coins/lightning_background_processor/src/lib.rs index 65fef106e4..0784dd16b0 100644 --- a/mm2src/coins/lightning_background_processor/src/lib.rs +++ b/mm2src/coins/lightning_background_processor/src/lib.rs @@ -340,6 +340,7 @@ mod tests { use bitcoin::blockdata::constants::genesis_block; use bitcoin::blockdata::transaction::{Transaction, TxOut}; use bitcoin::network::constants::Network; + use db_common::sqlite::rusqlite::Connection; use lightning::chain::channelmonitor::ANTI_REORG_DELAY; use lightning::chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager}; use lightning::chain::transaction::OutPoint; @@ -445,6 +446,7 @@ mod tests { let persister = Arc::new(LightningPersister::new( PathBuf::from(format!("{}_persister_{}", persist_dir, i)), None, + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), )); let seed = [i as u8; 32]; let network = Network::Testnet; diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 10e686425a..0ce666b5d1 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -529,6 +529,7 @@ mod tests { use bitcoin::blockdata::block::{Block, BlockHeader}; use bitcoin::hashes::hex::FromHex; use bitcoin::Txid; + use db_common::sqlite::rusqlite::Connection; use lightning::chain::chainmonitor::Persist; use lightning::chain::transaction::OutPoint; use lightning::chain::ChannelMonitorUpdateErr; @@ -539,6 +540,7 @@ mod tests { use lightning::{check_added_monitors, check_closed_broadcast, check_closed_event}; use std::fs; use std::path::PathBuf; + use std::sync::{Arc, Mutex}; impl Drop for LightningPersister { fn drop(&mut self) { @@ -557,8 +559,16 @@ mod tests { #[test] fn test_filesystem_persister() { // Create the nodes, giving them LightningPersisters for data persisters. - let persister_0 = LightningPersister::new(PathBuf::from("test_filesystem_persister_0"), None); - let persister_1 = LightningPersister::new(PathBuf::from("test_filesystem_persister_1"), None); + let persister_0 = LightningPersister::new( + PathBuf::from("test_filesystem_persister_0"), + None, + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + let persister_1 = LightningPersister::new( + PathBuf::from("test_filesystem_persister_1"), + None, + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); let chanmon_cfgs = create_chanmon_cfgs(2); let mut node_cfgs = create_node_cfgs(2, &chanmon_cfgs); let chain_mon_0 = test_utils::TestChainMonitor::new( @@ -713,7 +723,11 @@ mod tests { // channel fails to open because the directories fail to be created. There // don't seem to be invalid filename characters on Unix that Rust doesn't // handle, hence why the test is Windows-only. - let persister = LightningPersister::new(PathBuf::from(":<>/"), None); + let persister = LightningPersister::new( + PathBuf::from(":<>/"), + None, + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); let test_txo = OutPoint { txid: Txid::from_hex("8984484a580b825b9972d7adb15050b3ab624ccd731946b3eeddb92f4e7ef6be").unwrap(), diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index d124278286..38071970b2 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -1415,6 +1415,7 @@ pub trait MmCoin: SwapOps + MarketCoinOps + fmt::Debug + Send + Sync + 'static { } #[derive(Clone, Debug)] +#[allow(clippy::large_enum_variant)] pub enum MmCoinEnum { UtxoCoin(UtxoStandardCoin), QtumCoin(QtumCoin), @@ -1425,7 +1426,7 @@ pub enum MmCoinEnum { Bch(BchCoin), SlpToken(SlpToken), #[cfg(not(target_arch = "wasm32"))] - LightningCoin(Box), + LightningCoin(LightningCoin), Test(TestCoin), } @@ -1459,7 +1460,7 @@ impl From for MmCoinEnum { #[cfg(not(target_arch = "wasm32"))] impl From for MmCoinEnum { - fn from(c: LightningCoin) -> MmCoinEnum { MmCoinEnum::LightningCoin(Box::new(c)) } + fn from(c: LightningCoin) -> MmCoinEnum { MmCoinEnum::LightningCoin(c) } } #[cfg(all(not(target_arch = "wasm32"), feature = "zhtlc"))] @@ -1479,7 +1480,7 @@ impl Deref for MmCoinEnum { MmCoinEnum::Bch(ref c) => c, MmCoinEnum::SlpToken(ref c) => c, #[cfg(not(target_arch = "wasm32"))] - MmCoinEnum::LightningCoin(ref c) => &**c, + MmCoinEnum::LightningCoin(ref c) => c, #[cfg(all(not(target_arch = "wasm32"), feature = "zhtlc"))] MmCoinEnum::ZCoin(ref c) => c, MmCoinEnum::Test(ref c) => c, From 16b80bc7123f4250986b45d9778388754c7a577c Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Fri, 4 Mar 2022 10:13:16 +0200 Subject: [PATCH 03/49] WIP: refactor start_lightning fn --- mm2src/coins/lightning.rs | 15 +++ mm2src/coins/lightning/ln_utils.rs | 162 ++++++++++++++++------------- 2 files changed, 103 insertions(+), 74 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index f571648d11..9fb0b1ca28 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -85,6 +85,21 @@ pub struct PlatformFields { } impl PlatformFields { + pub fn new( + platform_coin: UtxoStandardCoin, + network: BlockchainNetwork, + default_fees_and_confirmations: PlatformCoinConfirmations, + ) -> Self { + PlatformFields { + platform_coin, + network, + default_fees_and_confirmations, + registered_txs: PaMutex::new(HashMap::new()), + registered_outputs: PaMutex::new(Vec::new()), + unsigned_funding_txs: PaMutex::new(HashMap::new()), + } + } + pub fn add_tx(&self, txid: &Txid, script_pubkey: &Script) { let mut registered_txs = self.registered_txs.lock(); match registered_txs.get_mut(txid) { diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index ca6610c023..79d346be62 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -31,11 +31,12 @@ use lightning_background_processor::BackgroundProcessor; use lightning_invoice::payment; use lightning_invoice::utils::DefaultRouter; use lightning_net_tokio::SocketDescriptor; -use lightning_persister::storage::FileSystemStorage; +use lightning_persister::storage::{FileSystemStorage, NodesAddressesMap}; use lightning_persister::LightningPersister; use parking_lot::Mutex as PaMutex; use rand::RngCore; use rpc::v1::types::H256; +use secp256k1::SecretKey; use std::cmp::Ordering; use std::collections::HashMap; use std::convert::TryInto; @@ -75,6 +76,8 @@ pub type InvoicePayer = payment::InvoicePayer, Router, Ar type Router = DefaultRouter, Arc>; +type NetworkGossip = NetGraphMsgHandler, Arc, Arc>; + // TODO: add TOR address option fn netaddress_from_ipaddr(addr: IpAddr, port: u16) -> Vec { if addr == Ipv4Addr::new(0, 0, 0, 0) || addr == Ipv4Addr::new(127, 0, 0, 1) { @@ -122,6 +125,60 @@ pub fn ln_data_backup_dir(ctx: &MmArc, path: Option, ticker: &str) -> Op }) } +async fn init_peer_manager( + ctx: MmArc, + listening_port: u16, + channel_manager: Arc, + network_gossip: Arc, + node_secret: SecretKey, + logger: Arc, +) -> EnableLightningResult> { + // The set (possibly empty) of socket addresses on which this node accepts incoming connections. + // If the user wishes to preserve privacy, addresses should likely contain only Tor Onion addresses. + let listening_addr = myipaddr(ctx).await.map_to_mm(EnableLightningError::InvalidAddress)?; + // If the listening port is used start_lightning should return an error early + let listener = TcpListener::bind(format!("{}:{}", listening_addr, listening_port)) + .await + .map_to_mm(|e| EnableLightningError::IOError(e.to_string()))?; + + // ephemeral_random_data is used to derive per-connection ephemeral keys + let mut ephemeral_bytes = [0; 32]; + rand::thread_rng().fill_bytes(&mut ephemeral_bytes); + let lightning_msg_handler = MessageHandler { + chan_handler: channel_manager, + route_handler: network_gossip, + }; + + // IgnoringMessageHandler is used as custom message types (experimental and application-specific messages) is not needed + let peer_manager: Arc = Arc::new(PeerManager::new( + lightning_msg_handler, + node_secret, + &ephemeral_bytes, + logger, + Arc::new(IgnoringMessageHandler {}), + )); + + // Initialize p2p networking + spawn(ln_p2p_loop(peer_manager.clone(), listener)); + + Ok(peer_manager) +} + +async fn get_open_channels_nodes_addresses( + persister: Arc, + channel_manager: Arc, +) -> EnableLightningResult { + let channels = channel_manager.list_channels(); + let mut nodes_addresses = persister.get_nodes_addresses().await?; + nodes_addresses.retain(|pubkey, _node_addr| { + channels + .iter() + .map(|chan| chan.counterparty.node_id) + .any(|node_id| node_id == *pubkey) + }); + Ok(nodes_addresses) +} + pub async fn start_lightning( ctx: &MmArc, platform_coin: UtxoStandardCoin, @@ -137,25 +194,12 @@ pub async fn start_lightning( )); } - // The set (possibly empty) of socket addresses on which this node accepts incoming connections. - // If the user wishes to preserve privacy, addresses should likely contain only Tor Onion addresses. - let listening_addr = myipaddr(ctx.clone()) - .await - .map_to_mm(EnableLightningError::InvalidAddress)?; - // If the listening port is used start_lightning should return an error early - let listener = TcpListener::bind(format!("{}:{}", listening_addr, params.listening_port)) - .await - .map_to_mm(|e| EnableLightningError::IOError(e.to_string()))?; - let network = protocol_conf.network.clone().into(); - let platform_fields = Arc::new(PlatformFields { - platform_coin: platform_coin.clone(), - network: protocol_conf.network, - default_fees_and_confirmations: protocol_conf.confirmations, - registered_txs: PaMutex::new(HashMap::new()), - registered_outputs: PaMutex::new(Vec::new()), - unsigned_funding_txs: PaMutex::new(HashMap::new()), - }); + let platform_fields = Arc::new(PlatformFields::new( + platform_coin.clone(), + protocol_conf.network, + protocol_conf.confirmations, + )); // Initialize the FeeEstimator. UtxoStandardCoin implements the FeeEstimator trait, so it'll act as our fee estimator. let fee_estimator = platform_fields.clone(); @@ -331,24 +375,15 @@ pub async fn start_lightning( }); // Initialize the PeerManager - // ephemeral_random_data is used to derive per-connection ephemeral keys - let mut ephemeral_bytes = [0; 32]; - rand::thread_rng().fill_bytes(&mut ephemeral_bytes); - let lightning_msg_handler = MessageHandler { - chan_handler: channel_manager.clone(), - route_handler: network_gossip.clone(), - }; - // IgnoringMessageHandler is used as custom message types (experimental and application-specific messages) is not needed - let peer_manager: Arc = Arc::new(PeerManager::new( - lightning_msg_handler, + let peer_manager = init_peer_manager( + ctx.clone(), + params.listening_port, + channel_manager.clone(), + network_gossip.clone(), keys_manager.get_node_secret(), - &ephemeral_bytes, logger.clone(), - Arc::new(IgnoringMessageHandler {}), - )); - - // Initialize p2p networking - spawn(ln_p2p_loop(peer_manager.clone(), listener)); + ) + .await?; // Update best block whenever there's a new chain tip or a block has been newly disconnected spawn(ln_best_block_update_loop( @@ -419,23 +454,10 @@ pub async fn start_lightning( logger, ); - // If node is restarting read other nodes data from disk and reconnect to channel nodes/peers if possible. - let mut open_channels_nodes_map = HashMap::new(); - if restarting_node { - let mut nodes_addresses = persister.get_nodes_addresses().await?; - for (pubkey, node_addr) in nodes_addresses.drain() { - if channel_manager - .list_channels() - .iter() - .map(|chan| chan.counterparty.node_id) - .any(|node_id| node_id == pubkey) - { - open_channels_nodes_map.insert(pubkey, node_addr); - } - } - } - let open_channels_nodes = Arc::new(PaMutex::new(open_channels_nodes_map)); - + // If channel_nodes_data file exists, read channels nodes data from disk and reconnect to channel nodes/peers if possible. + let open_channels_nodes = Arc::new(PaMutex::new( + get_open_channels_nodes_addresses(persister.clone(), channel_manager.clone()).await?, + )); if restarting_node { spawn(connect_to_nodes_loop(open_channels_nodes.clone(), peer_manager.clone())); } @@ -445,7 +467,6 @@ pub async fn start_lightning( channel_manager.clone(), params.node_name, params.node_color, - listening_addr, params.listening_port, )); @@ -816,31 +837,24 @@ async fn ln_node_announcement_loop( channel_manager: Arc, node_name: [u8; 32], node_color: [u8; 3], - addr: IpAddr, port: u16, ) { - let addresses = netaddress_from_ipaddr(addr, port); loop { - let addresses_to_announce = if addresses.is_empty() { - // Right now if the node is behind NAT the external ip is fetched on every loop - // If the node does not announce a public IP, it will not be displayed on the network graph, - // and other nodes will not be able to open a channel with it. But it can open channels with other nodes. - match fetch_external_ip().await { - Ok(ip) => { - log::debug!("Fetch real IP successfully: {}:{}", ip, port); - netaddress_from_ipaddr(ip, port) - }, - Err(e) => { - log::error!("Error while fetching external ip for node announcement: {}", e); - Timer::sleep(BROADCAST_NODE_ANNOUNCEMENT_INTERVAL as f64).await; - continue; - }, - } - } else { - addresses.clone() + // Right now if the node is behind NAT the external ip is fetched on every loop + // If the node does not announce a public IP, it will not be displayed on the network graph, + // and other nodes will not be able to open a channel with it. But it can open channels with other nodes. + let addresses = match fetch_external_ip().await { + Ok(ip) => { + log::debug!("Fetch real IP successfully: {}:{}", ip, port); + netaddress_from_ipaddr(ip, port) + }, + Err(e) => { + log::error!("Error while fetching external ip for node announcement: {}", e); + Timer::sleep(BROADCAST_NODE_ANNOUNCEMENT_INTERVAL as f64).await; + continue; + }, }; - - channel_manager.broadcast_node_announcement(node_color, node_name, addresses_to_announce); + channel_manager.broadcast_node_announcement(node_color, node_name, addresses); Timer::sleep(BROADCAST_NODE_ANNOUNCEMENT_INTERVAL as f64).await; } From 4f432a31f6fe38a4dd3b44462e8f4b6b6bc6abba Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Mon, 7 Mar 2022 23:20:56 +0200 Subject: [PATCH 04/49] WIP: continue refactoring start_lightning fn --- mm2src/coins/lightning/ln_utils.rs | 130 +++++++++--------- mm2src/coins/lightning_persister/src/lib.rs | 10 +- .../coins/lightning_persister/src/storage.rs | 3 +- 3 files changed, 77 insertions(+), 66 deletions(-) diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 79d346be62..8a284660e0 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -6,7 +6,6 @@ use crate::utxo::rpc_clients::{electrum_script_hash, BestBlock as RpcBestBlock, use crate::utxo::utxo_standard::UtxoStandardCoin; use crate::DerivationMethod; use bitcoin::blockdata::block::BlockHeader; -use bitcoin::blockdata::constants::genesis_block; use bitcoin::blockdata::transaction::Transaction; use bitcoin::consensus::encode::deserialize; use bitcoin::hash_types::{BlockHash, TxMerkleNode, Txid}; @@ -125,6 +124,63 @@ pub fn ln_data_backup_dir(ctx: &MmArc, path: Option, ticker: &str) -> Op }) } +async fn init_persister( + ctx: &MmArc, + ticker: String, + backup_path: Option, +) -> EnableLightningResult> { + let ln_data_dir = ln_data_dir(ctx, &ticker); + let ln_data_backup_dir = ln_data_backup_dir(ctx, backup_path, &ticker); + let persister = Arc::new(LightningPersister::new( + ln_data_dir, + ln_data_backup_dir, + ctx.sqlite_connection + .ok_or(MmError::new(EnableLightningError::SqlError( + "sqlite_connection is not initialized".into(), + )))? + .clone(), + )); + let is_initialized = persister.is_fs_initialized().await?; + if !is_initialized { + persister.init_fs().await?; + } + Ok(persister) +} + +fn init_keys_manager(ctx: &MmArc) -> EnableLightningResult> { + // The current time is used to derive random numbers from the seed where required, to ensure all random generation is unique across restarts. + let seed: [u8; 32] = ctx.secp256k1_key_pair().private().secret.into(); + let cur = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_to_mm(|e| EnableLightningError::SystemTimeError(e.to_string()))?; + + Ok(Arc::new(KeysManager::new(&seed, cur.as_secs(), cur.subsec_nanos()))) +} + +async fn persist_network_graph_loop(persister: Arc, network_graph: Arc) { + loop { + if let Err(e) = persister.save_network_graph(network_graph.clone()).await { + log::warn!( + "Failed to persist network graph error: {}, please check disk space and permissions", + e + ); + } + Timer::sleep(NETWORK_GRAPH_PERSIST_INTERVAL as f64).await; + } +} + +async fn persist_scorer_loop(persister: Arc, scorer: Arc>) { + loop { + if let Err(e) = persister.save_scorer(scorer.clone()).await { + log::warn!( + "Failed to persist scorer error: {}, please check disk space and permissions", + e + ); + } + Timer::sleep(SCORER_PERSIST_INTERVAL as f64).await; + } +} + async fn init_peer_manager( ctx: MmArc, listening_port: u16, @@ -211,23 +267,8 @@ pub async fn start_lightning( // broadcaster. let broadcaster = Arc::new(platform_coin); - // Initialize Persist - let ticker = conf.ticker.clone(); - let ln_data_dir = ln_data_dir(ctx, &ticker); - let ln_data_backup_dir = ln_data_backup_dir(ctx, params.backup_path, &ticker); - let persister = Arc::new(LightningPersister::new( - ln_data_dir, - ln_data_backup_dir, - ctx.sqlite_connection - .ok_or(MmError::new(EnableLightningError::SqlError( - "sqlite_connection is not initialized".into(), - )))? - .clone(), - )); - let is_initialized = persister.is_fs_initialized().await?; - if !is_initialized { - persister.init_fs().await?; - } + // Initialize Persister + let persister = init_persister(ctx, conf.ticker.clone(), params.backup_path).await?; // Initialize the Filter. PlatformFields implements the Filter trait, we can use it to construct the filter. let filter = Some(platform_fields.clone()); @@ -241,15 +282,8 @@ pub async fn start_lightning( persister.clone(), )); - let seed: [u8; 32] = ctx.secp256k1_key_pair().private().secret.into(); - - // The current time is used to derive random numbers from the seed where required, to ensure all random generation is unique across restarts. - let cur = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .map_to_mm(|e| EnableLightningError::SystemTimeError(e.to_string()))?; - // Initialize the KeysManager - let keys_manager = Arc::new(KeysManager::new(&seed, cur.as_secs(), cur.subsec_nanos())); + let keys_manager = init_keys_manager(ctx)?; // Read ChannelMonitor state from disk, important for lightning node is restarting and has at least 1 channel let mut channelmonitors = persister @@ -263,7 +297,6 @@ pub async fn start_lightning( } } - let mut restarting_node = true; // TODO: Right now it's safe to unwrap here, when implementing Native client for lightning whenever filter is used // the code it's used in will be a part of the electrum client implementation only let rpc_client = match &filter.clone().unwrap().platform_coin.as_ref().rpc_client { @@ -301,7 +334,6 @@ pub async fn start_lightning( .map_to_mm(|e| EnableLightningError::IOError(e.to_string()))? } else { // Initialize the ChannelManager to starting a new node without history - restarting_node = false; let chain_params = ChainParameters { network, best_block: BestBlock::new(best_block_hash, best_block.height as u32), @@ -322,7 +354,7 @@ pub async fn start_lightning( let channel_manager: Arc = Arc::new(channel_manager); // Sync ChannelMonitors and ChannelManager to chain tip if the node is restarting and has open channels - if restarting_node && channel_manager_blockhash != best_block_hash { + if channel_manager_blockhash != best_block_hash { process_txs_unconfirmations( filter.clone().unwrap().clone(), chain_monitor.clone(), @@ -350,29 +382,13 @@ pub async fn start_lightning( } // Initialize the NetGraphMsgHandler. This is used for providing routes to send payments over - let default_network_graph = NetworkGraph::new(genesis_block(network).header.block_hash()); - let network_graph = Arc::new(persister.get_network_graph().await.unwrap_or(default_network_graph)); + let network_graph = Arc::new(persister.get_network_graph(network).await?); + spawn(persist_network_graph_loop(persister.clone(), network_graph.clone())); let network_gossip = Arc::new(NetGraphMsgHandler::new( network_graph.clone(), None::>, logger.clone(), )); - let network_graph_persister = persister.clone(); - let network_graph_persist = network_graph.clone(); - spawn(async move { - loop { - if let Err(e) = network_graph_persister - .save_network_graph(network_graph_persist.clone()) - .await - { - log::warn!( - "Failed to persist network graph error: {}, please check disk space and permissions", - e - ); - } - Timer::sleep(NETWORK_GRAPH_PERSIST_INTERVAL as f64).await; - } - }); // Initialize the PeerManager let peer_manager = init_peer_manager( @@ -409,20 +425,8 @@ pub async fn start_lightning( )); // Initialize routing Scorer - let scorer = Arc::new(Mutex::new(persister.get_scorer().await.unwrap_or_default())); - let scorer_persister = persister.clone(); - let scorer_persist = scorer.clone(); - spawn(async move { - loop { - if let Err(e) = scorer_persister.save_scorer(scorer_persist.clone()).await { - log::warn!( - "Failed to persist scorer error: {}, please check disk space and permissions", - e - ); - } - Timer::sleep(SCORER_PERSIST_INTERVAL as f64).await; - } - }); + let scorer = Arc::new(Mutex::new(persister.get_scorer().await?)); + spawn(persist_scorer_loop(persister.clone(), scorer.clone())); // Create InvoicePayer let router = DefaultRouter::new(network_graph, logger.clone()); @@ -458,9 +462,7 @@ pub async fn start_lightning( let open_channels_nodes = Arc::new(PaMutex::new( get_open_channels_nodes_addresses(persister.clone(), channel_manager.clone()).await?, )); - if restarting_node { - spawn(connect_to_nodes_loop(open_channels_nodes.clone(), peer_manager.clone())); - } + spawn(connect_to_nodes_loop(open_channels_nodes.clone(), peer_manager.clone())); // Broadcast Node Announcement spawn(ln_node_announcement_loop( diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 0ce666b5d1..03e7eeeaba 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -16,8 +16,10 @@ extern crate serde_json; use crate::storage::{FileSystemStorage, NodesAddressesMap, NodesAddressesMapShared, SqlStorage}; use crate::util::DiskWriteable; use async_trait::async_trait; +use bitcoin::blockdata::constants::genesis_block; use bitcoin::hash_types::{BlockHash, Txid}; use bitcoin::hashes::hex::{FromHex, ToHex}; +use bitcoin::Network; use common::async_blocking; use common::fs::check_dir_operations; use db_common::sqlite::rusqlite::{Connection, Error as SqlError, NO_PARAMS}; @@ -406,8 +408,11 @@ impl FileSystemStorage for LightningPersister { .await } - async fn get_network_graph(&self) -> Result { + async fn get_network_graph(&self, network: Network) -> Result { let path = self.network_graph_path(); + if !path.exists() { + return Ok(NetworkGraph::new(genesis_block(network).header.block_hash())); + } async_blocking(move || { let file = fs::File::open(path)?; common::log::info!("Reading the saved lightning network graph from file, this can take some time!"); @@ -428,6 +433,9 @@ impl FileSystemStorage for LightningPersister { async fn get_scorer(&self) -> Result { let path = self.scorer_path(); + if !path.exists() { + return Ok(Scorer::default()); + } async_blocking(move || { let file = fs::File::open(path)?; Scorer::read(&mut BufReader::new(file)) diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 57de01904b..8ac89295cb 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -1,4 +1,5 @@ use async_trait::async_trait; +use bitcoin::Network; use lightning::ln::channelmanager::ChannelDetails; use lightning::routing::network_graph::NetworkGraph; use lightning::routing::scoring::Scorer; @@ -24,7 +25,7 @@ pub trait FileSystemStorage { async fn save_nodes_addresses(&self, nodes_addresses: NodesAddressesMapShared) -> Result<(), Self::Error>; - async fn get_network_graph(&self) -> Result; + async fn get_network_graph(&self, network: Network) -> Result; async fn save_network_graph(&self, network_graph: Arc) -> Result<(), Self::Error>; From eb62785a576ed7058ca523ab6b51891fb24ac58a Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Tue, 8 Mar 2022 01:47:26 +0200 Subject: [PATCH 05/49] WIP: refactoring, add init_channel_manager fn --- mm2src/coins/lightning/ln_utils.rs | 260 +++++++++++++++-------------- 1 file changed, 134 insertions(+), 126 deletions(-) diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 8a284660e0..1a58cc5992 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -157,6 +157,129 @@ fn init_keys_manager(ctx: &MmArc) -> EnableLightningResult> { Ok(Arc::new(KeysManager::new(&seed, cur.as_secs(), cur.subsec_nanos()))) } +async fn init_channel_manager( + platform_fields: Arc, + logger: Arc, + persister: Arc, + keys_manager: Arc, + conf: LightningCoinConf, +) -> EnableLightningResult<(Arc, Arc)> { + // Initialize the FeeEstimator. UtxoStandardCoin implements the FeeEstimator trait, so it'll act as our fee estimator. + let fee_estimator = platform_fields.clone(); + + // Initialize the BroadcasterInterface. UtxoStandardCoin implements the BroadcasterInterface trait, so it'll act as our transaction + // broadcaster. + let broadcaster = Arc::new(platform_fields.platform_coin.clone()); + + // Initialize the ChainMonitor + let chain_monitor: Arc = Arc::new(chainmonitor::ChainMonitor::new( + Some(platform_fields.clone()), + broadcaster.clone(), + logger.clone(), + fee_estimator.clone(), + persister.clone(), + )); + + // Read ChannelMonitor state from disk, important for lightning node is restarting and has at least 1 channel + let mut channelmonitors = persister + .read_channelmonitors(keys_manager.clone()) + .map_to_mm(|e| EnableLightningError::IOError(e.to_string()))?; + + // This is used for Electrum only to prepare for chain synchronization + for (_, chan_mon) in channelmonitors.iter() { + chan_mon.load_outputs_to_watch(&platform_fields); + } + + let rpc_client = match &platform_fields.platform_coin.as_ref().rpc_client { + UtxoRpcClientEnum::Electrum(c) => c.clone(), + UtxoRpcClientEnum::Native(_) => { + return MmError::err(EnableLightningError::UnsupportedMode( + "Lightning network".into(), + "electrum".into(), + )) + }, + }; + let best_header = get_best_header(&rpc_client).await?; + let best_block = RpcBestBlock::from(best_header.clone()); + let best_block_hash = BlockHash::from_hash( + sha256d::Hash::from_slice(&best_block.hash.0).map_to_mm(|e| EnableLightningError::HashError(e.to_string()))?, + ); + let (channel_manager_blockhash, channel_manager) = { + let user_config = conf.into(); + if let Ok(mut f) = File::open(persister.manager_path()) { + let mut channel_monitor_mut_references = Vec::new(); + for (_, channel_monitor) in channelmonitors.iter_mut() { + channel_monitor_mut_references.push(channel_monitor); + } + // Read ChannelManager data from the file + let read_args = ChannelManagerReadArgs::new( + keys_manager.clone(), + fee_estimator.clone(), + chain_monitor.clone(), + broadcaster.clone(), + logger.clone(), + user_config, + channel_monitor_mut_references, + ); + <(BlockHash, ChannelManager)>::read(&mut f, read_args) + .map_to_mm(|e| EnableLightningError::IOError(e.to_string()))? + } else { + // Initialize the ChannelManager to starting a new node without history + let chain_params = ChainParameters { + network: platform_fields.network.clone().into(), + best_block: BestBlock::new(best_block_hash, best_block.height as u32), + }; + let new_channel_manager = channelmanager::ChannelManager::new( + fee_estimator.clone(), + chain_monitor.clone(), + broadcaster.clone(), + logger.clone(), + keys_manager.clone(), + user_config, + chain_params, + ); + (best_block_hash, new_channel_manager) + } + }; + + let channel_manager: Arc = Arc::new(channel_manager); + + // Sync ChannelMonitors and ChannelManager to chain tip if the node is restarting and has open channels + if channel_manager_blockhash != best_block_hash { + process_txs_unconfirmations(platform_fields.clone(), chain_monitor.clone(), channel_manager.clone()).await; + process_txs_confirmations( + // It's safe to use unwrap here for now until implementing Native Client for Lightning + platform_fields.clone(), + rpc_client.clone(), + chain_monitor.clone(), + channel_manager.clone(), + best_header.block_height(), + ) + .await; + update_best_block(chain_monitor.clone(), channel_manager.clone(), best_header).await; + } + + // Give ChannelMonitors to ChainMonitor + for (_, channel_monitor) in channelmonitors.drain(..) { + let funding_outpoint = channel_monitor.get_funding_txo().0; + chain_monitor + .watch_channel(funding_outpoint, channel_monitor) + .map_to_mm(|e| EnableLightningError::IOError(format!("{:?}", e)))?; + } + + // Update best block whenever there's a new chain tip or a block has been newly disconnected + spawn(ln_best_block_update_loop( + // It's safe to use unwrap here for now until implementing Native Client for Lightning + platform_fields, + chain_monitor.clone(), + channel_manager.clone(), + rpc_client.clone(), + best_block, + )); + + Ok((chain_monitor, channel_manager)) +} + async fn persist_network_graph_loop(persister: Arc, network_graph: Arc) { loop { if let Err(e) = persister.save_network_graph(network_graph.clone()).await { @@ -257,130 +380,15 @@ pub async fn start_lightning( protocol_conf.confirmations, )); - // Initialize the FeeEstimator. UtxoStandardCoin implements the FeeEstimator trait, so it'll act as our fee estimator. - let fee_estimator = platform_fields.clone(); - // Initialize the Logger let logger = ctx.log.0.clone(); - // Initialize the BroadcasterInterface. UtxoStandardCoin implements the BroadcasterInterface trait, so it'll act as our transaction - // broadcaster. - let broadcaster = Arc::new(platform_coin); - // Initialize Persister let persister = init_persister(ctx, conf.ticker.clone(), params.backup_path).await?; - // Initialize the Filter. PlatformFields implements the Filter trait, we can use it to construct the filter. - let filter = Some(platform_fields.clone()); - - // Initialize the ChainMonitor - let chain_monitor: Arc = Arc::new(chainmonitor::ChainMonitor::new( - filter.clone(), - broadcaster.clone(), - logger.clone(), - fee_estimator.clone(), - persister.clone(), - )); - // Initialize the KeysManager let keys_manager = init_keys_manager(ctx)?; - // Read ChannelMonitor state from disk, important for lightning node is restarting and has at least 1 channel - let mut channelmonitors = persister - .read_channelmonitors(keys_manager.clone()) - .map_to_mm(|e| EnableLightningError::IOError(e.to_string()))?; - - // This is used for Electrum only to prepare for chain synchronization - if let Some(ref filter) = filter { - for (_, chan_mon) in channelmonitors.iter() { - chan_mon.load_outputs_to_watch(filter); - } - } - - // TODO: Right now it's safe to unwrap here, when implementing Native client for lightning whenever filter is used - // the code it's used in will be a part of the electrum client implementation only - let rpc_client = match &filter.clone().unwrap().platform_coin.as_ref().rpc_client { - UtxoRpcClientEnum::Electrum(c) => c.clone(), - UtxoRpcClientEnum::Native(_) => { - return MmError::err(EnableLightningError::UnsupportedMode( - "Lightning network".into(), - "electrum".into(), - )) - }, - }; - let best_header = get_best_header(&rpc_client).await?; - let best_block = RpcBestBlock::from(best_header.clone()); - let best_block_hash = BlockHash::from_hash( - sha256d::Hash::from_slice(&best_block.hash.0).map_to_mm(|e| EnableLightningError::HashError(e.to_string()))?, - ); - let (channel_manager_blockhash, channel_manager) = { - let user_config = conf.clone().into(); - if let Ok(mut f) = File::open(persister.manager_path()) { - let mut channel_monitor_mut_references = Vec::new(); - for (_, channel_monitor) in channelmonitors.iter_mut() { - channel_monitor_mut_references.push(channel_monitor); - } - // Read ChannelManager data from the file - let read_args = ChannelManagerReadArgs::new( - keys_manager.clone(), - fee_estimator.clone(), - chain_monitor.clone(), - broadcaster.clone(), - logger.clone(), - user_config, - channel_monitor_mut_references, - ); - <(BlockHash, ChannelManager)>::read(&mut f, read_args) - .map_to_mm(|e| EnableLightningError::IOError(e.to_string()))? - } else { - // Initialize the ChannelManager to starting a new node without history - let chain_params = ChainParameters { - network, - best_block: BestBlock::new(best_block_hash, best_block.height as u32), - }; - let new_channel_manager = channelmanager::ChannelManager::new( - fee_estimator.clone(), - chain_monitor.clone(), - broadcaster.clone(), - logger.clone(), - keys_manager.clone(), - user_config, - chain_params, - ); - (best_block_hash, new_channel_manager) - } - }; - - let channel_manager: Arc = Arc::new(channel_manager); - - // Sync ChannelMonitors and ChannelManager to chain tip if the node is restarting and has open channels - if channel_manager_blockhash != best_block_hash { - process_txs_unconfirmations( - filter.clone().unwrap().clone(), - chain_monitor.clone(), - channel_manager.clone(), - ) - .await; - process_txs_confirmations( - // It's safe to use unwrap here for now until implementing Native Client for Lightning - filter.clone().unwrap().clone(), - rpc_client.clone(), - chain_monitor.clone(), - channel_manager.clone(), - best_header.block_height(), - ) - .await; - update_best_block(chain_monitor.clone(), channel_manager.clone(), best_header).await; - } - - // Give ChannelMonitors to ChainMonitor - for (_, channel_monitor) in channelmonitors.drain(..) { - let funding_outpoint = channel_monitor.get_funding_txo().0; - chain_monitor - .watch_channel(funding_outpoint, channel_monitor) - .map_to_mm(|e| EnableLightningError::IOError(format!("{:?}", e)))?; - } - // Initialize the NetGraphMsgHandler. This is used for providing routes to send payments over let network_graph = Arc::new(persister.get_network_graph(network).await?); spawn(persist_network_graph_loop(persister.clone(), network_graph.clone())); @@ -390,6 +398,16 @@ pub async fn start_lightning( logger.clone(), )); + // Initialize the ChannelManager + let (chain_monitor, channel_manager) = init_channel_manager( + platform_fields.clone(), + logger.clone(), + persister.clone(), + keys_manager.clone(), + conf.clone(), + ) + .await?; + // Initialize the PeerManager let peer_manager = init_peer_manager( ctx.clone(), @@ -401,23 +419,13 @@ pub async fn start_lightning( ) .await?; - // Update best block whenever there's a new chain tip or a block has been newly disconnected - spawn(ln_best_block_update_loop( - // It's safe to use unwrap here for now until implementing Native Client for Lightning - filter.clone().unwrap(), - chain_monitor.clone(), - channel_manager.clone(), - rpc_client.clone(), - best_block, - )); - let inbound_payments = Arc::new(PaMutex::new(HashMap::new())); let outbound_payments = Arc::new(PaMutex::new(HashMap::new())); // Initialize the event handler let event_handler = Arc::new(ln_events::LightningEventHandler::new( // It's safe to use unwrap here for now until implementing Native Client for Lightning - filter.clone().unwrap(), + platform_fields.clone(), channel_manager.clone(), keys_manager.clone(), inbound_payments.clone(), From 88009fa8ff329a7801dfd68fb94c29e5529c2827 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Tue, 8 Mar 2022 19:20:58 +0200 Subject: [PATCH 06/49] WIP: refactoring, ln_p2p --- mm2src/coins/lightning.rs | 6 +- mm2src/coins/lightning/ln_connections.rs | 103 ----------- mm2src/coins/lightning/ln_p2p.rs | 211 +++++++++++++++++++++++ mm2src/coins/lightning/ln_utils.rs | 121 +------------ 4 files changed, 221 insertions(+), 220 deletions(-) delete mode 100644 mm2src/coins/lightning/ln_connections.rs create mode 100644 mm2src/coins/lightning/ln_p2p.rs diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 9fb0b1ca28..b0b5ec210f 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -35,7 +35,6 @@ use lightning_invoice::Invoice; use lightning_persister::storage::{FileSystemStorage, NodesAddressesMapShared}; use lightning_persister::LightningPersister; use ln_conf::{ChannelOptions, LightningCoinConf, PlatformCoinConfirmations}; -use ln_connections::{connect_to_node, ConnectToNodeRes}; use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelError, CloseChannelResult, ConnectToNodeError, ConnectToNodeResult, EnableLightningError, EnableLightningResult, GenerateInvoiceError, GenerateInvoiceResult, GetChannelDetailsError, GetChannelDetailsResult, @@ -43,8 +42,9 @@ use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelErr ListPaymentsError, ListPaymentsResult, OpenChannelError, OpenChannelResult, SendPaymentError, SendPaymentResult}; use ln_events::LightningEventHandler; +use ln_p2p::{connect_to_node, ConnectToNodeRes, PeerManager}; use ln_serialization::{InvoiceForRPC, NodeAddress, PublicKeyForRPC}; -use ln_utils::{ChainMonitor, ChannelManager, InvoicePayer, PeerManager}; +use ln_utils::{ChainMonitor, ChannelManager, InvoicePayer}; use parking_lot::Mutex as PaMutex; use rpc::v1::types::{Bytes as BytesJson, H256 as H256Json}; use script::{Builder, TransactionInputSigner}; @@ -59,9 +59,9 @@ use std::str::FromStr; use std::sync::Arc; pub mod ln_conf; -mod ln_connections; pub mod ln_errors; mod ln_events; +mod ln_p2p; mod ln_rpc; mod ln_serialization; pub mod ln_utils; diff --git a/mm2src/coins/lightning/ln_connections.rs b/mm2src/coins/lightning/ln_connections.rs deleted file mode 100644 index cc725d082b..0000000000 --- a/mm2src/coins/lightning/ln_connections.rs +++ /dev/null @@ -1,103 +0,0 @@ -use super::*; -use common::executor::{spawn, Timer}; -use derive_more::Display; -use lightning_persister::storage::NodesAddressesMapShared; -use tokio::net::TcpListener; - -const TRY_RECONNECTING_TO_NODE_INTERVAL: f64 = 60.; - -pub async fn ln_p2p_loop(peer_manager: Arc, listener: TcpListener) { - loop { - let peer_mgr = peer_manager.clone(); - let tcp_stream = match listener.accept().await { - Ok((stream, addr)) => { - log::debug!("New incoming lightning connection from node address: {}", addr); - stream - }, - Err(e) => { - log::error!("Error on accepting lightning connection: {}", e); - continue; - }, - }; - if let Ok(stream) = tcp_stream.into_std() { - spawn(async move { - lightning_net_tokio::setup_inbound(peer_mgr.clone(), stream).await; - }); - }; - } -} - -#[derive(Display)] -pub enum ConnectToNodeRes { - #[display(fmt = "Already connected to node: {}@{}", _0, _1)] - AlreadyConnected(String, String), - #[display(fmt = "Connected successfully to node : {}@{}", _0, _1)] - ConnectedSuccessfully(String, String), -} - -pub async fn connect_to_node( - pubkey: PublicKey, - node_addr: SocketAddr, - peer_manager: Arc, -) -> ConnectToNodeResult { - if peer_manager.get_peer_node_ids().contains(&pubkey) { - return Ok(ConnectToNodeRes::AlreadyConnected( - pubkey.to_string(), - node_addr.to_string(), - )); - } - - match lightning_net_tokio::connect_outbound(Arc::clone(&peer_manager), pubkey, node_addr).await { - Some(connection_closed_future) => { - let mut connection_closed_future = Box::pin(connection_closed_future); - loop { - // Make sure the connection is still established. - match futures::poll!(&mut connection_closed_future) { - std::task::Poll::Ready(_) => { - return MmError::err(ConnectToNodeError::ConnectionError(format!( - "Node {} disconnected before finishing the handshake", - pubkey - ))); - }, - std::task::Poll::Pending => {}, - } - - match peer_manager.get_peer_node_ids().contains(&pubkey) { - true => break, - // Wait for the handshake to complete if false. - false => Timer::sleep_ms(10).await, - } - } - }, - None => { - return MmError::err(ConnectToNodeError::ConnectionError(format!( - "Failed to connect to node: {}", - pubkey - ))) - }, - } - - Ok(ConnectToNodeRes::ConnectedSuccessfully( - pubkey.to_string(), - node_addr.to_string(), - )) -} - -pub async fn connect_to_nodes_loop(open_channels_nodes: NodesAddressesMapShared, peer_manager: Arc) { - loop { - let open_channels_nodes = open_channels_nodes.lock().clone(); - for (pubkey, node_addr) in open_channels_nodes { - let peer_manager = peer_manager.clone(); - match connect_to_node(pubkey, node_addr, peer_manager.clone()).await { - Ok(res) => { - if let ConnectToNodeRes::ConnectedSuccessfully(_, _) = res { - log::info!("{}", res.to_string()); - } - }, - Err(e) => log::error!("{}", e.to_string()), - } - } - - Timer::sleep(TRY_RECONNECTING_TO_NODE_INTERVAL).await; - } -} diff --git a/mm2src/coins/lightning/ln_p2p.rs b/mm2src/coins/lightning/ln_p2p.rs new file mode 100644 index 0000000000..784d267b1b --- /dev/null +++ b/mm2src/coins/lightning/ln_p2p.rs @@ -0,0 +1,211 @@ +use super::*; +use common::executor::{spawn, Timer}; +use common::ip_addr::fetch_external_ip; +use common::log::LogState; +use derive_more::Display; +use lightning::chain::Access; +use lightning::ln::msgs::NetAddress; +use lightning::ln::peer_handler::{IgnoringMessageHandler, MessageHandler, SimpleArcPeerManager}; +use lightning::routing::network_graph::{NetGraphMsgHandler, NetworkGraph}; +use lightning_net_tokio::SocketDescriptor; +use lightning_persister::storage::NodesAddressesMapShared; +use rand::RngCore; +use secp256k1::SecretKey; +use std::net::{IpAddr, Ipv4Addr}; +use tokio::net::TcpListener; + +const TRY_RECONNECTING_TO_NODE_INTERVAL: f64 = 60.; +const BROADCAST_NODE_ANNOUNCEMENT_INTERVAL: u64 = 600; + +type NetworkGossip = NetGraphMsgHandler, Arc, Arc>; + +pub type PeerManager = SimpleArcPeerManager< + SocketDescriptor, + ChainMonitor, + UtxoStandardCoin, + PlatformFields, + dyn Access + Send + Sync, + LogState, +>; + +#[derive(Display)] +pub enum ConnectToNodeRes { + #[display(fmt = "Already connected to node: {}@{}", _0, _1)] + AlreadyConnected(String, String), + #[display(fmt = "Connected successfully to node : {}@{}", _0, _1)] + ConnectedSuccessfully(String, String), +} + +pub async fn connect_to_node( + pubkey: PublicKey, + node_addr: SocketAddr, + peer_manager: Arc, +) -> ConnectToNodeResult { + if peer_manager.get_peer_node_ids().contains(&pubkey) { + return Ok(ConnectToNodeRes::AlreadyConnected( + pubkey.to_string(), + node_addr.to_string(), + )); + } + + match lightning_net_tokio::connect_outbound(Arc::clone(&peer_manager), pubkey, node_addr).await { + Some(connection_closed_future) => { + let mut connection_closed_future = Box::pin(connection_closed_future); + loop { + // Make sure the connection is still established. + match futures::poll!(&mut connection_closed_future) { + std::task::Poll::Ready(_) => { + return MmError::err(ConnectToNodeError::ConnectionError(format!( + "Node {} disconnected before finishing the handshake", + pubkey + ))); + }, + std::task::Poll::Pending => {}, + } + + match peer_manager.get_peer_node_ids().contains(&pubkey) { + true => break, + // Wait for the handshake to complete if false. + false => Timer::sleep_ms(10).await, + } + } + }, + None => { + return MmError::err(ConnectToNodeError::ConnectionError(format!( + "Failed to connect to node: {}", + pubkey + ))) + }, + } + + Ok(ConnectToNodeRes::ConnectedSuccessfully( + pubkey.to_string(), + node_addr.to_string(), + )) +} + +pub async fn connect_to_nodes_loop(open_channels_nodes: NodesAddressesMapShared, peer_manager: Arc) { + loop { + let open_channels_nodes = open_channels_nodes.lock().clone(); + for (pubkey, node_addr) in open_channels_nodes { + let peer_manager = peer_manager.clone(); + match connect_to_node(pubkey, node_addr, peer_manager.clone()).await { + Ok(res) => { + if let ConnectToNodeRes::ConnectedSuccessfully(_, _) = res { + log::info!("{}", res.to_string()); + } + }, + Err(e) => log::error!("{}", e.to_string()), + } + } + + Timer::sleep(TRY_RECONNECTING_TO_NODE_INTERVAL).await; + } +} + +// TODO: add TOR address option +fn netaddress_from_ipaddr(addr: IpAddr, port: u16) -> Vec { + if addr == Ipv4Addr::new(0, 0, 0, 0) || addr == Ipv4Addr::new(127, 0, 0, 1) { + return Vec::new(); + } + let mut addresses = Vec::new(); + let address = match addr { + IpAddr::V4(addr) => NetAddress::IPv4 { + addr: u32::from(addr).to_be_bytes(), + port, + }, + IpAddr::V6(addr) => NetAddress::IPv6 { + addr: u128::from(addr).to_be_bytes(), + port, + }, + }; + addresses.push(address); + addresses +} + +pub async fn ln_node_announcement_loop( + channel_manager: Arc, + node_name: [u8; 32], + node_color: [u8; 3], + port: u16, +) { + loop { + // Right now if the node is behind NAT the external ip is fetched on every loop + // If the node does not announce a public IP, it will not be displayed on the network graph, + // and other nodes will not be able to open a channel with it. But it can open channels with other nodes. + let addresses = match fetch_external_ip().await { + Ok(ip) => { + log::debug!("Fetch real IP successfully: {}:{}", ip, port); + netaddress_from_ipaddr(ip, port) + }, + Err(e) => { + log::error!("Error while fetching external ip for node announcement: {}", e); + Timer::sleep(BROADCAST_NODE_ANNOUNCEMENT_INTERVAL as f64).await; + continue; + }, + }; + channel_manager.broadcast_node_announcement(node_color, node_name, addresses); + + Timer::sleep(BROADCAST_NODE_ANNOUNCEMENT_INTERVAL as f64).await; + } +} + +async fn ln_p2p_loop(peer_manager: Arc, listener: TcpListener) { + loop { + let peer_mgr = peer_manager.clone(); + let tcp_stream = match listener.accept().await { + Ok((stream, addr)) => { + log::debug!("New incoming lightning connection from node address: {}", addr); + stream + }, + Err(e) => { + log::error!("Error on accepting lightning connection: {}", e); + continue; + }, + }; + if let Ok(stream) = tcp_stream.into_std() { + spawn(async move { + lightning_net_tokio::setup_inbound(peer_mgr.clone(), stream).await; + }); + }; + } +} + +pub async fn init_peer_manager( + ctx: MmArc, + listening_port: u16, + channel_manager: Arc, + network_gossip: Arc, + node_secret: SecretKey, + logger: Arc, +) -> EnableLightningResult> { + // The set (possibly empty) of socket addresses on which this node accepts incoming connections. + // If the user wishes to preserve privacy, addresses should likely contain only Tor Onion addresses. + let listening_addr = myipaddr(ctx).await.map_to_mm(EnableLightningError::InvalidAddress)?; + // If the listening port is used start_lightning should return an error early + let listener = TcpListener::bind(format!("{}:{}", listening_addr, listening_port)) + .await + .map_to_mm(|e| EnableLightningError::IOError(e.to_string()))?; + + // ephemeral_random_data is used to derive per-connection ephemeral keys + let mut ephemeral_bytes = [0; 32]; + rand::thread_rng().fill_bytes(&mut ephemeral_bytes); + let lightning_msg_handler = MessageHandler { + chan_handler: channel_manager, + route_handler: network_gossip, + }; + + // IgnoringMessageHandler is used as custom message types (experimental and application-specific messages) is not needed + let peer_manager: Arc = Arc::new(PeerManager::new( + lightning_msg_handler, + node_secret, + &ephemeral_bytes, + logger, + Arc::new(IgnoringMessageHandler {}), + )); + + // Initialize p2p networking + spawn(ln_p2p_loop(peer_manager.clone(), listener)); + + Ok(peer_manager) +} diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 1a58cc5992..ea0e16e490 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -1,6 +1,6 @@ use super::*; use crate::lightning::ln_conf::{LightningCoinConf, LightningProtocolConf}; -use crate::lightning::ln_connections::{connect_to_nodes_loop, ln_p2p_loop}; +use crate::lightning::ln_p2p::{connect_to_nodes_loop, init_peer_manager, ln_node_announcement_loop}; use crate::utxo::rpc_clients::{electrum_script_hash, BestBlock as RpcBestBlock, ElectrumBlockHeader, ElectrumClient, ElectrumNonce, UtxoRpcError}; use crate::utxo::utxo_standard::UtxoStandardCoin; @@ -11,7 +11,6 @@ use bitcoin::consensus::encode::deserialize; use bitcoin::hash_types::{BlockHash, TxMerkleNode, Txid}; use bitcoin_hashes::{sha256d, Hash}; use common::executor::{spawn, Timer}; -use common::ip_addr::fetch_external_ip; use common::jsonrpc_client::JsonRpcErrorType; use common::log; use common::log::LogState; @@ -21,33 +20,25 @@ use lightning::chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager use lightning::chain::{chainmonitor, Access, BestBlock, Confirm, Watch}; use lightning::ln::channelmanager; use lightning::ln::channelmanager::{ChainParameters, ChannelManagerReadArgs, SimpleArcChannelManager}; -use lightning::ln::msgs::NetAddress; -use lightning::ln::peer_handler::{IgnoringMessageHandler, MessageHandler, SimpleArcPeerManager}; use lightning::routing::network_graph::{NetGraphMsgHandler, NetworkGraph}; use lightning::routing::scoring::Scorer; use lightning::util::ser::ReadableArgs; use lightning_background_processor::BackgroundProcessor; use lightning_invoice::payment; use lightning_invoice::utils::DefaultRouter; -use lightning_net_tokio::SocketDescriptor; use lightning_persister::storage::{FileSystemStorage, NodesAddressesMap}; use lightning_persister::LightningPersister; use parking_lot::Mutex as PaMutex; -use rand::RngCore; use rpc::v1::types::H256; -use secp256k1::SecretKey; use std::cmp::Ordering; use std::collections::HashMap; use std::convert::TryInto; use std::fs::File; -use std::net::{IpAddr, Ipv4Addr}; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::SystemTime; -use tokio::net::TcpListener; const CHECK_FOR_NEW_BEST_BLOCK_INTERVAL: u64 = 60; -const BROADCAST_NODE_ANNOUNCEMENT_INTERVAL: u64 = 600; const NETWORK_GRAPH_PERSIST_INTERVAL: u64 = 600; const SCORER_PERSIST_INTERVAL: u64 = 600; @@ -62,40 +53,9 @@ pub type ChainMonitor = chainmonitor::ChainMonitor< pub type ChannelManager = SimpleArcChannelManager; -pub type PeerManager = SimpleArcPeerManager< - SocketDescriptor, - ChainMonitor, - UtxoStandardCoin, - PlatformFields, - dyn Access + Send + Sync, - LogState, ->; - -pub type InvoicePayer = payment::InvoicePayer, Router, Arc>, Arc, E>; - type Router = DefaultRouter, Arc>; -type NetworkGossip = NetGraphMsgHandler, Arc, Arc>; - -// TODO: add TOR address option -fn netaddress_from_ipaddr(addr: IpAddr, port: u16) -> Vec { - if addr == Ipv4Addr::new(0, 0, 0, 0) || addr == Ipv4Addr::new(127, 0, 0, 1) { - return Vec::new(); - } - let mut addresses = Vec::new(); - let address = match addr { - IpAddr::V4(addr) => NetAddress::IPv4 { - addr: u32::from(addr).to_be_bytes(), - port, - }, - IpAddr::V6(addr) => NetAddress::IPv6 { - addr: u128::from(addr).to_be_bytes(), - port, - }, - }; - addresses.push(address); - addresses -} +pub type InvoicePayer = payment::InvoicePayer, Router, Arc>, Arc, E>; #[derive(Clone, Debug, Deserialize, Serialize)] pub struct LightningParams { @@ -304,45 +264,6 @@ async fn persist_scorer_loop(persister: Arc, scorer: Arc, - network_gossip: Arc, - node_secret: SecretKey, - logger: Arc, -) -> EnableLightningResult> { - // The set (possibly empty) of socket addresses on which this node accepts incoming connections. - // If the user wishes to preserve privacy, addresses should likely contain only Tor Onion addresses. - let listening_addr = myipaddr(ctx).await.map_to_mm(EnableLightningError::InvalidAddress)?; - // If the listening port is used start_lightning should return an error early - let listener = TcpListener::bind(format!("{}:{}", listening_addr, listening_port)) - .await - .map_to_mm(|e| EnableLightningError::IOError(e.to_string()))?; - - // ephemeral_random_data is used to derive per-connection ephemeral keys - let mut ephemeral_bytes = [0; 32]; - rand::thread_rng().fill_bytes(&mut ephemeral_bytes); - let lightning_msg_handler = MessageHandler { - chan_handler: channel_manager, - route_handler: network_gossip, - }; - - // IgnoringMessageHandler is used as custom message types (experimental and application-specific messages) is not needed - let peer_manager: Arc = Arc::new(PeerManager::new( - lightning_msg_handler, - node_secret, - &ephemeral_bytes, - logger, - Arc::new(IgnoringMessageHandler {}), - )); - - // Initialize p2p networking - spawn(ln_p2p_loop(peer_manager.clone(), listener)); - - Ok(peer_manager) -} - async fn get_open_channels_nodes_addresses( persister: Arc, channel_manager: Arc, @@ -373,10 +294,9 @@ pub async fn start_lightning( )); } - let network = protocol_conf.network.clone().into(); let platform_fields = Arc::new(PlatformFields::new( platform_coin.clone(), - protocol_conf.network, + protocol_conf.network.clone(), protocol_conf.confirmations, )); @@ -390,7 +310,7 @@ pub async fn start_lightning( let keys_manager = init_keys_manager(ctx)?; // Initialize the NetGraphMsgHandler. This is used for providing routes to send payments over - let network_graph = Arc::new(persister.get_network_graph(network).await?); + let network_graph = Arc::new(persister.get_network_graph(protocol_conf.network.into()).await?); spawn(persist_network_graph_loop(persister.clone(), network_graph.clone())); let network_gossip = Arc::new(NetGraphMsgHandler::new( network_graph.clone(), @@ -456,7 +376,7 @@ pub async fn start_lightning( // Start Background Processing. Runs tasks periodically in the background to keep LN node operational. // InvoicePayer will act as our event handler as it handles some of the payments related events before // delegating it to LightningEventHandler. - let background_processor = BackgroundProcessor::start( + let background_processor = Arc::new(BackgroundProcessor::start( persist_channel_manager_callback, invoice_payer.clone(), chain_monitor.clone(), @@ -464,7 +384,7 @@ pub async fn start_lightning( Some(network_gossip), peer_manager.clone(), logger, - ); + )); // If channel_nodes_data file exists, read channels nodes data from disk and reconnect to channel nodes/peers if possible. let open_channels_nodes = Arc::new(PaMutex::new( @@ -484,7 +404,7 @@ pub async fn start_lightning( platform_fields, conf, peer_manager, - background_processor: Arc::new(background_processor), + background_processor, channel_manager, chain_monitor, keys_manager, @@ -842,30 +762,3 @@ async fn ln_best_block_update_loop( Timer::sleep(CHECK_FOR_NEW_BEST_BLOCK_INTERVAL as f64).await; } } - -async fn ln_node_announcement_loop( - channel_manager: Arc, - node_name: [u8; 32], - node_color: [u8; 3], - port: u16, -) { - loop { - // Right now if the node is behind NAT the external ip is fetched on every loop - // If the node does not announce a public IP, it will not be displayed on the network graph, - // and other nodes will not be able to open a channel with it. But it can open channels with other nodes. - let addresses = match fetch_external_ip().await { - Ok(ip) => { - log::debug!("Fetch real IP successfully: {}:{}", ip, port); - netaddress_from_ipaddr(ip, port) - }, - Err(e) => { - log::error!("Error while fetching external ip for node announcement: {}", e); - Timer::sleep(BROADCAST_NODE_ANNOUNCEMENT_INTERVAL as f64).await; - continue; - }, - }; - channel_manager.broadcast_node_announcement(node_color, node_name, addresses); - - Timer::sleep(BROADCAST_NODE_ANNOUNCEMENT_INTERVAL as f64).await; - } -} From b13d64f6c64da4889226fd7e1012dde36639d65c Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Wed, 9 Mar 2022 14:29:32 +0200 Subject: [PATCH 07/49] WIP: refactoring, ln_platform --- mm2src/coins/lightning.rs | 65 +-- mm2src/coins/lightning/ln_events.rs | 22 +- mm2src/coins/lightning/ln_p2p.rs | 10 +- mm2src/coins/lightning/ln_platform.rs | 576 ++++++++++++++++++++++++++ mm2src/coins/lightning/ln_rpc.rs | 169 -------- mm2src/coins/lightning/ln_utils.rs | 418 ++----------------- 6 files changed, 627 insertions(+), 633 deletions(-) create mode 100644 mm2src/coins/lightning/ln_platform.rs delete mode 100644 mm2src/coins/lightning/ln_rpc.rs diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index b0b5ec210f..f45fdb3342 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -8,8 +8,6 @@ use crate::{BalanceFut, CoinBalance, FeeApproxStage, FoundSwapTxSpend, HistorySy ValidatePaymentInput, WithdrawError, WithdrawFut, WithdrawRequest}; use async_trait::async_trait; use bigdecimal::BigDecimal; -use bitcoin::blockdata::script::Script; -use bitcoin::hash_types::Txid; use bitcoin::hashes::Hash; use bitcoin_hashes::sha256::Hash as Sha256; use chain::TransactionOutput; @@ -25,7 +23,6 @@ use keys::{AddressHashEnum, KeyPair}; use lightning::chain::channelmonitor::Balance; use lightning::chain::keysinterface::KeysInterface; use lightning::chain::keysinterface::KeysManager; -use lightning::chain::WatchedOutput; use lightning::ln::channelmanager::{ChannelDetails, MIN_FINAL_CLTV_EXPIRY}; use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; use lightning::util::config::UserConfig; @@ -43,6 +40,7 @@ use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelErr SendPaymentResult}; use ln_events::LightningEventHandler; use ln_p2p::{connect_to_node, ConnectToNodeRes, PeerManager}; +use ln_platform::Platform; use ln_serialization::{InvoiceForRPC, NodeAddress, PublicKeyForRPC}; use ln_utils::{ChainMonitor, ChannelManager, InvoicePayer}; use parking_lot::Mutex as PaMutex; @@ -62,64 +60,13 @@ pub mod ln_conf; pub mod ln_errors; mod ln_events; mod ln_p2p; -mod ln_rpc; +mod ln_platform; mod ln_serialization; pub mod ln_utils; type PaymentsMap = HashMap; type PaymentsMapShared = Arc>; -pub struct PlatformFields { - pub platform_coin: UtxoStandardCoin, - /// Main/testnet/signet/regtest Needed for lightning node to know which network to connect to - pub network: BlockchainNetwork, - // Default fees to and confirmation targets to be used for FeeEstimator. Default fees are used when the call for - // estimate_fee_sat fails. - pub default_fees_and_confirmations: PlatformCoinConfirmations, - // This cache stores the transactions that the LN node has interest in. - pub registered_txs: PaMutex>>, - // This cache stores the outputs that the LN node has interest in. - pub registered_outputs: PaMutex>, - // This cache stores transactions to be broadcasted once the other node accepts the channel - pub unsigned_funding_txs: PaMutex>, -} - -impl PlatformFields { - pub fn new( - platform_coin: UtxoStandardCoin, - network: BlockchainNetwork, - default_fees_and_confirmations: PlatformCoinConfirmations, - ) -> Self { - PlatformFields { - platform_coin, - network, - default_fees_and_confirmations, - registered_txs: PaMutex::new(HashMap::new()), - registered_outputs: PaMutex::new(Vec::new()), - unsigned_funding_txs: PaMutex::new(HashMap::new()), - } - } - - pub fn add_tx(&self, txid: &Txid, script_pubkey: &Script) { - let mut registered_txs = self.registered_txs.lock(); - match registered_txs.get_mut(txid) { - Some(h) => { - h.insert(script_pubkey.clone()); - }, - None => { - let mut script_pubkeys = HashSet::new(); - script_pubkeys.insert(script_pubkey.clone()); - registered_txs.insert(*txid, script_pubkeys); - }, - } - } - - pub fn add_output(&self, output: WatchedOutput) { - let mut registered_outputs = self.registered_outputs.lock(); - registered_outputs.push(output); - } -} - #[derive(Clone, Serialize)] #[serde(rename_all = "lowercase")] pub enum HTLCStatus { @@ -139,7 +86,7 @@ pub struct PaymentInfo { #[derive(Clone)] pub struct LightningCoin { - pub platform_fields: Arc, + pub platform: Arc, pub conf: LightningCoinConf, /// The lightning node peer manager that takes care of connecting to peers, etc.. pub peer_manager: Arc, @@ -170,7 +117,7 @@ impl fmt::Debug for LightningCoin { } impl LightningCoin { - fn platform_coin(&self) -> &UtxoStandardCoin { &self.platform_fields.platform_coin } + fn platform_coin(&self) -> &UtxoStandardCoin { &self.platform.coin } fn my_node_id(&self) -> String { self.channel_manager.get_our_node_id().to_string() } @@ -668,7 +615,7 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes .await?; { - let mut unsigned_funding_txs = ln_coin.platform_fields.unsigned_funding_txs.lock(); + let mut unsigned_funding_txs = ln_coin.platform.unsigned_funding_txs.lock(); unsigned_funding_txs.insert(temp_channel_id, unsigned); } @@ -817,7 +764,7 @@ pub async fn generate_invoice( node_pubkey )); } - let network = ln_coin.platform_fields.network.clone().into(); + let network = ln_coin.platform.network.clone().into(); let invoice = create_invoice_from_channelmanager( &ln_coin.channel_manager, ln_coin.keys_manager, diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index d71b2f7fc6..ba5820d491 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -16,7 +16,7 @@ use std::sync::Arc; use utxo_signer::with_key_pair::sign_tx; pub struct LightningEventHandler { - filter: Arc, + platform: Arc, channel_manager: Arc, keys_manager: Arc, inbound_payments: PaymentsMapShared, @@ -49,7 +49,7 @@ impl EventHandler for LightningEventHandler { Event::PaymentForwarded { fee_earned_msat, claim_from_onchain_tx } => log::info!( "Recieved a fee of {} milli-satoshis for a successfully forwarded payment through our {} lightning node. Was the forwarded HTLC claimed by our counterparty via an on-chain transaction?: {}", fee_earned_msat.unwrap_or_default(), - self.filter.platform_coin.ticker(), + self.platform.coin.ticker(), claim_from_onchain_tx, ), // Todo: Use storage to store channels history @@ -92,11 +92,11 @@ impl EventHandler for LightningEventHandler { fn sign_funding_transaction( temp_channel_id: [u8; 32], output_script: &Script, - filter: Arc, + platform: Arc, ) -> OpenChannelResult { - let coin = &filter.platform_coin; + let coin = &platform.coin; let mut unsigned = { - let unsigned_funding_txs = filter.unsigned_funding_txs.lock(); + let unsigned_funding_txs = platform.unsigned_funding_txs.lock(); unsigned_funding_txs .get(&temp_channel_id) .ok_or_else(|| { @@ -126,14 +126,14 @@ fn sign_funding_transaction( impl LightningEventHandler { pub fn new( - filter: Arc, + platform: Arc, channel_manager: Arc, keys_manager: Arc, inbound_payments: PaymentsMapShared, outbound_payments: PaymentsMapShared, ) -> Self { LightningEventHandler { - filter, + platform, channel_manager, keys_manager, inbound_payments, @@ -146,7 +146,7 @@ impl LightningEventHandler { "Handling FundingGenerationReady event for temporary_channel_id: {}", hex::encode(temporary_channel_id) ); - let funding_tx = match sign_funding_transaction(temporary_channel_id, output_script, self.filter.clone()) { + let funding_tx = match sign_funding_transaction(temporary_channel_id, output_script, self.platform.clone()) { Ok(tx) => tx, Err(e) => { log::error!( @@ -259,7 +259,7 @@ impl LightningEventHandler { fn handle_spendable_outputs(&self, outputs: &[SpendableOutputDescriptor]) { log::info!("Handling SpendableOutputs event!"); - let platform_coin = &self.filter.platform_coin; + let platform_coin = &self.platform.coin; // Todo: add support for Hardware wallets for funding transactions and spending spendable outputs (channel closing transactions) let my_address = match platform_coin.as_ref().derivation_method.iguana_or_err() { Ok(addr) => addr, @@ -269,7 +269,7 @@ impl LightningEventHandler { }, }; let change_destination_script = Builder::build_witness_script(&my_address.hash).to_bytes().take().into(); - let feerate_sat_per_1000_weight = self.filter.get_est_sat_per_1000_weight(ConfirmationTarget::Normal); + let feerate_sat_per_1000_weight = self.platform.get_est_sat_per_1000_weight(ConfirmationTarget::Normal); let output_descriptors = &outputs.iter().collect::>(); let spending_tx = match self.keys_manager.spend_spendable_outputs( output_descriptors, @@ -284,6 +284,6 @@ impl LightningEventHandler { return; }, }; - platform_coin.broadcast_transaction(&spending_tx); + self.platform.broadcast_transaction(&spending_tx); } } diff --git a/mm2src/coins/lightning/ln_p2p.rs b/mm2src/coins/lightning/ln_p2p.rs index 784d267b1b..d614ac3457 100644 --- a/mm2src/coins/lightning/ln_p2p.rs +++ b/mm2src/coins/lightning/ln_p2p.rs @@ -19,14 +19,8 @@ const BROADCAST_NODE_ANNOUNCEMENT_INTERVAL: u64 = 600; type NetworkGossip = NetGraphMsgHandler, Arc, Arc>; -pub type PeerManager = SimpleArcPeerManager< - SocketDescriptor, - ChainMonitor, - UtxoStandardCoin, - PlatformFields, - dyn Access + Send + Sync, - LogState, ->; +pub type PeerManager = + SimpleArcPeerManager; #[derive(Display)] pub enum ConnectToNodeRes { diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs new file mode 100644 index 0000000000..d35395245c --- /dev/null +++ b/mm2src/coins/lightning/ln_platform.rs @@ -0,0 +1,576 @@ +use super::*; +use crate::utxo::rpc_clients::{electrum_script_hash, BestBlock as RpcBestBlock, BlockHashOrHeight, + ElectrumBlockHeader, ElectrumClient, ElectrumNonce, EstimateFeeMethod, + UtxoRpcClientEnum, UtxoRpcError}; +use crate::utxo::utxo_standard::UtxoStandardCoin; +use crate::{MarketCoinOps, MmCoin}; +use bitcoin::blockdata::block::BlockHeader; +use bitcoin::blockdata::script::Script; +use bitcoin::blockdata::transaction::Transaction; +use bitcoin::consensus::encode::{deserialize, serialize_hex}; +use bitcoin::hash_types::{BlockHash, TxMerkleNode, Txid}; +use bitcoin_hashes::{sha256d, Hash}; +use common::executor::{spawn, Timer}; +use common::jsonrpc_client::JsonRpcErrorType; +use common::log; +use derive_more::Display; +use futures::compat::Future01CompatExt; +use keys::hash::H256; +use lightning::chain::{chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator}, + Confirm, Filter, WatchedOutput}; +use rpc::v1::types::H256 as H256Json; +use std::cmp; +use std::convert::{TryFrom, TryInto}; + +const CHECK_FOR_NEW_BEST_BLOCK_INTERVAL: u64 = 60; +const MIN_ALLOWED_FEE_PER_1000_WEIGHT: u32 = 253; + +#[derive(Debug, Display)] +pub enum FindWatchedOutputSpendError { + #[display(fmt = "Can't convert transaction: {}", _0)] + TransactionConvertionErr(String), + #[display(fmt = "Can't deserialize block header: {}", _0)] + BlockHeaderDeserializeErr(String), +} + +async fn find_watched_output_spend_with_header( + electrum_client: &ElectrumClient, + output: &WatchedOutput, +) -> Result, FindWatchedOutputSpendError> { + // from_block parameter is not used in find_output_spend for electrum clients + let utxo_client: UtxoRpcClientEnum = electrum_client.clone().into(); + let output_spend = match utxo_client + .find_output_spend( + H256::from(output.outpoint.txid.as_hash().into_inner()), + output.script_pubkey.as_ref(), + output.outpoint.index.into(), + BlockHashOrHeight::Hash(Default::default()), + ) + .compat() + .await + { + Ok(Some(output)) => output, + _ => return Ok(None), + }; + + if let BlockHashOrHeight::Height(height) = output_spend.spent_in_block { + if let Ok(header) = electrum_client.blockchain_block_header(height as u64).compat().await { + match deserialize(&header) { + Ok(h) => { + let spending_tx = match Transaction::try_from(output_spend.spending_tx) { + Ok(tx) => tx, + Err(e) => return Err(FindWatchedOutputSpendError::TransactionConvertionErr(e.to_string())), + }; + return Ok(Some((h, output_spend.input_index, spending_tx, height as u64))); + }, + Err(e) => return Err(FindWatchedOutputSpendError::BlockHeaderDeserializeErr(e.to_string())), + } + } + } + Ok(None) +} + +pub async fn get_best_header(best_header_listener: &ElectrumClient) -> EnableLightningResult { + best_header_listener + .blockchain_headers_subscribe() + .compat() + .await + .map_to_mm(|e| EnableLightningError::RpcError(e.to_string())) +} + +pub async fn update_best_block( + chain_monitor: Arc, + channel_manager: Arc, + best_header: ElectrumBlockHeader, +) { + { + let (new_best_header, new_best_height) = match best_header { + ElectrumBlockHeader::V12(h) => { + let nonce = match h.nonce { + ElectrumNonce::Number(n) => n as u32, + ElectrumNonce::Hash(_) => { + return; + }, + }; + let prev_blockhash = match sha256d::Hash::from_slice(&h.prev_block_hash.0) { + Ok(h) => h, + Err(e) => { + log::error!("Error while parsing previous block hash for lightning node: {}", e); + return; + }, + }; + let merkle_root = match sha256d::Hash::from_slice(&h.merkle_root.0) { + Ok(h) => h, + Err(e) => { + log::error!("Error while parsing merkle root for lightning node: {}", e); + return; + }, + }; + ( + BlockHeader { + version: h.version as i32, + prev_blockhash: BlockHash::from_hash(prev_blockhash), + merkle_root: TxMerkleNode::from_hash(merkle_root), + time: h.timestamp as u32, + bits: h.bits as u32, + nonce, + }, + h.block_height as u32, + ) + }, + ElectrumBlockHeader::V14(h) => { + let block_header = match deserialize(&h.hex.into_vec()) { + Ok(header) => header, + Err(e) => { + log::error!("Block header deserialization error: {}", e.to_string()); + return; + }, + }; + (block_header, h.height as u32) + }, + }; + channel_manager.best_block_updated(&new_best_header, new_best_height); + chain_monitor.best_block_updated(&new_best_header, new_best_height); + } +} + +pub async fn ln_best_block_update_loop( + platform: Arc, + chain_monitor: Arc, + channel_manager: Arc, + best_header_listener: ElectrumClient, + best_block: RpcBestBlock, +) { + let mut current_best_block = best_block; + loop { + let best_header = match get_best_header(&best_header_listener).await { + Ok(h) => h, + Err(e) => { + log::error!("Error while requesting best header for lightning node: {}", e); + Timer::sleep(CHECK_FOR_NEW_BEST_BLOCK_INTERVAL as f64).await; + continue; + }, + }; + if current_best_block != best_header.clone().into() { + platform + .process_txs_unconfirmations(chain_monitor.clone(), channel_manager.clone()) + .await; + platform + .process_txs_confirmations( + best_header_listener.clone(), + chain_monitor.clone(), + channel_manager.clone(), + best_header.block_height(), + ) + .await; + current_best_block = best_header.clone().into(); + update_best_block(chain_monitor.clone(), channel_manager.clone(), best_header).await; + } + Timer::sleep(CHECK_FOR_NEW_BEST_BLOCK_INTERVAL as f64).await; + } +} + +struct ConfirmedTransactionInfo { + txid: Txid, + header: BlockHeader, + index: usize, + transaction: Transaction, + height: u32, +} + +impl ConfirmedTransactionInfo { + fn new(txid: Txid, header: BlockHeader, index: usize, transaction: Transaction, height: u32) -> Self { + ConfirmedTransactionInfo { + txid, + header, + index, + transaction, + height, + } + } +} + +pub struct Platform { + pub coin: UtxoStandardCoin, + /// Main/testnet/signet/regtest Needed for lightning node to know which network to connect to + pub network: BlockchainNetwork, + // Default fees to and confirmation targets to be used for FeeEstimator. Default fees are used when the call for + // estimate_fee_sat fails. + pub default_fees_and_confirmations: PlatformCoinConfirmations, + // This cache stores the transactions that the LN node has interest in. + pub registered_txs: PaMutex>>, + // This cache stores the outputs that the LN node has interest in. + pub registered_outputs: PaMutex>, + // This cache stores transactions to be broadcasted once the other node accepts the channel + pub unsigned_funding_txs: PaMutex>, +} + +impl Platform { + pub fn new( + coin: UtxoStandardCoin, + network: BlockchainNetwork, + default_fees_and_confirmations: PlatformCoinConfirmations, + ) -> Self { + Platform { + coin, + network, + default_fees_and_confirmations, + registered_txs: PaMutex::new(HashMap::new()), + registered_outputs: PaMutex::new(Vec::new()), + unsigned_funding_txs: PaMutex::new(HashMap::new()), + } + } + + pub fn add_tx(&self, txid: &Txid, script_pubkey: &Script) { + let mut registered_txs = self.registered_txs.lock(); + match registered_txs.get_mut(txid) { + Some(h) => { + h.insert(script_pubkey.clone()); + }, + None => { + let mut script_pubkeys = HashSet::new(); + script_pubkeys.insert(script_pubkey.clone()); + registered_txs.insert(*txid, script_pubkeys); + }, + } + } + + pub fn add_output(&self, output: WatchedOutput) { + let mut registered_outputs = self.registered_outputs.lock(); + registered_outputs.push(output); + } + + async fn process_tx_for_unconfirmation(&self, txid: Txid, monitor: Arc) + where + T: Confirm, + { + if let Err(err) = self + .coin + .as_ref() + .rpc_client + .get_transaction_bytes(&H256Json::from(txid.as_hash().into_inner()).reversed()) + .compat() + .await + .map_err(|e| e.into_inner()) + { + if let UtxoRpcError::ResponseParseError(ref json_err) = err { + if let JsonRpcErrorType::Response(_, json) = &json_err.error { + if let Some(message) = json["message"].as_str() { + if message.contains("'code': -5") { + log::info!( + "Transaction {} is not found on chain :{}. The transaction will be re-broadcasted.", + txid, + err + ); + monitor.transaction_unconfirmed(&txid); + } + } + } + } + log::error!( + "Error while trying to check if the transaction {} is discarded or not :{}", + txid, + err + ); + } + } + + pub async fn process_txs_unconfirmations( + &self, + chain_monitor: Arc, + channel_manager: Arc, + ) { + // Retrieve channel manager transaction IDs to check the chain for un-confirmations + let channel_manager_relevant_txids = channel_manager.get_relevant_txids(); + for txid in channel_manager_relevant_txids { + self.process_tx_for_unconfirmation(txid, channel_manager.clone()).await; + } + + // Retrieve chain monitor transaction IDs to check the chain for un-confirmations + let chain_monitor_relevant_txids = chain_monitor.get_relevant_txids(); + for txid in chain_monitor_relevant_txids { + self.process_tx_for_unconfirmation(txid, chain_monitor.clone()).await; + } + } + + async fn get_confirmed_registered_txs( + &self, + client: &ElectrumClient, + current_height: u64, + ) -> Vec { + let registered_txs = self.registered_txs.lock().clone(); + let mut confirmed_registered_txs = Vec::new(); + for (txid, scripts) in registered_txs { + let rpc_txid = H256Json::from(txid.as_hash().into_inner()).reversed(); + match self + .coin + .as_ref() + .rpc_client + .get_transaction_bytes(&rpc_txid) + .compat() + .await + { + Ok(bytes) => { + let transaction: Transaction = match deserialize(&bytes.into_vec()) { + Ok(tx) => tx, + Err(e) => { + log::error!("Transaction deserialization error: {}", e.to_string()); + continue; + }, + }; + for (_, vout) in transaction.output.iter().enumerate() { + if scripts.contains(&vout.script_pubkey) { + let script_hash = hex::encode(electrum_script_hash(vout.script_pubkey.as_ref())); + let history = client + .scripthash_get_history(&script_hash) + .compat() + .await + .unwrap_or_default(); + for item in history { + if item.tx_hash == rpc_txid { + // If a new block mined the transaction while running process_txs_confirmations it will be confirmed later in ln_best_block_update_loop + if item.height > 0 && item.height <= current_height as i64 { + let height: u64 = match item.height.try_into() { + Ok(h) => h, + Err(e) => { + log::error!("Block height convertion to u64 error: {}", e.to_string()); + continue; + }, + }; + let header = match client.blockchain_block_header(height).compat().await { + Ok(block_header) => match deserialize(&block_header) { + Ok(h) => h, + Err(e) => { + log::error!( + "Block header deserialization error: {}", + e.to_string() + ); + continue; + }, + }, + Err(_) => continue, + }; + let index = match client + .blockchain_transaction_get_merkle(rpc_txid, height) + .compat() + .await + { + Ok(merkle_branch) => merkle_branch.pos, + Err(e) => { + log::error!( + "Error getting transaction position in the block: {}", + e.to_string() + ); + continue; + }, + }; + let confirmed_transaction_info = ConfirmedTransactionInfo::new( + txid, + header, + index, + transaction.clone(), + height as u32, + ); + confirmed_registered_txs.push(confirmed_transaction_info); + self.registered_txs.lock().remove(&txid); + } + } + } + } + } + }, + Err(e) => { + log::error!("Error getting transaction {} from chain: {}", txid, e); + continue; + }, + }; + } + confirmed_registered_txs + } + + async fn append_spent_registered_output_txs( + &self, + transactions_to_confirm: &mut Vec, + client: &ElectrumClient, + ) { + let mut outputs_to_remove = Vec::new(); + let registered_outputs = self.registered_outputs.lock().clone(); + for output in registered_outputs { + let result = match find_watched_output_spend_with_header(client, &output).await { + Ok(res) => res, + Err(e) => { + log::error!( + "Error while trying to find if the registered output {:?} is spent: {}", + output.outpoint, + e + ); + continue; + }, + }; + if let Some((header, _, tx, height)) = result { + if !transactions_to_confirm.iter().any(|info| info.txid == tx.txid()) { + let rpc_txid = H256Json::from(tx.txid().as_hash().into_inner()).reversed(); + let index = match client + .blockchain_transaction_get_merkle(rpc_txid, height) + .compat() + .await + { + Ok(merkle_branch) => merkle_branch.pos, + Err(e) => { + log::error!("Error getting transaction position in the block: {}", e.to_string()); + continue; + }, + }; + let confirmed_transaction_info = + ConfirmedTransactionInfo::new(tx.txid(), header, index, tx, height as u32); + transactions_to_confirm.push(confirmed_transaction_info); + } + outputs_to_remove.push(output); + } + } + self.registered_outputs + .lock() + .retain(|output| !outputs_to_remove.contains(output)); + } + + pub async fn process_txs_confirmations( + &self, + client: ElectrumClient, + chain_monitor: Arc, + channel_manager: Arc, + current_height: u64, + ) { + let mut transactions_to_confirm = self.get_confirmed_registered_txs(&client, current_height).await; + self.append_spent_registered_output_txs(&mut transactions_to_confirm, &client) + .await; + + transactions_to_confirm.sort_by(|a, b| { + let block_order = a.height.cmp(&b.height); + match block_order { + cmp::Ordering::Equal => a.index.cmp(&b.index), + _ => block_order, + } + }); + + for confirmed_transaction_info in transactions_to_confirm { + channel_manager.transactions_confirmed( + &confirmed_transaction_info.header, + &[( + confirmed_transaction_info.index, + &confirmed_transaction_info.transaction, + )], + confirmed_transaction_info.height, + ); + chain_monitor.transactions_confirmed( + &confirmed_transaction_info.header, + &[( + confirmed_transaction_info.index, + &confirmed_transaction_info.transaction, + )], + confirmed_transaction_info.height, + ); + } + } +} + +impl FeeEstimator for Platform { + // Gets estimated satoshis of fee required per 1000 Weight-Units. + fn get_est_sat_per_1000_weight(&self, confirmation_target: ConfirmationTarget) -> u32 { + let platform_coin = &self.coin; + + let default_fee = match confirmation_target { + ConfirmationTarget::Background => self.default_fees_and_confirmations.background.default_feerate, + ConfirmationTarget::Normal => self.default_fees_and_confirmations.normal.default_feerate, + ConfirmationTarget::HighPriority => self.default_fees_and_confirmations.high_priority.default_feerate, + } * 4; + + let conf = &platform_coin.as_ref().conf; + let n_blocks = match confirmation_target { + ConfirmationTarget::Background => self.default_fees_and_confirmations.background.n_blocks, + ConfirmationTarget::Normal => self.default_fees_and_confirmations.normal.n_blocks, + ConfirmationTarget::HighPriority => self.default_fees_and_confirmations.high_priority.n_blocks, + }; + let fee_per_kb = tokio::task::block_in_place(move || { + platform_coin + .as_ref() + .rpc_client + .estimate_fee_sat( + platform_coin.decimals(), + // Todo: when implementing Native client detect_fee_method should be used for Native and + // EstimateFeeMethod::Standard for Electrum + &EstimateFeeMethod::Standard, + &conf.estimate_fee_mode, + n_blocks, + ) + .wait() + .unwrap_or(default_fee) + }); + // Must be no smaller than 253 (ie 1 satoshi-per-byte rounded up to ensure later round-downs don’t put us below 1 satoshi-per-byte). + // https://docs.rs/lightning/0.0.101/lightning/chain/chaininterface/trait.FeeEstimator.html#tymethod.get_est_sat_per_1000_weight + cmp::max((fee_per_kb as f64 / 4.0).ceil() as u32, MIN_ALLOWED_FEE_PER_1000_WEIGHT) + } +} + +impl BroadcasterInterface for Platform { + fn broadcast_transaction(&self, tx: &Transaction) { + let tx_hex = serialize_hex(tx); + log::debug!("Trying to broadcast transaction: {}", tx_hex); + let tx_id = tx.txid(); + let fut = self.coin.send_raw_tx(&tx_hex); + spawn(async move { + match fut.compat().await { + Ok(id) => log::info!("Transaction broadcasted successfully: {:?} ", id), + Err(e) => log::error!("Broadcast transaction {} failed: {}", tx_id, e), + } + }); + } +} + +impl Filter for Platform { + // Watches for this transaction on-chain + fn register_tx(&self, txid: &Txid, script_pubkey: &Script) { self.add_tx(txid, script_pubkey); } + + // Watches for any transactions that spend this output on-chain + fn register_output(&self, output: WatchedOutput) -> Option<(usize, Transaction)> { + self.add_output(output.clone()); + + let block_hash = match output.block_hash { + Some(h) => H256Json::from(h.as_hash().into_inner()), + None => return None, + }; + + let client = &self.coin.as_ref().rpc_client; + // Although this works for both native and electrum clients as the block hash is available, + // the filter interface which includes register_output and register_tx should be used for electrum clients only, + // this is the reason for initializing the filter as an option in the start_lightning function as it will be None + // when implementing lightning for native clients + let output_spend_fut = tokio::task::block_in_place(move || { + client + .find_output_spend( + H256::from(output.outpoint.txid.as_hash().into_inner()), + output.script_pubkey.as_ref(), + output.outpoint.index.into(), + BlockHashOrHeight::Hash(block_hash), + ) + .wait() + }); + + match output_spend_fut { + Ok(Some(spent_output_info)) => { + let spending_tx = match Transaction::try_from(spent_output_info.spending_tx) { + Ok(tx) => tx, + Err(e) => { + log::error!("Can't convert transaction error: {}", e.to_string()); + return None; + }, + }; + Some((spent_output_info.input_index, spending_tx)) + }, + Ok(None) => None, + Err(e) => { + log::error!("Error when calling register_output: {}", e); + None + }, + } + } +} diff --git a/mm2src/coins/lightning/ln_rpc.rs b/mm2src/coins/lightning/ln_rpc.rs deleted file mode 100644 index 7fa6b22f61..0000000000 --- a/mm2src/coins/lightning/ln_rpc.rs +++ /dev/null @@ -1,169 +0,0 @@ -use super::*; -use crate::utxo::rpc_clients::{BlockHashOrHeight, ElectrumClient, EstimateFeeMethod, UtxoRpcClientEnum}; -use crate::utxo::utxo_standard::UtxoStandardCoin; -use crate::{MarketCoinOps, MmCoin}; -use bitcoin::blockdata::block::BlockHeader; -use bitcoin::blockdata::script::Script; -use bitcoin::blockdata::transaction::Transaction; -use bitcoin::consensus::encode; -use bitcoin::hash_types::Txid; -use bitcoin_hashes::Hash; -use common::executor::spawn; -use common::log; -use derive_more::Display; -use futures::compat::Future01CompatExt; -use keys::hash::H256; -use lightning::chain::{chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator}, - Filter, WatchedOutput}; -use rpc::v1::types::H256 as H256Json; -use std::cmp; -use std::convert::TryFrom; - -const MIN_ALLOWED_FEE_PER_1000_WEIGHT: u32 = 253; - -impl FeeEstimator for PlatformFields { - // Gets estimated satoshis of fee required per 1000 Weight-Units. - fn get_est_sat_per_1000_weight(&self, confirmation_target: ConfirmationTarget) -> u32 { - let platform_coin = &self.platform_coin; - - let default_fee = match confirmation_target { - ConfirmationTarget::Background => self.default_fees_and_confirmations.background.default_feerate, - ConfirmationTarget::Normal => self.default_fees_and_confirmations.normal.default_feerate, - ConfirmationTarget::HighPriority => self.default_fees_and_confirmations.high_priority.default_feerate, - } * 4; - - let conf = &platform_coin.as_ref().conf; - let n_blocks = match confirmation_target { - ConfirmationTarget::Background => self.default_fees_and_confirmations.background.n_blocks, - ConfirmationTarget::Normal => self.default_fees_and_confirmations.normal.n_blocks, - ConfirmationTarget::HighPriority => self.default_fees_and_confirmations.high_priority.n_blocks, - }; - let fee_per_kb = tokio::task::block_in_place(move || { - platform_coin - .as_ref() - .rpc_client - .estimate_fee_sat( - platform_coin.decimals(), - // Todo: when implementing Native client detect_fee_method should be used for Native and - // EstimateFeeMethod::Standard for Electrum - &EstimateFeeMethod::Standard, - &conf.estimate_fee_mode, - n_blocks, - ) - .wait() - .unwrap_or(default_fee) - }); - // Must be no smaller than 253 (ie 1 satoshi-per-byte rounded up to ensure later round-downs don’t put us below 1 satoshi-per-byte). - // https://docs.rs/lightning/0.0.101/lightning/chain/chaininterface/trait.FeeEstimator.html#tymethod.get_est_sat_per_1000_weight - cmp::max((fee_per_kb as f64 / 4.0).ceil() as u32, MIN_ALLOWED_FEE_PER_1000_WEIGHT) - } -} - -impl BroadcasterInterface for UtxoStandardCoin { - fn broadcast_transaction(&self, tx: &Transaction) { - let tx_hex = encode::serialize_hex(tx); - log::debug!("Trying to broadcast transaction: {}", tx_hex); - let tx_id = tx.txid(); - let fut = self.send_raw_tx(&tx_hex); - spawn(async move { - match fut.compat().await { - Ok(id) => log::info!("Transaction broadcasted successfully: {:?} ", id), - Err(e) => log::error!("Broadcast transaction {} failed: {}", tx_id, e), - } - }); - } -} - -#[derive(Debug, Display)] -pub enum FindWatchedOutputSpendError { - #[display(fmt = "Can't convert transaction: {}", _0)] - TransactionConvertionErr(String), - #[display(fmt = "Can't deserialize block header: {}", _0)] - BlockHeaderDeserializeErr(String), -} - -pub async fn find_watched_output_spend_with_header( - electrum_client: &ElectrumClient, - output: &WatchedOutput, -) -> Result, FindWatchedOutputSpendError> { - // from_block parameter is not used in find_output_spend for electrum clients - let utxo_client: UtxoRpcClientEnum = electrum_client.clone().into(); - let output_spend = match utxo_client - .find_output_spend( - H256::from(output.outpoint.txid.as_hash().into_inner()), - output.script_pubkey.as_ref(), - output.outpoint.index.into(), - BlockHashOrHeight::Hash(Default::default()), - ) - .compat() - .await - { - Ok(Some(output)) => output, - _ => return Ok(None), - }; - - if let BlockHashOrHeight::Height(height) = output_spend.spent_in_block { - if let Ok(header) = electrum_client.blockchain_block_header(height as u64).compat().await { - match encode::deserialize(&header) { - Ok(h) => { - let spending_tx = match Transaction::try_from(output_spend.spending_tx) { - Ok(tx) => tx, - Err(e) => return Err(FindWatchedOutputSpendError::TransactionConvertionErr(e.to_string())), - }; - return Ok(Some((h, output_spend.input_index, spending_tx, height as u64))); - }, - Err(e) => return Err(FindWatchedOutputSpendError::BlockHeaderDeserializeErr(e.to_string())), - } - } - } - Ok(None) -} - -impl Filter for PlatformFields { - // Watches for this transaction on-chain - fn register_tx(&self, txid: &Txid, script_pubkey: &Script) { self.add_tx(txid, script_pubkey); } - - // Watches for any transactions that spend this output on-chain - fn register_output(&self, output: WatchedOutput) -> Option<(usize, Transaction)> { - self.add_output(output.clone()); - - let block_hash = match output.block_hash { - Some(h) => H256Json::from(h.as_hash().into_inner()), - None => return None, - }; - - let client = &self.platform_coin.as_ref().rpc_client; - // Although this works for both native and electrum clients as the block hash is available, - // the filter interface which includes register_output and register_tx should be used for electrum clients only, - // this is the reason for initializing the filter as an option in the start_lightning function as it will be None - // when implementing lightning for native clients - let output_spend_fut = tokio::task::block_in_place(move || { - client - .find_output_spend( - H256::from(output.outpoint.txid.as_hash().into_inner()), - output.script_pubkey.as_ref(), - output.outpoint.index.into(), - BlockHashOrHeight::Hash(block_hash), - ) - .wait() - }); - - match output_spend_fut { - Ok(Some(spent_output_info)) => { - let spending_tx = match Transaction::try_from(spent_output_info.spending_tx) { - Ok(tx) => tx, - Err(e) => { - log::error!("Can't convert transaction error: {}", e.to_string()); - return None; - }, - }; - Some((spent_output_info.input_index, spending_tx)) - }, - Ok(None) => None, - Err(e) => { - log::error!("Error when calling register_output: {}", e); - None - }, - } - } -} diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index ea0e16e490..c2ef2d70bb 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -1,23 +1,18 @@ use super::*; use crate::lightning::ln_conf::{LightningCoinConf, LightningProtocolConf}; use crate::lightning::ln_p2p::{connect_to_nodes_loop, init_peer_manager, ln_node_announcement_loop}; -use crate::utxo::rpc_clients::{electrum_script_hash, BestBlock as RpcBestBlock, ElectrumBlockHeader, ElectrumClient, - ElectrumNonce, UtxoRpcError}; +use crate::lightning::ln_platform::{get_best_header, ln_best_block_update_loop, update_best_block}; +use crate::utxo::rpc_clients::BestBlock as RpcBestBlock; use crate::utxo::utxo_standard::UtxoStandardCoin; use crate::DerivationMethod; -use bitcoin::blockdata::block::BlockHeader; -use bitcoin::blockdata::transaction::Transaction; -use bitcoin::consensus::encode::deserialize; -use bitcoin::hash_types::{BlockHash, TxMerkleNode, Txid}; +use bitcoin::hash_types::BlockHash; use bitcoin_hashes::{sha256d, Hash}; use common::executor::{spawn, Timer}; -use common::jsonrpc_client::JsonRpcErrorType; use common::log; use common::log::LogState; use common::mm_ctx::MmArc; -use futures::compat::Future01CompatExt; use lightning::chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager}; -use lightning::chain::{chainmonitor, Access, BestBlock, Confirm, Watch}; +use lightning::chain::{chainmonitor, Access, BestBlock, Watch}; use lightning::ln::channelmanager; use lightning::ln::channelmanager::{ChainParameters, ChannelManagerReadArgs, SimpleArcChannelManager}; use lightning::routing::network_graph::{NetGraphMsgHandler, NetworkGraph}; @@ -29,29 +24,25 @@ use lightning_invoice::utils::DefaultRouter; use lightning_persister::storage::{FileSystemStorage, NodesAddressesMap}; use lightning_persister::LightningPersister; use parking_lot::Mutex as PaMutex; -use rpc::v1::types::H256; -use std::cmp::Ordering; use std::collections::HashMap; -use std::convert::TryInto; use std::fs::File; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::SystemTime; -const CHECK_FOR_NEW_BEST_BLOCK_INTERVAL: u64 = 60; const NETWORK_GRAPH_PERSIST_INTERVAL: u64 = 600; const SCORER_PERSIST_INTERVAL: u64 = 600; pub type ChainMonitor = chainmonitor::ChainMonitor< InMemorySigner, - Arc, - Arc, - Arc, + Arc, + Arc, + Arc, Arc, Arc, >; -pub type ChannelManager = SimpleArcChannelManager; +pub type ChannelManager = SimpleArcChannelManager; type Router = DefaultRouter, Arc>; @@ -118,22 +109,22 @@ fn init_keys_manager(ctx: &MmArc) -> EnableLightningResult> { } async fn init_channel_manager( - platform_fields: Arc, + platform: Arc, logger: Arc, persister: Arc, keys_manager: Arc, conf: LightningCoinConf, ) -> EnableLightningResult<(Arc, Arc)> { // Initialize the FeeEstimator. UtxoStandardCoin implements the FeeEstimator trait, so it'll act as our fee estimator. - let fee_estimator = platform_fields.clone(); + let fee_estimator = platform.clone(); // Initialize the BroadcasterInterface. UtxoStandardCoin implements the BroadcasterInterface trait, so it'll act as our transaction // broadcaster. - let broadcaster = Arc::new(platform_fields.platform_coin.clone()); + let broadcaster = platform.clone(); // Initialize the ChainMonitor let chain_monitor: Arc = Arc::new(chainmonitor::ChainMonitor::new( - Some(platform_fields.clone()), + Some(platform.clone()), broadcaster.clone(), logger.clone(), fee_estimator.clone(), @@ -147,10 +138,10 @@ async fn init_channel_manager( // This is used for Electrum only to prepare for chain synchronization for (_, chan_mon) in channelmonitors.iter() { - chan_mon.load_outputs_to_watch(&platform_fields); + chan_mon.load_outputs_to_watch(&platform); } - let rpc_client = match &platform_fields.platform_coin.as_ref().rpc_client { + let rpc_client = match &platform.coin.as_ref().rpc_client { UtxoRpcClientEnum::Electrum(c) => c.clone(), UtxoRpcClientEnum::Native(_) => { return MmError::err(EnableLightningError::UnsupportedMode( @@ -186,7 +177,7 @@ async fn init_channel_manager( } else { // Initialize the ChannelManager to starting a new node without history let chain_params = ChainParameters { - network: platform_fields.network.clone().into(), + network: platform.network.clone().into(), best_block: BestBlock::new(best_block_hash, best_block.height as u32), }; let new_channel_manager = channelmanager::ChannelManager::new( @@ -206,16 +197,18 @@ async fn init_channel_manager( // Sync ChannelMonitors and ChannelManager to chain tip if the node is restarting and has open channels if channel_manager_blockhash != best_block_hash { - process_txs_unconfirmations(platform_fields.clone(), chain_monitor.clone(), channel_manager.clone()).await; - process_txs_confirmations( - // It's safe to use unwrap here for now until implementing Native Client for Lightning - platform_fields.clone(), - rpc_client.clone(), - chain_monitor.clone(), - channel_manager.clone(), - best_header.block_height(), - ) - .await; + platform + .process_txs_unconfirmations(chain_monitor.clone(), channel_manager.clone()) + .await; + platform + .process_txs_confirmations( + // It's safe to use unwrap here for now until implementing Native Client for Lightning + rpc_client.clone(), + chain_monitor.clone(), + channel_manager.clone(), + best_header.block_height(), + ) + .await; update_best_block(chain_monitor.clone(), channel_manager.clone(), best_header).await; } @@ -230,7 +223,7 @@ async fn init_channel_manager( // Update best block whenever there's a new chain tip or a block has been newly disconnected spawn(ln_best_block_update_loop( // It's safe to use unwrap here for now until implementing Native Client for Lightning - platform_fields, + platform, chain_monitor.clone(), channel_manager.clone(), rpc_client.clone(), @@ -294,7 +287,7 @@ pub async fn start_lightning( )); } - let platform_fields = Arc::new(PlatformFields::new( + let platform = Arc::new(Platform::new( platform_coin.clone(), protocol_conf.network.clone(), protocol_conf.confirmations, @@ -320,7 +313,7 @@ pub async fn start_lightning( // Initialize the ChannelManager let (chain_monitor, channel_manager) = init_channel_manager( - platform_fields.clone(), + platform.clone(), logger.clone(), persister.clone(), keys_manager.clone(), @@ -345,7 +338,7 @@ pub async fn start_lightning( // Initialize the event handler let event_handler = Arc::new(ln_events::LightningEventHandler::new( // It's safe to use unwrap here for now until implementing Native Client for Lightning - platform_fields.clone(), + platform.clone(), channel_manager.clone(), keys_manager.clone(), inbound_payments.clone(), @@ -401,7 +394,7 @@ pub async fn start_lightning( )); Ok(LightningCoin { - platform_fields, + platform, conf, peer_manager, background_processor, @@ -415,350 +408,3 @@ pub async fn start_lightning( open_channels_nodes, }) } - -struct ConfirmedTransactionInfo { - txid: Txid, - header: BlockHeader, - index: usize, - transaction: Transaction, - height: u32, -} - -impl ConfirmedTransactionInfo { - fn new(txid: Txid, header: BlockHeader, index: usize, transaction: Transaction, height: u32) -> Self { - ConfirmedTransactionInfo { - txid, - header, - index, - transaction, - height, - } - } -} - -async fn process_tx_for_unconfirmation(txid: Txid, filter: Arc, monitor: Arc) -where - T: Confirm, -{ - if let Err(err) = filter - .platform_coin - .as_ref() - .rpc_client - .get_transaction_bytes(&H256::from(txid.as_hash().into_inner()).reversed()) - .compat() - .await - .map_err(|e| e.into_inner()) - { - if let UtxoRpcError::ResponseParseError(ref json_err) = err { - if let JsonRpcErrorType::Response(_, json) = &json_err.error { - if let Some(message) = json["message"].as_str() { - if message.contains("'code': -5") { - log::info!( - "Transaction {} is not found on chain :{}. The transaction will be re-broadcasted.", - txid, - err - ); - monitor.transaction_unconfirmed(&txid); - } - } - } - } - log::error!( - "Error while trying to check if the transaction {} is discarded or not :{}", - txid, - err - ); - } -} - -async fn process_txs_unconfirmations( - filter: Arc, - chain_monitor: Arc, - channel_manager: Arc, -) { - // Retrieve channel manager transaction IDs to check the chain for un-confirmations - let channel_manager_relevant_txids = channel_manager.get_relevant_txids(); - for txid in channel_manager_relevant_txids { - process_tx_for_unconfirmation(txid, filter.clone(), channel_manager.clone()).await; - } - - // Retrieve chain monitor transaction IDs to check the chain for un-confirmations - let chain_monitor_relevant_txids = chain_monitor.get_relevant_txids(); - for txid in chain_monitor_relevant_txids { - process_tx_for_unconfirmation(txid, filter.clone(), chain_monitor.clone()).await; - } -} - -async fn get_confirmed_registered_txs( - filter: Arc, - client: &ElectrumClient, - current_height: u64, -) -> Vec { - let registered_txs = filter.registered_txs.lock().clone(); - let mut confirmed_registered_txs = Vec::new(); - for (txid, scripts) in registered_txs { - let rpc_txid = H256::from(txid.as_hash().into_inner()).reversed(); - match filter - .platform_coin - .as_ref() - .rpc_client - .get_transaction_bytes(&rpc_txid) - .compat() - .await - { - Ok(bytes) => { - let transaction: Transaction = match deserialize(&bytes.into_vec()) { - Ok(tx) => tx, - Err(e) => { - log::error!("Transaction deserialization error: {}", e.to_string()); - continue; - }, - }; - for (_, vout) in transaction.output.iter().enumerate() { - if scripts.contains(&vout.script_pubkey) { - let script_hash = hex::encode(electrum_script_hash(vout.script_pubkey.as_ref())); - let history = client - .scripthash_get_history(&script_hash) - .compat() - .await - .unwrap_or_default(); - for item in history { - if item.tx_hash == rpc_txid { - // If a new block mined the transaction while running process_txs_confirmations it will be confirmed later in ln_best_block_update_loop - if item.height > 0 && item.height <= current_height as i64 { - let height: u64 = match item.height.try_into() { - Ok(h) => h, - Err(e) => { - log::error!("Block height convertion to u64 error: {}", e.to_string()); - continue; - }, - }; - let header = match client.blockchain_block_header(height).compat().await { - Ok(block_header) => match deserialize(&block_header) { - Ok(h) => h, - Err(e) => { - log::error!("Block header deserialization error: {}", e.to_string()); - continue; - }, - }, - Err(_) => continue, - }; - let index = match client - .blockchain_transaction_get_merkle(rpc_txid, height) - .compat() - .await - { - Ok(merkle_branch) => merkle_branch.pos, - Err(e) => { - log::error!( - "Error getting transaction position in the block: {}", - e.to_string() - ); - continue; - }, - }; - let confirmed_transaction_info = ConfirmedTransactionInfo::new( - txid, - header, - index, - transaction.clone(), - height as u32, - ); - confirmed_registered_txs.push(confirmed_transaction_info); - filter.registered_txs.lock().remove(&txid); - } - } - } - } - } - }, - Err(e) => { - log::error!("Error getting transaction {} from chain: {}", txid, e); - continue; - }, - }; - } - confirmed_registered_txs -} - -async fn append_spent_registered_output_txs( - transactions_to_confirm: &mut Vec, - filter: Arc, - client: &ElectrumClient, -) { - let mut outputs_to_remove = Vec::new(); - let registered_outputs = filter.registered_outputs.lock().clone(); - for output in registered_outputs { - let result = match ln_rpc::find_watched_output_spend_with_header(client, &output).await { - Ok(res) => res, - Err(e) => { - log::error!( - "Error while trying to find if the registered output {:?} is spent: {}", - output.outpoint, - e - ); - continue; - }, - }; - if let Some((header, _, tx, height)) = result { - if !transactions_to_confirm.iter().any(|info| info.txid == tx.txid()) { - let rpc_txid = H256::from(tx.txid().as_hash().into_inner()).reversed(); - let index = match client - .blockchain_transaction_get_merkle(rpc_txid, height) - .compat() - .await - { - Ok(merkle_branch) => merkle_branch.pos, - Err(e) => { - log::error!("Error getting transaction position in the block: {}", e.to_string()); - continue; - }, - }; - let confirmed_transaction_info = - ConfirmedTransactionInfo::new(tx.txid(), header, index, tx, height as u32); - transactions_to_confirm.push(confirmed_transaction_info); - } - outputs_to_remove.push(output); - } - } - filter - .registered_outputs - .lock() - .retain(|output| !outputs_to_remove.contains(output)); -} - -async fn process_txs_confirmations( - filter: Arc, - client: ElectrumClient, - chain_monitor: Arc, - channel_manager: Arc, - current_height: u64, -) { - let mut transactions_to_confirm = get_confirmed_registered_txs(filter.clone(), &client, current_height).await; - append_spent_registered_output_txs(&mut transactions_to_confirm, filter.clone(), &client).await; - - transactions_to_confirm.sort_by(|a, b| { - let block_order = a.height.cmp(&b.height); - match block_order { - Ordering::Equal => a.index.cmp(&b.index), - _ => block_order, - } - }); - - for confirmed_transaction_info in transactions_to_confirm { - channel_manager.transactions_confirmed( - &confirmed_transaction_info.header, - &[( - confirmed_transaction_info.index, - &confirmed_transaction_info.transaction, - )], - confirmed_transaction_info.height, - ); - chain_monitor.transactions_confirmed( - &confirmed_transaction_info.header, - &[( - confirmed_transaction_info.index, - &confirmed_transaction_info.transaction, - )], - confirmed_transaction_info.height, - ); - } -} - -async fn get_best_header(best_header_listener: &ElectrumClient) -> EnableLightningResult { - best_header_listener - .blockchain_headers_subscribe() - .compat() - .await - .map_to_mm(|e| EnableLightningError::RpcError(e.to_string())) -} - -async fn update_best_block( - chain_monitor: Arc, - channel_manager: Arc, - best_header: ElectrumBlockHeader, -) { - { - let (new_best_header, new_best_height) = match best_header { - ElectrumBlockHeader::V12(h) => { - let nonce = match h.nonce { - ElectrumNonce::Number(n) => n as u32, - ElectrumNonce::Hash(_) => { - return; - }, - }; - let prev_blockhash = match sha256d::Hash::from_slice(&h.prev_block_hash.0) { - Ok(h) => h, - Err(e) => { - log::error!("Error while parsing previous block hash for lightning node: {}", e); - return; - }, - }; - let merkle_root = match sha256d::Hash::from_slice(&h.merkle_root.0) { - Ok(h) => h, - Err(e) => { - log::error!("Error while parsing merkle root for lightning node: {}", e); - return; - }, - }; - ( - BlockHeader { - version: h.version as i32, - prev_blockhash: BlockHash::from_hash(prev_blockhash), - merkle_root: TxMerkleNode::from_hash(merkle_root), - time: h.timestamp as u32, - bits: h.bits as u32, - nonce, - }, - h.block_height as u32, - ) - }, - ElectrumBlockHeader::V14(h) => { - let block_header = match deserialize(&h.hex.into_vec()) { - Ok(header) => header, - Err(e) => { - log::error!("Block header deserialization error: {}", e.to_string()); - return; - }, - }; - (block_header, h.height as u32) - }, - }; - channel_manager.best_block_updated(&new_best_header, new_best_height); - chain_monitor.best_block_updated(&new_best_header, new_best_height); - } -} - -async fn ln_best_block_update_loop( - filter: Arc, - chain_monitor: Arc, - channel_manager: Arc, - best_header_listener: ElectrumClient, - best_block: RpcBestBlock, -) { - let mut current_best_block = best_block; - loop { - let best_header = match get_best_header(&best_header_listener).await { - Ok(h) => h, - Err(e) => { - log::error!("Error while requesting best header for lightning node: {}", e); - Timer::sleep(CHECK_FOR_NEW_BEST_BLOCK_INTERVAL as f64).await; - continue; - }, - }; - if current_best_block != best_header.clone().into() { - process_txs_unconfirmations(filter.clone(), chain_monitor.clone(), channel_manager.clone()).await; - process_txs_confirmations( - filter.clone(), - best_header_listener.clone(), - chain_monitor.clone(), - channel_manager.clone(), - best_header.block_height(), - ) - .await; - current_best_block = best_header.clone().into(); - update_best_block(chain_monitor.clone(), channel_manager.clone(), best_header).await; - } - Timer::sleep(CHECK_FOR_NEW_BEST_BLOCK_INTERVAL as f64).await; - } -} From 1ff8f8660204b0552e248dc3fd2db3a50ef67d0d Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Wed, 9 Mar 2022 20:18:19 +0200 Subject: [PATCH 08/49] Refactor completed --- mm2src/coins/lightning.rs | 228 +++++++++++++++--- mm2src/coins/lightning/ln_utils.rs | 192 +-------------- .../src/lightning_activation.rs | 3 +- 3 files changed, 211 insertions(+), 212 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index f45fdb3342..80b0a47809 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -1,4 +1,12 @@ -use super::{lp_coinfind_or_err, MmCoinEnum}; +pub mod ln_conf; +pub mod ln_errors; +mod ln_events; +mod ln_p2p; +mod ln_platform; +mod ln_serialization; +mod ln_utils; + +use super::{lp_coinfind_or_err, DerivationMethod, MmCoinEnum}; use crate::utxo::rpc_clients::UtxoRpcClientEnum; use crate::utxo::utxo_common::{big_decimal_from_sat_unsigned, UtxoTxBuilder}; use crate::utxo::{sat_from_big_decimal, BlockchainNetwork, FeePolicy, UtxoCommonOps, UtxoTxGenerationOps}; @@ -11,8 +19,9 @@ use bigdecimal::BigDecimal; use bitcoin::hashes::Hash; use bitcoin_hashes::sha256::Hash as Sha256; use chain::TransactionOutput; +use common::executor::spawn; use common::ip_addr::myipaddr; -use common::log::LogOnError; +use common::log::{LogOnError, LogState}; use common::mm_ctx::MmArc; use common::mm_error::prelude::*; use common::mm_number::MmNumber; @@ -23,15 +32,19 @@ use keys::{AddressHashEnum, KeyPair}; use lightning::chain::channelmonitor::Balance; use lightning::chain::keysinterface::KeysInterface; use lightning::chain::keysinterface::KeysManager; +use lightning::chain::Access; use lightning::ln::channelmanager::{ChannelDetails, MIN_FINAL_CLTV_EXPIRY}; use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; +use lightning::routing::network_graph::{NetGraphMsgHandler, NetworkGraph}; +use lightning::routing::scoring::Scorer; use lightning::util::config::UserConfig; use lightning_background_processor::BackgroundProcessor; -use lightning_invoice::utils::create_invoice_from_channelmanager; +use lightning_invoice::payment; +use lightning_invoice::utils::{create_invoice_from_channelmanager, DefaultRouter}; use lightning_invoice::Invoice; use lightning_persister::storage::{FileSystemStorage, NodesAddressesMapShared}; use lightning_persister::LightningPersister; -use ln_conf::{ChannelOptions, LightningCoinConf, PlatformCoinConfirmations}; +use ln_conf::{ChannelOptions, LightningCoinConf, LightningProtocolConf, PlatformCoinConfirmations}; use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelError, CloseChannelResult, ConnectToNodeError, ConnectToNodeResult, EnableLightningError, EnableLightningResult, GenerateInvoiceError, GenerateInvoiceResult, GetChannelDetailsError, GetChannelDetailsResult, @@ -42,7 +55,7 @@ use ln_events::LightningEventHandler; use ln_p2p::{connect_to_node, ConnectToNodeRes, PeerManager}; use ln_platform::Platform; use ln_serialization::{InvoiceForRPC, NodeAddress, PublicKeyForRPC}; -use ln_utils::{ChainMonitor, ChannelManager, InvoicePayer}; +use ln_utils::{ChainMonitor, ChannelManager}; use parking_lot::Mutex as PaMutex; use rpc::v1::types::{Bytes as BytesJson, H256 as H256Json}; use script::{Builder, TransactionInputSigner}; @@ -54,36 +67,13 @@ use std::collections::{HashMap, HashSet}; use std::fmt; use std::net::SocketAddr; use std::str::FromStr; -use std::sync::Arc; - -pub mod ln_conf; -pub mod ln_errors; -mod ln_events; -mod ln_p2p; -mod ln_platform; -mod ln_serialization; -pub mod ln_utils; +use std::sync::{Arc, Mutex}; +type Router = DefaultRouter, Arc>; +type InvoicePayer = payment::InvoicePayer, Router, Arc>, Arc, E>; type PaymentsMap = HashMap; type PaymentsMapShared = Arc>; -#[derive(Clone, Serialize)] -#[serde(rename_all = "lowercase")] -pub enum HTLCStatus { - Pending, - Succeeded, - Failed, -} - -#[derive(Clone)] -pub struct PaymentInfo { - pub preimage: Option, - pub secret: Option, - pub status: HTLCStatus, - pub amt_msat: Option, - pub fee_paid_msat: Option, -} - #[derive(Clone)] pub struct LightningCoin { pub platform: Arc, @@ -116,6 +106,23 @@ impl fmt::Debug for LightningCoin { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "LightningCoin {{ conf: {:?} }}", self.conf) } } +#[derive(Clone, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum HTLCStatus { + Pending, + Succeeded, + Failed, +} + +#[derive(Clone)] +pub struct PaymentInfo { + pub preimage: Option, + pub secret: Option, + pub status: HTLCStatus, + pub amt_msat: Option, + pub fee_paid_msat: Option, +} + impl LightningCoin { fn platform_coin(&self) -> &UtxoStandardCoin { &self.platform.coin } @@ -479,6 +486,165 @@ impl MmCoin for LightningCoin { fn is_coin_protocol_supported(&self, _info: &Option>) -> bool { unimplemented!() } } +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct LightningParams { + // The listening port for the p2p LN node + pub listening_port: u16, + // Printable human-readable string to describe this node to other users. + pub node_name: [u8; 32], + // Node's RGB color. This is used for showing the node in a network graph with the desired color. + pub node_color: [u8; 3], + // Invoice Payer is initialized while starting the lightning node, and it requires the number of payment retries that + // it should do before considering a payment failed or partially failed. If not provided the number of retries will be 5 + // as this is a good default value. + pub payment_retries: Option, + // Node's backup path for channels and other data that requires backup. + pub backup_path: Option, +} + +pub async fn start_lightning( + ctx: &MmArc, + platform_coin: UtxoStandardCoin, + protocol_conf: LightningProtocolConf, + conf: LightningCoinConf, + params: LightningParams, +) -> EnableLightningResult { + // Todo: add support for Hardware wallets for funding transactions and spending spendable outputs (channel closing transactions) + if let DerivationMethod::HDWallet(_) = platform_coin.as_ref().derivation_method { + return MmError::err(EnableLightningError::UnsupportedMode( + "'start_lightning'".into(), + "iguana".into(), + )); + } + + let platform = Arc::new(Platform::new( + platform_coin.clone(), + protocol_conf.network.clone(), + protocol_conf.confirmations, + )); + + // Initialize the Logger + let logger = ctx.log.0.clone(); + + // Initialize Persister + let persister = ln_utils::init_persister(ctx, conf.ticker.clone(), params.backup_path).await?; + + // Initialize the KeysManager + let keys_manager = ln_utils::init_keys_manager(ctx)?; + + // Initialize the NetGraphMsgHandler. This is used for providing routes to send payments over + let network_graph = Arc::new(persister.get_network_graph(protocol_conf.network.into()).await?); + spawn(ln_utils::persist_network_graph_loop( + persister.clone(), + network_graph.clone(), + )); + let network_gossip = Arc::new(NetGraphMsgHandler::new( + network_graph.clone(), + None::>, + logger.clone(), + )); + + // Initialize the ChannelManager + let (chain_monitor, channel_manager) = ln_utils::init_channel_manager( + platform.clone(), + logger.clone(), + persister.clone(), + keys_manager.clone(), + conf.clone().into(), + ) + .await?; + + // Initialize the PeerManager + let peer_manager = ln_p2p::init_peer_manager( + ctx.clone(), + params.listening_port, + channel_manager.clone(), + network_gossip.clone(), + keys_manager.get_node_secret(), + logger.clone(), + ) + .await?; + + let inbound_payments = Arc::new(PaMutex::new(HashMap::new())); + let outbound_payments = Arc::new(PaMutex::new(HashMap::new())); + + // Initialize the event handler + let event_handler = Arc::new(ln_events::LightningEventHandler::new( + // It's safe to use unwrap here for now until implementing Native Client for Lightning + platform.clone(), + channel_manager.clone(), + keys_manager.clone(), + inbound_payments.clone(), + outbound_payments.clone(), + )); + + // Initialize routing Scorer + let scorer = Arc::new(Mutex::new(persister.get_scorer().await?)); + spawn(ln_utils::persist_scorer_loop(persister.clone(), scorer.clone())); + + // Create InvoicePayer + let router = DefaultRouter::new(network_graph, logger.clone()); + let invoice_payer = Arc::new(InvoicePayer::new( + channel_manager.clone(), + router, + scorer, + logger.clone(), + event_handler, + payment::RetryAttempts(params.payment_retries.unwrap_or(5)), + )); + + // Persist ChannelManager + // Note: if the ChannelManager is not persisted properly to disk, there is risk of channels force closing the next time LN starts up + let channel_manager_persister = persister.clone(); + let persist_channel_manager_callback = + move |node: &ChannelManager| channel_manager_persister.persist_manager(&*node); + + // Start Background Processing. Runs tasks periodically in the background to keep LN node operational. + // InvoicePayer will act as our event handler as it handles some of the payments related events before + // delegating it to LightningEventHandler. + let background_processor = Arc::new(BackgroundProcessor::start( + persist_channel_manager_callback, + invoice_payer.clone(), + chain_monitor.clone(), + channel_manager.clone(), + Some(network_gossip), + peer_manager.clone(), + logger, + )); + + // If channel_nodes_data file exists, read channels nodes data from disk and reconnect to channel nodes/peers if possible. + let open_channels_nodes = Arc::new(PaMutex::new( + ln_utils::get_open_channels_nodes_addresses(persister.clone(), channel_manager.clone()).await?, + )); + spawn(ln_p2p::connect_to_nodes_loop( + open_channels_nodes.clone(), + peer_manager.clone(), + )); + + // Broadcast Node Announcement + spawn(ln_p2p::ln_node_announcement_loop( + channel_manager.clone(), + params.node_name, + params.node_color, + params.listening_port, + )); + + Ok(LightningCoin { + platform, + conf, + peer_manager, + background_processor, + channel_manager, + chain_monitor, + keys_manager, + invoice_payer, + persister, + inbound_payments, + outbound_payments, + open_channels_nodes, + }) +} + #[derive(Deserialize)] pub struct ConnectToNodeRequest { pub coin: String, diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index c2ef2d70bb..e5fbc242e7 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -1,30 +1,22 @@ use super::*; -use crate::lightning::ln_conf::{LightningCoinConf, LightningProtocolConf}; -use crate::lightning::ln_p2p::{connect_to_nodes_loop, init_peer_manager, ln_node_announcement_loop}; use crate::lightning::ln_platform::{get_best_header, ln_best_block_update_loop, update_best_block}; use crate::utxo::rpc_clients::BestBlock as RpcBestBlock; -use crate::utxo::utxo_standard::UtxoStandardCoin; -use crate::DerivationMethod; use bitcoin::hash_types::BlockHash; use bitcoin_hashes::{sha256d, Hash}; use common::executor::{spawn, Timer}; use common::log; use common::log::LogState; use common::mm_ctx::MmArc; -use lightning::chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager}; -use lightning::chain::{chainmonitor, Access, BestBlock, Watch}; +use lightning::chain::keysinterface::{InMemorySigner, KeysManager}; +use lightning::chain::{chainmonitor, BestBlock, Watch}; use lightning::ln::channelmanager; use lightning::ln::channelmanager::{ChainParameters, ChannelManagerReadArgs, SimpleArcChannelManager}; -use lightning::routing::network_graph::{NetGraphMsgHandler, NetworkGraph}; +use lightning::routing::network_graph::NetworkGraph; use lightning::routing::scoring::Scorer; +use lightning::util::config::UserConfig; use lightning::util::ser::ReadableArgs; -use lightning_background_processor::BackgroundProcessor; -use lightning_invoice::payment; -use lightning_invoice::utils::DefaultRouter; use lightning_persister::storage::{FileSystemStorage, NodesAddressesMap}; use lightning_persister::LightningPersister; -use parking_lot::Mutex as PaMutex; -use std::collections::HashMap; use std::fs::File; use std::path::PathBuf; use std::sync::{Arc, Mutex}; @@ -44,29 +36,9 @@ pub type ChainMonitor = chainmonitor::ChainMonitor< pub type ChannelManager = SimpleArcChannelManager; -type Router = DefaultRouter, Arc>; +fn ln_data_dir(ctx: &MmArc, ticker: &str) -> PathBuf { ctx.dbdir().join("LIGHTNING").join(ticker) } -pub type InvoicePayer = payment::InvoicePayer, Router, Arc>, Arc, E>; - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct LightningParams { - // The listening port for the p2p LN node - pub listening_port: u16, - // Printable human-readable string to describe this node to other users. - pub node_name: [u8; 32], - // Node's RGB color. This is used for showing the node in a network graph with the desired color. - pub node_color: [u8; 3], - // Invoice Payer is initialized while starting the lightning node, and it requires the number of payment retries that - // it should do before considering a payment failed or partially failed. If not provided the number of retries will be 5 - // as this is a good default value. - pub payment_retries: Option, - // Node's backup path for channels and other data that requires backup. - pub backup_path: Option, -} - -pub fn ln_data_dir(ctx: &MmArc, ticker: &str) -> PathBuf { ctx.dbdir().join("LIGHTNING").join(ticker) } - -pub fn ln_data_backup_dir(ctx: &MmArc, path: Option, ticker: &str) -> Option { +fn ln_data_backup_dir(ctx: &MmArc, path: Option, ticker: &str) -> Option { path.map(|p| { PathBuf::from(&p) .join(&hex::encode(&**ctx.rmd160())) @@ -75,7 +47,7 @@ pub fn ln_data_backup_dir(ctx: &MmArc, path: Option, ticker: &str) -> Op }) } -async fn init_persister( +pub async fn init_persister( ctx: &MmArc, ticker: String, backup_path: Option, @@ -98,7 +70,7 @@ async fn init_persister( Ok(persister) } -fn init_keys_manager(ctx: &MmArc) -> EnableLightningResult> { +pub fn init_keys_manager(ctx: &MmArc) -> EnableLightningResult> { // The current time is used to derive random numbers from the seed where required, to ensure all random generation is unique across restarts. let seed: [u8; 32] = ctx.secp256k1_key_pair().private().secret.into(); let cur = SystemTime::now() @@ -108,12 +80,12 @@ fn init_keys_manager(ctx: &MmArc) -> EnableLightningResult> { Ok(Arc::new(KeysManager::new(&seed, cur.as_secs(), cur.subsec_nanos()))) } -async fn init_channel_manager( +pub async fn init_channel_manager( platform: Arc, logger: Arc, persister: Arc, keys_manager: Arc, - conf: LightningCoinConf, + user_config: UserConfig, ) -> EnableLightningResult<(Arc, Arc)> { // Initialize the FeeEstimator. UtxoStandardCoin implements the FeeEstimator trait, so it'll act as our fee estimator. let fee_estimator = platform.clone(); @@ -156,7 +128,6 @@ async fn init_channel_manager( sha256d::Hash::from_slice(&best_block.hash.0).map_to_mm(|e| EnableLightningError::HashError(e.to_string()))?, ); let (channel_manager_blockhash, channel_manager) = { - let user_config = conf.into(); if let Ok(mut f) = File::open(persister.manager_path()) { let mut channel_monitor_mut_references = Vec::new(); for (_, channel_monitor) in channelmonitors.iter_mut() { @@ -233,7 +204,7 @@ async fn init_channel_manager( Ok((chain_monitor, channel_manager)) } -async fn persist_network_graph_loop(persister: Arc, network_graph: Arc) { +pub async fn persist_network_graph_loop(persister: Arc, network_graph: Arc) { loop { if let Err(e) = persister.save_network_graph(network_graph.clone()).await { log::warn!( @@ -245,7 +216,7 @@ async fn persist_network_graph_loop(persister: Arc, network_ } } -async fn persist_scorer_loop(persister: Arc, scorer: Arc>) { +pub async fn persist_scorer_loop(persister: Arc, scorer: Arc>) { loop { if let Err(e) = persister.save_scorer(scorer.clone()).await { log::warn!( @@ -257,7 +228,7 @@ async fn persist_scorer_loop(persister: Arc, scorer: Arc, channel_manager: Arc, ) -> EnableLightningResult { @@ -271,140 +242,3 @@ async fn get_open_channels_nodes_addresses( }); Ok(nodes_addresses) } - -pub async fn start_lightning( - ctx: &MmArc, - platform_coin: UtxoStandardCoin, - protocol_conf: LightningProtocolConf, - conf: LightningCoinConf, - params: LightningParams, -) -> EnableLightningResult { - // Todo: add support for Hardware wallets for funding transactions and spending spendable outputs (channel closing transactions) - if let DerivationMethod::HDWallet(_) = platform_coin.as_ref().derivation_method { - return MmError::err(EnableLightningError::UnsupportedMode( - "'start_lightning'".into(), - "iguana".into(), - )); - } - - let platform = Arc::new(Platform::new( - platform_coin.clone(), - protocol_conf.network.clone(), - protocol_conf.confirmations, - )); - - // Initialize the Logger - let logger = ctx.log.0.clone(); - - // Initialize Persister - let persister = init_persister(ctx, conf.ticker.clone(), params.backup_path).await?; - - // Initialize the KeysManager - let keys_manager = init_keys_manager(ctx)?; - - // Initialize the NetGraphMsgHandler. This is used for providing routes to send payments over - let network_graph = Arc::new(persister.get_network_graph(protocol_conf.network.into()).await?); - spawn(persist_network_graph_loop(persister.clone(), network_graph.clone())); - let network_gossip = Arc::new(NetGraphMsgHandler::new( - network_graph.clone(), - None::>, - logger.clone(), - )); - - // Initialize the ChannelManager - let (chain_monitor, channel_manager) = init_channel_manager( - platform.clone(), - logger.clone(), - persister.clone(), - keys_manager.clone(), - conf.clone(), - ) - .await?; - - // Initialize the PeerManager - let peer_manager = init_peer_manager( - ctx.clone(), - params.listening_port, - channel_manager.clone(), - network_gossip.clone(), - keys_manager.get_node_secret(), - logger.clone(), - ) - .await?; - - let inbound_payments = Arc::new(PaMutex::new(HashMap::new())); - let outbound_payments = Arc::new(PaMutex::new(HashMap::new())); - - // Initialize the event handler - let event_handler = Arc::new(ln_events::LightningEventHandler::new( - // It's safe to use unwrap here for now until implementing Native Client for Lightning - platform.clone(), - channel_manager.clone(), - keys_manager.clone(), - inbound_payments.clone(), - outbound_payments.clone(), - )); - - // Initialize routing Scorer - let scorer = Arc::new(Mutex::new(persister.get_scorer().await?)); - spawn(persist_scorer_loop(persister.clone(), scorer.clone())); - - // Create InvoicePayer - let router = DefaultRouter::new(network_graph, logger.clone()); - let invoice_payer = Arc::new(InvoicePayer::new( - channel_manager.clone(), - router, - scorer, - logger.clone(), - event_handler, - payment::RetryAttempts(params.payment_retries.unwrap_or(5)), - )); - - // Persist ChannelManager - // Note: if the ChannelManager is not persisted properly to disk, there is risk of channels force closing the next time LN starts up - let channel_manager_persister = persister.clone(); - let persist_channel_manager_callback = - move |node: &ChannelManager| channel_manager_persister.persist_manager(&*node); - - // Start Background Processing. Runs tasks periodically in the background to keep LN node operational. - // InvoicePayer will act as our event handler as it handles some of the payments related events before - // delegating it to LightningEventHandler. - let background_processor = Arc::new(BackgroundProcessor::start( - persist_channel_manager_callback, - invoice_payer.clone(), - chain_monitor.clone(), - channel_manager.clone(), - Some(network_gossip), - peer_manager.clone(), - logger, - )); - - // If channel_nodes_data file exists, read channels nodes data from disk and reconnect to channel nodes/peers if possible. - let open_channels_nodes = Arc::new(PaMutex::new( - get_open_channels_nodes_addresses(persister.clone(), channel_manager.clone()).await?, - )); - spawn(connect_to_nodes_loop(open_channels_nodes.clone(), peer_manager.clone())); - - // Broadcast Node Announcement - spawn(ln_node_announcement_loop( - channel_manager.clone(), - params.node_name, - params.node_color, - params.listening_port, - )); - - Ok(LightningCoin { - platform, - conf, - peer_manager, - background_processor, - channel_manager, - chain_monitor, - keys_manager, - invoice_payer, - persister, - inbound_payments, - outbound_payments, - open_channels_nodes, - }) -} diff --git a/mm2src/coins_activation/src/lightning_activation.rs b/mm2src/coins_activation/src/lightning_activation.rs index 100c7b93cf..3273b76377 100644 --- a/mm2src/coins_activation/src/lightning_activation.rs +++ b/mm2src/coins_activation/src/lightning_activation.rs @@ -3,8 +3,7 @@ use crate::prelude::*; use async_trait::async_trait; use coins::lightning::ln_conf::{LightningCoinConf, LightningProtocolConf}; use coins::lightning::ln_errors::EnableLightningError; -use coins::lightning::ln_utils::{start_lightning, LightningParams}; -use coins::lightning::LightningCoin; +use coins::lightning::{start_lightning, LightningCoin, LightningParams}; use coins::utxo::utxo_standard::UtxoStandardCoin; use coins::utxo::UtxoCommonOps; use coins::{BalanceError, CoinBalance, CoinProtocol, MarketCoinOps, MmCoinEnum}; From ccd53a65bf83d1586479e395646eb531addf7206 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Fri, 11 Mar 2022 20:35:22 +0200 Subject: [PATCH 09/49] WIP: use user_channel_id as channel id in rpc calls to avoid using temp id --- mm2src/coins/lightning.rs | 26 ++++--- mm2src/coins/lightning/ln_errors.rs | 16 +++- mm2src/coins/lightning/ln_events.rs | 28 ++++--- mm2src/coins/lightning/ln_platform.rs | 2 +- mm2src/coins/lightning/ln_utils.rs | 8 +- mm2src/coins/lightning_persister/src/lib.rs | 73 ++++++++++++++++++- .../coins/lightning_persister/src/storage.rs | 2 + 7 files changed, 130 insertions(+), 25 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 80b0a47809..d7fbcc215d 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -42,7 +42,7 @@ use lightning_background_processor::BackgroundProcessor; use lightning_invoice::payment; use lightning_invoice::utils::{create_invoice_from_channelmanager, DefaultRouter}; use lightning_invoice::Invoice; -use lightning_persister::storage::{FileSystemStorage, NodesAddressesMapShared}; +use lightning_persister::storage::{FileSystemStorage, NodesAddressesMapShared, SqlStorage}; use lightning_persister::LightningPersister; use ln_conf::{ChannelOptions, LightningCoinConf, LightningProtocolConf, PlatformCoinConfirmations}; use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelError, CloseChannelResult, @@ -185,6 +185,9 @@ impl LightningCoin { fee_paid_msat: None, })) } + + // Todo: implement GetHistoryCoinType instead + fn storage_ticker(&self) -> String { self.ticker().replace('-', "_") } } #[async_trait] @@ -702,7 +705,7 @@ pub struct OpenChannelRequest { #[derive(Serialize)] pub struct OpenChannelResponse { - temporary_channel_id: H256Json, + rpc_channel_id: u64, node_address: NodeAddress, } @@ -773,16 +776,19 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes user_config.own_channel_config.our_htlc_minimum_msat = min; } - let temp_channel_id = async_blocking(move || { + let storage_ticker = ln_coin.storage_ticker(); + let rpc_channel_id = ln_coin.persister.get_number_of_channels_in_sql(&storage_ticker).await? as u64; + + async_blocking(move || { channel_manager - .create_channel(node_pubkey, amount_in_sat, push_msat, 1, Some(user_config)) + .create_channel(node_pubkey, amount_in_sat, push_msat, rpc_channel_id, Some(user_config)) .map_to_mm(|e| OpenChannelError::FailureToOpenChannel(node_pubkey.to_string(), format!("{:?}", e))) }) .await?; { let mut unsigned_funding_txs = ln_coin.platform.unsigned_funding_txs.lock(); - unsigned_funding_txs.insert(temp_channel_id, unsigned); + unsigned_funding_txs.insert(rpc_channel_id, unsigned); } // Saving node data to reconnect to it on restart @@ -793,7 +799,7 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes .await?; Ok(OpenChannelResponse { - temporary_channel_id: temp_channel_id.into(), + rpc_channel_id, node_address: req.node_address, }) } @@ -806,6 +812,7 @@ pub struct ListChannelsRequest { #[derive(Serialize)] pub struct ChannelDetailsForRPC { pub channel_id: H256Json, + pub rpc_channel_id: u64, pub counterparty_node_id: PublicKeyForRPC, pub funding_tx: Option, pub funding_tx_output_index: Option, @@ -829,6 +836,7 @@ impl From for ChannelDetailsForRPC { fn from(details: ChannelDetails) -> ChannelDetailsForRPC { ChannelDetailsForRPC { channel_id: details.channel_id.into(), + rpc_channel_id: details.user_channel_id, counterparty_node_id: PublicKeyForRPC(details.counterparty.node_id), funding_tx: details .funding_txo @@ -870,7 +878,7 @@ pub async fn list_channels(ctx: MmArc, req: ListChannelsRequest) -> ListChannels #[derive(Deserialize)] pub struct GetChannelDetailsRequest { pub coin: String, - pub channel_id: H256Json, + pub rpc_channel_id: u64, } #[derive(Serialize)] @@ -891,8 +899,8 @@ pub async fn get_channel_details( .channel_manager .list_channels() .into_iter() - .find(|chan| chan.channel_id == req.channel_id.0) - .ok_or(GetChannelDetailsError::NoSuchChannel(req.channel_id))? + .find(|chan| chan.user_channel_id == req.rpc_channel_id) + .ok_or(GetChannelDetailsError::NoSuchChannel(req.rpc_channel_id))? .into(); Ok(GetChannelDetailsResponse { channel_details }) diff --git a/mm2src/coins/lightning/ln_errors.rs b/mm2src/coins/lightning/ln_errors.rs index e8b36cd962..17c62028e8 100644 --- a/mm2src/coins/lightning/ln_errors.rs +++ b/mm2src/coins/lightning/ln_errors.rs @@ -3,6 +3,7 @@ use crate::utxo::GenerateTxError; use crate::{BalanceError, CoinFindError, DerivationMethodNotSupported, NumConversError, PrivKeyNotAllowed}; use common::mm_error::prelude::*; use common::HttpStatusCode; +use db_common::sqlite::rusqlite::Error as SqlError; use derive_more::Display; use http::StatusCode; use lightning_invoice::SignOrCreationError; @@ -68,6 +69,10 @@ impl From for EnableLightningError { fn from(err: std::io::Error) -> EnableLightningError { EnableLightningError::IOError(err.to_string()) } } +impl From for EnableLightningError { + fn from(err: SqlError) -> EnableLightningError { EnableLightningError::SqlError(err.to_string()) } +} + #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] pub enum ConnectToNodeError { @@ -130,6 +135,8 @@ pub enum OpenChannelError { InternalError(String), #[display(fmt = "I/O error {}", _0)] IOError(String), + #[display(fmt = "SQL error {}", _0)] + SqlError(String), ConnectToNodeError(String), #[display(fmt = "No such coin {}", _0)] NoSuchCoin(String), @@ -151,6 +158,7 @@ impl HttpStatusCode for OpenChannelError { | OpenChannelError::InternalError(_) | OpenChannelError::GenerateTxErr(_) | OpenChannelError::IOError(_) + | OpenChannelError::SqlError(_) | OpenChannelError::InvalidPath(_) | OpenChannelError::ConvertTxErr(_) => StatusCode::INTERNAL_SERVER_ERROR, OpenChannelError::NoSuchCoin(_) | OpenChannelError::BalanceError(_) => StatusCode::PRECONDITION_REQUIRED, @@ -202,6 +210,10 @@ impl From for OpenChannelError { fn from(err: std::io::Error) -> OpenChannelError { OpenChannelError::IOError(err.to_string()) } } +impl From for OpenChannelError { + fn from(err: SqlError) -> OpenChannelError { OpenChannelError::SqlError(err.to_string()) } +} + #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] pub enum ListChannelsError { @@ -235,8 +247,8 @@ pub enum GetChannelDetailsError { UnsupportedCoin(String), #[display(fmt = "No such coin {}", _0)] NoSuchCoin(String), - #[display(fmt = "Channel with id: {:?} is not found", _0)] - NoSuchChannel(H256Json), + #[display(fmt = "Channel with rpc id: {} is not found", _0)] + NoSuchChannel(u64), } impl HttpStatusCode for GetChannelDetailsError { diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index ba5820d491..1e0ead482f 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -29,8 +29,9 @@ impl EventHandler for LightningEventHandler { Event::FundingGenerationReady { temporary_channel_id, output_script, + user_channel_id, .. - } => self.handle_funding_generation_ready(*temporary_channel_id, output_script), + } => self.handle_funding_generation_ready(*temporary_channel_id, output_script, *user_channel_id), Event::PaymentReceived { payment_hash, amt, @@ -90,7 +91,7 @@ impl EventHandler for LightningEventHandler { // Generates the raw funding transaction with one output equal to the channel value. fn sign_funding_transaction( - temp_channel_id: [u8; 32], + user_channel_id: u64, output_script: &Script, platform: Arc, ) -> OpenChannelResult { @@ -98,11 +99,11 @@ fn sign_funding_transaction( let mut unsigned = { let unsigned_funding_txs = platform.unsigned_funding_txs.lock(); unsigned_funding_txs - .get(&temp_channel_id) + .get(&user_channel_id) .ok_or_else(|| { OpenChannelError::InternalError(format!( - "Unsigned funding tx not found for temporary channel id: {}", - hex::encode(temp_channel_id) + "Unsigned funding tx not found for internal channel id: {}", + user_channel_id )) })? .clone() @@ -141,17 +142,22 @@ impl LightningEventHandler { } } - fn handle_funding_generation_ready(&self, temporary_channel_id: [u8; 32], output_script: &Script) { + fn handle_funding_generation_ready( + &self, + temporary_channel_id: [u8; 32], + output_script: &Script, + user_channel_id: u64, + ) { log::info!( - "Handling FundingGenerationReady event for temporary_channel_id: {}", - hex::encode(temporary_channel_id) + "Handling FundingGenerationReady event for internal channel id: {}", + user_channel_id ); - let funding_tx = match sign_funding_transaction(temporary_channel_id, output_script, self.platform.clone()) { + let funding_tx = match sign_funding_transaction(user_channel_id, output_script, self.platform.clone()) { Ok(tx) => tx, Err(e) => { log::error!( - "Error generating funding transaction for temporary channel id {:?}: {}", - temporary_channel_id, + "Error generating funding transaction for internal channel id {}: {}", + user_channel_id, e.to_string() ); return; diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index d35395245c..a65e614010 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -202,7 +202,7 @@ pub struct Platform { // This cache stores the outputs that the LN node has interest in. pub registered_outputs: PaMutex>, // This cache stores transactions to be broadcasted once the other node accepts the channel - pub unsigned_funding_txs: PaMutex>, + pub unsigned_funding_txs: PaMutex>, } impl Platform { diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index e5fbc242e7..5f3fbc6461 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -15,7 +15,7 @@ use lightning::routing::network_graph::NetworkGraph; use lightning::routing::scoring::Scorer; use lightning::util::config::UserConfig; use lightning::util::ser::ReadableArgs; -use lightning_persister::storage::{FileSystemStorage, NodesAddressesMap}; +use lightning_persister::storage::{FileSystemStorage, NodesAddressesMap, SqlStorage}; use lightning_persister::LightningPersister; use std::fs::File; use std::path::PathBuf; @@ -67,6 +67,12 @@ pub async fn init_persister( if !is_initialized { persister.init_fs().await?; } + // Todo: use get_history_coin_type and storage_ticker functions - should probably use platform orderbook_ticker + // this way we will have channels history and payments history for BTC instead of BTC-segwit for example + let is_sql_initialized = persister.is_sql_initialized(&ticker.replace('-', "_")).await?; + if !is_sql_initialized { + persister.init_sql(&ticker.replace('-', "_")).await?; + } Ok(persister) } diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 03e7eeeaba..7d48016bca 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -117,6 +117,15 @@ fn insert_channel_in_history_sql(for_coin: &str) -> Result { Ok(sql) } +fn get_num_channel_rows_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "SELECT COUNT(*) FROM ".to_owned() + &table_name + ";"; + + Ok(sql) +} + impl LightningPersister { /// Initialize a new LightningPersister and set the path to the individual channels' /// files. @@ -527,16 +536,29 @@ impl SqlStorage for LightningPersister { }) .await } + + async fn get_number_of_channels_in_sql(&self, for_coin: &str) -> Result { + let sql = get_num_channel_rows_sql(for_coin)?; + let sqlite_connection = self.sqlite_connection.clone(); + + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + let count: u32 = conn.query_row(&sql, NO_PARAMS, |r| r.get(0))?; + Ok(count) + }) + .await + } } #[cfg(test)] mod tests { + use super::*; extern crate bitcoin; extern crate lightning; - use crate::LightningPersister; use bitcoin::blockdata::block::{Block, BlockHeader}; use bitcoin::hashes::hex::FromHex; use bitcoin::Txid; + use common::block_on; use db_common::sqlite::rusqlite::Connection; use lightning::chain::chainmonitor::Persist; use lightning::chain::transaction::OutPoint; @@ -749,4 +771,53 @@ mod tests { nodes[1].node.get_and_clear_pending_msg_events(); added_monitors.clear(); } + + #[test] + fn test_init_sql_collection() { + let for_coin = "init_sql_collection"; + let persister = LightningPersister::new( + PathBuf::from("test_filesystem_persister"), + None, + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + let initialized = block_on(persister.is_sql_initialized(for_coin)).unwrap(); + assert!(!initialized); + + block_on(persister.init_sql(for_coin)).unwrap(); + // repetitive init must not fail + block_on(persister.init_sql(for_coin)).unwrap(); + + let initialized = block_on(persister.is_sql_initialized(for_coin)).unwrap(); + assert!(initialized); + } + + #[test] + fn test_add_channel_to_sql() { + let chanmon_cfgs = create_chanmon_cfgs(2); + let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); + let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); + let nodes = create_network(2, &node_cfgs, &node_chanmgrs); + create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()); + + let for_coin = "add_channel"; + let persister = LightningPersister::new( + PathBuf::from("test_filesystem_persister"), + None, + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + + block_on(persister.init_sql(for_coin)).unwrap(); + let number_of_rows = block_on(persister.get_number_of_channels_in_sql(for_coin)).unwrap(); + assert_eq!(number_of_rows, 0); + + let node_1_channel_details = nodes[0].node.list_channels()[0].clone(); + block_on(persister.add_channel_to_history(for_coin, node_1_channel_details)).unwrap(); + let number_of_rows = block_on(persister.get_number_of_channels_in_sql(for_coin)).unwrap(); + assert_eq!(number_of_rows, 1); + + let node_2_channel_details = nodes[1].node.list_channels()[0].clone(); + block_on(persister.add_channel_to_history(for_coin, node_2_channel_details)).unwrap(); + let number_of_rows = block_on(persister.get_number_of_channels_in_sql(for_coin)).unwrap(); + assert_eq!(number_of_rows, 2); + } } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 8ac89295cb..abdadbc6ad 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -44,4 +44,6 @@ pub trait SqlStorage { async fn is_sql_initialized(&self, for_coin: &str) -> Result; async fn add_channel_to_history(&self, for_coin: &str, channel_details: ChannelDetails) -> Result<(), Self::Error>; + + async fn get_number_of_channels_in_sql(&self, for_coin: &str) -> Result; } From 628921e1cd205321b7d3309a97a01029ca25b19b Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Tue, 15 Mar 2022 00:38:55 +0200 Subject: [PATCH 10/49] Channels in sql WIP: add pending outbound channel to sql --- mm2src/coins/lightning.rs | 24 ++- mm2src/coins/lightning_persister/src/lib.rs | 138 ++++++++++-------- .../coins/lightning_persister/src/storage.rs | 38 ++++- 3 files changed, 131 insertions(+), 69 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index d7fbcc215d..22bfcc77c5 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -42,7 +42,7 @@ use lightning_background_processor::BackgroundProcessor; use lightning_invoice::payment; use lightning_invoice::utils::{create_invoice_from_channelmanager, DefaultRouter}; use lightning_invoice::Invoice; -use lightning_persister::storage::{FileSystemStorage, NodesAddressesMapShared, SqlStorage}; +use lightning_persister::storage::{FileSystemStorage, NodesAddressesMapShared, PendingChannelForSql, SqlStorage}; use lightning_persister::LightningPersister; use ln_conf::{ChannelOptions, LightningCoinConf, LightningProtocolConf, PlatformCoinConfirmations}; use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelError, CloseChannelResult, @@ -777,9 +777,9 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes } let storage_ticker = ln_coin.storage_ticker(); - let rpc_channel_id = ln_coin.persister.get_number_of_channels_in_sql(&storage_ticker).await? as u64; + let rpc_channel_id = ln_coin.persister.get_last_channel_rpc_id(&storage_ticker).await? as u64 + 1; - async_blocking(move || { + let temp_channel_id = async_blocking(move || { channel_manager .create_channel(node_pubkey, amount_in_sat, push_msat, rpc_channel_id, Some(user_config)) .map_to_mm(|e| OpenChannelError::FailureToOpenChannel(node_pubkey.to_string(), format!("{:?}", e))) @@ -791,6 +791,15 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes unsigned_funding_txs.insert(rpc_channel_id, unsigned); } + let pending_channel_details = PendingChannelForSql::new( + rpc_channel_id, + temp_channel_id, + node_pubkey, + node_addr, + true, + user_config.channel_options.announced_channel, + ); + // Saving node data to reconnect to it on restart ln_coin.open_channels_nodes.lock().insert(node_pubkey, node_addr); ln_coin @@ -798,6 +807,11 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes .save_nodes_addresses(ln_coin.open_channels_nodes) .await?; + ln_coin + .persister + .add_pending_channel_to_sql(&storage_ticker, &pending_channel_details) + .await?; + Ok(OpenChannelResponse { rpc_channel_id, node_address: req.node_address, @@ -811,8 +825,8 @@ pub struct ListChannelsRequest { #[derive(Serialize)] pub struct ChannelDetailsForRPC { - pub channel_id: H256Json, pub rpc_channel_id: u64, + pub channel_id: H256Json, pub counterparty_node_id: PublicKeyForRPC, pub funding_tx: Option, pub funding_tx_output_index: Option, @@ -835,8 +849,8 @@ pub struct ChannelDetailsForRPC { impl From for ChannelDetailsForRPC { fn from(details: ChannelDetails) -> ChannelDetailsForRPC { ChannelDetailsForRPC { - channel_id: details.channel_id.into(), rpc_channel_id: details.user_channel_id, + channel_id: details.channel_id.into(), counterparty_node_id: PublicKeyForRPC(details.counterparty.node_id), funding_tx: details .funding_txo diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 7d48016bca..1b79806517 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -13,7 +13,7 @@ extern crate lightning; extern crate secp256k1; extern crate serde_json; -use crate::storage::{FileSystemStorage, NodesAddressesMap, NodesAddressesMapShared, SqlStorage}; +use crate::storage::{FileSystemStorage, NodesAddressesMap, NodesAddressesMapShared, PendingChannelForSql, SqlStorage}; use crate::util::DiskWriteable; use async_trait::async_trait; use bitcoin::blockdata::constants::genesis_block; @@ -30,7 +30,7 @@ use lightning::chain::chainmonitor; use lightning::chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate}; use lightning::chain::keysinterface::{KeysInterface, Sign}; use lightning::chain::transaction::OutPoint; -use lightning::ln::channelmanager::{ChannelDetails, ChannelManager}; +use lightning::ln::channelmanager::ChannelManager; use lightning::routing::network_graph::NetworkGraph; use lightning::routing::scoring::Scorer; use lightning::util::logger::Logger; @@ -91,37 +91,42 @@ fn create_channels_history_table_sql(for_coin: &str) -> Result + &table_name + " ( id INTEGER NOT NULL PRIMARY KEY, + rpc_id INTEGER NOT NULL UNIQUE, channel_id VARCHAR(255) NOT NULL, counterparty_node_id VARCHAR(255) NOT NULL, + counterparty_node_address VARCHAR(255) NOT NULL, funding_tx VARCHAR(255), funding_tx_output_index INTEGER, funding_tx_value_sats INTEGER NOT NULL, closing_tx VARCHAR(255), closing_tx_output_index INTEGER, - closing_balance_sats INTEGER, + closing_tx_value_sats INTEGER, is_outbound INTEGER NOT NULL, + is_public INTEGER NOT NULL, + is_pending INTEGER NOT NULL, + is_open INTEGER NOT NULL, is_closed INTEGER NOT NULL );"; Ok(sql) } -fn insert_channel_in_history_sql(for_coin: &str) -> Result { +fn insert_channel_in_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; let sql = "INSERT INTO ".to_owned() + &table_name - + " (channel_id, counterparty_node_id, funding_tx, funding_tx_output_index, funding_tx_value_sats, closing_tx, closing_tx_output_index, closing_balance_sats, is_outbound, is_closed) VALUES (?1, ?2, ?3, ?4, ?5, ?6,?7, ?8, ?9, ?10);"; + + " (rpc_id, channel_id, counterparty_node_id, counterparty_node_address, funding_tx, funding_tx_output_index, funding_tx_value_sats, closing_tx, closing_tx_output_index, closing_tx_value_sats, is_outbound, is_public, is_pending, is_open, is_closed) VALUES (?1, ?2, ?3, ?4, ?5, ?6,?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15);"; Ok(sql) } -fn get_num_channel_rows_sql(for_coin: &str) -> Result { +fn get_last_channel_rpc_id_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "SELECT COUNT(*) FROM ".to_owned() + &table_name + ";"; + let sql = "SELECT MAX(rpc_id) FROM ".to_owned() + &table_name + ";"; Ok(sql) } @@ -493,52 +498,59 @@ impl SqlStorage for LightningPersister { .await } - async fn add_channel_to_history(&self, for_coin: &str, channel_details: ChannelDetails) -> Result<(), Self::Error> { + async fn add_pending_channel_to_sql( + &self, + for_coin: &str, + details: &PendingChannelForSql, + ) -> Result<(), Self::Error> { let for_coin = for_coin.to_owned(); - let sqlite_connection = self.sqlite_connection.clone(); + let rpc_id = details.rpc_id.to_string(); + let channel_id = hex::encode(details.channel_id); + let counterparty_node_id = details.counterparty_node_id.to_string(); + let counterparty_node_address = details.counterparty_node_address.to_string(); + let funding_tx = String::default(); + let funding_tx_output_index = String::default(); + let funding_tx_value_sats = String::default(); + let closing_tx = String::default(); + let closing_tx_output_index = String::default(); + let closing_tx_value_sats = String::default(); + let is_outbound = (details.is_outbound as i32).to_string(); + let is_public = (details.is_public as i32).to_string(); + let is_pending = 1_u8.to_string(); + let is_open = 0_u8.to_string(); + let is_closed = 0_u8.to_string(); + + let params = [ + rpc_id, + channel_id, + counterparty_node_id, + counterparty_node_address, + funding_tx, + funding_tx_output_index, + funding_tx_value_sats, + closing_tx, + closing_tx_output_index, + closing_tx_value_sats, + is_outbound, + is_public, + is_pending, + is_open, + is_closed, + ]; + let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { let mut conn = sqlite_connection.lock().unwrap(); let sql_transaction = conn.transaction()?; - - let channel_id = hex::encode(channel_details.channel_id); - let counterparty_node_id = channel_details.counterparty.node_id.to_string(); - let funding_tx = channel_details - .funding_txo - .map(|outpoint| outpoint.txid.to_string()) - .unwrap_or_default(); - let funding_tx_output_index = channel_details - .funding_txo - .map(|outpoint| outpoint.index.to_string()) - .unwrap_or_default(); - let funding_tx_value_sats = channel_details.channel_value_satoshis.to_string(); - let closing_tx = String::default(); - let closing_tx_output_index = String::default(); - let closing_balance_sats = String::default(); - let is_outbound = (channel_details.is_outbound as i32).to_string(); - let is_closed = 0_u8.to_string(); - - let params = [ - channel_id, - counterparty_node_id, - funding_tx, - funding_tx_output_index, - funding_tx_value_sats, - closing_tx, - closing_tx_output_index, - closing_balance_sats, - is_outbound, - is_closed, - ]; - sql_transaction.execute(&insert_channel_in_history_sql(&for_coin)?, ¶ms)?; + sql_transaction.execute(&insert_channel_in_sql(&for_coin)?, ¶ms)?; sql_transaction.commit()?; Ok(()) }) .await } - async fn get_number_of_channels_in_sql(&self, for_coin: &str) -> Result { - let sql = get_num_channel_rows_sql(for_coin)?; + async fn get_last_channel_rpc_id(&self, for_coin: &str) -> Result { + let sql = get_last_channel_rpc_id_sql(for_coin)?; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { @@ -792,13 +804,7 @@ mod tests { } #[test] - fn test_add_channel_to_sql() { - let chanmon_cfgs = create_chanmon_cfgs(2); - let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); - let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); - let nodes = create_network(2, &node_cfgs, &node_chanmgrs); - create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()); - + fn test_add_pending_channel_to_sql() { let for_coin = "add_channel"; let persister = LightningPersister::new( PathBuf::from("test_filesystem_persister"), @@ -807,17 +813,27 @@ mod tests { ); block_on(persister.init_sql(for_coin)).unwrap(); - let number_of_rows = block_on(persister.get_number_of_channels_in_sql(for_coin)).unwrap(); - assert_eq!(number_of_rows, 0); - - let node_1_channel_details = nodes[0].node.list_channels()[0].clone(); - block_on(persister.add_channel_to_history(for_coin, node_1_channel_details)).unwrap(); - let number_of_rows = block_on(persister.get_number_of_channels_in_sql(for_coin)).unwrap(); - assert_eq!(number_of_rows, 1); - - let node_2_channel_details = nodes[1].node.list_channels()[0].clone(); - block_on(persister.add_channel_to_history(for_coin, node_2_channel_details)).unwrap(); - let number_of_rows = block_on(persister.get_number_of_channels_in_sql(for_coin)).unwrap(); - assert_eq!(number_of_rows, 2); + + let pending_chan_1_details = PendingChannelForSql::new( + 1, + [0; 32], + PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), + "127.0.0.1:9735".parse().unwrap(), + true, + true, + ); + block_on(persister.add_pending_channel_to_sql(for_coin, &pending_chan_1_details)).unwrap(); + let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id(for_coin)).unwrap(); + assert_eq!(last_channel_rpc_id, 1); + + // must fail because we are adding channel with the same rpc_id + block_on(persister.add_pending_channel_to_sql(for_coin, &pending_chan_1_details)).unwrap_err(); + assert_eq!(last_channel_rpc_id, 1); + + let mut pending_chan_2_details = pending_chan_1_details; + pending_chan_2_details.rpc_id = 2; + block_on(persister.add_pending_channel_to_sql(for_coin, &pending_chan_2_details)).unwrap(); + let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id(for_coin)).unwrap(); + assert_eq!(last_channel_rpc_id, 2); } } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index abdadbc6ad..d5461ef261 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -1,6 +1,5 @@ use async_trait::async_trait; use bitcoin::Network; -use lightning::ln::channelmanager::ChannelDetails; use lightning::routing::network_graph::NetworkGraph; use lightning::routing::scoring::Scorer; use parking_lot::Mutex as PaMutex; @@ -34,6 +33,35 @@ pub trait FileSystemStorage { async fn save_scorer(&self, scorer: Arc>) -> Result<(), Self::Error>; } +pub struct PendingChannelForSql { + pub rpc_id: u64, + pub channel_id: [u8; 32], + pub counterparty_node_id: PublicKey, + pub counterparty_node_address: SocketAddr, + pub is_outbound: bool, + pub is_public: bool, +} + +impl PendingChannelForSql { + pub fn new( + rpc_id: u64, + channel_id: [u8; 32], + counterparty_node_id: PublicKey, + counterparty_node_address: SocketAddr, + is_outbound: bool, + is_public: bool, + ) -> Self { + PendingChannelForSql { + rpc_id, + channel_id, + counterparty_node_id, + counterparty_node_address, + is_outbound, + is_public, + } + } +} + #[async_trait] pub trait SqlStorage { type Error; @@ -43,7 +71,11 @@ pub trait SqlStorage { async fn is_sql_initialized(&self, for_coin: &str) -> Result; - async fn add_channel_to_history(&self, for_coin: &str, channel_details: ChannelDetails) -> Result<(), Self::Error>; + async fn add_pending_channel_to_sql( + &self, + for_coin: &str, + details: &PendingChannelForSql, + ) -> Result<(), Self::Error>; - async fn get_number_of_channels_in_sql(&self, for_coin: &str) -> Result; + async fn get_last_channel_rpc_id(&self, for_coin: &str) -> Result; } From 35667fa2d6301f3f7712d495263237506c226696 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Wed, 16 Mar 2022 03:24:28 +0200 Subject: [PATCH 11/49] Channels sql WIP: get channel and add funding_tx --- mm2src/coins/lightning.rs | 8 +- mm2src/coins/lightning_persister/src/lib.rs | 162 ++++++++++++++---- .../coins/lightning_persister/src/storage.rs | 53 ++++-- 3 files changed, 168 insertions(+), 55 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 22bfcc77c5..27e6f23f28 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -42,7 +42,7 @@ use lightning_background_processor::BackgroundProcessor; use lightning_invoice::payment; use lightning_invoice::utils::{create_invoice_from_channelmanager, DefaultRouter}; use lightning_invoice::Invoice; -use lightning_persister::storage::{FileSystemStorage, NodesAddressesMapShared, PendingChannelForSql, SqlStorage}; +use lightning_persister::storage::{FileSystemStorage, NodesAddressesMapShared, SqlChannelDetails, SqlStorage}; use lightning_persister::LightningPersister; use ln_conf::{ChannelOptions, LightningCoinConf, LightningProtocolConf, PlatformCoinConfirmations}; use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelError, CloseChannelResult, @@ -791,11 +791,11 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes unsigned_funding_txs.insert(rpc_channel_id, unsigned); } - let pending_channel_details = PendingChannelForSql::new( - rpc_channel_id, + let pending_channel_details = SqlChannelDetails::new( temp_channel_id, node_pubkey, node_addr, + amount_in_sat, true, user_config.channel_options.announced_channel, ); @@ -809,7 +809,7 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes ln_coin .persister - .add_pending_channel_to_sql(&storage_ticker, &pending_channel_details) + .add_channel_to_sql(&storage_ticker, rpc_channel_id, pending_channel_details) .await?; Ok(OpenChannelResponse { diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 1b79806517..3d11262eb1 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -13,7 +13,7 @@ extern crate lightning; extern crate secp256k1; extern crate serde_json; -use crate::storage::{FileSystemStorage, NodesAddressesMap, NodesAddressesMapShared, PendingChannelForSql, SqlStorage}; +use crate::storage::{FileSystemStorage, NodesAddressesMap, NodesAddressesMapShared, SqlChannelDetails, SqlStorage}; use crate::util::DiskWriteable; use async_trait::async_trait; use bitcoin::blockdata::constants::genesis_block; @@ -22,7 +22,7 @@ use bitcoin::hashes::hex::{FromHex, ToHex}; use bitcoin::Network; use common::async_blocking; use common::fs::check_dir_operations; -use db_common::sqlite::rusqlite::{Connection, Error as SqlError, NO_PARAMS}; +use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, NO_PARAMS}; use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use lightning::chain; use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator}; @@ -95,12 +95,10 @@ fn create_channels_history_table_sql(for_coin: &str) -> Result channel_id VARCHAR(255) NOT NULL, counterparty_node_id VARCHAR(255) NOT NULL, counterparty_node_address VARCHAR(255) NOT NULL, - funding_tx VARCHAR(255), - funding_tx_output_index INTEGER, - funding_tx_value_sats INTEGER NOT NULL, - closing_tx VARCHAR(255), - closing_tx_output_index INTEGER, - closing_tx_value_sats INTEGER, + funding_tx VARCHAR(255) NOT NULL, + initial_balance INTEGER NOT NULL, + closing_tx VARCHAR(255) NOT NULL, + closing_balance INTEGER NOT NULL, is_outbound INTEGER NOT NULL, is_public INTEGER NOT NULL, is_pending INTEGER NOT NULL, @@ -117,11 +115,38 @@ fn insert_channel_in_sql(for_coin: &str) -> Result { let sql = "INSERT INTO ".to_owned() + &table_name - + " (rpc_id, channel_id, counterparty_node_id, counterparty_node_address, funding_tx, funding_tx_output_index, funding_tx_value_sats, closing_tx, closing_tx_output_index, closing_tx_value_sats, is_outbound, is_public, is_pending, is_open, is_closed) VALUES (?1, ?2, ?3, ?4, ?5, ?6,?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15);"; + + " (rpc_id, channel_id, counterparty_node_id, counterparty_node_address, funding_tx, initial_balance, closing_tx, closing_balance, is_outbound, is_public, is_pending, is_open, is_closed) VALUES (?1, ?2, ?3, ?4, ?5, ?6,?7, ?8, ?9, ?10, ?11, ?12, ?13);"; Ok(sql) } +fn select_channel_from_table_by_rpc_id_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "SELECT channel_id, counterparty_node_id, counterparty_node_address, funding_tx, initial_balance, closing_tx, closing_balance, is_outbound, is_public, is_pending, is_open, is_closed FROM ".to_owned() + &table_name + " WHERE rpc_id=?1;"; + + Ok(sql) +} + +fn channel_details_from_row(row: &Row<'_>) -> Result { + let channel_details = SqlChannelDetails { + channel_id: row.get(0)?, + counterparty_node_id: row.get(1)?, + counterparty_node_address: row.get(2)?, + funding_tx: row.get(3)?, + initial_balance: row.get::<_, u32>(4)? as u64, + closing_tx: row.get(5)?, + closing_balance: row.get::<_, u32>(6)? as u64, + is_outbound: row.get(7)?, + is_public: row.get(8)?, + is_pending: row.get(9)?, + is_open: row.get(10)?, + is_closed: row.get(11)?, + }; + Ok(channel_details) +} + fn get_last_channel_rpc_id_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; @@ -131,6 +156,15 @@ fn get_last_channel_rpc_id_sql(for_coin: &str) -> Result { Ok(sql) } +fn update_funding_tx_in_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "UPDATE ".to_owned() + &table_name + " SET funding_tx = ?2, initial_balance = ?3 WHERE rpc_id = ?1;"; + + Ok(sql) +} + impl LightningPersister { /// Initialize a new LightningPersister and set the path to the individual channels' /// files. @@ -498,27 +532,26 @@ impl SqlStorage for LightningPersister { .await } - async fn add_pending_channel_to_sql( + async fn add_channel_to_sql( &self, for_coin: &str, - details: &PendingChannelForSql, + rpc_id: u64, + details: SqlChannelDetails, ) -> Result<(), Self::Error> { let for_coin = for_coin.to_owned(); - let rpc_id = details.rpc_id.to_string(); - let channel_id = hex::encode(details.channel_id); - let counterparty_node_id = details.counterparty_node_id.to_string(); - let counterparty_node_address = details.counterparty_node_address.to_string(); - let funding_tx = String::default(); - let funding_tx_output_index = String::default(); - let funding_tx_value_sats = String::default(); - let closing_tx = String::default(); - let closing_tx_output_index = String::default(); - let closing_tx_value_sats = String::default(); + let rpc_id = rpc_id.to_string(); + let channel_id = details.channel_id; + let counterparty_node_id = details.counterparty_node_id; + let counterparty_node_address = details.counterparty_node_address; + let funding_tx = details.funding_tx; + let initial_balance = details.initial_balance.to_string(); + let closing_tx = details.closing_tx; + let closing_balance = details.closing_balance.to_string(); let is_outbound = (details.is_outbound as i32).to_string(); let is_public = (details.is_public as i32).to_string(); - let is_pending = 1_u8.to_string(); - let is_open = 0_u8.to_string(); - let is_closed = 0_u8.to_string(); + let is_pending = (details.is_pending as i32).to_string(); + let is_open = (details.is_open as i32).to_string(); + let is_closed = (details.is_closed as i32).to_string(); let params = [ rpc_id, @@ -526,11 +559,9 @@ impl SqlStorage for LightningPersister { counterparty_node_id, counterparty_node_address, funding_tx, - funding_tx_output_index, - funding_tx_value_sats, + initial_balance, closing_tx, - closing_tx_output_index, - closing_tx_value_sats, + closing_balance, is_outbound, is_public, is_pending, @@ -549,6 +580,22 @@ impl SqlStorage for LightningPersister { .await } + async fn get_channel_from_sql( + &self, + for_coin: &str, + rpc_id: u64, + ) -> Result, Self::Error> { + let params = [rpc_id.to_string()]; + let sql = select_channel_from_table_by_rpc_id_sql(for_coin)?; + let sqlite_connection = self.sqlite_connection.clone(); + + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + query_single_row(&conn, &sql, params, channel_details_from_row) + }) + .await + } + async fn get_last_channel_rpc_id(&self, for_coin: &str) -> Result { let sql = get_last_channel_rpc_id_sql(for_coin)?; let sqlite_connection = self.sqlite_connection.clone(); @@ -560,6 +607,30 @@ impl SqlStorage for LightningPersister { }) .await } + + async fn add_funding_tx_to_sql( + &self, + for_coin: &str, + rpc_id: u64, + funding_tx: String, + initial_balance: u64, + ) -> Result<(), Self::Error> { + let for_coin = for_coin.to_owned(); + let rpc_id = rpc_id.to_string(); + let initial_balance = initial_balance.to_string(); + + let params = [rpc_id, funding_tx, initial_balance]; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + sql_transaction.execute(&update_funding_tx_in_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } } #[cfg(test)] @@ -804,8 +875,8 @@ mod tests { } #[test] - fn test_add_pending_channel_to_sql() { - let for_coin = "add_channel"; + fn test_add_get_channel_sql() { + let for_coin = "add_get_channel"; let persister = LightningPersister::new( PathBuf::from("test_filesystem_persister"), None, @@ -814,26 +885,43 @@ mod tests { block_on(persister.init_sql(for_coin)).unwrap(); - let pending_chan_1_details = PendingChannelForSql::new( - 1, + let channel = block_on(persister.get_channel_from_sql(for_coin, 1)).unwrap(); + assert!(channel.is_none()); + + let mut expected_channel_details = SqlChannelDetails::new( [0; 32], PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), "127.0.0.1:9735".parse().unwrap(), + 2000, true, true, ); - block_on(persister.add_pending_channel_to_sql(for_coin, &pending_chan_1_details)).unwrap(); + block_on(persister.add_channel_to_sql(for_coin, 1, expected_channel_details.clone())).unwrap(); let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id(for_coin)).unwrap(); assert_eq!(last_channel_rpc_id, 1); + let actual_channel_details = block_on(persister.get_channel_from_sql(for_coin, 1)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); + // must fail because we are adding channel with the same rpc_id - block_on(persister.add_pending_channel_to_sql(for_coin, &pending_chan_1_details)).unwrap_err(); + block_on(persister.add_channel_to_sql(for_coin, 1, expected_channel_details.clone())).unwrap_err(); assert_eq!(last_channel_rpc_id, 1); - let mut pending_chan_2_details = pending_chan_1_details; - pending_chan_2_details.rpc_id = 2; - block_on(persister.add_pending_channel_to_sql(for_coin, &pending_chan_2_details)).unwrap(); + block_on(persister.add_channel_to_sql(for_coin, 2, expected_channel_details.clone())).unwrap(); let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id(for_coin)).unwrap(); assert_eq!(last_channel_rpc_id, 2); + + block_on(persister.add_funding_tx_to_sql( + for_coin, + 2, + "04d3f3931b7644094bbdaadbf341183d35bb7251cea2c91f5058cee28b7c2f20".into(), + 3000, + )) + .unwrap(); + expected_channel_details.funding_tx = "04d3f3931b7644094bbdaadbf341183d35bb7251cea2c91f5058cee28b7c2f20".into(); + expected_channel_details.initial_balance = 3000; + + let actual_channel_details = block_on(persister.get_channel_from_sql(for_coin, 2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details) } } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index d5461ef261..2e52e46c05 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -33,31 +33,44 @@ pub trait FileSystemStorage { async fn save_scorer(&self, scorer: Arc>) -> Result<(), Self::Error>; } -pub struct PendingChannelForSql { - pub rpc_id: u64, - pub channel_id: [u8; 32], - pub counterparty_node_id: PublicKey, - pub counterparty_node_address: SocketAddr, +#[derive(Clone, Debug, PartialEq)] +pub struct SqlChannelDetails { + pub channel_id: String, + pub counterparty_node_id: String, + pub counterparty_node_address: String, + pub funding_tx: String, + pub initial_balance: u64, + pub closing_tx: String, + pub closing_balance: u64, pub is_outbound: bool, pub is_public: bool, + pub is_pending: bool, + pub is_open: bool, + pub is_closed: bool, } -impl PendingChannelForSql { +impl SqlChannelDetails { pub fn new( - rpc_id: u64, channel_id: [u8; 32], counterparty_node_id: PublicKey, counterparty_node_address: SocketAddr, + initial_balance: u64, is_outbound: bool, is_public: bool, ) -> Self { - PendingChannelForSql { - rpc_id, - channel_id, - counterparty_node_id, - counterparty_node_address, + SqlChannelDetails { + channel_id: hex::encode(channel_id), + counterparty_node_id: counterparty_node_id.to_string(), + counterparty_node_address: counterparty_node_address.to_string(), + funding_tx: "".into(), + initial_balance, + closing_tx: "".into(), + closing_balance: 0, is_outbound, is_public, + is_pending: true, + is_open: false, + is_closed: false, } } } @@ -71,11 +84,23 @@ pub trait SqlStorage { async fn is_sql_initialized(&self, for_coin: &str) -> Result; - async fn add_pending_channel_to_sql( + async fn add_channel_to_sql( &self, for_coin: &str, - details: &PendingChannelForSql, + rpc_id: u64, + details: SqlChannelDetails, ) -> Result<(), Self::Error>; + async fn get_channel_from_sql(&self, for_coin: &str, rpc_id: u64) + -> Result, Self::Error>; + async fn get_last_channel_rpc_id(&self, for_coin: &str) -> Result; + + async fn add_funding_tx_to_sql( + &self, + for_coin: &str, + rpc_id: u64, + funding_tx: String, + initial_balance: u64, + ) -> Result<(), Self::Error>; } From 5069239c49933d17ec5c147dbf99ffa67b85af1f Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Wed, 16 Mar 2022 22:16:09 +0200 Subject: [PATCH 12/49] Channels sql WIP: add funding tx to sql after FundingGenerationReady event --- mm2src/coins/lightning.rs | 9 ++-- mm2src/coins/lightning/ln_events.rs | 19 ++++++++- mm2src/coins/lightning/ln_utils.rs | 10 +++-- mm2src/coins/lightning_persister/src/lib.rs | 41 +++++++++---------- .../coins/lightning_persister/src/storage.rs | 17 +++----- 5 files changed, 50 insertions(+), 46 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 27e6f23f28..206129aebe 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -185,9 +185,6 @@ impl LightningCoin { fee_paid_msat: None, })) } - - // Todo: implement GetHistoryCoinType instead - fn storage_ticker(&self) -> String { self.ticker().replace('-', "_") } } #[async_trait] @@ -577,6 +574,7 @@ pub async fn start_lightning( platform.clone(), channel_manager.clone(), keys_manager.clone(), + persister.clone(), inbound_payments.clone(), outbound_payments.clone(), )); @@ -776,8 +774,7 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes user_config.own_channel_config.our_htlc_minimum_msat = min; } - let storage_ticker = ln_coin.storage_ticker(); - let rpc_channel_id = ln_coin.persister.get_last_channel_rpc_id(&storage_ticker).await? as u64 + 1; + let rpc_channel_id = ln_coin.persister.get_last_channel_rpc_id().await? as u64 + 1; let temp_channel_id = async_blocking(move || { channel_manager @@ -809,7 +806,7 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes ln_coin .persister - .add_channel_to_sql(&storage_ticker, rpc_channel_id, pending_channel_details) + .add_channel_to_sql(rpc_channel_id, pending_channel_details) .await?; Ok(OpenChannelResponse { diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 1e0ead482f..2779e1a1b2 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -19,6 +19,7 @@ pub struct LightningEventHandler { platform: Arc, channel_manager: Arc, keys_manager: Arc, + persister: Arc, inbound_payments: PaymentsMapShared, outbound_payments: PaymentsMapShared, } @@ -28,10 +29,10 @@ impl EventHandler for LightningEventHandler { match event { Event::FundingGenerationReady { temporary_channel_id, + channel_value_satoshis, output_script, user_channel_id, - .. - } => self.handle_funding_generation_ready(*temporary_channel_id, output_script, *user_channel_id), + } => self.handle_funding_generation_ready(*temporary_channel_id, *channel_value_satoshis, output_script, *user_channel_id), Event::PaymentReceived { payment_hash, amt, @@ -130,6 +131,7 @@ impl LightningEventHandler { platform: Arc, channel_manager: Arc, keys_manager: Arc, + persister: Arc, inbound_payments: PaymentsMapShared, outbound_payments: PaymentsMapShared, ) -> Self { @@ -137,6 +139,7 @@ impl LightningEventHandler { platform, channel_manager, keys_manager, + persister, inbound_payments, outbound_payments, } @@ -145,6 +148,7 @@ impl LightningEventHandler { fn handle_funding_generation_ready( &self, temporary_channel_id: [u8; 32], + channel_value_satoshis: u64, output_script: &Script, user_channel_id: u64, ) { @@ -163,13 +167,24 @@ impl LightningEventHandler { return; }, }; + let funding_txid = funding_tx.txid(); // Give the funding transaction back to LDK for opening the channel. if let Err(e) = self .channel_manager .funding_transaction_generated(&temporary_channel_id, funding_tx) { log::error!("{:?}", e); + return; } + let persister = self.persister.clone(); + spawn(async move { + if let Err(e) = persister + .add_funding_tx_to_sql(user_channel_id, funding_txid.to_string(), channel_value_satoshis) + .await + { + log::error!("{}", e); + } + }); } fn handle_payment_received(&self, payment_hash: PaymentHash, amt: u64, purpose: &PaymentPurpose) { diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 5f3fbc6461..5b159ae594 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -55,6 +55,10 @@ pub async fn init_persister( let ln_data_dir = ln_data_dir(ctx, &ticker); let ln_data_backup_dir = ln_data_backup_dir(ctx, backup_path, &ticker); let persister = Arc::new(LightningPersister::new( + // Todo: Maybe implement GetHistoryCoinType instead / use get_history_coin_type and storage_ticker functions + // should probably use platform orderbook_ticker this way we will have channels history and payments history + // for BTC instead of BTC-segwit for example + ticker.replace('-', "_"), ln_data_dir, ln_data_backup_dir, ctx.sqlite_connection @@ -67,11 +71,9 @@ pub async fn init_persister( if !is_initialized { persister.init_fs().await?; } - // Todo: use get_history_coin_type and storage_ticker functions - should probably use platform orderbook_ticker - // this way we will have channels history and payments history for BTC instead of BTC-segwit for example - let is_sql_initialized = persister.is_sql_initialized(&ticker.replace('-', "_")).await?; + let is_sql_initialized = persister.is_sql_initialized().await?; if !is_sql_initialized { - persister.init_sql(&ticker.replace('-', "_")).await?; + persister.init_sql().await?; } Ok(persister) } diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 3d11262eb1..232c3b0f49 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -60,6 +60,7 @@ use std::sync::{Arc, Mutex}; /// LightningPersister. pub struct LightningPersister { + storage_ticker: String, main_path: PathBuf, backup_path: Option, sqlite_connection: Arc>, @@ -168,8 +169,14 @@ fn update_funding_tx_in_sql(for_coin: &str) -> Result { impl LightningPersister { /// Initialize a new LightningPersister and set the path to the individual channels' /// files. - pub fn new(main_path: PathBuf, backup_path: Option, sqlite_connection: Arc>) -> Self { + pub fn new( + storage_ticker: String, + main_path: PathBuf, + backup_path: Option, + sqlite_connection: Arc>, + ) -> Self { Self { + storage_ticker, main_path, backup_path, sqlite_connection, @@ -507,9 +514,9 @@ impl FileSystemStorage for LightningPersister { impl SqlStorage for LightningPersister { type Error = SqlError; - async fn init_sql(&self, for_coin: &str) -> Result<(), Self::Error> { + async fn init_sql(&self) -> Result<(), Self::Error> { let sqlite_connection = self.sqlite_connection.clone(); - let sql_channels_history = create_channels_history_table_sql(for_coin)?; + let sql_channels_history = create_channels_history_table_sql(self.storage_ticker.as_str())?; async_blocking(move || { let conn = sqlite_connection.lock().unwrap(); conn.execute(&sql_channels_history, NO_PARAMS).map(|_| ())?; @@ -518,8 +525,8 @@ impl SqlStorage for LightningPersister { .await } - async fn is_sql_initialized(&self, for_coin: &str) -> Result { - let channels_history_table = channels_history_table(for_coin); + async fn is_sql_initialized(&self) -> Result { + let channels_history_table = channels_history_table(self.storage_ticker.as_str()); validate_table_name(&channels_history_table)?; let sqlite_connection = self.sqlite_connection.clone(); @@ -532,13 +539,8 @@ impl SqlStorage for LightningPersister { .await } - async fn add_channel_to_sql( - &self, - for_coin: &str, - rpc_id: u64, - details: SqlChannelDetails, - ) -> Result<(), Self::Error> { - let for_coin = for_coin.to_owned(); + async fn add_channel_to_sql(&self, rpc_id: u64, details: SqlChannelDetails) -> Result<(), Self::Error> { + let for_coin = self.storage_ticker.clone(); let rpc_id = rpc_id.to_string(); let channel_id = details.channel_id; let counterparty_node_id = details.counterparty_node_id; @@ -580,13 +582,9 @@ impl SqlStorage for LightningPersister { .await } - async fn get_channel_from_sql( - &self, - for_coin: &str, - rpc_id: u64, - ) -> Result, Self::Error> { + async fn get_channel_from_sql(&self, rpc_id: u64) -> Result, Self::Error> { let params = [rpc_id.to_string()]; - let sql = select_channel_from_table_by_rpc_id_sql(for_coin)?; + let sql = select_channel_from_table_by_rpc_id_sql(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { @@ -596,8 +594,8 @@ impl SqlStorage for LightningPersister { .await } - async fn get_last_channel_rpc_id(&self, for_coin: &str) -> Result { - let sql = get_last_channel_rpc_id_sql(for_coin)?; + async fn get_last_channel_rpc_id(&self) -> Result { + let sql = get_last_channel_rpc_id_sql(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { @@ -610,12 +608,11 @@ impl SqlStorage for LightningPersister { async fn add_funding_tx_to_sql( &self, - for_coin: &str, rpc_id: u64, funding_tx: String, initial_balance: u64, ) -> Result<(), Self::Error> { - let for_coin = for_coin.to_owned(); + let for_coin = self.storage_ticker.clone(); let rpc_id = rpc_id.to_string(); let initial_balance = initial_balance.to_string(); diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 2e52e46c05..723fa4c71e 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -80,25 +80,18 @@ pub trait SqlStorage { type Error; /// Initializes dirs/collection/tables in storage for a specified coin - async fn init_sql(&self, for_coin: &str) -> Result<(), Self::Error>; + async fn init_sql(&self) -> Result<(), Self::Error>; - async fn is_sql_initialized(&self, for_coin: &str) -> Result; + async fn is_sql_initialized(&self) -> Result; - async fn add_channel_to_sql( - &self, - for_coin: &str, - rpc_id: u64, - details: SqlChannelDetails, - ) -> Result<(), Self::Error>; + async fn add_channel_to_sql(&self, rpc_id: u64, details: SqlChannelDetails) -> Result<(), Self::Error>; - async fn get_channel_from_sql(&self, for_coin: &str, rpc_id: u64) - -> Result, Self::Error>; + async fn get_channel_from_sql(&self, rpc_id: u64) -> Result, Self::Error>; - async fn get_last_channel_rpc_id(&self, for_coin: &str) -> Result; + async fn get_last_channel_rpc_id(&self) -> Result; async fn add_funding_tx_to_sql( &self, - for_coin: &str, rpc_id: u64, funding_tx: String, initial_balance: u64, From 15047e04c0066fc5e7515b9cde70cf2bfb37e9fc Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Fri, 18 Mar 2022 02:11:55 +0200 Subject: [PATCH 13/49] Channels sql WIP: add closure_reason to sql table --- mm2src/coins/lightning/ln_events.rs | 17 +++--- .../lightning_background_processor/src/lib.rs | 1 + mm2src/coins/lightning_persister/src/lib.rs | 59 +++++++++++-------- .../coins/lightning_persister/src/storage.rs | 2 + 4 files changed, 48 insertions(+), 31 deletions(-) diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 2779e1a1b2..440f34c048 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -6,7 +6,7 @@ use common::log; use core::time::Duration; use lightning::chain::chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator}; use lightning::chain::keysinterface::SpendableOutputDescriptor; -use lightning::util::events::{Event, EventHandler, PaymentPurpose}; +use lightning::util::events::{ClosureReason, Event, EventHandler, PaymentPurpose}; use rand::Rng; use script::{Builder, SignatureVersion}; use secp256k1::Secp256k1; @@ -54,12 +54,7 @@ impl EventHandler for LightningEventHandler { self.platform.coin.ticker(), claim_from_onchain_tx, ), - // Todo: Use storage to store channels history - Event::ChannelClosed { channel_id, reason, .. } => log::info!( - "Channel: {} closed for the following reason: {}", - hex::encode(channel_id), - reason - ), + Event::ChannelClosed { channel_id, user_channel_id, reason } => self.handle_channel_closed(*channel_id, *user_channel_id, reason), // Todo: Add spent UTXOs to RecentlySpentOutPoints if it's not discarded Event::DiscardFunding { channel_id, transaction } => log::info!( "Discarding funding tx: {} for channel {}", @@ -255,6 +250,14 @@ impl LightningEventHandler { } } + fn handle_channel_closed(&self, channel_id: [u8; 32], _user_channel_id: u64, reason: &ClosureReason) { + log::info!( + "Channel: {} closed for the following reason: {}", + hex::encode(channel_id), + reason + ) + } + fn handle_payment_failed(&self, payment_hash: &PaymentHash) { log::info!( "Handling PaymentFailed event for payment_hash: {}", diff --git a/mm2src/coins/lightning_background_processor/src/lib.rs b/mm2src/coins/lightning_background_processor/src/lib.rs index 0784dd16b0..0c3f9f0fb2 100644 --- a/mm2src/coins/lightning_background_processor/src/lib.rs +++ b/mm2src/coins/lightning_background_processor/src/lib.rs @@ -444,6 +444,7 @@ mod tests { let chain_source = Arc::new(test_utils::TestChainSource::new(Network::Testnet)); let logger = Arc::new(test_utils::TestLogger::with_id(format!("node {}", i))); let persister = Arc::new(LightningPersister::new( + format!("node_{}_ticker", i), PathBuf::from(format!("{}_persister_{}", persist_dir, i)), None, Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 232c3b0f49..a434bd2ff0 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -100,6 +100,7 @@ fn create_channels_history_table_sql(for_coin: &str) -> Result initial_balance INTEGER NOT NULL, closing_tx VARCHAR(255) NOT NULL, closing_balance INTEGER NOT NULL, + closure_reason TEXT NOT NULL, is_outbound INTEGER NOT NULL, is_public INTEGER NOT NULL, is_pending INTEGER NOT NULL, @@ -116,7 +117,7 @@ fn insert_channel_in_sql(for_coin: &str) -> Result { let sql = "INSERT INTO ".to_owned() + &table_name - + " (rpc_id, channel_id, counterparty_node_id, counterparty_node_address, funding_tx, initial_balance, closing_tx, closing_balance, is_outbound, is_public, is_pending, is_open, is_closed) VALUES (?1, ?2, ?3, ?4, ?5, ?6,?7, ?8, ?9, ?10, ?11, ?12, ?13);"; + + " (rpc_id, channel_id, counterparty_node_id, counterparty_node_address, funding_tx, initial_balance, closing_tx, closing_balance, closure_reason, is_outbound, is_public, is_pending, is_open, is_closed) VALUES (?1, ?2, ?3, ?4, ?5, ?6,?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14);"; Ok(sql) } @@ -125,7 +126,7 @@ fn select_channel_from_table_by_rpc_id_sql(for_coin: &str) -> Result) -> Result(4)? as u64, closing_tx: row.get(5)?, closing_balance: row.get::<_, u32>(6)? as u64, - is_outbound: row.get(7)?, - is_public: row.get(8)?, - is_pending: row.get(9)?, - is_open: row.get(10)?, - is_closed: row.get(11)?, + closure_reason: row.get(7)?, + is_outbound: row.get(8)?, + is_public: row.get(9)?, + is_pending: row.get(10)?, + is_open: row.get(11)?, + is_closed: row.get(12)?, }; Ok(channel_details) } @@ -549,6 +551,7 @@ impl SqlStorage for LightningPersister { let initial_balance = details.initial_balance.to_string(); let closing_tx = details.closing_tx; let closing_balance = details.closing_balance.to_string(); + let closure_reason = details.closure_reason; let is_outbound = (details.is_outbound as i32).to_string(); let is_public = (details.is_public as i32).to_string(); let is_pending = (details.is_pending as i32).to_string(); @@ -564,6 +567,7 @@ impl SqlStorage for LightningPersister { initial_balance, closing_tx, closing_balance, + closure_reason, is_outbound, is_public, is_pending, @@ -670,11 +674,13 @@ mod tests { fn test_filesystem_persister() { // Create the nodes, giving them LightningPersisters for data persisters. let persister_0 = LightningPersister::new( + "test_filesystem_persister_0".into(), PathBuf::from("test_filesystem_persister_0"), None, Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), ); let persister_1 = LightningPersister::new( + "test_filesystem_persister_1".into(), PathBuf::from("test_filesystem_persister_1"), None, Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), @@ -774,7 +780,12 @@ mod tests { #[cfg(not(target_os = "windows"))] #[test] fn test_readonly_dir_perm_failure() { - let persister = LightningPersister::new(PathBuf::from("test_readonly_dir_perm_failure"), None); + let persister = LightningPersister::new( + "test_readonly_dir_perm_failure".into(), + PathBuf::from("test_readonly_dir_perm_failure"), + None, + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); fs::create_dir_all(&persister.main_path).unwrap(); // Set up a dummy channel and force close. This will produce a monitor @@ -834,6 +845,7 @@ mod tests { // don't seem to be invalid filename characters on Unix that Rust doesn't // handle, hence why the test is Windows-only. let persister = LightningPersister::new( + "test_fail_on_open".into(), PathBuf::from(":<>/"), None, Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), @@ -854,35 +866,35 @@ mod tests { #[test] fn test_init_sql_collection() { - let for_coin = "init_sql_collection"; let persister = LightningPersister::new( + "init_sql_collection".into(), PathBuf::from("test_filesystem_persister"), None, Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), ); - let initialized = block_on(persister.is_sql_initialized(for_coin)).unwrap(); + let initialized = block_on(persister.is_sql_initialized()).unwrap(); assert!(!initialized); - block_on(persister.init_sql(for_coin)).unwrap(); + block_on(persister.init_sql()).unwrap(); // repetitive init must not fail - block_on(persister.init_sql(for_coin)).unwrap(); + block_on(persister.init_sql()).unwrap(); - let initialized = block_on(persister.is_sql_initialized(for_coin)).unwrap(); + let initialized = block_on(persister.is_sql_initialized()).unwrap(); assert!(initialized); } #[test] fn test_add_get_channel_sql() { - let for_coin = "add_get_channel"; let persister = LightningPersister::new( + "add_get_channel".into(), PathBuf::from("test_filesystem_persister"), None, Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), ); - block_on(persister.init_sql(for_coin)).unwrap(); + block_on(persister.init_sql()).unwrap(); - let channel = block_on(persister.get_channel_from_sql(for_coin, 1)).unwrap(); + let channel = block_on(persister.get_channel_from_sql(1)).unwrap(); assert!(channel.is_none()); let mut expected_channel_details = SqlChannelDetails::new( @@ -893,23 +905,22 @@ mod tests { true, true, ); - block_on(persister.add_channel_to_sql(for_coin, 1, expected_channel_details.clone())).unwrap(); - let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id(for_coin)).unwrap(); + block_on(persister.add_channel_to_sql(1, expected_channel_details.clone())).unwrap(); + let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id()).unwrap(); assert_eq!(last_channel_rpc_id, 1); - let actual_channel_details = block_on(persister.get_channel_from_sql(for_coin, 1)).unwrap().unwrap(); + let actual_channel_details = block_on(persister.get_channel_from_sql(1)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); // must fail because we are adding channel with the same rpc_id - block_on(persister.add_channel_to_sql(for_coin, 1, expected_channel_details.clone())).unwrap_err(); + block_on(persister.add_channel_to_sql(1, expected_channel_details.clone())).unwrap_err(); assert_eq!(last_channel_rpc_id, 1); - block_on(persister.add_channel_to_sql(for_coin, 2, expected_channel_details.clone())).unwrap(); - let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id(for_coin)).unwrap(); + block_on(persister.add_channel_to_sql(2, expected_channel_details.clone())).unwrap(); + let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id()).unwrap(); assert_eq!(last_channel_rpc_id, 2); block_on(persister.add_funding_tx_to_sql( - for_coin, 2, "04d3f3931b7644094bbdaadbf341183d35bb7251cea2c91f5058cee28b7c2f20".into(), 3000, @@ -918,7 +929,7 @@ mod tests { expected_channel_details.funding_tx = "04d3f3931b7644094bbdaadbf341183d35bb7251cea2c91f5058cee28b7c2f20".into(); expected_channel_details.initial_balance = 3000; - let actual_channel_details = block_on(persister.get_channel_from_sql(for_coin, 2)).unwrap().unwrap(); + let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details) } } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 723fa4c71e..e1b0f095db 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -42,6 +42,7 @@ pub struct SqlChannelDetails { pub initial_balance: u64, pub closing_tx: String, pub closing_balance: u64, + pub closure_reason: String, pub is_outbound: bool, pub is_public: bool, pub is_pending: bool, @@ -66,6 +67,7 @@ impl SqlChannelDetails { initial_balance, closing_tx: "".into(), closing_balance: 0, + closure_reason: "".into(), is_outbound, is_public, is_pending: true, From 4378f7dbdeb77548ec9aca11793cdc064c45a600 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Fri, 18 Mar 2022 05:39:18 +0200 Subject: [PATCH 14/49] Channels sql WIP: update channel to closed in sql --- mm2src/coins/lightning.rs | 1 - mm2src/coins/lightning/ln_events.rs | 19 +++-- mm2src/coins/lightning_persister/src/lib.rs | 69 ++++++++++++------- .../coins/lightning_persister/src/storage.rs | 9 +-- 4 files changed, 62 insertions(+), 36 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 206129aebe..0ac6d8d8dc 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -791,7 +791,6 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes let pending_channel_details = SqlChannelDetails::new( temp_channel_id, node_pubkey, - node_addr, amount_in_sat, true, user_config.channel_options.announced_channel, diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 440f34c048..268a1eb5de 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -6,7 +6,7 @@ use common::log; use core::time::Duration; use lightning::chain::chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator}; use lightning::chain::keysinterface::SpendableOutputDescriptor; -use lightning::util::events::{ClosureReason, Event, EventHandler, PaymentPurpose}; +use lightning::util::events::{Event, EventHandler, PaymentPurpose}; use rand::Rng; use script::{Builder, SignatureVersion}; use secp256k1::Secp256k1; @@ -54,7 +54,7 @@ impl EventHandler for LightningEventHandler { self.platform.coin.ticker(), claim_from_onchain_tx, ), - Event::ChannelClosed { channel_id, user_channel_id, reason } => self.handle_channel_closed(*channel_id, *user_channel_id, reason), + Event::ChannelClosed { channel_id, user_channel_id, reason } => self.handle_channel_closed(*channel_id, *user_channel_id, reason.to_string()), // Todo: Add spent UTXOs to RecentlySpentOutPoints if it's not discarded Event::DiscardFunding { channel_id, transaction } => log::info!( "Discarding funding tx: {} for channel {}", @@ -250,12 +250,23 @@ impl LightningEventHandler { } } - fn handle_channel_closed(&self, channel_id: [u8; 32], _user_channel_id: u64, reason: &ClosureReason) { + fn handle_channel_closed(&self, channel_id: [u8; 32], user_channel_id: u64, reason: String) { log::info!( "Channel: {} closed for the following reason: {}", hex::encode(channel_id), reason - ) + ); + let persister = self.persister.clone(); + // Todo: Handle inbound channels closure case after updating to latest version of rust-lightning + // as it has a new OpenChannelRequest event where we can give an inbound channel a user_channel_id + // other than 0 in sql + if user_channel_id != 0 { + spawn(async move { + if let Err(e) = persister.update_channel_to_closed(user_channel_id, reason).await { + log::error!("{}", e); + } + }); + } } fn handle_payment_failed(&self, payment_hash: &PaymentHash) { diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index a434bd2ff0..01affd4fab 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -95,7 +95,6 @@ fn create_channels_history_table_sql(for_coin: &str) -> Result rpc_id INTEGER NOT NULL UNIQUE, channel_id VARCHAR(255) NOT NULL, counterparty_node_id VARCHAR(255) NOT NULL, - counterparty_node_address VARCHAR(255) NOT NULL, funding_tx VARCHAR(255) NOT NULL, initial_balance INTEGER NOT NULL, closing_tx VARCHAR(255) NOT NULL, @@ -103,8 +102,6 @@ fn create_channels_history_table_sql(for_coin: &str) -> Result closure_reason TEXT NOT NULL, is_outbound INTEGER NOT NULL, is_public INTEGER NOT NULL, - is_pending INTEGER NOT NULL, - is_open INTEGER NOT NULL, is_closed INTEGER NOT NULL );"; @@ -117,7 +114,7 @@ fn insert_channel_in_sql(for_coin: &str) -> Result { let sql = "INSERT INTO ".to_owned() + &table_name - + " (rpc_id, channel_id, counterparty_node_id, counterparty_node_address, funding_tx, initial_balance, closing_tx, closing_balance, closure_reason, is_outbound, is_public, is_pending, is_open, is_closed) VALUES (?1, ?2, ?3, ?4, ?5, ?6,?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14);"; + + " (rpc_id, channel_id, counterparty_node_id, funding_tx, initial_balance, closing_tx, closing_balance, closure_reason, is_outbound, is_public, is_closed) VALUES (?1, ?2, ?3, ?4, ?5, ?6,?7, ?8, ?9, ?10, ?11);"; Ok(sql) } @@ -126,7 +123,7 @@ fn select_channel_from_table_by_rpc_id_sql(for_coin: &str) -> Result) -> Result(4)? as u64, - closing_tx: row.get(5)?, - closing_balance: row.get::<_, u32>(6)? as u64, - closure_reason: row.get(7)?, - is_outbound: row.get(8)?, - is_public: row.get(9)?, - is_pending: row.get(10)?, - is_open: row.get(11)?, - is_closed: row.get(12)?, + funding_tx: row.get(2)?, + initial_balance: row.get::<_, u32>(3)? as u64, + closing_tx: row.get(4)?, + closing_balance: row.get::<_, u32>(5)? as u64, + closure_reason: row.get(6)?, + is_outbound: row.get(7)?, + is_public: row.get(8)?, + is_closed: row.get(9)?, }; Ok(channel_details) } @@ -168,6 +162,15 @@ fn update_funding_tx_in_sql(for_coin: &str) -> Result { Ok(sql) } +fn update_channel_to_closed_in_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "UPDATE ".to_owned() + &table_name + " SET closure_reason = ?2, is_closed = ?3 WHERE rpc_id = ?1;"; + + Ok(sql) +} + impl LightningPersister { /// Initialize a new LightningPersister and set the path to the individual channels' /// files. @@ -546,7 +549,6 @@ impl SqlStorage for LightningPersister { let rpc_id = rpc_id.to_string(); let channel_id = details.channel_id; let counterparty_node_id = details.counterparty_node_id; - let counterparty_node_address = details.counterparty_node_address; let funding_tx = details.funding_tx; let initial_balance = details.initial_balance.to_string(); let closing_tx = details.closing_tx; @@ -554,15 +556,12 @@ impl SqlStorage for LightningPersister { let closure_reason = details.closure_reason; let is_outbound = (details.is_outbound as i32).to_string(); let is_public = (details.is_public as i32).to_string(); - let is_pending = (details.is_pending as i32).to_string(); - let is_open = (details.is_open as i32).to_string(); let is_closed = (details.is_closed as i32).to_string(); let params = [ rpc_id, channel_id, counterparty_node_id, - counterparty_node_address, funding_tx, initial_balance, closing_tx, @@ -570,8 +569,6 @@ impl SqlStorage for LightningPersister { closure_reason, is_outbound, is_public, - is_pending, - is_open, is_closed, ]; @@ -632,6 +629,24 @@ impl SqlStorage for LightningPersister { }) .await } + + async fn update_channel_to_closed(&self, rpc_id: u64, closure_reason: String) -> Result<(), Self::Error> { + let for_coin = self.storage_ticker.clone(); + let rpc_id = rpc_id.to_string(); + let is_closed = "1".to_string(); + + let params = [rpc_id, closure_reason, is_closed]; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + sql_transaction.execute(&update_channel_to_closed_in_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } } #[cfg(test)] @@ -900,7 +915,6 @@ mod tests { let mut expected_channel_details = SqlChannelDetails::new( [0; 32], PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), - "127.0.0.1:9735".parse().unwrap(), 2000, true, true, @@ -930,6 +944,13 @@ mod tests { expected_channel_details.initial_balance = 3000; let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); - assert_eq!(expected_channel_details, actual_channel_details) + assert_eq!(expected_channel_details, actual_channel_details); + + block_on(persister.update_channel_to_closed(2, "the channel was cooperatively closed".into())).unwrap(); + expected_channel_details.closure_reason = "the channel was cooperatively closed".into(); + expected_channel_details.is_closed = true; + + let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); } } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index e1b0f095db..49ea0811f4 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -37,7 +37,6 @@ pub trait FileSystemStorage { pub struct SqlChannelDetails { pub channel_id: String, pub counterparty_node_id: String, - pub counterparty_node_address: String, pub funding_tx: String, pub initial_balance: u64, pub closing_tx: String, @@ -45,8 +44,6 @@ pub struct SqlChannelDetails { pub closure_reason: String, pub is_outbound: bool, pub is_public: bool, - pub is_pending: bool, - pub is_open: bool, pub is_closed: bool, } @@ -54,7 +51,6 @@ impl SqlChannelDetails { pub fn new( channel_id: [u8; 32], counterparty_node_id: PublicKey, - counterparty_node_address: SocketAddr, initial_balance: u64, is_outbound: bool, is_public: bool, @@ -62,7 +58,6 @@ impl SqlChannelDetails { SqlChannelDetails { channel_id: hex::encode(channel_id), counterparty_node_id: counterparty_node_id.to_string(), - counterparty_node_address: counterparty_node_address.to_string(), funding_tx: "".into(), initial_balance, closing_tx: "".into(), @@ -70,8 +65,6 @@ impl SqlChannelDetails { closure_reason: "".into(), is_outbound, is_public, - is_pending: true, - is_open: false, is_closed: false, } } @@ -98,4 +91,6 @@ pub trait SqlStorage { funding_tx: String, initial_balance: u64, ) -> Result<(), Self::Error>; + + async fn update_channel_to_closed(&self, rpc_id: u64, closure_reason: String) -> Result<(), Self::Error>; } From fdc54f760ae1015a802c1bf7f87991af536dc615 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Fri, 18 Mar 2022 23:26:30 +0200 Subject: [PATCH 15/49] Channels sql WIP: get closed channels in list_channels response --- Cargo.lock | 1 + mm2src/coins/lightning.rs | 12 +++-- mm2src/coins/lightning/ln_errors.rs | 7 +++ mm2src/coins/lightning_persister/Cargo.toml | 1 + mm2src/coins/lightning_persister/src/lib.rs | 48 ++++++++++++++++--- .../coins/lightning_persister/src/storage.rs | 5 +- mm2src/mm2_tests/lightning_tests.rs | 21 +++++--- 7 files changed, 78 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 13ff219c44..ccb49ca76a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3014,6 +3014,7 @@ dependencies = [ "lightning", "parking_lot 0.11.1", "secp256k1", + "serde", "serde_json", "winapi 0.3.9", ] diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 0ac6d8d8dc..c67fb371d8 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -866,7 +866,8 @@ impl From for ChannelDetailsForRPC { #[derive(Serialize)] pub struct ListChannelsResponse { - channels: Vec, + open_channels: Vec, + closed_channels: Vec, } pub async fn list_channels(ctx: MmArc, req: ListChannelsRequest) -> ListChannelsResult { @@ -875,14 +876,19 @@ pub async fn list_channels(ctx: MmArc, req: ListChannelsRequest) -> ListChannels MmCoinEnum::LightningCoin(c) => c, _ => return MmError::err(ListChannelsError::UnsupportedCoin(coin.ticker().to_string())), }; - let channels = ln_coin + let open_channels = ln_coin .channel_manager .list_channels() .into_iter() .map(From::from) .collect(); - Ok(ListChannelsResponse { channels }) + let closed_channels = ln_coin.persister.get_closed_channels().await?; + + Ok(ListChannelsResponse { + open_channels, + closed_channels, + }) } #[derive(Deserialize)] diff --git a/mm2src/coins/lightning/ln_errors.rs b/mm2src/coins/lightning/ln_errors.rs index 17c62028e8..77c7c04e36 100644 --- a/mm2src/coins/lightning/ln_errors.rs +++ b/mm2src/coins/lightning/ln_errors.rs @@ -221,6 +221,8 @@ pub enum ListChannelsError { UnsupportedCoin(String), #[display(fmt = "No such coin {}", _0)] NoSuchCoin(String), + #[display(fmt = "SQL error {}", _0)] + SqlError(String), } impl HttpStatusCode for ListChannelsError { @@ -228,6 +230,7 @@ impl HttpStatusCode for ListChannelsError { match self { ListChannelsError::UnsupportedCoin(_) => StatusCode::BAD_REQUEST, ListChannelsError::NoSuchCoin(_) => StatusCode::PRECONDITION_REQUIRED, + ListChannelsError::SqlError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -240,6 +243,10 @@ impl From for ListChannelsError { } } +impl From for ListChannelsError { + fn from(err: SqlError) -> ListChannelsError { ListChannelsError::SqlError(err.to_string()) } +} + #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] pub enum GetChannelDetailsError { diff --git a/mm2src/coins/lightning_persister/Cargo.toml b/mm2src/coins/lightning_persister/Cargo.toml index 203ced48aa..da839a950a 100644 --- a/mm2src/coins/lightning_persister/Cargo.toml +++ b/mm2src/coins/lightning_persister/Cargo.toml @@ -19,6 +19,7 @@ lightning = "0.0.104" libc = "0.2" parking_lot = { version = "0.11", features = ["nightly"] } secp256k1 = { version = "0.20" } +serde = "1.0" serde_json = "1.0" [target.'cfg(windows)'.dependencies] diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 01affd4fab..dbf00ae4dc 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -108,7 +108,7 @@ fn create_channels_history_table_sql(for_coin: &str) -> Result Ok(sql) } -fn insert_channel_in_sql(for_coin: &str) -> Result { +fn insert_channel_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; @@ -148,12 +148,12 @@ fn get_last_channel_rpc_id_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "SELECT MAX(rpc_id) FROM ".to_owned() + &table_name + ";"; + let sql = "SELECT IFNULL(MAX(rpc_id), 0) FROM ".to_owned() + &table_name + ";"; Ok(sql) } -fn update_funding_tx_in_sql(for_coin: &str) -> Result { +fn update_funding_tx_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; @@ -162,7 +162,7 @@ fn update_funding_tx_in_sql(for_coin: &str) -> Result { Ok(sql) } -fn update_channel_to_closed_in_sql(for_coin: &str) -> Result { +fn update_channel_to_closed_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; @@ -171,6 +171,15 @@ fn update_channel_to_closed_in_sql(for_coin: &str) -> Result { Ok(sql) } +fn get_closed_channels_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "SELECT channel_id, counterparty_node_id, funding_tx, initial_balance, closing_tx, closing_balance, closure_reason, is_outbound, is_public, is_closed FROM ".to_owned() + &table_name + " WHERE is_closed = 1;"; + + Ok(sql) +} + impl LightningPersister { /// Initialize a new LightningPersister and set the path to the individual channels' /// files. @@ -576,7 +585,7 @@ impl SqlStorage for LightningPersister { async_blocking(move || { let mut conn = sqlite_connection.lock().unwrap(); let sql_transaction = conn.transaction()?; - sql_transaction.execute(&insert_channel_in_sql(&for_coin)?, ¶ms)?; + sql_transaction.execute(&insert_channel_sql(&for_coin)?, ¶ms)?; sql_transaction.commit()?; Ok(()) }) @@ -623,7 +632,7 @@ impl SqlStorage for LightningPersister { async_blocking(move || { let mut conn = sqlite_connection.lock().unwrap(); let sql_transaction = conn.transaction()?; - sql_transaction.execute(&update_funding_tx_in_sql(&for_coin)?, ¶ms)?; + sql_transaction.execute(&update_funding_tx_sql(&for_coin)?, ¶ms)?; sql_transaction.commit()?; Ok(()) }) @@ -641,12 +650,26 @@ impl SqlStorage for LightningPersister { async_blocking(move || { let mut conn = sqlite_connection.lock().unwrap(); let sql_transaction = conn.transaction()?; - sql_transaction.execute(&update_channel_to_closed_in_sql(&for_coin)?, ¶ms)?; + sql_transaction.execute(&update_channel_to_closed_sql(&for_coin)?, ¶ms)?; sql_transaction.commit()?; Ok(()) }) .await } + + async fn get_closed_channels(&self) -> Result, Self::Error> { + let sql = get_closed_channels_sql(self.storage_ticker.as_str())?; + let sqlite_connection = self.sqlite_connection.clone(); + + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query(NO_PARAMS)?; + let result = rows.mapped(channel_details_from_row).collect::>()?; + Ok(result) + }) + .await + } } #[cfg(test)] @@ -909,6 +932,9 @@ mod tests { block_on(persister.init_sql()).unwrap(); + let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id()).unwrap(); + assert_eq!(last_channel_rpc_id, 0); + let channel = block_on(persister.get_channel_from_sql(1)).unwrap(); assert!(channel.is_none()); @@ -952,5 +978,13 @@ mod tests { let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); + + let closed_channels = block_on(persister.get_closed_channels()).unwrap(); + assert_eq!(closed_channels.len(), 1); + assert_eq!(expected_channel_details, closed_channels[0]); + + block_on(persister.update_channel_to_closed(1, "the channel was cooperatively closed".into())).unwrap(); + let closed_channels = block_on(persister.get_closed_channels()).unwrap(); + assert_eq!(closed_channels.len(), 2); } } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 49ea0811f4..407790d52d 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -4,6 +4,7 @@ use lightning::routing::network_graph::NetworkGraph; use lightning::routing::scoring::Scorer; use parking_lot::Mutex as PaMutex; use secp256k1::PublicKey; +use serde::Serialize; use std::collections::HashMap; use std::net::SocketAddr; use std::sync::{Arc, Mutex}; @@ -33,7 +34,7 @@ pub trait FileSystemStorage { async fn save_scorer(&self, scorer: Arc>) -> Result<(), Self::Error>; } -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Serialize)] pub struct SqlChannelDetails { pub channel_id: String, pub counterparty_node_id: String, @@ -93,4 +94,6 @@ pub trait SqlStorage { ) -> Result<(), Self::Error>; async fn update_channel_to_closed(&self, rpc_id: u64, closure_reason: String) -> Result<(), Self::Error>; + + async fn get_closed_channels(&self) -> Result, Self::Error>; } diff --git a/mm2src/mm2_tests/lightning_tests.rs b/mm2src/mm2_tests/lightning_tests.rs index 6706eeec79..a63b002845 100644 --- a/mm2src/mm2_tests/lightning_tests.rs +++ b/mm2src/mm2_tests/lightning_tests.rs @@ -266,11 +266,17 @@ fn test_open_channel() { let list_channels_node_1_res: Json = json::from_str(&list_channels_node_1.1).unwrap(); log!("list_channels_node_1_res "[list_channels_node_1_res]); assert_eq!( - list_channels_node_1_res["result"]["channels"][0]["counterparty_node_id"], + list_channels_node_1_res["result"]["open_channels"][0]["counterparty_node_id"], node_2_id ); - assert_eq!(list_channels_node_1_res["result"]["channels"][0]["is_outbound"], false); - assert_eq!(list_channels_node_1_res["result"]["channels"][0]["balance_msat"], 0); + assert_eq!( + list_channels_node_1_res["result"]["open_channels"][0]["is_outbound"], + false + ); + assert_eq!( + list_channels_node_1_res["result"]["open_channels"][0]["balance_msat"], + 0 + ); let list_channels_node_2 = block_on(mm_node_2.rpc(json! ({ "userpass": mm_node_2.userpass, @@ -288,12 +294,15 @@ fn test_open_channel() { ); let list_channels_node_2_res: Json = json::from_str(&list_channels_node_2.1).unwrap(); assert_eq!( - list_channels_node_2_res["result"]["channels"][0]["counterparty_node_id"], + list_channels_node_2_res["result"]["open_channels"][0]["counterparty_node_id"], node_1_id ); - assert_eq!(list_channels_node_2_res["result"]["channels"][0]["is_outbound"], true); assert_eq!( - list_channels_node_2_res["result"]["channels"][0]["balance_msat"], + list_channels_node_2_res["result"]["open_channels"][0]["is_outbound"], + true + ); + assert_eq!( + list_channels_node_2_res["result"]["open_channels"][0]["balance_msat"], 2000000 ); From 4c6b47936649c829af7eea98d3a6099e7a0a8912 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Sat, 19 Mar 2022 01:13:39 +0200 Subject: [PATCH 16/49] Channels sql WIP: get closed channel by rpc_id in get_channel_details response --- mm2src/coins/lightning.rs | 28 +++++++++----- mm2src/coins/lightning/ln_errors.rs | 7 ++++ mm2src/coins/lightning_persister/src/lib.rs | 37 ++++++++++--------- .../coins/lightning_persister/src/storage.rs | 5 ++- 4 files changed, 49 insertions(+), 28 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index c67fb371d8..09b2460c0d 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -789,6 +789,7 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes } let pending_channel_details = SqlChannelDetails::new( + rpc_channel_id, temp_channel_id, node_pubkey, amount_in_sat, @@ -803,10 +804,7 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes .save_nodes_addresses(ln_coin.open_channels_nodes) .await?; - ln_coin - .persister - .add_channel_to_sql(rpc_channel_id, pending_channel_details) - .await?; + ln_coin.persister.add_channel_to_sql(pending_channel_details).await?; Ok(OpenChannelResponse { rpc_channel_id, @@ -898,8 +896,10 @@ pub struct GetChannelDetailsRequest { } #[derive(Serialize)] -pub struct GetChannelDetailsResponse { - channel_details: ChannelDetailsForRPC, +#[serde(tag = "status", content = "details")] +pub enum GetChannelDetailsResponse { + Open(ChannelDetailsForRPC), + Closed(SqlChannelDetails), } pub async fn get_channel_details( @@ -911,15 +911,23 @@ pub async fn get_channel_details( MmCoinEnum::LightningCoin(c) => c, _ => return MmError::err(GetChannelDetailsError::UnsupportedCoin(coin.ticker().to_string())), }; - let channel_details = ln_coin + let channel_details = match ln_coin .channel_manager .list_channels() .into_iter() .find(|chan| chan.user_channel_id == req.rpc_channel_id) - .ok_or(GetChannelDetailsError::NoSuchChannel(req.rpc_channel_id))? - .into(); + { + Some(details) => GetChannelDetailsResponse::Open(details.into()), + None => GetChannelDetailsResponse::Closed( + ln_coin + .persister + .get_channel_from_sql(req.rpc_channel_id) + .await? + .ok_or(GetChannelDetailsError::NoSuchChannel(req.rpc_channel_id))?, + ), + }; - Ok(GetChannelDetailsResponse { channel_details }) + Ok(channel_details) } #[derive(Deserialize)] diff --git a/mm2src/coins/lightning/ln_errors.rs b/mm2src/coins/lightning/ln_errors.rs index 77c7c04e36..ea97157599 100644 --- a/mm2src/coins/lightning/ln_errors.rs +++ b/mm2src/coins/lightning/ln_errors.rs @@ -256,6 +256,8 @@ pub enum GetChannelDetailsError { NoSuchCoin(String), #[display(fmt = "Channel with rpc id: {} is not found", _0)] NoSuchChannel(u64), + #[display(fmt = "SQL error {}", _0)] + SqlError(String), } impl HttpStatusCode for GetChannelDetailsError { @@ -264,6 +266,7 @@ impl HttpStatusCode for GetChannelDetailsError { GetChannelDetailsError::UnsupportedCoin(_) => StatusCode::BAD_REQUEST, GetChannelDetailsError::NoSuchCoin(_) => StatusCode::PRECONDITION_REQUIRED, GetChannelDetailsError::NoSuchChannel(_) => StatusCode::NOT_FOUND, + GetChannelDetailsError::SqlError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -276,6 +279,10 @@ impl From for GetChannelDetailsError { } } +impl From for GetChannelDetailsError { + fn from(err: SqlError) -> GetChannelDetailsError { GetChannelDetailsError::SqlError(err.to_string()) } +} + #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] pub enum GenerateInvoiceError { diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index dbf00ae4dc..75e2dc270c 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -123,23 +123,24 @@ fn select_channel_from_table_by_rpc_id_sql(for_coin: &str) -> Result) -> Result { let channel_details = SqlChannelDetails { - channel_id: row.get(0)?, - counterparty_node_id: row.get(1)?, - funding_tx: row.get(2)?, - initial_balance: row.get::<_, u32>(3)? as u64, - closing_tx: row.get(4)?, - closing_balance: row.get::<_, u32>(5)? as u64, - closure_reason: row.get(6)?, - is_outbound: row.get(7)?, - is_public: row.get(8)?, - is_closed: row.get(9)?, + rpc_id: row.get::<_, u32>(0)? as u64, + channel_id: row.get(1)?, + counterparty_node_id: row.get(2)?, + funding_tx: row.get(3)?, + initial_balance: row.get::<_, u32>(4)? as u64, + closing_tx: row.get(5)?, + closing_balance: row.get::<_, u32>(6)? as u64, + closure_reason: row.get(7)?, + is_outbound: row.get(8)?, + is_public: row.get(9)?, + is_closed: row.get(10)?, }; Ok(channel_details) } @@ -175,7 +176,7 @@ fn get_closed_channels_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "SELECT channel_id, counterparty_node_id, funding_tx, initial_balance, closing_tx, closing_balance, closure_reason, is_outbound, is_public, is_closed FROM ".to_owned() + &table_name + " WHERE is_closed = 1;"; + let sql = "SELECT rpc_id, channel_id, counterparty_node_id, funding_tx, initial_balance, closing_tx, closing_balance, closure_reason, is_outbound, is_public, is_closed FROM ".to_owned() + &table_name + " WHERE is_closed = 1;"; Ok(sql) } @@ -553,9 +554,9 @@ impl SqlStorage for LightningPersister { .await } - async fn add_channel_to_sql(&self, rpc_id: u64, details: SqlChannelDetails) -> Result<(), Self::Error> { + async fn add_channel_to_sql(&self, details: SqlChannelDetails) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); - let rpc_id = rpc_id.to_string(); + let rpc_id = details.rpc_id.to_string(); let channel_id = details.channel_id; let counterparty_node_id = details.counterparty_node_id; let funding_tx = details.funding_tx; @@ -939,13 +940,14 @@ mod tests { assert!(channel.is_none()); let mut expected_channel_details = SqlChannelDetails::new( + 1, [0; 32], PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), 2000, true, true, ); - block_on(persister.add_channel_to_sql(1, expected_channel_details.clone())).unwrap(); + block_on(persister.add_channel_to_sql(expected_channel_details.clone())).unwrap(); let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id()).unwrap(); assert_eq!(last_channel_rpc_id, 1); @@ -953,10 +955,11 @@ mod tests { assert_eq!(expected_channel_details, actual_channel_details); // must fail because we are adding channel with the same rpc_id - block_on(persister.add_channel_to_sql(1, expected_channel_details.clone())).unwrap_err(); + block_on(persister.add_channel_to_sql(expected_channel_details.clone())).unwrap_err(); assert_eq!(last_channel_rpc_id, 1); - block_on(persister.add_channel_to_sql(2, expected_channel_details.clone())).unwrap(); + expected_channel_details.rpc_id = 2; + block_on(persister.add_channel_to_sql(expected_channel_details.clone())).unwrap(); let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id()).unwrap(); assert_eq!(last_channel_rpc_id, 2); diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 407790d52d..1737533f41 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -36,6 +36,7 @@ pub trait FileSystemStorage { #[derive(Clone, Debug, PartialEq, Serialize)] pub struct SqlChannelDetails { + pub rpc_id: u64, pub channel_id: String, pub counterparty_node_id: String, pub funding_tx: String, @@ -50,6 +51,7 @@ pub struct SqlChannelDetails { impl SqlChannelDetails { pub fn new( + rpc_id: u64, channel_id: [u8; 32], counterparty_node_id: PublicKey, initial_balance: u64, @@ -57,6 +59,7 @@ impl SqlChannelDetails { is_public: bool, ) -> Self { SqlChannelDetails { + rpc_id, channel_id: hex::encode(channel_id), counterparty_node_id: counterparty_node_id.to_string(), funding_tx: "".into(), @@ -80,7 +83,7 @@ pub trait SqlStorage { async fn is_sql_initialized(&self) -> Result; - async fn add_channel_to_sql(&self, rpc_id: u64, details: SqlChannelDetails) -> Result<(), Self::Error>; + async fn add_channel_to_sql(&self, details: SqlChannelDetails) -> Result<(), Self::Error>; async fn get_channel_from_sql(&self, rpc_id: u64) -> Result, Self::Error>; From be882bfde28d56e4df3b07cf200783af148958c0 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Sat, 19 Mar 2022 06:15:27 +0200 Subject: [PATCH 17/49] Channels sql WIP: add closing transaction to sql --- mm2src/coins/lightning.rs | 1 - mm2src/coins/lightning/ln_events.rs | 64 +++++++++-- mm2src/coins/lightning_persister/src/lib.rs | 103 ++++++++++++------ .../coins/lightning_persister/src/storage.rs | 29 +++-- 4 files changed, 142 insertions(+), 55 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 09b2460c0d..df966a020e 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -792,7 +792,6 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes rpc_channel_id, temp_channel_id, node_pubkey, - amount_in_sat, true, user_config.channel_options.announced_channel, ); diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 268a1eb5de..58af2005f0 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -2,8 +2,9 @@ use super::*; use bitcoin::blockdata::script::Script; use bitcoin::blockdata::transaction::Transaction; use common::executor::{spawn, Timer}; -use common::log; +use common::{log, now_ms}; use core::time::Duration; +use futures::compat::Future01CompatExt; use lightning::chain::chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator}; use lightning::chain::keysinterface::SpendableOutputDescriptor; use lightning::util::events::{Event, EventHandler, PaymentPurpose}; @@ -171,14 +172,19 @@ impl LightningEventHandler { log::error!("{:?}", e); return; } + let platform = self.platform.clone(); let persister = self.persister.clone(); spawn(async move { - if let Err(e) = persister - .add_funding_tx_to_sql(user_channel_id, funding_txid.to_string(), channel_value_satoshis) + let current_block = platform.coin.current_block().compat().await.unwrap_or_default(); + persister + .add_funding_tx_to_sql( + user_channel_id, + funding_txid.to_string(), + channel_value_satoshis, + current_block, + ) .await - { - log::error!("{}", e); - } + .error_log(); }); } @@ -257,13 +263,55 @@ impl LightningEventHandler { reason ); let persister = self.persister.clone(); + let platform = self.platform.clone(); // Todo: Handle inbound channels closure case after updating to latest version of rust-lightning // as it has a new OpenChannelRequest event where we can give an inbound channel a user_channel_id // other than 0 in sql if user_channel_id != 0 { spawn(async move { - if let Err(e) = persister.update_channel_to_closed(user_channel_id, reason).await { - log::error!("{}", e); + persister + .update_channel_to_closed(user_channel_id, reason) + .await + .error_log(); + if let Ok(Some(channel_details)) = persister + .get_channel_from_sql(user_channel_id) + .await + .error_log_passthrough() + { + if let Some(tx_id) = channel_details.funding_tx { + if let Ok(tx_hash) = H256Json::from_str(&tx_id).error_log_passthrough() { + if let Ok(funding_tx_bytes) = platform + .coin + .as_ref() + .rpc_client + .get_transaction_bytes(&tx_hash) + .compat() + .await + .error_log_passthrough() + { + if let Ok(TransactionEnum::UtxoTx(closing_tx)) = platform + .coin + .wait_for_tx_spend( + &funding_tx_bytes.into_vec(), + (now_ms() / 1000) + 3600, + channel_details.funding_generated_in_block.unwrap_or_default(), + &None, + ) + .compat() + .await + .error_log_passthrough() + { + persister + .add_closing_tx_to_sql( + user_channel_id, + closing_tx.hash().reversed().to_string(), + ) + .await + .error_log(); + } + } + } + } } }); } diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 75e2dc270c..be4c75fd9a 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -95,11 +95,12 @@ fn create_channels_history_table_sql(for_coin: &str) -> Result rpc_id INTEGER NOT NULL UNIQUE, channel_id VARCHAR(255) NOT NULL, counterparty_node_id VARCHAR(255) NOT NULL, - funding_tx VARCHAR(255) NOT NULL, - initial_balance INTEGER NOT NULL, - closing_tx VARCHAR(255) NOT NULL, - closing_balance INTEGER NOT NULL, - closure_reason TEXT NOT NULL, + funding_tx VARCHAR(255), + funding_value INTEGER, + funding_generated_in_block Integer, + closing_tx VARCHAR(255), + closing_balance INTEGER, + closure_reason TEXT, is_outbound INTEGER NOT NULL, is_public INTEGER NOT NULL, is_closed INTEGER NOT NULL @@ -114,7 +115,7 @@ fn insert_channel_sql(for_coin: &str) -> Result { let sql = "INSERT INTO ".to_owned() + &table_name - + " (rpc_id, channel_id, counterparty_node_id, funding_tx, initial_balance, closing_tx, closing_balance, closure_reason, is_outbound, is_public, is_closed) VALUES (?1, ?2, ?3, ?4, ?5, ?6,?7, ?8, ?9, ?10, ?11);"; + + " (rpc_id, channel_id, counterparty_node_id, is_outbound, is_public, is_closed) VALUES (?1, ?2, ?3, ?4, ?5, ?6);"; Ok(sql) } @@ -123,7 +124,7 @@ fn select_channel_from_table_by_rpc_id_sql(for_coin: &str) -> Result) -> Result(0)? as u64, channel_id: row.get(1)?, counterparty_node_id: row.get(2)?, - funding_tx: row.get(3)?, - initial_balance: row.get::<_, u32>(4)? as u64, - closing_tx: row.get(5)?, - closing_balance: row.get::<_, u32>(6)? as u64, - closure_reason: row.get(7)?, - is_outbound: row.get(8)?, - is_public: row.get(9)?, - is_closed: row.get(10)?, + funding_tx: row.get(3).ok(), + funding_value: row.get::<_, u32>(4).ok().map(|v| v as u64), + funding_generated_in_block: row.get::<_, u32>(5).ok().map(|v| v as u64), + closing_tx: row.get(6).ok(), + closing_balance: row.get::<_, u32>(7).ok().map(|b| b as u64), + closure_reason: row.get(8).ok(), + is_outbound: row.get(9)?, + is_public: row.get(10)?, + is_closed: row.get(11)?, }; Ok(channel_details) } @@ -158,7 +160,9 @@ fn update_funding_tx_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "UPDATE ".to_owned() + &table_name + " SET funding_tx = ?2, initial_balance = ?3 WHERE rpc_id = ?1;"; + let sql = "UPDATE ".to_owned() + + &table_name + + " SET funding_tx = ?2, funding_value = ?3, funding_generated_in_block = ?4 WHERE rpc_id = ?1;"; Ok(sql) } @@ -172,11 +176,20 @@ fn update_channel_to_closed_sql(for_coin: &str) -> Result { Ok(sql) } +fn update_closing_tx_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "UPDATE ".to_owned() + &table_name + " SET closing_tx = ?2 WHERE rpc_id = ?1;"; + + Ok(sql) +} + fn get_closed_channels_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "SELECT rpc_id, channel_id, counterparty_node_id, funding_tx, initial_balance, closing_tx, closing_balance, closure_reason, is_outbound, is_public, is_closed FROM ".to_owned() + &table_name + " WHERE is_closed = 1;"; + let sql = "SELECT rpc_id, channel_id, counterparty_node_id, funding_tx, funding_value, funding_generated_in_block, closing_tx, closing_balance, closure_reason, is_outbound, is_public, is_closed FROM ".to_owned() + &table_name + " WHERE is_closed = 1;"; Ok(sql) } @@ -559,11 +572,6 @@ impl SqlStorage for LightningPersister { let rpc_id = details.rpc_id.to_string(); let channel_id = details.channel_id; let counterparty_node_id = details.counterparty_node_id; - let funding_tx = details.funding_tx; - let initial_balance = details.initial_balance.to_string(); - let closing_tx = details.closing_tx; - let closing_balance = details.closing_balance.to_string(); - let closure_reason = details.closure_reason; let is_outbound = (details.is_outbound as i32).to_string(); let is_public = (details.is_public as i32).to_string(); let is_closed = (details.is_closed as i32).to_string(); @@ -572,11 +580,6 @@ impl SqlStorage for LightningPersister { rpc_id, channel_id, counterparty_node_id, - funding_tx, - initial_balance, - closing_tx, - closing_balance, - closure_reason, is_outbound, is_public, is_closed, @@ -621,13 +624,15 @@ impl SqlStorage for LightningPersister { &self, rpc_id: u64, funding_tx: String, - initial_balance: u64, + funding_value: u64, + funding_generated_in_block: u64, ) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); let rpc_id = rpc_id.to_string(); - let initial_balance = initial_balance.to_string(); + let funding_value = funding_value.to_string(); + let funding_generated_in_block = funding_generated_in_block.to_string(); - let params = [rpc_id, funding_tx, initial_balance]; + let params = [rpc_id, funding_tx, funding_value, funding_generated_in_block]; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { @@ -658,6 +663,23 @@ impl SqlStorage for LightningPersister { .await } + async fn add_closing_tx_to_sql(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error> { + let for_coin = self.storage_ticker.clone(); + let rpc_id = rpc_id.to_string(); + + let params = [rpc_id, closing_tx]; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + sql_transaction.execute(&update_closing_tx_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + async fn get_closed_channels(&self) -> Result, Self::Error> { let sql = get_closed_channels_sql(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); @@ -943,7 +965,6 @@ mod tests { 1, [0; 32], PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), - 2000, true, true, ); @@ -967,16 +988,19 @@ mod tests { 2, "04d3f3931b7644094bbdaadbf341183d35bb7251cea2c91f5058cee28b7c2f20".into(), 3000, + 50000, )) .unwrap(); - expected_channel_details.funding_tx = "04d3f3931b7644094bbdaadbf341183d35bb7251cea2c91f5058cee28b7c2f20".into(); - expected_channel_details.initial_balance = 3000; + expected_channel_details.funding_tx = + Some("04d3f3931b7644094bbdaadbf341183d35bb7251cea2c91f5058cee28b7c2f20".into()); + expected_channel_details.funding_value = Some(3000); + expected_channel_details.funding_generated_in_block = Some(50000); let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); block_on(persister.update_channel_to_closed(2, "the channel was cooperatively closed".into())).unwrap(); - expected_channel_details.closure_reason = "the channel was cooperatively closed".into(); + expected_channel_details.closure_reason = Some("the channel was cooperatively closed".into()); expected_channel_details.is_closed = true; let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); @@ -989,5 +1013,16 @@ mod tests { block_on(persister.update_channel_to_closed(1, "the channel was cooperatively closed".into())).unwrap(); let closed_channels = block_on(persister.get_closed_channels()).unwrap(); assert_eq!(closed_channels.len(), 2); + + block_on(persister.add_closing_tx_to_sql( + 2, + "04d3f3931b7644094bbdaadbf341183d35bb7251cea2c91f5058cee28b7c2f20".into(), + )) + .unwrap(); + expected_channel_details.closing_tx = + Some("04d3f3931b7644094bbdaadbf341183d35bb7251cea2c91f5058cee28b7c2f20".into()); + + let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details) } } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 1737533f41..e3861b72dd 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -39,11 +39,13 @@ pub struct SqlChannelDetails { pub rpc_id: u64, pub channel_id: String, pub counterparty_node_id: String, - pub funding_tx: String, - pub initial_balance: u64, - pub closing_tx: String, - pub closing_balance: u64, - pub closure_reason: String, + pub funding_tx: Option, + pub funding_value: Option, + pub closing_tx: Option, + pub closing_balance: Option, + pub closure_reason: Option, + #[serde(skip_serializing)] + pub funding_generated_in_block: Option, pub is_outbound: bool, pub is_public: bool, pub is_closed: bool, @@ -54,7 +56,6 @@ impl SqlChannelDetails { rpc_id: u64, channel_id: [u8; 32], counterparty_node_id: PublicKey, - initial_balance: u64, is_outbound: bool, is_public: bool, ) -> Self { @@ -62,11 +63,12 @@ impl SqlChannelDetails { rpc_id, channel_id: hex::encode(channel_id), counterparty_node_id: counterparty_node_id.to_string(), - funding_tx: "".into(), - initial_balance, - closing_tx: "".into(), - closing_balance: 0, - closure_reason: "".into(), + funding_tx: None, + funding_value: None, + funding_generated_in_block: None, + closing_tx: None, + closing_balance: None, + closure_reason: None, is_outbound, is_public, is_closed: false, @@ -93,10 +95,13 @@ pub trait SqlStorage { &self, rpc_id: u64, funding_tx: String, - initial_balance: u64, + funding_value: u64, + funding_generated_in_block: u64, ) -> Result<(), Self::Error>; async fn update_channel_to_closed(&self, rpc_id: u64, closure_reason: String) -> Result<(), Self::Error>; + async fn add_closing_tx_to_sql(&self, rpc_id: u64, closing_tx_: String) -> Result<(), Self::Error>; + async fn get_closed_channels(&self) -> Result, Self::Error>; } From ab8dc450d709a442dec0deeb601c2954177c61f4 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Mon, 21 Mar 2022 15:43:19 +0200 Subject: [PATCH 18/49] outbound channels history in sql completed --- mm2src/coins/lightning/ln_events.rs | 23 +++++ mm2src/coins/lightning_persister/src/lib.rs | 84 +++++++++++++++---- .../coins/lightning_persister/src/storage.rs | 19 ++++- 3 files changed, 109 insertions(+), 17 deletions(-) diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 58af2005f0..15da121a06 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -367,6 +367,29 @@ impl LightningEventHandler { return; }, }; + self.platform.broadcast_transaction(&spending_tx); + + for output in outputs { + let (closing_txid, claimed_balance) = match output { + SpendableOutputDescriptor::StaticOutput { outpoint, output } => { + (outpoint.txid.to_string(), output.value) + }, + SpendableOutputDescriptor::DelayedPaymentOutput(descriptor) => { + (descriptor.outpoint.txid.to_string(), descriptor.output.value) + }, + SpendableOutputDescriptor::StaticPaymentOutput(descriptor) => { + (descriptor.outpoint.txid.to_string(), descriptor.output.value) + }, + }; + let claiming_txid = spending_tx.txid().to_string(); + let persister = self.persister.clone(); + spawn(async move { + persister + .add_claiming_tx_to_sql(closing_txid, claiming_txid, claimed_balance) + .await + .error_log(); + }); + } } } diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index be4c75fd9a..a3237a3336 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -99,8 +99,9 @@ fn create_channels_history_table_sql(for_coin: &str) -> Result funding_value INTEGER, funding_generated_in_block Integer, closing_tx VARCHAR(255), - closing_balance INTEGER, closure_reason TEXT, + claiming_tx VARCHAR(255), + claimed_balance INTEGER, is_outbound INTEGER NOT NULL, is_public INTEGER NOT NULL, is_closed INTEGER NOT NULL @@ -124,7 +125,7 @@ fn select_channel_from_table_by_rpc_id_sql(for_coin: &str) -> Result) -> Result(4).ok().map(|v| v as u64), funding_generated_in_block: row.get::<_, u32>(5).ok().map(|v| v as u64), closing_tx: row.get(6).ok(), - closing_balance: row.get::<_, u32>(7).ok().map(|b| b as u64), - closure_reason: row.get(8).ok(), - is_outbound: row.get(9)?, - is_public: row.get(10)?, - is_closed: row.get(11)?, + closure_reason: row.get(7).ok(), + claiming_tx: row.get(8).ok(), + claimed_balance: row.get::<_, u32>(9).ok().map(|b| b as u64), + is_outbound: row.get(10)?, + is_public: row.get(11)?, + is_closed: row.get(12)?, }; Ok(channel_details) } @@ -189,7 +191,16 @@ fn get_closed_channels_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "SELECT rpc_id, channel_id, counterparty_node_id, funding_tx, funding_value, funding_generated_in_block, closing_tx, closing_balance, closure_reason, is_outbound, is_public, is_closed FROM ".to_owned() + &table_name + " WHERE is_closed = 1;"; + let sql = "SELECT rpc_id, channel_id, counterparty_node_id, funding_tx, funding_value, funding_generated_in_block, closing_tx, closure_reason, claiming_tx, claimed_balance, is_outbound, is_public, is_closed FROM ".to_owned() + &table_name + " WHERE is_closed = 1;"; + + Ok(sql) +} + +fn update_claiming_tx_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "UPDATE ".to_owned() + &table_name + " SET claiming_tx = ?2, claimed_balance = ?3 WHERE closing_tx = ?1;"; Ok(sql) } @@ -508,7 +519,11 @@ impl FileSystemStorage for LightningPersister { async fn save_network_graph(&self, network_graph: Arc) -> Result<(), Self::Error> { let path = self.network_graph_path(); async_blocking(move || { - let file = fs::OpenOptions::new().create(true).write(true).open(path)?; + let file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path)?; network_graph.write(&mut BufWriter::new(file)) }) .await @@ -531,7 +546,11 @@ impl FileSystemStorage for LightningPersister { let path = self.scorer_path(); async_blocking(move || { let scorer = scorer.lock().unwrap(); - let file = fs::OpenOptions::new().create(true).write(true).open(path)?; + let file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path)?; scorer.write(&mut BufWriter::new(file)) }) .await @@ -693,6 +712,28 @@ impl SqlStorage for LightningPersister { }) .await } + + async fn add_claiming_tx_to_sql( + &self, + closing_tx: String, + claiming_tx: String, + claimed_balance: u64, + ) -> Result<(), Self::Error> { + let for_coin = self.storage_ticker.clone(); + let claimed_balance = claimed_balance.to_string(); + + let params = [closing_tx, claiming_tx, claimed_balance]; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + sql_transaction.execute(&update_claiming_tx_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } } #[cfg(test)] @@ -986,13 +1027,13 @@ mod tests { block_on(persister.add_funding_tx_to_sql( 2, - "04d3f3931b7644094bbdaadbf341183d35bb7251cea2c91f5058cee28b7c2f20".into(), + "9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into(), 3000, 50000, )) .unwrap(); expected_channel_details.funding_tx = - Some("04d3f3931b7644094bbdaadbf341183d35bb7251cea2c91f5058cee28b7c2f20".into()); + Some("9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into()); expected_channel_details.funding_value = Some(3000); expected_channel_details.funding_generated_in_block = Some(50000); @@ -1016,13 +1057,26 @@ mod tests { block_on(persister.add_closing_tx_to_sql( 2, - "04d3f3931b7644094bbdaadbf341183d35bb7251cea2c91f5058cee28b7c2f20".into(), + "5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into(), )) .unwrap(); expected_channel_details.closing_tx = - Some("04d3f3931b7644094bbdaadbf341183d35bb7251cea2c91f5058cee28b7c2f20".into()); + Some("5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into()); let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); - assert_eq!(expected_channel_details, actual_channel_details) + assert_eq!(expected_channel_details, actual_channel_details); + + block_on(persister.add_claiming_tx_to_sql( + "5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into(), + "97f061634a4a7b0b0c2b95648f86b1c39b95e0cf5073f07725b7143c095b612a".into(), + 2000, + )) + .unwrap(); + expected_channel_details.claiming_tx = + Some("97f061634a4a7b0b0c2b95648f86b1c39b95e0cf5073f07725b7143c095b612a".into()); + expected_channel_details.claimed_balance = Some(2000); + + let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); } } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index e3861b72dd..168b9ebb81 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -39,11 +39,18 @@ pub struct SqlChannelDetails { pub rpc_id: u64, pub channel_id: String, pub counterparty_node_id: String, + #[serde(skip_serializing_if = "Option::is_none")] pub funding_tx: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub funding_value: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub closing_tx: Option, - pub closing_balance: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub closure_reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub claiming_tx: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub claimed_balance: Option, #[serde(skip_serializing)] pub funding_generated_in_block: Option, pub is_outbound: bool, @@ -67,8 +74,9 @@ impl SqlChannelDetails { funding_value: None, funding_generated_in_block: None, closing_tx: None, - closing_balance: None, closure_reason: None, + claiming_tx: None, + claimed_balance: None, is_outbound, is_public, is_closed: false, @@ -104,4 +112,11 @@ pub trait SqlStorage { async fn add_closing_tx_to_sql(&self, rpc_id: u64, closing_tx_: String) -> Result<(), Self::Error>; async fn get_closed_channels(&self) -> Result, Self::Error>; + + async fn add_claiming_tx_to_sql( + &self, + closing_tx: String, + claiming_tx: String, + claimed_balance: u64, + ) -> Result<(), Self::Error>; } From 5eb93068530098092f6c809bbbdcd0629bed8c90 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Tue, 22 Mar 2022 23:13:25 +0200 Subject: [PATCH 19/49] Payments in sql WIP: add/get payment to/from sql --- Cargo.lock | 1 + mm2src/coins/lightning.rs | 30 +-- mm2src/coins/lightning/ln_events.rs | 2 +- mm2src/coins/lightning_persister/Cargo.toml | 1 + mm2src/coins/lightning_persister/src/lib.rs | 173 +++++++++++++++++- .../coins/lightning_persister/src/storage.rs | 43 +++++ 6 files changed, 223 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ccb49ca76a..06defcd585 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3009,6 +3009,7 @@ dependencies = [ "bitcoin", "common", "db_common", + "derive_more", "hex 0.4.3", "libc", "lightning", diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index df966a020e..224bf4526f 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -34,7 +34,7 @@ use lightning::chain::keysinterface::KeysInterface; use lightning::chain::keysinterface::KeysManager; use lightning::chain::Access; use lightning::ln::channelmanager::{ChannelDetails, MIN_FINAL_CLTV_EXPIRY}; -use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; +use lightning::ln::{PaymentHash, PaymentPreimage}; use lightning::routing::network_graph::{NetGraphMsgHandler, NetworkGraph}; use lightning::routing::scoring::Scorer; use lightning::util::config::UserConfig; @@ -42,7 +42,8 @@ use lightning_background_processor::BackgroundProcessor; use lightning_invoice::payment; use lightning_invoice::utils::{create_invoice_from_channelmanager, DefaultRouter}; use lightning_invoice::Invoice; -use lightning_persister::storage::{FileSystemStorage, NodesAddressesMapShared, SqlChannelDetails, SqlStorage}; +use lightning_persister::storage::{FileSystemStorage, HTLCStatus, NodesAddressesMapShared, PaymentInfo, + SqlChannelDetails, SqlStorage}; use lightning_persister::LightningPersister; use ln_conf::{ChannelOptions, LightningCoinConf, LightningProtocolConf, PlatformCoinConfirmations}; use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelError, CloseChannelResult, @@ -106,23 +107,6 @@ impl fmt::Debug for LightningCoin { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "LightningCoin {{ conf: {:?} }}", self.conf) } } -#[derive(Clone, Serialize)] -#[serde(rename_all = "lowercase")] -pub enum HTLCStatus { - Pending, - Succeeded, - Failed, -} - -#[derive(Clone)] -pub struct PaymentInfo { - pub preimage: Option, - pub secret: Option, - pub status: HTLCStatus, - pub amt_msat: Option, - pub fee_paid_msat: Option, -} - impl LightningCoin { fn platform_coin(&self) -> &UtxoStandardCoin { &self.platform.coin } @@ -153,9 +137,9 @@ impl LightningCoin { Ok((payment_hash, PaymentInfo { preimage: None, secret: payment_secret, - status: HTLCStatus::Pending, amt_msat: invoice.amount_milli_satoshis(), fee_paid_msat: None, + status: HTLCStatus::Pending, })) } @@ -180,9 +164,9 @@ impl LightningCoin { Ok((payment_hash, PaymentInfo { preimage: Some(payment_preimage), secret: None, - status: HTLCStatus::Pending, amt_msat: Some(amount_msat), fee_paid_msat: None, + status: HTLCStatus::Pending, })) } } @@ -1041,17 +1025,17 @@ pub struct ListPaymentsReq { #[derive(Serialize)] pub struct PaymentInfoForRPC { - status: HTLCStatus, amount_in_msat: Option, fee_paid_msat: Option, + status: HTLCStatus, } impl From for PaymentInfoForRPC { fn from(info: PaymentInfo) -> Self { PaymentInfoForRPC { - status: info.status, amount_in_msat: info.amt_msat, fee_paid_msat: info.fee_paid_msat, + status: info.status, } } } diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 15da121a06..390538e098 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -226,9 +226,9 @@ impl LightningEventHandler { e.insert(PaymentInfo { preimage: Some(payment_preimage), secret: payment_secret, - status, amt_msat: Some(amt), fee_paid_msat: None, + status, }); }, } diff --git a/mm2src/coins/lightning_persister/Cargo.toml b/mm2src/coins/lightning_persister/Cargo.toml index da839a950a..48d5595d4c 100644 --- a/mm2src/coins/lightning_persister/Cargo.toml +++ b/mm2src/coins/lightning_persister/Cargo.toml @@ -14,6 +14,7 @@ async-trait = "0.1" bitcoin = "0.27.1" common = { path = "../../common" } db_common = { path = "../../db_common" } +derive_more = "0.99" hex = "0.4.2" lightning = "0.0.104" libc = "0.2" diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index a3237a3336..e4ebb227d1 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -13,7 +13,8 @@ extern crate lightning; extern crate secp256k1; extern crate serde_json; -use crate::storage::{FileSystemStorage, NodesAddressesMap, NodesAddressesMapShared, SqlChannelDetails, SqlStorage}; +use crate::storage::{FileSystemStorage, HTLCStatus, NodesAddressesMap, NodesAddressesMapShared, PaymentInfo, + SqlChannelDetails, SqlStorage}; use crate::util::DiskWriteable; use async_trait::async_trait; use bitcoin::blockdata::constants::genesis_block; @@ -22,7 +23,7 @@ use bitcoin::hashes::hex::{FromHex, ToHex}; use bitcoin::Network; use common::async_blocking; use common::fs::check_dir_operations; -use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, NO_PARAMS}; +use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, ToSql, NO_PARAMS}; use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use lightning::chain; use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator}; @@ -31,12 +32,14 @@ use lightning::chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate}; use lightning::chain::keysinterface::{KeysInterface, Sign}; use lightning::chain::transaction::OutPoint; use lightning::ln::channelmanager::ChannelManager; +use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; use lightning::routing::network_graph::NetworkGraph; use lightning::routing::scoring::Scorer; use lightning::util::logger::Logger; use lightning::util::ser::{Readable, ReadableArgs, Writeable}; use secp256k1::PublicKey; use std::collections::HashMap; +use std::convert::TryInto; use std::fs; use std::io::{BufReader, BufWriter, Cursor, Error}; use std::net::SocketAddr; @@ -84,6 +87,8 @@ where fn channels_history_table(ticker: &str) -> String { ticker.to_owned() + "_channels_history" } +fn payments_history_table(ticker: &str) -> String { ticker.to_owned() + "_payments_history" } + fn create_channels_history_table_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; @@ -110,6 +115,26 @@ fn create_channels_history_table_sql(for_coin: &str) -> Result Ok(sql) } +fn create_payments_history_table_sql(for_coin: &str) -> Result { + let table_name = payments_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "CREATE TABLE IF NOT EXISTS ".to_owned() + + &table_name + + " ( + id INTEGER NOT NULL PRIMARY KEY, + payment_hash VARCHAR(255) NOT NULL UNIQUE, + preimage VARCHAR(255), + secret VARCHAR(255), + amount_msat INTEGER, + fee_paid_msat INTEGER, + is_outbound INTEGER NOT NULL, + status VARCHAR(255) NOT NULL + );"; + + Ok(sql) +} + fn insert_channel_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; @@ -121,6 +146,17 @@ fn insert_channel_sql(for_coin: &str) -> Result { Ok(sql) } +fn insert_payment_sql(for_coin: &str) -> Result { + let table_name = payments_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "INSERT INTO ".to_owned() + + &table_name + + " (payment_hash, preimage, secret, amount_msat, fee_paid_msat, is_outbound, status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);"; + + Ok(sql) +} + fn select_channel_from_table_by_rpc_id_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; @@ -130,6 +166,17 @@ fn select_channel_from_table_by_rpc_id_sql(for_coin: &str) -> Result Result { + let table_name = payments_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "SELECT preimage, secret, amount_msat, fee_paid_msat, status FROM ".to_owned() + + &table_name + + " WHERE payment_hash=?1;"; + + Ok(sql) +} + fn channel_details_from_row(row: &Row<'_>) -> Result { let channel_details = SqlChannelDetails { rpc_id: row.get::<_, u32>(0)? as u64, @@ -149,6 +196,31 @@ fn channel_details_from_row(row: &Row<'_>) -> Result) -> Result { + let payment_info = PaymentInfo { + preimage: row.get::<_, String>(0).ok().map(|p| { + PaymentPreimage( + hex::decode(p) + .expect("Preimage decoding should not fail!") + .try_into() + .expect("String should be 64 characters!"), + ) + }), + secret: row.get::<_, String>(1).ok().map(|s| { + PaymentSecret( + hex::decode(s) + .expect("Secret decoding should not fail!") + .try_into() + .expect("String should be 64 characters!"), + ) + }), + amt_msat: row.get::<_, u32>(2).ok().map(|v| v as u64), + fee_paid_msat: row.get::<_, u32>(3).ok().map(|v| v as u64), + status: HTLCStatus::from_str(&row.get::<_, String>(4)?)?, + }; + Ok(payment_info) +} + fn get_last_channel_rpc_id_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; @@ -564,9 +636,11 @@ impl SqlStorage for LightningPersister { async fn init_sql(&self) -> Result<(), Self::Error> { let sqlite_connection = self.sqlite_connection.clone(); let sql_channels_history = create_channels_history_table_sql(self.storage_ticker.as_str())?; + let sql_payments_history = create_payments_history_table_sql(self.storage_ticker.as_str())?; async_blocking(move || { let conn = sqlite_connection.lock().unwrap(); conn.execute(&sql_channels_history, NO_PARAMS).map(|_| ())?; + conn.execute(&sql_payments_history, NO_PARAMS).map(|_| ())?; Ok(()) }) .await @@ -575,13 +649,17 @@ impl SqlStorage for LightningPersister { async fn is_sql_initialized(&self) -> Result { let channels_history_table = channels_history_table(self.storage_ticker.as_str()); validate_table_name(&channels_history_table)?; + let payments_history_table = payments_history_table(self.storage_ticker.as_str()); + validate_table_name(&payments_history_table)?; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { let conn = sqlite_connection.lock().unwrap(); let channels_history_initialized = query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [channels_history_table], string_from_row)?; - Ok(channels_history_initialized.is_some()) + let payments_history_initialized = + query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [payments_history_table], string_from_row)?; + Ok(channels_history_initialized.is_some() && payments_history_initialized.is_some()) }) .await } @@ -615,6 +693,41 @@ impl SqlStorage for LightningPersister { .await } + async fn add_payment_to_sql( + &self, + hash: PaymentHash, + info: PaymentInfo, + is_outbound: bool, + ) -> Result<(), Self::Error> { + let for_coin = self.storage_ticker.clone(); + let payment_hash = hex::encode(hash.0); + let preimage = info.preimage.map(|p| hex::encode(p.0)); + let secret = info.secret.map(|s| hex::encode(s.0)); + let amount_msat = info.amt_msat.map(|a| a as u32); + let fee_paid_msat = info.fee_paid_msat.map(|f| f as u32); + let is_outbound = is_outbound as i32; + let status = info.status.to_string(); + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let params = [ + &payment_hash as &dyn ToSql, + &preimage as &dyn ToSql, + &secret as &dyn ToSql, + &amount_msat as &dyn ToSql, + &fee_paid_msat as &dyn ToSql, + &is_outbound as &dyn ToSql, + &status as &dyn ToSql, + ]; + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + sql_transaction.execute(&insert_payment_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + async fn get_channel_from_sql(&self, rpc_id: u64) -> Result, Self::Error> { let params = [rpc_id.to_string()]; let sql = select_channel_from_table_by_rpc_id_sql(self.storage_ticker.as_str())?; @@ -627,6 +740,18 @@ impl SqlStorage for LightningPersister { .await } + async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error> { + let params = [hex::encode(hash.0)]; + let sql = select_payment_from_table_by_hash_sql(self.storage_ticker.as_str())?; + let sqlite_connection = self.sqlite_connection.clone(); + + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + query_single_row(&conn, &sql, params, payment_info_from_row) + }) + .await + } + async fn get_last_channel_rpc_id(&self) -> Result { let sql = get_last_channel_rpc_id_sql(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); @@ -1079,4 +1204,46 @@ mod tests { let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); } + + #[test] + fn test_add_get_payment_sql() { + let persister = LightningPersister::new( + "add_get_payment".into(), + PathBuf::from("test_filesystem_persister"), + None, + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + + block_on(persister.init_sql()).unwrap(); + + let payment = block_on(persister.get_payment_from_sql(PaymentHash([0; 32]))).unwrap(); + assert!(payment.is_none()); + + let mut expected_payment_info = PaymentInfo { + preimage: Some(PaymentPreimage([2; 32])), + secret: Some(PaymentSecret([3; 32])), + amt_msat: Some(2000), + fee_paid_msat: Some(100), + status: HTLCStatus::Failed, + }; + block_on(persister.add_payment_to_sql(PaymentHash([0; 32]), expected_payment_info.clone(), true)).unwrap(); + + let actual_payment_info = block_on(persister.get_payment_from_sql(PaymentHash([0; 32]))) + .unwrap() + .unwrap(); + assert_eq!(expected_payment_info, actual_payment_info); + + // must fail because we are adding payment with the same hash + block_on(persister.add_payment_to_sql(PaymentHash([0; 32]), expected_payment_info.clone(), true)).unwrap_err(); + + expected_payment_info.secret = None; + expected_payment_info.amt_msat = None; + expected_payment_info.status = HTLCStatus::Succeeded; + block_on(persister.add_payment_to_sql(PaymentHash([1; 32]), expected_payment_info.clone(), true)).unwrap(); + + let actual_payment_info = block_on(persister.get_payment_from_sql(PaymentHash([1; 32]))) + .unwrap() + .unwrap(); + assert_eq!(expected_payment_info, actual_payment_info); + } } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 168b9ebb81..127f9ba77c 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -1,5 +1,8 @@ use async_trait::async_trait; use bitcoin::Network; +use db_common::sqlite::rusqlite::types::FromSqlError; +use derive_more::Display; +use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; use lightning::routing::network_graph::NetworkGraph; use lightning::routing::scoring::Scorer; use parking_lot::Mutex as PaMutex; @@ -7,6 +10,7 @@ use secp256k1::PublicKey; use serde::Serialize; use std::collections::HashMap; use std::net::SocketAddr; +use std::str::FromStr; use std::sync::{Arc, Mutex}; pub type NodesAddressesMap = HashMap; @@ -84,6 +88,36 @@ impl SqlChannelDetails { } } +#[derive(Clone, Debug, Display, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum HTLCStatus { + Pending, + Succeeded, + Failed, +} + +impl FromStr for HTLCStatus { + type Err = FromSqlError; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "pending" => Ok(HTLCStatus::Pending), + "succeeded" => Ok(HTLCStatus::Succeeded), + "failed" => Ok(HTLCStatus::Failed), + _ => Err(FromSqlError::InvalidType), + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct PaymentInfo { + pub preimage: Option, + pub secret: Option, + pub amt_msat: Option, + pub fee_paid_msat: Option, + pub status: HTLCStatus, +} + #[async_trait] pub trait SqlStorage { type Error; @@ -95,8 +129,17 @@ pub trait SqlStorage { async fn add_channel_to_sql(&self, details: SqlChannelDetails) -> Result<(), Self::Error>; + async fn add_payment_to_sql( + &self, + hash: PaymentHash, + info: PaymentInfo, + is_outbound: bool, + ) -> Result<(), Self::Error>; + async fn get_channel_from_sql(&self, rpc_id: u64) -> Result, Self::Error>; + async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error>; + async fn get_last_channel_rpc_id(&self) -> Result; async fn add_funding_tx_to_sql( From 61c9493a653e8dab1af1ddfe9988311ecb676c4e Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Wed, 23 Mar 2022 01:55:56 +0200 Subject: [PATCH 20/49] Payments in sql completed --- mm2src/coins/lightning.rs | 84 +++++------- mm2src/coins/lightning/ln_errors.rs | 23 +++- mm2src/coins/lightning/ln_events.rs | 99 ++++++++------ mm2src/coins/lightning_persister/src/lib.rs | 121 ++++++++++++++---- .../coins/lightning_persister/src/storage.rs | 20 ++- 5 files changed, 223 insertions(+), 124 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 224bf4526f..15cc50d7f9 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -42,7 +42,7 @@ use lightning_background_processor::BackgroundProcessor; use lightning_invoice::payment; use lightning_invoice::utils::{create_invoice_from_channelmanager, DefaultRouter}; use lightning_invoice::Invoice; -use lightning_persister::storage::{FileSystemStorage, HTLCStatus, NodesAddressesMapShared, PaymentInfo, +use lightning_persister::storage::{FileSystemStorage, HTLCStatus, NodesAddressesMapShared, PaymentInfo, PaymentType, SqlChannelDetails, SqlStorage}; use lightning_persister::LightningPersister; use ln_conf::{ChannelOptions, LightningCoinConf, LightningProtocolConf, PlatformCoinConfirmations}; @@ -72,8 +72,6 @@ use std::sync::{Arc, Mutex}; type Router = DefaultRouter, Arc>; type InvoicePayer = payment::InvoicePayer, Router, Arc>, Arc, E>; -type PaymentsMap = HashMap; -type PaymentsMapShared = Arc>; #[derive(Clone)] pub struct LightningCoin { @@ -94,10 +92,6 @@ pub struct LightningCoin { pub invoice_payer: Arc>>, /// The lightning node persister that takes care of writing/reading data from storage. pub persister: Arc, - /// The mutex storing the inbound payments info. - pub inbound_payments: PaymentsMapShared, - /// The mutex storing the outbound payments info. - pub outbound_payments: PaymentsMapShared, /// The mutex storing the addresses of the nodes that the lightning node has open channels with, /// these addresses are used for reconnecting. pub open_channels_nodes: NodesAddressesMapShared, @@ -128,19 +122,20 @@ impl LightningCoin { }) } - fn pay_invoice(&self, invoice: Invoice) -> SendPaymentResult<(PaymentHash, PaymentInfo)> { + fn pay_invoice(&self, invoice: Invoice) -> SendPaymentResult { self.invoice_payer .pay_invoice(&invoice) .map_to_mm(|e| SendPaymentError::PaymentError(format!("{:?}", e)))?; let payment_hash = PaymentHash((*invoice.payment_hash()).into_inner()); let payment_secret = Some(*invoice.payment_secret()); - Ok((payment_hash, PaymentInfo { + Ok(PaymentInfo { + payment_hash, preimage: None, secret: payment_secret, amt_msat: invoice.amount_milli_satoshis(), fee_paid_msat: None, status: HTLCStatus::Pending, - })) + }) } fn keysend( @@ -148,7 +143,7 @@ impl LightningCoin { destination: PublicKey, amount_msat: u64, final_cltv_expiry_delta: u32, - ) -> SendPaymentResult<(PaymentHash, PaymentInfo)> { + ) -> SendPaymentResult { if final_cltv_expiry_delta < MIN_FINAL_CLTV_EXPIRY { return MmError::err(SendPaymentError::CLTVExpiryError( final_cltv_expiry_delta, @@ -161,13 +156,14 @@ impl LightningCoin { .map_to_mm(|e| SendPaymentError::PaymentError(format!("{:?}", e)))?; let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0).into_inner()); - Ok((payment_hash, PaymentInfo { + Ok(PaymentInfo { + payment_hash, preimage: Some(payment_preimage), secret: None, amt_msat: Some(amount_msat), fee_paid_msat: None, status: HTLCStatus::Pending, - })) + }) } } @@ -549,9 +545,6 @@ pub async fn start_lightning( ) .await?; - let inbound_payments = Arc::new(PaMutex::new(HashMap::new())); - let outbound_payments = Arc::new(PaMutex::new(HashMap::new())); - // Initialize the event handler let event_handler = Arc::new(ln_events::LightningEventHandler::new( // It's safe to use unwrap here for now until implementing Native Client for Lightning @@ -559,8 +552,6 @@ pub async fn start_lightning( channel_manager.clone(), keys_manager.clone(), persister.clone(), - inbound_payments.clone(), - outbound_payments.clone(), )); // Initialize routing Scorer @@ -624,8 +615,6 @@ pub async fn start_lightning( keys_manager, invoice_payer, persister, - inbound_payments, - outbound_payments, open_channels_nodes, }) } @@ -1003,7 +992,7 @@ pub async fn send_payment(ctx: MmArc, req: SendPaymentReq) -> SendPaymentResult< node_pubkey )); } - let (payment_hash, payment_info) = match req.payment { + let payment_info = match req.payment { Payment::Invoice { invoice } => ln_coin.pay_invoice(invoice.into())?, Payment::Keysend { destination, @@ -1011,10 +1000,12 @@ pub async fn send_payment(ctx: MmArc, req: SendPaymentReq) -> SendPaymentResult< expiry, } => ln_coin.keysend(destination.into(), amount_in_msat, expiry)?, }; - let mut outbound_payments = ln_coin.outbound_payments.lock(); - outbound_payments.insert(payment_hash, payment_info); + ln_coin + .persister + .add_or_update_payment_in_sql(payment_info.clone(), PaymentType::OutboundPayment) + .await?; Ok(SendPaymentResponse { - payment_hash: payment_hash.0.into(), + payment_hash: payment_info.payment_hash.0.into(), }) } @@ -1042,8 +1033,8 @@ impl From for PaymentInfoForRPC { #[derive(Serialize)] pub struct ListPaymentsResponse { - pub inbound_payments: HashMap, - pub outbound_payments: HashMap, + pub inbound_payments: Vec, + pub outbound_payments: Vec, } pub async fn list_payments(ctx: MmArc, req: ListPaymentsReq) -> ListPaymentsResult { @@ -1053,18 +1044,18 @@ pub async fn list_payments(ctx: MmArc, req: ListPaymentsReq) -> ListPaymentsResu _ => return MmError::err(ListPaymentsError::UnsupportedCoin(coin.ticker().to_string())), }; let inbound_payments = ln_coin - .inbound_payments - .lock() - .clone() + .persister + .get_inbound_payments() + .await? .into_iter() - .map(|(hash, info)| (hash.0.into(), info.into())) + .map(|(info, _)| info.into()) .collect(); let outbound_payments = ln_coin - .outbound_payments - .lock() - .clone() + .persister + .get_outbound_payments() + .await? .into_iter() - .map(|(hash, info)| (hash.0.into(), info.into())) + .map(|(info, _)| info.into()) .collect(); Ok(ListPaymentsResponse { @@ -1079,14 +1070,6 @@ pub struct GetPaymentDetailsRequest { pub payment_hash: H256Json, } -#[derive(Serialize)] -enum PaymentType { - #[serde(rename = "Outbound Payment")] - OutboundPayment, - #[serde(rename = "Inbound Payment")] - InboundPayment, -} - #[derive(Serialize)] pub struct GetPaymentDetailsResponse { payment_type: PaymentType, @@ -1103,17 +1086,14 @@ pub async fn get_payment_details( _ => return MmError::err(GetPaymentDetailsError::UnsupportedCoin(coin.ticker().to_string())), }; - if let Some(payment_info) = ln_coin.outbound_payments.lock().get(&PaymentHash(req.payment_hash.0)) { - return Ok(GetPaymentDetailsResponse { - payment_type: PaymentType::OutboundPayment, - payment_details: payment_info.clone().into(), - }); - } - - if let Some(payment_info) = ln_coin.inbound_payments.lock().get(&PaymentHash(req.payment_hash.0)) { + if let Some((payment_info, payment_type)) = ln_coin + .persister + .get_payment_from_sql(PaymentHash(req.payment_hash.0)) + .await? + { return Ok(GetPaymentDetailsResponse { - payment_type: PaymentType::InboundPayment, - payment_details: payment_info.clone().into(), + payment_type, + payment_details: payment_info.into(), }); } diff --git a/mm2src/coins/lightning/ln_errors.rs b/mm2src/coins/lightning/ln_errors.rs index ea97157599..5aeb820300 100644 --- a/mm2src/coins/lightning/ln_errors.rs +++ b/mm2src/coins/lightning/ln_errors.rs @@ -329,6 +329,8 @@ pub enum SendPaymentError { PaymentError(String), #[display(fmt = "Final cltv expiry delta {} is below the required minimum of {}", _0, _1)] CLTVExpiryError(u32, u32), + #[display(fmt = "SQL error {}", _0)] + SqlError(String), } impl HttpStatusCode for SendPaymentError { @@ -338,7 +340,8 @@ impl HttpStatusCode for SendPaymentError { SendPaymentError::NoSuchCoin(_) => StatusCode::PRECONDITION_REQUIRED, SendPaymentError::PaymentError(_) | SendPaymentError::NoRouteFound(_) - | SendPaymentError::CLTVExpiryError(_, _) => StatusCode::INTERNAL_SERVER_ERROR, + | SendPaymentError::CLTVExpiryError(_, _) + | SendPaymentError::SqlError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -351,6 +354,10 @@ impl From for SendPaymentError { } } +impl From for SendPaymentError { + fn from(err: SqlError) -> SendPaymentError { SendPaymentError::SqlError(err.to_string()) } +} + #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] pub enum ListPaymentsError { @@ -358,6 +365,8 @@ pub enum ListPaymentsError { UnsupportedCoin(String), #[display(fmt = "No such coin {}", _0)] NoSuchCoin(String), + #[display(fmt = "SQL error {}", _0)] + SqlError(String), } impl HttpStatusCode for ListPaymentsError { @@ -365,6 +374,7 @@ impl HttpStatusCode for ListPaymentsError { match self { ListPaymentsError::UnsupportedCoin(_) => StatusCode::BAD_REQUEST, ListPaymentsError::NoSuchCoin(_) => StatusCode::PRECONDITION_REQUIRED, + ListPaymentsError::SqlError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -377,6 +387,10 @@ impl From for ListPaymentsError { } } +impl From for ListPaymentsError { + fn from(err: SqlError) -> ListPaymentsError { ListPaymentsError::SqlError(err.to_string()) } +} + #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] pub enum GetPaymentDetailsError { @@ -386,6 +400,8 @@ pub enum GetPaymentDetailsError { NoSuchCoin(String), #[display(fmt = "Payment with hash: {:?} is not found", _0)] NoSuchPayment(H256Json), + #[display(fmt = "SQL error {}", _0)] + SqlError(String), } impl HttpStatusCode for GetPaymentDetailsError { @@ -394,6 +410,7 @@ impl HttpStatusCode for GetPaymentDetailsError { GetPaymentDetailsError::UnsupportedCoin(_) => StatusCode::BAD_REQUEST, GetPaymentDetailsError::NoSuchCoin(_) => StatusCode::PRECONDITION_REQUIRED, GetPaymentDetailsError::NoSuchPayment(_) => StatusCode::NOT_FOUND, + GetPaymentDetailsError::SqlError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -406,6 +423,10 @@ impl From for GetPaymentDetailsError { } } +impl From for GetPaymentDetailsError { + fn from(err: SqlError) -> GetPaymentDetailsError { GetPaymentDetailsError::SqlError(err.to_string()) } +} + #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] pub enum CloseChannelError { diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 390538e098..9951d655d9 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -11,7 +11,6 @@ use lightning::util::events::{Event, EventHandler, PaymentPurpose}; use rand::Rng; use script::{Builder, SignatureVersion}; use secp256k1::Secp256k1; -use std::collections::hash_map::Entry; use std::convert::TryFrom; use std::sync::Arc; use utxo_signer::with_key_pair::sign_tx; @@ -21,8 +20,6 @@ pub struct LightningEventHandler { channel_manager: Arc, keys_manager: Arc, persister: Arc, - inbound_payments: PaymentsMapShared, - outbound_payments: PaymentsMapShared, } impl EventHandler for LightningEventHandler { @@ -45,7 +42,7 @@ impl EventHandler for LightningEventHandler { fee_paid_msat, .. } => self.handle_payment_sent(*payment_preimage, *payment_hash, *fee_paid_msat), - Event::PaymentFailed { payment_hash, .. } => self.handle_payment_failed(payment_hash), + Event::PaymentFailed { payment_hash, .. } => self.handle_payment_failed(*payment_hash), Event::PendingHTLCsForwardable { time_forwardable } => self.handle_pending_htlcs_forwards(*time_forwardable), Event::SpendableOutputs { outputs } => self.handle_spendable_outputs(outputs), // Todo: an RPC for total amount earned @@ -128,16 +125,12 @@ impl LightningEventHandler { channel_manager: Arc, keys_manager: Arc, persister: Arc, - inbound_payments: PaymentsMapShared, - outbound_payments: PaymentsMapShared, ) -> Self { LightningEventHandler { platform, channel_manager, keys_manager, persister, - inbound_payments, - outbound_payments, } } @@ -214,24 +207,23 @@ impl LightningEventHandler { }, false => HTLCStatus::Failed, }; - let mut payments = self.inbound_payments.lock(); - match payments.entry(payment_hash) { - Entry::Occupied(mut e) => { - let payment = e.get_mut(); - payment.status = status; - payment.preimage = Some(payment_preimage); - payment.secret = payment_secret; - }, - Entry::Vacant(e) => { - e.insert(PaymentInfo { - preimage: Some(payment_preimage), - secret: payment_secret, - amt_msat: Some(amt), - fee_paid_msat: None, - status, - }); - }, - } + let payment_info = PaymentInfo { + payment_hash, + preimage: Some(payment_preimage), + secret: payment_secret, + amt_msat: Some(amt), + fee_paid_msat: None, + status, + }; + let persister = self.persister.clone(); + spawn(async move { + if let Err(e) = persister + .add_or_update_payment_in_sql(payment_info, PaymentType::InboundPayment) + .await + { + log::error!("{}", e); + } + }); } fn handle_payment_sent( @@ -244,16 +236,31 @@ impl LightningEventHandler { "Handling PaymentSent event for payment_hash: {}", hex::encode(payment_hash.0) ); - if let Some(payment) = self.outbound_payments.lock().get_mut(&payment_hash) { - payment.preimage = Some(payment_preimage); - payment.status = HTLCStatus::Succeeded; - payment.fee_paid_msat = fee_paid_msat; - log::info!( - "Successfully sent payment of {} millisatoshis with payment hash {}", - payment.amt_msat.unwrap_or_default(), - hex::encode(payment_hash.0) - ); - } + let persister = self.persister.clone(); + spawn(async move { + if let Ok(Some((mut payment_info, payment_type))) = persister + .get_payment_from_sql(payment_hash) + .await + .error_log_passthrough() + { + payment_info.preimage = Some(payment_preimage); + payment_info.status = HTLCStatus::Succeeded; + payment_info.fee_paid_msat = fee_paid_msat; + let amt_msat = payment_info.amt_msat; + if let Err(e) = persister + .add_or_update_payment_in_sql(payment_info, payment_type) + .await + .error_log_passthrough() + { + log::error!("{}", e); + } + log::info!( + "Successfully sent payment of {} millisatoshis with payment hash {}", + amt_msat.unwrap_or_default(), + hex::encode(payment_hash.0) + ); + } + }); } fn handle_channel_closed(&self, channel_id: [u8; 32], user_channel_id: u64, reason: String) { @@ -317,16 +324,24 @@ impl LightningEventHandler { } } - fn handle_payment_failed(&self, payment_hash: &PaymentHash) { + fn handle_payment_failed(&self, payment_hash: PaymentHash) { log::info!( "Handling PaymentFailed event for payment_hash: {}", hex::encode(payment_hash.0) ); - let mut outbound_payments = self.outbound_payments.lock(); - let outbound_payment = outbound_payments.get_mut(payment_hash); - if let Some(payment) = outbound_payment { - payment.status = HTLCStatus::Failed; - } + let persister = self.persister.clone(); + spawn(async move { + if let Ok(Some((mut payment_info, payment_type))) = persister + .get_payment_from_sql(payment_hash) + .await + .error_log_passthrough() + { + payment_info.status = HTLCStatus::Failed; + if let Err(e) = persister.add_or_update_payment_in_sql(payment_info, payment_type).await { + log::error!("{}", e); + } + } + }); } fn handle_pending_htlcs_forwards(&self, time_forwardable: Duration) { diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index e4ebb227d1..cce4f5643a 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -14,7 +14,7 @@ extern crate secp256k1; extern crate serde_json; use crate::storage::{FileSystemStorage, HTLCStatus, NodesAddressesMap, NodesAddressesMapShared, PaymentInfo, - SqlChannelDetails, SqlStorage}; + PaymentType, SqlChannelDetails, SqlStorage}; use crate::util::DiskWriteable; use async_trait::async_trait; use bitcoin::blockdata::constants::genesis_block; @@ -146,11 +146,11 @@ fn insert_channel_sql(for_coin: &str) -> Result { Ok(sql) } -fn insert_payment_sql(for_coin: &str) -> Result { +fn insert_or_update_payment_sql(for_coin: &str) -> Result { let table_name = payments_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "INSERT INTO ".to_owned() + let sql = "INSERT OR REPLACE INTO ".to_owned() + &table_name + " (payment_hash, preimage, secret, amount_msat, fee_paid_msat, is_outbound, status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);"; @@ -170,7 +170,7 @@ fn select_payment_from_table_by_hash_sql(for_coin: &str) -> Result) -> Result) -> Result { +fn payment_info_from_row(row: &Row<'_>) -> Result<(PaymentInfo, PaymentType), SqlError> { let payment_info = PaymentInfo { - preimage: row.get::<_, String>(0).ok().map(|p| { + payment_hash: PaymentHash( + hex::decode(row.get::<_, String>(0)?) + .expect("Payment hash decoding should not fail!") + .try_into() + .expect("String should be 64 characters!"), + ), + preimage: row.get::<_, String>(1).ok().map(|p| { PaymentPreimage( hex::decode(p) .expect("Preimage decoding should not fail!") @@ -206,7 +212,7 @@ fn payment_info_from_row(row: &Row<'_>) -> Result { .expect("String should be 64 characters!"), ) }), - secret: row.get::<_, String>(1).ok().map(|s| { + secret: row.get::<_, String>(2).ok().map(|s| { PaymentSecret( hex::decode(s) .expect("Secret decoding should not fail!") @@ -214,11 +220,16 @@ fn payment_info_from_row(row: &Row<'_>) -> Result { .expect("String should be 64 characters!"), ) }), - amt_msat: row.get::<_, u32>(2).ok().map(|v| v as u64), - fee_paid_msat: row.get::<_, u32>(3).ok().map(|v| v as u64), - status: HTLCStatus::from_str(&row.get::<_, String>(4)?)?, + amt_msat: row.get::<_, u32>(3).ok().map(|v| v as u64), + fee_paid_msat: row.get::<_, u32>(4).ok().map(|v| v as u64), + status: HTLCStatus::from_str(&row.get::<_, String>(5)?)?, + }; + let is_outbound = row.get::<_, bool>(6)?; + let payment_type = match is_outbound { + true => PaymentType::OutboundPayment, + false => PaymentType::InboundPayment, }; - Ok(payment_info) + Ok((payment_info, payment_type)) } fn get_last_channel_rpc_id_sql(for_coin: &str) -> Result { @@ -268,6 +279,28 @@ fn get_closed_channels_sql(for_coin: &str) -> Result { Ok(sql) } +fn get_outbound_payments_sql(for_coin: &str) -> Result { + let table_name = payments_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "SELECT payment_hash, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound FROM ".to_owned() + + &table_name + + " WHERE is_outbound = 1;"; + + Ok(sql) +} + +fn get_inbound_payments_sql(for_coin: &str) -> Result { + let table_name = payments_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "SELECT payment_hash, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound FROM ".to_owned() + + &table_name + + " WHERE is_outbound = 0;"; + + Ok(sql) +} + fn update_claiming_tx_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; @@ -693,19 +726,21 @@ impl SqlStorage for LightningPersister { .await } - async fn add_payment_to_sql( + async fn add_or_update_payment_in_sql( &self, - hash: PaymentHash, info: PaymentInfo, - is_outbound: bool, + payment_type: PaymentType, ) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); - let payment_hash = hex::encode(hash.0); + let payment_hash = hex::encode(info.payment_hash.0); let preimage = info.preimage.map(|p| hex::encode(p.0)); let secret = info.secret.map(|s| hex::encode(s.0)); let amount_msat = info.amt_msat.map(|a| a as u32); let fee_paid_msat = info.fee_paid_msat.map(|f| f as u32); - let is_outbound = is_outbound as i32; + let is_outbound = match payment_type { + PaymentType::OutboundPayment => true as i32, + PaymentType::InboundPayment => false as i32, + }; let status = info.status.to_string(); let sqlite_connection = self.sqlite_connection.clone(); @@ -721,7 +756,7 @@ impl SqlStorage for LightningPersister { ]; let mut conn = sqlite_connection.lock().unwrap(); let sql_transaction = conn.transaction()?; - sql_transaction.execute(&insert_payment_sql(&for_coin)?, ¶ms)?; + sql_transaction.execute(&insert_or_update_payment_sql(&for_coin)?, ¶ms)?; sql_transaction.commit()?; Ok(()) }) @@ -740,7 +775,7 @@ impl SqlStorage for LightningPersister { .await } - async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error> { + async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error> { let params = [hex::encode(hash.0)]; let sql = select_payment_from_table_by_hash_sql(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); @@ -838,6 +873,34 @@ impl SqlStorage for LightningPersister { .await } + async fn get_outbound_payments(&self) -> Result, Self::Error> { + let sql = get_outbound_payments_sql(self.storage_ticker.as_str())?; + let sqlite_connection = self.sqlite_connection.clone(); + + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query(NO_PARAMS)?; + let result = rows.mapped(payment_info_from_row).collect::>()?; + Ok(result) + }) + .await + } + + async fn get_inbound_payments(&self) -> Result, Self::Error> { + let sql = get_inbound_payments_sql(self.storage_ticker.as_str())?; + let sqlite_connection = self.sqlite_connection.clone(); + + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query(NO_PARAMS)?; + let result = rows.mapped(payment_info_from_row).collect::>()?; + Ok(result) + }) + .await + } + async fn add_claiming_tx_to_sql( &self, closing_tx: String, @@ -1220,30 +1283,38 @@ mod tests { assert!(payment.is_none()); let mut expected_payment_info = PaymentInfo { + payment_hash: PaymentHash([0; 32]), preimage: Some(PaymentPreimage([2; 32])), secret: Some(PaymentSecret([3; 32])), amt_msat: Some(2000), fee_paid_msat: Some(100), status: HTLCStatus::Failed, }; - block_on(persister.add_payment_to_sql(PaymentHash([0; 32]), expected_payment_info.clone(), true)).unwrap(); + block_on(persister.add_or_update_payment_in_sql(expected_payment_info.clone(), PaymentType::InboundPayment)) + .unwrap(); let actual_payment_info = block_on(persister.get_payment_from_sql(PaymentHash([0; 32]))) .unwrap() .unwrap(); - assert_eq!(expected_payment_info, actual_payment_info); - - // must fail because we are adding payment with the same hash - block_on(persister.add_payment_to_sql(PaymentHash([0; 32]), expected_payment_info.clone(), true)).unwrap_err(); + assert_eq!(expected_payment_info, actual_payment_info.0); + expected_payment_info.payment_hash = PaymentHash([1; 32]); expected_payment_info.secret = None; expected_payment_info.amt_msat = None; expected_payment_info.status = HTLCStatus::Succeeded; - block_on(persister.add_payment_to_sql(PaymentHash([1; 32]), expected_payment_info.clone(), true)).unwrap(); + block_on(persister.add_or_update_payment_in_sql(expected_payment_info.clone(), PaymentType::OutboundPayment)) + .unwrap(); let actual_payment_info = block_on(persister.get_payment_from_sql(PaymentHash([1; 32]))) .unwrap() .unwrap(); - assert_eq!(expected_payment_info, actual_payment_info); + assert_eq!(expected_payment_info, actual_payment_info.0); + + // Update the first payment to outbound + expected_payment_info.payment_hash = PaymentHash([0; 32]); + block_on(persister.add_or_update_payment_in_sql(expected_payment_info.clone(), PaymentType::OutboundPayment)) + .unwrap(); + let outbound_payments = block_on(persister.get_outbound_payments()).unwrap(); + assert_eq!(outbound_payments.len(), 2); } } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 127f9ba77c..75780b284b 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -109,8 +109,17 @@ impl FromStr for HTLCStatus { } } +#[derive(Serialize)] +pub enum PaymentType { + #[serde(rename = "Outbound Payment")] + OutboundPayment, + #[serde(rename = "Inbound Payment")] + InboundPayment, +} + #[derive(Clone, Debug, PartialEq)] pub struct PaymentInfo { + pub payment_hash: PaymentHash, pub preimage: Option, pub secret: Option, pub amt_msat: Option, @@ -129,16 +138,15 @@ pub trait SqlStorage { async fn add_channel_to_sql(&self, details: SqlChannelDetails) -> Result<(), Self::Error>; - async fn add_payment_to_sql( + async fn add_or_update_payment_in_sql( &self, - hash: PaymentHash, info: PaymentInfo, - is_outbound: bool, + payment_type: PaymentType, ) -> Result<(), Self::Error>; async fn get_channel_from_sql(&self, rpc_id: u64) -> Result, Self::Error>; - async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error>; + async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error>; async fn get_last_channel_rpc_id(&self) -> Result; @@ -156,6 +164,10 @@ pub trait SqlStorage { async fn get_closed_channels(&self) -> Result, Self::Error>; + async fn get_outbound_payments(&self) -> Result, Self::Error>; + + async fn get_inbound_payments(&self) -> Result, Self::Error>; + async fn add_claiming_tx_to_sql( &self, closing_tx: String, From 9936e259c649c819d0a75a82da2347dfd7dc29c6 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Wed, 23 Mar 2022 23:23:12 +0200 Subject: [PATCH 21/49] update rust-lightning to v0.0.105 --- Cargo.lock | 18 ++++----- mm2src/coins/Cargo.toml | 6 +-- mm2src/coins/lightning.rs | 20 ++++++---- mm2src/coins/lightning/ln_events.rs | 2 + mm2src/coins/lightning/ln_utils.rs | 3 +- .../lightning_background_processor/Cargo.toml | 10 ++--- .../lightning_background_processor/src/lib.rs | 40 ++++++++++++------- mm2src/coins/lightning_persister/Cargo.toml | 6 +-- mm2src/coins/lightning_persister/src/lib.rs | 30 +++++++++----- .../coins/lightning_persister/src/storage.rs | 6 +-- mm2src/common/Cargo.toml | 2 +- 11 files changed, 86 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d56ee71c2..a54d028e7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3015,19 +3015,19 @@ dependencies = [ [[package]] name = "lightning" -version = "0.0.104" +version = "0.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0113e6b5a55b7ead30fb0a992b787e69a0551fa15b7eed93c99490eb018ab793" +checksum = "a10e7439623b293d000fc875627704210d8d0ff5b7badbb689f2a3d51afc618f" dependencies = [ "bitcoin", - "hex 0.3.2", + "hex 0.4.3", "regex 0.1.80", "secp256k1", ] [[package]] name = "lightning-background-processor" -version = "0.0.104" +version = "0.0.105" dependencies = [ "bitcoin", "db_common", @@ -3038,9 +3038,9 @@ dependencies = [ [[package]] name = "lightning-invoice" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2531e38818b3872b9acbcc3ff499f93962d2ff23ee0d05fc386ef70b993a38a0" +checksum = "ce661ad7182c2b258d1506e264095d2b4c71436a91b72834d7fb87b7e92e06c0" dependencies = [ "bech32", "bitcoin_hashes", @@ -3051,9 +3051,9 @@ dependencies = [ [[package]] name = "lightning-net-tokio" -version = "0.0.104" +version = "0.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1851ab90c2739929526c56e06d6b07def4508bfce70f190757fde2de648d1d9c" +checksum = "e6eb1bcdf0a29baccb13d514e4a8761c9d8038ffd107f27e45182f0177f5e007" dependencies = [ "bitcoin", "lightning", @@ -3062,7 +3062,7 @@ dependencies = [ [[package]] name = "lightning-persister" -version = "0.0.104" +version = "0.0.105" dependencies = [ "async-trait", "bitcoin", diff --git a/mm2src/coins/Cargo.toml b/mm2src/coins/Cargo.toml index 57f1b2c7db..79c03f2a8a 100644 --- a/mm2src/coins/Cargo.toml +++ b/mm2src/coins/Cargo.toml @@ -46,9 +46,9 @@ jsonrpc-core = "8.0.1" keys = { path = "../mm2_bitcoin/keys" } lazy_static = "1.4" libc = "0.2" -lightning = "0.0.104" +lightning = "0.0.105" lightning-background-processor = { path = "lightning_background_processor" } -lightning-invoice = "0.12.0" +lightning-invoice = "0.13.0" metrics = "0.12" mocktopus = "0.7.0" num-traits = "0.2" @@ -88,7 +88,7 @@ web-sys = { version = "0.3.55", features = ["console", "Headers", "Request", "Re [target.'cfg(not(target_arch = "wasm32"))'.dependencies] dirs = { version = "1" } lightning-persister = { path = "lightning_persister" } -lightning-net-tokio = "0.0.104" +lightning-net-tokio = "0.0.105" rusqlite = { version = "0.24.2", features = ["bundled"], optional = true } rust-ini = { version = "0.13" } rustls = { version = "0.19", features = ["dangerous_configuration"] } diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 15cc50d7f9..8dbf22a5bd 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -30,20 +30,18 @@ use futures::{FutureExt, TryFutureExt}; use futures01::Future; use keys::{AddressHashEnum, KeyPair}; use lightning::chain::channelmonitor::Balance; -use lightning::chain::keysinterface::KeysInterface; -use lightning::chain::keysinterface::KeysManager; +use lightning::chain::keysinterface::{KeysInterface, KeysManager, Recipient}; use lightning::chain::Access; use lightning::ln::channelmanager::{ChannelDetails, MIN_FINAL_CLTV_EXPIRY}; use lightning::ln::{PaymentHash, PaymentPreimage}; use lightning::routing::network_graph::{NetGraphMsgHandler, NetworkGraph}; -use lightning::routing::scoring::Scorer; use lightning::util::config::UserConfig; use lightning_background_processor::BackgroundProcessor; use lightning_invoice::payment; use lightning_invoice::utils::{create_invoice_from_channelmanager, DefaultRouter}; use lightning_invoice::Invoice; use lightning_persister::storage::{FileSystemStorage, HTLCStatus, NodesAddressesMapShared, PaymentInfo, PaymentType, - SqlChannelDetails, SqlStorage}; + Scorer, SqlChannelDetails, SqlStorage}; use lightning_persister::LightningPersister; use ln_conf::{ChannelOptions, LightningCoinConf, LightningProtocolConf, PlatformCoinConfirmations}; use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelError, CloseChannelResult, @@ -374,7 +372,13 @@ impl MarketCoinOps for LightningCoin { fn current_block(&self) -> Box + Send> { Box::new(futures01::future::ok(0)) } - fn display_priv_key(&self) -> Result { Ok(self.keys_manager.get_node_secret().to_string()) } + fn display_priv_key(&self) -> Result { + Ok(self + .keys_manager + .get_node_secret(Recipient::Node) + .map_err(|_| "Unsupported recipient".to_string())? + .to_string()) + } // Todo: Implement this when implementing swaps for lightning as it's is used only for swaps fn min_tx_amount(&self) -> BigDecimal { unimplemented!() } @@ -540,7 +544,9 @@ pub async fn start_lightning( params.listening_port, channel_manager.clone(), network_gossip.clone(), - keys_manager.get_node_secret(), + keys_manager + .get_node_secret(Recipient::Node) + .map_to_mm(|_| EnableLightningError::UnsupportedMode("'start_lightning'".into(), "local node".into()))?, logger.clone(), ) .await?; @@ -555,7 +561,7 @@ pub async fn start_lightning( )); // Initialize routing Scorer - let scorer = Arc::new(Mutex::new(persister.get_scorer().await?)); + let scorer = Arc::new(Mutex::new(persister.get_scorer(network_graph.clone()).await?)); spawn(ln_utils::persist_scorer_loop(persister.clone(), scorer.clone())); // Create InvoicePayer diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 9951d655d9..2a93d4be2d 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -79,6 +79,8 @@ impl EventHandler for LightningEventHandler { rejected_by_dest, all_paths_failed, ), + // Todo: Handle inbound channels + Event::OpenChannelRequest { .. } => {}, } } } diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 5b159ae594..341895e364 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -12,10 +12,9 @@ use lightning::chain::{chainmonitor, BestBlock, Watch}; use lightning::ln::channelmanager; use lightning::ln::channelmanager::{ChainParameters, ChannelManagerReadArgs, SimpleArcChannelManager}; use lightning::routing::network_graph::NetworkGraph; -use lightning::routing::scoring::Scorer; use lightning::util::config::UserConfig; use lightning::util::ser::ReadableArgs; -use lightning_persister::storage::{FileSystemStorage, NodesAddressesMap, SqlStorage}; +use lightning_persister::storage::{FileSystemStorage, NodesAddressesMap, Scorer, SqlStorage}; use lightning_persister::LightningPersister; use std::fs::File; use std::path::PathBuf; diff --git a/mm2src/coins/lightning_background_processor/Cargo.toml b/mm2src/coins/lightning_background_processor/Cargo.toml index 3a535e56ec..8c950bfc4c 100644 --- a/mm2src/coins/lightning_background_processor/Cargo.toml +++ b/mm2src/coins/lightning_background_processor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lightning-background-processor" -version = "0.0.104" +version = "0.0.105" authors = ["Valentine Wallace "] license = "MIT OR Apache-2.0" repository = "http://github.com/lightningdevkit/rust-lightning" @@ -11,10 +11,10 @@ edition = "2018" [dependencies] bitcoin = "0.27.1" -lightning = { version = "0.0.104", features = ["std"] } +lightning = { version = "0.0.105", features = ["std"] } [dev-dependencies] db_common = { path = "../../db_common" } -lightning = { version = "0.0.104", features = ["_test_utils"] } -lightning-invoice = "0.12.0" -lightning-persister = { version = "0.0.104", path = "../lightning_persister" } +lightning = { version = "0.0.105", features = ["_test_utils"] } +lightning-invoice = "0.13.0" +lightning-persister = { version = "0.0.105", path = "../lightning_persister" } diff --git a/mm2src/coins/lightning_background_processor/src/lib.rs b/mm2src/coins/lightning_background_processor/src/lib.rs index 0c3f9f0fb2..c563f1a75f 100644 --- a/mm2src/coins/lightning_background_processor/src/lib.rs +++ b/mm2src/coins/lightning_background_processor/src/lib.rs @@ -57,7 +57,7 @@ const FRESHNESS_TIMER: u64 = 60; const FRESHNESS_TIMER: u64 = 1; #[cfg(all(not(test), not(debug_assertions)))] -const PING_TIMER: u64 = 5; +const PING_TIMER: u64 = 10; /// Signature operations take a lot longer without compiler optimisations. /// Increasing the ping timer allows for this but slower devices will be disconnected if the /// timeout is reached. @@ -229,10 +229,16 @@ impl BackgroundProcessor { let mut have_pruned = false; loop { - peer_manager.process_events(); + peer_manager.process_events(); // Note that this may block on ChannelManager's locking channel_manager.process_pending_events(&event_handler); chain_monitor.process_pending_events(&event_handler); + + // We wait up to 100ms, but track how long it takes to detect being put to sleep, + // see `await_start`'s use below. + let await_start = Instant::now(); let updates_available = channel_manager.await_persistable_update_timeout(Duration::from_millis(100)); + let await_time = await_start.elapsed(); + if updates_available { log_trace!(logger, "Persisting ChannelManager..."); persister.persist_manager(&*channel_manager)?; @@ -241,25 +247,27 @@ impl BackgroundProcessor { // Exit the loop if the background processor was requested to stop. if stop_thread.load(Ordering::Acquire) { log_trace!(logger, "Terminating background processor."); - return Ok(()); + break; } if last_freshness_call.elapsed().as_secs() > FRESHNESS_TIMER { log_trace!(logger, "Calling ChannelManager's timer_tick_occurred"); channel_manager.timer_tick_occurred(); last_freshness_call = Instant::now(); } - if last_ping_call.elapsed().as_secs() > PING_TIMER * 2 { + if await_time > Duration::from_secs(1) { // On various platforms, we may be starved of CPU cycles for several reasons. // E.g. on iOS, if we've been in the background, we will be entirely paused. // Similarly, if we're on a desktop platform and the device has been asleep, we // may not get any cycles. - // In any case, if we've been entirely paused for more than double our ping - // timer, we should have disconnected all sockets by now (and they're probably - // dead anyway), so disconnect them by calling `timer_tick_occurred()` twice. - log_trace!( - logger, - "Awoke after more than double our ping timer, disconnecting peers." - ); + // We detect this by checking if our max-100ms-sleep, above, ran longer than a + // full second, at which point we assume sockets may have been killed (they + // appear to be at least on some platforms, even if it has only been a second). + // Note that we have to take care to not get here just because user event + // processing was slow at the top of the loop. For example, the sample client + // may call Bitcoin Core RPCs during event handling, which very often takes + // more than a handful of seconds to complete, and shouldn't disconnect all our + // peers. + log_trace!(logger, "100ms sleep took more than a second, disconnecting peers."); peer_manager.disconnect_all_peers(); last_ping_call = Instant::now(); } else if last_ping_call.elapsed().as_secs() > PING_TIMER { @@ -281,6 +289,10 @@ impl BackgroundProcessor { } } } + // After we exit, ensure we persist the ChannelManager one final time - this avoids + // some races where users quit while channel updates were in-flight, with + // ChannelMonitor update(s) persisted without a corresponding ChannelManager update. + persister.persist_manager(&*channel_manager) }); Self { stop_thread: stop_thread_clone, @@ -342,7 +354,7 @@ mod tests { use bitcoin::network::constants::Network; use db_common::sqlite::rusqlite::Connection; use lightning::chain::channelmonitor::ANTI_REORG_DELAY; - use lightning::chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager}; + use lightning::chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager, Recipient}; use lightning::chain::transaction::OutPoint; use lightning::chain::{chainmonitor, BestBlock, Confirm}; use lightning::get_event_msg; @@ -484,7 +496,7 @@ mod tests { }; let peer_manager = Arc::new(PeerManager::new( msg_handler, - keys_manager.get_node_secret(), + keys_manager.get_node_secret(Recipient::Node).unwrap(), &seed, logger.clone(), IgnoringMessageHandler {}, @@ -906,7 +918,7 @@ mod tests { Arc, >| node_0_persister.persist_manager(node); let router = DefaultRouter::new(Arc::clone(&nodes[0].network_graph), Arc::clone(&nodes[0].logger)); - let scorer = Arc::new(Mutex::new(test_utils::TestScorer::default())); + let scorer = Arc::new(Mutex::new(test_utils::TestScorer::with_penalty(0))); let invoice_payer = Arc::new(InvoicePayer::new( Arc::clone(&nodes[0].node), router, diff --git a/mm2src/coins/lightning_persister/Cargo.toml b/mm2src/coins/lightning_persister/Cargo.toml index 5663003985..33e9e8a80f 100644 --- a/mm2src/coins/lightning_persister/Cargo.toml +++ b/mm2src/coins/lightning_persister/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lightning-persister" -version = "0.0.104" +version = "0.0.105" edition = "2018" authors = ["Valentine Wallace", "Matt Corallo"] license = "MIT OR Apache-2.0" @@ -16,7 +16,7 @@ common = { path = "../../common" } db_common = { path = "../../db_common" } derive_more = "0.99" hex = "0.4.2" -lightning = "0.0.104" +lightning = "0.0.105" libc = "0.2" parking_lot = { version = "0.12.0", features = ["nightly"] } secp256k1 = { version = "0.20" } @@ -27,4 +27,4 @@ serde_json = "1.0" winapi = { version = "0.3", features = ["winbase"] } [dev-dependencies] -lightning = { version = "0.0.104", features = ["_test_utils"] } \ No newline at end of file +lightning = { version = "0.0.105", features = ["_test_utils"] } \ No newline at end of file diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index cce4f5643a..e63a0888c1 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -14,7 +14,7 @@ extern crate secp256k1; extern crate serde_json; use crate::storage::{FileSystemStorage, HTLCStatus, NodesAddressesMap, NodesAddressesMapShared, PaymentInfo, - PaymentType, SqlChannelDetails, SqlStorage}; + PaymentType, Scorer, SqlChannelDetails, SqlStorage}; use crate::util::DiskWriteable; use async_trait::async_trait; use bitcoin::blockdata::constants::genesis_block; @@ -23,8 +23,9 @@ use bitcoin::hashes::hex::{FromHex, ToHex}; use bitcoin::Network; use common::async_blocking; use common::fs::check_dir_operations; -use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, ToSql, NO_PARAMS}; -use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; +use db_common::sqlite::rusqlite::{Error as SqlError, Row, ToSql, NO_PARAMS}; +use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, SqliteConnShared, + CHECK_TABLE_EXISTS_SQL}; use lightning::chain; use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator}; use lightning::chain::chainmonitor; @@ -34,7 +35,7 @@ use lightning::chain::transaction::OutPoint; use lightning::ln::channelmanager::ChannelManager; use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; use lightning::routing::network_graph::NetworkGraph; -use lightning::routing::scoring::Scorer; +use lightning::routing::scoring::ProbabilisticScoringParameters; use lightning::util::logger::Logger; use lightning::util::ser::{Readable, ReadableArgs, Writeable}; use secp256k1::PublicKey; @@ -66,7 +67,7 @@ pub struct LightningPersister { storage_ticker: String, main_path: PathBuf, backup_path: Option, - sqlite_connection: Arc>, + sqlite_connection: SqliteConnShared, } impl DiskWriteable for ChannelMonitor { @@ -317,7 +318,7 @@ impl LightningPersister { storage_ticker: String, main_path: PathBuf, backup_path: Option, - sqlite_connection: Arc>, + sqlite_connection: SqliteConnShared, ) -> Self { Self { storage_ticker, @@ -423,6 +424,12 @@ impl LightningPersister { "Invalid ChannelMonitor file name", )); } + if filename.unwrap().ends_with(".tmp") { + // If we were in the middle of committing an new update and crashed, it should be + // safe to ignore the update - we should never have returned to the caller and + // irrevocably committed to the new state in any way. + continue; + } let txid = Txid::from_hex(filename.unwrap().split_at(64).0); if txid.is_err() { @@ -634,15 +641,18 @@ impl FileSystemStorage for LightningPersister { .await } - async fn get_scorer(&self) -> Result { + async fn get_scorer(&self, network_graph: Arc) -> Result { let path = self.scorer_path(); if !path.exists() { - return Ok(Scorer::default()); + return Ok(Scorer::new(ProbabilisticScoringParameters::default(), network_graph)); } async_blocking(move || { let file = fs::File::open(path)?; - Scorer::read(&mut BufReader::new(file)) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) + Scorer::read( + &mut BufReader::new(file), + (ProbabilisticScoringParameters::default(), network_graph), + ) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) }) .await } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 75780b284b..d450017a9d 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -4,7 +4,7 @@ use db_common::sqlite::rusqlite::types::FromSqlError; use derive_more::Display; use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; use lightning::routing::network_graph::NetworkGraph; -use lightning::routing::scoring::Scorer; +use lightning::routing::scoring::ProbabilisticScorer; use parking_lot::Mutex as PaMutex; use secp256k1::PublicKey; use serde::Serialize; @@ -15,7 +15,7 @@ use std::sync::{Arc, Mutex}; pub type NodesAddressesMap = HashMap; pub type NodesAddressesMapShared = Arc>; - +pub type Scorer = ProbabilisticScorer>; #[async_trait] pub trait FileSystemStorage { type Error; @@ -33,7 +33,7 @@ pub trait FileSystemStorage { async fn save_network_graph(&self, network_graph: Arc) -> Result<(), Self::Error>; - async fn get_scorer(&self) -> Result; + async fn get_scorer(&self, network_graph: Arc) -> Result; async fn save_scorer(&self, scorer: Arc>) -> Result<(), Self::Error>; } diff --git a/mm2src/common/Cargo.toml b/mm2src/common/Cargo.toml index d07aa57c21..f92d0ba453 100644 --- a/mm2src/common/Cargo.toml +++ b/mm2src/common/Cargo.toml @@ -37,7 +37,7 @@ http-body = "0.1" itertools = "0.8" keys = { path = "../mm2_bitcoin/keys" } lazy_static = "1.4" -lightning = "0.0.104" +lightning = "0.0.105" log = "0.4.8" num-bigint = { version = "0.2", features = ["serde", "std"] } num-rational = { version = "0.2", features = ["serde", "bigint", "bigint-std"] } From 7bd7b92fa609c2558df50c2cf1766c913c66ca53 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Thu, 24 Mar 2022 14:21:06 +0200 Subject: [PATCH 22/49] set manually_accept_inbound_channels to true to be able to add them to sql in the future --- mm2src/coins/lightning/ln_conf.rs | 2 + mm2src/coins/lightning/ln_events.rs | 70 +++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/mm2src/coins/lightning/ln_conf.rs b/mm2src/coins/lightning/ln_conf.rs index 582dfa3855..4db186fe2e 100644 --- a/mm2src/coins/lightning/ln_conf.rs +++ b/mm2src/coins/lightning/ln_conf.rs @@ -241,6 +241,8 @@ impl From for UserConfig { if let Some(accept_inbound) = conf.accept_inbound_channels { user_config.accept_inbound_channels = accept_inbound; } + // This allows OpenChannelRequest event to be fired + user_config.manually_accept_inbound_channels = true; user_config } diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 2a93d4be2d..2449f38f50 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -30,35 +30,53 @@ impl EventHandler for LightningEventHandler { channel_value_satoshis, output_script, user_channel_id, - } => self.handle_funding_generation_ready(*temporary_channel_id, *channel_value_satoshis, output_script, *user_channel_id), + } => self.handle_funding_generation_ready( + *temporary_channel_id, + *channel_value_satoshis, + output_script, + *user_channel_id, + ), + Event::PaymentReceived { payment_hash, amt, purpose, } => self.handle_payment_received(*payment_hash, *amt, purpose), + Event::PaymentSent { payment_preimage, payment_hash, fee_paid_msat, .. } => self.handle_payment_sent(*payment_preimage, *payment_hash, *fee_paid_msat), + Event::PaymentFailed { payment_hash, .. } => self.handle_payment_failed(*payment_hash), + Event::PendingHTLCsForwardable { time_forwardable } => self.handle_pending_htlcs_forwards(*time_forwardable), + Event::SpendableOutputs { outputs } => self.handle_spendable_outputs(outputs), + // Todo: an RPC for total amount earned Event::PaymentForwarded { fee_earned_msat, claim_from_onchain_tx } => log::info!( - "Recieved a fee of {} milli-satoshis for a successfully forwarded payment through our {} lightning node. Was the forwarded HTLC claimed by our counterparty via an on-chain transaction?: {}", - fee_earned_msat.unwrap_or_default(), - self.platform.coin.ticker(), - claim_from_onchain_tx, - ), - Event::ChannelClosed { channel_id, user_channel_id, reason } => self.handle_channel_closed(*channel_id, *user_channel_id, reason.to_string()), + "Recieved a fee of {} milli-satoshis for a successfully forwarded payment through our {} lightning node. Was the forwarded HTLC claimed by our counterparty via an on-chain transaction?: {}", + fee_earned_msat.unwrap_or_default(), + self.platform.coin.ticker(), + claim_from_onchain_tx, + ), + + Event::ChannelClosed { + channel_id, + user_channel_id, + reason, + } => self.handle_channel_closed(*channel_id, *user_channel_id, reason.to_string()), + // Todo: Add spent UTXOs to RecentlySpentOutPoints if it's not discarded Event::DiscardFunding { channel_id, transaction } => log::info!( - "Discarding funding tx: {} for channel {}", - transaction.txid().to_string(), - hex::encode(channel_id), - ), + "Discarding funding tx: {} for channel {}", + transaction.txid().to_string(), + hex::encode(channel_id), + ), + // Handling updating channel penalties after successfully routing a payment along a path is done by the InvoicePayer. Event::PaymentPathSuccessful { payment_id, @@ -70,17 +88,41 @@ impl EventHandler for LightningEventHandler { payment_hash.map(|h| hex::encode(h.0)).unwrap_or_default(), hex::encode(payment_id.0) ), + // Handling updating channel penalties after a payment fails to route through a channel is done by the InvoicePayer. // Also abandoning or retrying a payment is handled by the InvoicePayer. - Event::PaymentPathFailed { payment_hash, rejected_by_dest, all_paths_failed, path, .. } => log::info!( + Event::PaymentPathFailed { + payment_hash, + rejected_by_dest, + all_paths_failed, + path, + .. + } => log::info!( "Payment path: {:?}, failed for payment hash: {}, Was rejected by destination?: {}, All paths failed?: {}", path.iter().map(|hop| hop.pubkey.to_string()).collect::>(), hex::encode(payment_hash.0), rejected_by_dest, all_paths_failed, ), - // Todo: Handle inbound channels - Event::OpenChannelRequest { .. } => {}, + + Event::OpenChannelRequest { + temporary_channel_id, + counterparty_node_id, + funding_satoshis, + push_msat, + } => { + log::info!( + "Handling OpenChannelRequest from node: {} with funding value: {} and starting balance: {}", + counterparty_node_id, + funding_satoshis, + push_msat, + ); + if self.channel_manager.accept_inbound_channel(temporary_channel_id).is_ok() { + // Todo: once the rust-lightning PR for user_channel_id in accept_inbound_channel is released + // use user_channel_id to get the funding tx here once the funding tx is available. + unimplemented!() + } + }, } } } From bf11e7a16647976701d0b71ef5df5d90e0bafc21 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Sat, 26 Mar 2022 00:08:16 +0200 Subject: [PATCH 23/49] correct calculation of claimed balances --- mm2src/coins/lightning.rs | 2 + mm2src/coins/lightning/ln_events.rs | 23 ++++++++- mm2src/coins/lightning/ln_platform.rs | 49 +++++++++++++------ mm2src/coins/lightning_persister/src/lib.rs | 10 ++-- .../coins/lightning_persister/src/storage.rs | 4 +- 5 files changed, 65 insertions(+), 23 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 8dbf22a5bd..3bd53031b9 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -1022,6 +1022,7 @@ pub struct ListPaymentsReq { #[derive(Serialize)] pub struct PaymentInfoForRPC { + payment_hash: H256Json, amount_in_msat: Option, fee_paid_msat: Option, status: HTLCStatus, @@ -1030,6 +1031,7 @@ pub struct PaymentInfoForRPC { impl From for PaymentInfoForRPC { fn from(info: PaymentInfo) -> Self { PaymentInfoForRPC { + payment_hash: info.payment_hash.0.into(), amount_in_msat: info.amt_msat, fee_paid_msat: info.fee_paid_msat, status: info.status, diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 2449f38f50..f61624a005 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -429,6 +429,23 @@ impl LightningEventHandler { self.platform.broadcast_transaction(&spending_tx); + let claiming_tx_inputs_value = outputs.iter().fold(0, |sum, output| match output { + SpendableOutputDescriptor::StaticOutput { output, .. } => sum + output.value, + SpendableOutputDescriptor::DelayedPaymentOutput(descriptor) => sum + descriptor.output.value, + SpendableOutputDescriptor::StaticPaymentOutput(descriptor) => sum + descriptor.output.value, + }); + let claiming_tx_outputs_value = spending_tx.output.iter().fold(0, |sum, txout| sum + txout.value); + if claiming_tx_inputs_value < claiming_tx_outputs_value { + log::error!( + "Claiming transaction input value {} can't be less than outputs value {}!", + claiming_tx_inputs_value, + claiming_tx_outputs_value + ); + return; + } + let claiming_tx_fee = claiming_tx_inputs_value - claiming_tx_outputs_value; + let claiming_tx_fee_per_channel = (claiming_tx_fee as f64) / (outputs.len() as f64); + for output in outputs { let (closing_txid, claimed_balance) = match output { SpendableOutputDescriptor::StaticOutput { outpoint, output } => { @@ -445,7 +462,11 @@ impl LightningEventHandler { let persister = self.persister.clone(); spawn(async move { persister - .add_claiming_tx_to_sql(closing_txid, claiming_txid, claimed_balance) + .add_claiming_tx_to_sql( + closing_txid, + claiming_txid, + (claimed_balance as f64) - claiming_tx_fee_per_channel, + ) .await .error_log(); }); diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index a65e614010..3680f0efce 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -240,10 +240,7 @@ impl Platform { registered_outputs.push(output); } - async fn process_tx_for_unconfirmation(&self, txid: Txid, monitor: Arc) - where - T: Confirm, - { + async fn check_if_tx_is_onchain(&self, txid: Txid) -> Result> { if let Err(err) = self .coin .as_ref() @@ -257,21 +254,35 @@ impl Platform { if let JsonRpcErrorType::Response(_, json) = &json_err.error { if let Some(message) = json["message"].as_str() { if message.contains("'code': -5") { - log::info!( - "Transaction {} is not found on chain :{}. The transaction will be re-broadcasted.", - txid, - err - ); - monitor.transaction_unconfirmed(&txid); + return Ok(false); } } } } - log::error!( + return Err(err.into()); + } + Ok(true) + } + + async fn process_tx_for_unconfirmation(&self, txid: Txid, monitor: Arc) + where + T: Confirm, + { + match self.check_if_tx_is_onchain(txid).await { + Ok(tx_is_onchain) => { + if !tx_is_onchain { + log::info!( + "Transaction {} is not found on chain. The transaction will be re-broadcasted.", + txid, + ); + monitor.transaction_unconfirmed(&txid); + } + }, + Err(e) => log::error!( "Error while trying to check if the transaction {} is discarded or not :{}", txid, - err - ); + e + ), } } @@ -513,14 +524,22 @@ impl FeeEstimator for Platform { impl BroadcasterInterface for Platform { fn broadcast_transaction(&self, tx: &Transaction) { + let txid = tx.txid(); + let fut = Box::new(async move { self.check_if_tx_is_onchain(txid).await }.boxed().compat()); + let is_tx_onchain = tokio::task::block_in_place(move || fut.wait()); + + if let Ok(true) = is_tx_onchain { + log::debug!("Transaction {} is found onchain!", txid); + return; + } + let tx_hex = serialize_hex(tx); log::debug!("Trying to broadcast transaction: {}", tx_hex); - let tx_id = tx.txid(); let fut = self.coin.send_raw_tx(&tx_hex); spawn(async move { match fut.compat().await { Ok(id) => log::info!("Transaction broadcasted successfully: {:?} ", id), - Err(e) => log::error!("Broadcast transaction {} failed: {}", tx_id, e), + Err(e) => log::error!("Broadcast transaction {} failed: {}", txid, e), } }); } diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index e63a0888c1..4b11c74966 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -107,7 +107,7 @@ fn create_channels_history_table_sql(for_coin: &str) -> Result closing_tx VARCHAR(255), closure_reason TEXT, claiming_tx VARCHAR(255), - claimed_balance INTEGER, + claimed_balance REAL, is_outbound INTEGER NOT NULL, is_public INTEGER NOT NULL, is_closed INTEGER NOT NULL @@ -189,7 +189,7 @@ fn channel_details_from_row(row: &Row<'_>) -> Result(9).ok().map(|b| b as u64), + claimed_balance: row.get::<_, f64>(9).ok(), is_outbound: row.get(10)?, is_public: row.get(11)?, is_closed: row.get(12)?, @@ -915,7 +915,7 @@ impl SqlStorage for LightningPersister { &self, closing_tx: String, claiming_tx: String, - claimed_balance: u64, + claimed_balance: f64, ) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); let claimed_balance = claimed_balance.to_string(); @@ -1267,12 +1267,12 @@ mod tests { block_on(persister.add_claiming_tx_to_sql( "5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into(), "97f061634a4a7b0b0c2b95648f86b1c39b95e0cf5073f07725b7143c095b612a".into(), - 2000, + 2000.333333, )) .unwrap(); expected_channel_details.claiming_tx = Some("97f061634a4a7b0b0c2b95648f86b1c39b95e0cf5073f07725b7143c095b612a".into()); - expected_channel_details.claimed_balance = Some(2000); + expected_channel_details.claimed_balance = Some(2000.333333); let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index d450017a9d..cb837e984c 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -54,7 +54,7 @@ pub struct SqlChannelDetails { #[serde(skip_serializing_if = "Option::is_none")] pub claiming_tx: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub claimed_balance: Option, + pub claimed_balance: Option, #[serde(skip_serializing)] pub funding_generated_in_block: Option, pub is_outbound: bool, @@ -172,6 +172,6 @@ pub trait SqlStorage { &self, closing_tx: String, claiming_tx: String, - claimed_balance: u64, + claimed_balance: f64, ) -> Result<(), Self::Error>; } From 25fc9e2c1238cd6906854548717140b3d463abbb Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Mon, 28 Mar 2022 12:06:18 +0200 Subject: [PATCH 24/49] add destination to payment info --- mm2src/coins/lightning.rs | 40 ++++++-- mm2src/coins/lightning/ln_events.rs | 15 ++- mm2src/coins/lightning/ln_utils.rs | 3 - mm2src/coins/lightning_persister/src/lib.rs | 97 +++++++++++-------- .../coins/lightning_persister/src/storage.rs | 19 ++-- 5 files changed, 102 insertions(+), 72 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 3bd53031b9..bc1292c6d8 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -125,9 +125,13 @@ impl LightningCoin { .pay_invoice(&invoice) .map_to_mm(|e| SendPaymentError::PaymentError(format!("{:?}", e)))?; let payment_hash = PaymentHash((*invoice.payment_hash()).into_inner()); + let payment_type = PaymentType::OutboundPayment { + destination: invoice.payee_pub_key().cloned(), + }; let payment_secret = Some(*invoice.payment_secret()); Ok(PaymentInfo { payment_hash, + payment_type, preimage: None, secret: payment_secret, amt_msat: invoice.amount_milli_satoshis(), @@ -153,9 +157,13 @@ impl LightningCoin { .pay_pubkey(destination, payment_preimage, amount_msat, final_cltv_expiry_delta) .map_to_mm(|e| SendPaymentError::PaymentError(format!("{:?}", e)))?; let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0).into_inner()); + let payment_type = PaymentType::OutboundPayment { + destination: Some(destination), + }; Ok(PaymentInfo { payment_hash, + payment_type, preimage: Some(payment_preimage), secret: None, amt_msat: Some(amount_msat), @@ -1008,7 +1016,7 @@ pub async fn send_payment(ctx: MmArc, req: SendPaymentReq) -> SendPaymentResult< }; ln_coin .persister - .add_or_update_payment_in_sql(payment_info.clone(), PaymentType::OutboundPayment) + .add_or_update_payment_in_sql(payment_info.clone()) .await?; Ok(SendPaymentResponse { payment_hash: payment_info.payment_hash.0.into(), @@ -1020,9 +1028,30 @@ pub struct ListPaymentsReq { pub coin: String, } +#[derive(Serialize)] +#[serde(tag = "type")] +pub enum PaymentTypeForRPC { + #[serde(rename = "Outbound Payment")] + OutboundPayment { destination: Option }, + #[serde(rename = "Inbound Payment")] + InboundPayment, +} + +impl From for PaymentTypeForRPC { + fn from(payment_type: PaymentType) -> Self { + match payment_type { + PaymentType::OutboundPayment { destination } => PaymentTypeForRPC::OutboundPayment { + destination: destination.map(PublicKeyForRPC), + }, + PaymentType::InboundPayment => PaymentTypeForRPC::InboundPayment, + } + } +} + #[derive(Serialize)] pub struct PaymentInfoForRPC { payment_hash: H256Json, + payment_type: PaymentTypeForRPC, amount_in_msat: Option, fee_paid_msat: Option, status: HTLCStatus, @@ -1032,6 +1061,7 @@ impl From for PaymentInfoForRPC { fn from(info: PaymentInfo) -> Self { PaymentInfoForRPC { payment_hash: info.payment_hash.0.into(), + payment_type: info.payment_type.into(), amount_in_msat: info.amt_msat, fee_paid_msat: info.fee_paid_msat, status: info.status, @@ -1056,14 +1086,14 @@ pub async fn list_payments(ctx: MmArc, req: ListPaymentsReq) -> ListPaymentsResu .get_inbound_payments() .await? .into_iter() - .map(|(info, _)| info.into()) + .map(From::from) .collect(); let outbound_payments = ln_coin .persister .get_outbound_payments() .await? .into_iter() - .map(|(info, _)| info.into()) + .map(From::from) .collect(); Ok(ListPaymentsResponse { @@ -1080,7 +1110,6 @@ pub struct GetPaymentDetailsRequest { #[derive(Serialize)] pub struct GetPaymentDetailsResponse { - payment_type: PaymentType, payment_details: PaymentInfoForRPC, } @@ -1094,13 +1123,12 @@ pub async fn get_payment_details( _ => return MmError::err(GetPaymentDetailsError::UnsupportedCoin(coin.ticker().to_string())), }; - if let Some((payment_info, payment_type)) = ln_coin + if let Some(payment_info) = ln_coin .persister .get_payment_from_sql(PaymentHash(req.payment_hash.0)) .await? { return Ok(GetPaymentDetailsResponse { - payment_type, payment_details: payment_info.into(), }); } diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index f61624a005..297bf3926b 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -120,7 +120,6 @@ impl EventHandler for LightningEventHandler { if self.channel_manager.accept_inbound_channel(temporary_channel_id).is_ok() { // Todo: once the rust-lightning PR for user_channel_id in accept_inbound_channel is released // use user_channel_id to get the funding tx here once the funding tx is available. - unimplemented!() } }, } @@ -253,6 +252,7 @@ impl LightningEventHandler { }; let payment_info = PaymentInfo { payment_hash, + payment_type: PaymentType::InboundPayment, preimage: Some(payment_preimage), secret: payment_secret, amt_msat: Some(amt), @@ -261,10 +261,7 @@ impl LightningEventHandler { }; let persister = self.persister.clone(); spawn(async move { - if let Err(e) = persister - .add_or_update_payment_in_sql(payment_info, PaymentType::InboundPayment) - .await - { + if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { log::error!("{}", e); } }); @@ -282,7 +279,7 @@ impl LightningEventHandler { ); let persister = self.persister.clone(); spawn(async move { - if let Ok(Some((mut payment_info, payment_type))) = persister + if let Ok(Some(mut payment_info)) = persister .get_payment_from_sql(payment_hash) .await .error_log_passthrough() @@ -292,7 +289,7 @@ impl LightningEventHandler { payment_info.fee_paid_msat = fee_paid_msat; let amt_msat = payment_info.amt_msat; if let Err(e) = persister - .add_or_update_payment_in_sql(payment_info, payment_type) + .add_or_update_payment_in_sql(payment_info) .await .error_log_passthrough() { @@ -375,13 +372,13 @@ impl LightningEventHandler { ); let persister = self.persister.clone(); spawn(async move { - if let Ok(Some((mut payment_info, payment_type))) = persister + if let Ok(Some(mut payment_info)) = persister .get_payment_from_sql(payment_hash) .await .error_log_passthrough() { payment_info.status = HTLCStatus::Failed; - if let Err(e) = persister.add_or_update_payment_in_sql(payment_info, payment_type).await { + if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { log::error!("{}", e); } } diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 341895e364..2a5229aad8 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -54,9 +54,6 @@ pub async fn init_persister( let ln_data_dir = ln_data_dir(ctx, &ticker); let ln_data_backup_dir = ln_data_backup_dir(ctx, backup_path, &ticker); let persister = Arc::new(LightningPersister::new( - // Todo: Maybe implement GetHistoryCoinType instead / use get_history_coin_type and storage_ticker functions - // should probably use platform orderbook_ticker this way we will have channels history and payments history - // for BTC instead of BTC-segwit for example ticker.replace('-', "_"), ln_data_dir, ln_data_backup_dir, diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 4b11c74966..cd6ad1b9c7 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -125,6 +125,7 @@ fn create_payments_history_table_sql(for_coin: &str) -> Result + " ( id INTEGER NOT NULL PRIMARY KEY, payment_hash VARCHAR(255) NOT NULL UNIQUE, + destination VARCHAR(255), preimage VARCHAR(255), secret VARCHAR(255), amount_msat INTEGER, @@ -153,7 +154,7 @@ fn insert_or_update_payment_sql(for_coin: &str) -> Result { let sql = "INSERT OR REPLACE INTO ".to_owned() + &table_name - + " (payment_hash, preimage, secret, amount_msat, fee_paid_msat, is_outbound, status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);"; + + " (payment_hash, destination, preimage, secret, amount_msat, fee_paid_msat, is_outbound, status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8);"; Ok(sql) } @@ -171,9 +172,11 @@ fn select_payment_from_table_by_hash_sql(for_coin: &str) -> Result) -> Result) -> Result<(PaymentInfo, PaymentType), SqlError> { +fn payment_info_from_row(row: &Row<'_>) -> Result { + let is_outbound = row.get::<_, bool>(7)?; + let payment_type = match is_outbound { + true => PaymentType::OutboundPayment { + destination: row + .get::<_, String>(1) + .ok() + .map(|d| PublicKey::from_str(&d).expect("PublicKey from str should not fail!")), + }, + false => PaymentType::InboundPayment, + }; let payment_info = PaymentInfo { payment_hash: PaymentHash( hex::decode(row.get::<_, String>(0)?) @@ -205,7 +218,8 @@ fn payment_info_from_row(row: &Row<'_>) -> Result<(PaymentInfo, PaymentType), Sq .try_into() .expect("String should be 64 characters!"), ), - preimage: row.get::<_, String>(1).ok().map(|p| { + payment_type, + preimage: row.get::<_, String>(2).ok().map(|p| { PaymentPreimage( hex::decode(p) .expect("Preimage decoding should not fail!") @@ -213,7 +227,7 @@ fn payment_info_from_row(row: &Row<'_>) -> Result<(PaymentInfo, PaymentType), Sq .expect("String should be 64 characters!"), ) }), - secret: row.get::<_, String>(2).ok().map(|s| { + secret: row.get::<_, String>(3).ok().map(|s| { PaymentSecret( hex::decode(s) .expect("Secret decoding should not fail!") @@ -221,16 +235,11 @@ fn payment_info_from_row(row: &Row<'_>) -> Result<(PaymentInfo, PaymentType), Sq .expect("String should be 64 characters!"), ) }), - amt_msat: row.get::<_, u32>(3).ok().map(|v| v as u64), - fee_paid_msat: row.get::<_, u32>(4).ok().map(|v| v as u64), - status: HTLCStatus::from_str(&row.get::<_, String>(5)?)?, + amt_msat: row.get::<_, u32>(4).ok().map(|v| v as u64), + fee_paid_msat: row.get::<_, u32>(5).ok().map(|v| v as u64), + status: HTLCStatus::from_str(&row.get::<_, String>(6)?)?, }; - let is_outbound = row.get::<_, bool>(6)?; - let payment_type = match is_outbound { - true => PaymentType::OutboundPayment, - false => PaymentType::InboundPayment, - }; - Ok((payment_info, payment_type)) + Ok(payment_info) } fn get_last_channel_rpc_id_sql(for_coin: &str) -> Result { @@ -284,9 +293,11 @@ fn get_outbound_payments_sql(for_coin: &str) -> Result { let table_name = payments_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "SELECT payment_hash, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound FROM ".to_owned() - + &table_name - + " WHERE is_outbound = 1;"; + let sql = + "SELECT payment_hash, destination, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound FROM " + .to_owned() + + &table_name + + " WHERE is_outbound = 1;"; Ok(sql) } @@ -295,9 +306,11 @@ fn get_inbound_payments_sql(for_coin: &str) -> Result { let table_name = payments_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "SELECT payment_hash, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound FROM ".to_owned() - + &table_name - + " WHERE is_outbound = 0;"; + let sql = + "SELECT payment_hash, destination, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound FROM " + .to_owned() + + &table_name + + " WHERE is_outbound = 0;"; Ok(sql) } @@ -736,27 +749,24 @@ impl SqlStorage for LightningPersister { .await } - async fn add_or_update_payment_in_sql( - &self, - info: PaymentInfo, - payment_type: PaymentType, - ) -> Result<(), Self::Error> { + async fn add_or_update_payment_in_sql(&self, info: PaymentInfo) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); let payment_hash = hex::encode(info.payment_hash.0); + let (is_outbound, destination) = match info.payment_type { + PaymentType::OutboundPayment { destination } => (true as i32, destination.map(|d| d.to_string())), + PaymentType::InboundPayment => (false as i32, None), + }; let preimage = info.preimage.map(|p| hex::encode(p.0)); let secret = info.secret.map(|s| hex::encode(s.0)); let amount_msat = info.amt_msat.map(|a| a as u32); let fee_paid_msat = info.fee_paid_msat.map(|f| f as u32); - let is_outbound = match payment_type { - PaymentType::OutboundPayment => true as i32, - PaymentType::InboundPayment => false as i32, - }; let status = info.status.to_string(); let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { let params = [ &payment_hash as &dyn ToSql, + &destination as &dyn ToSql, &preimage as &dyn ToSql, &secret as &dyn ToSql, &amount_msat as &dyn ToSql, @@ -785,7 +795,7 @@ impl SqlStorage for LightningPersister { .await } - async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error> { + async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error> { let params = [hex::encode(hash.0)]; let sql = select_payment_from_table_by_hash_sql(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); @@ -883,7 +893,7 @@ impl SqlStorage for LightningPersister { .await } - async fn get_outbound_payments(&self) -> Result, Self::Error> { + async fn get_outbound_payments(&self) -> Result, Self::Error> { let sql = get_outbound_payments_sql(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); @@ -897,7 +907,7 @@ impl SqlStorage for LightningPersister { .await } - async fn get_inbound_payments(&self) -> Result, Self::Error> { + async fn get_inbound_payments(&self) -> Result, Self::Error> { let sql = get_inbound_payments_sql(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); @@ -1294,36 +1304,39 @@ mod tests { let mut expected_payment_info = PaymentInfo { payment_hash: PaymentHash([0; 32]), + payment_type: PaymentType::InboundPayment, preimage: Some(PaymentPreimage([2; 32])), secret: Some(PaymentSecret([3; 32])), amt_msat: Some(2000), fee_paid_msat: Some(100), status: HTLCStatus::Failed, }; - block_on(persister.add_or_update_payment_in_sql(expected_payment_info.clone(), PaymentType::InboundPayment)) - .unwrap(); + block_on(persister.add_or_update_payment_in_sql(expected_payment_info.clone())).unwrap(); let actual_payment_info = block_on(persister.get_payment_from_sql(PaymentHash([0; 32]))) .unwrap() .unwrap(); - assert_eq!(expected_payment_info, actual_payment_info.0); + assert_eq!(expected_payment_info, actual_payment_info); expected_payment_info.payment_hash = PaymentHash([1; 32]); + expected_payment_info.payment_type = PaymentType::OutboundPayment { + destination: Some( + PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), + ), + }; expected_payment_info.secret = None; expected_payment_info.amt_msat = None; expected_payment_info.status = HTLCStatus::Succeeded; - block_on(persister.add_or_update_payment_in_sql(expected_payment_info.clone(), PaymentType::OutboundPayment)) - .unwrap(); + block_on(persister.add_or_update_payment_in_sql(expected_payment_info.clone())).unwrap(); let actual_payment_info = block_on(persister.get_payment_from_sql(PaymentHash([1; 32]))) .unwrap() .unwrap(); - assert_eq!(expected_payment_info, actual_payment_info.0); + assert_eq!(expected_payment_info, actual_payment_info); // Update the first payment to outbound expected_payment_info.payment_hash = PaymentHash([0; 32]); - block_on(persister.add_or_update_payment_in_sql(expected_payment_info.clone(), PaymentType::OutboundPayment)) - .unwrap(); + block_on(persister.add_or_update_payment_in_sql(expected_payment_info.clone())).unwrap(); let outbound_payments = block_on(persister.get_outbound_payments()).unwrap(); assert_eq!(outbound_payments.len(), 2); } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index cb837e984c..eb765a4471 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -109,17 +109,16 @@ impl FromStr for HTLCStatus { } } -#[derive(Serialize)] +#[derive(Clone, Debug, PartialEq)] pub enum PaymentType { - #[serde(rename = "Outbound Payment")] - OutboundPayment, - #[serde(rename = "Inbound Payment")] + OutboundPayment { destination: Option }, InboundPayment, } #[derive(Clone, Debug, PartialEq)] pub struct PaymentInfo { pub payment_hash: PaymentHash, + pub payment_type: PaymentType, pub preimage: Option, pub secret: Option, pub amt_msat: Option, @@ -138,15 +137,11 @@ pub trait SqlStorage { async fn add_channel_to_sql(&self, details: SqlChannelDetails) -> Result<(), Self::Error>; - async fn add_or_update_payment_in_sql( - &self, - info: PaymentInfo, - payment_type: PaymentType, - ) -> Result<(), Self::Error>; + async fn add_or_update_payment_in_sql(&self, info: PaymentInfo) -> Result<(), Self::Error>; async fn get_channel_from_sql(&self, rpc_id: u64) -> Result, Self::Error>; - async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error>; + async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error>; async fn get_last_channel_rpc_id(&self) -> Result; @@ -164,9 +159,9 @@ pub trait SqlStorage { async fn get_closed_channels(&self) -> Result, Self::Error>; - async fn get_outbound_payments(&self) -> Result, Self::Error>; + async fn get_outbound_payments(&self) -> Result, Self::Error>; - async fn get_inbound_payments(&self) -> Result, Self::Error>; + async fn get_inbound_payments(&self) -> Result, Self::Error>; async fn add_claiming_tx_to_sql( &self, From 892e2b97311f12b32c120e226abfef0679d7639d Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Mon, 28 Mar 2022 14:44:24 +0200 Subject: [PATCH 25/49] add description to payment info --- mm2src/coins/lightning.rs | 34 ++++++++++++-- mm2src/coins/lightning/ln_errors.rs | 10 +++- mm2src/coins/lightning/ln_events.rs | 47 +++++++++++++------ mm2src/coins/lightning_persister/src/lib.rs | 25 ++++++---- .../coins/lightning_persister/src/storage.rs | 1 + 5 files changed, 88 insertions(+), 29 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index bc1292c6d8..f62edcdc5f 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -39,7 +39,7 @@ use lightning::util::config::UserConfig; use lightning_background_processor::BackgroundProcessor; use lightning_invoice::payment; use lightning_invoice::utils::{create_invoice_from_channelmanager, DefaultRouter}; -use lightning_invoice::Invoice; +use lightning_invoice::{Invoice, InvoiceDescription}; use lightning_persister::storage::{FileSystemStorage, HTLCStatus, NodesAddressesMapShared, PaymentInfo, PaymentType, Scorer, SqlChannelDetails, SqlStorage}; use lightning_persister::LightningPersister; @@ -128,10 +128,15 @@ impl LightningCoin { let payment_type = PaymentType::OutboundPayment { destination: invoice.payee_pub_key().cloned(), }; + let description = Some(match invoice.description() { + InvoiceDescription::Direct(d) => d.to_string(), + InvoiceDescription::Hash(h) => hex::encode(h.0.into_inner()), + }); let payment_secret = Some(*invoice.payment_secret()); Ok(PaymentInfo { payment_hash, payment_type, + description, preimage: None, secret: payment_secret, amt_msat: invoice.amount_milli_satoshis(), @@ -164,6 +169,7 @@ impl LightningCoin { Ok(PaymentInfo { payment_hash, payment_type, + description: None, preimage: Some(payment_preimage), secret: None, amt_msat: Some(amount_msat), @@ -954,10 +960,22 @@ pub async fn generate_invoice( ln_coin.keys_manager, network, req.amount_in_msat, - req.description, + req.description.clone(), )?; + let payment_hash = invoice.payment_hash().into_inner(); + let payment_info = PaymentInfo { + payment_hash: PaymentHash(payment_hash), + payment_type: PaymentType::InboundPayment, + description: Some(req.description), + preimage: None, + secret: Some(*invoice.payment_secret()), + amt_msat: req.amount_in_msat, + fee_paid_msat: None, + status: HTLCStatus::Pending, + }; + ln_coin.persister.add_or_update_payment_in_sql(payment_info).await?; Ok(GenerateInvoiceResponse { - payment_hash: invoice.payment_hash().into_inner().into(), + payment_hash: payment_hash.into(), invoice: invoice.into(), }) } @@ -1032,7 +1050,10 @@ pub struct ListPaymentsReq { #[serde(tag = "type")] pub enum PaymentTypeForRPC { #[serde(rename = "Outbound Payment")] - OutboundPayment { destination: Option }, + OutboundPayment { + #[serde(skip_serializing_if = "Option::is_none")] + destination: Option, + }, #[serde(rename = "Inbound Payment")] InboundPayment, } @@ -1052,7 +1073,11 @@ impl From for PaymentTypeForRPC { pub struct PaymentInfoForRPC { payment_hash: H256Json, payment_type: PaymentTypeForRPC, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + #[serde(skip_serializing_if = "Option::is_none")] amount_in_msat: Option, + #[serde(skip_serializing_if = "Option::is_none")] fee_paid_msat: Option, status: HTLCStatus, } @@ -1062,6 +1087,7 @@ impl From for PaymentInfoForRPC { PaymentInfoForRPC { payment_hash: info.payment_hash.0.into(), payment_type: info.payment_type.into(), + description: info.description, amount_in_msat: info.amt_msat, fee_paid_msat: info.fee_paid_msat, status: info.status, diff --git a/mm2src/coins/lightning/ln_errors.rs b/mm2src/coins/lightning/ln_errors.rs index e61941665b..a43da0508a 100644 --- a/mm2src/coins/lightning/ln_errors.rs +++ b/mm2src/coins/lightning/ln_errors.rs @@ -292,6 +292,8 @@ pub enum GenerateInvoiceError { NoSuchCoin(String), #[display(fmt = "Invoice signing or creation error: {}", _0)] SignOrCreationError(String), + #[display(fmt = "SQL error {}", _0)] + SqlError(String), } impl HttpStatusCode for GenerateInvoiceError { @@ -299,7 +301,9 @@ impl HttpStatusCode for GenerateInvoiceError { match self { GenerateInvoiceError::UnsupportedCoin(_) => StatusCode::BAD_REQUEST, GenerateInvoiceError::NoSuchCoin(_) => StatusCode::PRECONDITION_REQUIRED, - GenerateInvoiceError::SignOrCreationError(_) => StatusCode::INTERNAL_SERVER_ERROR, + GenerateInvoiceError::SignOrCreationError(_) | GenerateInvoiceError::SqlError(_) => { + StatusCode::INTERNAL_SERVER_ERROR + }, } } } @@ -316,6 +320,10 @@ impl From for GenerateInvoiceError { fn from(e: SignOrCreationError) -> Self { GenerateInvoiceError::SignOrCreationError(e.to_string()) } } +impl From for GenerateInvoiceError { + fn from(err: SqlError) -> GenerateInvoiceError { GenerateInvoiceError::SqlError(err.to_string()) } +} + #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] pub enum SendPaymentError { diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 297bf3926b..445ae545c2 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -250,21 +250,40 @@ impl LightningEventHandler { }, false => HTLCStatus::Failed, }; - let payment_info = PaymentInfo { - payment_hash, - payment_type: PaymentType::InboundPayment, - preimage: Some(payment_preimage), - secret: payment_secret, - amt_msat: Some(amt), - fee_paid_msat: None, - status, - }; let persister = self.persister.clone(); - spawn(async move { - if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { - log::error!("{}", e); - } - }); + match purpose { + PaymentPurpose::InvoicePayment { .. } => spawn(async move { + if let Ok(Some(mut payment_info)) = persister + .get_payment_from_sql(payment_hash) + .await + .error_log_passthrough() + { + payment_info.preimage = Some(payment_preimage); + payment_info.status = HTLCStatus::Succeeded; + payment_info.amt_msat = Some(amt); + if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { + log::error!("{}", e); + } + } + }), + PaymentPurpose::SpontaneousPayment(_) => { + let payment_info = PaymentInfo { + payment_hash, + payment_type: PaymentType::InboundPayment, + description: None, + preimage: Some(payment_preimage), + secret: payment_secret, + amt_msat: Some(amt), + fee_paid_msat: None, + status, + }; + spawn(async move { + if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { + log::error!("{}", e); + } + }); + }, + } } fn handle_payment_sent( diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index cd6ad1b9c7..228907694d 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -126,6 +126,7 @@ fn create_payments_history_table_sql(for_coin: &str) -> Result id INTEGER NOT NULL PRIMARY KEY, payment_hash VARCHAR(255) NOT NULL UNIQUE, destination VARCHAR(255), + description VARCHAR(641), preimage VARCHAR(255), secret VARCHAR(255), amount_msat INTEGER, @@ -154,7 +155,7 @@ fn insert_or_update_payment_sql(for_coin: &str) -> Result { let sql = "INSERT OR REPLACE INTO ".to_owned() + &table_name - + " (payment_hash, destination, preimage, secret, amount_msat, fee_paid_msat, is_outbound, status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8);"; + + " (payment_hash, destination, description, preimage, secret, amount_msat, fee_paid_msat, is_outbound, status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9);"; Ok(sql) } @@ -173,7 +174,7 @@ fn select_payment_from_table_by_hash_sql(for_coin: &str) -> Result) -> Result) -> Result { - let is_outbound = row.get::<_, bool>(7)?; + let is_outbound = row.get::<_, bool>(8)?; let payment_type = match is_outbound { true => PaymentType::OutboundPayment { destination: row @@ -219,7 +220,8 @@ fn payment_info_from_row(row: &Row<'_>) -> Result { .expect("String should be 64 characters!"), ), payment_type, - preimage: row.get::<_, String>(2).ok().map(|p| { + description: row.get(2).ok(), + preimage: row.get::<_, String>(3).ok().map(|p| { PaymentPreimage( hex::decode(p) .expect("Preimage decoding should not fail!") @@ -227,7 +229,7 @@ fn payment_info_from_row(row: &Row<'_>) -> Result { .expect("String should be 64 characters!"), ) }), - secret: row.get::<_, String>(3).ok().map(|s| { + secret: row.get::<_, String>(4).ok().map(|s| { PaymentSecret( hex::decode(s) .expect("Secret decoding should not fail!") @@ -235,9 +237,9 @@ fn payment_info_from_row(row: &Row<'_>) -> Result { .expect("String should be 64 characters!"), ) }), - amt_msat: row.get::<_, u32>(4).ok().map(|v| v as u64), - fee_paid_msat: row.get::<_, u32>(5).ok().map(|v| v as u64), - status: HTLCStatus::from_str(&row.get::<_, String>(6)?)?, + amt_msat: row.get::<_, u32>(5).ok().map(|v| v as u64), + fee_paid_msat: row.get::<_, u32>(6).ok().map(|v| v as u64), + status: HTLCStatus::from_str(&row.get::<_, String>(7)?)?, }; Ok(payment_info) } @@ -294,7 +296,7 @@ fn get_outbound_payments_sql(for_coin: &str) -> Result { validate_table_name(&table_name)?; let sql = - "SELECT payment_hash, destination, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound FROM " + "SELECT payment_hash, destination, description, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound FROM " .to_owned() + &table_name + " WHERE is_outbound = 1;"; @@ -307,7 +309,7 @@ fn get_inbound_payments_sql(for_coin: &str) -> Result { validate_table_name(&table_name)?; let sql = - "SELECT payment_hash, destination, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound FROM " + "SELECT payment_hash, destination, description, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound FROM " .to_owned() + &table_name + " WHERE is_outbound = 0;"; @@ -756,6 +758,7 @@ impl SqlStorage for LightningPersister { PaymentType::OutboundPayment { destination } => (true as i32, destination.map(|d| d.to_string())), PaymentType::InboundPayment => (false as i32, None), }; + let description = info.description; let preimage = info.preimage.map(|p| hex::encode(p.0)); let secret = info.secret.map(|s| hex::encode(s.0)); let amount_msat = info.amt_msat.map(|a| a as u32); @@ -767,6 +770,7 @@ impl SqlStorage for LightningPersister { let params = [ &payment_hash as &dyn ToSql, &destination as &dyn ToSql, + &description as &dyn ToSql, &preimage as &dyn ToSql, &secret as &dyn ToSql, &amount_msat as &dyn ToSql, @@ -1305,6 +1309,7 @@ mod tests { let mut expected_payment_info = PaymentInfo { payment_hash: PaymentHash([0; 32]), payment_type: PaymentType::InboundPayment, + description: Some("test payment".into()), preimage: Some(PaymentPreimage([2; 32])), secret: Some(PaymentSecret([3; 32])), amt_msat: Some(2000), diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index eb765a4471..d19204d234 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -119,6 +119,7 @@ pub enum PaymentType { pub struct PaymentInfo { pub payment_hash: PaymentHash, pub payment_type: PaymentType, + pub description: Option, pub preimage: Option, pub secret: Option, pub amt_msat: Option, From 316200a107cf275bda8b54e462ed5705e43353b6 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Mon, 28 Mar 2022 16:11:55 +0200 Subject: [PATCH 26/49] add created/updated at to payment info, wip: list payments by filter --- mm2src/coins/lightning.rs | 30 +++++++++++++++++-- mm2src/coins/lightning/ln_events.rs | 5 ++++ mm2src/coins/lightning_persister/src/lib.rs | 23 ++++++++++---- .../coins/lightning_persister/src/storage.rs | 6 ++-- mm2src/rpc/dispatcher/dispatcher.rs | 4 +-- 5 files changed, 55 insertions(+), 13 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index f62edcdc5f..b9bc02f078 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -25,7 +25,7 @@ use common::log::{LogOnError, LogState}; use common::mm_ctx::MmArc; use common::mm_error::prelude::*; use common::mm_number::MmNumber; -use common::{async_blocking, log}; +use common::{async_blocking, log, now_ms}; use futures::{FutureExt, TryFutureExt}; use futures01::Future; use keys::{AddressHashEnum, KeyPair}; @@ -142,6 +142,8 @@ impl LightningCoin { amt_msat: invoice.amount_milli_satoshis(), fee_paid_msat: None, status: HTLCStatus::Pending, + created_at: now_ms() / 1000, + last_updated: now_ms() / 1000, }) } @@ -175,6 +177,8 @@ impl LightningCoin { amt_msat: Some(amount_msat), fee_paid_msat: None, status: HTLCStatus::Pending, + created_at: now_ms() / 1000, + last_updated: now_ms() / 1000, }) } } @@ -972,6 +976,8 @@ pub async fn generate_invoice( amt_msat: req.amount_in_msat, fee_paid_msat: None, status: HTLCStatus::Pending, + created_at: now_ms() / 1000, + last_updated: now_ms() / 1000, }; ln_coin.persister.add_or_update_payment_in_sql(payment_info).await?; Ok(GenerateInvoiceResponse { @@ -1041,12 +1047,26 @@ pub async fn send_payment(ctx: MmArc, req: SendPaymentReq) -> SendPaymentResult< }) } +#[derive(Deserialize)] +pub struct PaymentsFilter { + pub payment_type: Option, + pub description: Option, + pub status: Option, + pub from_amount_msat: Option, + pub to_amount_msat: Option, + pub from_fee_paid_msat: Option, + pub to_fee_paid_msat: Option, + pub from_timestamp: Option, + pub to_timestamp: Option, +} + #[derive(Deserialize)] pub struct ListPaymentsReq { pub coin: String, + pub filter: Option, } -#[derive(Serialize)] +#[derive(Deserialize, Serialize)] #[serde(tag = "type")] pub enum PaymentTypeForRPC { #[serde(rename = "Outbound Payment")] @@ -1080,6 +1100,8 @@ pub struct PaymentInfoForRPC { #[serde(skip_serializing_if = "Option::is_none")] fee_paid_msat: Option, status: HTLCStatus, + created_at: u64, + last_updated: u64, } impl From for PaymentInfoForRPC { @@ -1091,6 +1113,8 @@ impl From for PaymentInfoForRPC { amount_in_msat: info.amt_msat, fee_paid_msat: info.fee_paid_msat, status: info.status, + created_at: info.created_at, + last_updated: info.last_updated, } } } @@ -1101,7 +1125,7 @@ pub struct ListPaymentsResponse { pub outbound_payments: Vec, } -pub async fn list_payments(ctx: MmArc, req: ListPaymentsReq) -> ListPaymentsResult { +pub async fn list_payments_by_filter(ctx: MmArc, req: ListPaymentsReq) -> ListPaymentsResult { let coin = lp_coinfind_or_err(&ctx, &req.coin).await?; let ln_coin = match coin { MmCoinEnum::LightningCoin(c) => c, diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 445ae545c2..64d3ab09d6 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -261,6 +261,7 @@ impl LightningEventHandler { payment_info.preimage = Some(payment_preimage); payment_info.status = HTLCStatus::Succeeded; payment_info.amt_msat = Some(amt); + payment_info.last_updated = now_ms() / 1000; if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { log::error!("{}", e); } @@ -276,6 +277,8 @@ impl LightningEventHandler { amt_msat: Some(amt), fee_paid_msat: None, status, + created_at: now_ms() / 1000, + last_updated: now_ms() / 1000, }; spawn(async move { if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { @@ -306,6 +309,7 @@ impl LightningEventHandler { payment_info.preimage = Some(payment_preimage); payment_info.status = HTLCStatus::Succeeded; payment_info.fee_paid_msat = fee_paid_msat; + payment_info.last_updated = now_ms() / 1000; let amt_msat = payment_info.amt_msat; if let Err(e) = persister .add_or_update_payment_in_sql(payment_info) @@ -397,6 +401,7 @@ impl LightningEventHandler { .error_log_passthrough() { payment_info.status = HTLCStatus::Failed; + payment_info.last_updated = now_ms() / 1000; if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { log::error!("{}", e); } diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 228907694d..f8f635bd7a 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -132,7 +132,9 @@ fn create_payments_history_table_sql(for_coin: &str) -> Result amount_msat INTEGER, fee_paid_msat INTEGER, is_outbound INTEGER NOT NULL, - status VARCHAR(255) NOT NULL + status VARCHAR(255) NOT NULL, + created_at INTEGER NOT NULL, + last_updated INTEGER NOT NULL );"; Ok(sql) @@ -155,7 +157,7 @@ fn insert_or_update_payment_sql(for_coin: &str) -> Result { let sql = "INSERT OR REPLACE INTO ".to_owned() + &table_name - + " (payment_hash, destination, description, preimage, secret, amount_msat, fee_paid_msat, is_outbound, status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9);"; + + " (payment_hash, destination, description, preimage, secret, amount_msat, fee_paid_msat, is_outbound, status, created_at, last_updated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11);"; Ok(sql) } @@ -174,7 +176,7 @@ fn select_payment_from_table_by_hash_sql(for_coin: &str) -> Result) -> Result { amt_msat: row.get::<_, u32>(5).ok().map(|v| v as u64), fee_paid_msat: row.get::<_, u32>(6).ok().map(|v| v as u64), status: HTLCStatus::from_str(&row.get::<_, String>(7)?)?, + created_at: row.get::<_, u32>(9)? as u64, + last_updated: row.get::<_, u32>(10)? as u64, }; Ok(payment_info) } @@ -296,7 +300,7 @@ fn get_outbound_payments_sql(for_coin: &str) -> Result { validate_table_name(&table_name)?; let sql = - "SELECT payment_hash, destination, description, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound FROM " + "SELECT payment_hash, destination, description, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound, created_at, last_updated FROM " .to_owned() + &table_name + " WHERE is_outbound = 1;"; @@ -309,7 +313,7 @@ fn get_inbound_payments_sql(for_coin: &str) -> Result { validate_table_name(&table_name)?; let sql = - "SELECT payment_hash, destination, description, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound FROM " + "SELECT payment_hash, destination, description, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound, created_at, last_updated FROM " .to_owned() + &table_name + " WHERE is_outbound = 0;"; @@ -764,6 +768,8 @@ impl SqlStorage for LightningPersister { let amount_msat = info.amt_msat.map(|a| a as u32); let fee_paid_msat = info.fee_paid_msat.map(|f| f as u32); let status = info.status.to_string(); + let created_at = info.created_at as u32; + let last_updated = info.last_updated as u32; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { @@ -777,6 +783,8 @@ impl SqlStorage for LightningPersister { &fee_paid_msat as &dyn ToSql, &is_outbound as &dyn ToSql, &status as &dyn ToSql, + &created_at as &dyn ToSql, + &last_updated as &dyn ToSql, ]; let mut conn = sqlite_connection.lock().unwrap(); let sql_transaction = conn.transaction()?; @@ -956,7 +964,7 @@ mod tests { use bitcoin::blockdata::block::{Block, BlockHeader}; use bitcoin::hashes::hex::FromHex; use bitcoin::Txid; - use common::block_on; + use common::{block_on, now_ms}; use db_common::sqlite::rusqlite::Connection; use lightning::chain::chainmonitor::Persist; use lightning::chain::transaction::OutPoint; @@ -1315,6 +1323,8 @@ mod tests { amt_msat: Some(2000), fee_paid_msat: Some(100), status: HTLCStatus::Failed, + created_at: now_ms() / 1000, + last_updated: now_ms() / 1000, }; block_on(persister.add_or_update_payment_in_sql(expected_payment_info.clone())).unwrap(); @@ -1332,6 +1342,7 @@ mod tests { expected_payment_info.secret = None; expected_payment_info.amt_msat = None; expected_payment_info.status = HTLCStatus::Succeeded; + expected_payment_info.last_updated = now_ms() / 1000; block_on(persister.add_or_update_payment_in_sql(expected_payment_info.clone())).unwrap(); let actual_payment_info = block_on(persister.get_payment_from_sql(PaymentHash([1; 32]))) diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index d19204d234..4289443096 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -7,7 +7,7 @@ use lightning::routing::network_graph::NetworkGraph; use lightning::routing::scoring::ProbabilisticScorer; use parking_lot::Mutex as PaMutex; use secp256k1::PublicKey; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::net::SocketAddr; use std::str::FromStr; @@ -88,7 +88,7 @@ impl SqlChannelDetails { } } -#[derive(Clone, Debug, Display, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, Display, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] pub enum HTLCStatus { Pending, @@ -125,6 +125,8 @@ pub struct PaymentInfo { pub amt_msat: Option, pub fee_paid_msat: Option, pub status: HTLCStatus, + pub created_at: u64, + pub last_updated: u64, } #[async_trait] diff --git a/mm2src/rpc/dispatcher/dispatcher.rs b/mm2src/rpc/dispatcher/dispatcher.rs index c4fe2a6851..cc675be89c 100644 --- a/mm2src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/rpc/dispatcher/dispatcher.rs @@ -32,7 +32,7 @@ use std::net::SocketAddr; cfg_native! { use coins::lightning::{close_channel, connect_to_lightning_node, generate_invoice, get_channel_details, - get_claimable_balances, get_payment_details, list_channels, list_payments, open_channel, + get_claimable_balances, get_payment_details, list_channels, list_payments_by_filter, open_channel, send_payment, LightningCoin}; } @@ -161,7 +161,7 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, get_claimable_balances).await, "get_payment_details" => handle_mmrpc(ctx, request, get_payment_details).await, "list_channels" => handle_mmrpc(ctx, request, list_channels).await, - "list_payments" => handle_mmrpc(ctx, request, list_payments).await, + "list_payments_by_filter" => handle_mmrpc(ctx, request, list_payments_by_filter).await, "open_channel" => handle_mmrpc(ctx, request, open_channel).await, "send_payment" => handle_mmrpc(ctx, request, send_payment).await, _ => MmError::err(DispatcherError::NoSuchMethod), From 70bcc0622bffbac7bf9f305f61f3904b941e5e8d Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Wed, 30 Mar 2022 15:12:40 +0200 Subject: [PATCH 27/49] list_payments_by_filter completed, pagination completed --- Cargo.lock | 1 + mm2src/coins/lightning.rs | 75 ++-- mm2src/coins/lightning_persister/Cargo.toml | 3 +- mm2src/coins/lightning_persister/src/lib.rs | 347 +++++++++++++++--- .../coins/lightning_persister/src/storage.rs | 29 +- mm2src/common/common.rs | 12 + 6 files changed, 401 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a54d028e7d..1af09c92dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3073,6 +3073,7 @@ dependencies = [ "libc", "lightning", "parking_lot 0.12.0", + "rand 0.7.3", "secp256k1", "serde", "serde_json", diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index b9bc02f078..dfefbdb1a0 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -25,7 +25,7 @@ use common::log::{LogOnError, LogState}; use common::mm_ctx::MmArc; use common::mm_error::prelude::*; use common::mm_number::MmNumber; -use common::{async_blocking, log, now_ms}; +use common::{async_blocking, calc_total_pages, log, now_ms, ten, PagingOptionsEnum}; use futures::{FutureExt, TryFutureExt}; use futures01::Future; use keys::{AddressHashEnum, KeyPair}; @@ -41,7 +41,7 @@ use lightning_invoice::payment; use lightning_invoice::utils::{create_invoice_from_channelmanager, DefaultRouter}; use lightning_invoice::{Invoice, InvoiceDescription}; use lightning_persister::storage::{FileSystemStorage, HTLCStatus, NodesAddressesMapShared, PaymentInfo, PaymentType, - Scorer, SqlChannelDetails, SqlStorage}; + PaymentsFilter, Scorer, SqlChannelDetails, SqlStorage}; use lightning_persister::LightningPersister; use ln_conf::{ChannelOptions, LightningCoinConf, LightningProtocolConf, PlatformCoinConfirmations}; use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelError, CloseChannelResult, @@ -1048,7 +1048,7 @@ pub async fn send_payment(ctx: MmArc, req: SendPaymentReq) -> SendPaymentResult< } #[derive(Deserialize)] -pub struct PaymentsFilter { +pub struct PaymentsFilterForRPC { pub payment_type: Option, pub description: Option, pub status: Option, @@ -1060,10 +1060,30 @@ pub struct PaymentsFilter { pub to_timestamp: Option, } +impl From for PaymentsFilter { + fn from(filter: PaymentsFilterForRPC) -> Self { + PaymentsFilter { + payment_type: filter.payment_type.map(From::from), + description: filter.description, + status: filter.status, + from_amount_msat: filter.from_amount_msat, + to_amount_msat: filter.to_amount_msat, + from_fee_paid_msat: filter.from_fee_paid_msat, + to_fee_paid_msat: filter.to_fee_paid_msat, + from_timestamp: filter.from_timestamp, + to_timestamp: filter.to_timestamp, + } + } +} + #[derive(Deserialize)] pub struct ListPaymentsReq { pub coin: String, - pub filter: Option, + pub filter: Option, + #[serde(default = "ten")] + limit: usize, + #[serde(default)] + paging_options: PagingOptionsEnum, } #[derive(Deserialize, Serialize)] @@ -1089,6 +1109,17 @@ impl From for PaymentTypeForRPC { } } +impl From for PaymentType { + fn from(payment_type: PaymentTypeForRPC) -> Self { + match payment_type { + PaymentTypeForRPC::OutboundPayment { destination } => PaymentType::OutboundPayment { + destination: destination.map(From::from), + }, + PaymentTypeForRPC::InboundPayment => PaymentType::InboundPayment, + } + } +} + #[derive(Serialize)] pub struct PaymentInfoForRPC { payment_hash: H256Json, @@ -1121,8 +1152,12 @@ impl From for PaymentInfoForRPC { #[derive(Serialize)] pub struct ListPaymentsResponse { - pub inbound_payments: Vec, - pub outbound_payments: Vec, + payments: Vec, + limit: usize, + skipped: usize, + total: usize, + total_pages: usize, + paging_options: PagingOptionsEnum, } pub async fn list_payments_by_filter(ctx: MmArc, req: ListPaymentsReq) -> ListPaymentsResult { @@ -1131,24 +1166,22 @@ pub async fn list_payments_by_filter(ctx: MmArc, req: ListPaymentsReq) -> ListPa MmCoinEnum::LightningCoin(c) => c, _ => return MmError::err(ListPaymentsError::UnsupportedCoin(coin.ticker().to_string())), }; - let inbound_payments = ln_coin + let get_payments_res = ln_coin .persister - .get_inbound_payments() - .await? - .into_iter() - .map(From::from) - .collect(); - let outbound_payments = ln_coin - .persister - .get_outbound_payments() - .await? - .into_iter() - .map(From::from) - .collect(); + .get_payments_by_filter( + req.filter.map(From::from), + req.paging_options.clone().map(|h| PaymentHash(h.0)), + req.limit, + ) + .await?; Ok(ListPaymentsResponse { - inbound_payments, - outbound_payments, + payments: get_payments_res.payments.into_iter().map(From::from).collect(), + limit: req.limit, + skipped: get_payments_res.skipped, + total: get_payments_res.total, + total_pages: calc_total_pages(get_payments_res.total, req.limit), + paging_options: req.paging_options, }) } diff --git a/mm2src/coins/lightning_persister/Cargo.toml b/mm2src/coins/lightning_persister/Cargo.toml index 33e9e8a80f..b474276f26 100644 --- a/mm2src/coins/lightning_persister/Cargo.toml +++ b/mm2src/coins/lightning_persister/Cargo.toml @@ -27,4 +27,5 @@ serde_json = "1.0" winapi = { version = "0.3", features = ["winbase"] } [dev-dependencies] -lightning = { version = "0.0.105", features = ["_test_utils"] } \ No newline at end of file +lightning = { version = "0.0.105", features = ["_test_utils"] } +rand = { version = "0.7", features = ["std", "small_rng"] } \ No newline at end of file diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index f8f635bd7a..b2df8f81c2 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -13,18 +13,19 @@ extern crate lightning; extern crate secp256k1; extern crate serde_json; -use crate::storage::{FileSystemStorage, HTLCStatus, NodesAddressesMap, NodesAddressesMapShared, PaymentInfo, - PaymentType, Scorer, SqlChannelDetails, SqlStorage}; +use crate::storage::{FileSystemStorage, GetPaymentsResult, HTLCStatus, NodesAddressesMap, NodesAddressesMapShared, + PaymentInfo, PaymentType, PaymentsFilter, Scorer, SqlChannelDetails, SqlStorage}; use crate::util::DiskWriteable; use async_trait::async_trait; use bitcoin::blockdata::constants::genesis_block; use bitcoin::hash_types::{BlockHash, Txid}; use bitcoin::hashes::hex::{FromHex, ToHex}; use bitcoin::Network; -use common::async_blocking; use common::fs::check_dir_operations; +use common::{async_blocking, PagingOptionsEnum}; use db_common::sqlite::rusqlite::{Error as SqlError, Row, ToSql, NO_PARAMS}; -use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, SqliteConnShared, +use db_common::sqlite::sql_builder::SqlBuilder; +use db_common::sqlite::{offset_by_id, query_single_row, string_from_row, validate_table_name, SqliteConnShared, CHECK_TABLE_EXISTS_SQL}; use lightning::chain; use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator}; @@ -295,30 +296,83 @@ fn get_closed_channels_sql(for_coin: &str) -> Result { Ok(sql) } -fn get_outbound_payments_sql(for_coin: &str) -> Result { +fn get_payments_builder_preimage(for_coin: &str) -> Result { let table_name = payments_history_table(for_coin); validate_table_name(&table_name)?; - let sql = - "SELECT payment_hash, destination, description, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound, created_at, last_updated FROM " - .to_owned() - + &table_name - + " WHERE is_outbound = 1;"; + Ok(SqlBuilder::select_from(table_name)) +} - Ok(sql) +fn finalize_get_payments_sql_builder(sql_builder: &mut SqlBuilder, offset: usize, limit: usize) { + sql_builder + .field("payment_hash") + .field("destination") + .field("description") + .field("preimage") + .field("secret") + .field("amount_msat") + .field("fee_paid_msat") + .field("status") + .field("is_outbound") + .field("created_at") + .field("last_updated"); + sql_builder.offset(offset); + sql_builder.limit(limit); + sql_builder.order_desc("last_updated"); } -fn get_inbound_payments_sql(for_coin: &str) -> Result { - let table_name = payments_history_table(for_coin); - validate_table_name(&table_name)?; +fn apply_get_payments_filter(builder: &mut SqlBuilder, params: &mut Vec<(&str, String)>, filter: PaymentsFilter) { + if let Some(payment_type) = filter.payment_type { + let (is_outbound, destination) = match payment_type { + PaymentType::OutboundPayment { destination } => (true as i32, destination.map(|d| d.to_string())), + PaymentType::InboundPayment => (false as i32, None), + }; + if let Some(dest) = destination { + builder.and_where(format!("destination LIKE '%{}%'", dest)); + } - let sql = - "SELECT payment_hash, destination, description, preimage, secret, amount_msat, fee_paid_msat, status, is_outbound, created_at, last_updated FROM " - .to_owned() - + &table_name - + " WHERE is_outbound = 0;"; + builder.and_where("is_outbound = :is_outbound"); + params.push((":is_outbound", is_outbound.to_string())); + } - Ok(sql) + if let Some(description) = filter.description { + builder.and_where(format!("description LIKE '%{}%'", description)); + } + + if let Some(status) = filter.status { + builder.and_where("status = :status"); + params.push((":status", status.to_string())); + } + + if let Some(from_amount) = filter.from_amount_msat { + builder.and_where("amount_msat >= :from_amount"); + params.push((":from_amount", from_amount.to_string())); + } + + if let Some(to_amount) = filter.to_amount_msat { + builder.and_where("amount_msat <= :to_amount"); + params.push((":to_amount", to_amount.to_string())); + } + + if let Some(from_fee) = filter.from_fee_paid_msat { + builder.and_where("fee_paid_msat >= :from_fee"); + params.push((":from_fee", from_fee.to_string())); + } + + if let Some(to_fee) = filter.to_fee_paid_msat { + builder.and_where("fee_paid_msat <= :to_fee"); + params.push((":to_fee", to_fee.to_string())); + } + + if let Some(from_time) = filter.from_timestamp { + builder.and_where("created_at >= :from_time"); + params.push((":from_time", from_time.to_string())); + } + + if let Some(to_time) = filter.to_timestamp { + builder.and_where("created_at <= :to_time"); + params.push((":to_time", to_time.to_string())); + } } fn update_claiming_tx_sql(for_coin: &str) -> Result { @@ -905,29 +959,67 @@ impl SqlStorage for LightningPersister { .await } - async fn get_outbound_payments(&self) -> Result, Self::Error> { - let sql = get_outbound_payments_sql(self.storage_ticker.as_str())?; + async fn get_payments_by_filter( + &self, + filter: Option, + paging: PagingOptionsEnum, + limit: usize, + ) -> Result { + let mut sql_builder = get_payments_builder_preimage(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { let conn = sqlite_connection.lock().unwrap(); - let mut stmt = conn.prepare(&sql)?; - let rows = stmt.query(NO_PARAMS)?; - let result = rows.mapped(payment_info_from_row).collect::>()?; - Ok(result) - }) - .await - } - async fn get_inbound_payments(&self) -> Result, Self::Error> { - let sql = get_inbound_payments_sql(self.storage_ticker.as_str())?; - let sqlite_connection = self.sqlite_connection.clone(); + let mut total_builder = sql_builder.clone(); + total_builder.count("id"); + let total_sql = total_builder.sql().expect("valid sql"); + let total: isize = conn.query_row(&total_sql, NO_PARAMS, |row| row.get(0))?; + let total = total.try_into().expect("count should be always above zero"); + + let offset = match paging { + PagingOptionsEnum::PageNumber(page) => (page.get() - 1) * limit, + PagingOptionsEnum::FromId(hash) => { + let hash_str = hex::encode(hash.0); + let params = [&hash_str]; + let maybe_offset = offset_by_id( + &conn, + &sql_builder, + params, + "payment_hash", + "last_updated DESC", + "payment_hash = ?1", + )?; + match maybe_offset { + Some(offset) => offset, + None => { + return Ok(GetPaymentsResult { + payments: vec![], + skipped: 0, + total, + }) + }, + } + }, + }; - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); + let mut params = vec![]; + if let Some(f) = filter { + apply_get_payments_filter(&mut sql_builder, &mut params, f); + } + let params_as_trait: Vec<_> = params.iter().map(|(key, value)| (*key, value as &dyn ToSql)).collect(); + finalize_get_payments_sql_builder(&mut sql_builder, offset, limit); + + let sql = sql_builder.sql().expect("valid sql"); let mut stmt = conn.prepare(&sql)?; - let rows = stmt.query(NO_PARAMS)?; - let result = rows.mapped(payment_info_from_row).collect::>()?; + let payments = stmt + .query_map_named(params_as_trait.as_slice(), payment_info_from_row)? + .collect::>()?; + let result = GetPaymentsResult { + payments, + skipped: offset, + total, + }; Ok(result) }) .await @@ -974,7 +1066,11 @@ mod tests { use lightning::util::events::{ClosureReason, MessageSendEventsProvider}; use lightning::util::test_utils; use lightning::{check_added_monitors, check_closed_broadcast, check_closed_event}; + use rand::distributions::Alphanumeric; + use rand::{Rng, RngCore}; + use secp256k1::{Secp256k1, SecretKey}; use std::fs; + use std::num::NonZeroUsize; use std::path::PathBuf; use std::sync::{Arc, Mutex}; @@ -989,6 +1085,57 @@ mod tests { } } + fn generate_random_payments(num: u64) -> Vec { + let mut rng = rand::thread_rng(); + let mut payments = vec![]; + let s = Secp256k1::new(); + let mut bytes = [0; 32]; + for _ in 0..num { + let payment_type = if let 0 = rng.gen::() % 2 { + PaymentType::InboundPayment + } else { + rng.fill_bytes(&mut bytes); + let secret = SecretKey::from_slice(&bytes).unwrap(); + let pubkey = PublicKey::from_secret_key(&s, &secret); + PaymentType::OutboundPayment { + destination: Some(pubkey), + } + }; + let status_rng: u8 = rng.gen(); + let status = if status_rng % 3 == 0 { + HTLCStatus::Succeeded + } else if status_rng % 3 == 1 { + HTLCStatus::Pending + } else { + HTLCStatus::Failed + }; + let description: String = rng.sample_iter(&Alphanumeric).take(30).map(char::from).collect(); + let info = PaymentInfo { + payment_hash: { + rng.fill_bytes(&mut bytes); + PaymentHash(bytes) + }, + payment_type, + description: Some(description), + preimage: { + rng.fill_bytes(&mut bytes); + Some(PaymentPreimage(bytes)) + }, + secret: { + rng.fill_bytes(&mut bytes); + Some(PaymentSecret(bytes)) + }, + amt_msat: Some(rng.gen::() as u64), + fee_paid_msat: Some(rng.gen::() as u64), + status, + created_at: rng.gen::() as u64, + last_updated: rng.gen::() as u64, + }; + payments.push(info); + } + payments + } + // Integration-test the LightningPersister. Test relaying a few payments // and check that the persisted data is updated the appropriate number of // times. @@ -1349,11 +1496,129 @@ mod tests { .unwrap() .unwrap(); assert_eq!(expected_payment_info, actual_payment_info); + } - // Update the first payment to outbound - expected_payment_info.payment_hash = PaymentHash([0; 32]); - block_on(persister.add_or_update_payment_in_sql(expected_payment_info.clone())).unwrap(); - let outbound_payments = block_on(persister.get_outbound_payments()).unwrap(); - assert_eq!(outbound_payments.len(), 2); + #[test] + fn test_get_payments_by_filter() { + let persister = LightningPersister::new( + "test_get_payments_by_filter".into(), + PathBuf::from("test_filesystem_persister"), + None, + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + + block_on(persister.init_sql()).unwrap(); + + let mut payments = generate_random_payments(100); + + for payment in payments.clone() { + block_on(persister.add_or_update_payment_in_sql(payment)).unwrap(); + } + + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); + let limit = 4; + + let result = block_on(persister.get_payments_by_filter(None, paging, limit)).unwrap(); + + payments.sort_by(|a, b| b.last_updated.cmp(&a.last_updated)); + let expected_payments = &payments[..4].to_vec(); + let actual_payments = &result.payments; + + assert_eq!(0, result.skipped); + assert_eq!(100, result.total); + assert_eq!(expected_payments, actual_payments); + + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(2).unwrap()); + let limit = 5; + + let result = block_on(persister.get_payments_by_filter(None, paging, limit)).unwrap(); + + let expected_payments = &payments[5..10].to_vec(); + let actual_payments = &result.payments; + + assert_eq!(5, result.skipped); + assert_eq!(100, result.total); + assert_eq!(expected_payments, actual_payments); + + let from_payment_hash = payments[20].payment_hash; + let paging = PagingOptionsEnum::FromId(from_payment_hash); + let limit = 3; + + let result = block_on(persister.get_payments_by_filter(None, paging, limit)).unwrap(); + + let expected_payments = &payments[21..24].to_vec(); + let actual_payments = &result.payments; + + assert_eq!(expected_payments, actual_payments); + + let mut filter = PaymentsFilter { + payment_type: Some(PaymentType::InboundPayment), + description: None, + status: None, + from_amount_msat: None, + to_amount_msat: None, + from_fee_paid_msat: None, + to_fee_paid_msat: None, + from_timestamp: None, + to_timestamp: None, + }; + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); + let limit = 10; + + let result = block_on(persister.get_payments_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); + let expected_payments_vec: Vec = payments + .iter() + .map(|p| p.clone()) + .filter(|p| p.payment_type == PaymentType::InboundPayment) + .collect(); + let expected_payments = if expected_payments_vec.len() > 10 { + expected_payments_vec[..10].to_vec() + } else { + expected_payments_vec.clone() + }; + let actual_payments = result.payments; + + assert_eq!(expected_payments, actual_payments); + + filter.status = Some(HTLCStatus::Succeeded); + let result = block_on(persister.get_payments_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); + let expected_payments_vec: Vec = expected_payments_vec + .iter() + .map(|p| p.clone()) + .filter(|p| p.status == HTLCStatus::Succeeded) + .collect(); + let expected_payments = if expected_payments_vec.len() > 10 { + expected_payments_vec[..10].to_vec() + } else { + expected_payments_vec + }; + let actual_payments = result.payments; + + assert_eq!(expected_payments, actual_payments); + + let description = payments[42].description.as_ref().unwrap(); + let substr = &description[5..10]; + filter.payment_type = None; + filter.status = None; + filter.description = Some(substr.to_string()); + let result = block_on(persister.get_payments_by_filter(Some(filter), paging, limit)).unwrap(); + let expected_payments_vec: Vec = payments + .iter() + .map(|p| p.clone()) + .filter(|p| { + if let Some(desc) = &p.description { + return desc.contains(&substr); + } + return false; + }) + .collect(); + let expected_payments = if expected_payments_vec.len() > 10 { + expected_payments_vec[..10].to_vec() + } else { + expected_payments_vec.clone() + }; + let actual_payments = result.payments; + + assert_eq!(expected_payments, actual_payments); } } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 4289443096..504786cf24 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -1,5 +1,6 @@ use async_trait::async_trait; use bitcoin::Network; +use common::PagingOptionsEnum; use db_common::sqlite::rusqlite::types::FromSqlError; use derive_more::Display; use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; @@ -129,6 +130,25 @@ pub struct PaymentInfo { pub last_updated: u64, } +#[derive(Clone)] +pub struct PaymentsFilter { + pub payment_type: Option, + pub description: Option, + pub status: Option, + pub from_amount_msat: Option, + pub to_amount_msat: Option, + pub from_fee_paid_msat: Option, + pub to_fee_paid_msat: Option, + pub from_timestamp: Option, + pub to_timestamp: Option, +} + +pub struct GetPaymentsResult { + pub payments: Vec, + pub skipped: usize, + pub total: usize, +} + #[async_trait] pub trait SqlStorage { type Error; @@ -162,9 +182,12 @@ pub trait SqlStorage { async fn get_closed_channels(&self) -> Result, Self::Error>; - async fn get_outbound_payments(&self) -> Result, Self::Error>; - - async fn get_inbound_payments(&self) -> Result, Self::Error>; + async fn get_payments_by_filter( + &self, + filter: Option, + paging: PagingOptionsEnum, + limit: usize, + ) -> Result; async fn add_claiming_tx_to_sql( &self, diff --git a/mm2src/common/common.rs b/mm2src/common/common.rs index fd03e2182a..3976ac12c2 100644 --- a/mm2src/common/common.rs +++ b/mm2src/common/common.rs @@ -1691,6 +1691,18 @@ pub enum PagingOptionsEnum { PageNumber(NonZeroUsize), } +impl PagingOptionsEnum { + pub fn map(self, f: F) -> PagingOptionsEnum + where + F: FnOnce(Id) -> U, + { + match self { + PagingOptionsEnum::FromId(id) => PagingOptionsEnum::FromId(f(id)), + PagingOptionsEnum::PageNumber(s) => PagingOptionsEnum::PageNumber(s), + } + } +} + impl Default for PagingOptionsEnum { fn default() -> Self { PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).expect("1 > 0")) } } From dc54493255dc0193dd40d9afe44419d2c68d4852 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Thu, 31 Mar 2022 15:07:06 +0200 Subject: [PATCH 28/49] list channels by filter for open and closed --- mm2src/coins/lightning.rs | 239 +++++++++- mm2src/coins/lightning_persister/src/lib.rs | 422 ++++++++++++++++-- .../coins/lightning_persister/src/storage.rs | 47 +- mm2src/rpc/dispatcher/dispatcher.rs | 5 +- 4 files changed, 671 insertions(+), 42 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index dfefbdb1a0..c727fcf771 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -40,8 +40,8 @@ use lightning_background_processor::BackgroundProcessor; use lightning_invoice::payment; use lightning_invoice::utils::{create_invoice_from_channelmanager, DefaultRouter}; use lightning_invoice::{Invoice, InvoiceDescription}; -use lightning_persister::storage::{FileSystemStorage, HTLCStatus, NodesAddressesMapShared, PaymentInfo, PaymentType, - PaymentsFilter, Scorer, SqlChannelDetails, SqlStorage}; +use lightning_persister::storage::{ClosedChannelsFilter, FileSystemStorage, HTLCStatus, NodesAddressesMapShared, + PaymentInfo, PaymentType, PaymentsFilter, Scorer, SqlChannelDetails, SqlStorage}; use lightning_persister::LightningPersister; use ln_conf::{ChannelOptions, LightningCoinConf, LightningProtocolConf, PlatformCoinConfirmations}; use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelError, CloseChannelResult, @@ -809,11 +809,147 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes } #[derive(Deserialize)] -pub struct ListChannelsRequest { +pub struct OpenChannelsFilter { + pub channel_id: Option, + pub counterparty_node_id: Option, + pub funding_tx: Option, + pub from_funding_value_sats: Option, + pub to_funding_value_sats: Option, + pub is_outbound: Option, + pub from_balance_msat: Option, + pub to_balance_msat: Option, + pub from_outbound_capacity_msat: Option, + pub to_outbound_capacity_msat: Option, + pub from_inbound_capacity_msat: Option, + pub to_inbound_capacity_msat: Option, + pub confirmed: Option, + pub is_usable: Option, + pub is_public: Option, +} + +fn apply_open_channel_filter(channel_details: &ChannelDetailsForRPC, filter: &OpenChannelsFilter) -> bool { + let is_channel_id = if let Some(channel_id) = filter.channel_id { + channel_details.channel_id == channel_id + } else { + true + }; + + let is_counterparty_node_id = if let Some(counterparty_node_id) = &filter.counterparty_node_id { + &channel_details.counterparty_node_id == counterparty_node_id + } else { + true + }; + + let is_funding_tx = if let Some(funding_tx) = filter.funding_tx { + if let Some(channel_details_funding_tx) = channel_details.funding_tx { + channel_details_funding_tx == funding_tx + } else { + false + } + } else { + true + }; + + let is_from_funding_value_sats = if let Some(from_funding_value_sats) = filter.from_funding_value_sats { + channel_details.funding_tx_value_sats >= from_funding_value_sats + } else { + true + }; + + let is_to_funding_value_sats = if let Some(to_funding_value_sats) = filter.to_funding_value_sats { + channel_details.funding_tx_value_sats <= to_funding_value_sats + } else { + true + }; + + let is_outbound = if let Some(is_outbound) = filter.is_outbound { + channel_details.is_outbound == is_outbound + } else { + true + }; + + let is_from_balance_msat = if let Some(from_balance_msat) = filter.from_balance_msat { + channel_details.balance_msat >= from_balance_msat + } else { + true + }; + + let is_to_balance_msat = if let Some(to_balance_msat) = filter.to_balance_msat { + channel_details.balance_msat <= to_balance_msat + } else { + true + }; + + let is_from_outbound_capacity_msat = if let Some(from_outbound_capacity_msat) = filter.from_outbound_capacity_msat { + channel_details.outbound_capacity_msat >= from_outbound_capacity_msat + } else { + true + }; + + let is_to_outbound_capacity_msat = if let Some(to_outbound_capacity_msat) = filter.to_outbound_capacity_msat { + channel_details.outbound_capacity_msat <= to_outbound_capacity_msat + } else { + true + }; + + let is_from_inbound_capacity_msat = if let Some(from_inbound_capacity_msat) = filter.from_inbound_capacity_msat { + channel_details.inbound_capacity_msat >= from_inbound_capacity_msat + } else { + true + }; + + let is_to_inbound_capacity_msat = if let Some(to_inbound_capacity_msat) = filter.to_inbound_capacity_msat { + channel_details.inbound_capacity_msat <= to_inbound_capacity_msat + } else { + true + }; + + let is_confirmed = if let Some(confirmed) = filter.confirmed { + channel_details.confirmed == confirmed + } else { + true + }; + + let is_usable = if let Some(is_usable) = filter.is_usable { + channel_details.is_usable == is_usable + } else { + true + }; + + let is_public = if let Some(is_public) = filter.is_public { + channel_details.is_public == is_public + } else { + true + }; + + is_channel_id + && is_counterparty_node_id + && is_funding_tx + && is_from_funding_value_sats + && is_to_funding_value_sats + && is_outbound + && is_from_balance_msat + && is_to_balance_msat + && is_from_outbound_capacity_msat + && is_to_outbound_capacity_msat + && is_from_inbound_capacity_msat + && is_to_inbound_capacity_msat + && is_confirmed + && is_usable + && is_public +} + +#[derive(Deserialize)] +pub struct ListOpenChannelsRequest { pub coin: String, + pub filter: Option, + #[serde(default = "ten")] + limit: usize, + #[serde(default)] + paging_options: PagingOptionsEnum, } -#[derive(Serialize)] +#[derive(Clone, Serialize)] pub struct ChannelDetailsForRPC { pub rpc_channel_id: u64, pub channel_id: H256Json, @@ -859,29 +995,110 @@ impl From for ChannelDetailsForRPC { } #[derive(Serialize)] -pub struct ListChannelsResponse { +pub struct ListOpenChannelsResponse { open_channels: Vec, - closed_channels: Vec, + limit: usize, + skipped: usize, + total: usize, + total_pages: usize, + paging_options: PagingOptionsEnum, } -pub async fn list_channels(ctx: MmArc, req: ListChannelsRequest) -> ListChannelsResult { +pub async fn list_open_channels_by_filter( + ctx: MmArc, + req: ListOpenChannelsRequest, +) -> ListChannelsResult { let coin = lp_coinfind_or_err(&ctx, &req.coin).await?; let ln_coin = match coin { MmCoinEnum::LightningCoin(c) => c, _ => return MmError::err(ListChannelsError::UnsupportedCoin(coin.ticker().to_string())), }; - let open_channels = ln_coin + let mut total_open_channels: Vec = ln_coin .channel_manager .list_channels() .into_iter() .map(From::from) .collect(); - let closed_channels = ln_coin.persister.get_closed_channels().await?; + total_open_channels.sort_by(|a, b| a.rpc_channel_id.cmp(&b.rpc_channel_id)); + + let total = total_open_channels.len(); + + let offset = match req.paging_options { + PagingOptionsEnum::PageNumber(page) => (page.get() - 1) * req.limit, + PagingOptionsEnum::FromId(rpc_id) => total_open_channels + .iter() + .position(|x| x.rpc_channel_id == rpc_id) + .map(|pos| pos + 1) + .unwrap_or_default(), + }; + + let open_channels_filtered = if let Some(ref filter) = req.filter { + total_open_channels + .into_iter() + .filter(|chan| apply_open_channel_filter(chan, filter)) + .collect() + } else { + total_open_channels + }; + + let open_channels = if offset + req.limit <= open_channels_filtered.len() { + open_channels_filtered[offset..offset + req.limit].to_vec() + } else { + open_channels_filtered[offset..].to_vec() + }; - Ok(ListChannelsResponse { + Ok(ListOpenChannelsResponse { open_channels, - closed_channels, + limit: req.limit, + skipped: offset, + total, + total_pages: calc_total_pages(total, req.limit), + paging_options: req.paging_options, + }) +} + +#[derive(Deserialize)] +pub struct ListClosedChannelsRequest { + pub coin: String, + pub filter: Option, + #[serde(default = "ten")] + limit: usize, + #[serde(default)] + paging_options: PagingOptionsEnum, +} + +#[derive(Serialize)] +pub struct ListClosedChannelsResponse { + closed_channels: Vec, + limit: usize, + skipped: usize, + total: usize, + total_pages: usize, + paging_options: PagingOptionsEnum, +} + +pub async fn list_closed_channels_by_filter( + ctx: MmArc, + req: ListClosedChannelsRequest, +) -> ListChannelsResult { + let coin = lp_coinfind_or_err(&ctx, &req.coin).await?; + let ln_coin = match coin { + MmCoinEnum::LightningCoin(c) => c, + _ => return MmError::err(ListChannelsError::UnsupportedCoin(coin.ticker().to_string())), + }; + let closed_channels_res = ln_coin + .persister + .get_closed_channels_by_filter(req.filter, req.paging_options.clone(), req.limit) + .await?; + + Ok(ListClosedChannelsResponse { + closed_channels: closed_channels_res.channels, + limit: req.limit, + skipped: closed_channels_res.skipped, + total: closed_channels_res.total, + total_pages: calc_total_pages(closed_channels_res.total, req.limit), + paging_options: req.paging_options, }) } diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index b2df8f81c2..db84d1cd04 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -13,8 +13,9 @@ extern crate lightning; extern crate secp256k1; extern crate serde_json; -use crate::storage::{FileSystemStorage, GetPaymentsResult, HTLCStatus, NodesAddressesMap, NodesAddressesMapShared, - PaymentInfo, PaymentType, PaymentsFilter, Scorer, SqlChannelDetails, SqlStorage}; +use crate::storage::{ChannelType, ChannelVisibility, ClosedChannelsFilter, FileSystemStorage, GetClosedChannelsResult, + GetPaymentsResult, HTLCStatus, NodesAddressesMap, NodesAddressesMapShared, PaymentInfo, + PaymentType, PaymentsFilter, Scorer, SqlChannelDetails, SqlStorage}; use crate::util::DiskWriteable; use async_trait::async_trait; use bitcoin::blockdata::constants::genesis_block; @@ -22,7 +23,7 @@ use bitcoin::hash_types::{BlockHash, Txid}; use bitcoin::hashes::hex::{FromHex, ToHex}; use bitcoin::Network; use common::fs::check_dir_operations; -use common::{async_blocking, PagingOptionsEnum}; +use common::{async_blocking, now_ms, PagingOptionsEnum}; use db_common::sqlite::rusqlite::{Error as SqlError, Row, ToSql, NO_PARAMS}; use db_common::sqlite::sql_builder::SqlBuilder; use db_common::sqlite::{offset_by_id, query_single_row, string_from_row, validate_table_name, SqliteConnShared, @@ -111,7 +112,9 @@ fn create_channels_history_table_sql(for_coin: &str) -> Result claimed_balance REAL, is_outbound INTEGER NOT NULL, is_public INTEGER NOT NULL, - is_closed INTEGER NOT NULL + is_closed INTEGER NOT NULL, + created_at INTEGER NOT NULL, + last_updated INTEGER NOT NULL );"; Ok(sql) @@ -147,7 +150,7 @@ fn insert_channel_sql(for_coin: &str) -> Result { let sql = "INSERT INTO ".to_owned() + &table_name - + " (rpc_id, channel_id, counterparty_node_id, is_outbound, is_public, is_closed) VALUES (?1, ?2, ?3, ?4, ?5, ?6);"; + + " (rpc_id, channel_id, counterparty_node_id, is_outbound, is_public, is_closed, created_at, last_updated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8);"; Ok(sql) } @@ -167,7 +170,7 @@ fn select_channel_from_table_by_rpc_id_sql(for_coin: &str) -> Result) -> Result(13)? as u64, + last_updated: row.get::<_, u32>(14)? as u64, }; Ok(channel_details) } @@ -264,7 +269,7 @@ fn update_funding_tx_sql(for_coin: &str) -> Result { let sql = "UPDATE ".to_owned() + &table_name - + " SET funding_tx = ?2, funding_value = ?3, funding_generated_in_block = ?4 WHERE rpc_id = ?1;"; + + " SET funding_tx = ?2, funding_value = ?3, funding_generated_in_block = ?4, last_updated = ?5 WHERE rpc_id = ?1;"; Ok(sql) } @@ -273,7 +278,9 @@ fn update_channel_to_closed_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "UPDATE ".to_owned() + &table_name + " SET closure_reason = ?2, is_closed = ?3 WHERE rpc_id = ?1;"; + let sql = "UPDATE ".to_owned() + + &table_name + + " SET closure_reason = ?2, is_closed = ?3, last_updated = ?4 WHERE rpc_id = ?1;"; Ok(sql) } @@ -282,18 +289,111 @@ fn update_closing_tx_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "UPDATE ".to_owned() + &table_name + " SET closing_tx = ?2 WHERE rpc_id = ?1;"; + let sql = "UPDATE ".to_owned() + &table_name + " SET closing_tx = ?2, last_updated = ?3 WHERE rpc_id = ?1;"; Ok(sql) } -fn get_closed_channels_sql(for_coin: &str) -> Result { +fn get_channels_builder_preimage(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "SELECT rpc_id, channel_id, counterparty_node_id, funding_tx, funding_value, funding_generated_in_block, closing_tx, closure_reason, claiming_tx, claimed_balance, is_outbound, is_public, is_closed FROM ".to_owned() + &table_name + " WHERE is_closed = 1;"; + let mut sql_builder = SqlBuilder::select_from(table_name); + sql_builder.and_where("is_closed = 1"); + Ok(sql_builder) +} - Ok(sql) +fn finalize_get_channels_sql_builder(sql_builder: &mut SqlBuilder, offset: usize, limit: usize) { + sql_builder + .field("rpc_id") + .field("channel_id") + .field("counterparty_node_id") + .field("funding_tx") + .field("funding_value") + .field("funding_generated_in_block") + .field("closing_tx") + .field("closure_reason") + .field("claiming_tx") + .field("claimed_balance") + .field("is_outbound") + .field("is_public") + .field("is_closed") + .field("created_at") + .field("last_updated"); + sql_builder.offset(offset); + sql_builder.limit(limit); + sql_builder.order_desc("last_updated"); +} + +fn apply_get_channels_filter(builder: &mut SqlBuilder, params: &mut Vec<(&str, String)>, filter: ClosedChannelsFilter) { + if let Some(channel_id) = filter.channel_id { + builder.and_where("channel_id = :channel_id"); + params.push((":channel_id", channel_id)); + } + + if let Some(counterparty_node_id) = filter.counterparty_node_id { + builder.and_where("counterparty_node_id = :counterparty_node_id"); + params.push((":counterparty_node_id", counterparty_node_id)); + } + + if let Some(funding_tx) = filter.funding_tx { + builder.and_where("funding_tx = :funding_tx"); + params.push((":funding_tx", funding_tx)); + } + + if let Some(from_funding_value) = filter.from_funding_value { + builder.and_where("funding_value >= :from_funding_value"); + params.push((":from_funding_value", from_funding_value.to_string())); + } + + if let Some(to_funding_value) = filter.to_funding_value { + builder.and_where("funding_value <= :to_funding_value"); + params.push((":to_funding_value", to_funding_value.to_string())); + } + + if let Some(closing_tx) = filter.closing_tx { + builder.and_where("closing_tx = :closing_tx"); + params.push((":closing_tx", closing_tx)); + } + + if let Some(closure_reason) = filter.closure_reason { + builder.and_where(format!("closure_reason LIKE '%{}%'", closure_reason)); + } + + if let Some(claiming_tx) = filter.claiming_tx { + builder.and_where("claiming_tx = :claiming_tx"); + params.push((":claiming_tx", claiming_tx)); + } + + if let Some(from_claimed_balance) = filter.from_claimed_balance { + builder.and_where("claimed_balance >= :from_claimed_balance"); + params.push((":from_claimed_balance", from_claimed_balance.to_string())); + } + + if let Some(to_claimed_balance) = filter.to_claimed_balance { + builder.and_where("claimed_balance <= :to_claimed_balance"); + params.push((":to_claimed_balance", to_claimed_balance.to_string())); + } + + if let Some(channel_type) = filter.channel_type { + let is_outbound = match channel_type { + ChannelType::Outbound => true as i32, + ChannelType::Inbound => false as i32, + }; + + builder.and_where("is_outbound = :is_outbound"); + params.push((":is_outbound", is_outbound.to_string())); + } + + if let Some(channel_visibility) = filter.channel_visibility { + let is_public = match channel_visibility { + ChannelVisibility::Public => true as i32, + ChannelVisibility::Private => false as i32, + }; + + builder.and_where("is_public = :is_public"); + params.push((":is_public", is_public.to_string())); + } } fn get_payments_builder_preimage(for_coin: &str) -> Result { @@ -328,7 +428,8 @@ fn apply_get_payments_filter(builder: &mut SqlBuilder, params: &mut Vec<(&str, S PaymentType::InboundPayment => (false as i32, None), }; if let Some(dest) = destination { - builder.and_where(format!("destination LIKE '%{}%'", dest)); + builder.and_where("destination = :dest"); + params.push((":dest", dest)); } builder.and_where("is_outbound = :is_outbound"); @@ -379,7 +480,9 @@ fn update_claiming_tx_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "UPDATE ".to_owned() + &table_name + " SET claiming_tx = ?2, claimed_balance = ?3 WHERE closing_tx = ?1;"; + let sql = "UPDATE ".to_owned() + + &table_name + + " SET claiming_tx = ?2, claimed_balance = ?3, last_updated = ?4 WHERE closing_tx = ?1;"; Ok(sql) } @@ -788,6 +891,8 @@ impl SqlStorage for LightningPersister { let is_outbound = (details.is_outbound as i32).to_string(); let is_public = (details.is_public as i32).to_string(); let is_closed = (details.is_closed as i32).to_string(); + let created_at = (details.created_at as u32).to_string(); + let last_updated = (details.last_updated as u32).to_string(); let params = [ rpc_id, @@ -796,6 +901,8 @@ impl SqlStorage for LightningPersister { is_outbound, is_public, is_closed, + created_at, + last_updated, ]; let sqlite_connection = self.sqlite_connection.clone(); @@ -896,8 +1003,15 @@ impl SqlStorage for LightningPersister { let rpc_id = rpc_id.to_string(); let funding_value = funding_value.to_string(); let funding_generated_in_block = funding_generated_in_block.to_string(); + let last_updated = (now_ms() / 1000).to_string(); - let params = [rpc_id, funding_tx, funding_value, funding_generated_in_block]; + let params = [ + rpc_id, + funding_tx, + funding_value, + funding_generated_in_block, + last_updated, + ]; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { @@ -914,8 +1028,9 @@ impl SqlStorage for LightningPersister { let for_coin = self.storage_ticker.clone(); let rpc_id = rpc_id.to_string(); let is_closed = "1".to_string(); + let last_updated = (now_ms() / 1000).to_string(); - let params = [rpc_id, closure_reason, is_closed]; + let params = [rpc_id, closure_reason, is_closed, last_updated]; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { @@ -931,8 +1046,9 @@ impl SqlStorage for LightningPersister { async fn add_closing_tx_to_sql(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); let rpc_id = rpc_id.to_string(); + let last_updated = (now_ms() / 1000).to_string(); - let params = [rpc_id, closing_tx]; + let params = [rpc_id, closing_tx, last_updated]; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { @@ -945,15 +1061,66 @@ impl SqlStorage for LightningPersister { .await } - async fn get_closed_channels(&self) -> Result, Self::Error> { - let sql = get_closed_channels_sql(self.storage_ticker.as_str())?; + async fn get_closed_channels_by_filter( + &self, + filter: Option, + paging: PagingOptionsEnum, + limit: usize, + ) -> Result { + let mut sql_builder = get_channels_builder_preimage(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { let conn = sqlite_connection.lock().unwrap(); + + let mut total_builder = sql_builder.clone(); + total_builder.count("id"); + let total_sql = total_builder.sql().expect("valid sql"); + let total: isize = conn.query_row(&total_sql, NO_PARAMS, |row| row.get(0))?; + let total = total.try_into().expect("count should be always above zero"); + + let offset = match paging { + PagingOptionsEnum::PageNumber(page) => (page.get() - 1) * limit, + PagingOptionsEnum::FromId(rpc_id) => { + let params = [rpc_id as u32]; + let maybe_offset = offset_by_id( + &conn, + &sql_builder, + params, + "rpc_id", + "last_updated DESC", + "rpc_id = ?1", + )?; + match maybe_offset { + Some(offset) => offset, + None => { + return Ok(GetClosedChannelsResult { + channels: vec![], + skipped: 0, + total, + }) + }, + } + }, + }; + + let mut params = vec![]; + if let Some(f) = filter { + apply_get_channels_filter(&mut sql_builder, &mut params, f); + } + let params_as_trait: Vec<_> = params.iter().map(|(key, value)| (*key, value as &dyn ToSql)).collect(); + finalize_get_channels_sql_builder(&mut sql_builder, offset, limit); + + let sql = sql_builder.sql().expect("valid sql"); let mut stmt = conn.prepare(&sql)?; - let rows = stmt.query(NO_PARAMS)?; - let result = rows.mapped(channel_details_from_row).collect::>()?; + let channels = stmt + .query_map_named(params_as_trait.as_slice(), channel_details_from_row)? + .collect::>()?; + let result = GetClosedChannelsResult { + channels, + skipped: offset, + total, + }; Ok(result) }) .await @@ -1033,8 +1200,9 @@ impl SqlStorage for LightningPersister { ) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); let claimed_balance = claimed_balance.to_string(); + let last_updated = (now_ms() / 1000).to_string(); - let params = [closing_tx, claiming_tx, claimed_balance]; + let params = [closing_tx, claiming_tx, claimed_balance, last_updated]; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { @@ -1085,6 +1253,58 @@ mod tests { } } + fn generate_random_channels(num: u64) -> Vec { + let mut rng = rand::thread_rng(); + let mut channels = vec![]; + let s = Secp256k1::new(); + let mut bytes = [0; 32]; + for i in 0..num { + let details = SqlChannelDetails { + rpc_id: i + 1, + channel_id: { + rng.fill_bytes(&mut bytes); + hex::encode(bytes) + }, + counterparty_node_id: { + rng.fill_bytes(&mut bytes); + let secret = SecretKey::from_slice(&bytes).unwrap(); + let pubkey = PublicKey::from_secret_key(&s, &secret); + pubkey.to_string() + }, + funding_tx: { + rng.fill_bytes(&mut bytes); + Some(hex::encode(bytes)) + }, + funding_value: Some(rng.gen::() as u64), + closing_tx: { + rng.fill_bytes(&mut bytes); + Some(hex::encode(bytes)) + }, + closure_reason: { + Some( + rng.sample_iter(&Alphanumeric) + .take(30) + .map(char::from) + .collect::(), + ) + }, + claiming_tx: { + rng.fill_bytes(&mut bytes); + Some(hex::encode(bytes)) + }, + claimed_balance: Some(rng.gen::()), + funding_generated_in_block: Some(rng.gen::() as u64), + is_outbound: rand::random(), + is_public: rand::random(), + is_closed: rand::random(), + created_at: rng.gen::() as u64, + last_updated: rng.gen::() as u64, + }; + channels.push(details); + } + channels + } + fn generate_random_payments(num: u64) -> Vec { let mut rng = rand::thread_rng(); let mut payments = vec![]; @@ -1414,13 +1634,15 @@ mod tests { let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); - let closed_channels = block_on(persister.get_closed_channels()).unwrap(); - assert_eq!(closed_channels.len(), 1); - assert_eq!(expected_channel_details, closed_channels[0]); + let closed_channels = + block_on(persister.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 10)).unwrap(); + assert_eq!(closed_channels.channels.len(), 1); + assert_eq!(expected_channel_details, closed_channels.channels[0]); block_on(persister.update_channel_to_closed(1, "the channel was cooperatively closed".into())).unwrap(); - let closed_channels = block_on(persister.get_closed_channels()).unwrap(); - assert_eq!(closed_channels.len(), 2); + let closed_channels = + block_on(persister.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 10)).unwrap(); + assert_eq!(closed_channels.channels.len(), 2); block_on(persister.add_closing_tx_to_sql( 2, @@ -1621,4 +1843,150 @@ mod tests { assert_eq!(expected_payments, actual_payments); } + + #[test] + fn test_get_channels_by_filter() { + let persister = LightningPersister::new( + "test_get_channels_by_filter".into(), + PathBuf::from("test_filesystem_persister"), + None, + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + + block_on(persister.init_sql()).unwrap(); + + let mut channels = generate_random_channels(100); + + for channel in channels.clone() { + block_on(persister.add_channel_to_sql(channel.clone())).unwrap(); + block_on(persister.add_funding_tx_to_sql( + channel.rpc_id, + channel.funding_tx.unwrap(), + channel.funding_value.unwrap(), + channel.funding_generated_in_block.unwrap(), + )) + .unwrap(); + block_on(persister.update_channel_to_closed(channel.rpc_id, channel.closure_reason.unwrap())).unwrap(); + block_on(persister.add_closing_tx_to_sql(channel.rpc_id, channel.closing_tx.clone().unwrap())).unwrap(); + block_on(persister.add_claiming_tx_to_sql( + channel.closing_tx.unwrap(), + channel.claiming_tx.unwrap(), + channel.claimed_balance.unwrap(), + )) + .unwrap(); + } + + // get all channels from SQL since updated_at changed from channels generated by generate_random_channels + channels = block_on(persister.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 100)) + .unwrap() + .channels; + assert_eq!(100, channels.len()); + + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); + let limit = 4; + + let result = block_on(persister.get_closed_channels_by_filter(None, paging, limit)).unwrap(); + + let expected_channels = &channels[..4].to_vec(); + let actual_channels = &result.channels; + + assert_eq!(0, result.skipped); + assert_eq!(100, result.total); + assert_eq!(expected_channels, actual_channels); + + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(2).unwrap()); + let limit = 5; + + let result = block_on(persister.get_closed_channels_by_filter(None, paging, limit)).unwrap(); + + let expected_channels = &channels[5..10].to_vec(); + let actual_channels = &result.channels; + + assert_eq!(5, result.skipped); + assert_eq!(100, result.total); + assert_eq!(expected_channels, actual_channels); + + let from_rpc_id = 20; + let paging = PagingOptionsEnum::FromId(from_rpc_id); + let limit = 3; + + let result = block_on(persister.get_closed_channels_by_filter(None, paging, limit)).unwrap(); + + channels.sort_by(|a, b| a.rpc_id.cmp(&b.rpc_id)); + + let expected_channels = &channels[20..23].to_vec(); + let actual_channels = &result.channels; + + assert_eq!(expected_channels, actual_channels); + + channels.sort_by(|a, b| b.last_updated.cmp(&a.last_updated)); + let mut filter = ClosedChannelsFilter { + channel_id: None, + counterparty_node_id: None, + funding_tx: None, + from_funding_value: None, + to_funding_value: None, + closing_tx: None, + closure_reason: None, + claiming_tx: None, + from_claimed_balance: None, + to_claimed_balance: None, + channel_type: Some(ChannelType::Outbound), + channel_visibility: None, + }; + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); + let limit = 10; + + let result = + block_on(persister.get_closed_channels_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); + let expected_channels_vec: Vec = channels + .iter() + .map(|chan| chan.clone()) + .filter(|chan| chan.is_outbound) + .collect(); + let expected_channels = if expected_channels_vec.len() > 10 { + expected_channels_vec[..10].to_vec() + } else { + expected_channels_vec.clone() + }; + let actual_channels = result.channels; + + assert_eq!(expected_channels, actual_channels); + + filter.channel_visibility = Some(ChannelVisibility::Public); + let result = + block_on(persister.get_closed_channels_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); + let expected_channels_vec: Vec = expected_channels_vec + .iter() + .map(|chan| chan.clone()) + .filter(|chan| chan.is_public) + .collect(); + let expected_channels = if expected_channels_vec.len() > 10 { + expected_channels_vec[..10].to_vec() + } else { + expected_channels_vec + }; + let actual_channels = result.channels; + + assert_eq!(expected_channels, actual_channels); + + let channel_id = channels[42].channel_id.clone(); + filter.channel_type = None; + filter.channel_visibility = None; + filter.channel_id = Some(channel_id.clone()); + let result = block_on(persister.get_closed_channels_by_filter(Some(filter), paging, limit)).unwrap(); + let expected_channels_vec: Vec = channels + .iter() + .map(|chan| chan.clone()) + .filter(|chan| chan.channel_id == channel_id) + .collect(); + let expected_channels = if expected_channels_vec.len() > 10 { + expected_channels_vec[..10].to_vec() + } else { + expected_channels_vec.clone() + }; + let actual_channels = result.channels; + + assert_eq!(expected_channels, actual_channels); + } } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 504786cf24..2349b10a1a 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -1,6 +1,6 @@ use async_trait::async_trait; use bitcoin::Network; -use common::PagingOptionsEnum; +use common::{now_ms, PagingOptionsEnum}; use db_common::sqlite::rusqlite::types::FromSqlError; use derive_more::Display; use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; @@ -61,6 +61,8 @@ pub struct SqlChannelDetails { pub is_outbound: bool, pub is_public: bool, pub is_closed: bool, + pub created_at: u64, + pub last_updated: u64, } impl SqlChannelDetails { @@ -85,10 +87,46 @@ impl SqlChannelDetails { is_outbound, is_public, is_closed: false, + created_at: now_ms() / 1000, + last_updated: now_ms() / 1000, } } } +#[derive(Clone, Deserialize)] +pub enum ChannelType { + Outbound, + Inbound, +} + +#[derive(Clone, Deserialize)] +pub enum ChannelVisibility { + Public, + Private, +} + +#[derive(Clone, Deserialize)] +pub struct ClosedChannelsFilter { + pub channel_id: Option, + pub counterparty_node_id: Option, + pub funding_tx: Option, + pub from_funding_value: Option, + pub to_funding_value: Option, + pub closing_tx: Option, + pub closure_reason: Option, + pub claiming_tx: Option, + pub from_claimed_balance: Option, + pub to_claimed_balance: Option, + pub channel_type: Option, + pub channel_visibility: Option, +} + +pub struct GetClosedChannelsResult { + pub channels: Vec, + pub skipped: usize, + pub total: usize, +} + #[derive(Clone, Debug, Deserialize, Display, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] pub enum HTLCStatus { @@ -180,7 +218,12 @@ pub trait SqlStorage { async fn add_closing_tx_to_sql(&self, rpc_id: u64, closing_tx_: String) -> Result<(), Self::Error>; - async fn get_closed_channels(&self) -> Result, Self::Error>; + async fn get_closed_channels_by_filter( + &self, + filter: Option, + paging: PagingOptionsEnum, + limit: usize, + ) -> Result; async fn get_payments_by_filter( &self, diff --git a/mm2src/rpc/dispatcher/dispatcher.rs b/mm2src/rpc/dispatcher/dispatcher.rs index cc675be89c..36f936836a 100644 --- a/mm2src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/rpc/dispatcher/dispatcher.rs @@ -32,7 +32,7 @@ use std::net::SocketAddr; cfg_native! { use coins::lightning::{close_channel, connect_to_lightning_node, generate_invoice, get_channel_details, - get_claimable_balances, get_payment_details, list_channels, list_payments_by_filter, open_channel, + get_claimable_balances, get_payment_details, list_closed_channels_by_filter, list_open_channels_by_filter, list_payments_by_filter, open_channel, send_payment, LightningCoin}; } @@ -160,7 +160,8 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, get_channel_details).await, "get_claimable_balances" => handle_mmrpc(ctx, request, get_claimable_balances).await, "get_payment_details" => handle_mmrpc(ctx, request, get_payment_details).await, - "list_channels" => handle_mmrpc(ctx, request, list_channels).await, + "list_closed_channels_by_filter" => handle_mmrpc(ctx, request, list_closed_channels_by_filter).await, + "list_open_channels_by_filter" => handle_mmrpc(ctx, request, list_open_channels_by_filter).await, "list_payments_by_filter" => handle_mmrpc(ctx, request, list_payments_by_filter).await, "open_channel" => handle_mmrpc(ctx, request, open_channel).await, "send_payment" => handle_mmrpc(ctx, request, send_payment).await, From 6b2e98fda5775cd857e69a2438f80d6bc0a552f7 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Thu, 31 Mar 2022 15:45:01 +0200 Subject: [PATCH 29/49] fix wasm build --- mm2src/coins/lp_coins.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index f7a473d5a8..0d05a4045d 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -1454,6 +1454,7 @@ pub enum MmCoinEnum { ZCoin(ZCoin), Bch(BchCoin), SlpToken(SlpToken), + #[cfg(not(target_arch = "wasm32"))] SolanaCoin(SolanaCoin), #[cfg(not(target_arch = "wasm32"))] SplToken(SplToken), From 27578e2f058f911b8cdb6093eb027563dd61fad5 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Sat, 2 Apr 2022 00:25:57 +0200 Subject: [PATCH 30/49] part of review fixes --- mm2src/coins/lightning/ln_errors.rs | 64 ++++++++++++++--------------- mm2src/coins/lightning/ln_events.rs | 16 +++----- mm2src/coins/lightning/ln_utils.rs | 2 +- 3 files changed, 39 insertions(+), 43 deletions(-) diff --git a/mm2src/coins/lightning/ln_errors.rs b/mm2src/coins/lightning/ln_errors.rs index a43da0508a..be92862b87 100644 --- a/mm2src/coins/lightning/ln_errors.rs +++ b/mm2src/coins/lightning/ln_errors.rs @@ -43,8 +43,8 @@ pub enum EnableLightningError { HashError(String), #[display(fmt = "RPC error {}", _0)] RpcError(String), - #[display(fmt = "SQL error {}", _0)] - SqlError(String), + #[display(fmt = "DB error {}", _0)] + DbError(String), ConnectToNodeError(String), } @@ -60,7 +60,7 @@ impl HttpStatusCode for EnableLightningError { | EnableLightningError::HashError(_) | EnableLightningError::ConnectToNodeError(_) | EnableLightningError::InvalidConfiguration(_) - | EnableLightningError::SqlError(_) => StatusCode::INTERNAL_SERVER_ERROR, + | EnableLightningError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -70,7 +70,7 @@ impl From for EnableLightningError { } impl From for EnableLightningError { - fn from(err: SqlError) -> EnableLightningError { EnableLightningError::SqlError(err.to_string()) } + fn from(err: SqlError) -> EnableLightningError { EnableLightningError::DbError(err.to_string()) } } #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] @@ -135,8 +135,8 @@ pub enum OpenChannelError { InternalError(String), #[display(fmt = "I/O error {}", _0)] IOError(String), - #[display(fmt = "SQL error {}", _0)] - SqlError(String), + #[display(fmt = "DB error {}", _0)] + DbError(String), ConnectToNodeError(String), #[display(fmt = "No such coin {}", _0)] NoSuchCoin(String), @@ -158,7 +158,7 @@ impl HttpStatusCode for OpenChannelError { | OpenChannelError::InternalError(_) | OpenChannelError::GenerateTxErr(_) | OpenChannelError::IOError(_) - | OpenChannelError::SqlError(_) + | OpenChannelError::DbError(_) | OpenChannelError::InvalidPath(_) | OpenChannelError::ConvertTxErr(_) => StatusCode::INTERNAL_SERVER_ERROR, OpenChannelError::NoSuchCoin(_) | OpenChannelError::BalanceError(_) => StatusCode::PRECONDITION_REQUIRED, @@ -211,7 +211,7 @@ impl From for OpenChannelError { } impl From for OpenChannelError { - fn from(err: SqlError) -> OpenChannelError { OpenChannelError::SqlError(err.to_string()) } + fn from(err: SqlError) -> OpenChannelError { OpenChannelError::DbError(err.to_string()) } } #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] @@ -221,8 +221,8 @@ pub enum ListChannelsError { UnsupportedCoin(String), #[display(fmt = "No such coin {}", _0)] NoSuchCoin(String), - #[display(fmt = "SQL error {}", _0)] - SqlError(String), + #[display(fmt = "DB error {}", _0)] + DbError(String), } impl HttpStatusCode for ListChannelsError { @@ -230,7 +230,7 @@ impl HttpStatusCode for ListChannelsError { match self { ListChannelsError::UnsupportedCoin(_) => StatusCode::BAD_REQUEST, ListChannelsError::NoSuchCoin(_) => StatusCode::PRECONDITION_REQUIRED, - ListChannelsError::SqlError(_) => StatusCode::INTERNAL_SERVER_ERROR, + ListChannelsError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -244,7 +244,7 @@ impl From for ListChannelsError { } impl From for ListChannelsError { - fn from(err: SqlError) -> ListChannelsError { ListChannelsError::SqlError(err.to_string()) } + fn from(err: SqlError) -> ListChannelsError { ListChannelsError::DbError(err.to_string()) } } #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] @@ -256,8 +256,8 @@ pub enum GetChannelDetailsError { NoSuchCoin(String), #[display(fmt = "Channel with rpc id: {} is not found", _0)] NoSuchChannel(u64), - #[display(fmt = "SQL error {}", _0)] - SqlError(String), + #[display(fmt = "DB error {}", _0)] + DbError(String), } impl HttpStatusCode for GetChannelDetailsError { @@ -266,7 +266,7 @@ impl HttpStatusCode for GetChannelDetailsError { GetChannelDetailsError::UnsupportedCoin(_) => StatusCode::BAD_REQUEST, GetChannelDetailsError::NoSuchCoin(_) => StatusCode::PRECONDITION_REQUIRED, GetChannelDetailsError::NoSuchChannel(_) => StatusCode::NOT_FOUND, - GetChannelDetailsError::SqlError(_) => StatusCode::INTERNAL_SERVER_ERROR, + GetChannelDetailsError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -280,7 +280,7 @@ impl From for GetChannelDetailsError { } impl From for GetChannelDetailsError { - fn from(err: SqlError) -> GetChannelDetailsError { GetChannelDetailsError::SqlError(err.to_string()) } + fn from(err: SqlError) -> GetChannelDetailsError { GetChannelDetailsError::DbError(err.to_string()) } } #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] @@ -292,8 +292,8 @@ pub enum GenerateInvoiceError { NoSuchCoin(String), #[display(fmt = "Invoice signing or creation error: {}", _0)] SignOrCreationError(String), - #[display(fmt = "SQL error {}", _0)] - SqlError(String), + #[display(fmt = "DB error {}", _0)] + DbError(String), } impl HttpStatusCode for GenerateInvoiceError { @@ -301,7 +301,7 @@ impl HttpStatusCode for GenerateInvoiceError { match self { GenerateInvoiceError::UnsupportedCoin(_) => StatusCode::BAD_REQUEST, GenerateInvoiceError::NoSuchCoin(_) => StatusCode::PRECONDITION_REQUIRED, - GenerateInvoiceError::SignOrCreationError(_) | GenerateInvoiceError::SqlError(_) => { + GenerateInvoiceError::SignOrCreationError(_) | GenerateInvoiceError::DbError(_) => { StatusCode::INTERNAL_SERVER_ERROR }, } @@ -321,7 +321,7 @@ impl From for GenerateInvoiceError { } impl From for GenerateInvoiceError { - fn from(err: SqlError) -> GenerateInvoiceError { GenerateInvoiceError::SqlError(err.to_string()) } + fn from(err: SqlError) -> GenerateInvoiceError { GenerateInvoiceError::DbError(err.to_string()) } } #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] @@ -337,8 +337,8 @@ pub enum SendPaymentError { PaymentError(String), #[display(fmt = "Final cltv expiry delta {} is below the required minimum of {}", _0, _1)] CLTVExpiryError(u32, u32), - #[display(fmt = "SQL error {}", _0)] - SqlError(String), + #[display(fmt = "DB error {}", _0)] + DbError(String), } impl HttpStatusCode for SendPaymentError { @@ -349,7 +349,7 @@ impl HttpStatusCode for SendPaymentError { SendPaymentError::PaymentError(_) | SendPaymentError::NoRouteFound(_) | SendPaymentError::CLTVExpiryError(_, _) - | SendPaymentError::SqlError(_) => StatusCode::INTERNAL_SERVER_ERROR, + | SendPaymentError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -363,7 +363,7 @@ impl From for SendPaymentError { } impl From for SendPaymentError { - fn from(err: SqlError) -> SendPaymentError { SendPaymentError::SqlError(err.to_string()) } + fn from(err: SqlError) -> SendPaymentError { SendPaymentError::DbError(err.to_string()) } } #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] @@ -373,8 +373,8 @@ pub enum ListPaymentsError { UnsupportedCoin(String), #[display(fmt = "No such coin {}", _0)] NoSuchCoin(String), - #[display(fmt = "SQL error {}", _0)] - SqlError(String), + #[display(fmt = "DB error {}", _0)] + DbError(String), } impl HttpStatusCode for ListPaymentsError { @@ -382,7 +382,7 @@ impl HttpStatusCode for ListPaymentsError { match self { ListPaymentsError::UnsupportedCoin(_) => StatusCode::BAD_REQUEST, ListPaymentsError::NoSuchCoin(_) => StatusCode::PRECONDITION_REQUIRED, - ListPaymentsError::SqlError(_) => StatusCode::INTERNAL_SERVER_ERROR, + ListPaymentsError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -396,7 +396,7 @@ impl From for ListPaymentsError { } impl From for ListPaymentsError { - fn from(err: SqlError) -> ListPaymentsError { ListPaymentsError::SqlError(err.to_string()) } + fn from(err: SqlError) -> ListPaymentsError { ListPaymentsError::DbError(err.to_string()) } } #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] @@ -408,8 +408,8 @@ pub enum GetPaymentDetailsError { NoSuchCoin(String), #[display(fmt = "Payment with hash: {:?} is not found", _0)] NoSuchPayment(H256Json), - #[display(fmt = "SQL error {}", _0)] - SqlError(String), + #[display(fmt = "DB error {}", _0)] + DbError(String), } impl HttpStatusCode for GetPaymentDetailsError { @@ -418,7 +418,7 @@ impl HttpStatusCode for GetPaymentDetailsError { GetPaymentDetailsError::UnsupportedCoin(_) => StatusCode::BAD_REQUEST, GetPaymentDetailsError::NoSuchCoin(_) => StatusCode::PRECONDITION_REQUIRED, GetPaymentDetailsError::NoSuchPayment(_) => StatusCode::NOT_FOUND, - GetPaymentDetailsError::SqlError(_) => StatusCode::INTERNAL_SERVER_ERROR, + GetPaymentDetailsError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -432,7 +432,7 @@ impl From for GetPaymentDetailsError { } impl From for GetPaymentDetailsError { - fn from(err: SqlError) -> GetPaymentDetailsError { GetPaymentDetailsError::SqlError(err.to_string()) } + fn from(err: SqlError) -> GetPaymentDetailsError { GetPaymentDetailsError::DbError(err.to_string()) } } #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 64d3ab09d6..d390d9192d 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -58,7 +58,7 @@ impl EventHandler for LightningEventHandler { // Todo: an RPC for total amount earned Event::PaymentForwarded { fee_earned_msat, claim_from_onchain_tx } => log::info!( - "Recieved a fee of {} milli-satoshis for a successfully forwarded payment through our {} lightning node. Was the forwarded HTLC claimed by our counterparty via an on-chain transaction?: {}", + "Received a fee of {} milli-satoshis for a successfully forwarded payment through our {} lightning node. Was the forwarded HTLC claimed by our counterparty via an on-chain transaction?: {}", fee_earned_msat.unwrap_or_default(), self.platform.coin.ticker(), claim_from_onchain_tx, @@ -263,7 +263,7 @@ impl LightningEventHandler { payment_info.amt_msat = Some(amt); payment_info.last_updated = now_ms() / 1000; if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { - log::error!("{}", e); + log::error!("Unable to update payment information in DB: {}", e); } } }), @@ -282,7 +282,7 @@ impl LightningEventHandler { }; spawn(async move { if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { - log::error!("{}", e); + log::error!("Unable to update payment information in DB: {}", e); } }); }, @@ -311,12 +311,8 @@ impl LightningEventHandler { payment_info.fee_paid_msat = fee_paid_msat; payment_info.last_updated = now_ms() / 1000; let amt_msat = payment_info.amt_msat; - if let Err(e) = persister - .add_or_update_payment_in_sql(payment_info) - .await - .error_log_passthrough() - { - log::error!("{}", e); + if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { + log::error!("Unable to update payment information in DB: {}", e); } log::info!( "Successfully sent payment of {} millisatoshis with payment hash {}", @@ -403,7 +399,7 @@ impl LightningEventHandler { payment_info.status = HTLCStatus::Failed; payment_info.last_updated = now_ms() / 1000; if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { - log::error!("{}", e); + log::error!("Unable to update payment information in DB: {}", e); } } }); diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 2a5229aad8..01f98b5242 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -58,7 +58,7 @@ pub async fn init_persister( ln_data_dir, ln_data_backup_dir, ctx.sqlite_connection - .ok_or(MmError::new(EnableLightningError::SqlError( + .ok_or(MmError::new(EnableLightningError::DbError( "sqlite_connection is not initialized".into(), )))? .clone(), From e680cd3931ad2369ee61956d48e403a02ee3418c Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Mon, 4 Apr 2022 12:35:33 +0200 Subject: [PATCH 31/49] part of review fixes: save_channel_closing_details function --- mm2src/coins/lightning/ln_errors.rs | 27 ++++++++ mm2src/coins/lightning/ln_events.rs | 103 ++++++++++++++++------------ 2 files changed, 87 insertions(+), 43 deletions(-) diff --git a/mm2src/coins/lightning/ln_errors.rs b/mm2src/coins/lightning/ln_errors.rs index be92862b87..60b7f67279 100644 --- a/mm2src/coins/lightning/ln_errors.rs +++ b/mm2src/coins/lightning/ln_errors.rs @@ -21,6 +21,7 @@ pub type ListPaymentsResult = Result>; pub type GetPaymentDetailsResult = Result>; pub type CloseChannelResult = Result>; pub type ClaimableBalancesResult = Result>; +pub type SaveChannelClosingResult = Result>; #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] @@ -489,3 +490,29 @@ impl From for ClaimableBalancesError { } } } + +#[derive(Display)] +pub enum SaveChannelClosingError { + #[display(fmt = "DB error: {}", _0)] + DbError(String), + #[display(fmt = "Channel with rpc id {} not found in DB", _0)] + ChannelNotFound(u64), + #[display(fmt = "Funding transaction hash is Null in DB")] + FundingTxNull, + #[display(fmt = "Error parsing funding transaction hash: {}", _0)] + FundingTxParseError(String), + #[display(fmt = "Error getting funding transaction bytes through RPC: {}", _0)] + RpcError(String), + #[display(fmt = "Error while waiting for the funding transaction to be spent: {}", _0)] + WaitForFundingTxSpendError(String), + #[display(fmt = "Wrong closing transaction type: {}", _0)] + WrongClosingTxType(String), +} + +impl From for SaveChannelClosingError { + fn from(err: SqlError) -> SaveChannelClosingError { SaveChannelClosingError::DbError(err.to_string()) } +} + +impl From for SaveChannelClosingError { + fn from(e: UtxoRpcError) -> Self { SaveChannelClosingError::RpcError(e.to_string()) } +} diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index d390d9192d..34990e51dc 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -1,4 +1,5 @@ use super::*; +use crate::lightning::ln_errors::{SaveChannelClosingError, SaveChannelClosingResult}; use bitcoin::blockdata::script::Script; use bitcoin::blockdata::transaction::Transaction; use common::executor::{spawn, Timer}; @@ -162,6 +163,63 @@ fn sign_funding_transaction( Transaction::try_from(signed).map_to_mm(|e| OpenChannelError::ConvertTxErr(e.to_string())) } +async fn save_channel_closing_details( + persister: Arc, + platform: Arc, + user_channel_id: u64, + reason: String, +) -> SaveChannelClosingResult<()> { + persister.update_channel_to_closed(user_channel_id, reason).await?; + + let channel_details = persister + .get_channel_from_sql(user_channel_id) + .await? + .ok_or_else(|| MmError::new(SaveChannelClosingError::ChannelNotFound(user_channel_id)))?; + + let tx_id = channel_details + .funding_tx + .ok_or_else(|| MmError::new(SaveChannelClosingError::FundingTxNull))?; + + let tx_hash = + H256Json::from_str(&tx_id).map_to_mm(|e| SaveChannelClosingError::FundingTxParseError(e.to_string()))?; + + let funding_tx_bytes = platform + .coin + .as_ref() + .rpc_client + .get_transaction_bytes(&tx_hash) + .compat() + .await?; + + let closing_tx_enum = platform + .coin + .wait_for_tx_spend( + &funding_tx_bytes.into_vec(), + (now_ms() / 1000) + 3600, + channel_details.funding_generated_in_block.unwrap_or_default(), + &None, + ) + .compat() + .await + .map_to_mm(SaveChannelClosingError::WaitForFundingTxSpendError)?; + + let closing_tx = match closing_tx_enum { + TransactionEnum::UtxoTx(tx) => tx, + e => { + return Err(MmError::new(SaveChannelClosingError::WrongClosingTxType(format!( + "{:?}", + e + )))) + }, + }; + + persister + .add_closing_tx_to_sql(user_channel_id, closing_tx.hash().reversed().to_string()) + .await?; + + Ok(()) +} + impl LightningEventHandler { pub fn new( platform: Arc, @@ -336,49 +394,8 @@ impl LightningEventHandler { // other than 0 in sql if user_channel_id != 0 { spawn(async move { - persister - .update_channel_to_closed(user_channel_id, reason) - .await - .error_log(); - if let Ok(Some(channel_details)) = persister - .get_channel_from_sql(user_channel_id) - .await - .error_log_passthrough() - { - if let Some(tx_id) = channel_details.funding_tx { - if let Ok(tx_hash) = H256Json::from_str(&tx_id).error_log_passthrough() { - if let Ok(funding_tx_bytes) = platform - .coin - .as_ref() - .rpc_client - .get_transaction_bytes(&tx_hash) - .compat() - .await - .error_log_passthrough() - { - if let Ok(TransactionEnum::UtxoTx(closing_tx)) = platform - .coin - .wait_for_tx_spend( - &funding_tx_bytes.into_vec(), - (now_ms() / 1000) + 3600, - channel_details.funding_generated_in_block.unwrap_or_default(), - &None, - ) - .compat() - .await - .error_log_passthrough() - { - persister - .add_closing_tx_to_sql( - user_channel_id, - closing_tx.hash().reversed().to_string(), - ) - .await - .error_log(); - } - } - } - } + if let Err(e) = save_channel_closing_details(persister, platform, user_channel_id, reason).await { + log::error!("Unable to update channel closing details in DB: {}", e); } }); } From 395af28f7f6e8c7d6c26f3d1a02b27d9404d5aa8 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Mon, 4 Apr 2022 15:54:39 +0200 Subject: [PATCH 32/49] review fixes wip: simplify find_watched_output_spend_with_header --- mm2src/coins/lightning/ln_platform.rs | 131 +++++++++++++++----------- 1 file changed, 76 insertions(+), 55 deletions(-) diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index 3680f0efce..9aa503d828 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -7,11 +7,11 @@ use crate::{MarketCoinOps, MmCoin}; use bitcoin::blockdata::block::BlockHeader; use bitcoin::blockdata::script::Script; use bitcoin::blockdata::transaction::Transaction; -use bitcoin::consensus::encode::{deserialize, serialize_hex}; +use bitcoin::consensus::encode::{deserialize, serialize_hex, Error as EncodeError}; use bitcoin::hash_types::{BlockHash, TxMerkleNode, Txid}; use bitcoin_hashes::{sha256d, Hash}; use common::executor::{spawn, Timer}; -use common::jsonrpc_client::JsonRpcErrorType; +use common::jsonrpc_client::{JsonRpcError, JsonRpcErrorType}; use common::log; use derive_more::Display; use futures::compat::Future01CompatExt; @@ -25,49 +25,65 @@ use std::convert::{TryFrom, TryInto}; const CHECK_FOR_NEW_BEST_BLOCK_INTERVAL: u64 = 60; const MIN_ALLOWED_FEE_PER_1000_WEIGHT: u32 = 253; -#[derive(Debug, Display)] +struct TxWithBlockInfo { + tx: Transaction, + block_header: BlockHeader, + block_height: u64, +} + +#[derive(Display)] pub enum FindWatchedOutputSpendError { - #[display(fmt = "Can't convert transaction: {}", _0)] - TransactionConvertionErr(String), - #[display(fmt = "Can't deserialize block header: {}", _0)] - BlockHeaderDeserializeErr(String), + #[display(fmt = "Spent output info has blockhash not height")] + HashNotHeight, + #[display(fmt = "Deserialization error: {}", _0)] + DeserializationErr(String), + #[display(fmt = "RPC error {}", _0)] + RpcError(String), +} + +impl From for FindWatchedOutputSpendError { + fn from(e: JsonRpcError) -> Self { FindWatchedOutputSpendError::RpcError(e.to_string()) } +} + +impl From for FindWatchedOutputSpendError { + fn from(e: EncodeError) -> Self { FindWatchedOutputSpendError::DeserializationErr(e.to_string()) } } async fn find_watched_output_spend_with_header( electrum_client: &ElectrumClient, output: &WatchedOutput, -) -> Result, FindWatchedOutputSpendError> { +) -> Result, FindWatchedOutputSpendError> { // from_block parameter is not used in find_output_spend for electrum clients let utxo_client: UtxoRpcClientEnum = electrum_client.clone().into(); + let tx_hash = H256::from(output.outpoint.txid.as_hash().into_inner()); let output_spend = match utxo_client .find_output_spend( - H256::from(output.outpoint.txid.as_hash().into_inner()), + tx_hash, output.script_pubkey.as_ref(), output.outpoint.index.into(), BlockHashOrHeight::Hash(Default::default()), ) .compat() .await + .map_err(FindWatchedOutputSpendError::RpcError)? { - Ok(Some(output)) => output, - _ => return Ok(None), + Some(output) => output, + None => return Ok(None), }; - if let BlockHashOrHeight::Height(height) = output_spend.spent_in_block { - if let Ok(header) = electrum_client.blockchain_block_header(height as u64).compat().await { - match deserialize(&header) { - Ok(h) => { - let spending_tx = match Transaction::try_from(output_spend.spending_tx) { - Ok(tx) => tx, - Err(e) => return Err(FindWatchedOutputSpendError::TransactionConvertionErr(e.to_string())), - }; - return Ok(Some((h, output_spend.input_index, spending_tx, height as u64))); - }, - Err(e) => return Err(FindWatchedOutputSpendError::BlockHeaderDeserializeErr(e.to_string())), - } - } - } - Ok(None) + let height = match output_spend.spent_in_block { + BlockHashOrHeight::Height(h) => h, + _ => return Err(FindWatchedOutputSpendError::HashNotHeight), + }; + let header = electrum_client.blockchain_block_header(height as u64).compat().await?; + let block_header = deserialize(&header)?; + let spending_tx = Transaction::try_from(output_spend.spending_tx)?; + + Ok(Some(TxWithBlockInfo { + tx: spending_tx, + block_header, + block_height: height as u64, + })) } pub async fn get_best_header(best_header_listener: &ElectrumClient) -> EnableLightningResult { @@ -407,36 +423,41 @@ impl Platform { let mut outputs_to_remove = Vec::new(); let registered_outputs = self.registered_outputs.lock().clone(); for output in registered_outputs { - let result = match find_watched_output_spend_with_header(client, &output).await { - Ok(res) => res, - Err(e) => { - log::error!( - "Error while trying to find if the registered output {:?} is spent: {}", - output.outpoint, - e - ); - continue; - }, - }; - if let Some((header, _, tx, height)) = result { - if !transactions_to_confirm.iter().any(|info| info.txid == tx.txid()) { - let rpc_txid = H256Json::from(tx.txid().as_hash().into_inner()).reversed(); - let index = match client - .blockchain_transaction_get_merkle(rpc_txid, height) - .compat() - .await + match find_watched_output_spend_with_header(client, &output).await { + Ok(Some(tx_info)) => { + if !transactions_to_confirm + .iter() + .any(|info| info.txid == tx_info.tx.txid()) { - Ok(merkle_branch) => merkle_branch.pos, - Err(e) => { - log::error!("Error getting transaction position in the block: {}", e.to_string()); - continue; - }, - }; - let confirmed_transaction_info = - ConfirmedTransactionInfo::new(tx.txid(), header, index, tx, height as u32); - transactions_to_confirm.push(confirmed_transaction_info); - } - outputs_to_remove.push(output); + let rpc_txid = H256Json::from(tx_info.tx.txid().as_hash().into_inner()).reversed(); + let index = match client + .blockchain_transaction_get_merkle(rpc_txid, tx_info.block_height) + .compat() + .await + { + Ok(merkle_branch) => merkle_branch.pos, + Err(e) => { + log::error!("Error getting transaction position in the block: {}", e.to_string()); + continue; + }, + }; + let confirmed_transaction_info = ConfirmedTransactionInfo::new( + tx_info.tx.txid(), + tx_info.block_header, + index, + tx_info.tx, + tx_info.block_height as u32, + ); + transactions_to_confirm.push(confirmed_transaction_info); + } + outputs_to_remove.push(output); + }, + Ok(None) => (), + Err(e) => log::error!( + "Error while trying to find if the registered output {:?} is spent: {}", + output.outpoint, + e + ), } } self.registered_outputs From 48256014a5cd8e9a2ee2b1cce06a120091231a8d Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Tue, 5 Apr 2022 17:54:42 +0200 Subject: [PATCH 33/49] review fixes --- mm2src/coins/lightning.rs | 124 +++++------------- mm2src/coins/lightning/ln_errors.rs | 2 - mm2src/coins/lightning/ln_events.rs | 14 +- .../coins/lightning_persister/src/storage.rs | 2 +- 4 files changed, 38 insertions(+), 104 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index f1c2470ca0..c6a6fedd3f 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -830,99 +830,43 @@ pub struct OpenChannelsFilter { } fn apply_open_channel_filter(channel_details: &ChannelDetailsForRPC, filter: &OpenChannelsFilter) -> bool { - let is_channel_id = if let Some(channel_id) = filter.channel_id { - channel_details.channel_id == channel_id - } else { - true - }; + let is_channel_id = filter.channel_id.is_none() || Some(&channel_details.channel_id) == filter.channel_id.as_ref(); - let is_counterparty_node_id = if let Some(counterparty_node_id) = &filter.counterparty_node_id { - &channel_details.counterparty_node_id == counterparty_node_id - } else { - true - }; + let is_counterparty_node_id = filter.counterparty_node_id.is_none() + || Some(&channel_details.counterparty_node_id) == filter.counterparty_node_id.as_ref(); - let is_funding_tx = if let Some(funding_tx) = filter.funding_tx { - if let Some(channel_details_funding_tx) = channel_details.funding_tx { - channel_details_funding_tx == funding_tx - } else { - false - } - } else { - true - }; + let is_funding_tx = filter.funding_tx.is_none() || channel_details.funding_tx == filter.funding_tx; - let is_from_funding_value_sats = if let Some(from_funding_value_sats) = filter.from_funding_value_sats { - channel_details.funding_tx_value_sats >= from_funding_value_sats - } else { - true - }; + let is_from_funding_value_sats = + Some(&channel_details.funding_tx_value_sats) >= filter.from_funding_value_sats.as_ref(); - let is_to_funding_value_sats = if let Some(to_funding_value_sats) = filter.to_funding_value_sats { - channel_details.funding_tx_value_sats <= to_funding_value_sats - } else { - true - }; + let is_to_funding_value_sats = filter.to_funding_value_sats.is_none() + || Some(&channel_details.funding_tx_value_sats) <= filter.to_funding_value_sats.as_ref(); - let is_outbound = if let Some(is_outbound) = filter.is_outbound { - channel_details.is_outbound == is_outbound - } else { - true - }; + let is_outbound = filter.is_outbound.is_none() || Some(&channel_details.is_outbound) == filter.is_outbound.as_ref(); - let is_from_balance_msat = if let Some(from_balance_msat) = filter.from_balance_msat { - channel_details.balance_msat >= from_balance_msat - } else { - true - }; + let is_from_balance_msat = Some(&channel_details.balance_msat) >= filter.from_balance_msat.as_ref(); - let is_to_balance_msat = if let Some(to_balance_msat) = filter.to_balance_msat { - channel_details.balance_msat <= to_balance_msat - } else { - true - }; + let is_to_balance_msat = + filter.to_balance_msat.is_none() || Some(&channel_details.balance_msat) <= filter.to_balance_msat.as_ref(); - let is_from_outbound_capacity_msat = if let Some(from_outbound_capacity_msat) = filter.from_outbound_capacity_msat { - channel_details.outbound_capacity_msat >= from_outbound_capacity_msat - } else { - true - }; + let is_from_outbound_capacity_msat = + Some(&channel_details.outbound_capacity_msat) >= filter.from_outbound_capacity_msat.as_ref(); - let is_to_outbound_capacity_msat = if let Some(to_outbound_capacity_msat) = filter.to_outbound_capacity_msat { - channel_details.outbound_capacity_msat <= to_outbound_capacity_msat - } else { - true - }; + let is_to_outbound_capacity_msat = filter.to_outbound_capacity_msat.is_none() + || Some(&channel_details.outbound_capacity_msat) <= filter.to_outbound_capacity_msat.as_ref(); - let is_from_inbound_capacity_msat = if let Some(from_inbound_capacity_msat) = filter.from_inbound_capacity_msat { - channel_details.inbound_capacity_msat >= from_inbound_capacity_msat - } else { - true - }; + let is_from_inbound_capacity_msat = + Some(&channel_details.inbound_capacity_msat) >= filter.from_inbound_capacity_msat.as_ref(); - let is_to_inbound_capacity_msat = if let Some(to_inbound_capacity_msat) = filter.to_inbound_capacity_msat { - channel_details.inbound_capacity_msat <= to_inbound_capacity_msat - } else { - true - }; + let is_to_inbound_capacity_msat = filter.to_inbound_capacity_msat.is_none() + || Some(&channel_details.inbound_capacity_msat) <= filter.to_inbound_capacity_msat.as_ref(); - let is_confirmed = if let Some(confirmed) = filter.confirmed { - channel_details.confirmed == confirmed - } else { - true - }; + let is_confirmed = filter.confirmed.is_none() || Some(&channel_details.confirmed) == filter.confirmed.as_ref(); - let is_usable = if let Some(is_usable) = filter.is_usable { - channel_details.is_usable == is_usable - } else { - true - }; + let is_usable = filter.is_usable.is_none() || Some(&channel_details.is_usable) == filter.is_usable.as_ref(); - let is_public = if let Some(is_public) = filter.is_public { - channel_details.is_public == is_public - } else { - true - }; + let is_public = filter.is_public.is_none() || Some(&channel_details.is_public) == filter.is_public.as_ref(); is_channel_id && is_counterparty_node_id @@ -1024,27 +968,27 @@ pub async fn list_open_channels_by_filter( total_open_channels.sort_by(|a, b| a.rpc_channel_id.cmp(&b.rpc_channel_id)); - let total = total_open_channels.len(); + let open_channels_filtered = if let Some(ref filter) = req.filter { + total_open_channels + .into_iter() + .filter(|chan| apply_open_channel_filter(chan, filter)) + .collect() + } else { + total_open_channels + }; let offset = match req.paging_options { PagingOptionsEnum::PageNumber(page) => (page.get() - 1) * req.limit, - PagingOptionsEnum::FromId(rpc_id) => total_open_channels + PagingOptionsEnum::FromId(rpc_id) => open_channels_filtered .iter() .position(|x| x.rpc_channel_id == rpc_id) .map(|pos| pos + 1) .unwrap_or_default(), }; - let open_channels_filtered = if let Some(ref filter) = req.filter { - total_open_channels - .into_iter() - .filter(|chan| apply_open_channel_filter(chan, filter)) - .collect() - } else { - total_open_channels - }; + let total = open_channels_filtered.len(); - let open_channels = if offset + req.limit <= open_channels_filtered.len() { + let open_channels = if offset + req.limit <= total { open_channels_filtered[offset..offset + req.limit].to_vec() } else { open_channels_filtered[offset..].to_vec() diff --git a/mm2src/coins/lightning/ln_errors.rs b/mm2src/coins/lightning/ln_errors.rs index 60b7f67279..eee54abd70 100644 --- a/mm2src/coins/lightning/ln_errors.rs +++ b/mm2src/coins/lightning/ln_errors.rs @@ -505,8 +505,6 @@ pub enum SaveChannelClosingError { RpcError(String), #[display(fmt = "Error while waiting for the funding transaction to be spent: {}", _0)] WaitForFundingTxSpendError(String), - #[display(fmt = "Wrong closing transaction type: {}", _0)] - WrongClosingTxType(String), } impl From for SaveChannelClosingError { diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 34990e51dc..3c86a46b11 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -191,7 +191,7 @@ async fn save_channel_closing_details( .compat() .await?; - let closing_tx_enum = platform + let closing_tx = platform .coin .wait_for_tx_spend( &funding_tx_bytes.into_vec(), @@ -203,18 +203,10 @@ async fn save_channel_closing_details( .await .map_to_mm(SaveChannelClosingError::WaitForFundingTxSpendError)?; - let closing_tx = match closing_tx_enum { - TransactionEnum::UtxoTx(tx) => tx, - e => { - return Err(MmError::new(SaveChannelClosingError::WrongClosingTxType(format!( - "{:?}", - e - )))) - }, - }; + let closing_tx_hash = format!("{:02x}", closing_tx.tx_hash()); persister - .add_closing_tx_to_sql(user_channel_id, closing_tx.hash().reversed().to_string()) + .add_closing_tx_to_sql(user_channel_id, closing_tx_hash) .await?; Ok(()) diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 2349b10a1a..5f96af048f 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -216,7 +216,7 @@ pub trait SqlStorage { async fn update_channel_to_closed(&self, rpc_id: u64, closure_reason: String) -> Result<(), Self::Error>; - async fn add_closing_tx_to_sql(&self, rpc_id: u64, closing_tx_: String) -> Result<(), Self::Error>; + async fn add_closing_tx_to_sql(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error>; async fn get_closed_channels_by_filter( &self, From 0b3691b50f6871666382fe6fda825e380af6ea03 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Wed, 6 Apr 2022 15:53:39 +0200 Subject: [PATCH 34/49] reviw fix for funding_generated_in_block --- mm2src/coins/eth.rs | 6 ++-- mm2src/coins/lightning/ln_errors.rs | 2 ++ mm2src/coins/lightning/ln_events.rs | 10 ++++-- mm2src/coins/lightning/ln_platform.rs | 34 +++++++++++++++--- mm2src/coins/lightning/ln_utils.rs | 3 ++ mm2src/coins/lightning_persister/src/lib.rs | 35 +++++++++++++++++++ .../coins/lightning_persister/src/storage.rs | 4 ++- 7 files changed, 82 insertions(+), 12 deletions(-) diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 3da456e72f..7cc6fced85 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -48,7 +48,7 @@ use std::collections::HashMap; use std::ops::Deref; use std::path::PathBuf; use std::str::FromStr; -use std::sync::atomic::{AtomicU64, Ordering as AtomicOrderding}; +use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; use std::sync::{Arc, Mutex}; use web3::types::{Action as TraceAction, BlockId, BlockNumber, Bytes, CallRequest, FilterBuilder, Log, Trace, TraceFilterBuilder, Transaction as Web3Transaction, TransactionId}; @@ -3077,13 +3077,13 @@ impl MmCoin for EthCoin { }) } - fn required_confirmations(&self) -> u64 { self.required_confirmations.load(AtomicOrderding::Relaxed) } + fn required_confirmations(&self) -> u64 { self.required_confirmations.load(AtomicOrdering::Relaxed) } fn requires_notarization(&self) -> bool { false } fn set_required_confirmations(&self, confirmations: u64) { self.required_confirmations - .store(confirmations, AtomicOrderding::Relaxed); + .store(confirmations, AtomicOrdering::Relaxed); } fn set_requires_notarization(&self, _requires_nota: bool) { diff --git a/mm2src/coins/lightning/ln_errors.rs b/mm2src/coins/lightning/ln_errors.rs index eee54abd70..983027e17f 100644 --- a/mm2src/coins/lightning/ln_errors.rs +++ b/mm2src/coins/lightning/ln_errors.rs @@ -497,6 +497,8 @@ pub enum SaveChannelClosingError { DbError(String), #[display(fmt = "Channel with rpc id {} not found in DB", _0)] ChannelNotFound(u64), + #[display(fmt = "funding_generated_in_block is Null in DB")] + BlockHeightNull, #[display(fmt = "Funding transaction hash is Null in DB")] FundingTxNull, #[display(fmt = "Error parsing funding transaction hash: {}", _0)] diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 3c86a46b11..44d94bdc06 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -176,6 +176,10 @@ async fn save_channel_closing_details( .await? .ok_or_else(|| MmError::new(SaveChannelClosingError::ChannelNotFound(user_channel_id)))?; + let from_block = channel_details + .funding_generated_in_block + .ok_or_else(|| MmError::new(SaveChannelClosingError::BlockHeightNull))?; + let tx_id = channel_details .funding_tx .ok_or_else(|| MmError::new(SaveChannelClosingError::FundingTxNull))?; @@ -196,7 +200,7 @@ async fn save_channel_closing_details( .wait_for_tx_spend( &funding_tx_bytes.into_vec(), (now_ms() / 1000) + 3600, - channel_details.funding_generated_in_block.unwrap_or_default(), + from_block, &None, ) .compat() @@ -261,13 +265,13 @@ impl LightningEventHandler { let platform = self.platform.clone(); let persister = self.persister.clone(); spawn(async move { - let current_block = platform.coin.current_block().compat().await.unwrap_or_default(); + let best_block_height = platform.best_block_height(); persister .add_funding_tx_to_sql( user_channel_id, funding_txid.to_string(), channel_value_satoshis, - current_block, + best_block_height, ) .await .error_log(); diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index 9aa503d828..17b7531ae2 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -21,6 +21,7 @@ use lightning::chain::{chaininterface::{BroadcasterInterface, ConfirmationTarget use rpc::v1::types::H256 as H256Json; use std::cmp; use std::convert::{TryFrom, TryInto}; +use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; const CHECK_FOR_NEW_BEST_BLOCK_INTERVAL: u64 = 60; const MIN_ALLOWED_FEE_PER_1000_WEIGHT: u32 = 253; @@ -152,6 +153,7 @@ pub async fn update_best_block( pub async fn ln_best_block_update_loop( platform: Arc, + persister: Arc, chain_monitor: Arc, channel_manager: Arc, best_header_listener: ElectrumClient, @@ -168,12 +170,14 @@ pub async fn ln_best_block_update_loop( }, }; if current_best_block != best_header.clone().into() { + platform.update_best_block_height(best_header.block_height()); platform .process_txs_unconfirmations(chain_monitor.clone(), channel_manager.clone()) .await; platform .process_txs_confirmations( best_header_listener.clone(), + persister.clone(), chain_monitor.clone(), channel_manager.clone(), best_header.block_height(), @@ -210,14 +214,16 @@ pub struct Platform { pub coin: UtxoStandardCoin, /// Main/testnet/signet/regtest Needed for lightning node to know which network to connect to pub network: BlockchainNetwork, - // Default fees to and confirmation targets to be used for FeeEstimator. Default fees are used when the call for - // estimate_fee_sat fails. + /// The best block height. + pub best_block_height: AtomicU64, + /// Default fees to and confirmation targets to be used for FeeEstimator. Default fees are used when the call for + /// estimate_fee_sat fails. pub default_fees_and_confirmations: PlatformCoinConfirmations, - // This cache stores the transactions that the LN node has interest in. + /// This cache stores the transactions that the LN node has interest in. pub registered_txs: PaMutex>>, - // This cache stores the outputs that the LN node has interest in. + /// This cache stores the outputs that the LN node has interest in. pub registered_outputs: PaMutex>, - // This cache stores transactions to be broadcasted once the other node accepts the channel + /// This cache stores transactions to be broadcasted once the other node accepts the channel pub unsigned_funding_txs: PaMutex>, } @@ -230,6 +236,7 @@ impl Platform { Platform { coin, network, + best_block_height: AtomicU64::new(0), default_fees_and_confirmations, registered_txs: PaMutex::new(HashMap::new()), registered_outputs: PaMutex::new(Vec::new()), @@ -237,6 +244,12 @@ impl Platform { } } + pub fn update_best_block_height(&self, new_height: u64) { + self.best_block_height.store(new_height, AtomicOrdering::Relaxed); + } + + pub fn best_block_height(&self) -> u64 { self.best_block_height.load(AtomicOrdering::Relaxed) } + pub fn add_tx(&self, txid: &Txid, script_pubkey: &Script) { let mut registered_txs = self.registered_txs.lock(); match registered_txs.get_mut(txid) { @@ -468,6 +481,7 @@ impl Platform { pub async fn process_txs_confirmations( &self, client: ElectrumClient, + persister: Arc, chain_monitor: Arc, channel_manager: Arc, current_height: u64, @@ -485,6 +499,16 @@ impl Platform { }); for confirmed_transaction_info in transactions_to_confirm { + let best_block_height = self.best_block_height(); + if let Err(e) = persister + .update_funding_tx_block_height( + confirmed_transaction_info.transaction.txid().to_string(), + best_block_height, + ) + .await + { + log::error!("Unable to update the funding tx block height in DB: {}", e); + } channel_manager.transactions_confirmed( &confirmed_transaction_info.header, &[( diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 01f98b5242..def40ba3b4 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -127,6 +127,7 @@ pub async fn init_channel_manager( }, }; let best_header = get_best_header(&rpc_client).await?; + platform.update_best_block_height(best_header.block_height()); let best_block = RpcBestBlock::from(best_header.clone()); let best_block_hash = BlockHash::from_hash( sha256d::Hash::from_slice(&best_block.hash.0).map_to_mm(|e| EnableLightningError::HashError(e.to_string()))?, @@ -179,6 +180,7 @@ pub async fn init_channel_manager( .process_txs_confirmations( // It's safe to use unwrap here for now until implementing Native Client for Lightning rpc_client.clone(), + persister.clone(), chain_monitor.clone(), channel_manager.clone(), best_header.block_height(), @@ -199,6 +201,7 @@ pub async fn init_channel_manager( spawn(ln_best_block_update_loop( // It's safe to use unwrap here for now until implementing Native Client for Lightning platform, + persister.clone(), chain_monitor.clone(), channel_manager.clone(), rpc_client.clone(), diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index db84d1cd04..b2ab551314 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -274,6 +274,15 @@ fn update_funding_tx_sql(for_coin: &str) -> Result { Ok(sql) } +fn update_funding_tx_block_height_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "UPDATE ".to_owned() + &table_name + " SET funding_generated_in_block = ?2 WHERE funding_tx = ?1;"; + + Ok(sql) +} + fn update_channel_to_closed_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; @@ -1024,6 +1033,22 @@ impl SqlStorage for LightningPersister { .await } + async fn update_funding_tx_block_height(&self, funding_tx: String, block_height: u64) -> Result<(), Self::Error> { + let for_coin = self.storage_ticker.clone(); + let generated_in_block = block_height as u32; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [&funding_tx as &dyn ToSql, &generated_in_block as &dyn ToSql]; + sql_transaction.execute(&update_funding_tx_block_height_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + async fn update_channel_to_closed(&self, rpc_id: u64, closure_reason: String) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); let rpc_id = rpc_id.to_string(); @@ -1627,6 +1652,16 @@ mod tests { let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); + block_on(persister.update_funding_tx_block_height( + "9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into(), + 50001, + )) + .unwrap(); + expected_channel_details.funding_generated_in_block = Some(50001); + + let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); + block_on(persister.update_channel_to_closed(2, "the channel was cooperatively closed".into())).unwrap(); expected_channel_details.closure_reason = Some("the channel was cooperatively closed".into()); expected_channel_details.is_closed = true; diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 5f96af048f..8a13027fe7 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -56,7 +56,7 @@ pub struct SqlChannelDetails { pub claiming_tx: Option, #[serde(skip_serializing_if = "Option::is_none")] pub claimed_balance: Option, - #[serde(skip_serializing)] + #[serde(skip_serializing_if = "Option::is_none")] pub funding_generated_in_block: Option, pub is_outbound: bool, pub is_public: bool, @@ -214,6 +214,8 @@ pub trait SqlStorage { funding_generated_in_block: u64, ) -> Result<(), Self::Error>; + async fn update_funding_tx_block_height(&self, funding_tx: String, block_height: u64) -> Result<(), Self::Error>; + async fn update_channel_to_closed(&self, rpc_id: u64, closure_reason: String) -> Result<(), Self::Error>; async fn add_closing_tx_to_sql(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error>; From 658c8b3b0366e3e5e5ffe358141eeb29e0b1db04 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Thu, 7 Apr 2022 16:34:24 +0200 Subject: [PATCH 35/49] fix saving closing tx to db issues if electrums are down --- mm2src/coins/lightning.rs | 2 +- mm2src/coins/lightning/ln_errors.rs | 6 --- mm2src/coins/lightning/ln_events.rs | 40 +++------------- mm2src/coins/lightning/ln_platform.rs | 48 +++++++++++++++++++ mm2src/coins/lightning/ln_utils.rs | 24 ++++++++++ mm2src/coins/lightning_persister/src/lib.rs | 34 ++++++++++++- .../coins/lightning_persister/src/storage.rs | 2 + 7 files changed, 114 insertions(+), 42 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index c6a6fedd3f..e96ef70e2d 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -531,7 +531,7 @@ pub async fn start_lightning( let logger = ctx.log.0.clone(); // Initialize Persister - let persister = ln_utils::init_persister(ctx, conf.ticker.clone(), params.backup_path).await?; + let persister = ln_utils::init_persister(ctx, platform.clone(), conf.ticker.clone(), params.backup_path).await?; // Initialize the KeysManager let keys_manager = ln_utils::init_keys_manager(ctx)?; diff --git a/mm2src/coins/lightning/ln_errors.rs b/mm2src/coins/lightning/ln_errors.rs index 983027e17f..d73c09b414 100644 --- a/mm2src/coins/lightning/ln_errors.rs +++ b/mm2src/coins/lightning/ln_errors.rs @@ -503,8 +503,6 @@ pub enum SaveChannelClosingError { FundingTxNull, #[display(fmt = "Error parsing funding transaction hash: {}", _0)] FundingTxParseError(String), - #[display(fmt = "Error getting funding transaction bytes through RPC: {}", _0)] - RpcError(String), #[display(fmt = "Error while waiting for the funding transaction to be spent: {}", _0)] WaitForFundingTxSpendError(String), } @@ -512,7 +510,3 @@ pub enum SaveChannelClosingError { impl From for SaveChannelClosingError { fn from(err: SqlError) -> SaveChannelClosingError { SaveChannelClosingError::DbError(err.to_string()) } } - -impl From for SaveChannelClosingError { - fn from(e: UtxoRpcError) -> Self { SaveChannelClosingError::RpcError(e.to_string()) } -} diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 44d94bdc06..e31e660168 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -5,7 +5,6 @@ use bitcoin::blockdata::transaction::Transaction; use common::executor::{spawn, Timer}; use common::{log, now_ms}; use core::time::Duration; -use futures::compat::Future01CompatExt; use lightning::chain::chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator}; use lightning::chain::keysinterface::SpendableOutputDescriptor; use lightning::util::events::{Event, EventHandler, PaymentPurpose}; @@ -176,38 +175,7 @@ async fn save_channel_closing_details( .await? .ok_or_else(|| MmError::new(SaveChannelClosingError::ChannelNotFound(user_channel_id)))?; - let from_block = channel_details - .funding_generated_in_block - .ok_or_else(|| MmError::new(SaveChannelClosingError::BlockHeightNull))?; - - let tx_id = channel_details - .funding_tx - .ok_or_else(|| MmError::new(SaveChannelClosingError::FundingTxNull))?; - - let tx_hash = - H256Json::from_str(&tx_id).map_to_mm(|e| SaveChannelClosingError::FundingTxParseError(e.to_string()))?; - - let funding_tx_bytes = platform - .coin - .as_ref() - .rpc_client - .get_transaction_bytes(&tx_hash) - .compat() - .await?; - - let closing_tx = platform - .coin - .wait_for_tx_spend( - &funding_tx_bytes.into_vec(), - (now_ms() / 1000) + 3600, - from_block, - &None, - ) - .compat() - .await - .map_to_mm(SaveChannelClosingError::WaitForFundingTxSpendError)?; - - let closing_tx_hash = format!("{:02x}", closing_tx.tx_hash()); + let closing_tx_hash = platform.get_channel_closing_tx(channel_details).await?; persister .add_closing_tx_to_sql(user_channel_id, closing_tx_hash) @@ -391,7 +359,11 @@ impl LightningEventHandler { if user_channel_id != 0 { spawn(async move { if let Err(e) = save_channel_closing_details(persister, platform, user_channel_id, reason).await { - log::error!("Unable to update channel closing details in DB: {}", e); + log::error!( + "Unable to update channel {} closing details in DB: {}", + user_channel_id, + e + ); } }); } diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index 17b7531ae2..98819cc08a 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -1,4 +1,5 @@ use super::*; +use crate::lightning::ln_errors::{SaveChannelClosingError, SaveChannelClosingResult}; use crate::utxo::rpc_clients::{electrum_script_hash, BestBlock as RpcBestBlock, BlockHashOrHeight, ElectrumBlockHeader, ElectrumClient, ElectrumNonce, EstimateFeeMethod, UtxoRpcClientEnum, UtxoRpcError}; @@ -25,6 +26,7 @@ use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; const CHECK_FOR_NEW_BEST_BLOCK_INTERVAL: u64 = 60; const MIN_ALLOWED_FEE_PER_1000_WEIGHT: u32 = 253; +const GET_FUNDING_TX_INTERVAL: f64 = 60.; struct TxWithBlockInfo { tx: Transaction, @@ -527,6 +529,52 @@ impl Platform { ); } } + + pub async fn get_channel_closing_tx(&self, channel_details: SqlChannelDetails) -> SaveChannelClosingResult { + let from_block = channel_details + .funding_generated_in_block + .ok_or_else(|| MmError::new(SaveChannelClosingError::BlockHeightNull))?; + + let tx_id = channel_details + .funding_tx + .ok_or_else(|| MmError::new(SaveChannelClosingError::FundingTxNull))?; + + let tx_hash = + H256Json::from_str(&tx_id).map_to_mm(|e| SaveChannelClosingError::FundingTxParseError(e.to_string()))?; + + let funding_tx_bytes = loop { + match self + .coin + .as_ref() + .rpc_client + .get_transaction_bytes(&tx_hash) + .compat() + .await + { + Ok(tx) => break tx, + Err(e) => { + log::error!("Error getting funding tx bytes: {}", e); + Timer::sleep(GET_FUNDING_TX_INTERVAL).await; + }, + } + }; + + let closing_tx = self + .coin + .wait_for_tx_spend( + &funding_tx_bytes.into_vec(), + (now_ms() / 1000) + 3600, + from_block, + &None, + ) + .compat() + .await + .map_to_mm(SaveChannelClosingError::WaitForFundingTxSpendError)?; + + let closing_tx_hash = format!("{:02x}", closing_tx.tx_hash()); + + Ok(closing_tx_hash) + } } impl FeeEstimator for Platform { diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index def40ba3b4..67f0ae02a5 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -48,6 +48,7 @@ fn ln_data_backup_dir(ctx: &MmArc, path: Option, ticker: &str) -> Option pub async fn init_persister( ctx: &MmArc, + platform: Arc, ticker: String, backup_path: Option, ) -> EnableLightningResult> { @@ -71,6 +72,29 @@ pub async fn init_persister( if !is_sql_initialized { persister.init_sql().await?; } + + let closed_channels_without_closing_tx = persister.get_closed_channels_with_no_closing_tx().await?; + for channel_details in closed_channels_without_closing_tx { + let platform = platform.clone(); + let persister = persister.clone(); + let user_channel_id = channel_details.rpc_id; + spawn(async move { + if let Ok(closing_tx_hash) = platform + .get_channel_closing_tx(channel_details) + .await + .error_log_passthrough() + { + if let Err(e) = persister.add_closing_tx_to_sql(user_channel_id, closing_tx_hash).await { + log::error!( + "Unable to update channel {} closing details in DB: {}", + user_channel_id, + e + ); + } + } + }); + } + Ok(persister) } diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index b2ab551314..514327e585 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -312,7 +312,7 @@ fn get_channels_builder_preimage(for_coin: &str) -> Result Ok(sql_builder) } -fn finalize_get_channels_sql_builder(sql_builder: &mut SqlBuilder, offset: usize, limit: usize) { +fn add_fields_to_get_channels_sql_builder(sql_builder: &mut SqlBuilder) { sql_builder .field("rpc_id") .field("channel_id") @@ -329,6 +329,9 @@ fn finalize_get_channels_sql_builder(sql_builder: &mut SqlBuilder, offset: usize .field("is_closed") .field("created_at") .field("last_updated"); +} + +fn finalize_get_channels_sql_builder(sql_builder: &mut SqlBuilder, offset: usize, limit: usize) { sql_builder.offset(offset); sql_builder.limit(limit); sql_builder.order_desc("last_updated"); @@ -1134,6 +1137,7 @@ impl SqlStorage for LightningPersister { apply_get_channels_filter(&mut sql_builder, &mut params, f); } let params_as_trait: Vec<_> = params.iter().map(|(key, value)| (*key, value as &dyn ToSql)).collect(); + add_fields_to_get_channels_sql_builder(&mut sql_builder); finalize_get_channels_sql_builder(&mut sql_builder, offset, limit); let sql = sql_builder.sql().expect("valid sql"); @@ -1151,6 +1155,25 @@ impl SqlStorage for LightningPersister { .await } + async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error> { + let mut builder = get_channels_builder_preimage(self.storage_ticker.as_str())?; + builder.and_where("closing_tx IS NULL"); + add_fields_to_get_channels_sql_builder(&mut builder); + let sql = builder.sql().expect("valid sql"); + let sqlite_connection = self.sqlite_connection.clone(); + + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + + let mut stmt = conn.prepare(&sql)?; + let result = stmt + .query_map_named(&[], channel_details_from_row)? + .collect::>()?; + Ok(result) + }) + .await + } + async fn get_payments_by_filter( &self, filter: Option, @@ -1669,6 +1692,9 @@ mod tests { let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); + let actual_channels = block_on(persister.get_closed_channels_with_no_closing_tx()).unwrap(); + assert_eq!(actual_channels.len(), 1); + let closed_channels = block_on(persister.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 10)).unwrap(); assert_eq!(closed_channels.channels.len(), 1); @@ -1679,6 +1705,9 @@ mod tests { block_on(persister.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 10)).unwrap(); assert_eq!(closed_channels.channels.len(), 2); + let actual_channels = block_on(persister.get_closed_channels_with_no_closing_tx()).unwrap(); + assert_eq!(actual_channels.len(), 2); + block_on(persister.add_closing_tx_to_sql( 2, "5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into(), @@ -1687,6 +1716,9 @@ mod tests { expected_channel_details.closing_tx = Some("5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into()); + let actual_channels = block_on(persister.get_closed_channels_with_no_closing_tx()).unwrap(); + assert_eq!(actual_channels.len(), 1); + let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 8a13027fe7..a3ff590519 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -227,6 +227,8 @@ pub trait SqlStorage { limit: usize, ) -> Result; + async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error>; + async fn get_payments_by_filter( &self, filter: Option, From 79982e259cdebd9c6eb7a7fec96522c097458be2 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Fri, 8 Apr 2022 15:22:36 +0200 Subject: [PATCH 36/49] review fixes: better grouping of SqlStorage trait functions + added descriptions --- mm2src/coins/lightning/ln_platform.rs | 1 + mm2src/coins/lightning_persister/src/lib.rs | 220 +++++++++--------- .../coins/lightning_persister/src/storage.rs | 56 +++-- 3 files changed, 150 insertions(+), 127 deletions(-) diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index 98819cc08a..dcc54dc6ea 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -230,6 +230,7 @@ pub struct Platform { } impl Platform { + #[inline] pub fn new( coin: UtxoStandardCoin, network: BlockchainNetwork, diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 514327e585..08002a585f 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -895,6 +895,18 @@ impl SqlStorage for LightningPersister { .await } + async fn get_last_channel_rpc_id(&self) -> Result { + let sql = get_last_channel_rpc_id_sql(self.storage_ticker.as_str())?; + let sqlite_connection = self.sqlite_connection.clone(); + + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + let count: u32 = conn.query_row(&sql, NO_PARAMS, |r| r.get(0))?; + Ok(count) + }) + .await + } + async fn add_channel_to_sql(&self, details: SqlChannelDetails) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); let rpc_id = details.rpc_id.to_string(); @@ -928,82 +940,6 @@ impl SqlStorage for LightningPersister { .await } - async fn add_or_update_payment_in_sql(&self, info: PaymentInfo) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let payment_hash = hex::encode(info.payment_hash.0); - let (is_outbound, destination) = match info.payment_type { - PaymentType::OutboundPayment { destination } => (true as i32, destination.map(|d| d.to_string())), - PaymentType::InboundPayment => (false as i32, None), - }; - let description = info.description; - let preimage = info.preimage.map(|p| hex::encode(p.0)); - let secret = info.secret.map(|s| hex::encode(s.0)); - let amount_msat = info.amt_msat.map(|a| a as u32); - let fee_paid_msat = info.fee_paid_msat.map(|f| f as u32); - let status = info.status.to_string(); - let created_at = info.created_at as u32; - let last_updated = info.last_updated as u32; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let params = [ - &payment_hash as &dyn ToSql, - &destination as &dyn ToSql, - &description as &dyn ToSql, - &preimage as &dyn ToSql, - &secret as &dyn ToSql, - &amount_msat as &dyn ToSql, - &fee_paid_msat as &dyn ToSql, - &is_outbound as &dyn ToSql, - &status as &dyn ToSql, - &created_at as &dyn ToSql, - &last_updated as &dyn ToSql, - ]; - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - sql_transaction.execute(&insert_or_update_payment_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn get_channel_from_sql(&self, rpc_id: u64) -> Result, Self::Error> { - let params = [rpc_id.to_string()]; - let sql = select_channel_from_table_by_rpc_id_sql(self.storage_ticker.as_str())?; - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - query_single_row(&conn, &sql, params, channel_details_from_row) - }) - .await - } - - async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error> { - let params = [hex::encode(hash.0)]; - let sql = select_payment_from_table_by_hash_sql(self.storage_ticker.as_str())?; - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - query_single_row(&conn, &sql, params, payment_info_from_row) - }) - .await - } - - async fn get_last_channel_rpc_id(&self) -> Result { - let sql = get_last_channel_rpc_id_sql(self.storage_ticker.as_str())?; - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - let count: u32 = conn.query_row(&sql, NO_PARAMS, |r| r.get(0))?; - Ok(count) - }) - .await - } - async fn add_funding_tx_to_sql( &self, rpc_id: u64, @@ -1071,6 +1007,25 @@ impl SqlStorage for LightningPersister { .await } + async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error> { + let mut builder = get_channels_builder_preimage(self.storage_ticker.as_str())?; + builder.and_where("closing_tx IS NULL"); + add_fields_to_get_channels_sql_builder(&mut builder); + let sql = builder.sql().expect("valid sql"); + let sqlite_connection = self.sqlite_connection.clone(); + + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + + let mut stmt = conn.prepare(&sql)?; + let result = stmt + .query_map_named(&[], channel_details_from_row)? + .collect::>()?; + Ok(result) + }) + .await + } + async fn add_closing_tx_to_sql(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); let rpc_id = rpc_id.to_string(); @@ -1089,6 +1044,41 @@ impl SqlStorage for LightningPersister { .await } + async fn add_claiming_tx_to_sql( + &self, + closing_tx: String, + claiming_tx: String, + claimed_balance: f64, + ) -> Result<(), Self::Error> { + let for_coin = self.storage_ticker.clone(); + let claimed_balance = claimed_balance.to_string(); + let last_updated = (now_ms() / 1000).to_string(); + + let params = [closing_tx, claiming_tx, claimed_balance, last_updated]; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + sql_transaction.execute(&update_claiming_tx_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn get_channel_from_sql(&self, rpc_id: u64) -> Result, Self::Error> { + let params = [rpc_id.to_string()]; + let sql = select_channel_from_table_by_rpc_id_sql(self.storage_ticker.as_str())?; + let sqlite_connection = self.sqlite_connection.clone(); + + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + query_single_row(&conn, &sql, params, channel_details_from_row) + }) + .await + } + async fn get_closed_channels_by_filter( &self, filter: Option, @@ -1155,21 +1145,54 @@ impl SqlStorage for LightningPersister { .await } - async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error> { - let mut builder = get_channels_builder_preimage(self.storage_ticker.as_str())?; - builder.and_where("closing_tx IS NULL"); - add_fields_to_get_channels_sql_builder(&mut builder); - let sql = builder.sql().expect("valid sql"); + async fn add_or_update_payment_in_sql(&self, info: PaymentInfo) -> Result<(), Self::Error> { + let for_coin = self.storage_ticker.clone(); + let payment_hash = hex::encode(info.payment_hash.0); + let (is_outbound, destination) = match info.payment_type { + PaymentType::OutboundPayment { destination } => (true as i32, destination.map(|d| d.to_string())), + PaymentType::InboundPayment => (false as i32, None), + }; + let description = info.description; + let preimage = info.preimage.map(|p| hex::encode(p.0)); + let secret = info.secret.map(|s| hex::encode(s.0)); + let amount_msat = info.amt_msat.map(|a| a as u32); + let fee_paid_msat = info.fee_paid_msat.map(|f| f as u32); + let status = info.status.to_string(); + let created_at = info.created_at as u32; + let last_updated = info.last_updated as u32; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let params = [ + &payment_hash as &dyn ToSql, + &destination as &dyn ToSql, + &description as &dyn ToSql, + &preimage as &dyn ToSql, + &secret as &dyn ToSql, + &amount_msat as &dyn ToSql, + &fee_paid_msat as &dyn ToSql, + &is_outbound as &dyn ToSql, + &status as &dyn ToSql, + &created_at as &dyn ToSql, + &last_updated as &dyn ToSql, + ]; + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + sql_transaction.execute(&insert_or_update_payment_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error> { + let params = [hex::encode(hash.0)]; + let sql = select_payment_from_table_by_hash_sql(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { let conn = sqlite_connection.lock().unwrap(); - - let mut stmt = conn.prepare(&sql)?; - let result = stmt - .query_map_named(&[], channel_details_from_row)? - .collect::>()?; - Ok(result) + query_single_row(&conn, &sql, params, payment_info_from_row) }) .await } @@ -1239,29 +1262,6 @@ impl SqlStorage for LightningPersister { }) .await } - - async fn add_claiming_tx_to_sql( - &self, - closing_tx: String, - claiming_tx: String, - claimed_balance: f64, - ) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let claimed_balance = claimed_balance.to_string(); - let last_updated = (now_ms() / 1000).to_string(); - - let params = [closing_tx, claiming_tx, claimed_balance, last_updated]; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - sql_transaction.execute(&update_claiming_tx_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } } #[cfg(test)] diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index a3ff590519..684b71cadb 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -191,21 +191,20 @@ pub struct GetPaymentsResult { pub trait SqlStorage { type Error; - /// Initializes dirs/collection/tables in storage for a specified coin + /// Initializes tables in DB. async fn init_sql(&self) -> Result<(), Self::Error>; + /// Checks if tables have been initialized or not in DB. async fn is_sql_initialized(&self) -> Result; - async fn add_channel_to_sql(&self, details: SqlChannelDetails) -> Result<(), Self::Error>; - - async fn add_or_update_payment_in_sql(&self, info: PaymentInfo) -> Result<(), Self::Error>; - - async fn get_channel_from_sql(&self, rpc_id: u64) -> Result, Self::Error>; - - async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error>; - + /// Gets the last added channel rpc_id. Can be used to deduce the rpc_id for a new channel to be added to DB. async fn get_last_channel_rpc_id(&self) -> Result; + /// Inserts a new channel record in the DB. The record's data is completed using add_funding_tx_to_sql, + /// add_closing_tx_to_sql, add_claiming_tx_to_sql when this information is available. + async fn add_channel_to_sql(&self, details: SqlChannelDetails) -> Result<(), Self::Error>; + + /// Updates a channel's DB record with the channel's funding transaction information. async fn add_funding_tx_to_sql( &self, rpc_id: u64, @@ -214,12 +213,35 @@ pub trait SqlStorage { funding_generated_in_block: u64, ) -> Result<(), Self::Error>; + /// Updates funding_tx_block_height value for a channel in the DB. Should be used to update the block height of + /// the funding tx when the transaction is confirmed on-chain. async fn update_funding_tx_block_height(&self, funding_tx: String, block_height: u64) -> Result<(), Self::Error>; + /// Updates the is_closed value for a channel in the DB to 1. async fn update_channel_to_closed(&self, rpc_id: u64, closure_reason: String) -> Result<(), Self::Error>; + /// Gets the list of closed channels records in the DB with no closing tx hashs saved yet. Can be used to check if + /// the closing tx hash needs to be fetched from the chain and saved to DB when initializing the persister. + async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error>; + + /// Updates a channel's DB record with the channel's closing transaction hash. async fn add_closing_tx_to_sql(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error>; + /// Updates a channel's DB record with information about the transaction responsible for claiming the channel's + /// closing balance back to the user's address. + async fn add_claiming_tx_to_sql( + &self, + closing_tx: String, + claiming_tx: String, + claimed_balance: f64, + ) -> Result<(), Self::Error>; + + /// Gets a channel record from DB by the channel's rpc_id. + async fn get_channel_from_sql(&self, rpc_id: u64) -> Result, Self::Error>; + + /// Gets the list of closed channels that match the provided filter criteria. The number of requested records is + /// specified by the limit parameter, the starting record to list from is specified by the paging parameter. The + /// total number of matched records along with the number of skipped records are also returned in the result. async fn get_closed_channels_by_filter( &self, filter: Option, @@ -227,19 +249,19 @@ pub trait SqlStorage { limit: usize, ) -> Result; - async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error>; + /// Inserts or updates a new payment record in the DB. + async fn add_or_update_payment_in_sql(&self, info: PaymentInfo) -> Result<(), Self::Error>; + + /// Gets a payment's record from DB by the payment's hash. + async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error>; + /// Gets the list of payments that match the provided filter criteria. The number of requested records is specified + /// by the limit parameter, the starting record to list from is specified by the paging parameter. The total number + /// of matched records along with the number of skipped records are also returned in the result. async fn get_payments_by_filter( &self, filter: Option, paging: PagingOptionsEnum, limit: usize, ) -> Result; - - async fn add_claiming_tx_to_sql( - &self, - closing_tx: String, - claiming_tx: String, - claimed_balance: f64, - ) -> Result<(), Self::Error>; } From 6d717fc0242f61a29ad9198b2c8725c446deb278 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Fri, 8 Apr 2022 17:16:01 +0200 Subject: [PATCH 37/49] review fixes: add get_open_channels_by_filter fn --- mm2src/coins/lightning.rs | 113 +++++++++++------- mm2src/coins/lightning/ln_events.rs | 24 ++-- mm2src/coins/lightning/ln_utils.rs | 10 +- mm2src/coins/lightning_persister/src/lib.rs | 88 +++++++------- .../coins/lightning_persister/src/storage.rs | 24 ++-- 5 files changed, 141 insertions(+), 118 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index e96ef70e2d..f41c18f336 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -40,8 +40,9 @@ use lightning_background_processor::BackgroundProcessor; use lightning_invoice::payment; use lightning_invoice::utils::{create_invoice_from_channelmanager, DefaultRouter}; use lightning_invoice::{Invoice, InvoiceDescription}; -use lightning_persister::storage::{ClosedChannelsFilter, FileSystemStorage, HTLCStatus, NodesAddressesMapShared, - PaymentInfo, PaymentType, PaymentsFilter, Scorer, SqlChannelDetails, SqlStorage}; +use lightning_persister::storage::{ClosedChannelsFilter, DbStorage, FileSystemStorage, HTLCStatus, + NodesAddressesMapShared, PaymentInfo, PaymentType, PaymentsFilter, Scorer, + SqlChannelDetails}; use lightning_persister::LightningPersister; use ln_conf::{ChannelOptions, LightningCoinConf, LightningProtocolConf, PlatformCoinConfirmations}; use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelError, CloseChannelResult, @@ -181,6 +182,54 @@ impl LightningCoin { last_updated: now_ms() / 1000, }) } + + async fn get_open_channels_by_filter( + &self, + filter: Option, + paging: PagingOptionsEnum, + limit: usize, + ) -> ListChannelsResult { + let mut total_open_channels: Vec = self + .channel_manager + .list_channels() + .into_iter() + .map(From::from) + .collect(); + + total_open_channels.sort_by(|a, b| a.rpc_channel_id.cmp(&b.rpc_channel_id)); + + let open_channels_filtered = if let Some(ref f) = filter { + total_open_channels + .into_iter() + .filter(|chan| apply_open_channel_filter(chan, f)) + .collect() + } else { + total_open_channels + }; + + let offset = match paging { + PagingOptionsEnum::PageNumber(page) => (page.get() - 1) * limit, + PagingOptionsEnum::FromId(rpc_id) => open_channels_filtered + .iter() + .position(|x| x.rpc_channel_id == rpc_id) + .map(|pos| pos + 1) + .unwrap_or_default(), + }; + + let total = open_channels_filtered.len(); + + let channels = if offset + limit <= total { + open_channels_filtered[offset..offset + limit].to_vec() + } else { + open_channels_filtered[offset..].to_vec() + }; + + Ok(GetOpenChannelsResult { + channels, + skipped: offset, + total, + }) + } } #[async_trait] @@ -802,7 +851,7 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes .save_nodes_addresses(ln_coin.open_channels_nodes) .await?; - ln_coin.persister.add_channel_to_sql(pending_channel_details).await?; + ln_coin.persister.add_channel_to_db(pending_channel_details).await?; Ok(OpenChannelResponse { rpc_channel_id, @@ -940,6 +989,12 @@ impl From for ChannelDetailsForRPC { } } +struct GetOpenChannelsResult { + pub channels: Vec, + pub skipped: usize, + pub total: usize, +} + #[derive(Serialize)] pub struct ListOpenChannelsResponse { open_channels: Vec, @@ -959,47 +1014,17 @@ pub async fn list_open_channels_by_filter( MmCoinEnum::LightningCoin(c) => c, _ => return MmError::err(ListChannelsError::UnsupportedCoin(coin.ticker().to_string())), }; - let mut total_open_channels: Vec = ln_coin - .channel_manager - .list_channels() - .into_iter() - .map(From::from) - .collect(); - - total_open_channels.sort_by(|a, b| a.rpc_channel_id.cmp(&b.rpc_channel_id)); - - let open_channels_filtered = if let Some(ref filter) = req.filter { - total_open_channels - .into_iter() - .filter(|chan| apply_open_channel_filter(chan, filter)) - .collect() - } else { - total_open_channels - }; - - let offset = match req.paging_options { - PagingOptionsEnum::PageNumber(page) => (page.get() - 1) * req.limit, - PagingOptionsEnum::FromId(rpc_id) => open_channels_filtered - .iter() - .position(|x| x.rpc_channel_id == rpc_id) - .map(|pos| pos + 1) - .unwrap_or_default(), - }; - let total = open_channels_filtered.len(); - - let open_channels = if offset + req.limit <= total { - open_channels_filtered[offset..offset + req.limit].to_vec() - } else { - open_channels_filtered[offset..].to_vec() - }; + let result = ln_coin + .get_open_channels_by_filter(req.filter, req.paging_options.clone(), req.limit) + .await?; Ok(ListOpenChannelsResponse { - open_channels, + open_channels: result.channels, limit: req.limit, - skipped: offset, - total, - total_pages: calc_total_pages(total, req.limit), + skipped: result.skipped, + total: result.total, + total_pages: calc_total_pages(result.total, req.limit), paging_options: req.paging_options, }) } @@ -1080,7 +1105,7 @@ pub async fn get_channel_details( None => GetChannelDetailsResponse::Closed( ln_coin .persister - .get_channel_from_sql(req.rpc_channel_id) + .get_channel_from_db(req.rpc_channel_id) .await? .ok_or(GetChannelDetailsError::NoSuchChannel(req.rpc_channel_id))?, ), @@ -1142,7 +1167,7 @@ pub async fn generate_invoice( created_at: now_ms() / 1000, last_updated: now_ms() / 1000, }; - ln_coin.persister.add_or_update_payment_in_sql(payment_info).await?; + ln_coin.persister.add_or_update_payment_in_db(payment_info).await?; Ok(GenerateInvoiceResponse { payment_hash: payment_hash.into(), invoice: invoice.into(), @@ -1203,7 +1228,7 @@ pub async fn send_payment(ctx: MmArc, req: SendPaymentReq) -> SendPaymentResult< }; ln_coin .persister - .add_or_update_payment_in_sql(payment_info.clone()) + .add_or_update_payment_in_db(payment_info.clone()) .await?; Ok(SendPaymentResponse { payment_hash: payment_info.payment_hash.0.into(), @@ -1371,7 +1396,7 @@ pub async fn get_payment_details( if let Some(payment_info) = ln_coin .persister - .get_payment_from_sql(PaymentHash(req.payment_hash.0)) + .get_payment_from_db(PaymentHash(req.payment_hash.0)) .await? { return Ok(GetPaymentDetailsResponse { diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index e31e660168..823251bd62 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -171,15 +171,13 @@ async fn save_channel_closing_details( persister.update_channel_to_closed(user_channel_id, reason).await?; let channel_details = persister - .get_channel_from_sql(user_channel_id) + .get_channel_from_db(user_channel_id) .await? .ok_or_else(|| MmError::new(SaveChannelClosingError::ChannelNotFound(user_channel_id)))?; let closing_tx_hash = platform.get_channel_closing_tx(channel_details).await?; - persister - .add_closing_tx_to_sql(user_channel_id, closing_tx_hash) - .await?; + persister.add_closing_tx_to_db(user_channel_id, closing_tx_hash).await?; Ok(()) } @@ -235,7 +233,7 @@ impl LightningEventHandler { spawn(async move { let best_block_height = platform.best_block_height(); persister - .add_funding_tx_to_sql( + .add_funding_tx_to_db( user_channel_id, funding_txid.to_string(), channel_value_satoshis, @@ -276,7 +274,7 @@ impl LightningEventHandler { match purpose { PaymentPurpose::InvoicePayment { .. } => spawn(async move { if let Ok(Some(mut payment_info)) = persister - .get_payment_from_sql(payment_hash) + .get_payment_from_db(payment_hash) .await .error_log_passthrough() { @@ -284,7 +282,7 @@ impl LightningEventHandler { payment_info.status = HTLCStatus::Succeeded; payment_info.amt_msat = Some(amt); payment_info.last_updated = now_ms() / 1000; - if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { + if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { log::error!("Unable to update payment information in DB: {}", e); } } @@ -303,7 +301,7 @@ impl LightningEventHandler { last_updated: now_ms() / 1000, }; spawn(async move { - if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { + if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { log::error!("Unable to update payment information in DB: {}", e); } }); @@ -324,7 +322,7 @@ impl LightningEventHandler { let persister = self.persister.clone(); spawn(async move { if let Ok(Some(mut payment_info)) = persister - .get_payment_from_sql(payment_hash) + .get_payment_from_db(payment_hash) .await .error_log_passthrough() { @@ -333,7 +331,7 @@ impl LightningEventHandler { payment_info.fee_paid_msat = fee_paid_msat; payment_info.last_updated = now_ms() / 1000; let amt_msat = payment_info.amt_msat; - if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { + if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { log::error!("Unable to update payment information in DB: {}", e); } log::info!( @@ -377,13 +375,13 @@ impl LightningEventHandler { let persister = self.persister.clone(); spawn(async move { if let Ok(Some(mut payment_info)) = persister - .get_payment_from_sql(payment_hash) + .get_payment_from_db(payment_hash) .await .error_log_passthrough() { payment_info.status = HTLCStatus::Failed; payment_info.last_updated = now_ms() / 1000; - if let Err(e) = persister.add_or_update_payment_in_sql(payment_info).await { + if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { log::error!("Unable to update payment information in DB: {}", e); } } @@ -464,7 +462,7 @@ impl LightningEventHandler { let persister = self.persister.clone(); spawn(async move { persister - .add_claiming_tx_to_sql( + .add_claiming_tx_to_db( closing_txid, claiming_txid, (claimed_balance as f64) - claiming_tx_fee_per_channel, diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 67f0ae02a5..1d7f44113c 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -14,7 +14,7 @@ use lightning::ln::channelmanager::{ChainParameters, ChannelManagerReadArgs, Sim use lightning::routing::network_graph::NetworkGraph; use lightning::util::config::UserConfig; use lightning::util::ser::ReadableArgs; -use lightning_persister::storage::{FileSystemStorage, NodesAddressesMap, Scorer, SqlStorage}; +use lightning_persister::storage::{DbStorage, FileSystemStorage, NodesAddressesMap, Scorer}; use lightning_persister::LightningPersister; use std::fs::File; use std::path::PathBuf; @@ -68,9 +68,9 @@ pub async fn init_persister( if !is_initialized { persister.init_fs().await?; } - let is_sql_initialized = persister.is_sql_initialized().await?; - if !is_sql_initialized { - persister.init_sql().await?; + let is_db_initialized = persister.is_db_initialized().await?; + if !is_db_initialized { + persister.init_db().await?; } let closed_channels_without_closing_tx = persister.get_closed_channels_with_no_closing_tx().await?; @@ -84,7 +84,7 @@ pub async fn init_persister( .await .error_log_passthrough() { - if let Err(e) = persister.add_closing_tx_to_sql(user_channel_id, closing_tx_hash).await { + if let Err(e) = persister.add_closing_tx_to_db(user_channel_id, closing_tx_hash).await { log::error!( "Unable to update channel {} closing details in DB: {}", user_channel_id, diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 08002a585f..b2b1537d1e 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -13,9 +13,9 @@ extern crate lightning; extern crate secp256k1; extern crate serde_json; -use crate::storage::{ChannelType, ChannelVisibility, ClosedChannelsFilter, FileSystemStorage, GetClosedChannelsResult, - GetPaymentsResult, HTLCStatus, NodesAddressesMap, NodesAddressesMapShared, PaymentInfo, - PaymentType, PaymentsFilter, Scorer, SqlChannelDetails, SqlStorage}; +use crate::storage::{ChannelType, ChannelVisibility, ClosedChannelsFilter, DbStorage, FileSystemStorage, + GetClosedChannelsResult, GetPaymentsResult, HTLCStatus, NodesAddressesMap, + NodesAddressesMapShared, PaymentInfo, PaymentType, PaymentsFilter, Scorer, SqlChannelDetails}; use crate::util::DiskWriteable; use async_trait::async_trait; use bitcoin::blockdata::constants::genesis_block; @@ -861,10 +861,10 @@ impl FileSystemStorage for LightningPersister { } #[async_trait] -impl SqlStorage for LightningPersister { +impl DbStorage for LightningPersister { type Error = SqlError; - async fn init_sql(&self) -> Result<(), Self::Error> { + async fn init_db(&self) -> Result<(), Self::Error> { let sqlite_connection = self.sqlite_connection.clone(); let sql_channels_history = create_channels_history_table_sql(self.storage_ticker.as_str())?; let sql_payments_history = create_payments_history_table_sql(self.storage_ticker.as_str())?; @@ -877,7 +877,7 @@ impl SqlStorage for LightningPersister { .await } - async fn is_sql_initialized(&self) -> Result { + async fn is_db_initialized(&self) -> Result { let channels_history_table = channels_history_table(self.storage_ticker.as_str()); validate_table_name(&channels_history_table)?; let payments_history_table = payments_history_table(self.storage_ticker.as_str()); @@ -907,7 +907,7 @@ impl SqlStorage for LightningPersister { .await } - async fn add_channel_to_sql(&self, details: SqlChannelDetails) -> Result<(), Self::Error> { + async fn add_channel_to_db(&self, details: SqlChannelDetails) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); let rpc_id = details.rpc_id.to_string(); let channel_id = details.channel_id; @@ -940,7 +940,7 @@ impl SqlStorage for LightningPersister { .await } - async fn add_funding_tx_to_sql( + async fn add_funding_tx_to_db( &self, rpc_id: u64, funding_tx: String, @@ -1026,7 +1026,7 @@ impl SqlStorage for LightningPersister { .await } - async fn add_closing_tx_to_sql(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error> { + async fn add_closing_tx_to_db(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); let rpc_id = rpc_id.to_string(); let last_updated = (now_ms() / 1000).to_string(); @@ -1044,7 +1044,7 @@ impl SqlStorage for LightningPersister { .await } - async fn add_claiming_tx_to_sql( + async fn add_claiming_tx_to_db( &self, closing_tx: String, claiming_tx: String, @@ -1067,7 +1067,7 @@ impl SqlStorage for LightningPersister { .await } - async fn get_channel_from_sql(&self, rpc_id: u64) -> Result, Self::Error> { + async fn get_channel_from_db(&self, rpc_id: u64) -> Result, Self::Error> { let params = [rpc_id.to_string()]; let sql = select_channel_from_table_by_rpc_id_sql(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); @@ -1145,7 +1145,7 @@ impl SqlStorage for LightningPersister { .await } - async fn add_or_update_payment_in_sql(&self, info: PaymentInfo) -> Result<(), Self::Error> { + async fn add_or_update_payment_in_db(&self, info: PaymentInfo) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); let payment_hash = hex::encode(info.payment_hash.0); let (is_outbound, destination) = match info.payment_type { @@ -1185,7 +1185,7 @@ impl SqlStorage for LightningPersister { .await } - async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error> { + async fn get_payment_from_db(&self, hash: PaymentHash) -> Result, Self::Error> { let params = [hex::encode(hash.0)]; let sql = select_payment_from_table_by_hash_sql(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); @@ -1609,14 +1609,14 @@ mod tests { None, Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), ); - let initialized = block_on(persister.is_sql_initialized()).unwrap(); + let initialized = block_on(persister.is_db_initialized()).unwrap(); assert!(!initialized); - block_on(persister.init_sql()).unwrap(); + block_on(persister.init_db()).unwrap(); // repetitive init must not fail - block_on(persister.init_sql()).unwrap(); + block_on(persister.init_db()).unwrap(); - let initialized = block_on(persister.is_sql_initialized()).unwrap(); + let initialized = block_on(persister.is_db_initialized()).unwrap(); assert!(initialized); } @@ -1629,12 +1629,12 @@ mod tests { Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), ); - block_on(persister.init_sql()).unwrap(); + block_on(persister.init_db()).unwrap(); let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id()).unwrap(); assert_eq!(last_channel_rpc_id, 0); - let channel = block_on(persister.get_channel_from_sql(1)).unwrap(); + let channel = block_on(persister.get_channel_from_db(1)).unwrap(); assert!(channel.is_none()); let mut expected_channel_details = SqlChannelDetails::new( @@ -1644,23 +1644,23 @@ mod tests { true, true, ); - block_on(persister.add_channel_to_sql(expected_channel_details.clone())).unwrap(); + block_on(persister.add_channel_to_db(expected_channel_details.clone())).unwrap(); let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id()).unwrap(); assert_eq!(last_channel_rpc_id, 1); - let actual_channel_details = block_on(persister.get_channel_from_sql(1)).unwrap().unwrap(); + let actual_channel_details = block_on(persister.get_channel_from_db(1)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); // must fail because we are adding channel with the same rpc_id - block_on(persister.add_channel_to_sql(expected_channel_details.clone())).unwrap_err(); + block_on(persister.add_channel_to_db(expected_channel_details.clone())).unwrap_err(); assert_eq!(last_channel_rpc_id, 1); expected_channel_details.rpc_id = 2; - block_on(persister.add_channel_to_sql(expected_channel_details.clone())).unwrap(); + block_on(persister.add_channel_to_db(expected_channel_details.clone())).unwrap(); let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id()).unwrap(); assert_eq!(last_channel_rpc_id, 2); - block_on(persister.add_funding_tx_to_sql( + block_on(persister.add_funding_tx_to_db( 2, "9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into(), 3000, @@ -1672,7 +1672,7 @@ mod tests { expected_channel_details.funding_value = Some(3000); expected_channel_details.funding_generated_in_block = Some(50000); - let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); + let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); block_on(persister.update_funding_tx_block_height( @@ -1682,14 +1682,14 @@ mod tests { .unwrap(); expected_channel_details.funding_generated_in_block = Some(50001); - let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); + let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); block_on(persister.update_channel_to_closed(2, "the channel was cooperatively closed".into())).unwrap(); expected_channel_details.closure_reason = Some("the channel was cooperatively closed".into()); expected_channel_details.is_closed = true; - let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); + let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); let actual_channels = block_on(persister.get_closed_channels_with_no_closing_tx()).unwrap(); @@ -1708,7 +1708,7 @@ mod tests { let actual_channels = block_on(persister.get_closed_channels_with_no_closing_tx()).unwrap(); assert_eq!(actual_channels.len(), 2); - block_on(persister.add_closing_tx_to_sql( + block_on(persister.add_closing_tx_to_db( 2, "5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into(), )) @@ -1719,10 +1719,10 @@ mod tests { let actual_channels = block_on(persister.get_closed_channels_with_no_closing_tx()).unwrap(); assert_eq!(actual_channels.len(), 1); - let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); + let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); - block_on(persister.add_claiming_tx_to_sql( + block_on(persister.add_claiming_tx_to_db( "5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into(), "97f061634a4a7b0b0c2b95648f86b1c39b95e0cf5073f07725b7143c095b612a".into(), 2000.333333, @@ -1732,7 +1732,7 @@ mod tests { Some("97f061634a4a7b0b0c2b95648f86b1c39b95e0cf5073f07725b7143c095b612a".into()); expected_channel_details.claimed_balance = Some(2000.333333); - let actual_channel_details = block_on(persister.get_channel_from_sql(2)).unwrap().unwrap(); + let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); assert_eq!(expected_channel_details, actual_channel_details); } @@ -1745,9 +1745,9 @@ mod tests { Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), ); - block_on(persister.init_sql()).unwrap(); + block_on(persister.init_db()).unwrap(); - let payment = block_on(persister.get_payment_from_sql(PaymentHash([0; 32]))).unwrap(); + let payment = block_on(persister.get_payment_from_db(PaymentHash([0; 32]))).unwrap(); assert!(payment.is_none()); let mut expected_payment_info = PaymentInfo { @@ -1762,9 +1762,9 @@ mod tests { created_at: now_ms() / 1000, last_updated: now_ms() / 1000, }; - block_on(persister.add_or_update_payment_in_sql(expected_payment_info.clone())).unwrap(); + block_on(persister.add_or_update_payment_in_db(expected_payment_info.clone())).unwrap(); - let actual_payment_info = block_on(persister.get_payment_from_sql(PaymentHash([0; 32]))) + let actual_payment_info = block_on(persister.get_payment_from_db(PaymentHash([0; 32]))) .unwrap() .unwrap(); assert_eq!(expected_payment_info, actual_payment_info); @@ -1779,9 +1779,9 @@ mod tests { expected_payment_info.amt_msat = None; expected_payment_info.status = HTLCStatus::Succeeded; expected_payment_info.last_updated = now_ms() / 1000; - block_on(persister.add_or_update_payment_in_sql(expected_payment_info.clone())).unwrap(); + block_on(persister.add_or_update_payment_in_db(expected_payment_info.clone())).unwrap(); - let actual_payment_info = block_on(persister.get_payment_from_sql(PaymentHash([1; 32]))) + let actual_payment_info = block_on(persister.get_payment_from_db(PaymentHash([1; 32]))) .unwrap() .unwrap(); assert_eq!(expected_payment_info, actual_payment_info); @@ -1796,12 +1796,12 @@ mod tests { Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), ); - block_on(persister.init_sql()).unwrap(); + block_on(persister.init_db()).unwrap(); let mut payments = generate_random_payments(100); for payment in payments.clone() { - block_on(persister.add_or_update_payment_in_sql(payment)).unwrap(); + block_on(persister.add_or_update_payment_in_db(payment)).unwrap(); } let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); @@ -1920,13 +1920,13 @@ mod tests { Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), ); - block_on(persister.init_sql()).unwrap(); + block_on(persister.init_db()).unwrap(); let mut channels = generate_random_channels(100); for channel in channels.clone() { - block_on(persister.add_channel_to_sql(channel.clone())).unwrap(); - block_on(persister.add_funding_tx_to_sql( + block_on(persister.add_channel_to_db(channel.clone())).unwrap(); + block_on(persister.add_funding_tx_to_db( channel.rpc_id, channel.funding_tx.unwrap(), channel.funding_value.unwrap(), @@ -1934,8 +1934,8 @@ mod tests { )) .unwrap(); block_on(persister.update_channel_to_closed(channel.rpc_id, channel.closure_reason.unwrap())).unwrap(); - block_on(persister.add_closing_tx_to_sql(channel.rpc_id, channel.closing_tx.clone().unwrap())).unwrap(); - block_on(persister.add_claiming_tx_to_sql( + block_on(persister.add_closing_tx_to_db(channel.rpc_id, channel.closing_tx.clone().unwrap())).unwrap(); + block_on(persister.add_claiming_tx_to_db( channel.closing_tx.unwrap(), channel.claiming_tx.unwrap(), channel.claimed_balance.unwrap(), diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 684b71cadb..96bbcadc64 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -188,24 +188,24 @@ pub struct GetPaymentsResult { } #[async_trait] -pub trait SqlStorage { +pub trait DbStorage { type Error; /// Initializes tables in DB. - async fn init_sql(&self) -> Result<(), Self::Error>; + async fn init_db(&self) -> Result<(), Self::Error>; /// Checks if tables have been initialized or not in DB. - async fn is_sql_initialized(&self) -> Result; + async fn is_db_initialized(&self) -> Result; /// Gets the last added channel rpc_id. Can be used to deduce the rpc_id for a new channel to be added to DB. async fn get_last_channel_rpc_id(&self) -> Result; - /// Inserts a new channel record in the DB. The record's data is completed using add_funding_tx_to_sql, - /// add_closing_tx_to_sql, add_claiming_tx_to_sql when this information is available. - async fn add_channel_to_sql(&self, details: SqlChannelDetails) -> Result<(), Self::Error>; + /// Inserts a new channel record in the DB. The record's data is completed using add_funding_tx_to_db, + /// add_closing_tx_to_db, add_claiming_tx_to_db when this information is available. + async fn add_channel_to_db(&self, details: SqlChannelDetails) -> Result<(), Self::Error>; /// Updates a channel's DB record with the channel's funding transaction information. - async fn add_funding_tx_to_sql( + async fn add_funding_tx_to_db( &self, rpc_id: u64, funding_tx: String, @@ -225,11 +225,11 @@ pub trait SqlStorage { async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error>; /// Updates a channel's DB record with the channel's closing transaction hash. - async fn add_closing_tx_to_sql(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error>; + async fn add_closing_tx_to_db(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error>; /// Updates a channel's DB record with information about the transaction responsible for claiming the channel's /// closing balance back to the user's address. - async fn add_claiming_tx_to_sql( + async fn add_claiming_tx_to_db( &self, closing_tx: String, claiming_tx: String, @@ -237,7 +237,7 @@ pub trait SqlStorage { ) -> Result<(), Self::Error>; /// Gets a channel record from DB by the channel's rpc_id. - async fn get_channel_from_sql(&self, rpc_id: u64) -> Result, Self::Error>; + async fn get_channel_from_db(&self, rpc_id: u64) -> Result, Self::Error>; /// Gets the list of closed channels that match the provided filter criteria. The number of requested records is /// specified by the limit parameter, the starting record to list from is specified by the paging parameter. The @@ -250,10 +250,10 @@ pub trait SqlStorage { ) -> Result; /// Inserts or updates a new payment record in the DB. - async fn add_or_update_payment_in_sql(&self, info: PaymentInfo) -> Result<(), Self::Error>; + async fn add_or_update_payment_in_db(&self, info: PaymentInfo) -> Result<(), Self::Error>; /// Gets a payment's record from DB by the payment's hash. - async fn get_payment_from_sql(&self, hash: PaymentHash) -> Result, Self::Error>; + async fn get_payment_from_db(&self, hash: PaymentHash) -> Result, Self::Error>; /// Gets the list of payments that match the provided filter criteria. The number of requested records is specified /// by the limit parameter, the starting record to list from is specified by the paging parameter. The total number From 152b2afafa5d8926aa7bd561e2ec433786b02600 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Mon, 11 Apr 2022 09:30:35 +0200 Subject: [PATCH 38/49] review fixes wip: some refactors --- mm2src/coins/lightning.rs | 2 +- mm2src/coins/lightning/ln_events.rs | 4 +- mm2src/coins/lightning/ln_p2p.rs | 76 ++++++++++++--------------- mm2src/coins/lightning/ln_platform.rs | 33 +++++------- 4 files changed, 51 insertions(+), 64 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index f41c18f336..38f74f7cf7 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -714,7 +714,7 @@ pub async fn connect_to_lightning_node(ctx: MmArc, req: ConnectToNodeRequest) -> // If a node that we have an open channel with changed it's address, "connect_to_lightning_node" // can be used to reconnect to the new address while saving this new address for reconnections. - if let ConnectToNodeRes::ConnectedSuccessfully(_, _) = res { + if let ConnectToNodeRes::ConnectedSuccessfully { .. } = res { if let Entry::Occupied(mut entry) = ln_coin.open_channels_nodes.lock().entry(node_pubkey) { entry.insert(node_addr); } diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 823251bd62..cb3a30daf7 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -427,8 +427,6 @@ impl LightningEventHandler { }, }; - self.platform.broadcast_transaction(&spending_tx); - let claiming_tx_inputs_value = outputs.iter().fold(0, |sum, output| match output { SpendableOutputDescriptor::StaticOutput { output, .. } => sum + output.value, SpendableOutputDescriptor::DelayedPaymentOutput(descriptor) => sum + descriptor.output.value, @@ -470,6 +468,8 @@ impl LightningEventHandler { .await .error_log(); }); + + self.platform.broadcast_transaction(&spending_tx); } } } diff --git a/mm2src/coins/lightning/ln_p2p.rs b/mm2src/coins/lightning/ln_p2p.rs index d614ac3457..b3c160da45 100644 --- a/mm2src/coins/lightning/ln_p2p.rs +++ b/mm2src/coins/lightning/ln_p2p.rs @@ -24,10 +24,10 @@ pub type PeerManager = #[derive(Display)] pub enum ConnectToNodeRes { - #[display(fmt = "Already connected to node: {}@{}", _0, _1)] - AlreadyConnected(String, String), - #[display(fmt = "Connected successfully to node : {}@{}", _0, _1)] - ConnectedSuccessfully(String, String), + #[display(fmt = "Already connected to node: {}@{}", pubkey, node_addr)] + AlreadyConnected { pubkey: PublicKey, node_addr: SocketAddr }, + #[display(fmt = "Connected successfully to node : {}@{}", pubkey, node_addr)] + ConnectedSuccessfully { pubkey: PublicKey, node_addr: SocketAddr }, } pub async fn connect_to_node( @@ -36,46 +36,40 @@ pub async fn connect_to_node( peer_manager: Arc, ) -> ConnectToNodeResult { if peer_manager.get_peer_node_ids().contains(&pubkey) { - return Ok(ConnectToNodeRes::AlreadyConnected( - pubkey.to_string(), - node_addr.to_string(), - )); + return Ok(ConnectToNodeRes::AlreadyConnected { pubkey, node_addr }); } - match lightning_net_tokio::connect_outbound(Arc::clone(&peer_manager), pubkey, node_addr).await { - Some(connection_closed_future) => { - let mut connection_closed_future = Box::pin(connection_closed_future); - loop { - // Make sure the connection is still established. - match futures::poll!(&mut connection_closed_future) { - std::task::Poll::Ready(_) => { - return MmError::err(ConnectToNodeError::ConnectionError(format!( - "Node {} disconnected before finishing the handshake", - pubkey - ))); - }, - std::task::Poll::Pending => {}, - } - - match peer_manager.get_peer_node_ids().contains(&pubkey) { - true => break, - // Wait for the handshake to complete if false. - false => Timer::sleep_ms(10).await, - } - } - }, - None => { - return MmError::err(ConnectToNodeError::ConnectionError(format!( - "Failed to connect to node: {}", - pubkey - ))) - }, + let mut connection_closed_future = + match lightning_net_tokio::connect_outbound(Arc::clone(&peer_manager), pubkey, node_addr).await { + Some(fut) => Box::pin(fut), + None => { + return MmError::err(ConnectToNodeError::ConnectionError(format!( + "Failed to connect to node: {}", + pubkey + ))) + }, + }; + + loop { + // Make sure the connection is still established. + match futures::poll!(&mut connection_closed_future) { + std::task::Poll::Ready(_) => { + return MmError::err(ConnectToNodeError::ConnectionError(format!( + "Node {} disconnected before finishing the handshake", + pubkey + ))); + }, + std::task::Poll::Pending => {}, + } + + match peer_manager.get_peer_node_ids().contains(&pubkey) { + true => break, + // Wait for the handshake to complete if false. + false => Timer::sleep_ms(10).await, + } } - Ok(ConnectToNodeRes::ConnectedSuccessfully( - pubkey.to_string(), - node_addr.to_string(), - )) + Ok(ConnectToNodeRes::ConnectedSuccessfully { pubkey, node_addr }) } pub async fn connect_to_nodes_loop(open_channels_nodes: NodesAddressesMapShared, peer_manager: Arc) { @@ -85,7 +79,7 @@ pub async fn connect_to_nodes_loop(open_channels_nodes: NodesAddressesMapShared, let peer_manager = peer_manager.clone(); match connect_to_node(pubkey, node_addr, peer_manager.clone()).await { Ok(res) => { - if let ConnectToNodeRes::ConnectedSuccessfully(_, _) = res { + if let ConnectToNodeRes::ConnectedSuccessfully { .. } = res { log::info!("{}", res.to_string()); } }, diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index dcc54dc6ea..e7481b8326 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -253,18 +253,12 @@ impl Platform { pub fn best_block_height(&self) -> u64 { self.best_block_height.load(AtomicOrdering::Relaxed) } - pub fn add_tx(&self, txid: &Txid, script_pubkey: &Script) { + pub fn add_tx(&self, txid: Txid, script_pubkey: Script) { let mut registered_txs = self.registered_txs.lock(); - match registered_txs.get_mut(txid) { - Some(h) => { - h.insert(script_pubkey.clone()); - }, - None => { - let mut script_pubkeys = HashSet::new(); - script_pubkeys.insert(script_pubkey.clone()); - registered_txs.insert(*txid, script_pubkeys); - }, - } + registered_txs + .entry(txid) + .or_insert_with(HashSet::new) + .insert(script_pubkey); } pub fn add_output(&self, output: WatchedOutput) { @@ -301,14 +295,13 @@ impl Platform { T: Confirm, { match self.check_if_tx_is_onchain(txid).await { - Ok(tx_is_onchain) => { - if !tx_is_onchain { - log::info!( - "Transaction {} is not found on chain. The transaction will be re-broadcasted.", - txid, - ); - monitor.transaction_unconfirmed(&txid); - } + Ok(true) => {}, + Ok(false) => { + log::info!( + "Transaction {} is not found on chain. The transaction will be re-broadcasted.", + txid, + ); + monitor.transaction_unconfirmed(&txid); }, Err(e) => log::error!( "Error while trying to check if the transaction {} is discarded or not :{}", @@ -641,7 +634,7 @@ impl BroadcasterInterface for Platform { impl Filter for Platform { // Watches for this transaction on-chain - fn register_tx(&self, txid: &Txid, script_pubkey: &Script) { self.add_tx(txid, script_pubkey); } + fn register_tx(&self, txid: &Txid, script_pubkey: &Script) { self.add_tx(*txid, script_pubkey.clone()); } // Watches for any transactions that spend this output on-chain fn register_output(&self, output: WatchedOutput) -> Option<(usize, Transaction)> { From c5b4c90372fe83c2530eb8e37def019c3004a204 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Mon, 11 Apr 2022 15:02:05 +0200 Subject: [PATCH 39/49] review fixes wip: default_fee_per_kb, refactors --- mm2src/coins/lightning/ln_conf.rs | 2 +- mm2src/coins/lightning/ln_platform.rs | 24 +++++------------------- mm2src/mm2_tests/lightning_tests.rs | 12 ++++++------ 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/mm2src/coins/lightning/ln_conf.rs b/mm2src/coins/lightning/ln_conf.rs index 4db186fe2e..3773631e9c 100644 --- a/mm2src/coins/lightning/ln_conf.rs +++ b/mm2src/coins/lightning/ln_conf.rs @@ -3,7 +3,7 @@ use lightning::util::config::{ChannelConfig, ChannelHandshakeConfig, ChannelHand #[derive(Clone, Debug, Deserialize, Serialize)] pub struct DefaultFeesAndConfirmations { - pub default_feerate: u64, + pub default_fee_per_kb: u64, pub n_blocks: u32, } diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index e7481b8326..76fa42ddc6 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -486,13 +486,7 @@ impl Platform { self.append_spent_registered_output_txs(&mut transactions_to_confirm, &client) .await; - transactions_to_confirm.sort_by(|a, b| { - let block_order = a.height.cmp(&b.height); - match block_order { - cmp::Ordering::Equal => a.index.cmp(&b.index), - _ => block_order, - } - }); + transactions_to_confirm.sort_by(|a, b| (a.height, a.index).cmp(&(b.height, b.index))); for confirmed_transaction_info in transactions_to_confirm { let best_block_height = self.best_block_height(); @@ -577,10 +571,10 @@ impl FeeEstimator for Platform { let platform_coin = &self.coin; let default_fee = match confirmation_target { - ConfirmationTarget::Background => self.default_fees_and_confirmations.background.default_feerate, - ConfirmationTarget::Normal => self.default_fees_and_confirmations.normal.default_feerate, - ConfirmationTarget::HighPriority => self.default_fees_and_confirmations.high_priority.default_feerate, - } * 4; + ConfirmationTarget::Background => self.default_fees_and_confirmations.background.default_fee_per_kb, + ConfirmationTarget::Normal => self.default_fees_and_confirmations.normal.default_fee_per_kb, + ConfirmationTarget::HighPriority => self.default_fees_and_confirmations.high_priority.default_fee_per_kb, + }; let conf = &platform_coin.as_ref().conf; let n_blocks = match confirmation_target { @@ -612,14 +606,6 @@ impl FeeEstimator for Platform { impl BroadcasterInterface for Platform { fn broadcast_transaction(&self, tx: &Transaction) { let txid = tx.txid(); - let fut = Box::new(async move { self.check_if_tx_is_onchain(txid).await }.boxed().compat()); - let is_tx_onchain = tokio::task::block_in_place(move || fut.wait()); - - if let Ok(true) = is_tx_onchain { - log::debug!("Transaction {} is found onchain!", txid); - return; - } - let tx_hex = serialize_hex(tx); log::debug!("Trying to broadcast transaction: {}", tx_hex); let fut = self.coin.send_raw_tx(&tx_hex); diff --git a/mm2src/mm2_tests/lightning_tests.rs b/mm2src/mm2_tests/lightning_tests.rs index a63b002845..ffa9a68914 100644 --- a/mm2src/mm2_tests/lightning_tests.rs +++ b/mm2src/mm2_tests/lightning_tests.rs @@ -49,15 +49,15 @@ fn start_lightning_nodes() -> (MarketMakerIt, MarketMakerIt, String, String) { "network": "testnet", "confirmations": { "background": { - "default_feerate": 253, + "default_fee_per_kb": 1012, "n_blocks": 12 }, "normal": { - "default_feerate": 2000, + "default_fee_per_kb": 8000, "n_blocks": 6 }, "high_priority": { - "default_feerate": 5000, + "default_fee_per_kb": 20000, "n_blocks": 1 } } @@ -150,15 +150,15 @@ fn test_enable_lightning() { "network": "testnet", "confirmations": { "background": { - "default_feerate": 253, + "default_fee_per_kb": 1012, "n_blocks": 12 }, "normal": { - "default_feerate": 2000, + "default_fee_per_kb": 8000, "n_blocks": 6 }, "high_priority": { - "default_feerate": 5000, + "default_fee_per_kb": 20000, "n_blocks": 1 } } From 55d2ab69d7fe4e2b6de290cc6938eb6cabf8c520 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Mon, 11 Apr 2022 15:56:03 +0200 Subject: [PATCH 40/49] review fixes: use try_loop_with_sleep macro wip --- mm2src/coins/lightning/ln_platform.rs | 197 ++++++++++++-------------- mm2src/coins/lp_coins.rs | 13 ++ mm2src/coins/utxo/utxo_common.rs | 13 -- 3 files changed, 102 insertions(+), 121 deletions(-) diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index 76fa42ddc6..de8e0cb84b 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -13,7 +13,7 @@ use bitcoin::hash_types::{BlockHash, TxMerkleNode, Txid}; use bitcoin_hashes::{sha256d, Hash}; use common::executor::{spawn, Timer}; use common::jsonrpc_client::{JsonRpcError, JsonRpcErrorType}; -use common::log; +use common::log::{debug, error, info}; use derive_more::Display; use futures::compat::Future01CompatExt; use keys::hash::H256; @@ -27,6 +27,7 @@ use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; const CHECK_FOR_NEW_BEST_BLOCK_INTERVAL: u64 = 60; const MIN_ALLOWED_FEE_PER_1000_WEIGHT: u32 = 253; const GET_FUNDING_TX_INTERVAL: f64 = 60.; +const TRY_LOOP_INTERVAL: f64 = 60.; struct TxWithBlockInfo { tx: Transaction, @@ -114,14 +115,14 @@ pub async fn update_best_block( let prev_blockhash = match sha256d::Hash::from_slice(&h.prev_block_hash.0) { Ok(h) => h, Err(e) => { - log::error!("Error while parsing previous block hash for lightning node: {}", e); + error!("Error while parsing previous block hash for lightning node: {}", e); return; }, }; let merkle_root = match sha256d::Hash::from_slice(&h.merkle_root.0) { Ok(h) => h, Err(e) => { - log::error!("Error while parsing merkle root for lightning node: {}", e); + error!("Error while parsing merkle root for lightning node: {}", e); return; }, }; @@ -141,7 +142,7 @@ pub async fn update_best_block( let block_header = match deserialize(&h.hex.into_vec()) { Ok(header) => header, Err(e) => { - log::error!("Block header deserialization error: {}", e.to_string()); + error!("Block header deserialization error: {}", e.to_string()); return; }, }; @@ -166,7 +167,7 @@ pub async fn ln_best_block_update_loop( let best_header = match get_best_header(&best_header_listener).await { Ok(h) => h, Err(e) => { - log::error!("Error while requesting best header for lightning node: {}", e); + error!("Error while requesting best header for lightning node: {}", e); Timer::sleep(CHECK_FOR_NEW_BEST_BLOCK_INTERVAL as f64).await; continue; }, @@ -297,16 +298,15 @@ impl Platform { match self.check_if_tx_is_onchain(txid).await { Ok(true) => {}, Ok(false) => { - log::info!( + info!( "Transaction {} is not found on chain. The transaction will be re-broadcasted.", txid, ); monitor.transaction_unconfirmed(&txid); }, - Err(e) => log::error!( + Err(e) => error!( "Error while trying to check if the transaction {} is discarded or not :{}", - txid, - e + txid, e ), } } @@ -338,88 +338,73 @@ impl Platform { let mut confirmed_registered_txs = Vec::new(); for (txid, scripts) in registered_txs { let rpc_txid = H256Json::from(txid.as_hash().into_inner()).reversed(); - match self - .coin - .as_ref() - .rpc_client - .get_transaction_bytes(&rpc_txid) - .compat() - .await - { - Ok(bytes) => { - let transaction: Transaction = match deserialize(&bytes.into_vec()) { - Ok(tx) => tx, - Err(e) => { - log::error!("Transaction deserialization error: {}", e.to_string()); - continue; - }, - }; - for (_, vout) in transaction.output.iter().enumerate() { - if scripts.contains(&vout.script_pubkey) { - let script_hash = hex::encode(electrum_script_hash(vout.script_pubkey.as_ref())); - let history = client - .scripthash_get_history(&script_hash) - .compat() - .await - .unwrap_or_default(); - for item in history { - if item.tx_hash == rpc_txid { - // If a new block mined the transaction while running process_txs_confirmations it will be confirmed later in ln_best_block_update_loop - if item.height > 0 && item.height <= current_height as i64 { - let height: u64 = match item.height.try_into() { - Ok(h) => h, - Err(e) => { - log::error!("Block height convertion to u64 error: {}", e.to_string()); - continue; - }, - }; - let header = match client.blockchain_block_header(height).compat().await { - Ok(block_header) => match deserialize(&block_header) { - Ok(h) => h, - Err(e) => { - log::error!( - "Block header deserialization error: {}", - e.to_string() - ); - continue; - }, - }, - Err(_) => continue, - }; - let index = match client - .blockchain_transaction_get_merkle(rpc_txid, height) - .compat() - .await - { - Ok(merkle_branch) => merkle_branch.pos, - Err(e) => { - log::error!( - "Error getting transaction position in the block: {}", - e.to_string() - ); - continue; - }, - }; - let confirmed_transaction_info = ConfirmedTransactionInfo::new( - txid, - header, - index, - transaction.clone(), - height as u32, - ); - confirmed_registered_txs.push(confirmed_transaction_info); - self.registered_txs.lock().remove(&txid); - } - } - } - } - } - }, + let bytes = try_loop_with_sleep!( + self.coin + .as_ref() + .rpc_client + .get_transaction_bytes(&rpc_txid) + .compat() + .await, + TRY_LOOP_INTERVAL + ); + let transaction: Transaction = match deserialize(&bytes.into_vec()) { + Ok(tx) => tx, Err(e) => { - log::error!("Error getting transaction {} from chain: {}", txid, e); + error!("Transaction deserialization error: {}", e.to_string()); continue; }, }; + for (_, vout) in transaction.output.iter().enumerate() { + if scripts.contains(&vout.script_pubkey) { + let script_hash = hex::encode(electrum_script_hash(vout.script_pubkey.as_ref())); + let history = try_loop_with_sleep!( + client.scripthash_get_history(&script_hash).compat().await, + TRY_LOOP_INTERVAL + ); + for item in history { + if item.tx_hash == rpc_txid { + // If a new block mined the transaction while running process_txs_confirmations it will be confirmed later in ln_best_block_update_loop + if item.height > 0 && item.height <= current_height as i64 { + let height: u64 = match item.height.try_into() { + Ok(h) => h, + Err(e) => { + error!("Block height convertion to u64 error: {}", e.to_string()); + continue; + }, + }; + let block_header = try_loop_with_sleep!( + client.blockchain_block_header(height).compat().await, + TRY_LOOP_INTERVAL + ); + let header = match deserialize(&block_header) { + Ok(h) => h, + Err(e) => { + error!("Block header deserialization error: {}", e.to_string()); + continue; + }, + }; + let index = try_loop_with_sleep!( + client + .blockchain_transaction_get_merkle(rpc_txid, height) + .compat() + .await, + TRY_LOOP_INTERVAL + ) + .pos; + let confirmed_transaction_info = ConfirmedTransactionInfo::new( + txid, + header, + index, + transaction.clone(), + height as u32, + ); + confirmed_registered_txs.push(confirmed_transaction_info); + self.registered_txs.lock().remove(&txid); + } + } + } + } + } } confirmed_registered_txs } @@ -439,17 +424,14 @@ impl Platform { .any(|info| info.txid == tx_info.tx.txid()) { let rpc_txid = H256Json::from(tx_info.tx.txid().as_hash().into_inner()).reversed(); - let index = match client - .blockchain_transaction_get_merkle(rpc_txid, tx_info.block_height) - .compat() - .await - { - Ok(merkle_branch) => merkle_branch.pos, - Err(e) => { - log::error!("Error getting transaction position in the block: {}", e.to_string()); - continue; - }, - }; + let index = try_loop_with_sleep!( + client + .blockchain_transaction_get_merkle(rpc_txid, tx_info.block_height) + .compat() + .await, + TRY_LOOP_INTERVAL + ) + .pos; let confirmed_transaction_info = ConfirmedTransactionInfo::new( tx_info.tx.txid(), tx_info.block_header, @@ -462,10 +444,9 @@ impl Platform { outputs_to_remove.push(output); }, Ok(None) => (), - Err(e) => log::error!( + Err(e) => error!( "Error while trying to find if the registered output {:?} is spent: {}", - output.outpoint, - e + output.outpoint, e ), } } @@ -497,7 +478,7 @@ impl Platform { ) .await { - log::error!("Unable to update the funding tx block height in DB: {}", e); + error!("Unable to update the funding tx block height in DB: {}", e); } channel_manager.transactions_confirmed( &confirmed_transaction_info.header, @@ -541,7 +522,7 @@ impl Platform { { Ok(tx) => break tx, Err(e) => { - log::error!("Error getting funding tx bytes: {}", e); + error!("Error getting funding tx bytes: {}", e); Timer::sleep(GET_FUNDING_TX_INTERVAL).await; }, } @@ -607,12 +588,12 @@ impl BroadcasterInterface for Platform { fn broadcast_transaction(&self, tx: &Transaction) { let txid = tx.txid(); let tx_hex = serialize_hex(tx); - log::debug!("Trying to broadcast transaction: {}", tx_hex); + debug!("Trying to broadcast transaction: {}", tx_hex); let fut = self.coin.send_raw_tx(&tx_hex); spawn(async move { match fut.compat().await { - Ok(id) => log::info!("Transaction broadcasted successfully: {:?} ", id), - Err(e) => log::error!("Broadcast transaction {} failed: {}", txid, e), + Ok(id) => info!("Transaction broadcasted successfully: {:?} ", id), + Err(e) => error!("Broadcast transaction {} failed: {}", txid, e), } }); } @@ -652,7 +633,7 @@ impl Filter for Platform { let spending_tx = match Transaction::try_from(spent_output_info.spending_tx) { Ok(tx) => tx, Err(e) => { - log::error!("Can't convert transaction error: {}", e.to_string()); + error!("Can't convert transaction error: {}", e.to_string()); return None; }, }; @@ -660,7 +641,7 @@ impl Filter for Platform { }, Ok(None) => None, Err(e) => { - log::error!("Error when calling register_output: {}", e); + error!("Error when calling register_output: {}", e); None }, } diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 0d05a4045d..cd25694932 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -97,6 +97,19 @@ macro_rules! try_f { }; } +macro_rules! try_loop_with_sleep { + ($e:expr, $delay: ident) => { + match $e { + Ok(res) => res, + Err(e) => { + error!("error {:?}", e); + Timer::sleep($delay).await; + continue; + }, + } + }; +} + pub mod coin_balance; #[doc(hidden)] #[cfg(test)] diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index d07c699a97..c1e4e6ada4 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3438,19 +3438,6 @@ where } } -macro_rules! try_loop_with_sleep { - ($e:expr, $delay: ident) => { - match $e { - Ok(res) => res, - Err(e) => { - error!("error {:?}", e); - Timer::sleep($delay).await; - continue; - }, - } - }; -} - pub async fn block_header_utxo_loop(weak: UtxoWeak, constructor: impl Fn(UtxoArc) -> T) where T: AsRef + UtxoCommonOps, From 7026db78297b996b38d73b303fa9744efa076359 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Wed, 13 Apr 2022 17:21:34 +0200 Subject: [PATCH 41/49] review fixes --- mm2src/coins/lightning/ln_errors.rs | 49 +++++ mm2src/coins/lightning/ln_events.rs | 77 +++---- mm2src/coins/lightning/ln_platform.rs | 276 ++++++++++---------------- mm2src/coins/lightning/ln_utils.rs | 1 - mm2src/coins/lp_coins.rs | 32 ++- mm2src/coins/utxo/utxo_common.rs | 9 +- 6 files changed, 230 insertions(+), 214 deletions(-) diff --git a/mm2src/coins/lightning/ln_errors.rs b/mm2src/coins/lightning/ln_errors.rs index d73c09b414..ff333f729c 100644 --- a/mm2src/coins/lightning/ln_errors.rs +++ b/mm2src/coins/lightning/ln_errors.rs @@ -1,6 +1,8 @@ use crate::utxo::rpc_clients::UtxoRpcError; use crate::utxo::GenerateTxError; use crate::{BalanceError, CoinFindError, NumConversError, PrivKeyNotAllowed, UnexpectedDerivationMethod}; +use bitcoin::consensus::encode; +use common::jsonrpc_client::JsonRpcError; use common::mm_error::prelude::*; use common::HttpStatusCode; use db_common::sqlite::rusqlite::Error as SqlError; @@ -510,3 +512,50 @@ pub enum SaveChannelClosingError { impl From for SaveChannelClosingError { fn from(err: SqlError) -> SaveChannelClosingError { SaveChannelClosingError::DbError(err.to_string()) } } + +#[derive(Debug)] +#[allow(clippy::large_enum_variant)] +pub enum GetTxError { + Rpc(UtxoRpcError), + TxDeserialization(encode::Error), +} + +impl From for GetTxError { + fn from(err: UtxoRpcError) -> GetTxError { GetTxError::Rpc(err) } +} + +impl From for GetTxError { + fn from(err: encode::Error) -> GetTxError { GetTxError::TxDeserialization(err) } +} + +#[derive(Debug)] +#[allow(clippy::large_enum_variant)] +pub enum GetHeaderError { + Rpc(JsonRpcError), + HeaderDeserialization(encode::Error), +} + +impl From for GetHeaderError { + fn from(err: JsonRpcError) -> GetHeaderError { GetHeaderError::Rpc(err) } +} + +impl From for GetHeaderError { + fn from(err: encode::Error) -> GetHeaderError { GetHeaderError::HeaderDeserialization(err) } +} + +#[derive(Debug)] +#[allow(clippy::large_enum_variant)] +pub enum FindWatchedOutputSpendError { + HashNotHeight, + DeserializationErr(encode::Error), + RpcError(String), + GetHeaderError(GetHeaderError), +} + +impl From for FindWatchedOutputSpendError { + fn from(err: JsonRpcError) -> Self { FindWatchedOutputSpendError::RpcError(err.to_string()) } +} + +impl From for FindWatchedOutputSpendError { + fn from(err: encode::Error) -> Self { FindWatchedOutputSpendError::DeserializationErr(err) } +} diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index cb3a30daf7..b82874c741 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -3,7 +3,8 @@ use crate::lightning::ln_errors::{SaveChannelClosingError, SaveChannelClosingRes use bitcoin::blockdata::script::Script; use bitcoin::blockdata::transaction::Transaction; use common::executor::{spawn, Timer}; -use common::{log, now_ms}; +use common::log::{error, info}; +use common::now_ms; use core::time::Duration; use lightning::chain::chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator}; use lightning::chain::keysinterface::SpendableOutputDescriptor; @@ -15,6 +16,8 @@ use std::convert::TryFrom; use std::sync::Arc; use utxo_signer::with_key_pair::sign_tx; +const TRY_LOOP_INTERVAL: f64 = 60.; + pub struct LightningEventHandler { platform: Arc, channel_manager: Arc, @@ -57,7 +60,7 @@ impl EventHandler for LightningEventHandler { Event::SpendableOutputs { outputs } => self.handle_spendable_outputs(outputs), // Todo: an RPC for total amount earned - Event::PaymentForwarded { fee_earned_msat, claim_from_onchain_tx } => log::info!( + Event::PaymentForwarded { fee_earned_msat, claim_from_onchain_tx } => info!( "Received a fee of {} milli-satoshis for a successfully forwarded payment through our {} lightning node. Was the forwarded HTLC claimed by our counterparty via an on-chain transaction?: {}", fee_earned_msat.unwrap_or_default(), self.platform.coin.ticker(), @@ -71,7 +74,7 @@ impl EventHandler for LightningEventHandler { } => self.handle_channel_closed(*channel_id, *user_channel_id, reason.to_string()), // Todo: Add spent UTXOs to RecentlySpentOutPoints if it's not discarded - Event::DiscardFunding { channel_id, transaction } => log::info!( + Event::DiscardFunding { channel_id, transaction } => info!( "Discarding funding tx: {} for channel {}", transaction.txid().to_string(), hex::encode(channel_id), @@ -82,7 +85,7 @@ impl EventHandler for LightningEventHandler { payment_id, payment_hash, path, - } => log::info!( + } => info!( "Payment path: {:?}, successful for payment hash: {}, payment id: {}", path.iter().map(|hop| hop.pubkey.to_string()).collect::>(), payment_hash.map(|h| hex::encode(h.0)).unwrap_or_default(), @@ -97,7 +100,7 @@ impl EventHandler for LightningEventHandler { all_paths_failed, path, .. - } => log::info!( + } => info!( "Payment path: {:?}, failed for payment hash: {}, Was rejected by destination?: {}, All paths failed?: {}", path.iter().map(|hop| hop.pubkey.to_string()).collect::>(), hex::encode(payment_hash.0), @@ -111,7 +114,7 @@ impl EventHandler for LightningEventHandler { funding_satoshis, push_msat, } => { - log::info!( + info!( "Handling OpenChannelRequest from node: {} with funding value: {} and starting balance: {}", counterparty_node_id, funding_satoshis, @@ -204,14 +207,14 @@ impl LightningEventHandler { output_script: &Script, user_channel_id: u64, ) { - log::info!( + info!( "Handling FundingGenerationReady event for internal channel id: {}", user_channel_id ); let funding_tx = match sign_funding_transaction(user_channel_id, output_script, self.platform.clone()) { Ok(tx) => tx, Err(e) => { - log::error!( + error!( "Error generating funding transaction for internal channel id {}: {}", user_channel_id, e.to_string() @@ -225,7 +228,7 @@ impl LightningEventHandler { .channel_manager .funding_transaction_generated(&temporary_channel_id, funding_tx) { - log::error!("{:?}", e); + error!("{:?}", e); return; } let platform = self.platform.clone(); @@ -245,7 +248,7 @@ impl LightningEventHandler { } fn handle_payment_received(&self, payment_hash: PaymentHash, amt: u64, purpose: &PaymentPurpose) { - log::info!( + info!( "Handling PaymentReceived event for payment_hash: {}", hex::encode(payment_hash.0) ); @@ -261,7 +264,7 @@ impl LightningEventHandler { }; let status = match self.channel_manager.claim_funds(payment_preimage) { true => { - log::info!( + info!( "Received an amount of {} millisatoshis for payment hash {}", amt, hex::encode(payment_hash.0) @@ -283,7 +286,7 @@ impl LightningEventHandler { payment_info.amt_msat = Some(amt); payment_info.last_updated = now_ms() / 1000; if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { - log::error!("Unable to update payment information in DB: {}", e); + error!("Unable to update payment information in DB: {}", e); } } }), @@ -302,7 +305,7 @@ impl LightningEventHandler { }; spawn(async move { if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { - log::error!("Unable to update payment information in DB: {}", e); + error!("Unable to update payment information in DB: {}", e); } }); }, @@ -315,7 +318,7 @@ impl LightningEventHandler { payment_hash: PaymentHash, fee_paid_msat: Option, ) { - log::info!( + info!( "Handling PaymentSent event for payment_hash: {}", hex::encode(payment_hash.0) ); @@ -332,9 +335,9 @@ impl LightningEventHandler { payment_info.last_updated = now_ms() / 1000; let amt_msat = payment_info.amt_msat; if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { - log::error!("Unable to update payment information in DB: {}", e); + error!("Unable to update payment information in DB: {}", e); } - log::info!( + info!( "Successfully sent payment of {} millisatoshis with payment hash {}", amt_msat.unwrap_or_default(), hex::encode(payment_hash.0) @@ -344,7 +347,7 @@ impl LightningEventHandler { } fn handle_channel_closed(&self, channel_id: [u8; 32], user_channel_id: u64, reason: String) { - log::info!( + info!( "Channel: {} closed for the following reason: {}", hex::encode(channel_id), reason @@ -357,10 +360,9 @@ impl LightningEventHandler { if user_channel_id != 0 { spawn(async move { if let Err(e) = save_channel_closing_details(persister, platform, user_channel_id, reason).await { - log::error!( + error!( "Unable to update channel {} closing details in DB: {}", - user_channel_id, - e + user_channel_id, e ); } }); @@ -368,7 +370,7 @@ impl LightningEventHandler { } fn handle_payment_failed(&self, payment_hash: PaymentHash) { - log::info!( + info!( "Handling PaymentFailed event for payment_hash: {}", hex::encode(payment_hash.0) ); @@ -382,14 +384,14 @@ impl LightningEventHandler { payment_info.status = HTLCStatus::Failed; payment_info.last_updated = now_ms() / 1000; if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { - log::error!("Unable to update payment information in DB: {}", e); + error!("Unable to update payment information in DB: {}", e); } } }); } fn handle_pending_htlcs_forwards(&self, time_forwardable: Duration) { - log::info!("Handling PendingHTLCsForwardable event!"); + info!("Handling PendingHTLCsForwardable event!"); let min_wait_time = time_forwardable.as_millis() as u32; let channel_manager = self.channel_manager.clone(); spawn(async move { @@ -400,13 +402,13 @@ impl LightningEventHandler { } fn handle_spendable_outputs(&self, outputs: &[SpendableOutputDescriptor]) { - log::info!("Handling SpendableOutputs event!"); + info!("Handling SpendableOutputs event!"); let platform_coin = &self.platform.coin; // Todo: add support for Hardware wallets for funding transactions and spending spendable outputs (channel closing transactions) let my_address = match platform_coin.as_ref().derivation_method.iguana_or_err() { Ok(addr) => addr, Err(e) => { - log::error!("{}", e); + error!("{}", e); return; }, }; @@ -422,7 +424,7 @@ impl LightningEventHandler { ) { Ok(tx) => tx, Err(_) => { - log::error!("Error spending spendable outputs"); + error!("Error spending spendable outputs"); return; }, }; @@ -434,10 +436,9 @@ impl LightningEventHandler { }); let claiming_tx_outputs_value = spending_tx.output.iter().fold(0, |sum, txout| sum + txout.value); if claiming_tx_inputs_value < claiming_tx_outputs_value { - log::error!( + error!( "Claiming transaction input value {} can't be less than outputs value {}!", - claiming_tx_inputs_value, - claiming_tx_outputs_value + claiming_tx_inputs_value, claiming_tx_outputs_value ); return; } @@ -459,14 +460,16 @@ impl LightningEventHandler { let claiming_txid = spending_tx.txid().to_string(); let persister = self.persister.clone(); spawn(async move { - persister - .add_claiming_tx_to_db( - closing_txid, - claiming_txid, - (claimed_balance as f64) - claiming_tx_fee_per_channel, - ) - .await - .error_log(); + ok_or_retry_after_sleep!( + persister + .add_claiming_tx_to_db( + closing_txid.clone(), + claiming_txid.clone(), + (claimed_balance as f64) - claiming_tx_fee_per_channel, + ) + .await, + TRY_LOOP_INTERVAL + ); }); self.platform.broadcast_transaction(&spending_tx); diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index de8e0cb84b..4a4c938b24 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -1,5 +1,6 @@ use super::*; -use crate::lightning::ln_errors::{SaveChannelClosingError, SaveChannelClosingResult}; +use crate::lightning::ln_errors::{FindWatchedOutputSpendError, GetHeaderError, GetTxError, SaveChannelClosingError, + SaveChannelClosingResult}; use crate::utxo::rpc_clients::{electrum_script_hash, BestBlock as RpcBestBlock, BlockHashOrHeight, ElectrumBlockHeader, ElectrumClient, ElectrumNonce, EstimateFeeMethod, UtxoRpcClientEnum, UtxoRpcError}; @@ -8,25 +9,23 @@ use crate::{MarketCoinOps, MmCoin}; use bitcoin::blockdata::block::BlockHeader; use bitcoin::blockdata::script::Script; use bitcoin::blockdata::transaction::Transaction; -use bitcoin::consensus::encode::{deserialize, serialize_hex, Error as EncodeError}; +use bitcoin::consensus::encode::{deserialize, serialize_hex}; use bitcoin::hash_types::{BlockHash, TxMerkleNode, Txid}; use bitcoin_hashes::{sha256d, Hash}; use common::executor::{spawn, Timer}; -use common::jsonrpc_client::{JsonRpcError, JsonRpcErrorType}; +use common::jsonrpc_client::JsonRpcErrorType; use common::log::{debug, error, info}; -use derive_more::Display; use futures::compat::Future01CompatExt; use keys::hash::H256; use lightning::chain::{chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator}, Confirm, Filter, WatchedOutput}; use rpc::v1::types::H256 as H256Json; use std::cmp; -use std::convert::{TryFrom, TryInto}; +use std::convert::TryFrom; use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; -const CHECK_FOR_NEW_BEST_BLOCK_INTERVAL: u64 = 60; +const CHECK_FOR_NEW_BEST_BLOCK_INTERVAL: f64 = 60.; const MIN_ALLOWED_FEE_PER_1000_WEIGHT: u32 = 253; -const GET_FUNDING_TX_INTERVAL: f64 = 60.; const TRY_LOOP_INTERVAL: f64 = 60.; struct TxWithBlockInfo { @@ -35,22 +34,10 @@ struct TxWithBlockInfo { block_height: u64, } -#[derive(Display)] -pub enum FindWatchedOutputSpendError { - #[display(fmt = "Spent output info has blockhash not height")] - HashNotHeight, - #[display(fmt = "Deserialization error: {}", _0)] - DeserializationErr(String), - #[display(fmt = "RPC error {}", _0)] - RpcError(String), -} - -impl From for FindWatchedOutputSpendError { - fn from(e: JsonRpcError) -> Self { FindWatchedOutputSpendError::RpcError(e.to_string()) } -} - -impl From for FindWatchedOutputSpendError { - fn from(e: EncodeError) -> Self { FindWatchedOutputSpendError::DeserializationErr(e.to_string()) } +async fn get_block_header(electrum_client: &ElectrumClient, height: u64) -> Result { + Ok(deserialize( + &electrum_client.blockchain_block_header(height).compat().await?, + )?) } async fn find_watched_output_spend_with_header( @@ -79,8 +66,9 @@ async fn find_watched_output_spend_with_header( BlockHashOrHeight::Height(h) => h, _ => return Err(FindWatchedOutputSpendError::HashNotHeight), }; - let header = electrum_client.blockchain_block_header(height as u64).compat().await?; - let block_header = deserialize(&header)?; + let block_header = get_block_header(electrum_client, height as u64) + .await + .map_err(FindWatchedOutputSpendError::GetHeaderError)?; let spending_tx = Transaction::try_from(output_spend.spending_tx)?; Ok(Some(TxWithBlockInfo { @@ -164,14 +152,7 @@ pub async fn ln_best_block_update_loop( ) { let mut current_best_block = best_block; loop { - let best_header = match get_best_header(&best_header_listener).await { - Ok(h) => h, - Err(e) => { - error!("Error while requesting best header for lightning node: {}", e); - Timer::sleep(CHECK_FOR_NEW_BEST_BLOCK_INTERVAL as f64).await; - continue; - }, - }; + let best_header = ok_or_continue_after_sleep!(get_best_header(&best_header_listener).await, TRY_LOOP_INTERVAL); if current_best_block != best_header.clone().into() { platform.update_best_block_height(best_header.block_height()); platform @@ -183,13 +164,12 @@ pub async fn ln_best_block_update_loop( persister.clone(), chain_monitor.clone(), channel_manager.clone(), - best_header.block_height(), ) .await; current_best_block = best_header.clone().into(); update_best_block(chain_monitor.clone(), channel_manager.clone(), best_header).await; } - Timer::sleep(CHECK_FOR_NEW_BEST_BLOCK_INTERVAL as f64).await; + Timer::sleep(CHECK_FOR_NEW_BEST_BLOCK_INTERVAL).await; } } @@ -267,8 +247,8 @@ impl Platform { registered_outputs.push(output); } - async fn check_if_tx_is_onchain(&self, txid: Txid) -> Result> { - if let Err(err) = self + async fn get_tx_if_onchain(&self, txid: Txid) -> Result, GetTxError> { + match self .coin .as_ref() .rpc_client @@ -277,27 +257,29 @@ impl Platform { .await .map_err(|e| e.into_inner()) { - if let UtxoRpcError::ResponseParseError(ref json_err) = err { - if let JsonRpcErrorType::Response(_, json) = &json_err.error { - if let Some(message) = json["message"].as_str() { - if message.contains("'code': -5") { - return Ok(false); + Ok(bytes) => Ok(Some(deserialize(&bytes.into_vec())?)), + Err(err) => { + if let UtxoRpcError::ResponseParseError(ref json_err) = err { + if let JsonRpcErrorType::Response(_, json) = &json_err.error { + if let Some(message) = json["message"].as_str() { + if message.contains("'code': -5") { + return Ok(None); + } } } } - } - return Err(err.into()); + Err(err.into()) + }, } - Ok(true) } async fn process_tx_for_unconfirmation(&self, txid: Txid, monitor: Arc) where T: Confirm, { - match self.check_if_tx_is_onchain(txid).await { - Ok(true) => {}, - Ok(false) => { + match self.get_tx_if_onchain(txid).await { + Ok(Some(_)) => {}, + Ok(None) => { info!( "Transaction {} is not found on chain. The transaction will be re-broadcasted.", txid, @@ -305,7 +287,7 @@ impl Platform { monitor.transaction_unconfirmed(&txid); }, Err(e) => error!( - "Error while trying to check if the transaction {} is discarded or not :{}", + "Error while trying to check if the transaction {} is discarded or not :{:?}", txid, e ), } @@ -329,61 +311,27 @@ impl Platform { } } - async fn get_confirmed_registered_txs( - &self, - client: &ElectrumClient, - current_height: u64, - ) -> Vec { + async fn get_confirmed_registered_txs(&self, client: &ElectrumClient) -> Vec { let registered_txs = self.registered_txs.lock().clone(); let mut confirmed_registered_txs = Vec::new(); for (txid, scripts) in registered_txs { - let rpc_txid = H256Json::from(txid.as_hash().into_inner()).reversed(); - let bytes = try_loop_with_sleep!( - self.coin - .as_ref() - .rpc_client - .get_transaction_bytes(&rpc_txid) - .compat() - .await, - TRY_LOOP_INTERVAL - ); - let transaction: Transaction = match deserialize(&bytes.into_vec()) { - Ok(tx) => tx, - Err(e) => { - error!("Transaction deserialization error: {}", e.to_string()); - continue; - }, - }; - for (_, vout) in transaction.output.iter().enumerate() { - if scripts.contains(&vout.script_pubkey) { - let script_hash = hex::encode(electrum_script_hash(vout.script_pubkey.as_ref())); - let history = try_loop_with_sleep!( - client.scripthash_get_history(&script_hash).compat().await, - TRY_LOOP_INTERVAL - ); - for item in history { - if item.tx_hash == rpc_txid { - // If a new block mined the transaction while running process_txs_confirmations it will be confirmed later in ln_best_block_update_loop - if item.height > 0 && item.height <= current_height as i64 { - let height: u64 = match item.height.try_into() { - Ok(h) => h, - Err(e) => { - error!("Block height convertion to u64 error: {}", e.to_string()); - continue; - }, - }; - let block_header = try_loop_with_sleep!( - client.blockchain_block_header(height).compat().await, - TRY_LOOP_INTERVAL - ); - let header = match deserialize(&block_header) { - Ok(h) => h, - Err(e) => { - error!("Block header deserialization error: {}", e.to_string()); - continue; - }, - }; - let index = try_loop_with_sleep!( + if let Some(transaction) = + ok_or_continue_after_sleep!(self.get_tx_if_onchain(txid).await, TRY_LOOP_INTERVAL) + { + for (_, vout) in transaction.output.iter().enumerate() { + if scripts.contains(&vout.script_pubkey) { + let script_hash = hex::encode(electrum_script_hash(vout.script_pubkey.as_ref())); + let history = ok_or_retry_after_sleep!( + client.scripthash_get_history(&script_hash).compat().await, + TRY_LOOP_INTERVAL + ); + for item in history { + let rpc_txid = H256Json::from(txid.as_hash().into_inner()).reversed(); + if item.tx_hash == rpc_txid && item.height > 0 { + let height = item.height as u64; + let header = + ok_or_retry_after_sleep!(get_block_header(client, height).await, TRY_LOOP_INTERVAL); + let index = ok_or_retry_after_sleep!( client .blockchain_transaction_get_merkle(rpc_txid, height) .compat() @@ -417,37 +365,33 @@ impl Platform { let mut outputs_to_remove = Vec::new(); let registered_outputs = self.registered_outputs.lock().clone(); for output in registered_outputs { - match find_watched_output_spend_with_header(client, &output).await { - Ok(Some(tx_info)) => { - if !transactions_to_confirm - .iter() - .any(|info| info.txid == tx_info.tx.txid()) - { - let rpc_txid = H256Json::from(tx_info.tx.txid().as_hash().into_inner()).reversed(); - let index = try_loop_with_sleep!( - client - .blockchain_transaction_get_merkle(rpc_txid, tx_info.block_height) - .compat() - .await, - TRY_LOOP_INTERVAL - ) - .pos; - let confirmed_transaction_info = ConfirmedTransactionInfo::new( - tx_info.tx.txid(), - tx_info.block_header, - index, - tx_info.tx, - tx_info.block_height as u32, - ); - transactions_to_confirm.push(confirmed_transaction_info); - } - outputs_to_remove.push(output); - }, - Ok(None) => (), - Err(e) => error!( - "Error while trying to find if the registered output {:?} is spent: {}", - output.outpoint, e - ), + if let Some(tx_info) = ok_or_continue_after_sleep!( + find_watched_output_spend_with_header(client, &output).await, + TRY_LOOP_INTERVAL + ) { + if !transactions_to_confirm + .iter() + .any(|info| info.txid == tx_info.tx.txid()) + { + let rpc_txid = H256Json::from(tx_info.tx.txid().as_hash().into_inner()).reversed(); + let index = ok_or_retry_after_sleep!( + client + .blockchain_transaction_get_merkle(rpc_txid, tx_info.block_height) + .compat() + .await, + TRY_LOOP_INTERVAL + ) + .pos; + let confirmed_transaction_info = ConfirmedTransactionInfo::new( + tx_info.tx.txid(), + tx_info.block_header, + index, + tx_info.tx, + tx_info.block_height as u32, + ); + transactions_to_confirm.push(confirmed_transaction_info); + } + outputs_to_remove.push(output); } } self.registered_outputs @@ -461,9 +405,8 @@ impl Platform { persister: Arc, chain_monitor: Arc, channel_manager: Arc, - current_height: u64, ) { - let mut transactions_to_confirm = self.get_confirmed_registered_txs(&client, current_height).await; + let mut transactions_to_confirm = self.get_confirmed_registered_txs(&client).await; self.append_spent_registered_output_txs(&mut transactions_to_confirm, &client) .await; @@ -511,22 +454,15 @@ impl Platform { let tx_hash = H256Json::from_str(&tx_id).map_to_mm(|e| SaveChannelClosingError::FundingTxParseError(e.to_string()))?; - let funding_tx_bytes = loop { - match self - .coin + let funding_tx_bytes = ok_or_retry_after_sleep!( + self.coin .as_ref() .rpc_client .get_transaction_bytes(&tx_hash) .compat() - .await - { - Ok(tx) => break tx, - Err(e) => { - error!("Error getting funding tx bytes: {}", e); - Timer::sleep(GET_FUNDING_TX_INTERVAL).await; - }, - } - }; + .await, + TRY_LOOP_INTERVAL + ); let closing_tx = self .coin @@ -617,33 +553,31 @@ impl Filter for Platform { // the filter interface which includes register_output and register_tx should be used for electrum clients only, // this is the reason for initializing the filter as an option in the start_lightning function as it will be None // when implementing lightning for native clients - let output_spend_fut = tokio::task::block_in_place(move || { - client - .find_output_spend( - H256::from(output.outpoint.txid.as_hash().into_inner()), - output.script_pubkey.as_ref(), - output.outpoint.index.into(), - BlockHashOrHeight::Hash(block_hash), - ) - .wait() + let output_spend_info = tokio::task::block_in_place(move || { + let delay = TRY_LOOP_INTERVAL as u64; + ok_or_retry_after_sleep_sync!( + client + .find_output_spend( + H256::from(output.outpoint.txid.as_hash().into_inner()), + output.script_pubkey.as_ref(), + output.outpoint.index.into(), + BlockHashOrHeight::Hash(block_hash), + ) + .wait(), + delay + ) }); - match output_spend_fut { - Ok(Some(spent_output_info)) => { - let spending_tx = match Transaction::try_from(spent_output_info.spending_tx) { - Ok(tx) => tx, - Err(e) => { - error!("Can't convert transaction error: {}", e.to_string()); - return None; - }, - }; - Some((spent_output_info.input_index, spending_tx)) - }, - Ok(None) => None, - Err(e) => { - error!("Error when calling register_output: {}", e); - None - }, + if let Some(info) = output_spend_info { + match Transaction::try_from(info.spending_tx) { + Ok(tx) => Some((info.input_index, tx)), + Err(e) => { + error!("Can't convert transaction error: {}", e.to_string()); + return None; + }, + }; } + + None } } diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 1d7f44113c..982afe89eb 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -207,7 +207,6 @@ pub async fn init_channel_manager( persister.clone(), chain_monitor.clone(), channel_manager.clone(), - best_header.block_height(), ) .await; update_best_block(chain_monitor.clone(), channel_manager.clone(), best_header).await; diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index cd25694932..22d9e6e250 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -97,7 +97,7 @@ macro_rules! try_f { }; } -macro_rules! try_loop_with_sleep { +macro_rules! ok_or_continue_after_sleep { ($e:expr, $delay: ident) => { match $e { Ok(res) => res, @@ -110,6 +110,36 @@ macro_rules! try_loop_with_sleep { }; } +macro_rules! ok_or_retry_after_sleep { + ($e:expr, $delay: ident) => { + loop { + match $e { + Ok(res) => break res, + Err(e) => { + error!("error {:?}", e); + Timer::sleep($delay).await; + continue; + }, + } + } + }; +} + +macro_rules! ok_or_retry_after_sleep_sync { + ($e:expr, $delay: ident) => { + loop { + match $e { + Ok(res) => break res, + Err(e) => { + error!("error {:?}", e); + std::thread::sleep(core::time::Duration::from_secs($delay)); + continue; + }, + } + } + }; +} + pub mod coin_balance; #[doc(hidden)] #[cfg(test)] diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index c1e4e6ada4..002fdfeb17 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3480,25 +3480,26 @@ where params.difficulty_check, params.constant_difficulty, ); - let height = try_loop_with_sleep!(coin.as_ref().rpc_client.get_block_count().compat().await, check_every); + let height = + ok_or_continue_after_sleep!(coin.as_ref().rpc_client.get_block_count().compat().await, check_every); let client = match &coin.as_ref().rpc_client { UtxoRpcClientEnum::Native(_) => break, UtxoRpcClientEnum::Electrum(client) => client, }; - let (block_registry, block_headers) = try_loop_with_sleep!( + let (block_registry, block_headers) = ok_or_continue_after_sleep!( client .retrieve_last_headers(blocks_limit_to_check, height) .compat() .await, check_every ); - try_loop_with_sleep!( + ok_or_continue_after_sleep!( validate_headers(block_headers, difficulty_check, constant_difficulty), check_every ); let ticker = coin.as_ref().conf.ticker.as_str(); - try_loop_with_sleep!( + ok_or_continue_after_sleep!( storage.add_block_headers_to_storage(ticker, block_registry).await, check_every ); From f1001a0bea615cd533b39f768842845a6b55c7ca Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Wed, 13 Apr 2022 17:55:24 +0200 Subject: [PATCH 42/49] remove wasm warnings --- mm2src/coins/lp_coins.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index cc234c4638..8ed5a523a2 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -110,6 +110,7 @@ macro_rules! ok_or_continue_after_sleep { }; } +#[cfg(not(target_arch = "wasm32"))] macro_rules! ok_or_retry_after_sleep { ($e:expr, $delay: ident) => { loop { @@ -125,6 +126,7 @@ macro_rules! ok_or_retry_after_sleep { }; } +#[cfg(not(target_arch = "wasm32"))] macro_rules! ok_or_retry_after_sleep_sync { ($e:expr, $delay: ident) => { loop { From 1800112bce9702f34ce59294b66eb5dbb7130812 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Mon, 18 Apr 2022 15:31:44 +0200 Subject: [PATCH 43/49] destination can't be None in OutboundPayment --- mm2src/coins/lightning.rs | 15 +++++---------- mm2src/coins/lightning_persister/src/lib.rs | 17 ++++++----------- mm2src/coins/lightning_persister/src/storage.rs | 2 +- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index b844540ac2..e7fa73c362 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -127,7 +127,7 @@ impl LightningCoin { .map_to_mm(|e| SendPaymentError::PaymentError(format!("{:?}", e)))?; let payment_hash = PaymentHash((*invoice.payment_hash()).into_inner()); let payment_type = PaymentType::OutboundPayment { - destination: invoice.payee_pub_key().cloned(), + destination: *invoice.payee_pub_key().unwrap_or(&invoice.recover_payee_pub_key()), }; let description = Some(match invoice.description() { InvoiceDescription::Direct(d) => d.to_string(), @@ -165,9 +165,7 @@ impl LightningCoin { .pay_pubkey(destination, payment_preimage, amount_msat, final_cltv_expiry_delta) .map_to_mm(|e| SendPaymentError::PaymentError(format!("{:?}", e)))?; let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0).into_inner()); - let payment_type = PaymentType::OutboundPayment { - destination: Some(destination), - }; + let payment_type = PaymentType::OutboundPayment { destination }; Ok(PaymentInfo { payment_hash, @@ -1282,10 +1280,7 @@ pub struct ListPaymentsReq { #[serde(tag = "type")] pub enum PaymentTypeForRPC { #[serde(rename = "Outbound Payment")] - OutboundPayment { - #[serde(skip_serializing_if = "Option::is_none")] - destination: Option, - }, + OutboundPayment { destination: PublicKeyForRPC }, #[serde(rename = "Inbound Payment")] InboundPayment, } @@ -1294,7 +1289,7 @@ impl From for PaymentTypeForRPC { fn from(payment_type: PaymentType) -> Self { match payment_type { PaymentType::OutboundPayment { destination } => PaymentTypeForRPC::OutboundPayment { - destination: destination.map(PublicKeyForRPC), + destination: PublicKeyForRPC(destination), }, PaymentType::InboundPayment => PaymentTypeForRPC::InboundPayment, } @@ -1305,7 +1300,7 @@ impl From for PaymentType { fn from(payment_type: PaymentTypeForRPC) -> Self { match payment_type { PaymentTypeForRPC::OutboundPayment { destination } => PaymentType::OutboundPayment { - destination: destination.map(From::from), + destination: destination.into(), }, PaymentTypeForRPC::InboundPayment => PaymentType::InboundPayment, } diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index b2b1537d1e..e6b35976d2 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -213,10 +213,7 @@ fn payment_info_from_row(row: &Row<'_>) -> Result { let is_outbound = row.get::<_, bool>(8)?; let payment_type = match is_outbound { true => PaymentType::OutboundPayment { - destination: row - .get::<_, String>(1) - .ok() - .map(|d| PublicKey::from_str(&d).expect("PublicKey from str should not fail!")), + destination: PublicKey::from_str(&row.get::<_, String>(1)?).expect("PublicKey from str should not fail!"), }, false => PaymentType::InboundPayment, }; @@ -436,7 +433,7 @@ fn finalize_get_payments_sql_builder(sql_builder: &mut SqlBuilder, offset: usize fn apply_get_payments_filter(builder: &mut SqlBuilder, params: &mut Vec<(&str, String)>, filter: PaymentsFilter) { if let Some(payment_type) = filter.payment_type { let (is_outbound, destination) = match payment_type { - PaymentType::OutboundPayment { destination } => (true as i32, destination.map(|d| d.to_string())), + PaymentType::OutboundPayment { destination } => (true as i32, Some(destination.to_string())), PaymentType::InboundPayment => (false as i32, None), }; if let Some(dest) = destination { @@ -1149,7 +1146,7 @@ impl DbStorage for LightningPersister { let for_coin = self.storage_ticker.clone(); let payment_hash = hex::encode(info.payment_hash.0); let (is_outbound, destination) = match info.payment_type { - PaymentType::OutboundPayment { destination } => (true as i32, destination.map(|d| d.to_string())), + PaymentType::OutboundPayment { destination } => (true as i32, Some(destination.to_string())), PaymentType::InboundPayment => (false as i32, None), }; let description = info.description; @@ -1364,9 +1361,8 @@ mod tests { } else { rng.fill_bytes(&mut bytes); let secret = SecretKey::from_slice(&bytes).unwrap(); - let pubkey = PublicKey::from_secret_key(&s, &secret); PaymentType::OutboundPayment { - destination: Some(pubkey), + destination: PublicKey::from_secret_key(&s, &secret), } }; let status_rng: u8 = rng.gen(); @@ -1771,9 +1767,8 @@ mod tests { expected_payment_info.payment_hash = PaymentHash([1; 32]); expected_payment_info.payment_type = PaymentType::OutboundPayment { - destination: Some( - PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), - ), + destination: PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9") + .unwrap(), }; expected_payment_info.secret = None; expected_payment_info.amt_msat = None; diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 96bbcadc64..8f58c2d12f 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -150,7 +150,7 @@ impl FromStr for HTLCStatus { #[derive(Clone, Debug, PartialEq)] pub enum PaymentType { - OutboundPayment { destination: Option }, + OutboundPayment { destination: PublicKey }, InboundPayment, } From d9e6b6a36caf15837c535b775e4db54a07b42029 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Mon, 18 Apr 2022 16:46:20 +0200 Subject: [PATCH 44/49] empty string for description instead of an option --- mm2src/coins/lightning.rs | 11 +++++------ mm2src/coins/lightning/ln_events.rs | 2 +- mm2src/coins/lightning_persister/src/lib.rs | 17 ++++++----------- mm2src/coins/lightning_persister/src/storage.rs | 2 +- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index e7fa73c362..6c9af7d48e 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -129,10 +129,10 @@ impl LightningCoin { let payment_type = PaymentType::OutboundPayment { destination: *invoice.payee_pub_key().unwrap_or(&invoice.recover_payee_pub_key()), }; - let description = Some(match invoice.description() { + let description = match invoice.description() { InvoiceDescription::Direct(d) => d.to_string(), InvoiceDescription::Hash(h) => hex::encode(h.0.into_inner()), - }); + }; let payment_secret = Some(*invoice.payment_secret()); Ok(PaymentInfo { payment_hash, @@ -170,7 +170,7 @@ impl LightningCoin { Ok(PaymentInfo { payment_hash, payment_type, - description: None, + description: "".into(), preimage: Some(payment_preimage), secret: None, amt_msat: Some(amount_msat), @@ -1160,7 +1160,7 @@ pub async fn generate_invoice( let payment_info = PaymentInfo { payment_hash: PaymentHash(payment_hash), payment_type: PaymentType::InboundPayment, - description: Some(req.description), + description: req.description, preimage: None, secret: Some(*invoice.payment_secret()), amt_msat: req.amount_in_msat, @@ -1311,8 +1311,7 @@ impl From for PaymentType { pub struct PaymentInfoForRPC { payment_hash: H256Json, payment_type: PaymentTypeForRPC, - #[serde(skip_serializing_if = "Option::is_none")] - description: Option, + description: String, #[serde(skip_serializing_if = "Option::is_none")] amount_in_msat: Option, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index b82874c741..2fc8c5966c 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -294,7 +294,7 @@ impl LightningEventHandler { let payment_info = PaymentInfo { payment_hash, payment_type: PaymentType::InboundPayment, - description: None, + description: "".into(), preimage: Some(payment_preimage), secret: payment_secret, amt_msat: Some(amt), diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index e6b35976d2..33302536e4 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -130,7 +130,7 @@ fn create_payments_history_table_sql(for_coin: &str) -> Result id INTEGER NOT NULL PRIMARY KEY, payment_hash VARCHAR(255) NOT NULL UNIQUE, destination VARCHAR(255), - description VARCHAR(641), + description VARCHAR(641) NOT NULL, preimage VARCHAR(255), secret VARCHAR(255), amount_msat INTEGER, @@ -225,7 +225,7 @@ fn payment_info_from_row(row: &Row<'_>) -> Result { .expect("String should be 64 characters!"), ), payment_type, - description: row.get(2).ok(), + description: row.get(2)?, preimage: row.get::<_, String>(3).ok().map(|p| { PaymentPreimage( hex::decode(p) @@ -1380,7 +1380,7 @@ mod tests { PaymentHash(bytes) }, payment_type, - description: Some(description), + description, preimage: { rng.fill_bytes(&mut bytes); Some(PaymentPreimage(bytes)) @@ -1749,7 +1749,7 @@ mod tests { let mut expected_payment_info = PaymentInfo { payment_hash: PaymentHash([0; 32]), payment_type: PaymentType::InboundPayment, - description: Some("test payment".into()), + description: "test payment".into(), preimage: Some(PaymentPreimage([2; 32])), secret: Some(PaymentSecret([3; 32])), amt_msat: Some(2000), @@ -1880,7 +1880,7 @@ mod tests { assert_eq!(expected_payments, actual_payments); - let description = payments[42].description.as_ref().unwrap(); + let description = &payments[42].description; let substr = &description[5..10]; filter.payment_type = None; filter.status = None; @@ -1889,12 +1889,7 @@ mod tests { let expected_payments_vec: Vec = payments .iter() .map(|p| p.clone()) - .filter(|p| { - if let Some(desc) = &p.description { - return desc.contains(&substr); - } - return false; - }) + .filter(|p| p.description.contains(&substr)) .collect(); let expected_payments = if expected_payments_vec.len() > 10 { expected_payments_vec[..10].to_vec() diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 8f58c2d12f..354b7bce61 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -158,7 +158,7 @@ pub enum PaymentType { pub struct PaymentInfo { pub payment_hash: PaymentHash, pub payment_type: PaymentType, - pub description: Option, + pub description: String, pub preimage: Option, pub secret: Option, pub amt_msat: Option, From 6ccb32feb9df40fa5a465adeba310d9723db0699 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Mon, 18 Apr 2022 18:02:19 +0200 Subject: [PATCH 45/49] Review fixes: refactors --- mm2src/coins/lightning/ln_platform.rs | 62 ++++++++++----------------- mm2src/coins/lightning/ln_utils.rs | 12 +++--- 2 files changed, 29 insertions(+), 45 deletions(-) diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index 4a4c938b24..48289a77fc 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -87,8 +87,8 @@ pub async fn get_best_header(best_header_listener: &ElectrumClient) -> EnableLig } pub async fn update_best_block( - chain_monitor: Arc, - channel_manager: Arc, + chain_monitor: &ChainMonitor, + channel_manager: &ChannelManager, best_header: ElectrumBlockHeader, ) { { @@ -156,18 +156,13 @@ pub async fn ln_best_block_update_loop( if current_best_block != best_header.clone().into() { platform.update_best_block_height(best_header.block_height()); platform - .process_txs_unconfirmations(chain_monitor.clone(), channel_manager.clone()) + .process_txs_unconfirmations(&chain_monitor, &channel_manager) .await; platform - .process_txs_confirmations( - best_header_listener.clone(), - persister.clone(), - chain_monitor.clone(), - channel_manager.clone(), - ) + .process_txs_confirmations(&best_header_listener, &persister, &chain_monitor, &channel_manager) .await; current_best_block = best_header.clone().into(); - update_best_block(chain_monitor.clone(), channel_manager.clone(), best_header).await; + update_best_block(&chain_monitor, &channel_manager, best_header).await; } Timer::sleep(CHECK_FOR_NEW_BEST_BLOCK_INTERVAL).await; } @@ -228,6 +223,8 @@ impl Platform { } } + fn rpc_client(&self) -> &UtxoRpcClientEnum { &self.coin.as_ref().rpc_client } + pub fn update_best_block_height(&self, new_height: u64) { self.best_block_height.store(new_height, AtomicOrdering::Relaxed); } @@ -248,11 +245,10 @@ impl Platform { } async fn get_tx_if_onchain(&self, txid: Txid) -> Result, GetTxError> { + let txid = H256Json::from(txid.as_hash().into_inner()).reversed(); match self - .coin - .as_ref() - .rpc_client - .get_transaction_bytes(&H256Json::from(txid.as_hash().into_inner()).reversed()) + .rpc_client() + .get_transaction_bytes(&txid) .compat() .await .map_err(|e| e.into_inner()) @@ -273,7 +269,7 @@ impl Platform { } } - async fn process_tx_for_unconfirmation(&self, txid: Txid, monitor: Arc) + async fn process_tx_for_unconfirmation(&self, txid: Txid, monitor: &T) where T: Confirm, { @@ -293,21 +289,17 @@ impl Platform { } } - pub async fn process_txs_unconfirmations( - &self, - chain_monitor: Arc, - channel_manager: Arc, - ) { + pub async fn process_txs_unconfirmations(&self, chain_monitor: &ChainMonitor, channel_manager: &ChannelManager) { // Retrieve channel manager transaction IDs to check the chain for un-confirmations let channel_manager_relevant_txids = channel_manager.get_relevant_txids(); for txid in channel_manager_relevant_txids { - self.process_tx_for_unconfirmation(txid, channel_manager.clone()).await; + self.process_tx_for_unconfirmation(txid, channel_manager).await; } // Retrieve chain monitor transaction IDs to check the chain for un-confirmations let chain_monitor_relevant_txids = chain_monitor.get_relevant_txids(); for txid in chain_monitor_relevant_txids { - self.process_tx_for_unconfirmation(txid, chain_monitor.clone()).await; + self.process_tx_for_unconfirmation(txid, chain_monitor).await; } } @@ -401,13 +393,13 @@ impl Platform { pub async fn process_txs_confirmations( &self, - client: ElectrumClient, - persister: Arc, - chain_monitor: Arc, - channel_manager: Arc, + client: &ElectrumClient, + persister: &LightningPersister, + chain_monitor: &ChainMonitor, + channel_manager: &ChannelManager, ) { - let mut transactions_to_confirm = self.get_confirmed_registered_txs(&client).await; - self.append_spent_registered_output_txs(&mut transactions_to_confirm, &client) + let mut transactions_to_confirm = self.get_confirmed_registered_txs(client).await; + self.append_spent_registered_output_txs(&mut transactions_to_confirm, client) .await; transactions_to_confirm.sort_by(|a, b| (a.height, a.index).cmp(&(b.height, b.index))); @@ -455,12 +447,7 @@ impl Platform { H256Json::from_str(&tx_id).map_to_mm(|e| SaveChannelClosingError::FundingTxParseError(e.to_string()))?; let funding_tx_bytes = ok_or_retry_after_sleep!( - self.coin - .as_ref() - .rpc_client - .get_transaction_bytes(&tx_hash) - .compat() - .await, + self.rpc_client().get_transaction_bytes(&tx_hash).compat().await, TRY_LOOP_INTERVAL ); @@ -500,9 +487,7 @@ impl FeeEstimator for Platform { ConfirmationTarget::HighPriority => self.default_fees_and_confirmations.high_priority.n_blocks, }; let fee_per_kb = tokio::task::block_in_place(move || { - platform_coin - .as_ref() - .rpc_client + self.rpc_client() .estimate_fee_sat( platform_coin.decimals(), // Todo: when implementing Native client detect_fee_method should be used for Native and @@ -548,7 +533,6 @@ impl Filter for Platform { None => return None, }; - let client = &self.coin.as_ref().rpc_client; // Although this works for both native and electrum clients as the block hash is available, // the filter interface which includes register_output and register_tx should be used for electrum clients only, // this is the reason for initializing the filter as an option in the start_lightning function as it will be None @@ -556,7 +540,7 @@ impl Filter for Platform { let output_spend_info = tokio::task::block_in_place(move || { let delay = TRY_LOOP_INTERVAL as u64; ok_or_retry_after_sleep_sync!( - client + self.rpc_client() .find_output_spend( H256::from(output.outpoint.txid.as_hash().into_inner()), output.script_pubkey.as_ref(), diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 982afe89eb..7c9114ef7d 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -198,18 +198,18 @@ pub async fn init_channel_manager( // Sync ChannelMonitors and ChannelManager to chain tip if the node is restarting and has open channels if channel_manager_blockhash != best_block_hash { platform - .process_txs_unconfirmations(chain_monitor.clone(), channel_manager.clone()) + .process_txs_unconfirmations(&chain_monitor, &channel_manager) .await; platform .process_txs_confirmations( // It's safe to use unwrap here for now until implementing Native Client for Lightning - rpc_client.clone(), - persister.clone(), - chain_monitor.clone(), - channel_manager.clone(), + &rpc_client, + &persister, + &chain_monitor, + &channel_manager, ) .await; - update_best_block(chain_monitor.clone(), channel_manager.clone(), best_header).await; + update_best_block(&chain_monitor, &channel_manager, best_header).await; } // Give ChannelMonitors to ChainMonitor From 21a04d7affa3089a90bdcdf3a7f10df810654c3e Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Tue, 19 Apr 2022 16:42:09 +0200 Subject: [PATCH 46/49] review fixes wip --- mm2src/coins/lightning.rs | 6 +- mm2src/coins/lightning/ln_platform.rs | 8 +- mm2src/coins/lightning/ln_utils.rs | 8 +- mm2src/coins/lightning_persister/src/lib.rs | 209 ++++++++++++++------ 4 files changed, 153 insertions(+), 78 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 6c9af7d48e..f64023e048 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -53,7 +53,7 @@ use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelErr SendPaymentResult}; use ln_events::LightningEventHandler; use ln_p2p::{connect_to_node, ConnectToNodeRes, PeerManager}; -use ln_platform::Platform; +use ln_platform::{h256_json_from_txid, Platform}; use ln_serialization::{InvoiceForRPC, NodeAddress, PublicKeyForRPC}; use ln_utils::{ChainMonitor, ChannelManager}; use parking_lot::Mutex as PaMutex; @@ -975,9 +975,7 @@ impl From for ChannelDetailsForRPC { rpc_channel_id: details.user_channel_id, channel_id: details.channel_id.into(), counterparty_node_id: PublicKeyForRPC(details.counterparty.node_id), - funding_tx: details - .funding_txo - .map(|tx| H256Json::from(tx.txid.as_hash().into_inner()).reversed()), + funding_tx: details.funding_txo.map(|tx| h256_json_from_txid(tx.txid)), funding_tx_output_index: details.funding_txo.map(|tx| tx.index), funding_tx_value_sats: details.channel_value_satoshis, is_outbound: details.is_outbound, diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index 48289a77fc..4721fb20b5 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -28,6 +28,8 @@ const CHECK_FOR_NEW_BEST_BLOCK_INTERVAL: f64 = 60.; const MIN_ALLOWED_FEE_PER_1000_WEIGHT: u32 = 253; const TRY_LOOP_INTERVAL: f64 = 60.; +pub fn h256_json_from_txid(txid: Txid) -> H256Json { H256Json::from(txid.as_hash().into_inner()).reversed() } + struct TxWithBlockInfo { tx: Transaction, block_header: BlockHeader, @@ -245,7 +247,7 @@ impl Platform { } async fn get_tx_if_onchain(&self, txid: Txid) -> Result, GetTxError> { - let txid = H256Json::from(txid.as_hash().into_inner()).reversed(); + let txid = h256_json_from_txid(txid); match self .rpc_client() .get_transaction_bytes(&txid) @@ -318,7 +320,7 @@ impl Platform { TRY_LOOP_INTERVAL ); for item in history { - let rpc_txid = H256Json::from(txid.as_hash().into_inner()).reversed(); + let rpc_txid = h256_json_from_txid(txid); if item.tx_hash == rpc_txid && item.height > 0 { let height = item.height as u64; let header = @@ -365,7 +367,7 @@ impl Platform { .iter() .any(|info| info.txid == tx_info.tx.txid()) { - let rpc_txid = H256Json::from(tx_info.tx.txid().as_hash().into_inner()).reversed(); + let rpc_txid = h256_json_from_txid(tx_info.tx.txid()); let index = ok_or_retry_after_sleep!( client .blockchain_transaction_get_merkle(rpc_txid, tx_info.block_height) diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 7c9114ef7d..cf9ad18c3c 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -201,13 +201,7 @@ pub async fn init_channel_manager( .process_txs_unconfirmations(&chain_monitor, &channel_manager) .await; platform - .process_txs_confirmations( - // It's safe to use unwrap here for now until implementing Native Client for Lightning - &rpc_client, - &persister, - &chain_monitor, - &channel_manager, - ) + .process_txs_confirmations(&rpc_client, &persister, &chain_monitor, &channel_manager) .await; update_best_block(&chain_monitor, &channel_manager, best_header).await; } diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 33302536e4..a8bb5fbbcc 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -96,26 +96,27 @@ fn create_channels_history_table_sql(for_coin: &str) -> Result let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "CREATE TABLE IF NOT EXISTS ".to_owned() - + &table_name - + " ( - id INTEGER NOT NULL PRIMARY KEY, - rpc_id INTEGER NOT NULL UNIQUE, - channel_id VARCHAR(255) NOT NULL, - counterparty_node_id VARCHAR(255) NOT NULL, - funding_tx VARCHAR(255), - funding_value INTEGER, - funding_generated_in_block Integer, - closing_tx VARCHAR(255), - closure_reason TEXT, - claiming_tx VARCHAR(255), - claimed_balance REAL, - is_outbound INTEGER NOT NULL, - is_public INTEGER NOT NULL, - is_closed INTEGER NOT NULL, - created_at INTEGER NOT NULL, - last_updated INTEGER NOT NULL - );"; + let sql = format!( + "CREATE TABLE IF NOT EXISTS {} ( + id INTEGER NOT NULL PRIMARY KEY, + rpc_id INTEGER NOT NULL UNIQUE, + channel_id VARCHAR(255) NOT NULL, + counterparty_node_id VARCHAR(255) NOT NULL, + funding_tx VARCHAR(255), + funding_value INTEGER, + funding_generated_in_block Integer, + closing_tx VARCHAR(255), + closure_reason TEXT, + claiming_tx VARCHAR(255), + claimed_balance REAL, + is_outbound INTEGER NOT NULL, + is_public INTEGER NOT NULL, + is_closed INTEGER NOT NULL, + created_at INTEGER NOT NULL, + last_updated INTEGER NOT NULL + );", + table_name + ); Ok(sql) } @@ -124,22 +125,23 @@ fn create_payments_history_table_sql(for_coin: &str) -> Result let table_name = payments_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "CREATE TABLE IF NOT EXISTS ".to_owned() - + &table_name - + " ( - id INTEGER NOT NULL PRIMARY KEY, - payment_hash VARCHAR(255) NOT NULL UNIQUE, - destination VARCHAR(255), - description VARCHAR(641) NOT NULL, - preimage VARCHAR(255), - secret VARCHAR(255), - amount_msat INTEGER, - fee_paid_msat INTEGER, - is_outbound INTEGER NOT NULL, - status VARCHAR(255) NOT NULL, - created_at INTEGER NOT NULL, - last_updated INTEGER NOT NULL - );"; + let sql = format!( + "CREATE TABLE IF NOT EXISTS {} ( + id INTEGER NOT NULL PRIMARY KEY, + payment_hash VARCHAR(255) NOT NULL UNIQUE, + destination VARCHAR(255), + description VARCHAR(641) NOT NULL, + preimage VARCHAR(255), + secret VARCHAR(255), + amount_msat INTEGER, + fee_paid_msat INTEGER, + is_outbound INTEGER NOT NULL, + status VARCHAR(255) NOT NULL, + created_at INTEGER NOT NULL, + last_updated INTEGER NOT NULL + );", + table_name + ); Ok(sql) } @@ -148,9 +150,21 @@ fn insert_channel_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "INSERT INTO ".to_owned() - + &table_name - + " (rpc_id, channel_id, counterparty_node_id, is_outbound, is_public, is_closed, created_at, last_updated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8);"; + let sql = format!( + "INSERT INTO {} ( + rpc_id, + channel_id, + counterparty_node_id, + is_outbound, + is_public, + is_closed, + created_at, + last_updated + ) VALUES ( + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8 + );", + table_name + ); Ok(sql) } @@ -159,9 +173,24 @@ fn insert_or_update_payment_sql(for_coin: &str) -> Result { let table_name = payments_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "INSERT OR REPLACE INTO ".to_owned() - + &table_name - + " (payment_hash, destination, description, preimage, secret, amount_msat, fee_paid_msat, is_outbound, status, created_at, last_updated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11);"; + let sql = format!( + "INSERT OR REPLACE INTO {} ( + payment_hash, + destination, + description, + preimage, + secret, + amount_msat, + fee_paid_msat, + is_outbound, + status, + created_at, + last_updated + ) VALUES ( + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 + );", + table_name + ); Ok(sql) } @@ -170,7 +199,29 @@ fn select_channel_from_table_by_rpc_id_sql(for_coin: &str) -> Result Result) -> Result) -> Result { let is_outbound = row.get::<_, bool>(8)?; - let payment_type = match is_outbound { - true => PaymentType::OutboundPayment { + let payment_type = if is_outbound { + PaymentType::OutboundPayment { destination: PublicKey::from_str(&row.get::<_, String>(1)?).expect("PublicKey from str should not fail!"), - }, - false => PaymentType::InboundPayment, + } + } else { + PaymentType::InboundPayment }; let payment_info = PaymentInfo { payment_hash: PaymentHash( @@ -255,7 +321,7 @@ fn get_last_channel_rpc_id_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "SELECT IFNULL(MAX(rpc_id), 0) FROM ".to_owned() + &table_name + ";"; + let sql = format!("SELECT IFNULL(MAX(rpc_id), 0) FROM {};", table_name); Ok(sql) } @@ -264,9 +330,16 @@ fn update_funding_tx_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "UPDATE ".to_owned() - + &table_name - + " SET funding_tx = ?2, funding_value = ?3, funding_generated_in_block = ?4, last_updated = ?5 WHERE rpc_id = ?1;"; + let sql = format!( + "UPDATE {} SET + funding_tx = ?2, + funding_value = ?3, + funding_generated_in_block = ?4, + last_updated = ?5 + WHERE + rpc_id = ?1;", + table_name + ); Ok(sql) } @@ -275,7 +348,10 @@ fn update_funding_tx_block_height_sql(for_coin: &str) -> Result Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "UPDATE ".to_owned() - + &table_name - + " SET closure_reason = ?2, is_closed = ?3, last_updated = ?4 WHERE rpc_id = ?1;"; + let sql = format!( + "UPDATE {} SET closure_reason = ?2, is_closed = ?3, last_updated = ?4 WHERE rpc_id = ?1;", + table_name + ); Ok(sql) } @@ -295,7 +372,10 @@ fn update_closing_tx_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "UPDATE ".to_owned() + &table_name + " SET closing_tx = ?2, last_updated = ?3 WHERE rpc_id = ?1;"; + let sql = format!( + "UPDATE {} SET closing_tx = ?2, last_updated = ?3 WHERE rpc_id = ?1;", + table_name + ); Ok(sql) } @@ -489,9 +569,10 @@ fn update_claiming_tx_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "UPDATE ".to_owned() - + &table_name - + " SET claiming_tx = ?2, claimed_balance = ?3, last_updated = ?4 WHERE closing_tx = ?1;"; + let sql = format!( + "UPDATE {} SET claiming_tx = ?2, claimed_balance = ?3, last_updated = ?4 WHERE closing_tx = ?1;", + table_name + ); Ok(sql) } From 9b3339a7e6e2e49ae98345de6162b4d6c31cbfed Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Wed, 20 Apr 2022 15:17:31 +0200 Subject: [PATCH 47/49] wip: more review fixes --- mm2src/coins/lightning_persister/src/lib.rs | 109 ++++++++++-------- .../coins/lightning_persister/src/storage.rs | 8 +- mm2src/coins/sql_tx_history_storage.rs | 87 ++++++++------ .../utxo/utxo_sql_block_header_storage.rs | 20 ++-- 4 files changed, 130 insertions(+), 94 deletions(-) diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index a8bb5fbbcc..cb3342a2ca 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -24,6 +24,7 @@ use bitcoin::hashes::hex::{FromHex, ToHex}; use bitcoin::Network; use common::fs::check_dir_operations; use common::{async_blocking, now_ms, PagingOptionsEnum}; +use db_common::sqlite::rusqlite::types::Type as SqlType; use db_common::sqlite::rusqlite::{Error as SqlError, Row, ToSql, NO_PARAMS}; use db_common::sqlite::sql_builder::SqlBuilder; use db_common::sqlite::{offset_by_id, query_single_row, string_from_row, validate_table_name, SqliteConnShared, @@ -258,13 +259,13 @@ fn channel_details_from_row(row: &Row<'_>) -> Result(0)? as u64, channel_id: row.get(1)?, counterparty_node_id: row.get(2)?, - funding_tx: row.get(3).ok(), - funding_value: row.get::<_, u32>(4).ok().map(|v| v as u64), - funding_generated_in_block: row.get::<_, u32>(5).ok().map(|v| v as u64), - closing_tx: row.get(6).ok(), - closure_reason: row.get(7).ok(), - claiming_tx: row.get(8).ok(), - claimed_balance: row.get::<_, f64>(9).ok(), + funding_tx: row.get(3)?, + funding_value: row.get::<_, Option>(4)?.map(|v| v as u64), + funding_generated_in_block: row.get::<_, Option>(5)?.map(|v| v as u64), + closing_tx: row.get(6)?, + closure_reason: row.get(7)?, + claiming_tx: row.get(8)?, + claimed_balance: row.get::<_, Option>(9)?, is_outbound: row.get(10)?, is_public: row.get(11)?, is_closed: row.get(12)?, @@ -278,38 +279,48 @@ fn payment_info_from_row(row: &Row<'_>) -> Result { let is_outbound = row.get::<_, bool>(8)?; let payment_type = if is_outbound { PaymentType::OutboundPayment { - destination: PublicKey::from_str(&row.get::<_, String>(1)?).expect("PublicKey from str should not fail!"), + destination: PublicKey::from_str(&row.get::<_, String>(1)?) + .map_err(|e| SqlError::FromSqlConversionFailure(1, SqlType::Text, Box::new(e)))?, } } else { PaymentType::InboundPayment }; + + let mut hash_slice = [0u8; 32]; + hex::decode_to_slice(row.get::<_, String>(0)?, &mut hash_slice as &mut [u8]) + .map_err(|e| SqlError::FromSqlConversionFailure(0, SqlType::Text, Box::new(e)))?; + let payment_hash = PaymentHash(hash_slice); + + let maybe_preimage = row.get::<_, Option>(3)?; + let preimage = match maybe_preimage { + Some(p) => { + let mut preimage_slice = [0u8; 32]; + hex::decode_to_slice(p, &mut preimage_slice as &mut [u8]) + .map_err(|e| SqlError::FromSqlConversionFailure(3, SqlType::Text, Box::new(e)))?; + Some(PaymentPreimage(preimage_slice)) + }, + None => None, + }; + + let maybe_secret = row.get::<_, Option>(4)?; + let secret = match maybe_secret { + Some(s) => { + let mut secret_slice = [0u8; 32]; + hex::decode_to_slice(s, &mut secret_slice as &mut [u8]) + .map_err(|e| SqlError::FromSqlConversionFailure(4, SqlType::Text, Box::new(e)))?; + Some(PaymentSecret(secret_slice)) + }, + None => None, + }; + let payment_info = PaymentInfo { - payment_hash: PaymentHash( - hex::decode(row.get::<_, String>(0)?) - .expect("Payment hash decoding should not fail!") - .try_into() - .expect("String should be 64 characters!"), - ), + payment_hash, payment_type, description: row.get(2)?, - preimage: row.get::<_, String>(3).ok().map(|p| { - PaymentPreimage( - hex::decode(p) - .expect("Preimage decoding should not fail!") - .try_into() - .expect("String should be 64 characters!"), - ) - }), - secret: row.get::<_, String>(4).ok().map(|s| { - PaymentSecret( - hex::decode(s) - .expect("Secret decoding should not fail!") - .try_into() - .expect("String should be 64 characters!"), - ) - }), - amt_msat: row.get::<_, u32>(5).ok().map(|v| v as u64), - fee_paid_msat: row.get::<_, u32>(6).ok().map(|v| v as u64), + preimage, + secret, + amt_msat: row.get::<_, Option>(5)?.map(|v| v as u64), + fee_paid_msat: row.get::<_, Option>(6)?.map(|v| v as u64), status: HTLCStatus::from_str(&row.get::<_, String>(7)?)?, created_at: row.get::<_, u32>(9)? as u64, last_updated: row.get::<_, u32>(10)? as u64, @@ -332,12 +343,12 @@ fn update_funding_tx_sql(for_coin: &str) -> Result { let sql = format!( "UPDATE {} SET - funding_tx = ?2, - funding_value = ?3, - funding_generated_in_block = ?4, - last_updated = ?5 + funding_tx = ?1, + funding_value = ?2, + funding_generated_in_block = ?3, + last_updated = ?4 WHERE - rpc_id = ?1;", + rpc_id = ?5;", table_name ); @@ -349,7 +360,7 @@ fn update_funding_tx_block_height_sql(for_coin: &str) -> Result Result { validate_table_name(&table_name)?; let sql = format!( - "UPDATE {} SET closure_reason = ?2, is_closed = ?3, last_updated = ?4 WHERE rpc_id = ?1;", + "UPDATE {} SET closure_reason = ?1, is_closed = ?2, last_updated = ?3 WHERE rpc_id = ?4;", table_name ); @@ -373,7 +384,7 @@ fn update_closing_tx_sql(for_coin: &str) -> Result { validate_table_name(&table_name)?; let sql = format!( - "UPDATE {} SET closing_tx = ?2, last_updated = ?3 WHERE rpc_id = ?1;", + "UPDATE {} SET closing_tx = ?1, last_updated = ?2 WHERE rpc_id = ?3;", table_name ); @@ -570,7 +581,7 @@ fn update_claiming_tx_sql(for_coin: &str) -> Result { validate_table_name(&table_name)?; let sql = format!( - "UPDATE {} SET claiming_tx = ?2, claimed_balance = ?3, last_updated = ?4 WHERE closing_tx = ?1;", + "UPDATE {} SET claiming_tx = ?1, claimed_balance = ?2, last_updated = ?3 WHERE closing_tx = ?4;", table_name ); @@ -1026,17 +1037,17 @@ impl DbStorage for LightningPersister { funding_generated_in_block: u64, ) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); - let rpc_id = rpc_id.to_string(); let funding_value = funding_value.to_string(); let funding_generated_in_block = funding_generated_in_block.to_string(); let last_updated = (now_ms() / 1000).to_string(); + let rpc_id = rpc_id.to_string(); let params = [ - rpc_id, funding_tx, funding_value, funding_generated_in_block, last_updated, + rpc_id, ]; let sqlite_connection = self.sqlite_connection.clone(); @@ -1058,7 +1069,7 @@ impl DbStorage for LightningPersister { async_blocking(move || { let mut conn = sqlite_connection.lock().unwrap(); let sql_transaction = conn.transaction()?; - let params = [&funding_tx as &dyn ToSql, &generated_in_block as &dyn ToSql]; + let params = [&generated_in_block as &dyn ToSql, &funding_tx as &dyn ToSql]; sql_transaction.execute(&update_funding_tx_block_height_sql(&for_coin)?, ¶ms)?; sql_transaction.commit()?; Ok(()) @@ -1068,11 +1079,11 @@ impl DbStorage for LightningPersister { async fn update_channel_to_closed(&self, rpc_id: u64, closure_reason: String) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); - let rpc_id = rpc_id.to_string(); let is_closed = "1".to_string(); let last_updated = (now_ms() / 1000).to_string(); + let rpc_id = rpc_id.to_string(); - let params = [rpc_id, closure_reason, is_closed, last_updated]; + let params = [closure_reason, is_closed, last_updated, rpc_id]; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { @@ -1106,10 +1117,10 @@ impl DbStorage for LightningPersister { async fn add_closing_tx_to_db(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error> { let for_coin = self.storage_ticker.clone(); - let rpc_id = rpc_id.to_string(); let last_updated = (now_ms() / 1000).to_string(); + let rpc_id = rpc_id.to_string(); - let params = [rpc_id, closing_tx, last_updated]; + let params = [closing_tx, last_updated, rpc_id]; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { @@ -1132,7 +1143,7 @@ impl DbStorage for LightningPersister { let claimed_balance = claimed_balance.to_string(); let last_updated = (now_ms() / 1000).to_string(); - let params = [closing_tx, claiming_tx, claimed_balance, last_updated]; + let params = [claiming_tx, claimed_balance, last_updated, closing_tx]; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index 354b7bce61..a333711bdb 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -139,10 +139,10 @@ impl FromStr for HTLCStatus { type Err = FromSqlError; fn from_str(s: &str) -> Result { - match s.to_lowercase().as_str() { - "pending" => Ok(HTLCStatus::Pending), - "succeeded" => Ok(HTLCStatus::Succeeded), - "failed" => Ok(HTLCStatus::Failed), + match s { + "Pending" => Ok(HTLCStatus::Pending), + "Succeeded" => Ok(HTLCStatus::Succeeded), + "Failed" => Ok(HTLCStatus::Failed), _ => Err(FromSqlError::InvalidType), } } diff --git a/mm2src/coins/sql_tx_history_storage.rs b/mm2src/coins/sql_tx_history_storage.rs index 85f934180c..2f729b777b 100644 --- a/mm2src/coins/sql_tx_history_storage.rs +++ b/mm2src/coins/sql_tx_history_storage.rs @@ -21,17 +21,18 @@ fn create_tx_history_table_sql(for_coin: &str) -> Result Result let table_name = tx_cache_table(for_coin); validate_table_name(&table_name)?; - let sql = "CREATE TABLE IF NOT EXISTS ".to_owned() - + &table_name - + " ( - tx_hash VARCHAR(255) NOT NULL UNIQUE, - tx_hex TEXT NOT NULL - );"; + let sql = format!( + "CREATE TABLE IF NOT EXISTS {} ( + tx_hash VARCHAR(255) NOT NULL UNIQUE, + tx_hex TEXT NOT NULL + );", + table_name + ); Ok(sql) } @@ -54,9 +56,19 @@ fn insert_tx_in_history_sql(for_coin: &str) -> Result> let table_name = tx_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "INSERT INTO ".to_owned() - + &table_name - + " (tx_hash, internal_id, block_height, confirmation_status, token_id, details_json) VALUES (?1, ?2, ?3, ?4, ?5, ?6);"; + let sql = format!( + "INSERT INTO {} ( + tx_hash, + internal_id, + block_height, + confirmation_status, + token_id, + details_json + ) VALUES ( + ?1, ?2, ?3, ?4, ?5, ?6 + );", + table_name + ); Ok(sql) } @@ -66,7 +78,10 @@ fn insert_tx_in_cache_sql(for_coin: &str) -> Result> { validate_table_name(&table_name)?; // We can simply ignore the repetitive attempt to insert the same tx_hash - let sql = "INSERT OR IGNORE INTO ".to_owned() + &table_name + " (tx_hash, tx_hex) VALUES (?1, ?2);"; + let sql = format!( + "INSERT OR IGNORE INTO {} (tx_hash, tx_hex) VALUES (?1, ?2);", + table_name + ); Ok(sql) } @@ -75,7 +90,7 @@ fn remove_tx_from_table_by_internal_id_sql(for_coin: &str) -> Result Result Result Result Result Result Result> let table_name = tx_history_table(for_coin); validate_table_name(&table_name)?; - let sql = "SELECT COUNT(DISTINCT tx_hash) FROM ".to_owned() + &table_name + ";"; + let sql = format!("SELECT COUNT(DISTINCT tx_hash) FROM {};", table_name); Ok(sql) } @@ -140,7 +161,7 @@ fn get_tx_hex_from_cache_sql(for_coin: &str) -> Result let table_name = tx_cache_table(for_coin); validate_table_name(&table_name)?; - let sql = "SELECT tx_hex FROM ".to_owned() + &table_name + " WHERE tx_hash = ?1 LIMIT 1;"; + let sql = format!("SELECT tx_hex FROM {} WHERE tx_hash = ?1 LIMIT 1;", table_name); Ok(sql) } @@ -172,7 +193,7 @@ impl SqliteTxHistoryStorage { fn is_table_empty(&self, table_name: &str) -> bool { validate_table_name(table_name).unwrap(); - let sql = "SELECT COUNT(id) FROM ".to_owned() + table_name + ";"; + let sql = format!("SELECT COUNT(id) FROM {};", table_name); let conn = self.0.lock().unwrap(); let rows_count: u32 = conn.query_row(&sql, NO_PARAMS, |row| row.get(0)).unwrap(); rows_count == 0 @@ -301,7 +322,7 @@ impl TxHistoryStorage for SqliteTxHistoryStorage { async_blocking(move || { let conn = selfi.0.lock().unwrap(); - query_single_row(&conn, &sql, params, tx_details_from_row).map_err(From::from) + query_single_row(&conn, &sql, params, tx_details_from_row).map_to_mm(SqlError::from) }) .await } diff --git a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs index 75dfe4f4eb..2e7546407f 100644 --- a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs @@ -26,12 +26,13 @@ fn get_table_name_and_validate(for_coin: &str) -> Result Result> { let table_name = get_table_name_and_validate(for_coin)?; - let sql = "CREATE TABLE IF NOT EXISTS ".to_owned() - + &table_name - + " ( - block_height INTEGER NOT NULL UNIQUE, - hex TEXT NOT NULL - );"; + let sql = format!( + "CREATE TABLE IF NOT EXISTS {} ( + block_height INTEGER NOT NULL UNIQUE, + hex TEXT NOT NULL + );", + table_name + ); Ok(sql) } @@ -39,13 +40,16 @@ fn create_block_header_cache_table_sql(for_coin: &str) -> Result Result> { let table_name = get_table_name_and_validate(for_coin)?; // We can simply ignore the repetitive attempt to insert the same block_height - let sql = "INSERT OR IGNORE INTO ".to_owned() + &table_name + " (block_height, hex) VALUES (?1, ?2);"; + let sql = format!( + "INSERT OR IGNORE INTO {} (block_height, hex) VALUES (?1, ?2);", + table_name + ); Ok(sql) } fn get_block_header_by_height(for_coin: &str) -> Result> { let table_name = get_table_name_and_validate(for_coin)?; - let sql = "SELECT hex FROM ".to_owned() + &table_name + " WHERE block_height=?1;"; + let sql = format!("SELECT hex FROM {} WHERE block_height=?1;", table_name); Ok(sql) } From 6d9df92ac789b9790d768e55fad259c0f1a42195 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Fri, 22 Apr 2022 14:39:42 +0200 Subject: [PATCH 48/49] review fixes: inline, refactors --- mm2src/coins/lightning.rs | 1 + mm2src/coins/lightning/ln_p2p.rs | 9 +++++---- mm2src/coins/lightning/ln_platform.rs | 5 +++++ mm2src/coins/lightning/ln_utils.rs | 2 ++ mm2src/coins/lightning_persister/src/lib.rs | 12 ++++++------ mm2src/coins/lightning_persister/src/storage.rs | 1 + mm2src/coins/sql_tx_history_storage.rs | 8 ++++---- mm2src/mm2_bitcoin/rpc/src/v1/types/hash.rs | 1 + 8 files changed, 25 insertions(+), 14 deletions(-) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index f64023e048..13a7cf54ae 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -103,6 +103,7 @@ impl fmt::Debug for LightningCoin { impl LightningCoin { fn platform_coin(&self) -> &UtxoStandardCoin { &self.platform.coin } + #[inline] fn my_node_id(&self) -> String { self.channel_manager.get_our_node_id().to_string() } fn get_balance_msat(&self) -> (u64, u64) { diff --git a/mm2src/coins/lightning/ln_p2p.rs b/mm2src/coins/lightning/ln_p2p.rs index b3c160da45..35977fb40d 100644 --- a/mm2src/coins/lightning/ln_p2p.rs +++ b/mm2src/coins/lightning/ln_p2p.rs @@ -62,11 +62,12 @@ pub async fn connect_to_node( std::task::Poll::Pending => {}, } - match peer_manager.get_peer_node_ids().contains(&pubkey) { - true => break, - // Wait for the handshake to complete if false. - false => Timer::sleep_ms(10).await, + if peer_manager.get_peer_node_ids().contains(&pubkey) { + break; } + + // Wait for the handshake to complete + Timer::sleep_ms(10).await; } Ok(ConnectToNodeRes::ConnectedSuccessfully { pubkey, node_addr }) diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index 4721fb20b5..3214c033c5 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -28,6 +28,7 @@ const CHECK_FOR_NEW_BEST_BLOCK_INTERVAL: f64 = 60.; const MIN_ALLOWED_FEE_PER_1000_WEIGHT: u32 = 253; const TRY_LOOP_INTERVAL: f64 = 60.; +#[inline] pub fn h256_json_from_txid(txid: Txid) -> H256Json { H256Json::from(txid.as_hash().into_inner()).reversed() } struct TxWithBlockInfo { @@ -225,12 +226,15 @@ impl Platform { } } + #[inline] fn rpc_client(&self) -> &UtxoRpcClientEnum { &self.coin.as_ref().rpc_client } + #[inline] pub fn update_best_block_height(&self, new_height: u64) { self.best_block_height.store(new_height, AtomicOrdering::Relaxed); } + #[inline] pub fn best_block_height(&self) -> u64 { self.best_block_height.load(AtomicOrdering::Relaxed) } pub fn add_tx(&self, txid: Txid, script_pubkey: Script) { @@ -524,6 +528,7 @@ impl BroadcasterInterface for Platform { impl Filter for Platform { // Watches for this transaction on-chain + #[inline] fn register_tx(&self, txid: &Txid, script_pubkey: &Script) { self.add_tx(*txid, script_pubkey.clone()); } // Watches for any transactions that spend this output on-chain diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index cf9ad18c3c..9467396590 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -35,8 +35,10 @@ pub type ChainMonitor = chainmonitor::ChainMonitor< pub type ChannelManager = SimpleArcChannelManager; +#[inline] fn ln_data_dir(ctx: &MmArc, ticker: &str) -> PathBuf { ctx.dbdir().join("LIGHTNING").join(ticker) } +#[inline] fn ln_data_backup_dir(ctx: &MmArc, path: Option, ticker: &str) -> Option { path.map(|p| { PathBuf::from(&p) diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index cb3342a2ca..9e680d8a6b 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -170,7 +170,7 @@ fn insert_channel_sql(for_coin: &str) -> Result { Ok(sql) } -fn insert_or_update_payment_sql(for_coin: &str) -> Result { +fn upsert_payment_sql(for_coin: &str) -> Result { let table_name = payments_history_table(for_coin); validate_table_name(&table_name)?; @@ -196,7 +196,7 @@ fn insert_or_update_payment_sql(for_coin: &str) -> Result { Ok(sql) } -fn select_channel_from_table_by_rpc_id_sql(for_coin: &str) -> Result { +fn select_channel_by_rpc_id_sql(for_coin: &str) -> Result { let table_name = channels_history_table(for_coin); validate_table_name(&table_name)?; @@ -227,7 +227,7 @@ fn select_channel_from_table_by_rpc_id_sql(for_coin: &str) -> Result Result { +fn select_payment_by_hash_sql(for_coin: &str) -> Result { let table_name = payments_history_table(for_coin); validate_table_name(&table_name)?; @@ -1158,7 +1158,7 @@ impl DbStorage for LightningPersister { async fn get_channel_from_db(&self, rpc_id: u64) -> Result, Self::Error> { let params = [rpc_id.to_string()]; - let sql = select_channel_from_table_by_rpc_id_sql(self.storage_ticker.as_str())?; + let sql = select_channel_by_rpc_id_sql(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { @@ -1267,7 +1267,7 @@ impl DbStorage for LightningPersister { ]; let mut conn = sqlite_connection.lock().unwrap(); let sql_transaction = conn.transaction()?; - sql_transaction.execute(&insert_or_update_payment_sql(&for_coin)?, ¶ms)?; + sql_transaction.execute(&upsert_payment_sql(&for_coin)?, ¶ms)?; sql_transaction.commit()?; Ok(()) }) @@ -1276,7 +1276,7 @@ impl DbStorage for LightningPersister { async fn get_payment_from_db(&self, hash: PaymentHash) -> Result, Self::Error> { let params = [hex::encode(hash.0)]; - let sql = select_payment_from_table_by_hash_sql(self.storage_ticker.as_str())?; + let sql = select_payment_by_hash_sql(self.storage_ticker.as_str())?; let sqlite_connection = self.sqlite_connection.clone(); async_blocking(move || { diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning_persister/src/storage.rs index a333711bdb..abf1bf4edb 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning_persister/src/storage.rs @@ -66,6 +66,7 @@ pub struct SqlChannelDetails { } impl SqlChannelDetails { + #[inline] pub fn new( rpc_id: u64, channel_id: [u8; 32], diff --git a/mm2src/coins/sql_tx_history_storage.rs b/mm2src/coins/sql_tx_history_storage.rs index 2f729b777b..bc7c2c5bea 100644 --- a/mm2src/coins/sql_tx_history_storage.rs +++ b/mm2src/coins/sql_tx_history_storage.rs @@ -86,7 +86,7 @@ fn insert_tx_in_cache_sql(for_coin: &str) -> Result> { Ok(sql) } -fn remove_tx_from_table_by_internal_id_sql(for_coin: &str) -> Result> { +fn remove_tx_by_internal_id_sql(for_coin: &str) -> Result> { let table_name = tx_history_table(for_coin); validate_table_name(&table_name)?; @@ -95,7 +95,7 @@ fn remove_tx_from_table_by_internal_id_sql(for_coin: &str) -> Result Result> { +fn select_tx_by_internal_id_sql(for_coin: &str) -> Result> { let table_name = tx_history_table(for_coin); validate_table_name(&table_name)?; @@ -292,7 +292,7 @@ impl TxHistoryStorage for SqliteTxHistoryStorage { for_coin: &str, internal_id: &BytesJson, ) -> Result> { - let sql = remove_tx_from_table_by_internal_id_sql(for_coin)?; + let sql = remove_tx_by_internal_id_sql(for_coin)?; let params = [format!("{:02x}", internal_id)]; let selfi = self.clone(); @@ -317,7 +317,7 @@ impl TxHistoryStorage for SqliteTxHistoryStorage { internal_id: &BytesJson, ) -> Result, MmError> { let params = [format!("{:02x}", internal_id)]; - let sql = select_tx_from_table_by_internal_id_sql(for_coin)?; + let sql = select_tx_by_internal_id_sql(for_coin)?; let selfi = self.clone(); async_blocking(move || { diff --git a/mm2src/mm2_bitcoin/rpc/src/v1/types/hash.rs b/mm2src/mm2_bitcoin/rpc/src/v1/types/hash.rs index 18ebb29081..21ec02a853 100644 --- a/mm2src/mm2_bitcoin/rpc/src/v1/types/hash.rs +++ b/mm2src/mm2_bitcoin/rpc/src/v1/types/hash.rs @@ -157,6 +157,7 @@ impl_hash!(H256, GlobalH256, 32); impl_hash!(H160, GlobalH160, 20); impl H256 { + #[inline] pub fn reversed(&self) -> Self { let mut result = *self; result.0.reverse(); From 88c68ea09e0d22f381b3964546a25af82e34b796 Mon Sep 17 00:00:00 2001 From: Omar Shamardy Date: Tue, 26 Apr 2022 18:06:13 +0200 Subject: [PATCH 49/49] sql_text_conversion_err, h256_slice_from_row --- Cargo.lock | 1 + mm2src/coins/lightning_persister/src/lib.rs | 40 ++++----------------- mm2src/db_common/Cargo.toml | 1 + mm2src/db_common/src/sqlite.rs | 34 ++++++++++++++++++ 4 files changed, 42 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 278a970545..0ab5764a76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1683,6 +1683,7 @@ checksum = "72aa14c04dfae8dd7d8a2b1cb7ca2152618cd01336dbfe704b8dcbf8d41dbd69" name = "db_common" version = "0.1.0" dependencies = [ + "hex 0.4.2", "log 0.4.14", "rusqlite", "sql-builder", diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs index 9e680d8a6b..7ca83b577c 100644 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ b/mm2src/coins/lightning_persister/src/lib.rs @@ -24,10 +24,10 @@ use bitcoin::hashes::hex::{FromHex, ToHex}; use bitcoin::Network; use common::fs::check_dir_operations; use common::{async_blocking, now_ms, PagingOptionsEnum}; -use db_common::sqlite::rusqlite::types::Type as SqlType; use db_common::sqlite::rusqlite::{Error as SqlError, Row, ToSql, NO_PARAMS}; use db_common::sqlite::sql_builder::SqlBuilder; -use db_common::sqlite::{offset_by_id, query_single_row, string_from_row, validate_table_name, SqliteConnShared, +use db_common::sqlite::{h256_option_slice_from_row, h256_slice_from_row, offset_by_id, query_single_row, + sql_text_conversion_err, string_from_row, validate_table_name, SqliteConnShared, CHECK_TABLE_EXISTS_SQL}; use lightning::chain; use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator}; @@ -279,46 +279,18 @@ fn payment_info_from_row(row: &Row<'_>) -> Result { let is_outbound = row.get::<_, bool>(8)?; let payment_type = if is_outbound { PaymentType::OutboundPayment { - destination: PublicKey::from_str(&row.get::<_, String>(1)?) - .map_err(|e| SqlError::FromSqlConversionFailure(1, SqlType::Text, Box::new(e)))?, + destination: PublicKey::from_str(&row.get::<_, String>(1)?).map_err(|e| sql_text_conversion_err(1, e))?, } } else { PaymentType::InboundPayment }; - let mut hash_slice = [0u8; 32]; - hex::decode_to_slice(row.get::<_, String>(0)?, &mut hash_slice as &mut [u8]) - .map_err(|e| SqlError::FromSqlConversionFailure(0, SqlType::Text, Box::new(e)))?; - let payment_hash = PaymentHash(hash_slice); - - let maybe_preimage = row.get::<_, Option>(3)?; - let preimage = match maybe_preimage { - Some(p) => { - let mut preimage_slice = [0u8; 32]; - hex::decode_to_slice(p, &mut preimage_slice as &mut [u8]) - .map_err(|e| SqlError::FromSqlConversionFailure(3, SqlType::Text, Box::new(e)))?; - Some(PaymentPreimage(preimage_slice)) - }, - None => None, - }; - - let maybe_secret = row.get::<_, Option>(4)?; - let secret = match maybe_secret { - Some(s) => { - let mut secret_slice = [0u8; 32]; - hex::decode_to_slice(s, &mut secret_slice as &mut [u8]) - .map_err(|e| SqlError::FromSqlConversionFailure(4, SqlType::Text, Box::new(e)))?; - Some(PaymentSecret(secret_slice)) - }, - None => None, - }; - let payment_info = PaymentInfo { - payment_hash, + payment_hash: PaymentHash(h256_slice_from_row::(row, 0)?), payment_type, description: row.get(2)?, - preimage, - secret, + preimage: h256_option_slice_from_row::(row, 3)?.map(PaymentPreimage), + secret: h256_option_slice_from_row::(row, 4)?.map(PaymentSecret), amt_msat: row.get::<_, Option>(5)?.map(|v| v as u64), fee_paid_msat: row.get::<_, Option>(6)?.map(|v| v as u64), status: HTLCStatus::from_str(&row.get::<_, String>(7)?)?, diff --git a/mm2src/db_common/Cargo.toml b/mm2src/db_common/Cargo.toml index bd5e18f8c1..f1c0e32921 100644 --- a/mm2src/db_common/Cargo.toml +++ b/mm2src/db_common/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +hex = "0.4.2" log = "0.4.8" uuid = { version = "0.7", features = ["serde", "v4"] } diff --git a/mm2src/db_common/src/sqlite.rs b/mm2src/db_common/src/sqlite.rs index 4e11afb1ab..ae383fad3d 100644 --- a/mm2src/db_common/src/sqlite.rs +++ b/mm2src/db_common/src/sqlite.rs @@ -2,6 +2,7 @@ pub use rusqlite; pub use sql_builder; use log::debug; +use rusqlite::types::{FromSql, Type as SqlType}; use rusqlite::{Connection, Error as SqlError, Result as SqlResult, Row, ToSql}; use sql_builder::SqlBuilder; use std::sync::{Arc, Mutex, Weak}; @@ -123,3 +124,36 @@ where let offset = maybe_offset?; Ok(Some(offset.try_into().expect("row index should be always above zero"))) } + +pub fn sql_text_conversion_err(field_id: usize, e: E) -> SqlError +where + E: std::error::Error + Send + Sync + 'static, +{ + SqlError::FromSqlConversionFailure(field_id, SqlType::Text, Box::new(e)) +} + +pub fn h256_slice_from_row(row: &Row<'_>, column_id: usize) -> Result<[u8; 32], SqlError> +where + T: AsRef<[u8]> + FromSql, +{ + let mut h256_slice = [0u8; 32]; + hex::decode_to_slice(row.get::<_, T>(column_id)?, &mut h256_slice as &mut [u8]) + .map_err(|e| sql_text_conversion_err(column_id, e))?; + Ok(h256_slice) +} + +pub fn h256_option_slice_from_row(row: &Row<'_>, column_id: usize) -> Result, SqlError> +where + T: AsRef<[u8]> + FromSql, +{ + let maybe_h256_slice = row.get::<_, Option>(column_id)?; + let res = match maybe_h256_slice { + Some(s) => { + let mut h256_slice = [0u8; 32]; + hex::decode_to_slice(s, &mut h256_slice as &mut [u8]).map_err(|e| sql_text_conversion_err(column_id, e))?; + Some(h256_slice) + }, + None => None, + }; + Ok(res) +}