feat(ckbtc): bump limit on concurrent withdrawals

rs/bitcoin/ckbtc/minter/src/lib.rs

@@ -907,9 +907,13 @@ pub async fn sign_transaction(
 
         let sighash = sighasher.sighash(input, &pkhash);
 
-        let sec1_signature =
-            management::sign_with_ecdsa(key_name.clone(), DerivationPath::new(path), sighash)
-                .await?;
+        let sec1_signature = management::sign_with_ecdsa(
+            key_name.clone(),
+            DerivationPath::new(path),
+            sighash,
+            &IC_CANISTER_RUNTIME,
+        )
+        .await?;
 
         signed_inputs.push(tx::SignedInput {
             signature: signature::EncodedSignature::from_sec1(&sec1_signature),
@@ -1247,6 +1251,13 @@ pub trait CanisterRuntime {
         to: Account,
         memo: Memo,
     ) -> Result<u64, UpdateBalanceError>;
+
+    async fn sign_with_ecdsa(
+        &self,
+        key_name: String,
+        derivation_path: DerivationPath,
+        message_hash: [u8; 32],
+    ) -> Result<Vec<u8>, CallError>;
 }
 
 #[derive(Copy, Clone)]
@@ -1294,6 +1305,29 @@ impl CanisterRuntime for IcCanisterRuntime {
     ) -> Result<u64, UpdateBalanceError> {
         updates::update_balance::mint(amount, to, memo).await
     }
+
+    async fn sign_with_ecdsa(
+        &self,
+        key_name: String,
+        derivation_path: DerivationPath,
+        message_hash: [u8; 32],
+    ) -> Result<Vec<u8>, CallError> {
+        use ic_cdk::api::management_canister::ecdsa::{
+            EcdsaCurve, EcdsaKeyId, SignWithEcdsaArgument,
+        };
+
+        ic_cdk::api::management_canister::ecdsa::sign_with_ecdsa(SignWithEcdsaArgument {
+            message_hash: message_hash.to_vec(),
+            derivation_path: derivation_path.into_inner(),
+            key_id: EcdsaKeyId {
+                curve: EcdsaCurve::Secp256k1,
+                name: key_name.clone(),
+            },
+        })
+        .await
+        .map(|(result,)| result.signature)
+        .map_err(|err| CallError::from_cdk_error("sign_with_ecdsa", err))
+    }
 }
 
 /// Time in nanoseconds since the epoch (1970-01-01).

rs/bitcoin/ckbtc/minter/src/management.rs

@@ -1,6 +1,6 @@
 //! This module contains async functions for interacting with the management canister.
 use crate::logs::P0;
-use crate::metrics::observe_get_utxos_latency;
+use crate::metrics::{observe_get_utxos_latency, observe_sign_with_ecdsa_latency};
 use crate::{tx, CanisterRuntime, ECDSAPublicKey, GetUtxosRequest, GetUtxosResponse, Network};
 use candid::{CandidType, Principal};
 use ic_btc_checker::{
@@ -168,7 +168,6 @@ pub async fn get_utxos<R: CanisterRuntime>(
         runtime.bitcoin_get_utxos(req).await
     }
 
-    // Record start time of method execution for metrics
     let start_time = runtime.time();
     let request = GetUtxosRequest {
         address: address.clone(),
@@ -261,32 +260,21 @@ pub async fn ecdsa_public_key(
 }
 
 /// Signs a message hash using the tECDSA API.
-pub async fn sign_with_ecdsa(
+pub async fn sign_with_ecdsa<R: CanisterRuntime>(
     key_name: String,
     derivation_path: DerivationPath,
     message_hash: [u8; 32],
+    runtime: &R,
 ) -> Result<Vec<u8>, CallError> {
-    use ic_cdk::api::management_canister::ecdsa::{
-        sign_with_ecdsa, EcdsaCurve, EcdsaKeyId, SignWithEcdsaArgument,
-    };
+    let start_time = runtime.time();
 
-    let result = sign_with_ecdsa(SignWithEcdsaArgument {
-        message_hash: message_hash.to_vec(),
-        derivation_path: derivation_path.into_inner(),
-        key_id: EcdsaKeyId {
-            curve: EcdsaCurve::Secp256k1,
-            name: key_name.clone(),
-        },
-    })
-    .await;
+    let result = runtime
+        .sign_with_ecdsa(key_name, derivation_path, message_hash)
+        .await;
 
-    match result {
-        Ok((reply,)) => Ok(reply.signature),
-        Err((code, msg)) => Err(CallError {
-            method: "sign_with_ecdsa".to_string(),
-            reason: Reason::from_reject(code, msg),
-        }),
-    }
+    observe_sign_with_ecdsa_latency(&result, start_time, runtime.time());
+
+    result
 }
 
 /// Check if the given Bitcoin address is blocked.

rs/bitcoin/ckbtc/minter/src/metrics.rs

@@ -7,15 +7,34 @@ use std::time::Duration;
 
 pub type NumUtxoPages = u32;
 
+#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd)]
+pub enum MetricsResult {
+    Ok,
+    Err,
+}
+
+impl MetricsResult {
+    pub fn as_str(&self) -> &str {
+        match self {
+            MetricsResult::Ok => "success",
+            MetricsResult::Err => "failure",
+        }
+    }
+}
+
 thread_local! {
     pub static GET_UTXOS_CLIENT_CALLS: Cell<u64> = Cell::default();
     pub static GET_UTXOS_MINTER_CALLS: Cell<u64> = Cell::default();
     pub static UPDATE_CALL_LATENCY: RefCell<BTreeMap<NumUtxoPages,LatencyHistogram>> = RefCell::default();
     pub static GET_UTXOS_CALL_LATENCY: RefCell<BTreeMap<(NumUtxoPages, CallSource),LatencyHistogram>> = RefCell::default();
     pub static GET_UTXOS_RESULT_SIZE: RefCell<BTreeMap<CallSource,NumUtxosHistogram>> = RefCell::default();
+    pub static SIGN_WITH_ECDSA_LATENCY: RefCell<BTreeMap<MetricsResult, LatencyHistogram>> = RefCell::default();
 }
 
-pub const BUCKETS_MS: [u64; 8] = [500, 1_000, 2_000, 4_000, 8_000, 16_000, 32_000, u64::MAX];
+pub const BUCKETS_DEFAULT_MS: [u64; 8] =
+    [500, 1_000, 2_000, 4_000, 8_000, 16_000, 32_000, u64::MAX];
+pub const BUCKETS_SIGN_WITH_ECDSA_MS: [u64; 8] =
+    [1_000, 2_000, 4_000, 6_000, 8_000, 12_000, 20_000, u64::MAX];
 pub const BUCKETS_UTXOS: [u64; 8] = [1, 4, 16, 64, 256, 1024, 4096, u64::MAX];
 
 pub struct NumUtxosHistogram(pub Histogram<8>);
@@ -30,7 +49,7 @@ pub struct LatencyHistogram(pub Histogram<8>);
 
 impl Default for LatencyHistogram {
     fn default() -> Self {
-        Self(Histogram::new(&BUCKETS_MS))
+        Self(Histogram::new(&BUCKETS_DEFAULT_MS))
     }
 }
 
@@ -128,6 +147,21 @@ pub fn observe_update_call_latency(num_new_utxos: usize, start_ns: u64, end_ns:
     });
 }
 
+pub fn observe_sign_with_ecdsa_latency<T, E>(result: &Result<T, E>, start_ns: u64, end_ns: u64) {
+    let metric_result = match result {
+        Ok(_) => MetricsResult::Ok,
+        Err(_) => MetricsResult::Err,
+    };
+    SIGN_WITH_ECDSA_LATENCY.with_borrow_mut(|metrics| {
+        metrics
+            .entry(metric_result)
+            .or_insert(LatencyHistogram(Histogram::new(
+                &BUCKETS_SIGN_WITH_ECDSA_MS,
+            )))
+            .observe_latency(start_ns, end_ns);
+    });
+}
+
 pub fn encode_metrics(
     metrics: &mut ic_metrics_encoder::MetricsEncoder<Vec<u8>>,
 ) -> std::io::Result<()> {
@@ -392,6 +426,22 @@ pub fn encode_metrics(
         Ok(())
     })?;
 
+    let mut histogram_vec = metrics.histogram_vec(
+        "ckbtc_minter_sign_with_ecdsa_latency",
+        "The latency of ckBTC minter `sign_with_ecdsa` calls in milliseconds.",
+    )?;
+
+    SIGN_WITH_ECDSA_LATENCY.with_borrow(|histograms| -> Result<(), Error> {
+        for (result, histogram) in histograms {
+            histogram_vec = histogram_vec.histogram(
+                &[("result", result.as_str())],
+                histogram.0.iter(),
+                histogram.0.sum() as f64,
+            )?;
+        }
+        Ok(())
+    })?;
+
     Ok(())
 }
 

rs/bitcoin/ckbtc/minter/src/test_fixtures.rs

@@ -117,6 +117,7 @@ pub mod mock {
     use candid::Principal;
     use ic_btc_checker::CheckTransactionResponse;
     use ic_btc_interface::Utxo;
+    use ic_management_canister_types_private::DerivationPath;
     use icrc_ledger_types::icrc1::account::Account;
     use icrc_ledger_types::icrc1::transfer::Memo;
     use mockall::mock;
@@ -134,6 +135,7 @@ pub mod mock {
             async fn bitcoin_get_utxos(&self, request: GetUtxosRequest) -> Result<GetUtxosResponse, CallError>;
             async fn check_transaction(&self, btc_checker_principal: Principal, utxo: &Utxo, cycle_payment: u128, ) -> Result<CheckTransactionResponse, CallError>;
             async fn mint_ckbtc(&self, amount: u64, to: Account, memo: Memo) -> Result<u64, UpdateBalanceError>;
+            async fn sign_with_ecdsa(&self, key_name: String, derivation_path: DerivationPath, message_hash: [u8; 32]) -> Result<Vec<u8>, CallError>;
         }
     }
 }

rs/bitcoin/ckbtc/minter/src/updates/retrieve_btc.rs

@@ -22,7 +22,7 @@ use icrc_ledger_types::icrc1::transfer::{TransferArg, TransferError};
 use icrc_ledger_types::icrc2::transfer_from::{TransferFromArgs, TransferFromError};
 use num_traits::cast::ToPrimitive;
 
-const MAX_CONCURRENT_PENDING_REQUESTS: usize = 1000;
+const MAX_CONCURRENT_PENDING_REQUESTS: usize = 5000;
 
 /// The arguments of the [retrieve_btc] endpoint.
 #[derive(Clone, Eq, PartialEq, Debug, CandidType, Deserialize)]

rs/bitcoin/ckbtc/minter/src/updates/tests.rs

@@ -1,7 +1,7 @@
 mod update_balance {
-    use crate::management::{get_utxos, CallError, CallSource};
-    use crate::metrics::LatencyHistogram;
+    use crate::management::{get_utxos, sign_with_ecdsa, CallError, CallSource};
     use crate::metrics::{Histogram, NumUtxoPages};
+    use crate::metrics::{LatencyHistogram, MetricsResult};
     use crate::state::{audit, eventlog::EventType, mutate_state, read_state, SuspendedReason};
     use crate::test_fixtures::{
         ecdsa_public_key, get_uxos_response, ignored_utxo, init_args, init_state, ledger_account,
@@ -16,6 +16,8 @@ mod update_balance {
     use crate::{storage, GetUtxosResponse, Timestamp};
     use ic_btc_checker::CheckTransactionResponse;
     use ic_btc_interface::{Page, Utxo};
+    use ic_cdk::api::call::RejectionCode;
+    use ic_management_canister_types_private::BoundedVec;
     use icrc_ledger_types::icrc1::account::Account;
     use std::iter;
     use std::time::Duration;
@@ -510,6 +512,90 @@ mod update_balance {
         })
     }
 
+    #[tokio::test]
+    async fn should_observe_sign_with_ecdsa_metrics() {
+        init_state_with_ecdsa_public_key();
+
+        async fn sign_with_ecdsa_with_latency(
+            latency: Duration,
+            result: MetricsResult,
+        ) -> Result<Vec<u8>, CallError> {
+            let key_name = "test_key".to_string();
+            let derivation_path = BoundedVec::new(vec![]);
+            let message_hash = [0u8; 32];
+
+            let mut runtime = MockCanisterRuntime::new();
+
+            mock_increasing_time(&mut runtime, NOW, latency);
+
+            let mock_result = match result {
+                MetricsResult::Ok => Ok(vec![]),
+                MetricsResult::Err => Err(CallError::from_cdk_error(
+                    "sign_with_ecdsa",
+                    (RejectionCode::Unknown, "mock error".to_string()),
+                )),
+            };
+            runtime.expect_sign_with_ecdsa().return_const(mock_result);
+
+            sign_with_ecdsa(key_name, derivation_path, message_hash, &runtime).await
+        }
+
+        let sign_with_ecdsa_ms = [
+            500, 1_000, 1_250, 2_500, 3_250, 4_000, 8_000, 15_000, 50_000,
+        ];
+        for millis in &sign_with_ecdsa_ms {
+            let result =
+                sign_with_ecdsa_with_latency(Duration::from_millis(*millis), MetricsResult::Ok)
+                    .await;
+            assert!(result.is_ok());
+        }
+
+        let result =
+            sign_with_ecdsa_with_latency(Duration::from_millis(5_000), MetricsResult::Err).await;
+        assert!(result.is_err());
+
+        let histogram = sign_with_ecdsa_histogram(MetricsResult::Ok);
+        assert_eq!(
+            histogram.iter().collect::<Vec<_>>(),
+            vec![
+                (1_000., 2.),
+                (2_000., 1.),
+                (4_000., 3.),
+                (6_000., 0.),
+                (8_000., 1.),
+                (12_000., 0.),
+                (20_000., 1.),
+                (f64::INFINITY, 1.)
+            ]
+        );
+        assert_eq!(histogram.sum(), sign_with_ecdsa_ms.iter().sum::<u64>());
+
+        let histogram = sign_with_ecdsa_histogram(MetricsResult::Err);
+        assert_eq!(
+            histogram.iter().collect::<Vec<_>>(),
+            vec![
+                (1_000., 0.),
+                (2_000., 0.),
+                (4_000., 0.),
+                (6_000., 1.),
+                (8_000., 0.),
+                (12_000., 0.),
+                (20_000., 0.),
+                (f64::INFINITY, 0.)
+            ]
+        );
+        assert_eq!(histogram.sum(), 5_000);
+    }
+
+    fn sign_with_ecdsa_histogram(result: MetricsResult) -> Histogram<8> {
+        crate::metrics::SIGN_WITH_ECDSA_LATENCY.with_borrow(|histograms| {
+            let &LatencyHistogram(histogram) = histograms
+                .get(&result)
+                .expect("No histogram for given metric result");
+            histogram
+        })
+    }
+
     async fn test_suspended_utxo_last_time_checked_timestamp(utxo: Utxo, reason: SuspendedReason) {
         init_state_with_ecdsa_public_key();
         let account = ledger_account();

rs/bitcoin/ckbtc/minter/tests/tests.rs

@@ -1969,6 +1969,15 @@ fn test_retrieve_btc_with_approval() {
 
     ckbtc.finalize_transaction(tx);
     assert_eq!(ckbtc.await_finalization(block_index, 10), txid);
+
+    ckbtc
+        .check_minter_metrics()
+        .assert_contains_metric_matching(
+            r#"ckbtc_minter_sign_with_ecdsa_latency_bucket\{result="success",le="(\d+|\+Inf)"\} 1 \d+"#,
+        )
+        .assert_does_not_contain_metric_matching(
+            r#"ckbtc_minter_sign_with_ecdsa_latency_bucket\{result="failure",le="(\d+|\+Inf)"\} 1 \d+"#
+        );
 }
 
 #[test]
@@ -2074,6 +2083,15 @@ fn test_retrieve_btc_with_approval_from_subaccount() {
             status_v2: Some(ckbtc.retrieve_btc_status_v2(block_index))
         }]
     );
+
+    ckbtc
+        .check_minter_metrics()
+        .assert_contains_metric_matching(
+            r#"ckbtc_minter_sign_with_ecdsa_latency_bucket\{result="success",le="(\d+|\+Inf)"\} 1 \d+"#,
+        )
+        .assert_does_not_contain_metric_matching(
+            r#"ckbtc_minter_sign_with_ecdsa_latency_bucket\{result="failure",le="(\d+|\+Inf)"\} 1 \d+"#
+        );
 }
 
 #[test]