openzeppelin_relayer/domain/transaction/evm/
status.rs

1//! This module contains the status-related functionality for EVM transactions.
2//! It includes methods for checking transaction status, determining when to resubmit
3//! or replace transactions with NOOPs, and updating transaction status in the repository.
4
5use alloy::network::ReceiptResponse;
6use chrono::{DateTime, Duration, Utc};
7use eyre::Result;
8use tracing::{debug, error, warn};
9
10use super::super::common::is_active_nonce_status;
11use super::EvmRelayerTransaction;
12use super::{
13    ensure_status, evm_transaction::TX_NONCE_RECONCILE_TRIGGER, get_age_since_status_change,
14    has_enough_confirmations, is_noop, is_too_early_to_resubmit, is_transaction_valid, make_noop,
15    too_many_attempts, too_many_noop_attempts,
16};
17use crate::constants::{
18    get_evm_min_age_for_hash_recovery, get_evm_pending_recovery_trigger_timeout,
19    get_evm_prepare_timeout, get_evm_resend_timeout, ARBITRUM_TIME_TO_RESUBMIT,
20    EVM_MIN_HASHES_FOR_RECOVERY, MAX_GAP_SCAN_RANGE,
21};
22use crate::domain::transaction::common::{
23    get_age_of_sent_at, is_final_state, is_pending_transaction,
24};
25use crate::domain::transaction::util::get_age_since_created;
26use crate::models::{EvmNetwork, NetworkRepoModel, NetworkType};
27use crate::repositories::{NetworkRepository, RelayerRepository};
28use crate::{
29    domain::transaction::evm::price_calculator::PriceCalculatorTrait,
30    jobs::{JobProducerTrait, StatusCheckContext},
31    models::{
32        NetworkTransactionData, RelayerRepoModel, TransactionError, TransactionRepoModel,
33        TransactionStatus, TransactionUpdateRequest,
34    },
35    repositories::{Repository, TransactionCounterTrait, TransactionRepository},
36    services::{provider::EvmProviderTrait, signer::Signer},
37    utils::{get_resubmit_timeout_for_speed, get_resubmit_timeout_with_backoff},
38};
39
40impl<P, RR, NR, TR, J, S, TCR, PC> EvmRelayerTransaction<P, RR, NR, TR, J, S, TCR, PC>
41where
42    P: EvmProviderTrait + Send + Sync,
43    RR: RelayerRepository + Repository<RelayerRepoModel, String> + Send + Sync + 'static,
44    NR: NetworkRepository + Repository<NetworkRepoModel, String> + Send + Sync + 'static,
45    TR: TransactionRepository + Repository<TransactionRepoModel, String> + Send + Sync + 'static,
46    J: JobProducerTrait + Send + Sync + 'static,
47    S: Signer + Send + Sync + 'static,
48    TCR: TransactionCounterTrait + Send + Sync + 'static,
49    PC: PriceCalculatorTrait + Send + Sync,
50{
51    pub(super) async fn check_transaction_status(
52        &self,
53        tx: &TransactionRepoModel,
54    ) -> Result<TransactionStatus, TransactionError> {
55        // Early return if transaction is already in a final state
56        if is_final_state(&tx.status) {
57            return Ok(tx.status.clone());
58        }
59
60        // Early return for Pending/Sent states - these are DB-only states
61        // that don't require on-chain queries and may not have a hash yet
62        match tx.status {
63            TransactionStatus::Pending | TransactionStatus::Sent => {
64                return Ok(tx.status.clone());
65            }
66            _ => {}
67        }
68
69        let evm_data = tx.network_data.get_evm_transaction_data()?;
70        let tx_hash = evm_data
71            .hash
72            .as_ref()
73            .ok_or(TransactionError::UnexpectedError(
74                "Transaction hash is missing".to_string(),
75            ))?;
76
77        let receipt_result = self.provider().get_transaction_receipt(tx_hash).await?;
78
79        if let Some(receipt) = receipt_result {
80            if !receipt.inner.status() {
81                return Ok(TransactionStatus::Failed);
82            }
83            let last_block_number = self.provider().get_block_number().await?;
84            let tx_block_number = receipt
85                .block_number
86                .ok_or(TransactionError::UnexpectedError(
87                    "Transaction receipt missing block number".to_string(),
88                ))?;
89
90            let network_model = self
91                .network_repository()
92                .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
93                .await?
94                .ok_or(TransactionError::UnexpectedError(format!(
95                    "Network with chain id {} not found",
96                    evm_data.chain_id
97                )))?;
98
99            let network = EvmNetwork::try_from(network_model).map_err(|e| {
100                TransactionError::UnexpectedError(format!(
101                    "Error converting network model to EvmNetwork: {e}"
102                ))
103            })?;
104
105            if !has_enough_confirmations(
106                tx_block_number,
107                last_block_number,
108                network.required_confirmations,
109            ) {
110                debug!(
111                    tx_id = %tx.id,
112                    relayer_id = %tx.relayer_id,
113                    tx_hash = %tx_hash,
114                    "transaction mined but not confirmed"
115                );
116                return Ok(TransactionStatus::Mined);
117            }
118            // For a user-cancelled transaction, attribute the terminal status by what
119            // ACTUALLY mined. Cancellation replaces the tx with a self-send NOOP
120            // (to == from). If that NOOP is what confirmed, the transaction was truly
121            // cancelled. If the original mined instead (the cancellation lost the nonce
122            // race), it executed for real and must be reported Confirmed, not Canceled.
123            // We key off the mined receipt rather than the locally-stored network_data,
124            // which may still describe the NOOP even while the original hash is what
125            // confirmed (e.g. before the replacement has been broadcast).
126            if tx.is_canceled == Some(true) {
127                if receipt.to == Some(receipt.from) {
128                    debug!(
129                        tx_id = %tx.id,
130                        relayer_id = %tx.relayer_id,
131                        tx_hash = %tx_hash,
132                        "cancellation NOOP confirmed on-chain; marking transaction as Canceled"
133                    );
134                    return Ok(TransactionStatus::Canceled);
135                }
136                debug!(
137                    tx_id = %tx.id,
138                    relayer_id = %tx.relayer_id,
139                    tx_hash = %tx_hash,
140                    "cancelled transaction's original mined before the NOOP could replace it; reporting Confirmed"
141                );
142            }
143            Ok(TransactionStatus::Confirmed)
144        } else {
145            debug!(
146                tx_id = %tx.id,
147                relayer_id = %tx.relayer_id,
148                tx_hash = %tx_hash,
149                "transaction not yet mined"
150            );
151
152            // FALLBACK: Try to find transaction by checking all historical hashes
153            // Only do this for transactions that have multiple resubmission attempts
154            // and have been stuck in Submitted for a while
155            if tx.hashes.len() > 1 && self.should_try_hash_recovery(tx)? {
156                if let Some(recovered_tx) = self
157                    .try_recover_with_historical_hashes(tx, &evm_data)
158                    .await?
159                {
160                    // Return the status from the recovered (updated) transaction
161                    return Ok(recovered_tx.status);
162                }
163            }
164
165            Ok(TransactionStatus::Submitted)
166        }
167    }
168
169    /// Determines if a transaction should be resubmitted.
170    pub(super) async fn should_resubmit(
171        &self,
172        tx: &TransactionRepoModel,
173    ) -> Result<bool, TransactionError> {
174        // Validate transaction is in correct state for resubmission
175        ensure_status(tx, TransactionStatus::Submitted, Some("should_resubmit"))?;
176
177        let evm_data = tx.network_data.get_evm_transaction_data()?;
178        let age = get_age_of_sent_at(tx)?;
179
180        // Check if network lacks mempool and determine appropriate timeout
181        let network_model = self
182            .network_repository()
183            .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
184            .await?
185            .ok_or(TransactionError::UnexpectedError(format!(
186                "Network with chain id {} not found",
187                evm_data.chain_id
188            )))?;
189
190        let network = EvmNetwork::try_from(network_model).map_err(|e| {
191            TransactionError::UnexpectedError(format!(
192                "Error converting network model to EvmNetwork: {e}"
193            ))
194        })?;
195
196        let timeout = match network.is_arbitrum() {
197            true => ARBITRUM_TIME_TO_RESUBMIT,
198            false => get_resubmit_timeout_for_speed(&evm_data.speed),
199        };
200
201        let timeout_with_backoff = match network.is_arbitrum() {
202            true => timeout, // Use base timeout without backoff for Arbitrum
203            false => get_resubmit_timeout_with_backoff(timeout, tx.hashes.len()),
204        };
205
206        if age > Duration::milliseconds(timeout_with_backoff) {
207            debug!(
208                tx_id = %tx.id,
209                relayer_id = %tx.relayer_id,
210                age_ms = %age.num_milliseconds(),
211                "transaction has been pending for too long, resubmitting"
212            );
213            return Ok(true);
214        }
215        Ok(false)
216    }
217
218    /// Determines if a transaction should be replaced with a NOOP transaction.
219    ///
220    /// Returns a tuple `(should_noop, reason)` where:
221    /// - `should_noop`: `true` if transaction should be replaced with NOOP
222    /// - `reason`: Optional reason string explaining why NOOP is needed (only set when `should_noop` is `true`)
223    ///
224    /// # Arguments
225    ///
226    /// * `tx` - The transaction to check
227    pub(super) async fn should_noop(
228        &self,
229        tx: &TransactionRepoModel,
230    ) -> Result<(bool, Option<String>), TransactionError> {
231        if too_many_noop_attempts(tx) {
232            debug!("Transaction has too many NOOP attempts already");
233            return Ok((false, None));
234        }
235
236        let evm_data = tx.network_data.get_evm_transaction_data()?;
237        if is_noop(&evm_data) {
238            return Ok((false, None));
239        }
240
241        let network_model = self
242            .network_repository()
243            .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
244            .await?
245            .ok_or(TransactionError::UnexpectedError(format!(
246                "Network with chain id {} not found",
247                evm_data.chain_id
248            )))?;
249
250        let network = EvmNetwork::try_from(network_model).map_err(|e| {
251            TransactionError::UnexpectedError(format!(
252                "Error converting network model to EvmNetwork: {e}"
253            ))
254        })?;
255
256        if network.is_rollup() && too_many_attempts(tx) {
257            let reason =
258                "Rollup transaction has too many attempts. Replacing with NOOP.".to_string();
259            debug!(
260                tx_id = %tx.id,
261                relayer_id = %tx.relayer_id,
262                reason = %reason,
263                "replacing transaction with NOOP"
264            );
265            return Ok((true, Some(reason)));
266        }
267
268        if !is_transaction_valid(&tx.created_at, &tx.valid_until) {
269            let reason = "Transaction is expired. Replacing with NOOP.".to_string();
270            debug!(
271                tx_id = %tx.id,
272                relayer_id = %tx.relayer_id,
273                reason = %reason,
274                "replacing transaction with NOOP"
275            );
276            return Ok((true, Some(reason)));
277        }
278
279        if tx.status == TransactionStatus::Pending {
280            let created_at = &tx.created_at;
281            let created_time = DateTime::parse_from_rfc3339(created_at)
282                .map_err(|e| {
283                    TransactionError::UnexpectedError(format!("Invalid created_at timestamp: {e}"))
284                })?
285                .with_timezone(&Utc);
286            let age = Utc::now().signed_duration_since(created_time);
287            if age > get_evm_prepare_timeout() {
288                let reason = format!(
289                    "Transaction in Pending state for over {} minutes. Replacing with NOOP.",
290                    get_evm_prepare_timeout().num_minutes()
291                );
292                debug!(
293                    tx_id = %tx.id,
294                    relayer_id = %tx.relayer_id,
295                    reason = %reason,
296                    "replacing transaction with NOOP"
297                );
298                return Ok((true, Some(reason)));
299            }
300        }
301
302        let latest_block = self.provider().get_block_by_number().await;
303        if let Ok(block) = latest_block {
304            let block_gas_limit = block.header.gas_limit;
305            if let Some(gas_limit) = evm_data.gas_limit {
306                if gas_limit > block_gas_limit {
307                    let reason = format!(
308                                "Transaction gas limit ({gas_limit}) exceeds block gas limit ({block_gas_limit}). Replacing with NOOP.",
309                            );
310                    warn!(
311                        tx_id = %tx.id,
312                        tx_gas_limit = %gas_limit,
313                        block_gas_limit = %block_gas_limit,
314                        "transaction gas limit exceeds block gas limit, replacing with NOOP"
315                    );
316                    return Ok((true, Some(reason)));
317                }
318            }
319        }
320
321        Ok((false, None))
322    }
323
324    /// Helper method that updates transaction status only if it's different from the current status.
325    pub(super) async fn update_transaction_status_if_needed(
326        &self,
327        tx: TransactionRepoModel,
328        new_status: TransactionStatus,
329        status_reason: Option<String>,
330    ) -> Result<TransactionRepoModel, TransactionError> {
331        if tx.status != new_status {
332            return self
333                .update_transaction_status(tx, new_status, status_reason)
334                .await;
335        }
336        Ok(tx)
337    }
338
339    /// Prepares a NOOP transaction update request.
340    pub(super) async fn prepare_noop_update_request(
341        &self,
342        tx: &TransactionRepoModel,
343        is_cancellation: bool,
344        reason: Option<String>,
345    ) -> Result<TransactionUpdateRequest, TransactionError> {
346        let mut evm_data = tx.network_data.get_evm_transaction_data()?;
347        let network_model = self
348            .network_repository()
349            .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
350            .await?
351            .ok_or(TransactionError::UnexpectedError(format!(
352                "Network with chain id {} not found",
353                evm_data.chain_id
354            )))?;
355
356        let network = EvmNetwork::try_from(network_model).map_err(|e| {
357            TransactionError::UnexpectedError(format!(
358                "Error converting network model to EvmNetwork: {e}"
359            ))
360        })?;
361
362        make_noop(&mut evm_data, &network, Some(self.provider())).await?;
363
364        let noop_count = tx.noop_count.unwrap_or(0) + 1;
365        let update_request = TransactionUpdateRequest {
366            network_data: Some(NetworkTransactionData::Evm(evm_data)),
367            noop_count: Some(noop_count),
368            status_reason: reason,
369            is_canceled: if is_cancellation {
370                Some(true)
371            } else {
372                tx.is_canceled
373            },
374            ..Default::default()
375        };
376        Ok(update_request)
377    }
378
379    /// Handles transactions in the Submitted state.
380    ///
381    /// Before resubmitting, checks whether the transaction's nonce is ahead of
382    /// the on-chain nonce. If so, the tx can never mine because there's a gap
383    /// below it — schedules a nonce health job to fill the gap instead of
384    /// resubmitting (which would be futile).
385    async fn handle_submitted_state(
386        &self,
387        tx: TransactionRepoModel,
388    ) -> Result<TransactionRepoModel, TransactionError> {
389        if self.should_resubmit(&tx).await? {
390            // Before resubmitting, check if there's a nonce gap blocking this tx.
391            // Only worth the RPC call when we're already going to resubmit (tx is stale).
392            if let Some(nonce_gap_detected) = self.detect_nonce_gap_ahead(&tx).await {
393                if nonce_gap_detected {
394                    // Tx can't mine — nonce gap below it. Trigger health job, skip resubmit.
395                    return self
396                        .update_transaction_status_if_needed(tx, TransactionStatus::Submitted, None)
397                        .await;
398                }
399            }
400
401            let resubmitted_tx = self.handle_resubmission(tx).await?;
402            return Ok(resubmitted_tx);
403        }
404
405        self.update_transaction_status_if_needed(tx, TransactionStatus::Submitted, None)
406            .await
407    }
408
409    /// Checks whether the tx is blocked by a nonce gap below it.
410    ///
411    /// 1. Fetches on-chain nonce (single RPC call).
412    /// 2. If `tx_nonce <= on_chain_nonce` — no gap, tx should be mineable.
413    /// 3. Otherwise scans `on_chain_nonce..tx_nonce` using the Redis nonce index
414    ///    to check if every slot has an active (Pending/Sent/Submitted/Mined) tx.
415    /// 4. If any slot is empty or has a terminal-status tx — that's a real gap.
416    ///    Schedules a nonce health job and returns `Some(true)`.
417    ///
418    /// Returns:
419    /// - `Some(true)` — gap confirmed, health job scheduled
420    /// - `Some(false)` — no gap (all slots filled or tx is next)
421    /// - `None` — couldn't determine (missing nonce, RPC/Redis error)
422    async fn detect_nonce_gap_ahead(&self, tx: &TransactionRepoModel) -> Option<bool> {
423        let evm_data = match tx.network_data.get_evm_transaction_data() {
424            Ok(d) => d,
425            Err(_) => return None,
426        };
427        let tx_nonce = match evm_data.nonce {
428            Some(n) => n,
429            None => return None,
430        };
431
432        let on_chain_nonce = match self
433            .provider()
434            .get_transaction_count(&self.relayer().address)
435            .await
436        {
437            Ok(n) => n,
438            Err(e) => {
439                debug!(
440                    tx_id = %tx.id,
441                    error = %e,
442                    "nonce gap check: failed to get on-chain nonce, skipping"
443                );
444                return None;
445            }
446        };
447
448        // tx is the next expected nonce or already behind — no gap possible.
449        if tx_nonce <= on_chain_nonce {
450            return Some(false);
451        }
452
453        // Cap the scan to avoid an unbounded MGET if tx_nonce is very far ahead.
454        // If the gap is larger, we scan what we can — the health job's
455        // detect_nonce_gaps will use the nonce hint to extend its own scan.
456        let scan_to = std::cmp::min(tx_nonce, on_chain_nonce + MAX_GAP_SCAN_RANGE);
457
458        let occupancy = match self
459            .transaction_repository()
460            .get_nonce_occupancy(&tx.relayer_id, on_chain_nonce, scan_to)
461            .await
462        {
463            Ok(o) => o,
464            Err(e) => {
465                debug!(
466                    tx_id = %tx.id,
467                    error = %e,
468                    "nonce gap check: occupancy lookup failed, skipping"
469                );
470                return None;
471            }
472        };
473
474        let gap_nonces: Vec<u64> = occupancy
475            .into_iter()
476            .filter(|(_, status)| !status.as_ref().is_some_and(is_active_nonce_status))
477            .map(|(nonce, _)| nonce)
478            .collect();
479
480        if gap_nonces.is_empty() {
481            // All slots between on-chain and tx_nonce are actively filled — no gap.
482            return Some(false);
483        }
484
485        warn!(
486            tx_id = %tx.id,
487            relayer_id = %tx.relayer_id,
488            tx_nonce = tx_nonce,
489            on_chain_nonce = on_chain_nonce,
490            gap_count = gap_nonces.len(),
491            gaps = ?gap_nonces,
492            "nonce gaps confirmed below tx, scheduling nonce health to fill"
493        );
494
495        if let Err(e) = self.schedule_relayer_nonce_health_job(tx).await {
496            warn!(
497                tx_id = %tx.id,
498                error = %e,
499                "failed to schedule nonce health job for nonce gap"
500            );
501        }
502
503        Some(true)
504    }
505
506    /// Processes transaction resubmission logic
507    async fn handle_resubmission(
508        &self,
509        tx: TransactionRepoModel,
510    ) -> Result<TransactionRepoModel, TransactionError> {
511        debug!(
512            tx_id = %tx.id,
513            relayer_id = %tx.relayer_id,
514            status = ?tx.status,
515            "scheduling resubmit job for transaction"
516        );
517
518        // Check if transaction gas limit exceeds block gas limit before resubmitting
519        let (should_noop, reason) = self.should_noop(&tx).await?;
520        let tx_to_process = if should_noop {
521            self.process_noop_transaction(&tx, reason).await?
522        } else {
523            tx
524        };
525
526        self.send_transaction_resubmit_job(&tx_to_process).await?;
527        Ok(tx_to_process)
528    }
529
530    /// Handles NOOP transaction processing before resubmission
531    async fn process_noop_transaction(
532        &self,
533        tx: &TransactionRepoModel,
534        reason: Option<String>,
535    ) -> Result<TransactionRepoModel, TransactionError> {
536        debug!(
537            tx_id = %tx.id,
538            relayer_id = %tx.relayer_id,
539            status = ?tx.status,
540            "preparing transaction NOOP before resubmission"
541        );
542        let update = self.prepare_noop_update_request(tx, false, reason).await?;
543        let updated_tx = self
544            .transaction_repository()
545            .partial_update(tx.id.clone(), update)
546            .await?;
547
548        let res = self.send_transaction_update_notification(&updated_tx).await;
549        if let Err(e) = res {
550            error!(
551                tx_id = %updated_tx.id,
552                relayer_id = %updated_tx.relayer_id,
553                status = ?updated_tx.status,
554                error = %e,
555                "sending transaction update notification failed for NOOP transaction"
556            );
557        }
558        Ok(updated_tx)
559    }
560
561    /// Handles transactions in the Pending state.
562    async fn handle_pending_state(
563        &self,
564        tx: TransactionRepoModel,
565    ) -> Result<TransactionRepoModel, TransactionError> {
566        let (should_noop, reason) = self.should_noop(&tx).await?;
567        if should_noop {
568            // For Pending state transactions, nonces are not yet assigned, so we mark as Failed
569            // instead of NOOP. This matches prepare_transaction behavior.
570            debug!(
571                tx_id = %tx.id,
572                relayer_id = %tx.relayer_id,
573                reason = %reason.as_ref().unwrap_or(&"unknown".to_string()),
574                "marking pending transaction as Failed (nonce not assigned, no NOOP needed)"
575            );
576            let update = TransactionUpdateRequest {
577                status: Some(TransactionStatus::Failed),
578                status_reason: reason,
579                ..Default::default()
580            };
581            let updated_tx = self
582                .transaction_repository()
583                .partial_update(tx.id.clone(), update)
584                .await?;
585
586            let res = self.send_transaction_update_notification(&updated_tx).await;
587            if let Err(e) = res {
588                error!(
589                    tx_id = %updated_tx.id,
590                    relayer_id = %updated_tx.relayer_id,
591                    status = ?updated_tx.status,
592                    error = %e,
593                    "sending transaction update notification failed for Pending state NOOP"
594                );
595            }
596            return Ok(updated_tx);
597        }
598
599        // Check if transaction is stuck in Pending (prepare job may have failed)
600        let age = get_age_since_created(&tx)?;
601        if age > get_evm_pending_recovery_trigger_timeout() {
602            warn!(
603                tx_id = %tx.id,
604                relayer_id = %tx.relayer_id,
605                age_seconds = age.num_seconds(),
606                "transaction stuck in Pending, queuing prepare job"
607            );
608
609            // Re-queue prepare job
610            self.send_transaction_request_job(&tx).await?;
611        }
612
613        Ok(tx)
614    }
615
616    /// Handles transactions in the Mined state.
617    async fn handle_mined_state(
618        &self,
619        tx: TransactionRepoModel,
620    ) -> Result<TransactionRepoModel, TransactionError> {
621        self.update_transaction_status_if_needed(tx, TransactionStatus::Mined, None)
622            .await
623    }
624
625    /// Handles transactions in final states (Confirmed, Failed, Expired).
626    async fn handle_final_state(
627        &self,
628        tx: TransactionRepoModel,
629        status: TransactionStatus,
630        status_reason: Option<String>,
631    ) -> Result<TransactionRepoModel, TransactionError> {
632        self.update_transaction_status_if_needed(tx, status, status_reason)
633            .await
634    }
635
636    /// Marks a transaction as Failed with a given reason.
637    async fn mark_as_failed(
638        &self,
639        tx: TransactionRepoModel,
640        reason: String,
641    ) -> Result<TransactionRepoModel, TransactionError> {
642        warn!(
643            tx_id = %tx.id,
644            relayer_id = %tx.relayer_id,
645            reason = %reason,
646            "force-failing transaction due to circuit breaker"
647        );
648
649        let update = TransactionUpdateRequest {
650            status: Some(TransactionStatus::Failed),
651            status_reason: Some(reason),
652            ..Default::default()
653        };
654
655        let updated_tx = self
656            .transaction_repository()
657            .partial_update(tx.id.clone(), update)
658            .await?;
659
660        // Send notification (best effort)
661        if let Err(e) = self.send_transaction_update_notification(&updated_tx).await {
662            error!(
663                tx_id = %updated_tx.id,
664                relayer_id = %updated_tx.relayer_id,
665                error = %e,
666                "failed to send notification for force-failed transaction"
667            );
668        }
669
670        Ok(updated_tx)
671    }
672
673    /// Reconciles a single transaction's nonce state against on-chain reality.
674    ///
675    /// This is the fast-path reconciliation triggered by nonce errors during submission.
676    /// It checks:
677    /// 1. Receipt for current tx hash — if found, defers to normal flow
678    /// 2. Historical hash recovery — if a different hash was mined, updates the tx
679    /// 3. On-chain nonce comparison — if the nonce was consumed externally, marks Failed
680    ///
681    /// Returns `Some(tx)` if recovery handled the transaction (caller should return early),
682    /// or `None` to continue with normal status flow.
683    async fn reconcile_tx_nonce_state(
684        &self,
685        tx: &TransactionRepoModel,
686    ) -> Result<Option<TransactionRepoModel>, TransactionError> {
687        let evm_data = tx.network_data.get_evm_transaction_data()?;
688
689        // Track whether any RPC call failed transiently. If so, we must NOT
690        // make the irreversible "consumed externally" determination in step 3,
691        // because the tx could be mined under a hash we failed to check.
692        let mut had_rpc_errors = false;
693
694        // 1. Check receipt for current tx hash — if found, normal flow handles it
695        if let Some(ref hash) = evm_data.hash {
696            match self.provider().get_transaction_receipt(hash).await {
697                Ok(Some(_)) => {
698                    debug!(
699                        tx_id = %tx.id,
700                        hash = %hash,
701                        "nonce recovery: receipt found for current hash, deferring to normal flow"
702                    );
703                    return Ok(None);
704                }
705                Ok(None) => {
706                    // No receipt for current hash — continue recovery
707                }
708                Err(e) => {
709                    warn!(
710                        tx_id = %tx.id,
711                        hash = %hash,
712                        error = %e,
713                        "nonce recovery: error checking receipt for current hash"
714                    );
715                    had_rpc_errors = true;
716                }
717            }
718        }
719
720        // 2. Try historical hash recovery (reuse existing method)
721        if tx.hashes.len() > 1 {
722            match self.try_recover_with_historical_hashes(tx, &evm_data).await {
723                Ok(Some(recovered_tx)) => {
724                    debug!(
725                        tx_id = %tx.id,
726                        "nonce recovery: recovered transaction via historical hash"
727                    );
728                    return Ok(Some(recovered_tx));
729                }
730                Ok(None) => {
731                    // No historical hash found — continue
732                }
733                Err(e) => {
734                    warn!(
735                        tx_id = %tx.id,
736                        error = %e,
737                        "nonce recovery: error during historical hash recovery"
738                    );
739                    had_rpc_errors = true;
740                }
741            }
742        }
743
744        // 3. Compare on-chain nonce to determine if nonce was consumed externally.
745        //    Only safe to make this determination if all hash checks succeeded.
746        //    If any RPC call failed, the tx might be mined under a hash we couldn't check.
747        if had_rpc_errors {
748            warn!(
749                tx_id = %tx.id,
750                "nonce recovery: skipping nonce comparison due to RPC errors during hash checks, deferring to normal flow"
751            );
752            return Ok(None);
753        }
754
755        let tx_nonce = match evm_data.nonce {
756            Some(n) => n,
757            None => {
758                // No nonce assigned — can't compare, defer to normal flow
759                return Ok(None);
760            }
761        };
762
763        let on_chain_nonce = self
764            .provider()
765            .get_transaction_count(&self.relayer().address)
766            .await
767            .map_err(|e| {
768                TransactionError::UnexpectedError(format!(
769                    "Failed to get on-chain nonce for recovery: {e}"
770                ))
771            })?;
772
773        if on_chain_nonce > tx_nonce {
774            // Nonce was consumed but no known hash found — consumed externally
775            let reason = format!(
776                "Nonce {tx_nonce} consumed externally (on-chain nonce: {on_chain_nonce}). \
777                 No matching transaction hash found on-chain."
778            );
779            warn!(
780                tx_id = %tx.id,
781                relayer_id = %tx.relayer_id,
782                tx_nonce = tx_nonce,
783                on_chain_nonce = on_chain_nonce,
784                "nonce recovery: nonce consumed externally, marking as Failed"
785            );
786
787            let updated_tx = self
788                .update_transaction_status(tx.clone(), TransactionStatus::Failed, Some(reason))
789                .await?;
790
791            // External nonce consumption may have left the internal transaction counter
792            // behind the on-chain nonce, or created gaps. Schedule a nonce health job
793            // to sync the counter and fill any gaps with NOOPs. Best-effort — failure
794            // here doesn't block the recovery; the periodic health check will catch it.
795            if let Err(e) = self.schedule_relayer_nonce_health_job(tx).await {
796                warn!(
797                    tx_id = %tx.id,
798                    error = %e,
799                    "nonce recovery: failed to schedule nonce health after external consumption"
800                );
801            }
802
803            return Ok(Some(updated_tx));
804        }
805
806        // on_chain_nonce <= tx_nonce: nonce not yet consumed, defer to normal status flow
807        debug!(
808            tx_id = %tx.id,
809            tx_nonce = tx_nonce,
810            on_chain_nonce = on_chain_nonce,
811            "nonce recovery: on-chain nonce not past tx nonce, deferring to normal flow"
812        );
813        Ok(None)
814    }
815
816    /// Handles circuit breaker safely based on transaction status.
817    ///
818    /// This method implements the safe circuit breaker logic:
819    /// - **Pending/Sent**: Safe to mark as Failed (never broadcast to network)
820    /// - **Submitted**: Must trigger NOOP to clear nonce slot (regardless of expiry)
821    ///
822    /// For Submitted transactions, we always issue a NOOP because the nonce slot is
823    /// occupied and the original transaction could still execute. Simply marking as
824    /// Failed/Expired would leave the nonce blocked and risk the relayer stopping.
825    ///
826    /// Note: NOOP transactions are filtered out before entering this function.
827    async fn handle_circuit_breaker_safely(
828        &self,
829        tx: TransactionRepoModel,
830        ctx: &StatusCheckContext,
831    ) -> Result<TransactionRepoModel, TransactionError> {
832        let reason = format!(
833            "Transaction status monitoring failed after {} consecutive errors (total: {}). \
834             Last status: {:?}.",
835            ctx.consecutive_failures, ctx.total_failures, tx.status
836        );
837
838        match tx.status {
839            TransactionStatus::Pending => {
840                // Pending: no nonce assigned yet - safe to mark as Failed
841                debug!(
842                    tx_id = %tx.id,
843                    relayer_id = %tx.relayer_id,
844                    "circuit breaker: Pending transaction (no nonce) - safe to mark as Failed"
845                );
846                self.mark_as_failed(tx, reason).await
847            }
848            TransactionStatus::Sent => {
849                // Sent: nonce assigned but never broadcast to network.
850                // If a nonce is assigned, we must issue a NOOP to clear the nonce slot
851                // rather than just marking as Failed (which would leak the nonce).
852                let has_nonce = tx
853                    .network_data
854                    .get_evm_transaction_data()
855                    .map(|d| d.nonce.is_some())
856                    .unwrap_or(false);
857
858                if has_nonce {
859                    warn!(
860                        tx_id = %tx.id,
861                        relayer_id = %tx.relayer_id,
862                        "circuit breaker: Sent transaction with nonce assigned - triggering NOOP to clear nonce slot"
863                    );
864                    let noop_reason = Some(format!(
865                        "{reason}. Replacing with NOOP to clear nonce slot (Sent state with assigned nonce)."
866                    ));
867                    let updated_tx = self.process_noop_transaction(&tx, noop_reason).await?;
868                    // Must use resubmit (not submit) — resubmit re-signs with new gas pricing,
869                    // producing fresh `raw` bytes for the NOOP. submit_transaction would
870                    // broadcast the stale `raw` bytes which still contain the original tx.
871                    self.send_transaction_resubmit_job(&updated_tx).await?;
872                    Ok(updated_tx)
873                } else {
874                    // Defensive: Sent without nonce shouldn't normally happen
875                    debug!(
876                        tx_id = %tx.id,
877                        relayer_id = %tx.relayer_id,
878                        "circuit breaker: Sent transaction without nonce - safe to mark as Failed"
879                    );
880                    self.mark_as_failed(tx, reason).await
881                }
882            }
883            TransactionStatus::Submitted => {
884                // Submitted transactions occupy a nonce slot and could still execute.
885                // Regardless of expiry status, we MUST issue a NOOP to:
886                // 1. Clear the nonce slot so subsequent transactions can proceed
887                // 2. Prevent the original transaction from executing later
888                // Note: NOOP transactions are filtered out before entering this function.
889                warn!(
890                    tx_id = %tx.id,
891                    relayer_id = %tx.relayer_id,
892                    "circuit breaker: Submitted transaction - triggering NOOP to safely clear nonce"
893                );
894                let noop_reason = Some(format!(
895                    "{reason}. Replacing with NOOP to clear nonce slot."
896                ));
897                let updated_tx = self.process_noop_transaction(&tx, noop_reason).await?;
898                self.send_transaction_resubmit_job(&updated_tx).await?;
899                Ok(updated_tx)
900            }
901            _ => {
902                // Final states shouldn't reach here, but handle gracefully
903                debug!(
904                    tx_id = %tx.id,
905                    relayer_id = %tx.relayer_id,
906                    status = ?tx.status,
907                    "circuit breaker: unexpected status, returning transaction unchanged"
908                );
909                Ok(tx)
910            }
911        }
912    }
913
914    /// Inherent status-handling method.
915    ///
916    /// This method encapsulates the full logic for handling transaction status,
917    /// including resubmission, NOOP replacement, timeout detection, and updating status.
918    pub async fn handle_status_impl(
919        &self,
920        tx: TransactionRepoModel,
921        context: Option<StatusCheckContext>,
922    ) -> Result<TransactionRepoModel, TransactionError> {
923        debug!(
924            tx_id = %tx.id,
925            relayer_id = %tx.relayer_id,
926            status = ?tx.status,
927            "checking transaction status"
928        );
929
930        // 1. Early return if final state
931        if is_final_state(&tx.status) {
932            debug!(
933                tx_id = %tx.id,
934                relayer_id = %tx.relayer_id,
935                status = ?tx.status,
936                "transaction already in final state"
937            );
938            return Ok(tx);
939        }
940
941        // 1.1. Check if circuit breaker should force finalization
942        // Skip circuit breaker for NOOP transactions - they're already safe (just clearing nonce)
943        // and should be handled by normal status logic which will eventually resolve them.
944        if let Some(ref ctx) = context {
945            let is_noop_tx = tx
946                .network_data
947                .get_evm_transaction_data()
948                .map(|data| is_noop(&data))
949                .unwrap_or(false);
950
951            if ctx.should_force_finalize() && !is_noop_tx {
952                warn!(
953                    tx_id = %tx.id,
954                    consecutive_failures = ctx.consecutive_failures,
955                    total_failures = ctx.total_failures,
956                    max_consecutive = ctx.max_consecutive_failures,
957                    status = ?tx.status,
958                    "circuit breaker triggered - handling safely based on transaction state"
959                );
960                return self.handle_circuit_breaker_safely(tx, ctx).await;
961            }
962
963            if ctx.should_force_finalize() && is_noop_tx {
964                debug!(
965                    tx_id = %tx.id,
966                    consecutive_failures = ctx.consecutive_failures,
967                    relayer_id = %tx.relayer_id,
968                    "circuit breaker would trigger but transaction is NOOP - continuing with normal status logic"
969                );
970            }
971        }
972
973        // 1.2. Check for nonce recovery hint in job metadata (one-shot signal from submission errors).
974        // This performs nonce reconciliation before normal status flow.
975        // The hint is in job_metadata, not the transaction — subsequent retries won't have it.
976        if let Some(ref ctx) = context {
977            if let Some(ref metadata) = ctx.job_metadata {
978                if let Some(hint) = metadata.get(TX_NONCE_RECONCILE_TRIGGER) {
979                    debug!(
980                        tx_id = %tx.id,
981                        hint = %hint,
982                        "nonce recovery hint detected - performing nonce reconciliation"
983                    );
984                    match self.reconcile_tx_nonce_state(&tx).await {
985                        Ok(Some(recovered_tx)) => {
986                            return Ok(recovered_tx);
987                        }
988                        Ok(None) => {
989                            // Recovery didn't resolve it — fall through to normal flow
990                            debug!(
991                                tx_id = %tx.id,
992                                "nonce recovery did not resolve transaction, continuing normal flow"
993                            );
994                        }
995                        Err(e) => {
996                            // Recovery failed — log and continue with normal flow
997                            warn!(
998                                tx_id = %tx.id,
999                                error = %e,
1000                                "nonce recovery failed, falling through to normal status flow"
1001                            );
1002                        }
1003                    }
1004                }
1005            }
1006        }
1007
1008        // 2. Check transaction status first
1009        // This allows fast transactions to update their status immediately,
1010        // even if they're young (<20s). For Pending/Sent states, this returns
1011        // early without querying the blockchain.
1012        let status = self.check_transaction_status(&tx).await?;
1013
1014        debug!(
1015            tx_id = %tx.id,
1016            previous_status = ?tx.status,
1017            new_status = ?status,
1018            relayer_id = %tx.relayer_id,
1019            "transaction status check completed"
1020        );
1021
1022        // 2.1. Reload transaction from DB if status changed
1023        // This ensures we have fresh data if check_transaction_status triggered a recovery
1024        // or any other update that modified the transaction in the database.
1025        let tx = if status != tx.status {
1026            debug!(
1027                tx_id = %tx.id,
1028                old_status = ?tx.status,
1029                new_status = ?status,
1030                relayer_id = %tx.relayer_id,
1031                "status changed during check, reloading transaction from DB to ensure fresh data"
1032            );
1033            self.transaction_repository()
1034                .get_by_id(tx.id.clone())
1035                .await?
1036        } else {
1037            tx
1038        };
1039
1040        // 3. Check if too early for resubmission on in-progress transactions
1041        // For Pending/Sent/Submitted states, defer resubmission logic and timeout checks
1042        // if the transaction is too young. Just update status and return.
1043        // For other states (Mined/Confirmed/Failed/etc), process immediately regardless of age.
1044        if is_too_early_to_resubmit(&tx)? && is_pending_transaction(&status) {
1045            // Update status if it changed, then return
1046            return self
1047                .update_transaction_status_if_needed(tx, status, None)
1048                .await;
1049        }
1050
1051        // 4. Handle based on status (including complex operations like resubmission)
1052        match status {
1053            TransactionStatus::Pending => self.handle_pending_state(tx).await,
1054            TransactionStatus::Sent => self.handle_sent_state(tx).await,
1055            TransactionStatus::Submitted => self.handle_submitted_state(tx).await,
1056            TransactionStatus::Mined => self.handle_mined_state(tx).await,
1057            TransactionStatus::Failed => {
1058                // Provide a descriptive status_reason when transitioning to Failed
1059                // from an on-chain receipt check (i.e., receipt status was false).
1060                let status_reason = if tx.status != TransactionStatus::Failed {
1061                    Some("Transaction reverted on-chain (receipt status: failed)".to_string())
1062                } else {
1063                    None
1064                };
1065                self.handle_final_state(tx, status, status_reason).await
1066            }
1067            TransactionStatus::Confirmed
1068            | TransactionStatus::Expired
1069            | TransactionStatus::Canceled => self.handle_final_state(tx, status, None).await,
1070        }
1071    }
1072
1073    /// Handle transactions stuck in Sent (prepared but not submitted)
1074    async fn handle_sent_state(
1075        &self,
1076        tx: TransactionRepoModel,
1077    ) -> Result<TransactionRepoModel, TransactionError> {
1078        debug!(
1079            tx_id = %tx.id,
1080            relayer_id = %tx.relayer_id,
1081            "handling Sent state"
1082        );
1083
1084        // Check if transaction should be replaced with NOOP (expired, too many attempts on rollup, etc.)
1085        let (should_noop, reason) = self.should_noop(&tx).await?;
1086        if should_noop {
1087            debug!(
1088                tx_id = %tx.id,
1089                relayer_id = %tx.relayer_id,
1090                "preparing NOOP for sent transaction"
1091            );
1092            let update = self.prepare_noop_update_request(&tx, false, reason).await?;
1093            let updated_tx = self
1094                .transaction_repository()
1095                .partial_update(tx.id.clone(), update)
1096                .await?;
1097
1098            self.send_transaction_submit_job(&updated_tx).await?;
1099            let res = self.send_transaction_update_notification(&updated_tx).await;
1100            if let Err(e) = res {
1101                error!(
1102                    tx_id = %updated_tx.id,
1103                    relayer_id = %updated_tx.relayer_id,
1104                    status = ?updated_tx.status,
1105                    error = %e,
1106                    "sending transaction update notification failed for Sent state NOOP"
1107                );
1108            }
1109            return Ok(updated_tx);
1110        }
1111
1112        // Transaction was prepared but submission job may have failed
1113        // Re-queue a resend job if it's been stuck for a while
1114        let age_since_sent = get_age_since_status_change(&tx)?;
1115
1116        if age_since_sent > get_evm_resend_timeout() {
1117            warn!(
1118                tx_id = %tx.id,
1119                relayer_id = %tx.relayer_id,
1120                age_seconds = age_since_sent.num_seconds(),
1121                "transaction stuck in Sent, queuing resubmit job with repricing"
1122            );
1123
1124            // Queue resubmit job to reprice the transaction for better acceptance
1125            self.send_transaction_resubmit_job(&tx).await?;
1126        }
1127
1128        self.update_transaction_status_if_needed(tx, TransactionStatus::Sent, None)
1129            .await
1130    }
1131
1132    /// Determines if we should attempt hash recovery for a stuck transaction.
1133    ///
1134    /// This is an expensive operation, so we only do it when:
1135    /// - Transaction has been in Submitted status for a while (> 2 minutes)
1136    /// - Transaction has had at least 2 resubmission attempts (hashes.len() > 1)
1137    /// - Haven't tried recovery too recently (to avoid repeated attempts)
1138    fn should_try_hash_recovery(
1139        &self,
1140        tx: &TransactionRepoModel,
1141    ) -> Result<bool, TransactionError> {
1142        // Only try recovery for transactions stuck in Submitted
1143        if tx.status != TransactionStatus::Submitted {
1144            return Ok(false);
1145        }
1146
1147        // Must have multiple hashes (indicating resubmissions happened)
1148        if tx.hashes.len() <= 1 {
1149            return Ok(false);
1150        }
1151
1152        // Only try if transaction has been stuck for a while
1153        let age = get_age_of_sent_at(tx)?;
1154        let min_age_for_recovery = get_evm_min_age_for_hash_recovery();
1155
1156        if age < min_age_for_recovery {
1157            return Ok(false);
1158        }
1159
1160        // Check if we've had enough resubmission attempts (more attempts = more likely to have wrong hash)
1161        // Only try recovery if we have at least 3 hashes (2 resubmissions)
1162        if tx.hashes.len() < EVM_MIN_HASHES_FOR_RECOVERY {
1163            return Ok(false);
1164        }
1165
1166        Ok(true)
1167    }
1168
1169    /// Attempts to recover transaction status by checking all historical hashes.
1170    ///
1171    /// When a transaction is resubmitted multiple times due to timeouts, the database
1172    /// may contain multiple hashes. The "current" hash (network_data.hash) might not
1173    /// be the one that actually got mined. This method checks all historical hashes
1174    /// to find if any were mined, and updates the database with the correct one.
1175    ///
1176    /// Returns the updated transaction model if recovery was successful, None otherwise.
1177    async fn try_recover_with_historical_hashes(
1178        &self,
1179        tx: &TransactionRepoModel,
1180        evm_data: &crate::models::EvmTransactionData,
1181    ) -> Result<Option<TransactionRepoModel>, TransactionError> {
1182        warn!(
1183            tx_id = %tx.id,
1184            relayer_id = %tx.relayer_id,
1185            current_hash = ?evm_data.hash,
1186            total_hashes = %tx.hashes.len(),
1187            "attempting hash recovery - checking historical hashes"
1188        );
1189
1190        // Check each historical hash (most recent first, since it's more likely)
1191        for (idx, historical_hash) in tx.hashes.iter().rev().enumerate() {
1192            // Skip if this is the current hash (already checked)
1193            if Some(historical_hash) == evm_data.hash.as_ref() {
1194                continue;
1195            }
1196
1197            debug!(
1198                tx_id = %tx.id,
1199                relayer_id = %tx.relayer_id,
1200                hash = %historical_hash,
1201                index = %idx,
1202                "checking historical hash"
1203            );
1204
1205            // Try to get receipt for this hash
1206            match self
1207                .provider()
1208                .get_transaction_receipt(historical_hash)
1209                .await
1210            {
1211                Ok(Some(receipt)) => {
1212                    warn!(
1213                        tx_id = %tx.id,
1214                        relayer_id = %tx.relayer_id,
1215                        mined_hash = %historical_hash,
1216                        wrong_hash = ?evm_data.hash,
1217                        block_number = ?receipt.block_number,
1218                        "RECOVERED: found mined transaction with historical hash - correcting database"
1219                    );
1220
1221                    // Update with correct hash and Mined status
1222                    // Let the normal status check flow handle confirmation checking
1223                    let updated_tx = self
1224                        .update_transaction_with_corrected_hash(
1225                            tx,
1226                            evm_data,
1227                            historical_hash,
1228                            TransactionStatus::Mined,
1229                        )
1230                        .await?;
1231
1232                    return Ok(Some(updated_tx));
1233                }
1234                Ok(None) => {
1235                    // This hash not found either, continue to next
1236                    continue;
1237                }
1238                Err(e) => {
1239                    // Network error, log but continue checking other hashes
1240                    warn!(
1241                        tx_id = %tx.id,
1242                        relayer_id = %tx.relayer_id,
1243                        hash = %historical_hash,
1244                        error = %e,
1245                        "error checking historical hash, continuing to next"
1246                    );
1247                    continue;
1248                }
1249            }
1250        }
1251
1252        // None of the historical hashes found on-chain
1253        debug!(
1254            tx_id = %tx.id,
1255            relayer_id = %tx.relayer_id,
1256            "hash recovery completed - no historical hashes found on-chain"
1257        );
1258        Ok(None)
1259    }
1260
1261    /// Updates transaction with the corrected hash and status
1262    ///
1263    /// Returns the updated transaction model and sends a notification about the status change.
1264    async fn update_transaction_with_corrected_hash(
1265        &self,
1266        tx: &TransactionRepoModel,
1267        evm_data: &crate::models::EvmTransactionData,
1268        correct_hash: &str,
1269        status: TransactionStatus,
1270    ) -> Result<TransactionRepoModel, TransactionError> {
1271        let mut corrected_data = evm_data.clone();
1272        corrected_data.hash = Some(correct_hash.to_string());
1273
1274        let updated_tx = self
1275            .transaction_repository()
1276            .partial_update(
1277                tx.id.clone(),
1278                TransactionUpdateRequest {
1279                    network_data: Some(NetworkTransactionData::Evm(corrected_data)),
1280                    status: Some(status),
1281                    ..Default::default()
1282                },
1283            )
1284            .await?;
1285
1286        // Send notification about the recovered transaction
1287        if let Err(e) = self.send_transaction_update_notification(&updated_tx).await {
1288            error!(
1289                tx_id = %updated_tx.id,
1290                relayer_id = %updated_tx.relayer_id,
1291                error = %e,
1292                "failed to send notification for hash recovery"
1293            );
1294        }
1295
1296        Ok(updated_tx)
1297    }
1298}
1299
1300#[cfg(test)]
1301mod tests {
1302    use crate::{
1303        config::{EvmNetworkConfig, NetworkConfigCommon},
1304        domain::transaction::evm::{EvmRelayerTransaction, MockPriceCalculatorTrait},
1305        jobs::MockJobProducerTrait,
1306        models::{
1307            evm::Speed, EvmTransactionData, NetworkConfigData, NetworkRepoModel,
1308            NetworkTransactionData, NetworkType, RelayerEvmPolicy, RelayerNetworkPolicy,
1309            RelayerRepoModel, RpcConfig, TransactionReceipt, TransactionRepoModel,
1310            TransactionStatus, U256,
1311        },
1312        repositories::{
1313            MockNetworkRepository, MockRelayerRepository, MockTransactionCounterTrait,
1314            MockTransactionRepository,
1315        },
1316        services::{provider::MockEvmProviderTrait, signer::MockSigner},
1317    };
1318    use alloy::{
1319        consensus::{Eip658Value, Receipt, ReceiptWithBloom},
1320        network::AnyReceiptEnvelope,
1321        primitives::{b256, Address, BlockHash, Bloom, TxHash},
1322    };
1323    use chrono::{Duration, Utc};
1324    use std::sync::Arc;
1325
1326    /// Helper struct holding all the mocks we often need
1327    pub struct TestMocks {
1328        pub provider: MockEvmProviderTrait,
1329        pub relayer_repo: MockRelayerRepository,
1330        pub network_repo: MockNetworkRepository,
1331        pub tx_repo: MockTransactionRepository,
1332        pub job_producer: MockJobProducerTrait,
1333        pub signer: MockSigner,
1334        pub counter: MockTransactionCounterTrait,
1335        pub price_calc: MockPriceCalculatorTrait,
1336    }
1337
1338    /// Returns a default `TestMocks` with zero-configuration stubs.
1339    /// You can override expectations in each test as needed.
1340    pub fn default_test_mocks() -> TestMocks {
1341        TestMocks {
1342            provider: MockEvmProviderTrait::new(),
1343            relayer_repo: MockRelayerRepository::new(),
1344            network_repo: MockNetworkRepository::new(),
1345            tx_repo: MockTransactionRepository::new(),
1346            job_producer: MockJobProducerTrait::new(),
1347            signer: MockSigner::new(),
1348            counter: MockTransactionCounterTrait::new(),
1349            price_calc: MockPriceCalculatorTrait::new(),
1350        }
1351    }
1352
1353    /// Returns a `TestMocks` with network repository configured for prepare_noop_update_request tests.
1354    pub fn default_test_mocks_with_network() -> TestMocks {
1355        let mut mocks = default_test_mocks();
1356        // Set up default expectation for get_by_chain_id that prepare_noop_update_request tests need
1357        mocks
1358            .network_repo
1359            .expect_get_by_chain_id()
1360            .returning(|network_type, chain_id| {
1361                if network_type == NetworkType::Evm && chain_id == 1 {
1362                    Ok(Some(create_test_network_model()))
1363                } else {
1364                    Ok(None)
1365                }
1366            });
1367        mocks
1368    }
1369
1370    /// Creates a test NetworkRepoModel for chain_id 1 (mainnet)
1371    pub fn create_test_network_model() -> NetworkRepoModel {
1372        let evm_config = EvmNetworkConfig {
1373            common: NetworkConfigCommon {
1374                network: "mainnet".to_string(),
1375                from: None,
1376                rpc_urls: Some(vec![RpcConfig::new("https://rpc.example.com".to_string())]),
1377                explorer_urls: Some(vec!["https://explorer.example.com".to_string()]),
1378                average_blocktime_ms: Some(12000),
1379                is_testnet: Some(false),
1380                tags: Some(vec!["mainnet".to_string()]),
1381            },
1382            chain_id: Some(1),
1383            required_confirmations: Some(12),
1384            features: Some(vec!["eip1559".to_string()]),
1385            symbol: Some("ETH".to_string()),
1386            gas_price_cache: None,
1387        };
1388        NetworkRepoModel {
1389            id: "evm:mainnet".to_string(),
1390            name: "mainnet".to_string(),
1391            network_type: NetworkType::Evm,
1392            config: NetworkConfigData::Evm(evm_config),
1393        }
1394    }
1395
1396    /// Creates a test NetworkRepoModel for chain_id 42161 (Arbitrum-like) with no-mempool tag
1397    pub fn create_test_no_mempool_network_model() -> NetworkRepoModel {
1398        let evm_config = EvmNetworkConfig {
1399            common: NetworkConfigCommon {
1400                network: "arbitrum".to_string(),
1401                from: None,
1402                rpc_urls: Some(vec![crate::models::RpcConfig::new(
1403                    "https://arb-rpc.example.com".to_string(),
1404                )]),
1405                explorer_urls: Some(vec!["https://arb-explorer.example.com".to_string()]),
1406                average_blocktime_ms: Some(1000),
1407                is_testnet: Some(false),
1408                tags: Some(vec![
1409                    "arbitrum".to_string(),
1410                    "rollup".to_string(),
1411                    "no-mempool".to_string(),
1412                ]),
1413            },
1414            chain_id: Some(42161),
1415            required_confirmations: Some(12),
1416            features: Some(vec!["eip1559".to_string()]),
1417            symbol: Some("ETH".to_string()),
1418            gas_price_cache: None,
1419        };
1420        NetworkRepoModel {
1421            id: "evm:arbitrum".to_string(),
1422            name: "arbitrum".to_string(),
1423            network_type: NetworkType::Evm,
1424            config: NetworkConfigData::Evm(evm_config),
1425        }
1426    }
1427
1428    /// Minimal "builder" for TransactionRepoModel.
1429    /// Allows quick creation of a test transaction with default fields,
1430    /// then updates them based on the provided status or overrides.
1431    pub fn make_test_transaction(status: TransactionStatus) -> TransactionRepoModel {
1432        TransactionRepoModel {
1433            id: "test-tx-id".to_string(),
1434            relayer_id: "test-relayer-id".to_string(),
1435            status,
1436            status_reason: None,
1437            created_at: Utc::now().to_rfc3339(),
1438            sent_at: None,
1439            confirmed_at: None,
1440            valid_until: None,
1441            delete_at: None,
1442            network_type: NetworkType::Evm,
1443            network_data: NetworkTransactionData::Evm(EvmTransactionData {
1444                chain_id: 1,
1445                from: "0xSender".to_string(),
1446                to: Some("0xRecipient".to_string()),
1447                value: U256::from(0),
1448                data: Some("0xData".to_string()),
1449                gas_limit: Some(21000),
1450                gas_price: Some(20000000000),
1451                max_fee_per_gas: None,
1452                max_priority_fee_per_gas: None,
1453                nonce: None,
1454                signature: None,
1455                hash: None,
1456                speed: Some(Speed::Fast),
1457                raw: None,
1458            }),
1459            priced_at: None,
1460            hashes: Vec::new(),
1461            noop_count: None,
1462            is_canceled: Some(false),
1463            metadata: None,
1464        }
1465    }
1466
1467    /// Minimal "builder" for EvmRelayerTransaction.
1468    /// Takes mock dependencies as arguments.
1469    pub fn make_test_evm_relayer_transaction(
1470        relayer: RelayerRepoModel,
1471        mocks: TestMocks,
1472    ) -> EvmRelayerTransaction<
1473        MockEvmProviderTrait,
1474        MockRelayerRepository,
1475        MockNetworkRepository,
1476        MockTransactionRepository,
1477        MockJobProducerTrait,
1478        MockSigner,
1479        MockTransactionCounterTrait,
1480        MockPriceCalculatorTrait,
1481    > {
1482        EvmRelayerTransaction::new(
1483            relayer,
1484            mocks.provider,
1485            Arc::new(mocks.relayer_repo),
1486            Arc::new(mocks.network_repo),
1487            Arc::new(mocks.tx_repo),
1488            Arc::new(mocks.counter),
1489            Arc::new(mocks.job_producer),
1490            mocks.price_calc,
1491            mocks.signer,
1492        )
1493        .unwrap()
1494    }
1495
1496    fn create_test_relayer() -> RelayerRepoModel {
1497        RelayerRepoModel {
1498            id: "test-relayer-id".to_string(),
1499            name: "Test Relayer".to_string(),
1500            paused: false,
1501            system_disabled: false,
1502            network: "test_network".to_string(),
1503            network_type: NetworkType::Evm,
1504            policies: RelayerNetworkPolicy::Evm(RelayerEvmPolicy::default()),
1505            signer_id: "test_signer".to_string(),
1506            address: "0x".to_string(),
1507            notification_id: None,
1508            custom_rpc_urls: None,
1509            ..Default::default()
1510        }
1511    }
1512
1513    fn make_mock_receipt(status: bool, block_number: Option<u64>) -> TransactionReceipt {
1514        // Use some placeholder values for minimal completeness
1515        let tx_hash = TxHash::from(b256!(
1516            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
1517        ));
1518        let block_hash = BlockHash::from(b256!(
1519            "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
1520        ));
1521        let from_address = Address::from([0x11; 20]);
1522
1523        TransactionReceipt {
1524            inner: alloy::rpc::types::TransactionReceipt {
1525                inner: AnyReceiptEnvelope {
1526                    inner: ReceiptWithBloom {
1527                        receipt: Receipt {
1528                            status: Eip658Value::Eip658(status), // determines success/fail
1529                            cumulative_gas_used: 0,
1530                            logs: vec![],
1531                        },
1532                        logs_bloom: Bloom::ZERO,
1533                    },
1534                    r#type: 0, // Legacy transaction type
1535                },
1536                transaction_hash: tx_hash,
1537                transaction_index: Some(0),
1538                block_hash: block_number.map(|_| block_hash), // only set if mined
1539                block_number,
1540                gas_used: 21000,
1541                effective_gas_price: 1000,
1542                blob_gas_used: None,
1543                blob_gas_price: None,
1544                from: from_address,
1545                to: None,
1546                contract_address: None,
1547            },
1548            other: Default::default(),
1549        }
1550    }
1551
1552    /// The fixed `from` address used by [`make_mock_receipt`].
1553    fn mock_receipt_from() -> Address {
1554        Address::from([0x11; 20])
1555    }
1556
1557    /// Like [`make_mock_receipt`] but sets the receipt's `to`, so cancellation
1558    /// attribution (a NOOP is a self-send: `to == from`) can be exercised.
1559    fn make_mock_receipt_with_to(
1560        status: bool,
1561        block_number: Option<u64>,
1562        to: Option<Address>,
1563    ) -> TransactionReceipt {
1564        let mut receipt = make_mock_receipt(status, block_number);
1565        receipt.inner.to = to;
1566        receipt
1567    }
1568
1569    // Tests for `check_transaction_status`
1570    mod check_transaction_status_tests {
1571        use super::*;
1572
1573        #[tokio::test]
1574        async fn test_not_mined() {
1575            let mut mocks = default_test_mocks();
1576            let relayer = create_test_relayer();
1577            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1578
1579            // Provide a hash so we can check for receipt
1580            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1581                evm_data.hash = Some("0xFakeHash".to_string());
1582            }
1583
1584            // Mock that get_transaction_receipt returns None (not mined)
1585            mocks
1586                .provider
1587                .expect_get_transaction_receipt()
1588                .returning(|_| Box::pin(async { Ok(None) }));
1589
1590            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1591
1592            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1593            assert_eq!(status, TransactionStatus::Submitted);
1594        }
1595
1596        #[tokio::test]
1597        async fn test_mined_but_not_confirmed() {
1598            let mut mocks = default_test_mocks();
1599            let relayer = create_test_relayer();
1600            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1601
1602            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1603                evm_data.hash = Some("0xFakeHash".to_string());
1604            }
1605
1606            // Mock a mined receipt with block_number = 100
1607            mocks
1608                .provider
1609                .expect_get_transaction_receipt()
1610                .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
1611
1612            // Mock block_number that hasn't reached the confirmation threshold
1613            mocks
1614                .provider
1615                .expect_get_block_number()
1616                .return_once(|| Box::pin(async { Ok(100) }));
1617
1618            // Mock network repository to return a test network model
1619            mocks
1620                .network_repo
1621                .expect_get_by_chain_id()
1622                .returning(|_, _| Ok(Some(create_test_network_model())));
1623
1624            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1625
1626            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1627            assert_eq!(status, TransactionStatus::Mined);
1628        }
1629
1630        #[tokio::test]
1631        async fn test_confirmed() {
1632            let mut mocks = default_test_mocks();
1633            let relayer = create_test_relayer();
1634            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1635
1636            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1637                evm_data.hash = Some("0xFakeHash".to_string());
1638            }
1639
1640            // Mock a mined receipt with block_number = 100
1641            mocks
1642                .provider
1643                .expect_get_transaction_receipt()
1644                .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
1645
1646            // Mock block_number that meets the confirmation threshold
1647            mocks
1648                .provider
1649                .expect_get_block_number()
1650                .return_once(|| Box::pin(async { Ok(113) }));
1651
1652            // Mock network repository to return a test network model
1653            mocks
1654                .network_repo
1655                .expect_get_by_chain_id()
1656                .returning(|_, _| Ok(Some(create_test_network_model())));
1657
1658            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1659
1660            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1661            assert_eq!(status, TransactionStatus::Confirmed);
1662        }
1663
1664        /// When a user-cancelled tx confirms and the mined tx is the NOOP (a self-send,
1665        /// receipt.to == receipt.from), it must terminate as Canceled.
1666        #[tokio::test]
1667        async fn test_cancellation_noop_confirmed_becomes_canceled() {
1668            let mut mocks = default_test_mocks();
1669            let relayer = create_test_relayer();
1670            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1671            tx.is_canceled = Some(true);
1672
1673            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1674                evm_data.hash = Some("0xNoopHash".to_string());
1675            }
1676
1677            // Mined receipt for a NOOP: to == from.
1678            let noop_to = Some(mock_receipt_from());
1679            mocks
1680                .provider
1681                .expect_get_transaction_receipt()
1682                .returning(move |_| {
1683                    Box::pin(async move {
1684                        Ok(Some(make_mock_receipt_with_to(true, Some(100), noop_to)))
1685                    })
1686                });
1687            mocks
1688                .provider
1689                .expect_get_block_number()
1690                .return_once(|| Box::pin(async { Ok(113) }));
1691            mocks
1692                .network_repo
1693                .expect_get_by_chain_id()
1694                .returning(|_, _| Ok(Some(create_test_network_model())));
1695
1696            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1697
1698            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1699            assert_eq!(status, TransactionStatus::Canceled);
1700        }
1701
1702        /// If a user-cancelled tx confirms but the ORIGINAL transaction is what mined
1703        /// (cancellation lost the nonce race: receipt.to != receipt.from), it actually
1704        /// executed and must be reported Confirmed, not Canceled.
1705        #[tokio::test]
1706        async fn test_cancellation_original_mined_stays_confirmed() {
1707            let mut mocks = default_test_mocks();
1708            let relayer = create_test_relayer();
1709            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1710            tx.is_canceled = Some(true);
1711
1712            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1713                evm_data.hash = Some("0xOriginalHash".to_string());
1714            }
1715
1716            // Mined receipt for the ORIGINAL transfer: to is a different address than from.
1717            let original_to = Some(Address::from([0x22; 20]));
1718            mocks
1719                .provider
1720                .expect_get_transaction_receipt()
1721                .returning(move |_| {
1722                    Box::pin(async move {
1723                        Ok(Some(make_mock_receipt_with_to(
1724                            true,
1725                            Some(100),
1726                            original_to,
1727                        )))
1728                    })
1729                });
1730            mocks
1731                .provider
1732                .expect_get_block_number()
1733                .return_once(|| Box::pin(async { Ok(113) }));
1734            mocks
1735                .network_repo
1736                .expect_get_by_chain_id()
1737                .returning(|_, _| Ok(Some(create_test_network_model())));
1738
1739            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1740
1741            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1742            assert_eq!(status, TransactionStatus::Confirmed);
1743        }
1744
1745        /// A confirmed self-send that is NOT a user cancellation (is_canceled=false) — e.g.
1746        /// a nonce-clearing NOOP from timeout handling — must still confirm normally.
1747        #[tokio::test]
1748        async fn test_non_cancellation_noop_confirmed_stays_confirmed() {
1749            let mut mocks = default_test_mocks();
1750            let relayer = create_test_relayer();
1751            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1752            // is_canceled stays Some(false) from the helper.
1753
1754            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1755                evm_data.hash = Some("0xNoopHash".to_string());
1756            }
1757
1758            // Even with a self-send receipt (to == from), a non-cancelled tx stays Confirmed.
1759            let noop_to = Some(mock_receipt_from());
1760            mocks
1761                .provider
1762                .expect_get_transaction_receipt()
1763                .returning(move |_| {
1764                    Box::pin(async move {
1765                        Ok(Some(make_mock_receipt_with_to(true, Some(100), noop_to)))
1766                    })
1767                });
1768            mocks
1769                .provider
1770                .expect_get_block_number()
1771                .return_once(|| Box::pin(async { Ok(113) }));
1772            mocks
1773                .network_repo
1774                .expect_get_by_chain_id()
1775                .returning(|_, _| Ok(Some(create_test_network_model())));
1776
1777            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1778
1779            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1780            assert_eq!(status, TransactionStatus::Confirmed);
1781        }
1782
1783        #[tokio::test]
1784        async fn test_failed() {
1785            let mut mocks = default_test_mocks();
1786            let relayer = create_test_relayer();
1787            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1788
1789            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1790                evm_data.hash = Some("0xFakeHash".to_string());
1791            }
1792
1793            // Mock a mined receipt with failure
1794            mocks
1795                .provider
1796                .expect_get_transaction_receipt()
1797                .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(false, Some(100)))) }));
1798
1799            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1800
1801            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1802            assert_eq!(status, TransactionStatus::Failed);
1803        }
1804    }
1805
1806    // Tests for `should_resubmit`
1807    mod should_resubmit_tests {
1808        use super::*;
1809        use crate::models::TransactionError;
1810
1811        #[tokio::test]
1812        async fn test_should_resubmit_true() {
1813            let mut mocks = default_test_mocks();
1814            let relayer = create_test_relayer();
1815
1816            // Set sent_at to 600 seconds ago to force resubmission
1817            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1818            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1819
1820            // Mock network repository to return a regular network model
1821            mocks
1822                .network_repo
1823                .expect_get_by_chain_id()
1824                .returning(|_, _| Ok(Some(create_test_network_model())));
1825
1826            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1827            let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1828            assert!(res, "Transaction should be resubmitted after timeout.");
1829        }
1830
1831        #[tokio::test]
1832        async fn test_should_resubmit_false() {
1833            let mut mocks = default_test_mocks();
1834            let relayer = create_test_relayer();
1835
1836            // Make a transaction with status Submitted but recently sent
1837            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1838            tx.sent_at = Some(Utc::now().to_rfc3339());
1839
1840            // Mock network repository to return a regular network model
1841            mocks
1842                .network_repo
1843                .expect_get_by_chain_id()
1844                .returning(|_, _| Ok(Some(create_test_network_model())));
1845
1846            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1847            let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1848            assert!(!res, "Transaction should not be resubmitted immediately.");
1849        }
1850
1851        #[tokio::test]
1852        async fn test_should_resubmit_true_for_no_mempool_network() {
1853            let mut mocks = default_test_mocks();
1854            let relayer = create_test_relayer();
1855
1856            // Set up a transaction that would normally be resubmitted (sent_at long ago)
1857            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1858            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1859
1860            // Set chain_id to match the no-mempool network
1861            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1862                evm_data.chain_id = 42161; // Arbitrum chain ID
1863            }
1864
1865            // Mock network repository to return a no-mempool network model
1866            mocks
1867                .network_repo
1868                .expect_get_by_chain_id()
1869                .returning(|_, _| Ok(Some(create_test_no_mempool_network_model())));
1870
1871            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1872            let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1873            assert!(
1874                res,
1875                "Transaction should be resubmitted for no-mempool networks."
1876            );
1877        }
1878
1879        #[tokio::test]
1880        async fn test_should_resubmit_network_not_found() {
1881            let mut mocks = default_test_mocks();
1882            let relayer = create_test_relayer();
1883
1884            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1885            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1886
1887            // Mock network repository to return None (network not found)
1888            mocks
1889                .network_repo
1890                .expect_get_by_chain_id()
1891                .returning(|_, _| Ok(None));
1892
1893            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1894            let result = evm_transaction.should_resubmit(&tx).await;
1895
1896            assert!(
1897                result.is_err(),
1898                "should_resubmit should return error when network not found"
1899            );
1900            let error = result.unwrap_err();
1901            match error {
1902                TransactionError::UnexpectedError(msg) => {
1903                    assert!(msg.contains("Network with chain id 1 not found"));
1904                }
1905                _ => panic!("Expected UnexpectedError for network not found"),
1906            }
1907        }
1908
1909        #[tokio::test]
1910        async fn test_should_resubmit_network_conversion_error() {
1911            let mut mocks = default_test_mocks();
1912            let relayer = create_test_relayer();
1913
1914            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1915            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1916
1917            // Create a network model with invalid EVM config (missing chain_id)
1918            let invalid_evm_config = EvmNetworkConfig {
1919                common: NetworkConfigCommon {
1920                    network: "invalid-network".to_string(),
1921                    from: None,
1922                    rpc_urls: Some(vec![crate::models::RpcConfig::new(
1923                        "https://rpc.example.com".to_string(),
1924                    )]),
1925                    explorer_urls: Some(vec!["https://explorer.example.com".to_string()]),
1926                    average_blocktime_ms: Some(12000),
1927                    is_testnet: Some(false),
1928                    tags: Some(vec!["testnet".to_string()]),
1929                },
1930                chain_id: None, // This will cause the conversion to fail
1931                required_confirmations: Some(12),
1932                features: Some(vec!["eip1559".to_string()]),
1933                symbol: Some("ETH".to_string()),
1934                gas_price_cache: None,
1935            };
1936            let invalid_network = NetworkRepoModel {
1937                id: "evm:invalid".to_string(),
1938                name: "invalid-network".to_string(),
1939                network_type: NetworkType::Evm,
1940                config: NetworkConfigData::Evm(invalid_evm_config),
1941            };
1942
1943            // Mock network repository to return the invalid network model
1944            mocks
1945                .network_repo
1946                .expect_get_by_chain_id()
1947                .returning(move |_, _| Ok(Some(invalid_network.clone())));
1948
1949            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1950            let result = evm_transaction.should_resubmit(&tx).await;
1951
1952            assert!(
1953                result.is_err(),
1954                "should_resubmit should return error when network conversion fails"
1955            );
1956            let error = result.unwrap_err();
1957            match error {
1958                TransactionError::UnexpectedError(msg) => {
1959                    assert!(msg.contains("Error converting network model to EvmNetwork"));
1960                }
1961                _ => panic!("Expected UnexpectedError for network conversion failure"),
1962            }
1963        }
1964    }
1965
1966    // Tests for `should_noop`
1967    mod should_noop_tests {
1968        use super::*;
1969
1970        #[tokio::test]
1971        async fn test_expired_transaction_triggers_noop() {
1972            let mut mocks = default_test_mocks();
1973            let relayer = create_test_relayer();
1974
1975            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1976            // Force the transaction to be "expired" by setting valid_until in the past
1977            tx.valid_until = Some((Utc::now() - Duration::seconds(10)).to_rfc3339());
1978
1979            // Mock network repository to return a test network model
1980            mocks
1981                .network_repo
1982                .expect_get_by_chain_id()
1983                .returning(|_, _| Ok(Some(create_test_network_model())));
1984
1985            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1986            let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
1987            assert!(res, "Expired transaction should be replaced with a NOOP.");
1988            assert!(
1989                reason.is_some(),
1990                "Reason should be provided for expired transaction"
1991            );
1992            assert!(
1993                reason.unwrap().contains("expired"),
1994                "Reason should mention expiration"
1995            );
1996        }
1997
1998        #[tokio::test]
1999        async fn test_too_many_noop_attempts_returns_false() {
2000            let mocks = default_test_mocks();
2001            let relayer = create_test_relayer();
2002
2003            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2004            tx.noop_count = Some(51); // Max is 50, so this should return false
2005
2006            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2007            let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2008            assert!(
2009                !res,
2010                "Transaction with too many NOOP attempts should not be replaced."
2011            );
2012            assert!(
2013                reason.is_none(),
2014                "Reason should not be provided when should_noop is false"
2015            );
2016        }
2017
2018        #[tokio::test]
2019        async fn test_already_noop_returns_false() {
2020            let mut mocks = default_test_mocks();
2021            let relayer = create_test_relayer();
2022
2023            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2024            // Make it a NOOP by setting to=None and value=0
2025            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2026                evm_data.to = None;
2027                evm_data.value = U256::from(0);
2028            }
2029
2030            mocks
2031                .network_repo
2032                .expect_get_by_chain_id()
2033                .returning(|_, _| Ok(Some(create_test_network_model())));
2034
2035            // Mock get_block_by_number for gas limit validation (won't be called since is_noop returns early, but needed for compilation)
2036            mocks.provider.expect_get_block_by_number().returning(|| {
2037                Box::pin(async {
2038                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2039                    let mut block: Block = Block::default();
2040                    block.header.gas_limit = 30_000_000u64;
2041                    Ok(AnyRpcBlock::from(block))
2042                })
2043            });
2044
2045            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2046            let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2047            assert!(
2048                !res,
2049                "Transaction that is already a NOOP should not be replaced."
2050            );
2051            assert!(
2052                reason.is_none(),
2053                "Reason should not be provided when should_noop is false"
2054            );
2055        }
2056
2057        #[tokio::test]
2058        async fn test_rollup_with_too_many_attempts_triggers_noop() {
2059            let mut mocks = default_test_mocks();
2060            let relayer = create_test_relayer();
2061
2062            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2063            // Set chain_id to Arbitrum (rollup network)
2064            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2065                evm_data.chain_id = 42161; // Arbitrum
2066            }
2067            // Set enough hashes to trigger too_many_attempts (> 50)
2068            tx.hashes = vec!["0xHash1".to_string(); 51];
2069
2070            // Mock network repository to return Arbitrum network
2071            mocks
2072                .network_repo
2073                .expect_get_by_chain_id()
2074                .returning(|_, _| Ok(Some(create_test_no_mempool_network_model())));
2075
2076            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2077            let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2078            assert!(
2079                res,
2080                "Rollup transaction with too many attempts should be replaced with NOOP."
2081            );
2082            assert!(
2083                reason.is_some(),
2084                "Reason should be provided for rollup transaction"
2085            );
2086            assert!(
2087                reason.unwrap().contains("too many attempts"),
2088                "Reason should mention too many attempts"
2089            );
2090        }
2091
2092        #[tokio::test]
2093        async fn test_pending_state_timeout_triggers_noop() {
2094            let mut mocks = default_test_mocks();
2095            let relayer = create_test_relayer();
2096
2097            let mut tx = make_test_transaction(TransactionStatus::Pending);
2098            // Set created_at to 3 minutes ago (> 2 minute timeout)
2099            tx.created_at = (Utc::now() - Duration::minutes(3)).to_rfc3339();
2100
2101            mocks
2102                .network_repo
2103                .expect_get_by_chain_id()
2104                .returning(|_, _| Ok(Some(create_test_network_model())));
2105
2106            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2107            let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2108            assert!(
2109                res,
2110                "Pending transaction stuck for >2 minutes should be replaced with NOOP."
2111            );
2112            assert!(
2113                reason.is_some(),
2114                "Reason should be provided for pending timeout"
2115            );
2116            assert!(
2117                reason.unwrap().contains("Pending state"),
2118                "Reason should mention Pending state"
2119            );
2120        }
2121
2122        #[tokio::test]
2123        async fn test_valid_transaction_returns_false() {
2124            let mut mocks = default_test_mocks();
2125            let relayer = create_test_relayer();
2126
2127            let tx = make_test_transaction(TransactionStatus::Submitted);
2128            // Transaction is recent, not expired, not on rollup, no issues
2129
2130            mocks
2131                .network_repo
2132                .expect_get_by_chain_id()
2133                .returning(|_, _| Ok(Some(create_test_network_model())));
2134
2135            // Mock get_block_by_number for gas limit validation
2136            mocks.provider.expect_get_block_by_number().returning(|| {
2137                Box::pin(async {
2138                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2139                    let mut block: Block = Block::default();
2140                    block.header.gas_limit = 30_000_000u64;
2141                    Ok(AnyRpcBlock::from(block))
2142                })
2143            });
2144
2145            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2146            let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2147            assert!(!res, "Valid transaction should not be replaced with NOOP.");
2148            assert!(
2149                reason.is_none(),
2150                "Reason should not be provided when should_noop is false"
2151            );
2152        }
2153    }
2154
2155    // Tests for `update_transaction_status_if_needed`
2156    mod update_transaction_status_tests {
2157        use super::*;
2158
2159        #[tokio::test]
2160        async fn test_no_update_when_status_is_same() {
2161            // Create mocks, relayer, and a transaction with status Submitted.
2162            let mocks = default_test_mocks();
2163            let relayer = create_test_relayer();
2164            let tx = make_test_transaction(TransactionStatus::Submitted);
2165            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2166
2167            // When new status is the same as current, update_transaction_status_if_needed
2168            // should simply return the original transaction.
2169            let updated_tx = evm_transaction
2170                .update_transaction_status_if_needed(tx.clone(), TransactionStatus::Submitted, None)
2171                .await
2172                .unwrap();
2173            assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2174            assert_eq!(updated_tx.id, tx.id);
2175        }
2176
2177        #[tokio::test]
2178        async fn test_updates_when_status_differs() {
2179            let mut mocks = default_test_mocks();
2180            let relayer = create_test_relayer();
2181            let tx = make_test_transaction(TransactionStatus::Submitted);
2182
2183            // Mock partial_update to return a transaction with new status
2184            mocks
2185                .tx_repo
2186                .expect_partial_update()
2187                .returning(|_, update| {
2188                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2189                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2190                    Ok(updated_tx)
2191                });
2192
2193            // Mock notification job
2194            mocks
2195                .job_producer
2196                .expect_produce_send_notification_job()
2197                .returning(|_, _| Box::pin(async { Ok(()) }));
2198
2199            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2200            let updated_tx = evm_transaction
2201                .update_transaction_status_if_needed(tx.clone(), TransactionStatus::Mined, None)
2202                .await
2203                .unwrap();
2204
2205            assert_eq!(updated_tx.status, TransactionStatus::Mined);
2206        }
2207
2208        #[tokio::test]
2209        async fn test_updates_with_status_reason() {
2210            let mut mocks = default_test_mocks();
2211            let relayer = create_test_relayer();
2212            let tx = make_test_transaction(TransactionStatus::Submitted);
2213
2214            mocks
2215                .tx_repo
2216                .expect_partial_update()
2217                .withf(|_, update| {
2218                    update.status == Some(TransactionStatus::Failed)
2219                        && update.status_reason == Some("Transaction reverted on-chain".to_string())
2220                })
2221                .returning(|_, update| {
2222                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2223                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2224                    updated_tx.status_reason = update.status_reason.clone();
2225                    Ok(updated_tx)
2226                });
2227
2228            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2229            let updated_tx = evm_transaction
2230                .update_transaction_status_if_needed(
2231                    tx.clone(),
2232                    TransactionStatus::Failed,
2233                    Some("Transaction reverted on-chain".to_string()),
2234                )
2235                .await
2236                .unwrap();
2237
2238            assert_eq!(updated_tx.status, TransactionStatus::Failed);
2239            assert_eq!(
2240                updated_tx.status_reason.as_deref(),
2241                Some("Transaction reverted on-chain")
2242            );
2243        }
2244    }
2245
2246    // Tests for `handle_sent_state`
2247    mod handle_sent_state_tests {
2248        use super::*;
2249
2250        #[tokio::test]
2251        async fn test_sent_state_recent_no_resend() {
2252            let mut mocks = default_test_mocks();
2253            let relayer = create_test_relayer();
2254
2255            let mut tx = make_test_transaction(TransactionStatus::Sent);
2256            // Set sent_at to recent (e.g., 10 seconds ago)
2257            tx.sent_at = Some((Utc::now() - Duration::seconds(10)).to_rfc3339());
2258
2259            // Mock network repository to return a test network model for should_noop check
2260            mocks
2261                .network_repo
2262                .expect_get_by_chain_id()
2263                .returning(|_, _| Ok(Some(create_test_network_model())));
2264
2265            // Mock get_block_by_number for gas limit validation in handle_sent_state
2266            mocks.provider.expect_get_block_by_number().returning(|| {
2267                Box::pin(async {
2268                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2269                    let mut block: Block = Block::default();
2270                    block.header.gas_limit = 30_000_000u64;
2271                    Ok(AnyRpcBlock::from(block))
2272                })
2273            });
2274
2275            // Mock status check job scheduling
2276            mocks
2277                .job_producer
2278                .expect_produce_check_transaction_status_job()
2279                .returning(|_, _| Box::pin(async { Ok(()) }));
2280
2281            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2282            let result = evm_transaction.handle_sent_state(tx.clone()).await.unwrap();
2283
2284            assert_eq!(result.status, TransactionStatus::Sent);
2285        }
2286
2287        #[tokio::test]
2288        async fn test_sent_state_stuck_schedules_resubmit() {
2289            let mut mocks = default_test_mocks();
2290            let relayer = create_test_relayer();
2291
2292            let mut tx = make_test_transaction(TransactionStatus::Sent);
2293            // Set sent_at to long ago (> 30 seconds for resend timeout)
2294            tx.sent_at = Some((Utc::now() - Duration::seconds(60)).to_rfc3339());
2295
2296            // Mock network repository to return a test network model for should_noop check
2297            mocks
2298                .network_repo
2299                .expect_get_by_chain_id()
2300                .returning(|_, _| Ok(Some(create_test_network_model())));
2301
2302            // Mock get_block_by_number for gas limit validation in handle_sent_state
2303            mocks.provider.expect_get_block_by_number().returning(|| {
2304                Box::pin(async {
2305                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2306                    let mut block: Block = Block::default();
2307                    block.header.gas_limit = 30_000_000u64;
2308                    Ok(AnyRpcBlock::from(block))
2309                })
2310            });
2311
2312            // Mock resubmit job scheduling
2313            mocks
2314                .job_producer
2315                .expect_produce_submit_transaction_job()
2316                .returning(|_, _| Box::pin(async { Ok(()) }));
2317
2318            // Mock status check job scheduling
2319            mocks
2320                .job_producer
2321                .expect_produce_check_transaction_status_job()
2322                .returning(|_, _| Box::pin(async { Ok(()) }));
2323
2324            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2325            let result = evm_transaction.handle_sent_state(tx.clone()).await.unwrap();
2326
2327            assert_eq!(result.status, TransactionStatus::Sent);
2328        }
2329    }
2330
2331    // Tests for `prepare_noop_update_request`
2332    mod prepare_noop_update_request_tests {
2333        use super::*;
2334
2335        #[tokio::test]
2336        async fn test_noop_request_without_cancellation() {
2337            // Create a transaction with an initial noop_count of 2 and is_canceled set to false.
2338            let mocks = default_test_mocks_with_network();
2339            let relayer = create_test_relayer();
2340            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2341            tx.noop_count = Some(2);
2342            tx.is_canceled = Some(false);
2343
2344            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2345            let update_req = evm_transaction
2346                .prepare_noop_update_request(&tx, false, None)
2347                .await
2348                .unwrap();
2349
2350            // NOOP count should be incremented: 2 becomes 3.
2351            assert_eq!(update_req.noop_count, Some(3));
2352            // When not cancelling, the is_canceled flag should remain as in the original transaction.
2353            assert_eq!(update_req.is_canceled, Some(false));
2354        }
2355
2356        #[tokio::test]
2357        async fn test_noop_request_with_cancellation() {
2358            // Create a transaction with no initial noop_count (None) and is_canceled false.
2359            let mocks = default_test_mocks_with_network();
2360            let relayer = create_test_relayer();
2361            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2362            tx.noop_count = None;
2363            tx.is_canceled = Some(false);
2364
2365            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2366            let update_req = evm_transaction
2367                .prepare_noop_update_request(&tx, true, None)
2368                .await
2369                .unwrap();
2370
2371            // NOOP count should default to 1.
2372            assert_eq!(update_req.noop_count, Some(1));
2373            // When cancelling, the is_canceled flag should be forced to true.
2374            assert_eq!(update_req.is_canceled, Some(true));
2375        }
2376    }
2377
2378    // Tests for `handle_submitted_state`
2379    mod handle_submitted_state_tests {
2380        use super::*;
2381
2382        #[tokio::test]
2383        async fn test_schedules_resubmit_job() {
2384            let mut mocks = default_test_mocks();
2385            let relayer = create_test_relayer();
2386
2387            // Set sent_at far in the past to force resubmission
2388            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2389            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2390
2391            // Mock network repository to return a test network model for should_noop check
2392            mocks
2393                .network_repo
2394                .expect_get_by_chain_id()
2395                .returning(|_, _| Ok(Some(create_test_network_model())));
2396
2397            // Mock get_block_by_number for gas limit validation
2398            mocks.provider.expect_get_block_by_number().returning(|| {
2399                Box::pin(async {
2400                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2401                    let mut block: Block = Block::default();
2402                    block.header.gas_limit = 30_000_000u64;
2403                    Ok(AnyRpcBlock::from(block))
2404                })
2405            });
2406
2407            // On-chain nonce <= tx nonce (no gap), so resubmission proceeds normally
2408            mocks
2409                .provider
2410                .expect_get_transaction_count()
2411                .returning(|_| Box::pin(async { Ok(10) }));
2412
2413            // Expect the resubmit job to be produced
2414            mocks
2415                .job_producer
2416                .expect_produce_submit_transaction_job()
2417                .returning(|_, _| Box::pin(async { Ok(()) }));
2418
2419            // Expect status check to be scheduled
2420            mocks
2421                .job_producer
2422                .expect_produce_check_transaction_status_job()
2423                .returning(|_, _| Box::pin(async { Ok(()) }));
2424
2425            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2426            let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2427
2428            // We remain in "Submitted" after scheduling the resubmit
2429            assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2430        }
2431
2432        /// When tx_nonce > on_chain_nonce and nonce slots are empty, the tx is
2433        /// blocked by a gap. Should schedule nonce health job and skip resubmission.
2434        #[tokio::test]
2435        async fn test_nonce_gap_detected_schedules_health_skips_resubmit() {
2436            let mut mocks = default_test_mocks();
2437            let relayer = create_test_relayer();
2438
2439            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2440            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2441            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
2442                nonce: Some(274),
2443                hash: Some("0xhash".to_string()),
2444                raw: Some(vec![1, 2, 3]),
2445                ..tx.network_data.get_evm_transaction_data().unwrap()
2446            });
2447
2448            // Mock network repository for should_resubmit
2449            mocks
2450                .network_repo
2451                .expect_get_by_chain_id()
2452                .returning(|_, _| Ok(Some(create_test_network_model())));
2453
2454            // On-chain nonce is 269, tx nonce is 274
2455            mocks
2456                .provider
2457                .expect_get_transaction_count()
2458                .returning(|_| Box::pin(async { Ok(269) }));
2459
2460            // Batch nonce scan: nonces 269-273 are all empty → confirmed gap
2461            mocks
2462                .tx_repo
2463                .expect_get_nonce_occupancy()
2464                .withf(|relayer_id, from, to| {
2465                    relayer_id == "test-relayer-id" && *from == 269 && *to == 274
2466                })
2467                .returning(|_, from, to| Ok((from..to).map(|n| (n, None)).collect()));
2468
2469            // Should schedule nonce health job
2470            mocks
2471                .job_producer
2472                .expect_produce_relayer_health_check_job()
2473                .withf(|job, _| {
2474                    job.metadata.as_ref().map_or(false, |m| {
2475                        m.get("health_check_action") == Some(&"nonce_health".to_string())
2476                    })
2477                })
2478                .returning(|_, _| Box::pin(async { Ok(()) }));
2479
2480            // Should NOT call produce_submit_transaction_job (resubmit skipped)
2481            // mockall will panic if unexpected calls are made
2482
2483            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2484            let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2485
2486            assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2487        }
2488
2489        /// When get_transaction_count fails, the gap check should be skipped
2490        /// and resubmission should proceed normally.
2491        #[tokio::test]
2492        async fn test_nonce_gap_check_rpc_failure_proceeds_to_resubmit() {
2493            let mut mocks = default_test_mocks();
2494            let relayer = create_test_relayer();
2495
2496            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2497            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2498
2499            mocks
2500                .network_repo
2501                .expect_get_by_chain_id()
2502                .returning(|_, _| Ok(Some(create_test_network_model())));
2503
2504            mocks.provider.expect_get_block_by_number().returning(|| {
2505                Box::pin(async {
2506                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2507                    let mut block: Block = Block::default();
2508                    block.header.gas_limit = 30_000_000u64;
2509                    Ok(AnyRpcBlock::from(block))
2510                })
2511            });
2512
2513            // RPC fails for nonce check — should be gracefully skipped
2514            mocks
2515                .provider
2516                .expect_get_transaction_count()
2517                .returning(|_| {
2518                    Box::pin(async {
2519                        Err(crate::services::provider::ProviderError::Other(
2520                            "rpc timeout".to_string(),
2521                        ))
2522                    })
2523                });
2524
2525            // Resubmission should still proceed
2526            mocks
2527                .job_producer
2528                .expect_produce_submit_transaction_job()
2529                .returning(|_, _| Box::pin(async { Ok(()) }));
2530
2531            mocks
2532                .job_producer
2533                .expect_produce_check_transaction_status_job()
2534                .returning(|_, _| Box::pin(async { Ok(()) }));
2535
2536            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2537            let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2538
2539            assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2540        }
2541
2542        /// When all nonce slots between on-chain and tx_nonce are filled by active
2543        /// transactions, no gap exists — resubmission proceeds normally.
2544        #[tokio::test]
2545        async fn test_no_gap_when_slots_filled_proceeds_to_resubmit() {
2546            let mut mocks = default_test_mocks();
2547            let relayer = create_test_relayer();
2548
2549            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2550            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2551            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
2552                nonce: Some(270),
2553                hash: Some("0xhash".to_string()),
2554                raw: Some(vec![1, 2, 3]),
2555                ..tx.network_data.get_evm_transaction_data().unwrap()
2556            });
2557
2558            mocks
2559                .network_repo
2560                .expect_get_by_chain_id()
2561                .returning(|_, _| Ok(Some(create_test_network_model())));
2562
2563            mocks.provider.expect_get_block_by_number().returning(|| {
2564                Box::pin(async {
2565                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2566                    let mut block: Block = Block::default();
2567                    block.header.gas_limit = 30_000_000u64;
2568                    Ok(AnyRpcBlock::from(block))
2569                })
2570            });
2571
2572            // tx_nonce=270, on_chain=269 → 1 slot to check
2573            mocks
2574                .provider
2575                .expect_get_transaction_count()
2576                .returning(|_| Box::pin(async { Ok(269) }));
2577
2578            // Nonce 269 has an active Submitted tx → no gap
2579            mocks
2580                .tx_repo
2581                .expect_get_nonce_occupancy()
2582                .returning(|_, from, to| {
2583                    Ok((from..to)
2584                        .map(|n| (n, Some(TransactionStatus::Submitted)))
2585                        .collect())
2586                });
2587
2588            // Should proceed to resubmit (no health job expected)
2589            mocks
2590                .job_producer
2591                .expect_produce_submit_transaction_job()
2592                .returning(|_, _| Box::pin(async { Ok(()) }));
2593
2594            mocks
2595                .job_producer
2596                .expect_produce_check_transaction_status_job()
2597                .returning(|_, _| Box::pin(async { Ok(()) }));
2598
2599            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2600            let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2601
2602            assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2603        }
2604    }
2605
2606    // Tests for `handle_pending_state`
2607    mod handle_pending_state_tests {
2608        use super::*;
2609
2610        #[tokio::test]
2611        async fn test_pending_state_no_noop() {
2612            // Create a pending transaction that is fresh (created now).
2613            let mut mocks = default_test_mocks();
2614            let relayer = create_test_relayer();
2615            let mut tx = make_test_transaction(TransactionStatus::Pending);
2616            tx.created_at = Utc::now().to_rfc3339(); // less than one minute old
2617
2618            // Mock network repository to return a test network model
2619            mocks
2620                .network_repo
2621                .expect_get_by_chain_id()
2622                .returning(|_, _| Ok(Some(create_test_network_model())));
2623
2624            // Mock get_block_by_number for gas limit validation
2625            mocks.provider.expect_get_block_by_number().returning(|| {
2626                Box::pin(async {
2627                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2628                    let mut block: Block = Block::default();
2629                    block.header.gas_limit = 30_000_000u64;
2630                    Ok(AnyRpcBlock::from(block))
2631                })
2632            });
2633
2634            // Expect status check to be scheduled when not doing NOOP
2635            mocks
2636                .job_producer
2637                .expect_produce_check_transaction_status_job()
2638                .returning(|_, _| Box::pin(async { Ok(()) }));
2639
2640            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2641            let result = evm_transaction
2642                .handle_pending_state(tx.clone())
2643                .await
2644                .unwrap();
2645
2646            // When should_noop returns false the original transaction is returned unchanged.
2647            assert_eq!(result.id, tx.id);
2648            assert_eq!(result.status, tx.status);
2649            assert_eq!(result.noop_count, tx.noop_count);
2650        }
2651
2652        #[tokio::test]
2653        async fn test_pending_state_with_noop() {
2654            // Create a pending transaction that is old (created 2 minutes ago)
2655            let mut mocks = default_test_mocks();
2656            let relayer = create_test_relayer();
2657            let mut tx = make_test_transaction(TransactionStatus::Pending);
2658            tx.created_at = (Utc::now() - Duration::minutes(2)).to_rfc3339();
2659
2660            // Mock network repository to return a test network model
2661            mocks
2662                .network_repo
2663                .expect_get_by_chain_id()
2664                .returning(|_, _| Ok(Some(create_test_network_model())));
2665
2666            // Mock get_block_by_number for gas limit validation
2667            mocks.provider.expect_get_block_by_number().returning(|| {
2668                Box::pin(async {
2669                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2670                    let mut block: Block = Block::default();
2671                    block.header.gas_limit = 30_000_000u64;
2672                    Ok(AnyRpcBlock::from(block))
2673                })
2674            });
2675
2676            // Expect partial_update to be called and simulate a Failed update
2677            // (Pending state transactions are marked as Failed, not NOOP, since nonces aren't assigned)
2678            let tx_clone = tx.clone();
2679            mocks
2680                .tx_repo
2681                .expect_partial_update()
2682                .withf(move |id, update| {
2683                    id == "test-tx-id"
2684                        && update.status == Some(TransactionStatus::Failed)
2685                        && update.status_reason.is_some()
2686                })
2687                .returning(move |_, update| {
2688                    let mut updated_tx = tx_clone.clone();
2689                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2690                    updated_tx.status_reason = update.status_reason.clone();
2691                    Ok(updated_tx)
2692                });
2693            // Expect that a notification is produced (no submit job needed for Failed status)
2694            mocks
2695                .job_producer
2696                .expect_produce_send_notification_job()
2697                .returning(|_, _| Box::pin(async { Ok(()) }));
2698
2699            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2700            let result = evm_transaction
2701                .handle_pending_state(tx.clone())
2702                .await
2703                .unwrap();
2704
2705            // Since should_noop returns true for pending timeout, transaction should be marked as Failed
2706            assert_eq!(result.status, TransactionStatus::Failed);
2707            assert!(result.status_reason.is_some());
2708            assert!(result.status_reason.unwrap().contains("Pending state"));
2709        }
2710    }
2711
2712    // Tests for `handle_mined_state`
2713    mod handle_mined_state_tests {
2714        use super::*;
2715
2716        #[tokio::test]
2717        async fn test_updates_status_and_schedules_check() {
2718            let mut mocks = default_test_mocks();
2719            let relayer = create_test_relayer();
2720            // Create a transaction in Submitted state (the mined branch is reached via status check).
2721            let tx = make_test_transaction(TransactionStatus::Submitted);
2722
2723            // Expect schedule_status_check to be called with delay 5.
2724            mocks
2725                .job_producer
2726                .expect_produce_check_transaction_status_job()
2727                .returning(|_, _| Box::pin(async { Ok(()) }));
2728            // Expect partial_update to update the transaction status to Mined.
2729            mocks
2730                .tx_repo
2731                .expect_partial_update()
2732                .returning(|_, update| {
2733                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2734                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2735                    Ok(updated_tx)
2736                });
2737
2738            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2739            let result = evm_transaction
2740                .handle_mined_state(tx.clone())
2741                .await
2742                .unwrap();
2743            assert_eq!(result.status, TransactionStatus::Mined);
2744        }
2745    }
2746
2747    // Tests for `handle_final_state`
2748    mod handle_final_state_tests {
2749        use super::*;
2750
2751        #[tokio::test]
2752        async fn test_final_state_confirmed() {
2753            let mut mocks = default_test_mocks();
2754            let relayer = create_test_relayer();
2755            let tx = make_test_transaction(TransactionStatus::Submitted);
2756
2757            // Expect partial_update to update status to Confirmed.
2758            mocks
2759                .tx_repo
2760                .expect_partial_update()
2761                .returning(|_, update| {
2762                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2763                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2764                    Ok(updated_tx)
2765                });
2766
2767            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2768            let result = evm_transaction
2769                .handle_final_state(tx.clone(), TransactionStatus::Confirmed, None)
2770                .await
2771                .unwrap();
2772            assert_eq!(result.status, TransactionStatus::Confirmed);
2773        }
2774
2775        #[tokio::test]
2776        async fn test_final_state_failed() {
2777            let mut mocks = default_test_mocks();
2778            let relayer = create_test_relayer();
2779            let tx = make_test_transaction(TransactionStatus::Submitted);
2780
2781            // Expect partial_update to update status to Failed with status_reason.
2782            mocks
2783                .tx_repo
2784                .expect_partial_update()
2785                .returning(|_, update| {
2786                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2787                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2788                    updated_tx.status_reason = update.status_reason.clone();
2789                    Ok(updated_tx)
2790                });
2791
2792            let reason = "Transaction reverted on-chain (receipt status: failed)".to_string();
2793            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2794            let result = evm_transaction
2795                .handle_final_state(tx.clone(), TransactionStatus::Failed, Some(reason.clone()))
2796                .await
2797                .unwrap();
2798            assert_eq!(result.status, TransactionStatus::Failed);
2799            assert_eq!(result.status_reason.as_deref(), Some(reason.as_str()));
2800        }
2801
2802        #[tokio::test]
2803        async fn test_final_state_expired() {
2804            let mut mocks = default_test_mocks();
2805            let relayer = create_test_relayer();
2806            let tx = make_test_transaction(TransactionStatus::Submitted);
2807
2808            // Expect partial_update to update status to Expired.
2809            mocks
2810                .tx_repo
2811                .expect_partial_update()
2812                .returning(|_, update| {
2813                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2814                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2815                    Ok(updated_tx)
2816                });
2817
2818            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2819            let result = evm_transaction
2820                .handle_final_state(tx.clone(), TransactionStatus::Expired, None)
2821                .await
2822                .unwrap();
2823            assert_eq!(result.status, TransactionStatus::Expired);
2824        }
2825    }
2826
2827    // Integration tests for `handle_status_impl`
2828    mod handle_status_impl_tests {
2829        use super::*;
2830
2831        #[tokio::test]
2832        async fn test_impl_submitted_branch() {
2833            let mut mocks = default_test_mocks();
2834            let relayer = create_test_relayer();
2835            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2836            tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
2837            // Set a dummy hash so check_transaction_status can proceed.
2838            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2839                evm_data.hash = Some("0xFakeHash".to_string());
2840            }
2841            // Simulate no receipt found.
2842            mocks
2843                .provider
2844                .expect_get_transaction_receipt()
2845                .returning(|_| Box::pin(async { Ok(None) }));
2846            // Mock network repository for should_resubmit check
2847            mocks
2848                .network_repo
2849                .expect_get_by_chain_id()
2850                .returning(|_, _| Ok(Some(create_test_network_model())));
2851            // Expect that a status check job is scheduled.
2852            mocks
2853                .job_producer
2854                .expect_produce_check_transaction_status_job()
2855                .returning(|_, _| Box::pin(async { Ok(()) }));
2856            // Expect update_transaction_status_if_needed to update status to Submitted.
2857            mocks
2858                .tx_repo
2859                .expect_partial_update()
2860                .returning(|_, update| {
2861                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2862                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2863                    Ok(updated_tx)
2864                });
2865
2866            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2867            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2868            assert_eq!(result.status, TransactionStatus::Submitted);
2869        }
2870
2871        #[tokio::test]
2872        async fn test_impl_mined_branch() {
2873            let mut mocks = default_test_mocks();
2874            let relayer = create_test_relayer();
2875            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2876            // Set created_at to be old enough to pass is_too_early_to_resubmit
2877            tx.created_at = (Utc::now() - Duration::minutes(1)).to_rfc3339();
2878            // Set a dummy hash.
2879            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2880                evm_data.hash = Some("0xFakeHash".to_string());
2881            }
2882            // Simulate a receipt with a block number of 100 and a successful receipt.
2883            mocks
2884                .provider
2885                .expect_get_transaction_receipt()
2886                .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
2887            // Simulate that the current block number is 100 (so confirmations are insufficient).
2888            mocks
2889                .provider
2890                .expect_get_block_number()
2891                .return_once(|| Box::pin(async { Ok(100) }));
2892            // Mock network repository to return a test network model
2893            mocks
2894                .network_repo
2895                .expect_get_by_chain_id()
2896                .returning(|_, _| Ok(Some(create_test_network_model())));
2897            // Mock the notification job that gets sent after status update
2898            mocks
2899                .job_producer
2900                .expect_produce_send_notification_job()
2901                .returning(|_, _| Box::pin(async { Ok(()) }));
2902            // Expect get_by_id to reload the transaction after status change
2903            mocks.tx_repo.expect_get_by_id().returning(|_| {
2904                let updated_tx = make_test_transaction(TransactionStatus::Mined);
2905                Ok(updated_tx)
2906            });
2907            // Expect update_transaction_status_if_needed to update status to Mined.
2908            mocks
2909                .tx_repo
2910                .expect_partial_update()
2911                .returning(|_, update| {
2912                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2913                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2914                    Ok(updated_tx)
2915                });
2916
2917            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2918            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2919            assert_eq!(result.status, TransactionStatus::Mined);
2920        }
2921
2922        #[tokio::test]
2923        async fn test_impl_final_confirmed_branch() {
2924            let mut mocks = default_test_mocks();
2925            let relayer = create_test_relayer();
2926            // Create a transaction with status Confirmed.
2927            let tx = make_test_transaction(TransactionStatus::Confirmed);
2928
2929            // In this branch, check_transaction_status returns the final status immediately,
2930            // so we expect partial_update to update the transaction status to Confirmed.
2931            mocks
2932                .tx_repo
2933                .expect_partial_update()
2934                .returning(|_, update| {
2935                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2936                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2937                    Ok(updated_tx)
2938                });
2939
2940            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2941            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2942            assert_eq!(result.status, TransactionStatus::Confirmed);
2943        }
2944
2945        #[tokio::test]
2946        async fn test_impl_final_failed_branch() {
2947            let mut mocks = default_test_mocks();
2948            let relayer = create_test_relayer();
2949            // Create a transaction with status Failed.
2950            let tx = make_test_transaction(TransactionStatus::Failed);
2951
2952            mocks
2953                .tx_repo
2954                .expect_partial_update()
2955                .returning(|_, update| {
2956                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2957                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2958                    Ok(updated_tx)
2959                });
2960
2961            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2962            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2963            assert_eq!(result.status, TransactionStatus::Failed);
2964        }
2965
2966        /// Verifies that a Submitted transaction with a failed on-chain receipt
2967        /// transitions to Failed status with a descriptive status_reason.
2968        #[tokio::test]
2969        async fn test_impl_submitted_to_failed_sets_status_reason() {
2970            let mut mocks = default_test_mocks();
2971            let relayer = create_test_relayer();
2972            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2973            tx.created_at = (Utc::now() - Duration::minutes(1)).to_rfc3339();
2974            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2975                evm_data.hash = Some("0xFakeHash".to_string());
2976            }
2977
2978            // Simulate a receipt with status=false (reverted on-chain).
2979            mocks
2980                .provider
2981                .expect_get_transaction_receipt()
2982                .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(false, Some(100)))) }));
2983
2984            // Mock get_by_id for the DB reload after status change.
2985            let tx_clone = tx.clone();
2986            mocks.tx_repo.expect_get_by_id().returning(move |_| {
2987                let mut reloaded = tx_clone.clone();
2988                reloaded.status = TransactionStatus::Submitted;
2989                Ok(reloaded)
2990            });
2991
2992            // Expect partial_update with status=Failed and a status_reason.
2993            mocks
2994                .tx_repo
2995                .expect_partial_update()
2996                .withf(|_, update| {
2997                    update.status == Some(TransactionStatus::Failed)
2998                        && update.status_reason.is_some()
2999                })
3000                .returning(|_, update| {
3001                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3002                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3003                    updated_tx.status_reason = update.status_reason.clone();
3004                    Ok(updated_tx)
3005                });
3006
3007            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3008            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3009            assert_eq!(result.status, TransactionStatus::Failed);
3010            assert!(result.status_reason.is_some());
3011            assert!(
3012                result
3013                    .status_reason
3014                    .as_ref()
3015                    .unwrap()
3016                    .contains("reverted on-chain"),
3017                "Expected on-chain revert reason, got: {:?}",
3018                result.status_reason
3019            );
3020        }
3021
3022        #[tokio::test]
3023        async fn test_impl_final_expired_branch() {
3024            let mut mocks = default_test_mocks();
3025            let relayer = create_test_relayer();
3026            // Create a transaction with status Expired.
3027            let tx = make_test_transaction(TransactionStatus::Expired);
3028
3029            mocks
3030                .tx_repo
3031                .expect_partial_update()
3032                .returning(|_, update| {
3033                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3034                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3035                    Ok(updated_tx)
3036                });
3037
3038            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3039            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3040            assert_eq!(result.status, TransactionStatus::Expired);
3041        }
3042    }
3043
3044    // Tests for circuit breaker functionality
3045    mod circuit_breaker_tests {
3046        use super::*;
3047        use crate::jobs::StatusCheckContext;
3048
3049        /// Helper to create a context that should trigger the circuit breaker
3050        fn create_triggered_context() -> StatusCheckContext {
3051            StatusCheckContext::new(
3052                30, // consecutive_failures: exceeds EVM threshold of 25
3053                50, // total_failures
3054                60, // total_retries
3055                25, // max_consecutive_failures (EVM default)
3056                75, // max_total_failures (EVM default)
3057                NetworkType::Evm,
3058            )
3059        }
3060
3061        /// Helper to create a context that should NOT trigger the circuit breaker
3062        fn create_safe_context() -> StatusCheckContext {
3063            StatusCheckContext::new(
3064                5,  // consecutive_failures: below threshold
3065                10, // total_failures
3066                15, // total_retries
3067                25, // max_consecutive_failures
3068                75, // max_total_failures
3069                NetworkType::Evm,
3070            )
3071        }
3072
3073        /// Helper to create a context that triggers via total failures (safety net)
3074        fn create_total_triggered_context() -> StatusCheckContext {
3075            StatusCheckContext::new(
3076                5,   // consecutive_failures: below threshold
3077                80,  // total_failures: exceeds EVM threshold of 75
3078                100, // total_retries
3079                25,  // max_consecutive_failures
3080                75,  // max_total_failures
3081                NetworkType::Evm,
3082            )
3083        }
3084
3085        #[tokio::test]
3086        async fn test_circuit_breaker_pending_marks_as_failed() {
3087            let mut mocks = default_test_mocks();
3088            let relayer = create_test_relayer();
3089            let tx = make_test_transaction(TransactionStatus::Pending);
3090
3091            // Expect partial_update to be called with Failed status
3092            mocks
3093                .tx_repo
3094                .expect_partial_update()
3095                .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3096                .returning(|_, update| {
3097                    let mut updated_tx = make_test_transaction(TransactionStatus::Pending);
3098                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3099                    updated_tx.status_reason = update.status_reason.clone();
3100                    Ok(updated_tx)
3101                });
3102
3103            // Mock notification (best effort, may or may not be called)
3104            mocks
3105                .job_producer
3106                .expect_produce_send_notification_job()
3107                .returning(|_, _| Box::pin(async { Ok(()) }));
3108
3109            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3110            let ctx = create_triggered_context();
3111
3112            let result = evm_transaction
3113                .handle_status_impl(tx, Some(ctx))
3114                .await
3115                .unwrap();
3116
3117            assert_eq!(result.status, TransactionStatus::Failed);
3118            assert!(result.status_reason.is_some());
3119            assert!(result.status_reason.unwrap().contains("consecutive errors"));
3120        }
3121
3122        #[tokio::test]
3123        async fn test_circuit_breaker_sent_marks_as_failed() {
3124            let mut mocks = default_test_mocks();
3125            let relayer = create_test_relayer();
3126            let tx = make_test_transaction(TransactionStatus::Sent);
3127
3128            // Expect partial_update to be called with Failed status
3129            mocks
3130                .tx_repo
3131                .expect_partial_update()
3132                .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3133                .returning(|_, update| {
3134                    let mut updated_tx = make_test_transaction(TransactionStatus::Sent);
3135                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3136                    updated_tx.status_reason = update.status_reason.clone();
3137                    Ok(updated_tx)
3138                });
3139
3140            // Mock notification
3141            mocks
3142                .job_producer
3143                .expect_produce_send_notification_job()
3144                .returning(|_, _| Box::pin(async { Ok(()) }));
3145
3146            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3147            let ctx = create_triggered_context();
3148
3149            let result = evm_transaction
3150                .handle_status_impl(tx, Some(ctx))
3151                .await
3152                .unwrap();
3153
3154            assert_eq!(result.status, TransactionStatus::Failed);
3155        }
3156
3157        #[tokio::test]
3158        async fn test_circuit_breaker_submitted_triggers_noop() {
3159            let mut mocks = default_test_mocks();
3160            let relayer = create_test_relayer();
3161            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3162            tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3163
3164            // Mock network repository for NOOP processing
3165            mocks
3166                .network_repo
3167                .expect_get_by_chain_id()
3168                .returning(|_, _| Ok(Some(create_test_network_model())));
3169
3170            // Expect partial_update to be called with NOOP indicator
3171            mocks
3172                .tx_repo
3173                .expect_partial_update()
3174                .returning(|_, update| {
3175                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3176                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3177                    updated_tx.status_reason = update.status_reason.clone();
3178                    updated_tx.noop_count = update.noop_count;
3179                    Ok(updated_tx)
3180                });
3181
3182            // Mock resubmit job (NOOP triggers resubmit)
3183            mocks
3184                .job_producer
3185                .expect_produce_submit_transaction_job()
3186                .returning(|_, _| Box::pin(async { Ok(()) }));
3187
3188            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3189            let ctx = create_triggered_context();
3190
3191            let result = evm_transaction
3192                .handle_status_impl(tx, Some(ctx))
3193                .await
3194                .unwrap();
3195
3196            // NOOP processing should succeed
3197            assert!(result.noop_count.is_some());
3198        }
3199
3200        #[tokio::test]
3201        async fn test_circuit_breaker_noop_tx_excluded() {
3202            let mut mocks = default_test_mocks();
3203            let relayer = create_test_relayer();
3204
3205            // Create a NOOP transaction (to: self, value: 0, data: "0x")
3206            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3207            tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3208            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3209                evm_data.to = Some(evm_data.from.clone()); // to == from (NOOP indicator)
3210                evm_data.value = U256::from(0);
3211                evm_data.data = Some("0x".to_string());
3212                evm_data.hash = Some("0xFakeHash".to_string());
3213            }
3214
3215            // NOOP transactions should NOT trigger circuit breaker
3216            // Instead, they should go through normal status checking
3217            mocks
3218                .provider
3219                .expect_get_transaction_receipt()
3220                .returning(|_| Box::pin(async { Ok(None) }));
3221
3222            mocks
3223                .network_repo
3224                .expect_get_by_chain_id()
3225                .returning(|_, _| Ok(Some(create_test_network_model())));
3226
3227            mocks
3228                .job_producer
3229                .expect_produce_check_transaction_status_job()
3230                .returning(|_, _| Box::pin(async { Ok(()) }));
3231
3232            // Mock resubmit job (may be triggered by normal status flow for stuck transactions)
3233            mocks
3234                .job_producer
3235                .expect_produce_submit_transaction_job()
3236                .returning(|_, _| Box::pin(async { Ok(()) }));
3237
3238            mocks
3239                .tx_repo
3240                .expect_partial_update()
3241                .returning(|_, update| {
3242                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3243                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3244                    Ok(updated_tx)
3245                });
3246
3247            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3248            let ctx = create_triggered_context();
3249
3250            let result = evm_transaction
3251                .handle_status_impl(tx, Some(ctx))
3252                .await
3253                .unwrap();
3254
3255            // NOOP tx should continue normal processing, not be force-failed
3256            assert_eq!(result.status, TransactionStatus::Submitted);
3257        }
3258
3259        #[tokio::test]
3260        async fn test_circuit_breaker_total_failures_triggers() {
3261            let mut mocks = default_test_mocks();
3262            let relayer = create_test_relayer();
3263            let tx = make_test_transaction(TransactionStatus::Pending);
3264
3265            // Expect partial_update to be called with Failed status
3266            mocks
3267                .tx_repo
3268                .expect_partial_update()
3269                .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3270                .returning(|_, update| {
3271                    let mut updated_tx = make_test_transaction(TransactionStatus::Pending);
3272                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3273                    updated_tx.status_reason = update.status_reason.clone();
3274                    Ok(updated_tx)
3275                });
3276
3277            mocks
3278                .job_producer
3279                .expect_produce_send_notification_job()
3280                .returning(|_, _| Box::pin(async { Ok(()) }));
3281
3282            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3283            // Use context that triggers via total failures (safety net)
3284            let ctx = create_total_triggered_context();
3285
3286            let result = evm_transaction
3287                .handle_status_impl(tx, Some(ctx))
3288                .await
3289                .unwrap();
3290
3291            assert_eq!(result.status, TransactionStatus::Failed);
3292            assert!(result.status_reason.is_some());
3293        }
3294
3295        #[tokio::test]
3296        async fn test_circuit_breaker_below_threshold_continues_normally() {
3297            let mut mocks = default_test_mocks();
3298            let relayer = create_test_relayer();
3299            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3300            tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3301
3302            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3303                evm_data.hash = Some("0xFakeHash".to_string());
3304            }
3305
3306            // Below threshold, should continue with normal status checking
3307            mocks
3308                .provider
3309                .expect_get_transaction_receipt()
3310                .returning(|_| Box::pin(async { Ok(None) }));
3311
3312            mocks
3313                .network_repo
3314                .expect_get_by_chain_id()
3315                .returning(|_, _| Ok(Some(create_test_network_model())));
3316
3317            mocks
3318                .job_producer
3319                .expect_produce_check_transaction_status_job()
3320                .returning(|_, _| Box::pin(async { Ok(()) }));
3321
3322            mocks
3323                .tx_repo
3324                .expect_partial_update()
3325                .returning(|_, update| {
3326                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3327                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3328                    Ok(updated_tx)
3329                });
3330
3331            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3332            let ctx = create_safe_context();
3333
3334            let result = evm_transaction
3335                .handle_status_impl(tx, Some(ctx))
3336                .await
3337                .unwrap();
3338
3339            // Should continue normal processing, not trigger circuit breaker
3340            assert_eq!(result.status, TransactionStatus::Submitted);
3341        }
3342
3343        #[tokio::test]
3344        async fn test_circuit_breaker_no_context_continues_normally() {
3345            let mut mocks = default_test_mocks();
3346            let relayer = create_test_relayer();
3347            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3348            tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3349
3350            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3351                evm_data.hash = Some("0xFakeHash".to_string());
3352            }
3353
3354            // No context means no circuit breaker, should continue normally
3355            mocks
3356                .provider
3357                .expect_get_transaction_receipt()
3358                .returning(|_| Box::pin(async { Ok(None) }));
3359
3360            mocks
3361                .network_repo
3362                .expect_get_by_chain_id()
3363                .returning(|_, _| Ok(Some(create_test_network_model())));
3364
3365            mocks
3366                .job_producer
3367                .expect_produce_check_transaction_status_job()
3368                .returning(|_, _| Box::pin(async { Ok(()) }));
3369
3370            mocks
3371                .tx_repo
3372                .expect_partial_update()
3373                .returning(|_, update| {
3374                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3375                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3376                    Ok(updated_tx)
3377                });
3378
3379            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3380
3381            // Pass None for context
3382            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3383
3384            // Should continue normal processing
3385            assert_eq!(result.status, TransactionStatus::Submitted);
3386        }
3387
3388        #[tokio::test]
3389        async fn test_circuit_breaker_final_state_early_return() {
3390            let mocks = default_test_mocks();
3391            let relayer = create_test_relayer();
3392            // Transaction is already in final state
3393            let tx = make_test_transaction(TransactionStatus::Confirmed);
3394
3395            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3396            let ctx = create_triggered_context();
3397
3398            // Even with triggered context, final states should return early
3399            let result = evm_transaction
3400                .handle_status_impl(tx, Some(ctx))
3401                .await
3402                .unwrap();
3403
3404            assert_eq!(result.status, TransactionStatus::Confirmed);
3405        }
3406    }
3407
3408    // Tests for hash recovery functions
3409    mod hash_recovery_tests {
3410        use super::*;
3411
3412        #[tokio::test]
3413        async fn test_should_try_hash_recovery_not_submitted() {
3414            let mocks = default_test_mocks();
3415            let relayer = create_test_relayer();
3416
3417            let mut tx = make_test_transaction(TransactionStatus::Sent);
3418            tx.hashes = vec![
3419                "0xHash1".to_string(),
3420                "0xHash2".to_string(),
3421                "0xHash3".to_string(),
3422            ];
3423
3424            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3425            let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3426
3427            assert!(
3428                !result,
3429                "Should not attempt recovery for non-Submitted transactions"
3430            );
3431        }
3432
3433        #[tokio::test]
3434        async fn test_should_try_hash_recovery_not_enough_hashes() {
3435            let mocks = default_test_mocks();
3436            let relayer = create_test_relayer();
3437
3438            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3439            tx.hashes = vec!["0xHash1".to_string()]; // Only 1 hash
3440            tx.sent_at = Some((Utc::now() - Duration::minutes(3)).to_rfc3339());
3441
3442            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3443            let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3444
3445            assert!(
3446                !result,
3447                "Should not attempt recovery with insufficient hashes"
3448            );
3449        }
3450
3451        #[tokio::test]
3452        async fn test_should_try_hash_recovery_too_recent() {
3453            let mocks = default_test_mocks();
3454            let relayer = create_test_relayer();
3455
3456            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3457            tx.hashes = vec![
3458                "0xHash1".to_string(),
3459                "0xHash2".to_string(),
3460                "0xHash3".to_string(),
3461            ];
3462            tx.sent_at = Some(Utc::now().to_rfc3339()); // Recent
3463
3464            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3465            let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3466
3467            assert!(
3468                !result,
3469                "Should not attempt recovery for recently sent transactions"
3470            );
3471        }
3472
3473        #[tokio::test]
3474        async fn test_should_try_hash_recovery_success() {
3475            let mocks = default_test_mocks();
3476            let relayer = create_test_relayer();
3477
3478            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3479            tx.hashes = vec![
3480                "0xHash1".to_string(),
3481                "0xHash2".to_string(),
3482                "0xHash3".to_string(),
3483            ];
3484            tx.sent_at = Some((Utc::now() - Duration::minutes(3)).to_rfc3339());
3485
3486            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3487            let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3488
3489            assert!(
3490                result,
3491                "Should attempt recovery for stuck transactions with multiple hashes"
3492            );
3493        }
3494
3495        #[tokio::test]
3496        async fn test_try_recover_no_historical_hash_found() {
3497            let mut mocks = default_test_mocks();
3498            let relayer = create_test_relayer();
3499
3500            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3501            tx.hashes = vec![
3502                "0xHash1".to_string(),
3503                "0xHash2".to_string(),
3504                "0xHash3".to_string(),
3505            ];
3506
3507            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3508                evm_data.hash = Some("0xHash3".to_string());
3509            }
3510
3511            // Mock provider to return None for all hash lookups
3512            mocks
3513                .provider
3514                .expect_get_transaction_receipt()
3515                .returning(|_| Box::pin(async { Ok(None) }));
3516
3517            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3518            let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3519            let result = evm_transaction
3520                .try_recover_with_historical_hashes(&tx, &evm_data)
3521                .await
3522                .unwrap();
3523
3524            assert!(
3525                result.is_none(),
3526                "Should return None when no historical hash is found"
3527            );
3528        }
3529
3530        #[tokio::test]
3531        async fn test_try_recover_finds_mined_historical_hash() {
3532            let mut mocks = default_test_mocks();
3533            let relayer = create_test_relayer();
3534
3535            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3536            tx.hashes = vec![
3537                "0xHash1".to_string(),
3538                "0xHash2".to_string(), // This one is mined
3539                "0xHash3".to_string(),
3540            ];
3541
3542            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3543                evm_data.hash = Some("0xHash3".to_string()); // Current hash (wrong one)
3544            }
3545
3546            // Mock provider to return None for Hash1 and Hash3, but receipt for Hash2
3547            mocks
3548                .provider
3549                .expect_get_transaction_receipt()
3550                .returning(|hash| {
3551                    if hash == "0xHash2" {
3552                        Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) })
3553                    } else {
3554                        Box::pin(async { Ok(None) })
3555                    }
3556                });
3557
3558            // Mock partial_update for correcting the hash
3559            let tx_clone = tx.clone();
3560            mocks
3561                .tx_repo
3562                .expect_partial_update()
3563                .returning(move |_, update| {
3564                    let mut updated_tx = tx_clone.clone();
3565                    if let Some(status) = update.status {
3566                        updated_tx.status = status;
3567                    }
3568                    if let Some(NetworkTransactionData::Evm(ref evm_data)) = update.network_data {
3569                        if let NetworkTransactionData::Evm(ref mut updated_evm) =
3570                            updated_tx.network_data
3571                        {
3572                            updated_evm.hash = evm_data.hash.clone();
3573                        }
3574                    }
3575                    Ok(updated_tx)
3576                });
3577
3578            // Mock notification job
3579            mocks
3580                .job_producer
3581                .expect_produce_send_notification_job()
3582                .returning(|_, _| Box::pin(async { Ok(()) }));
3583
3584            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3585            let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3586            let result = evm_transaction
3587                .try_recover_with_historical_hashes(&tx, &evm_data)
3588                .await
3589                .unwrap();
3590
3591            assert!(result.is_some(), "Should recover the transaction");
3592            let recovered_tx = result.unwrap();
3593            assert_eq!(recovered_tx.status, TransactionStatus::Mined);
3594        }
3595
3596        #[tokio::test]
3597        async fn test_try_recover_network_error_continues() {
3598            let mut mocks = default_test_mocks();
3599            let relayer = create_test_relayer();
3600
3601            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3602            tx.hashes = vec![
3603                "0xHash1".to_string(),
3604                "0xHash2".to_string(), // Network error
3605                "0xHash3".to_string(), // This one is mined
3606            ];
3607
3608            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3609                evm_data.hash = Some("0xHash1".to_string());
3610            }
3611
3612            // Mock provider to return error for Hash2, receipt for Hash3
3613            mocks
3614                .provider
3615                .expect_get_transaction_receipt()
3616                .returning(|hash| {
3617                    if hash == "0xHash2" {
3618                        Box::pin(async { Err(crate::services::provider::ProviderError::Timeout) })
3619                    } else if hash == "0xHash3" {
3620                        Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) })
3621                    } else {
3622                        Box::pin(async { Ok(None) })
3623                    }
3624                });
3625
3626            // Mock partial_update for correcting the hash
3627            let tx_clone = tx.clone();
3628            mocks
3629                .tx_repo
3630                .expect_partial_update()
3631                .returning(move |_, update| {
3632                    let mut updated_tx = tx_clone.clone();
3633                    if let Some(status) = update.status {
3634                        updated_tx.status = status;
3635                    }
3636                    Ok(updated_tx)
3637                });
3638
3639            // Mock notification job
3640            mocks
3641                .job_producer
3642                .expect_produce_send_notification_job()
3643                .returning(|_, _| Box::pin(async { Ok(()) }));
3644
3645            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3646            let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3647            let result = evm_transaction
3648                .try_recover_with_historical_hashes(&tx, &evm_data)
3649                .await
3650                .unwrap();
3651
3652            assert!(
3653                result.is_some(),
3654                "Should continue checking after network error and find mined hash"
3655            );
3656        }
3657
3658        #[tokio::test]
3659        async fn test_update_transaction_with_corrected_hash() {
3660            let mut mocks = default_test_mocks();
3661            let relayer = create_test_relayer();
3662
3663            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3664            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3665                evm_data.hash = Some("0xWrongHash".to_string());
3666            }
3667
3668            // Mock partial_update
3669            mocks
3670                .tx_repo
3671                .expect_partial_update()
3672                .returning(move |_, update| {
3673                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3674                    if let Some(status) = update.status {
3675                        updated_tx.status = status;
3676                    }
3677                    if let Some(NetworkTransactionData::Evm(ref evm_data)) = update.network_data {
3678                        if let NetworkTransactionData::Evm(ref mut updated_evm) =
3679                            updated_tx.network_data
3680                        {
3681                            updated_evm.hash = evm_data.hash.clone();
3682                        }
3683                    }
3684                    Ok(updated_tx)
3685                });
3686
3687            // Mock notification job
3688            mocks
3689                .job_producer
3690                .expect_produce_send_notification_job()
3691                .returning(|_, _| Box::pin(async { Ok(()) }));
3692
3693            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3694            let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3695            let result = evm_transaction
3696                .update_transaction_with_corrected_hash(
3697                    &tx,
3698                    &evm_data,
3699                    "0xCorrectHash",
3700                    TransactionStatus::Mined,
3701                )
3702                .await
3703                .unwrap();
3704
3705            assert_eq!(result.status, TransactionStatus::Mined);
3706            if let NetworkTransactionData::Evm(ref updated_evm) = result.network_data {
3707                assert_eq!(updated_evm.hash.as_ref().unwrap(), "0xCorrectHash");
3708            }
3709        }
3710    }
3711
3712    // Tests for check_transaction_status edge cases
3713    mod check_transaction_status_edge_cases {
3714        use super::*;
3715
3716        #[tokio::test]
3717        async fn test_missing_hash_returns_error() {
3718            let mocks = default_test_mocks();
3719            let relayer = create_test_relayer();
3720
3721            let tx = make_test_transaction(TransactionStatus::Submitted);
3722            // Hash is None by default
3723
3724            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3725            let result = evm_transaction.check_transaction_status(&tx).await;
3726
3727            assert!(result.is_err(), "Should return error when hash is missing");
3728        }
3729
3730        #[tokio::test]
3731        async fn test_pending_status_early_return() {
3732            let mocks = default_test_mocks();
3733            let relayer = create_test_relayer();
3734
3735            let tx = make_test_transaction(TransactionStatus::Pending);
3736
3737            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3738            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
3739
3740            assert_eq!(
3741                status,
3742                TransactionStatus::Pending,
3743                "Should return Pending without querying blockchain"
3744            );
3745        }
3746
3747        #[tokio::test]
3748        async fn test_sent_status_early_return() {
3749            let mocks = default_test_mocks();
3750            let relayer = create_test_relayer();
3751
3752            let tx = make_test_transaction(TransactionStatus::Sent);
3753
3754            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3755            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
3756
3757            assert_eq!(
3758                status,
3759                TransactionStatus::Sent,
3760                "Should return Sent without querying blockchain"
3761            );
3762        }
3763
3764        #[tokio::test]
3765        async fn test_final_state_early_return() {
3766            let mocks = default_test_mocks();
3767            let relayer = create_test_relayer();
3768
3769            let tx = make_test_transaction(TransactionStatus::Confirmed);
3770
3771            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3772            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
3773
3774            assert_eq!(
3775                status,
3776                TransactionStatus::Confirmed,
3777                "Should return final state without querying blockchain"
3778            );
3779        }
3780    }
3781
3782    mod nonce_recovery_tests {
3783        use super::*;
3784        use crate::domain::transaction::evm::evm_transaction::TX_NONCE_RECONCILE_TRIGGER;
3785        use crate::jobs::StatusCheckContext;
3786
3787        /// Test reconcile_tx_nonce_state with on_chain_nonce > tx_nonce → marks Failed
3788        #[tokio::test]
3789        async fn test_nonce_recovery_nonce_consumed_externally() {
3790            let mut mocks = default_test_mocks();
3791            let relayer = create_test_relayer();
3792
3793            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3794            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3795                nonce: Some(5),
3796                hash: Some("0xhash".to_string()),
3797                raw: Some(vec![1, 2, 3]),
3798                ..tx.network_data.get_evm_transaction_data().unwrap()
3799            });
3800            tx.sent_at = Some(Utc::now().to_rfc3339());
3801
3802            // No receipt for current hash
3803            mocks
3804                .provider
3805                .expect_get_transaction_receipt()
3806                .returning(|_| Box::pin(async { Ok(None) }));
3807
3808            // On-chain nonce is 10, tx nonce is 5 → consumed externally
3809            mocks
3810                .provider
3811                .expect_get_transaction_count()
3812                .returning(|_| Box::pin(async { Ok(10) }));
3813
3814            // Should update to Failed status
3815            let tx_clone = tx.clone();
3816            mocks
3817                .tx_repo
3818                .expect_partial_update()
3819                .withf(|_, update| {
3820                    update.status == Some(TransactionStatus::Failed)
3821                        && update
3822                            .status_reason
3823                            .as_ref()
3824                            .map(|r| r.contains("consumed externally"))
3825                            .unwrap_or(false)
3826                })
3827                .returning(move |_, update| {
3828                    let mut updated_tx = tx_clone.clone();
3829                    updated_tx.status = update.status.unwrap();
3830                    updated_tx.status_reason = update.status_reason.clone();
3831                    Ok(updated_tx)
3832                });
3833
3834            // Should schedule nonce health job after detecting external consumption
3835            mocks
3836                .job_producer
3837                .expect_produce_relayer_health_check_job()
3838                .withf(|job, scheduled_on| {
3839                    job.relayer_id == "test-relayer-id"
3840                        && job.metadata.as_ref().map_or(false, |m| {
3841                            m.get("health_check_action") == Some(&"nonce_health".to_string())
3842                        })
3843                        && scheduled_on.is_none()
3844                })
3845                .returning(|_, _| Box::pin(async { Ok(()) }));
3846
3847            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3848            let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3849
3850            assert!(result.is_ok());
3851            let recovered = result.unwrap();
3852            assert!(recovered.is_some(), "Expected Some(tx) for consumed nonce");
3853            assert_eq!(recovered.unwrap().status, TransactionStatus::Failed);
3854        }
3855
3856        /// Test reconcile_tx_nonce_state with on_chain_nonce <= tx_nonce → returns None
3857        #[tokio::test]
3858        async fn test_nonce_recovery_nonce_not_consumed() {
3859            let mut mocks = default_test_mocks();
3860            let relayer = create_test_relayer();
3861
3862            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3863            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3864                nonce: Some(5),
3865                hash: Some("0xhash".to_string()),
3866                raw: Some(vec![1, 2, 3]),
3867                ..tx.network_data.get_evm_transaction_data().unwrap()
3868            });
3869
3870            // No receipt for current hash
3871            mocks
3872                .provider
3873                .expect_get_transaction_receipt()
3874                .returning(|_| Box::pin(async { Ok(None) }));
3875
3876            // On-chain nonce is 5, same as tx nonce → not consumed yet
3877            mocks
3878                .provider
3879                .expect_get_transaction_count()
3880                .returning(|_| Box::pin(async { Ok(5) }));
3881
3882            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3883            let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3884
3885            assert!(result.is_ok());
3886            assert!(
3887                result.unwrap().is_none(),
3888                "Expected None when nonce not consumed"
3889            );
3890        }
3891
3892        /// Test reconcile_tx_nonce_state with receipt found → returns None (defer to normal flow)
3893        #[tokio::test]
3894        async fn test_nonce_recovery_receipt_found() {
3895            let mut mocks = default_test_mocks();
3896            let relayer = create_test_relayer();
3897
3898            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3899            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3900                nonce: Some(5),
3901                hash: Some("0xhash".to_string()),
3902                raw: Some(vec![1, 2, 3]),
3903                ..tx.network_data.get_evm_transaction_data().unwrap()
3904            });
3905
3906            // Receipt exists for current hash — defer to normal flow
3907            mocks
3908                .provider
3909                .expect_get_transaction_receipt()
3910                .returning(|_| {
3911                    let receipt = make_mock_receipt(true, Some(100));
3912                    Box::pin(async move { Ok(Some(receipt)) })
3913                });
3914
3915            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3916            let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3917
3918            assert!(result.is_ok());
3919            assert!(
3920                result.unwrap().is_none(),
3921                "Expected None when receipt found — defer to normal flow"
3922            );
3923        }
3924
3925        /// Test reconcile_tx_nonce_state with RPC errors during receipt check → returns None
3926        /// Must NOT proceed to force-fail via nonce comparison when hash checks were incomplete
3927        #[tokio::test]
3928        async fn test_nonce_recovery_rpc_error_prevents_force_fail() {
3929            let mut mocks = default_test_mocks();
3930            let relayer = create_test_relayer();
3931
3932            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3933            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3934                nonce: Some(5),
3935                hash: Some("0xhash".to_string()),
3936                raw: Some(vec![1, 2, 3]),
3937                ..tx.network_data.get_evm_transaction_data().unwrap()
3938            });
3939
3940            // Receipt check FAILS with RPC error
3941            mocks
3942                .provider
3943                .expect_get_transaction_receipt()
3944                .returning(|_| {
3945                    Box::pin(async {
3946                        Err(crate::services::provider::ProviderError::Other(
3947                            "RPC timeout".to_string(),
3948                        ))
3949                    })
3950                });
3951
3952            // get_transaction_count should NOT be called — we bail before reaching it
3953            // (no expectation set = will panic if called)
3954
3955            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3956            let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3957
3958            assert!(result.is_ok());
3959            assert!(
3960                result.unwrap().is_none(),
3961                "Expected None when RPC errors occurred — must not force-fail on incomplete data"
3962            );
3963        }
3964
3965        /// Test handle_status_impl with nonce_error_hint metadata triggers recovery
3966        #[tokio::test]
3967        async fn test_handle_status_impl_nonce_recovery_hint() {
3968            let mut mocks = default_test_mocks();
3969            let relayer = create_test_relayer();
3970
3971            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3972            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3973                nonce: Some(5),
3974                hash: Some("0xhash".to_string()),
3975                raw: Some(vec![1, 2, 3]),
3976                ..tx.network_data.get_evm_transaction_data().unwrap()
3977            });
3978            tx.sent_at = Some(Utc::now().to_rfc3339());
3979
3980            // No receipt for current hash
3981            mocks
3982                .provider
3983                .expect_get_transaction_receipt()
3984                .returning(|_| Box::pin(async { Ok(None) }));
3985
3986            // On-chain nonce > tx nonce → consumed externally
3987            mocks
3988                .provider
3989                .expect_get_transaction_count()
3990                .returning(|_| Box::pin(async { Ok(10) }));
3991
3992            // Should update to Failed
3993            let tx_clone = tx.clone();
3994            mocks
3995                .tx_repo
3996                .expect_partial_update()
3997                .returning(move |_, update| {
3998                    let mut updated_tx = tx_clone.clone();
3999                    if let Some(status) = update.status {
4000                        updated_tx.status = status;
4001                    }
4002                    updated_tx.status_reason = update.status_reason.clone();
4003                    Ok(updated_tx)
4004                });
4005
4006            // Should schedule nonce health job after detecting external consumption
4007            mocks
4008                .job_producer
4009                .expect_produce_relayer_health_check_job()
4010                .returning(|_, _| Box::pin(async { Ok(()) }));
4011
4012            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4013
4014            // Build context with nonce_error_hint metadata
4015            let mut metadata = std::collections::HashMap::new();
4016            metadata.insert(
4017                TX_NONCE_RECONCILE_TRIGGER.to_string(),
4018                "NonceTooLow".to_string(),
4019            );
4020            let context = StatusCheckContext::default().with_job_metadata(Some(metadata));
4021
4022            let result = evm_transaction.handle_status_impl(tx, Some(context)).await;
4023            assert!(result.is_ok());
4024            assert_eq!(result.unwrap().status, TransactionStatus::Failed);
4025        }
4026    }
4027
4028    mod circuit_breaker_sent_state_tests {
4029        use super::*;
4030        use crate::jobs::StatusCheckContext;
4031
4032        /// Test circuit breaker on Sent tx with nonce → issues NOOP + submit job
4033        #[tokio::test]
4034        async fn test_circuit_breaker_sent_with_nonce_issues_noop() {
4035            let mut mocks = default_test_mocks_with_network();
4036            let relayer = create_test_relayer();
4037
4038            let mut tx = make_test_transaction(TransactionStatus::Sent);
4039            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4040                nonce: Some(5),
4041                hash: Some("0xhash".to_string()),
4042                raw: Some(vec![1, 2, 3]),
4043                ..tx.network_data.get_evm_transaction_data().unwrap()
4044            });
4045            tx.sent_at = Some(Utc::now().to_rfc3339());
4046
4047            // process_noop_transaction calls prepare_noop_update_request → partial_update
4048            let tx_clone = tx.clone();
4049            mocks
4050                .tx_repo
4051                .expect_partial_update()
4052                .returning(move |_, update| {
4053                    let mut updated_tx = tx_clone.clone();
4054                    if let Some(status) = update.status {
4055                        updated_tx.status = status;
4056                    }
4057                    updated_tx.status_reason = update.status_reason.clone();
4058                    Ok(updated_tx)
4059                });
4060
4061            // Should produce a submit job (for the NOOP)
4062            mocks
4063                .job_producer
4064                .expect_produce_submit_transaction_job()
4065                .times(1)
4066                .returning(|_, _| Box::pin(async { Ok(()) }));
4067
4068            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4069
4070            // Circuit breaker context that should trigger
4071            let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
4072
4073            let result = evm_transaction
4074                .handle_circuit_breaker_safely(tx, &ctx)
4075                .await;
4076            assert!(result.is_ok());
4077        }
4078
4079        /// Test circuit breaker on Sent tx without nonce → marks Failed
4080        #[tokio::test]
4081        async fn test_circuit_breaker_sent_without_nonce_marks_failed() {
4082            let mut mocks = default_test_mocks();
4083            let relayer = create_test_relayer();
4084
4085            let mut tx = make_test_transaction(TransactionStatus::Sent);
4086            // No nonce assigned
4087            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4088                nonce: None,
4089                hash: None,
4090                raw: None,
4091                ..tx.network_data.get_evm_transaction_data().unwrap()
4092            });
4093            tx.sent_at = Some(Utc::now().to_rfc3339());
4094
4095            // Should mark as Failed
4096            let tx_clone = tx.clone();
4097            mocks
4098                .tx_repo
4099                .expect_partial_update()
4100                .withf(|_, update| update.status == Some(TransactionStatus::Failed))
4101                .returning(move |_, update| {
4102                    let mut updated_tx = tx_clone.clone();
4103                    updated_tx.status = update.status.unwrap();
4104                    Ok(updated_tx)
4105                });
4106
4107            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4108
4109            let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
4110
4111            let result = evm_transaction
4112                .handle_circuit_breaker_safely(tx, &ctx)
4113                .await;
4114            assert!(result.is_ok());
4115            assert_eq!(result.unwrap().status, TransactionStatus::Failed);
4116        }
4117
4118        /// Test circuit breaker on Pending tx → marks Failed (unchanged behavior)
4119        #[tokio::test]
4120        async fn test_circuit_breaker_pending_marks_failed() {
4121            let mut mocks = default_test_mocks();
4122            let relayer = create_test_relayer();
4123
4124            let tx = make_test_transaction(TransactionStatus::Pending);
4125
4126            // Should mark as Failed
4127            let tx_clone = tx.clone();
4128            mocks
4129                .tx_repo
4130                .expect_partial_update()
4131                .withf(|_, update| update.status == Some(TransactionStatus::Failed))
4132                .returning(move |_, update| {
4133                    let mut updated_tx = tx_clone.clone();
4134                    updated_tx.status = update.status.unwrap();
4135                    Ok(updated_tx)
4136                });
4137
4138            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4139
4140            let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
4141
4142            let result = evm_transaction
4143                .handle_circuit_breaker_safely(tx, &ctx)
4144                .await;
4145            assert!(result.is_ok());
4146            assert_eq!(result.unwrap().status, TransactionStatus::Failed);
4147        }
4148    }
4149}