1use alloy::network::ReceiptResponse;
6use chrono::{DateTime, Duration, Utc};
7use eyre::Result;
8use tracing::{debug, error, warn};
9
10use super::super::common::is_active_nonce_status;
11use super::EvmRelayerTransaction;
12use super::{
13 ensure_status, evm_transaction::TX_NONCE_RECONCILE_TRIGGER, get_age_since_status_change,
14 has_enough_confirmations, is_noop, is_too_early_to_resubmit, is_transaction_valid, make_noop,
15 too_many_attempts, too_many_noop_attempts,
16};
17use crate::constants::{
18 get_evm_min_age_for_hash_recovery, get_evm_pending_recovery_trigger_timeout,
19 get_evm_prepare_timeout, get_evm_resend_timeout, ARBITRUM_TIME_TO_RESUBMIT,
20 EVM_MIN_HASHES_FOR_RECOVERY, MAX_GAP_SCAN_RANGE,
21};
22use crate::domain::transaction::common::{
23 get_age_of_sent_at, is_final_state, is_pending_transaction,
24};
25use crate::domain::transaction::util::get_age_since_created;
26use crate::models::{EvmNetwork, NetworkRepoModel, NetworkType};
27use crate::repositories::{NetworkRepository, RelayerRepository};
28use crate::{
29 domain::transaction::evm::price_calculator::PriceCalculatorTrait,
30 jobs::{JobProducerTrait, StatusCheckContext},
31 models::{
32 NetworkTransactionData, RelayerRepoModel, TransactionError, TransactionRepoModel,
33 TransactionStatus, TransactionUpdateRequest,
34 },
35 repositories::{Repository, TransactionCounterTrait, TransactionRepository},
36 services::{provider::EvmProviderTrait, signer::Signer},
37 utils::{get_resubmit_timeout_for_speed, get_resubmit_timeout_with_backoff},
38};
39
40impl<P, RR, NR, TR, J, S, TCR, PC> EvmRelayerTransaction<P, RR, NR, TR, J, S, TCR, PC>
41where
42 P: EvmProviderTrait + Send + Sync,
43 RR: RelayerRepository + Repository<RelayerRepoModel, String> + Send + Sync + 'static,
44 NR: NetworkRepository + Repository<NetworkRepoModel, String> + Send + Sync + 'static,
45 TR: TransactionRepository + Repository<TransactionRepoModel, String> + Send + Sync + 'static,
46 J: JobProducerTrait + Send + Sync + 'static,
47 S: Signer + Send + Sync + 'static,
48 TCR: TransactionCounterTrait + Send + Sync + 'static,
49 PC: PriceCalculatorTrait + Send + Sync,
50{
51 pub(super) async fn check_transaction_status(
52 &self,
53 tx: &TransactionRepoModel,
54 ) -> Result<TransactionStatus, TransactionError> {
55 if is_final_state(&tx.status) {
57 return Ok(tx.status.clone());
58 }
59
60 match tx.status {
63 TransactionStatus::Pending | TransactionStatus::Sent => {
64 return Ok(tx.status.clone());
65 }
66 _ => {}
67 }
68
69 let evm_data = tx.network_data.get_evm_transaction_data()?;
70 let tx_hash = evm_data
71 .hash
72 .as_ref()
73 .ok_or(TransactionError::UnexpectedError(
74 "Transaction hash is missing".to_string(),
75 ))?;
76
77 let receipt_result = self.provider().get_transaction_receipt(tx_hash).await?;
78
79 if let Some(receipt) = receipt_result {
80 if !receipt.inner.status() {
81 return Ok(TransactionStatus::Failed);
82 }
83 let last_block_number = self.provider().get_block_number().await?;
84 let tx_block_number = receipt
85 .block_number
86 .ok_or(TransactionError::UnexpectedError(
87 "Transaction receipt missing block number".to_string(),
88 ))?;
89
90 let network_model = self
91 .network_repository()
92 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
93 .await?
94 .ok_or(TransactionError::UnexpectedError(format!(
95 "Network with chain id {} not found",
96 evm_data.chain_id
97 )))?;
98
99 let network = EvmNetwork::try_from(network_model).map_err(|e| {
100 TransactionError::UnexpectedError(format!(
101 "Error converting network model to EvmNetwork: {e}"
102 ))
103 })?;
104
105 if !has_enough_confirmations(
106 tx_block_number,
107 last_block_number,
108 network.required_confirmations,
109 ) {
110 debug!(
111 tx_id = %tx.id,
112 relayer_id = %tx.relayer_id,
113 tx_hash = %tx_hash,
114 "transaction mined but not confirmed"
115 );
116 return Ok(TransactionStatus::Mined);
117 }
118 if tx.is_canceled == Some(true) {
127 if receipt.to == Some(receipt.from) {
128 debug!(
129 tx_id = %tx.id,
130 relayer_id = %tx.relayer_id,
131 tx_hash = %tx_hash,
132 "cancellation NOOP confirmed on-chain; marking transaction as Canceled"
133 );
134 return Ok(TransactionStatus::Canceled);
135 }
136 debug!(
137 tx_id = %tx.id,
138 relayer_id = %tx.relayer_id,
139 tx_hash = %tx_hash,
140 "cancelled transaction's original mined before the NOOP could replace it; reporting Confirmed"
141 );
142 }
143 Ok(TransactionStatus::Confirmed)
144 } else {
145 debug!(
146 tx_id = %tx.id,
147 relayer_id = %tx.relayer_id,
148 tx_hash = %tx_hash,
149 "transaction not yet mined"
150 );
151
152 if tx.hashes.len() > 1 && self.should_try_hash_recovery(tx)? {
156 if let Some(recovered_tx) = self
157 .try_recover_with_historical_hashes(tx, &evm_data)
158 .await?
159 {
160 return Ok(recovered_tx.status);
162 }
163 }
164
165 Ok(TransactionStatus::Submitted)
166 }
167 }
168
169 pub(super) async fn should_resubmit(
171 &self,
172 tx: &TransactionRepoModel,
173 ) -> Result<bool, TransactionError> {
174 ensure_status(tx, TransactionStatus::Submitted, Some("should_resubmit"))?;
176
177 let evm_data = tx.network_data.get_evm_transaction_data()?;
178 let age = get_age_of_sent_at(tx)?;
179
180 let network_model = self
182 .network_repository()
183 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
184 .await?
185 .ok_or(TransactionError::UnexpectedError(format!(
186 "Network with chain id {} not found",
187 evm_data.chain_id
188 )))?;
189
190 let network = EvmNetwork::try_from(network_model).map_err(|e| {
191 TransactionError::UnexpectedError(format!(
192 "Error converting network model to EvmNetwork: {e}"
193 ))
194 })?;
195
196 let timeout = match network.is_arbitrum() {
197 true => ARBITRUM_TIME_TO_RESUBMIT,
198 false => get_resubmit_timeout_for_speed(&evm_data.speed),
199 };
200
201 let timeout_with_backoff = match network.is_arbitrum() {
202 true => timeout, false => get_resubmit_timeout_with_backoff(timeout, tx.hashes.len()),
204 };
205
206 if age > Duration::milliseconds(timeout_with_backoff) {
207 debug!(
208 tx_id = %tx.id,
209 relayer_id = %tx.relayer_id,
210 age_ms = %age.num_milliseconds(),
211 "transaction has been pending for too long, resubmitting"
212 );
213 return Ok(true);
214 }
215 Ok(false)
216 }
217
218 pub(super) async fn should_noop(
228 &self,
229 tx: &TransactionRepoModel,
230 ) -> Result<(bool, Option<String>), TransactionError> {
231 if too_many_noop_attempts(tx) {
232 debug!("Transaction has too many NOOP attempts already");
233 return Ok((false, None));
234 }
235
236 let evm_data = tx.network_data.get_evm_transaction_data()?;
237 if is_noop(&evm_data) {
238 return Ok((false, None));
239 }
240
241 let network_model = self
242 .network_repository()
243 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
244 .await?
245 .ok_or(TransactionError::UnexpectedError(format!(
246 "Network with chain id {} not found",
247 evm_data.chain_id
248 )))?;
249
250 let network = EvmNetwork::try_from(network_model).map_err(|e| {
251 TransactionError::UnexpectedError(format!(
252 "Error converting network model to EvmNetwork: {e}"
253 ))
254 })?;
255
256 if network.is_rollup() && too_many_attempts(tx) {
257 let reason =
258 "Rollup transaction has too many attempts. Replacing with NOOP.".to_string();
259 debug!(
260 tx_id = %tx.id,
261 relayer_id = %tx.relayer_id,
262 reason = %reason,
263 "replacing transaction with NOOP"
264 );
265 return Ok((true, Some(reason)));
266 }
267
268 if !is_transaction_valid(&tx.created_at, &tx.valid_until) {
269 let reason = "Transaction is expired. Replacing with NOOP.".to_string();
270 debug!(
271 tx_id = %tx.id,
272 relayer_id = %tx.relayer_id,
273 reason = %reason,
274 "replacing transaction with NOOP"
275 );
276 return Ok((true, Some(reason)));
277 }
278
279 if tx.status == TransactionStatus::Pending {
280 let created_at = &tx.created_at;
281 let created_time = DateTime::parse_from_rfc3339(created_at)
282 .map_err(|e| {
283 TransactionError::UnexpectedError(format!("Invalid created_at timestamp: {e}"))
284 })?
285 .with_timezone(&Utc);
286 let age = Utc::now().signed_duration_since(created_time);
287 if age > get_evm_prepare_timeout() {
288 let reason = format!(
289 "Transaction in Pending state for over {} minutes. Replacing with NOOP.",
290 get_evm_prepare_timeout().num_minutes()
291 );
292 debug!(
293 tx_id = %tx.id,
294 relayer_id = %tx.relayer_id,
295 reason = %reason,
296 "replacing transaction with NOOP"
297 );
298 return Ok((true, Some(reason)));
299 }
300 }
301
302 let latest_block = self.provider().get_block_by_number().await;
303 if let Ok(block) = latest_block {
304 let block_gas_limit = block.header.gas_limit;
305 if let Some(gas_limit) = evm_data.gas_limit {
306 if gas_limit > block_gas_limit {
307 let reason = format!(
308 "Transaction gas limit ({gas_limit}) exceeds block gas limit ({block_gas_limit}). Replacing with NOOP.",
309 );
310 warn!(
311 tx_id = %tx.id,
312 tx_gas_limit = %gas_limit,
313 block_gas_limit = %block_gas_limit,
314 "transaction gas limit exceeds block gas limit, replacing with NOOP"
315 );
316 return Ok((true, Some(reason)));
317 }
318 }
319 }
320
321 Ok((false, None))
322 }
323
324 pub(super) async fn update_transaction_status_if_needed(
326 &self,
327 tx: TransactionRepoModel,
328 new_status: TransactionStatus,
329 status_reason: Option<String>,
330 ) -> Result<TransactionRepoModel, TransactionError> {
331 if tx.status != new_status {
332 return self
333 .update_transaction_status(tx, new_status, status_reason)
334 .await;
335 }
336 Ok(tx)
337 }
338
339 pub(super) async fn prepare_noop_update_request(
341 &self,
342 tx: &TransactionRepoModel,
343 is_cancellation: bool,
344 reason: Option<String>,
345 ) -> Result<TransactionUpdateRequest, TransactionError> {
346 let mut evm_data = tx.network_data.get_evm_transaction_data()?;
347 let network_model = self
348 .network_repository()
349 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
350 .await?
351 .ok_or(TransactionError::UnexpectedError(format!(
352 "Network with chain id {} not found",
353 evm_data.chain_id
354 )))?;
355
356 let network = EvmNetwork::try_from(network_model).map_err(|e| {
357 TransactionError::UnexpectedError(format!(
358 "Error converting network model to EvmNetwork: {e}"
359 ))
360 })?;
361
362 make_noop(&mut evm_data, &network, Some(self.provider())).await?;
363
364 let noop_count = tx.noop_count.unwrap_or(0) + 1;
365 let update_request = TransactionUpdateRequest {
366 network_data: Some(NetworkTransactionData::Evm(evm_data)),
367 noop_count: Some(noop_count),
368 status_reason: reason,
369 is_canceled: if is_cancellation {
370 Some(true)
371 } else {
372 tx.is_canceled
373 },
374 ..Default::default()
375 };
376 Ok(update_request)
377 }
378
379 async fn handle_submitted_state(
386 &self,
387 tx: TransactionRepoModel,
388 ) -> Result<TransactionRepoModel, TransactionError> {
389 if self.should_resubmit(&tx).await? {
390 if let Some(nonce_gap_detected) = self.detect_nonce_gap_ahead(&tx).await {
393 if nonce_gap_detected {
394 return self
396 .update_transaction_status_if_needed(tx, TransactionStatus::Submitted, None)
397 .await;
398 }
399 }
400
401 let resubmitted_tx = self.handle_resubmission(tx).await?;
402 return Ok(resubmitted_tx);
403 }
404
405 self.update_transaction_status_if_needed(tx, TransactionStatus::Submitted, None)
406 .await
407 }
408
409 async fn detect_nonce_gap_ahead(&self, tx: &TransactionRepoModel) -> Option<bool> {
423 let evm_data = match tx.network_data.get_evm_transaction_data() {
424 Ok(d) => d,
425 Err(_) => return None,
426 };
427 let tx_nonce = match evm_data.nonce {
428 Some(n) => n,
429 None => return None,
430 };
431
432 let on_chain_nonce = match self
433 .provider()
434 .get_transaction_count(&self.relayer().address)
435 .await
436 {
437 Ok(n) => n,
438 Err(e) => {
439 debug!(
440 tx_id = %tx.id,
441 error = %e,
442 "nonce gap check: failed to get on-chain nonce, skipping"
443 );
444 return None;
445 }
446 };
447
448 if tx_nonce <= on_chain_nonce {
450 return Some(false);
451 }
452
453 let scan_to = std::cmp::min(tx_nonce, on_chain_nonce + MAX_GAP_SCAN_RANGE);
457
458 let occupancy = match self
459 .transaction_repository()
460 .get_nonce_occupancy(&tx.relayer_id, on_chain_nonce, scan_to)
461 .await
462 {
463 Ok(o) => o,
464 Err(e) => {
465 debug!(
466 tx_id = %tx.id,
467 error = %e,
468 "nonce gap check: occupancy lookup failed, skipping"
469 );
470 return None;
471 }
472 };
473
474 let gap_nonces: Vec<u64> = occupancy
475 .into_iter()
476 .filter(|(_, status)| !status.as_ref().is_some_and(is_active_nonce_status))
477 .map(|(nonce, _)| nonce)
478 .collect();
479
480 if gap_nonces.is_empty() {
481 return Some(false);
483 }
484
485 warn!(
486 tx_id = %tx.id,
487 relayer_id = %tx.relayer_id,
488 tx_nonce = tx_nonce,
489 on_chain_nonce = on_chain_nonce,
490 gap_count = gap_nonces.len(),
491 gaps = ?gap_nonces,
492 "nonce gaps confirmed below tx, scheduling nonce health to fill"
493 );
494
495 if let Err(e) = self.schedule_relayer_nonce_health_job(tx).await {
496 warn!(
497 tx_id = %tx.id,
498 error = %e,
499 "failed to schedule nonce health job for nonce gap"
500 );
501 }
502
503 Some(true)
504 }
505
506 async fn handle_resubmission(
508 &self,
509 tx: TransactionRepoModel,
510 ) -> Result<TransactionRepoModel, TransactionError> {
511 debug!(
512 tx_id = %tx.id,
513 relayer_id = %tx.relayer_id,
514 status = ?tx.status,
515 "scheduling resubmit job for transaction"
516 );
517
518 let (should_noop, reason) = self.should_noop(&tx).await?;
520 let tx_to_process = if should_noop {
521 self.process_noop_transaction(&tx, reason).await?
522 } else {
523 tx
524 };
525
526 self.send_transaction_resubmit_job(&tx_to_process).await?;
527 Ok(tx_to_process)
528 }
529
530 async fn process_noop_transaction(
532 &self,
533 tx: &TransactionRepoModel,
534 reason: Option<String>,
535 ) -> Result<TransactionRepoModel, TransactionError> {
536 debug!(
537 tx_id = %tx.id,
538 relayer_id = %tx.relayer_id,
539 status = ?tx.status,
540 "preparing transaction NOOP before resubmission"
541 );
542 let update = self.prepare_noop_update_request(tx, false, reason).await?;
543 let updated_tx = self
544 .transaction_repository()
545 .partial_update(tx.id.clone(), update)
546 .await?;
547
548 let res = self.send_transaction_update_notification(&updated_tx).await;
549 if let Err(e) = res {
550 error!(
551 tx_id = %updated_tx.id,
552 relayer_id = %updated_tx.relayer_id,
553 status = ?updated_tx.status,
554 error = %e,
555 "sending transaction update notification failed for NOOP transaction"
556 );
557 }
558 Ok(updated_tx)
559 }
560
561 async fn handle_pending_state(
563 &self,
564 tx: TransactionRepoModel,
565 ) -> Result<TransactionRepoModel, TransactionError> {
566 let (should_noop, reason) = self.should_noop(&tx).await?;
567 if should_noop {
568 debug!(
571 tx_id = %tx.id,
572 relayer_id = %tx.relayer_id,
573 reason = %reason.as_ref().unwrap_or(&"unknown".to_string()),
574 "marking pending transaction as Failed (nonce not assigned, no NOOP needed)"
575 );
576 let update = TransactionUpdateRequest {
577 status: Some(TransactionStatus::Failed),
578 status_reason: reason,
579 ..Default::default()
580 };
581 let updated_tx = self
582 .transaction_repository()
583 .partial_update(tx.id.clone(), update)
584 .await?;
585
586 let res = self.send_transaction_update_notification(&updated_tx).await;
587 if let Err(e) = res {
588 error!(
589 tx_id = %updated_tx.id,
590 relayer_id = %updated_tx.relayer_id,
591 status = ?updated_tx.status,
592 error = %e,
593 "sending transaction update notification failed for Pending state NOOP"
594 );
595 }
596 return Ok(updated_tx);
597 }
598
599 let age = get_age_since_created(&tx)?;
601 if age > get_evm_pending_recovery_trigger_timeout() {
602 warn!(
603 tx_id = %tx.id,
604 relayer_id = %tx.relayer_id,
605 age_seconds = age.num_seconds(),
606 "transaction stuck in Pending, queuing prepare job"
607 );
608
609 self.send_transaction_request_job(&tx).await?;
611 }
612
613 Ok(tx)
614 }
615
616 async fn handle_mined_state(
618 &self,
619 tx: TransactionRepoModel,
620 ) -> Result<TransactionRepoModel, TransactionError> {
621 self.update_transaction_status_if_needed(tx, TransactionStatus::Mined, None)
622 .await
623 }
624
625 async fn handle_final_state(
627 &self,
628 tx: TransactionRepoModel,
629 status: TransactionStatus,
630 status_reason: Option<String>,
631 ) -> Result<TransactionRepoModel, TransactionError> {
632 self.update_transaction_status_if_needed(tx, status, status_reason)
633 .await
634 }
635
636 async fn mark_as_failed(
638 &self,
639 tx: TransactionRepoModel,
640 reason: String,
641 ) -> Result<TransactionRepoModel, TransactionError> {
642 warn!(
643 tx_id = %tx.id,
644 relayer_id = %tx.relayer_id,
645 reason = %reason,
646 "force-failing transaction due to circuit breaker"
647 );
648
649 let update = TransactionUpdateRequest {
650 status: Some(TransactionStatus::Failed),
651 status_reason: Some(reason),
652 ..Default::default()
653 };
654
655 let updated_tx = self
656 .transaction_repository()
657 .partial_update(tx.id.clone(), update)
658 .await?;
659
660 if let Err(e) = self.send_transaction_update_notification(&updated_tx).await {
662 error!(
663 tx_id = %updated_tx.id,
664 relayer_id = %updated_tx.relayer_id,
665 error = %e,
666 "failed to send notification for force-failed transaction"
667 );
668 }
669
670 Ok(updated_tx)
671 }
672
673 async fn reconcile_tx_nonce_state(
684 &self,
685 tx: &TransactionRepoModel,
686 ) -> Result<Option<TransactionRepoModel>, TransactionError> {
687 let evm_data = tx.network_data.get_evm_transaction_data()?;
688
689 let mut had_rpc_errors = false;
693
694 if let Some(ref hash) = evm_data.hash {
696 match self.provider().get_transaction_receipt(hash).await {
697 Ok(Some(_)) => {
698 debug!(
699 tx_id = %tx.id,
700 hash = %hash,
701 "nonce recovery: receipt found for current hash, deferring to normal flow"
702 );
703 return Ok(None);
704 }
705 Ok(None) => {
706 }
708 Err(e) => {
709 warn!(
710 tx_id = %tx.id,
711 hash = %hash,
712 error = %e,
713 "nonce recovery: error checking receipt for current hash"
714 );
715 had_rpc_errors = true;
716 }
717 }
718 }
719
720 if tx.hashes.len() > 1 {
722 match self.try_recover_with_historical_hashes(tx, &evm_data).await {
723 Ok(Some(recovered_tx)) => {
724 debug!(
725 tx_id = %tx.id,
726 "nonce recovery: recovered transaction via historical hash"
727 );
728 return Ok(Some(recovered_tx));
729 }
730 Ok(None) => {
731 }
733 Err(e) => {
734 warn!(
735 tx_id = %tx.id,
736 error = %e,
737 "nonce recovery: error during historical hash recovery"
738 );
739 had_rpc_errors = true;
740 }
741 }
742 }
743
744 if had_rpc_errors {
748 warn!(
749 tx_id = %tx.id,
750 "nonce recovery: skipping nonce comparison due to RPC errors during hash checks, deferring to normal flow"
751 );
752 return Ok(None);
753 }
754
755 let tx_nonce = match evm_data.nonce {
756 Some(n) => n,
757 None => {
758 return Ok(None);
760 }
761 };
762
763 let on_chain_nonce = self
764 .provider()
765 .get_transaction_count(&self.relayer().address)
766 .await
767 .map_err(|e| {
768 TransactionError::UnexpectedError(format!(
769 "Failed to get on-chain nonce for recovery: {e}"
770 ))
771 })?;
772
773 if on_chain_nonce > tx_nonce {
774 let reason = format!(
776 "Nonce {tx_nonce} consumed externally (on-chain nonce: {on_chain_nonce}). \
777 No matching transaction hash found on-chain."
778 );
779 warn!(
780 tx_id = %tx.id,
781 relayer_id = %tx.relayer_id,
782 tx_nonce = tx_nonce,
783 on_chain_nonce = on_chain_nonce,
784 "nonce recovery: nonce consumed externally, marking as Failed"
785 );
786
787 let updated_tx = self
788 .update_transaction_status(tx.clone(), TransactionStatus::Failed, Some(reason))
789 .await?;
790
791 if let Err(e) = self.schedule_relayer_nonce_health_job(tx).await {
796 warn!(
797 tx_id = %tx.id,
798 error = %e,
799 "nonce recovery: failed to schedule nonce health after external consumption"
800 );
801 }
802
803 return Ok(Some(updated_tx));
804 }
805
806 debug!(
808 tx_id = %tx.id,
809 tx_nonce = tx_nonce,
810 on_chain_nonce = on_chain_nonce,
811 "nonce recovery: on-chain nonce not past tx nonce, deferring to normal flow"
812 );
813 Ok(None)
814 }
815
816 async fn handle_circuit_breaker_safely(
828 &self,
829 tx: TransactionRepoModel,
830 ctx: &StatusCheckContext,
831 ) -> Result<TransactionRepoModel, TransactionError> {
832 let reason = format!(
833 "Transaction status monitoring failed after {} consecutive errors (total: {}). \
834 Last status: {:?}.",
835 ctx.consecutive_failures, ctx.total_failures, tx.status
836 );
837
838 match tx.status {
839 TransactionStatus::Pending => {
840 debug!(
842 tx_id = %tx.id,
843 relayer_id = %tx.relayer_id,
844 "circuit breaker: Pending transaction (no nonce) - safe to mark as Failed"
845 );
846 self.mark_as_failed(tx, reason).await
847 }
848 TransactionStatus::Sent => {
849 let has_nonce = tx
853 .network_data
854 .get_evm_transaction_data()
855 .map(|d| d.nonce.is_some())
856 .unwrap_or(false);
857
858 if has_nonce {
859 warn!(
860 tx_id = %tx.id,
861 relayer_id = %tx.relayer_id,
862 "circuit breaker: Sent transaction with nonce assigned - triggering NOOP to clear nonce slot"
863 );
864 let noop_reason = Some(format!(
865 "{reason}. Replacing with NOOP to clear nonce slot (Sent state with assigned nonce)."
866 ));
867 let updated_tx = self.process_noop_transaction(&tx, noop_reason).await?;
868 self.send_transaction_resubmit_job(&updated_tx).await?;
872 Ok(updated_tx)
873 } else {
874 debug!(
876 tx_id = %tx.id,
877 relayer_id = %tx.relayer_id,
878 "circuit breaker: Sent transaction without nonce - safe to mark as Failed"
879 );
880 self.mark_as_failed(tx, reason).await
881 }
882 }
883 TransactionStatus::Submitted => {
884 warn!(
890 tx_id = %tx.id,
891 relayer_id = %tx.relayer_id,
892 "circuit breaker: Submitted transaction - triggering NOOP to safely clear nonce"
893 );
894 let noop_reason = Some(format!(
895 "{reason}. Replacing with NOOP to clear nonce slot."
896 ));
897 let updated_tx = self.process_noop_transaction(&tx, noop_reason).await?;
898 self.send_transaction_resubmit_job(&updated_tx).await?;
899 Ok(updated_tx)
900 }
901 _ => {
902 debug!(
904 tx_id = %tx.id,
905 relayer_id = %tx.relayer_id,
906 status = ?tx.status,
907 "circuit breaker: unexpected status, returning transaction unchanged"
908 );
909 Ok(tx)
910 }
911 }
912 }
913
914 pub async fn handle_status_impl(
919 &self,
920 tx: TransactionRepoModel,
921 context: Option<StatusCheckContext>,
922 ) -> Result<TransactionRepoModel, TransactionError> {
923 debug!(
924 tx_id = %tx.id,
925 relayer_id = %tx.relayer_id,
926 status = ?tx.status,
927 "checking transaction status"
928 );
929
930 if is_final_state(&tx.status) {
932 debug!(
933 tx_id = %tx.id,
934 relayer_id = %tx.relayer_id,
935 status = ?tx.status,
936 "transaction already in final state"
937 );
938 return Ok(tx);
939 }
940
941 if let Some(ref ctx) = context {
945 let is_noop_tx = tx
946 .network_data
947 .get_evm_transaction_data()
948 .map(|data| is_noop(&data))
949 .unwrap_or(false);
950
951 if ctx.should_force_finalize() && !is_noop_tx {
952 warn!(
953 tx_id = %tx.id,
954 consecutive_failures = ctx.consecutive_failures,
955 total_failures = ctx.total_failures,
956 max_consecutive = ctx.max_consecutive_failures,
957 status = ?tx.status,
958 "circuit breaker triggered - handling safely based on transaction state"
959 );
960 return self.handle_circuit_breaker_safely(tx, ctx).await;
961 }
962
963 if ctx.should_force_finalize() && is_noop_tx {
964 debug!(
965 tx_id = %tx.id,
966 consecutive_failures = ctx.consecutive_failures,
967 relayer_id = %tx.relayer_id,
968 "circuit breaker would trigger but transaction is NOOP - continuing with normal status logic"
969 );
970 }
971 }
972
973 if let Some(ref ctx) = context {
977 if let Some(ref metadata) = ctx.job_metadata {
978 if let Some(hint) = metadata.get(TX_NONCE_RECONCILE_TRIGGER) {
979 debug!(
980 tx_id = %tx.id,
981 hint = %hint,
982 "nonce recovery hint detected - performing nonce reconciliation"
983 );
984 match self.reconcile_tx_nonce_state(&tx).await {
985 Ok(Some(recovered_tx)) => {
986 return Ok(recovered_tx);
987 }
988 Ok(None) => {
989 debug!(
991 tx_id = %tx.id,
992 "nonce recovery did not resolve transaction, continuing normal flow"
993 );
994 }
995 Err(e) => {
996 warn!(
998 tx_id = %tx.id,
999 error = %e,
1000 "nonce recovery failed, falling through to normal status flow"
1001 );
1002 }
1003 }
1004 }
1005 }
1006 }
1007
1008 let status = self.check_transaction_status(&tx).await?;
1013
1014 debug!(
1015 tx_id = %tx.id,
1016 previous_status = ?tx.status,
1017 new_status = ?status,
1018 relayer_id = %tx.relayer_id,
1019 "transaction status check completed"
1020 );
1021
1022 let tx = if status != tx.status {
1026 debug!(
1027 tx_id = %tx.id,
1028 old_status = ?tx.status,
1029 new_status = ?status,
1030 relayer_id = %tx.relayer_id,
1031 "status changed during check, reloading transaction from DB to ensure fresh data"
1032 );
1033 self.transaction_repository()
1034 .get_by_id(tx.id.clone())
1035 .await?
1036 } else {
1037 tx
1038 };
1039
1040 if is_too_early_to_resubmit(&tx)? && is_pending_transaction(&status) {
1045 return self
1047 .update_transaction_status_if_needed(tx, status, None)
1048 .await;
1049 }
1050
1051 match status {
1053 TransactionStatus::Pending => self.handle_pending_state(tx).await,
1054 TransactionStatus::Sent => self.handle_sent_state(tx).await,
1055 TransactionStatus::Submitted => self.handle_submitted_state(tx).await,
1056 TransactionStatus::Mined => self.handle_mined_state(tx).await,
1057 TransactionStatus::Failed => {
1058 let status_reason = if tx.status != TransactionStatus::Failed {
1061 Some("Transaction reverted on-chain (receipt status: failed)".to_string())
1062 } else {
1063 None
1064 };
1065 self.handle_final_state(tx, status, status_reason).await
1066 }
1067 TransactionStatus::Confirmed
1068 | TransactionStatus::Expired
1069 | TransactionStatus::Canceled => self.handle_final_state(tx, status, None).await,
1070 }
1071 }
1072
1073 async fn handle_sent_state(
1075 &self,
1076 tx: TransactionRepoModel,
1077 ) -> Result<TransactionRepoModel, TransactionError> {
1078 debug!(
1079 tx_id = %tx.id,
1080 relayer_id = %tx.relayer_id,
1081 "handling Sent state"
1082 );
1083
1084 let (should_noop, reason) = self.should_noop(&tx).await?;
1086 if should_noop {
1087 debug!(
1088 tx_id = %tx.id,
1089 relayer_id = %tx.relayer_id,
1090 "preparing NOOP for sent transaction"
1091 );
1092 let update = self.prepare_noop_update_request(&tx, false, reason).await?;
1093 let updated_tx = self
1094 .transaction_repository()
1095 .partial_update(tx.id.clone(), update)
1096 .await?;
1097
1098 self.send_transaction_submit_job(&updated_tx).await?;
1099 let res = self.send_transaction_update_notification(&updated_tx).await;
1100 if let Err(e) = res {
1101 error!(
1102 tx_id = %updated_tx.id,
1103 relayer_id = %updated_tx.relayer_id,
1104 status = ?updated_tx.status,
1105 error = %e,
1106 "sending transaction update notification failed for Sent state NOOP"
1107 );
1108 }
1109 return Ok(updated_tx);
1110 }
1111
1112 let age_since_sent = get_age_since_status_change(&tx)?;
1115
1116 if age_since_sent > get_evm_resend_timeout() {
1117 warn!(
1118 tx_id = %tx.id,
1119 relayer_id = %tx.relayer_id,
1120 age_seconds = age_since_sent.num_seconds(),
1121 "transaction stuck in Sent, queuing resubmit job with repricing"
1122 );
1123
1124 self.send_transaction_resubmit_job(&tx).await?;
1126 }
1127
1128 self.update_transaction_status_if_needed(tx, TransactionStatus::Sent, None)
1129 .await
1130 }
1131
1132 fn should_try_hash_recovery(
1139 &self,
1140 tx: &TransactionRepoModel,
1141 ) -> Result<bool, TransactionError> {
1142 if tx.status != TransactionStatus::Submitted {
1144 return Ok(false);
1145 }
1146
1147 if tx.hashes.len() <= 1 {
1149 return Ok(false);
1150 }
1151
1152 let age = get_age_of_sent_at(tx)?;
1154 let min_age_for_recovery = get_evm_min_age_for_hash_recovery();
1155
1156 if age < min_age_for_recovery {
1157 return Ok(false);
1158 }
1159
1160 if tx.hashes.len() < EVM_MIN_HASHES_FOR_RECOVERY {
1163 return Ok(false);
1164 }
1165
1166 Ok(true)
1167 }
1168
1169 async fn try_recover_with_historical_hashes(
1178 &self,
1179 tx: &TransactionRepoModel,
1180 evm_data: &crate::models::EvmTransactionData,
1181 ) -> Result<Option<TransactionRepoModel>, TransactionError> {
1182 warn!(
1183 tx_id = %tx.id,
1184 relayer_id = %tx.relayer_id,
1185 current_hash = ?evm_data.hash,
1186 total_hashes = %tx.hashes.len(),
1187 "attempting hash recovery - checking historical hashes"
1188 );
1189
1190 for (idx, historical_hash) in tx.hashes.iter().rev().enumerate() {
1192 if Some(historical_hash) == evm_data.hash.as_ref() {
1194 continue;
1195 }
1196
1197 debug!(
1198 tx_id = %tx.id,
1199 relayer_id = %tx.relayer_id,
1200 hash = %historical_hash,
1201 index = %idx,
1202 "checking historical hash"
1203 );
1204
1205 match self
1207 .provider()
1208 .get_transaction_receipt(historical_hash)
1209 .await
1210 {
1211 Ok(Some(receipt)) => {
1212 warn!(
1213 tx_id = %tx.id,
1214 relayer_id = %tx.relayer_id,
1215 mined_hash = %historical_hash,
1216 wrong_hash = ?evm_data.hash,
1217 block_number = ?receipt.block_number,
1218 "RECOVERED: found mined transaction with historical hash - correcting database"
1219 );
1220
1221 let updated_tx = self
1224 .update_transaction_with_corrected_hash(
1225 tx,
1226 evm_data,
1227 historical_hash,
1228 TransactionStatus::Mined,
1229 )
1230 .await?;
1231
1232 return Ok(Some(updated_tx));
1233 }
1234 Ok(None) => {
1235 continue;
1237 }
1238 Err(e) => {
1239 warn!(
1241 tx_id = %tx.id,
1242 relayer_id = %tx.relayer_id,
1243 hash = %historical_hash,
1244 error = %e,
1245 "error checking historical hash, continuing to next"
1246 );
1247 continue;
1248 }
1249 }
1250 }
1251
1252 debug!(
1254 tx_id = %tx.id,
1255 relayer_id = %tx.relayer_id,
1256 "hash recovery completed - no historical hashes found on-chain"
1257 );
1258 Ok(None)
1259 }
1260
1261 async fn update_transaction_with_corrected_hash(
1265 &self,
1266 tx: &TransactionRepoModel,
1267 evm_data: &crate::models::EvmTransactionData,
1268 correct_hash: &str,
1269 status: TransactionStatus,
1270 ) -> Result<TransactionRepoModel, TransactionError> {
1271 let mut corrected_data = evm_data.clone();
1272 corrected_data.hash = Some(correct_hash.to_string());
1273
1274 let updated_tx = self
1275 .transaction_repository()
1276 .partial_update(
1277 tx.id.clone(),
1278 TransactionUpdateRequest {
1279 network_data: Some(NetworkTransactionData::Evm(corrected_data)),
1280 status: Some(status),
1281 ..Default::default()
1282 },
1283 )
1284 .await?;
1285
1286 if let Err(e) = self.send_transaction_update_notification(&updated_tx).await {
1288 error!(
1289 tx_id = %updated_tx.id,
1290 relayer_id = %updated_tx.relayer_id,
1291 error = %e,
1292 "failed to send notification for hash recovery"
1293 );
1294 }
1295
1296 Ok(updated_tx)
1297 }
1298}
1299
1300#[cfg(test)]
1301mod tests {
1302 use crate::{
1303 config::{EvmNetworkConfig, NetworkConfigCommon},
1304 domain::transaction::evm::{EvmRelayerTransaction, MockPriceCalculatorTrait},
1305 jobs::MockJobProducerTrait,
1306 models::{
1307 evm::Speed, EvmTransactionData, NetworkConfigData, NetworkRepoModel,
1308 NetworkTransactionData, NetworkType, RelayerEvmPolicy, RelayerNetworkPolicy,
1309 RelayerRepoModel, RpcConfig, TransactionReceipt, TransactionRepoModel,
1310 TransactionStatus, U256,
1311 },
1312 repositories::{
1313 MockNetworkRepository, MockRelayerRepository, MockTransactionCounterTrait,
1314 MockTransactionRepository,
1315 },
1316 services::{provider::MockEvmProviderTrait, signer::MockSigner},
1317 };
1318 use alloy::{
1319 consensus::{Eip658Value, Receipt, ReceiptWithBloom},
1320 network::AnyReceiptEnvelope,
1321 primitives::{b256, Address, BlockHash, Bloom, TxHash},
1322 };
1323 use chrono::{Duration, Utc};
1324 use std::sync::Arc;
1325
1326 pub struct TestMocks {
1328 pub provider: MockEvmProviderTrait,
1329 pub relayer_repo: MockRelayerRepository,
1330 pub network_repo: MockNetworkRepository,
1331 pub tx_repo: MockTransactionRepository,
1332 pub job_producer: MockJobProducerTrait,
1333 pub signer: MockSigner,
1334 pub counter: MockTransactionCounterTrait,
1335 pub price_calc: MockPriceCalculatorTrait,
1336 }
1337
1338 pub fn default_test_mocks() -> TestMocks {
1341 TestMocks {
1342 provider: MockEvmProviderTrait::new(),
1343 relayer_repo: MockRelayerRepository::new(),
1344 network_repo: MockNetworkRepository::new(),
1345 tx_repo: MockTransactionRepository::new(),
1346 job_producer: MockJobProducerTrait::new(),
1347 signer: MockSigner::new(),
1348 counter: MockTransactionCounterTrait::new(),
1349 price_calc: MockPriceCalculatorTrait::new(),
1350 }
1351 }
1352
1353 pub fn default_test_mocks_with_network() -> TestMocks {
1355 let mut mocks = default_test_mocks();
1356 mocks
1358 .network_repo
1359 .expect_get_by_chain_id()
1360 .returning(|network_type, chain_id| {
1361 if network_type == NetworkType::Evm && chain_id == 1 {
1362 Ok(Some(create_test_network_model()))
1363 } else {
1364 Ok(None)
1365 }
1366 });
1367 mocks
1368 }
1369
1370 pub fn create_test_network_model() -> NetworkRepoModel {
1372 let evm_config = EvmNetworkConfig {
1373 common: NetworkConfigCommon {
1374 network: "mainnet".to_string(),
1375 from: None,
1376 rpc_urls: Some(vec![RpcConfig::new("https://rpc.example.com".to_string())]),
1377 explorer_urls: Some(vec!["https://explorer.example.com".to_string()]),
1378 average_blocktime_ms: Some(12000),
1379 is_testnet: Some(false),
1380 tags: Some(vec!["mainnet".to_string()]),
1381 },
1382 chain_id: Some(1),
1383 required_confirmations: Some(12),
1384 features: Some(vec!["eip1559".to_string()]),
1385 symbol: Some("ETH".to_string()),
1386 gas_price_cache: None,
1387 };
1388 NetworkRepoModel {
1389 id: "evm:mainnet".to_string(),
1390 name: "mainnet".to_string(),
1391 network_type: NetworkType::Evm,
1392 config: NetworkConfigData::Evm(evm_config),
1393 }
1394 }
1395
1396 pub fn create_test_no_mempool_network_model() -> NetworkRepoModel {
1398 let evm_config = EvmNetworkConfig {
1399 common: NetworkConfigCommon {
1400 network: "arbitrum".to_string(),
1401 from: None,
1402 rpc_urls: Some(vec![crate::models::RpcConfig::new(
1403 "https://arb-rpc.example.com".to_string(),
1404 )]),
1405 explorer_urls: Some(vec!["https://arb-explorer.example.com".to_string()]),
1406 average_blocktime_ms: Some(1000),
1407 is_testnet: Some(false),
1408 tags: Some(vec![
1409 "arbitrum".to_string(),
1410 "rollup".to_string(),
1411 "no-mempool".to_string(),
1412 ]),
1413 },
1414 chain_id: Some(42161),
1415 required_confirmations: Some(12),
1416 features: Some(vec!["eip1559".to_string()]),
1417 symbol: Some("ETH".to_string()),
1418 gas_price_cache: None,
1419 };
1420 NetworkRepoModel {
1421 id: "evm:arbitrum".to_string(),
1422 name: "arbitrum".to_string(),
1423 network_type: NetworkType::Evm,
1424 config: NetworkConfigData::Evm(evm_config),
1425 }
1426 }
1427
1428 pub fn make_test_transaction(status: TransactionStatus) -> TransactionRepoModel {
1432 TransactionRepoModel {
1433 id: "test-tx-id".to_string(),
1434 relayer_id: "test-relayer-id".to_string(),
1435 status,
1436 status_reason: None,
1437 created_at: Utc::now().to_rfc3339(),
1438 sent_at: None,
1439 confirmed_at: None,
1440 valid_until: None,
1441 delete_at: None,
1442 network_type: NetworkType::Evm,
1443 network_data: NetworkTransactionData::Evm(EvmTransactionData {
1444 chain_id: 1,
1445 from: "0xSender".to_string(),
1446 to: Some("0xRecipient".to_string()),
1447 value: U256::from(0),
1448 data: Some("0xData".to_string()),
1449 gas_limit: Some(21000),
1450 gas_price: Some(20000000000),
1451 max_fee_per_gas: None,
1452 max_priority_fee_per_gas: None,
1453 nonce: None,
1454 signature: None,
1455 hash: None,
1456 speed: Some(Speed::Fast),
1457 raw: None,
1458 }),
1459 priced_at: None,
1460 hashes: Vec::new(),
1461 noop_count: None,
1462 is_canceled: Some(false),
1463 metadata: None,
1464 }
1465 }
1466
1467 pub fn make_test_evm_relayer_transaction(
1470 relayer: RelayerRepoModel,
1471 mocks: TestMocks,
1472 ) -> EvmRelayerTransaction<
1473 MockEvmProviderTrait,
1474 MockRelayerRepository,
1475 MockNetworkRepository,
1476 MockTransactionRepository,
1477 MockJobProducerTrait,
1478 MockSigner,
1479 MockTransactionCounterTrait,
1480 MockPriceCalculatorTrait,
1481 > {
1482 EvmRelayerTransaction::new(
1483 relayer,
1484 mocks.provider,
1485 Arc::new(mocks.relayer_repo),
1486 Arc::new(mocks.network_repo),
1487 Arc::new(mocks.tx_repo),
1488 Arc::new(mocks.counter),
1489 Arc::new(mocks.job_producer),
1490 mocks.price_calc,
1491 mocks.signer,
1492 )
1493 .unwrap()
1494 }
1495
1496 fn create_test_relayer() -> RelayerRepoModel {
1497 RelayerRepoModel {
1498 id: "test-relayer-id".to_string(),
1499 name: "Test Relayer".to_string(),
1500 paused: false,
1501 system_disabled: false,
1502 network: "test_network".to_string(),
1503 network_type: NetworkType::Evm,
1504 policies: RelayerNetworkPolicy::Evm(RelayerEvmPolicy::default()),
1505 signer_id: "test_signer".to_string(),
1506 address: "0x".to_string(),
1507 notification_id: None,
1508 custom_rpc_urls: None,
1509 ..Default::default()
1510 }
1511 }
1512
1513 fn make_mock_receipt(status: bool, block_number: Option<u64>) -> TransactionReceipt {
1514 let tx_hash = TxHash::from(b256!(
1516 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
1517 ));
1518 let block_hash = BlockHash::from(b256!(
1519 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
1520 ));
1521 let from_address = Address::from([0x11; 20]);
1522
1523 TransactionReceipt {
1524 inner: alloy::rpc::types::TransactionReceipt {
1525 inner: AnyReceiptEnvelope {
1526 inner: ReceiptWithBloom {
1527 receipt: Receipt {
1528 status: Eip658Value::Eip658(status), cumulative_gas_used: 0,
1530 logs: vec![],
1531 },
1532 logs_bloom: Bloom::ZERO,
1533 },
1534 r#type: 0, },
1536 transaction_hash: tx_hash,
1537 transaction_index: Some(0),
1538 block_hash: block_number.map(|_| block_hash), block_number,
1540 gas_used: 21000,
1541 effective_gas_price: 1000,
1542 blob_gas_used: None,
1543 blob_gas_price: None,
1544 from: from_address,
1545 to: None,
1546 contract_address: None,
1547 },
1548 other: Default::default(),
1549 }
1550 }
1551
1552 fn mock_receipt_from() -> Address {
1554 Address::from([0x11; 20])
1555 }
1556
1557 fn make_mock_receipt_with_to(
1560 status: bool,
1561 block_number: Option<u64>,
1562 to: Option<Address>,
1563 ) -> TransactionReceipt {
1564 let mut receipt = make_mock_receipt(status, block_number);
1565 receipt.inner.to = to;
1566 receipt
1567 }
1568
1569 mod check_transaction_status_tests {
1571 use super::*;
1572
1573 #[tokio::test]
1574 async fn test_not_mined() {
1575 let mut mocks = default_test_mocks();
1576 let relayer = create_test_relayer();
1577 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1578
1579 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1581 evm_data.hash = Some("0xFakeHash".to_string());
1582 }
1583
1584 mocks
1586 .provider
1587 .expect_get_transaction_receipt()
1588 .returning(|_| Box::pin(async { Ok(None) }));
1589
1590 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1591
1592 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1593 assert_eq!(status, TransactionStatus::Submitted);
1594 }
1595
1596 #[tokio::test]
1597 async fn test_mined_but_not_confirmed() {
1598 let mut mocks = default_test_mocks();
1599 let relayer = create_test_relayer();
1600 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1601
1602 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1603 evm_data.hash = Some("0xFakeHash".to_string());
1604 }
1605
1606 mocks
1608 .provider
1609 .expect_get_transaction_receipt()
1610 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
1611
1612 mocks
1614 .provider
1615 .expect_get_block_number()
1616 .return_once(|| Box::pin(async { Ok(100) }));
1617
1618 mocks
1620 .network_repo
1621 .expect_get_by_chain_id()
1622 .returning(|_, _| Ok(Some(create_test_network_model())));
1623
1624 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1625
1626 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1627 assert_eq!(status, TransactionStatus::Mined);
1628 }
1629
1630 #[tokio::test]
1631 async fn test_confirmed() {
1632 let mut mocks = default_test_mocks();
1633 let relayer = create_test_relayer();
1634 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1635
1636 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1637 evm_data.hash = Some("0xFakeHash".to_string());
1638 }
1639
1640 mocks
1642 .provider
1643 .expect_get_transaction_receipt()
1644 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
1645
1646 mocks
1648 .provider
1649 .expect_get_block_number()
1650 .return_once(|| Box::pin(async { Ok(113) }));
1651
1652 mocks
1654 .network_repo
1655 .expect_get_by_chain_id()
1656 .returning(|_, _| Ok(Some(create_test_network_model())));
1657
1658 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1659
1660 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1661 assert_eq!(status, TransactionStatus::Confirmed);
1662 }
1663
1664 #[tokio::test]
1667 async fn test_cancellation_noop_confirmed_becomes_canceled() {
1668 let mut mocks = default_test_mocks();
1669 let relayer = create_test_relayer();
1670 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1671 tx.is_canceled = Some(true);
1672
1673 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1674 evm_data.hash = Some("0xNoopHash".to_string());
1675 }
1676
1677 let noop_to = Some(mock_receipt_from());
1679 mocks
1680 .provider
1681 .expect_get_transaction_receipt()
1682 .returning(move |_| {
1683 Box::pin(async move {
1684 Ok(Some(make_mock_receipt_with_to(true, Some(100), noop_to)))
1685 })
1686 });
1687 mocks
1688 .provider
1689 .expect_get_block_number()
1690 .return_once(|| Box::pin(async { Ok(113) }));
1691 mocks
1692 .network_repo
1693 .expect_get_by_chain_id()
1694 .returning(|_, _| Ok(Some(create_test_network_model())));
1695
1696 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1697
1698 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1699 assert_eq!(status, TransactionStatus::Canceled);
1700 }
1701
1702 #[tokio::test]
1706 async fn test_cancellation_original_mined_stays_confirmed() {
1707 let mut mocks = default_test_mocks();
1708 let relayer = create_test_relayer();
1709 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1710 tx.is_canceled = Some(true);
1711
1712 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1713 evm_data.hash = Some("0xOriginalHash".to_string());
1714 }
1715
1716 let original_to = Some(Address::from([0x22; 20]));
1718 mocks
1719 .provider
1720 .expect_get_transaction_receipt()
1721 .returning(move |_| {
1722 Box::pin(async move {
1723 Ok(Some(make_mock_receipt_with_to(
1724 true,
1725 Some(100),
1726 original_to,
1727 )))
1728 })
1729 });
1730 mocks
1731 .provider
1732 .expect_get_block_number()
1733 .return_once(|| Box::pin(async { Ok(113) }));
1734 mocks
1735 .network_repo
1736 .expect_get_by_chain_id()
1737 .returning(|_, _| Ok(Some(create_test_network_model())));
1738
1739 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1740
1741 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1742 assert_eq!(status, TransactionStatus::Confirmed);
1743 }
1744
1745 #[tokio::test]
1748 async fn test_non_cancellation_noop_confirmed_stays_confirmed() {
1749 let mut mocks = default_test_mocks();
1750 let relayer = create_test_relayer();
1751 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1752 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1755 evm_data.hash = Some("0xNoopHash".to_string());
1756 }
1757
1758 let noop_to = Some(mock_receipt_from());
1760 mocks
1761 .provider
1762 .expect_get_transaction_receipt()
1763 .returning(move |_| {
1764 Box::pin(async move {
1765 Ok(Some(make_mock_receipt_with_to(true, Some(100), noop_to)))
1766 })
1767 });
1768 mocks
1769 .provider
1770 .expect_get_block_number()
1771 .return_once(|| Box::pin(async { Ok(113) }));
1772 mocks
1773 .network_repo
1774 .expect_get_by_chain_id()
1775 .returning(|_, _| Ok(Some(create_test_network_model())));
1776
1777 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1778
1779 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1780 assert_eq!(status, TransactionStatus::Confirmed);
1781 }
1782
1783 #[tokio::test]
1784 async fn test_failed() {
1785 let mut mocks = default_test_mocks();
1786 let relayer = create_test_relayer();
1787 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1788
1789 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1790 evm_data.hash = Some("0xFakeHash".to_string());
1791 }
1792
1793 mocks
1795 .provider
1796 .expect_get_transaction_receipt()
1797 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(false, Some(100)))) }));
1798
1799 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1800
1801 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1802 assert_eq!(status, TransactionStatus::Failed);
1803 }
1804 }
1805
1806 mod should_resubmit_tests {
1808 use super::*;
1809 use crate::models::TransactionError;
1810
1811 #[tokio::test]
1812 async fn test_should_resubmit_true() {
1813 let mut mocks = default_test_mocks();
1814 let relayer = create_test_relayer();
1815
1816 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1818 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1819
1820 mocks
1822 .network_repo
1823 .expect_get_by_chain_id()
1824 .returning(|_, _| Ok(Some(create_test_network_model())));
1825
1826 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1827 let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1828 assert!(res, "Transaction should be resubmitted after timeout.");
1829 }
1830
1831 #[tokio::test]
1832 async fn test_should_resubmit_false() {
1833 let mut mocks = default_test_mocks();
1834 let relayer = create_test_relayer();
1835
1836 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1838 tx.sent_at = Some(Utc::now().to_rfc3339());
1839
1840 mocks
1842 .network_repo
1843 .expect_get_by_chain_id()
1844 .returning(|_, _| Ok(Some(create_test_network_model())));
1845
1846 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1847 let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1848 assert!(!res, "Transaction should not be resubmitted immediately.");
1849 }
1850
1851 #[tokio::test]
1852 async fn test_should_resubmit_true_for_no_mempool_network() {
1853 let mut mocks = default_test_mocks();
1854 let relayer = create_test_relayer();
1855
1856 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1858 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1859
1860 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1862 evm_data.chain_id = 42161; }
1864
1865 mocks
1867 .network_repo
1868 .expect_get_by_chain_id()
1869 .returning(|_, _| Ok(Some(create_test_no_mempool_network_model())));
1870
1871 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1872 let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1873 assert!(
1874 res,
1875 "Transaction should be resubmitted for no-mempool networks."
1876 );
1877 }
1878
1879 #[tokio::test]
1880 async fn test_should_resubmit_network_not_found() {
1881 let mut mocks = default_test_mocks();
1882 let relayer = create_test_relayer();
1883
1884 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1885 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1886
1887 mocks
1889 .network_repo
1890 .expect_get_by_chain_id()
1891 .returning(|_, _| Ok(None));
1892
1893 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1894 let result = evm_transaction.should_resubmit(&tx).await;
1895
1896 assert!(
1897 result.is_err(),
1898 "should_resubmit should return error when network not found"
1899 );
1900 let error = result.unwrap_err();
1901 match error {
1902 TransactionError::UnexpectedError(msg) => {
1903 assert!(msg.contains("Network with chain id 1 not found"));
1904 }
1905 _ => panic!("Expected UnexpectedError for network not found"),
1906 }
1907 }
1908
1909 #[tokio::test]
1910 async fn test_should_resubmit_network_conversion_error() {
1911 let mut mocks = default_test_mocks();
1912 let relayer = create_test_relayer();
1913
1914 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1915 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1916
1917 let invalid_evm_config = EvmNetworkConfig {
1919 common: NetworkConfigCommon {
1920 network: "invalid-network".to_string(),
1921 from: None,
1922 rpc_urls: Some(vec![crate::models::RpcConfig::new(
1923 "https://rpc.example.com".to_string(),
1924 )]),
1925 explorer_urls: Some(vec!["https://explorer.example.com".to_string()]),
1926 average_blocktime_ms: Some(12000),
1927 is_testnet: Some(false),
1928 tags: Some(vec!["testnet".to_string()]),
1929 },
1930 chain_id: None, required_confirmations: Some(12),
1932 features: Some(vec!["eip1559".to_string()]),
1933 symbol: Some("ETH".to_string()),
1934 gas_price_cache: None,
1935 };
1936 let invalid_network = NetworkRepoModel {
1937 id: "evm:invalid".to_string(),
1938 name: "invalid-network".to_string(),
1939 network_type: NetworkType::Evm,
1940 config: NetworkConfigData::Evm(invalid_evm_config),
1941 };
1942
1943 mocks
1945 .network_repo
1946 .expect_get_by_chain_id()
1947 .returning(move |_, _| Ok(Some(invalid_network.clone())));
1948
1949 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1950 let result = evm_transaction.should_resubmit(&tx).await;
1951
1952 assert!(
1953 result.is_err(),
1954 "should_resubmit should return error when network conversion fails"
1955 );
1956 let error = result.unwrap_err();
1957 match error {
1958 TransactionError::UnexpectedError(msg) => {
1959 assert!(msg.contains("Error converting network model to EvmNetwork"));
1960 }
1961 _ => panic!("Expected UnexpectedError for network conversion failure"),
1962 }
1963 }
1964 }
1965
1966 mod should_noop_tests {
1968 use super::*;
1969
1970 #[tokio::test]
1971 async fn test_expired_transaction_triggers_noop() {
1972 let mut mocks = default_test_mocks();
1973 let relayer = create_test_relayer();
1974
1975 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1976 tx.valid_until = Some((Utc::now() - Duration::seconds(10)).to_rfc3339());
1978
1979 mocks
1981 .network_repo
1982 .expect_get_by_chain_id()
1983 .returning(|_, _| Ok(Some(create_test_network_model())));
1984
1985 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1986 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
1987 assert!(res, "Expired transaction should be replaced with a NOOP.");
1988 assert!(
1989 reason.is_some(),
1990 "Reason should be provided for expired transaction"
1991 );
1992 assert!(
1993 reason.unwrap().contains("expired"),
1994 "Reason should mention expiration"
1995 );
1996 }
1997
1998 #[tokio::test]
1999 async fn test_too_many_noop_attempts_returns_false() {
2000 let mocks = default_test_mocks();
2001 let relayer = create_test_relayer();
2002
2003 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2004 tx.noop_count = Some(51); let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2007 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2008 assert!(
2009 !res,
2010 "Transaction with too many NOOP attempts should not be replaced."
2011 );
2012 assert!(
2013 reason.is_none(),
2014 "Reason should not be provided when should_noop is false"
2015 );
2016 }
2017
2018 #[tokio::test]
2019 async fn test_already_noop_returns_false() {
2020 let mut mocks = default_test_mocks();
2021 let relayer = create_test_relayer();
2022
2023 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2024 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2026 evm_data.to = None;
2027 evm_data.value = U256::from(0);
2028 }
2029
2030 mocks
2031 .network_repo
2032 .expect_get_by_chain_id()
2033 .returning(|_, _| Ok(Some(create_test_network_model())));
2034
2035 mocks.provider.expect_get_block_by_number().returning(|| {
2037 Box::pin(async {
2038 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2039 let mut block: Block = Block::default();
2040 block.header.gas_limit = 30_000_000u64;
2041 Ok(AnyRpcBlock::from(block))
2042 })
2043 });
2044
2045 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2046 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2047 assert!(
2048 !res,
2049 "Transaction that is already a NOOP should not be replaced."
2050 );
2051 assert!(
2052 reason.is_none(),
2053 "Reason should not be provided when should_noop is false"
2054 );
2055 }
2056
2057 #[tokio::test]
2058 async fn test_rollup_with_too_many_attempts_triggers_noop() {
2059 let mut mocks = default_test_mocks();
2060 let relayer = create_test_relayer();
2061
2062 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2063 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2065 evm_data.chain_id = 42161; }
2067 tx.hashes = vec!["0xHash1".to_string(); 51];
2069
2070 mocks
2072 .network_repo
2073 .expect_get_by_chain_id()
2074 .returning(|_, _| Ok(Some(create_test_no_mempool_network_model())));
2075
2076 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2077 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2078 assert!(
2079 res,
2080 "Rollup transaction with too many attempts should be replaced with NOOP."
2081 );
2082 assert!(
2083 reason.is_some(),
2084 "Reason should be provided for rollup transaction"
2085 );
2086 assert!(
2087 reason.unwrap().contains("too many attempts"),
2088 "Reason should mention too many attempts"
2089 );
2090 }
2091
2092 #[tokio::test]
2093 async fn test_pending_state_timeout_triggers_noop() {
2094 let mut mocks = default_test_mocks();
2095 let relayer = create_test_relayer();
2096
2097 let mut tx = make_test_transaction(TransactionStatus::Pending);
2098 tx.created_at = (Utc::now() - Duration::minutes(3)).to_rfc3339();
2100
2101 mocks
2102 .network_repo
2103 .expect_get_by_chain_id()
2104 .returning(|_, _| Ok(Some(create_test_network_model())));
2105
2106 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2107 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2108 assert!(
2109 res,
2110 "Pending transaction stuck for >2 minutes should be replaced with NOOP."
2111 );
2112 assert!(
2113 reason.is_some(),
2114 "Reason should be provided for pending timeout"
2115 );
2116 assert!(
2117 reason.unwrap().contains("Pending state"),
2118 "Reason should mention Pending state"
2119 );
2120 }
2121
2122 #[tokio::test]
2123 async fn test_valid_transaction_returns_false() {
2124 let mut mocks = default_test_mocks();
2125 let relayer = create_test_relayer();
2126
2127 let tx = make_test_transaction(TransactionStatus::Submitted);
2128 mocks
2131 .network_repo
2132 .expect_get_by_chain_id()
2133 .returning(|_, _| Ok(Some(create_test_network_model())));
2134
2135 mocks.provider.expect_get_block_by_number().returning(|| {
2137 Box::pin(async {
2138 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2139 let mut block: Block = Block::default();
2140 block.header.gas_limit = 30_000_000u64;
2141 Ok(AnyRpcBlock::from(block))
2142 })
2143 });
2144
2145 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2146 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2147 assert!(!res, "Valid transaction should not be replaced with NOOP.");
2148 assert!(
2149 reason.is_none(),
2150 "Reason should not be provided when should_noop is false"
2151 );
2152 }
2153 }
2154
2155 mod update_transaction_status_tests {
2157 use super::*;
2158
2159 #[tokio::test]
2160 async fn test_no_update_when_status_is_same() {
2161 let mocks = default_test_mocks();
2163 let relayer = create_test_relayer();
2164 let tx = make_test_transaction(TransactionStatus::Submitted);
2165 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2166
2167 let updated_tx = evm_transaction
2170 .update_transaction_status_if_needed(tx.clone(), TransactionStatus::Submitted, None)
2171 .await
2172 .unwrap();
2173 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2174 assert_eq!(updated_tx.id, tx.id);
2175 }
2176
2177 #[tokio::test]
2178 async fn test_updates_when_status_differs() {
2179 let mut mocks = default_test_mocks();
2180 let relayer = create_test_relayer();
2181 let tx = make_test_transaction(TransactionStatus::Submitted);
2182
2183 mocks
2185 .tx_repo
2186 .expect_partial_update()
2187 .returning(|_, update| {
2188 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2189 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2190 Ok(updated_tx)
2191 });
2192
2193 mocks
2195 .job_producer
2196 .expect_produce_send_notification_job()
2197 .returning(|_, _| Box::pin(async { Ok(()) }));
2198
2199 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2200 let updated_tx = evm_transaction
2201 .update_transaction_status_if_needed(tx.clone(), TransactionStatus::Mined, None)
2202 .await
2203 .unwrap();
2204
2205 assert_eq!(updated_tx.status, TransactionStatus::Mined);
2206 }
2207
2208 #[tokio::test]
2209 async fn test_updates_with_status_reason() {
2210 let mut mocks = default_test_mocks();
2211 let relayer = create_test_relayer();
2212 let tx = make_test_transaction(TransactionStatus::Submitted);
2213
2214 mocks
2215 .tx_repo
2216 .expect_partial_update()
2217 .withf(|_, update| {
2218 update.status == Some(TransactionStatus::Failed)
2219 && update.status_reason == Some("Transaction reverted on-chain".to_string())
2220 })
2221 .returning(|_, update| {
2222 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2223 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2224 updated_tx.status_reason = update.status_reason.clone();
2225 Ok(updated_tx)
2226 });
2227
2228 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2229 let updated_tx = evm_transaction
2230 .update_transaction_status_if_needed(
2231 tx.clone(),
2232 TransactionStatus::Failed,
2233 Some("Transaction reverted on-chain".to_string()),
2234 )
2235 .await
2236 .unwrap();
2237
2238 assert_eq!(updated_tx.status, TransactionStatus::Failed);
2239 assert_eq!(
2240 updated_tx.status_reason.as_deref(),
2241 Some("Transaction reverted on-chain")
2242 );
2243 }
2244 }
2245
2246 mod handle_sent_state_tests {
2248 use super::*;
2249
2250 #[tokio::test]
2251 async fn test_sent_state_recent_no_resend() {
2252 let mut mocks = default_test_mocks();
2253 let relayer = create_test_relayer();
2254
2255 let mut tx = make_test_transaction(TransactionStatus::Sent);
2256 tx.sent_at = Some((Utc::now() - Duration::seconds(10)).to_rfc3339());
2258
2259 mocks
2261 .network_repo
2262 .expect_get_by_chain_id()
2263 .returning(|_, _| Ok(Some(create_test_network_model())));
2264
2265 mocks.provider.expect_get_block_by_number().returning(|| {
2267 Box::pin(async {
2268 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2269 let mut block: Block = Block::default();
2270 block.header.gas_limit = 30_000_000u64;
2271 Ok(AnyRpcBlock::from(block))
2272 })
2273 });
2274
2275 mocks
2277 .job_producer
2278 .expect_produce_check_transaction_status_job()
2279 .returning(|_, _| Box::pin(async { Ok(()) }));
2280
2281 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2282 let result = evm_transaction.handle_sent_state(tx.clone()).await.unwrap();
2283
2284 assert_eq!(result.status, TransactionStatus::Sent);
2285 }
2286
2287 #[tokio::test]
2288 async fn test_sent_state_stuck_schedules_resubmit() {
2289 let mut mocks = default_test_mocks();
2290 let relayer = create_test_relayer();
2291
2292 let mut tx = make_test_transaction(TransactionStatus::Sent);
2293 tx.sent_at = Some((Utc::now() - Duration::seconds(60)).to_rfc3339());
2295
2296 mocks
2298 .network_repo
2299 .expect_get_by_chain_id()
2300 .returning(|_, _| Ok(Some(create_test_network_model())));
2301
2302 mocks.provider.expect_get_block_by_number().returning(|| {
2304 Box::pin(async {
2305 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2306 let mut block: Block = Block::default();
2307 block.header.gas_limit = 30_000_000u64;
2308 Ok(AnyRpcBlock::from(block))
2309 })
2310 });
2311
2312 mocks
2314 .job_producer
2315 .expect_produce_submit_transaction_job()
2316 .returning(|_, _| Box::pin(async { Ok(()) }));
2317
2318 mocks
2320 .job_producer
2321 .expect_produce_check_transaction_status_job()
2322 .returning(|_, _| Box::pin(async { Ok(()) }));
2323
2324 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2325 let result = evm_transaction.handle_sent_state(tx.clone()).await.unwrap();
2326
2327 assert_eq!(result.status, TransactionStatus::Sent);
2328 }
2329 }
2330
2331 mod prepare_noop_update_request_tests {
2333 use super::*;
2334
2335 #[tokio::test]
2336 async fn test_noop_request_without_cancellation() {
2337 let mocks = default_test_mocks_with_network();
2339 let relayer = create_test_relayer();
2340 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2341 tx.noop_count = Some(2);
2342 tx.is_canceled = Some(false);
2343
2344 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2345 let update_req = evm_transaction
2346 .prepare_noop_update_request(&tx, false, None)
2347 .await
2348 .unwrap();
2349
2350 assert_eq!(update_req.noop_count, Some(3));
2352 assert_eq!(update_req.is_canceled, Some(false));
2354 }
2355
2356 #[tokio::test]
2357 async fn test_noop_request_with_cancellation() {
2358 let mocks = default_test_mocks_with_network();
2360 let relayer = create_test_relayer();
2361 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2362 tx.noop_count = None;
2363 tx.is_canceled = Some(false);
2364
2365 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2366 let update_req = evm_transaction
2367 .prepare_noop_update_request(&tx, true, None)
2368 .await
2369 .unwrap();
2370
2371 assert_eq!(update_req.noop_count, Some(1));
2373 assert_eq!(update_req.is_canceled, Some(true));
2375 }
2376 }
2377
2378 mod handle_submitted_state_tests {
2380 use super::*;
2381
2382 #[tokio::test]
2383 async fn test_schedules_resubmit_job() {
2384 let mut mocks = default_test_mocks();
2385 let relayer = create_test_relayer();
2386
2387 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2389 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2390
2391 mocks
2393 .network_repo
2394 .expect_get_by_chain_id()
2395 .returning(|_, _| Ok(Some(create_test_network_model())));
2396
2397 mocks.provider.expect_get_block_by_number().returning(|| {
2399 Box::pin(async {
2400 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2401 let mut block: Block = Block::default();
2402 block.header.gas_limit = 30_000_000u64;
2403 Ok(AnyRpcBlock::from(block))
2404 })
2405 });
2406
2407 mocks
2409 .provider
2410 .expect_get_transaction_count()
2411 .returning(|_| Box::pin(async { Ok(10) }));
2412
2413 mocks
2415 .job_producer
2416 .expect_produce_submit_transaction_job()
2417 .returning(|_, _| Box::pin(async { Ok(()) }));
2418
2419 mocks
2421 .job_producer
2422 .expect_produce_check_transaction_status_job()
2423 .returning(|_, _| Box::pin(async { Ok(()) }));
2424
2425 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2426 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2427
2428 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2430 }
2431
2432 #[tokio::test]
2435 async fn test_nonce_gap_detected_schedules_health_skips_resubmit() {
2436 let mut mocks = default_test_mocks();
2437 let relayer = create_test_relayer();
2438
2439 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2440 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2441 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
2442 nonce: Some(274),
2443 hash: Some("0xhash".to_string()),
2444 raw: Some(vec![1, 2, 3]),
2445 ..tx.network_data.get_evm_transaction_data().unwrap()
2446 });
2447
2448 mocks
2450 .network_repo
2451 .expect_get_by_chain_id()
2452 .returning(|_, _| Ok(Some(create_test_network_model())));
2453
2454 mocks
2456 .provider
2457 .expect_get_transaction_count()
2458 .returning(|_| Box::pin(async { Ok(269) }));
2459
2460 mocks
2462 .tx_repo
2463 .expect_get_nonce_occupancy()
2464 .withf(|relayer_id, from, to| {
2465 relayer_id == "test-relayer-id" && *from == 269 && *to == 274
2466 })
2467 .returning(|_, from, to| Ok((from..to).map(|n| (n, None)).collect()));
2468
2469 mocks
2471 .job_producer
2472 .expect_produce_relayer_health_check_job()
2473 .withf(|job, _| {
2474 job.metadata.as_ref().map_or(false, |m| {
2475 m.get("health_check_action") == Some(&"nonce_health".to_string())
2476 })
2477 })
2478 .returning(|_, _| Box::pin(async { Ok(()) }));
2479
2480 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2484 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2485
2486 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2487 }
2488
2489 #[tokio::test]
2492 async fn test_nonce_gap_check_rpc_failure_proceeds_to_resubmit() {
2493 let mut mocks = default_test_mocks();
2494 let relayer = create_test_relayer();
2495
2496 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2497 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2498
2499 mocks
2500 .network_repo
2501 .expect_get_by_chain_id()
2502 .returning(|_, _| Ok(Some(create_test_network_model())));
2503
2504 mocks.provider.expect_get_block_by_number().returning(|| {
2505 Box::pin(async {
2506 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2507 let mut block: Block = Block::default();
2508 block.header.gas_limit = 30_000_000u64;
2509 Ok(AnyRpcBlock::from(block))
2510 })
2511 });
2512
2513 mocks
2515 .provider
2516 .expect_get_transaction_count()
2517 .returning(|_| {
2518 Box::pin(async {
2519 Err(crate::services::provider::ProviderError::Other(
2520 "rpc timeout".to_string(),
2521 ))
2522 })
2523 });
2524
2525 mocks
2527 .job_producer
2528 .expect_produce_submit_transaction_job()
2529 .returning(|_, _| Box::pin(async { Ok(()) }));
2530
2531 mocks
2532 .job_producer
2533 .expect_produce_check_transaction_status_job()
2534 .returning(|_, _| Box::pin(async { Ok(()) }));
2535
2536 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2537 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2538
2539 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2540 }
2541
2542 #[tokio::test]
2545 async fn test_no_gap_when_slots_filled_proceeds_to_resubmit() {
2546 let mut mocks = default_test_mocks();
2547 let relayer = create_test_relayer();
2548
2549 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2550 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2551 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
2552 nonce: Some(270),
2553 hash: Some("0xhash".to_string()),
2554 raw: Some(vec![1, 2, 3]),
2555 ..tx.network_data.get_evm_transaction_data().unwrap()
2556 });
2557
2558 mocks
2559 .network_repo
2560 .expect_get_by_chain_id()
2561 .returning(|_, _| Ok(Some(create_test_network_model())));
2562
2563 mocks.provider.expect_get_block_by_number().returning(|| {
2564 Box::pin(async {
2565 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2566 let mut block: Block = Block::default();
2567 block.header.gas_limit = 30_000_000u64;
2568 Ok(AnyRpcBlock::from(block))
2569 })
2570 });
2571
2572 mocks
2574 .provider
2575 .expect_get_transaction_count()
2576 .returning(|_| Box::pin(async { Ok(269) }));
2577
2578 mocks
2580 .tx_repo
2581 .expect_get_nonce_occupancy()
2582 .returning(|_, from, to| {
2583 Ok((from..to)
2584 .map(|n| (n, Some(TransactionStatus::Submitted)))
2585 .collect())
2586 });
2587
2588 mocks
2590 .job_producer
2591 .expect_produce_submit_transaction_job()
2592 .returning(|_, _| Box::pin(async { Ok(()) }));
2593
2594 mocks
2595 .job_producer
2596 .expect_produce_check_transaction_status_job()
2597 .returning(|_, _| Box::pin(async { Ok(()) }));
2598
2599 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2600 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2601
2602 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2603 }
2604 }
2605
2606 mod handle_pending_state_tests {
2608 use super::*;
2609
2610 #[tokio::test]
2611 async fn test_pending_state_no_noop() {
2612 let mut mocks = default_test_mocks();
2614 let relayer = create_test_relayer();
2615 let mut tx = make_test_transaction(TransactionStatus::Pending);
2616 tx.created_at = Utc::now().to_rfc3339(); mocks
2620 .network_repo
2621 .expect_get_by_chain_id()
2622 .returning(|_, _| Ok(Some(create_test_network_model())));
2623
2624 mocks.provider.expect_get_block_by_number().returning(|| {
2626 Box::pin(async {
2627 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2628 let mut block: Block = Block::default();
2629 block.header.gas_limit = 30_000_000u64;
2630 Ok(AnyRpcBlock::from(block))
2631 })
2632 });
2633
2634 mocks
2636 .job_producer
2637 .expect_produce_check_transaction_status_job()
2638 .returning(|_, _| Box::pin(async { Ok(()) }));
2639
2640 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2641 let result = evm_transaction
2642 .handle_pending_state(tx.clone())
2643 .await
2644 .unwrap();
2645
2646 assert_eq!(result.id, tx.id);
2648 assert_eq!(result.status, tx.status);
2649 assert_eq!(result.noop_count, tx.noop_count);
2650 }
2651
2652 #[tokio::test]
2653 async fn test_pending_state_with_noop() {
2654 let mut mocks = default_test_mocks();
2656 let relayer = create_test_relayer();
2657 let mut tx = make_test_transaction(TransactionStatus::Pending);
2658 tx.created_at = (Utc::now() - Duration::minutes(2)).to_rfc3339();
2659
2660 mocks
2662 .network_repo
2663 .expect_get_by_chain_id()
2664 .returning(|_, _| Ok(Some(create_test_network_model())));
2665
2666 mocks.provider.expect_get_block_by_number().returning(|| {
2668 Box::pin(async {
2669 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2670 let mut block: Block = Block::default();
2671 block.header.gas_limit = 30_000_000u64;
2672 Ok(AnyRpcBlock::from(block))
2673 })
2674 });
2675
2676 let tx_clone = tx.clone();
2679 mocks
2680 .tx_repo
2681 .expect_partial_update()
2682 .withf(move |id, update| {
2683 id == "test-tx-id"
2684 && update.status == Some(TransactionStatus::Failed)
2685 && update.status_reason.is_some()
2686 })
2687 .returning(move |_, update| {
2688 let mut updated_tx = tx_clone.clone();
2689 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2690 updated_tx.status_reason = update.status_reason.clone();
2691 Ok(updated_tx)
2692 });
2693 mocks
2695 .job_producer
2696 .expect_produce_send_notification_job()
2697 .returning(|_, _| Box::pin(async { Ok(()) }));
2698
2699 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2700 let result = evm_transaction
2701 .handle_pending_state(tx.clone())
2702 .await
2703 .unwrap();
2704
2705 assert_eq!(result.status, TransactionStatus::Failed);
2707 assert!(result.status_reason.is_some());
2708 assert!(result.status_reason.unwrap().contains("Pending state"));
2709 }
2710 }
2711
2712 mod handle_mined_state_tests {
2714 use super::*;
2715
2716 #[tokio::test]
2717 async fn test_updates_status_and_schedules_check() {
2718 let mut mocks = default_test_mocks();
2719 let relayer = create_test_relayer();
2720 let tx = make_test_transaction(TransactionStatus::Submitted);
2722
2723 mocks
2725 .job_producer
2726 .expect_produce_check_transaction_status_job()
2727 .returning(|_, _| Box::pin(async { Ok(()) }));
2728 mocks
2730 .tx_repo
2731 .expect_partial_update()
2732 .returning(|_, update| {
2733 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2734 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2735 Ok(updated_tx)
2736 });
2737
2738 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2739 let result = evm_transaction
2740 .handle_mined_state(tx.clone())
2741 .await
2742 .unwrap();
2743 assert_eq!(result.status, TransactionStatus::Mined);
2744 }
2745 }
2746
2747 mod handle_final_state_tests {
2749 use super::*;
2750
2751 #[tokio::test]
2752 async fn test_final_state_confirmed() {
2753 let mut mocks = default_test_mocks();
2754 let relayer = create_test_relayer();
2755 let tx = make_test_transaction(TransactionStatus::Submitted);
2756
2757 mocks
2759 .tx_repo
2760 .expect_partial_update()
2761 .returning(|_, update| {
2762 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2763 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2764 Ok(updated_tx)
2765 });
2766
2767 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2768 let result = evm_transaction
2769 .handle_final_state(tx.clone(), TransactionStatus::Confirmed, None)
2770 .await
2771 .unwrap();
2772 assert_eq!(result.status, TransactionStatus::Confirmed);
2773 }
2774
2775 #[tokio::test]
2776 async fn test_final_state_failed() {
2777 let mut mocks = default_test_mocks();
2778 let relayer = create_test_relayer();
2779 let tx = make_test_transaction(TransactionStatus::Submitted);
2780
2781 mocks
2783 .tx_repo
2784 .expect_partial_update()
2785 .returning(|_, update| {
2786 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2787 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2788 updated_tx.status_reason = update.status_reason.clone();
2789 Ok(updated_tx)
2790 });
2791
2792 let reason = "Transaction reverted on-chain (receipt status: failed)".to_string();
2793 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2794 let result = evm_transaction
2795 .handle_final_state(tx.clone(), TransactionStatus::Failed, Some(reason.clone()))
2796 .await
2797 .unwrap();
2798 assert_eq!(result.status, TransactionStatus::Failed);
2799 assert_eq!(result.status_reason.as_deref(), Some(reason.as_str()));
2800 }
2801
2802 #[tokio::test]
2803 async fn test_final_state_expired() {
2804 let mut mocks = default_test_mocks();
2805 let relayer = create_test_relayer();
2806 let tx = make_test_transaction(TransactionStatus::Submitted);
2807
2808 mocks
2810 .tx_repo
2811 .expect_partial_update()
2812 .returning(|_, update| {
2813 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2814 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2815 Ok(updated_tx)
2816 });
2817
2818 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2819 let result = evm_transaction
2820 .handle_final_state(tx.clone(), TransactionStatus::Expired, None)
2821 .await
2822 .unwrap();
2823 assert_eq!(result.status, TransactionStatus::Expired);
2824 }
2825 }
2826
2827 mod handle_status_impl_tests {
2829 use super::*;
2830
2831 #[tokio::test]
2832 async fn test_impl_submitted_branch() {
2833 let mut mocks = default_test_mocks();
2834 let relayer = create_test_relayer();
2835 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2836 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
2837 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2839 evm_data.hash = Some("0xFakeHash".to_string());
2840 }
2841 mocks
2843 .provider
2844 .expect_get_transaction_receipt()
2845 .returning(|_| Box::pin(async { Ok(None) }));
2846 mocks
2848 .network_repo
2849 .expect_get_by_chain_id()
2850 .returning(|_, _| Ok(Some(create_test_network_model())));
2851 mocks
2853 .job_producer
2854 .expect_produce_check_transaction_status_job()
2855 .returning(|_, _| Box::pin(async { Ok(()) }));
2856 mocks
2858 .tx_repo
2859 .expect_partial_update()
2860 .returning(|_, update| {
2861 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2862 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2863 Ok(updated_tx)
2864 });
2865
2866 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2867 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2868 assert_eq!(result.status, TransactionStatus::Submitted);
2869 }
2870
2871 #[tokio::test]
2872 async fn test_impl_mined_branch() {
2873 let mut mocks = default_test_mocks();
2874 let relayer = create_test_relayer();
2875 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2876 tx.created_at = (Utc::now() - Duration::minutes(1)).to_rfc3339();
2878 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2880 evm_data.hash = Some("0xFakeHash".to_string());
2881 }
2882 mocks
2884 .provider
2885 .expect_get_transaction_receipt()
2886 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
2887 mocks
2889 .provider
2890 .expect_get_block_number()
2891 .return_once(|| Box::pin(async { Ok(100) }));
2892 mocks
2894 .network_repo
2895 .expect_get_by_chain_id()
2896 .returning(|_, _| Ok(Some(create_test_network_model())));
2897 mocks
2899 .job_producer
2900 .expect_produce_send_notification_job()
2901 .returning(|_, _| Box::pin(async { Ok(()) }));
2902 mocks.tx_repo.expect_get_by_id().returning(|_| {
2904 let updated_tx = make_test_transaction(TransactionStatus::Mined);
2905 Ok(updated_tx)
2906 });
2907 mocks
2909 .tx_repo
2910 .expect_partial_update()
2911 .returning(|_, update| {
2912 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2913 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2914 Ok(updated_tx)
2915 });
2916
2917 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2918 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2919 assert_eq!(result.status, TransactionStatus::Mined);
2920 }
2921
2922 #[tokio::test]
2923 async fn test_impl_final_confirmed_branch() {
2924 let mut mocks = default_test_mocks();
2925 let relayer = create_test_relayer();
2926 let tx = make_test_transaction(TransactionStatus::Confirmed);
2928
2929 mocks
2932 .tx_repo
2933 .expect_partial_update()
2934 .returning(|_, update| {
2935 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2936 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2937 Ok(updated_tx)
2938 });
2939
2940 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2941 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2942 assert_eq!(result.status, TransactionStatus::Confirmed);
2943 }
2944
2945 #[tokio::test]
2946 async fn test_impl_final_failed_branch() {
2947 let mut mocks = default_test_mocks();
2948 let relayer = create_test_relayer();
2949 let tx = make_test_transaction(TransactionStatus::Failed);
2951
2952 mocks
2953 .tx_repo
2954 .expect_partial_update()
2955 .returning(|_, update| {
2956 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2957 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2958 Ok(updated_tx)
2959 });
2960
2961 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2962 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2963 assert_eq!(result.status, TransactionStatus::Failed);
2964 }
2965
2966 #[tokio::test]
2969 async fn test_impl_submitted_to_failed_sets_status_reason() {
2970 let mut mocks = default_test_mocks();
2971 let relayer = create_test_relayer();
2972 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2973 tx.created_at = (Utc::now() - Duration::minutes(1)).to_rfc3339();
2974 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2975 evm_data.hash = Some("0xFakeHash".to_string());
2976 }
2977
2978 mocks
2980 .provider
2981 .expect_get_transaction_receipt()
2982 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(false, Some(100)))) }));
2983
2984 let tx_clone = tx.clone();
2986 mocks.tx_repo.expect_get_by_id().returning(move |_| {
2987 let mut reloaded = tx_clone.clone();
2988 reloaded.status = TransactionStatus::Submitted;
2989 Ok(reloaded)
2990 });
2991
2992 mocks
2994 .tx_repo
2995 .expect_partial_update()
2996 .withf(|_, update| {
2997 update.status == Some(TransactionStatus::Failed)
2998 && update.status_reason.is_some()
2999 })
3000 .returning(|_, update| {
3001 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3002 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3003 updated_tx.status_reason = update.status_reason.clone();
3004 Ok(updated_tx)
3005 });
3006
3007 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3008 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3009 assert_eq!(result.status, TransactionStatus::Failed);
3010 assert!(result.status_reason.is_some());
3011 assert!(
3012 result
3013 .status_reason
3014 .as_ref()
3015 .unwrap()
3016 .contains("reverted on-chain"),
3017 "Expected on-chain revert reason, got: {:?}",
3018 result.status_reason
3019 );
3020 }
3021
3022 #[tokio::test]
3023 async fn test_impl_final_expired_branch() {
3024 let mut mocks = default_test_mocks();
3025 let relayer = create_test_relayer();
3026 let tx = make_test_transaction(TransactionStatus::Expired);
3028
3029 mocks
3030 .tx_repo
3031 .expect_partial_update()
3032 .returning(|_, update| {
3033 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3034 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3035 Ok(updated_tx)
3036 });
3037
3038 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3039 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3040 assert_eq!(result.status, TransactionStatus::Expired);
3041 }
3042 }
3043
3044 mod circuit_breaker_tests {
3046 use super::*;
3047 use crate::jobs::StatusCheckContext;
3048
3049 fn create_triggered_context() -> StatusCheckContext {
3051 StatusCheckContext::new(
3052 30, 50, 60, 25, 75, NetworkType::Evm,
3058 )
3059 }
3060
3061 fn create_safe_context() -> StatusCheckContext {
3063 StatusCheckContext::new(
3064 5, 10, 15, 25, 75, NetworkType::Evm,
3070 )
3071 }
3072
3073 fn create_total_triggered_context() -> StatusCheckContext {
3075 StatusCheckContext::new(
3076 5, 80, 100, 25, 75, NetworkType::Evm,
3082 )
3083 }
3084
3085 #[tokio::test]
3086 async fn test_circuit_breaker_pending_marks_as_failed() {
3087 let mut mocks = default_test_mocks();
3088 let relayer = create_test_relayer();
3089 let tx = make_test_transaction(TransactionStatus::Pending);
3090
3091 mocks
3093 .tx_repo
3094 .expect_partial_update()
3095 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3096 .returning(|_, update| {
3097 let mut updated_tx = make_test_transaction(TransactionStatus::Pending);
3098 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3099 updated_tx.status_reason = update.status_reason.clone();
3100 Ok(updated_tx)
3101 });
3102
3103 mocks
3105 .job_producer
3106 .expect_produce_send_notification_job()
3107 .returning(|_, _| Box::pin(async { Ok(()) }));
3108
3109 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3110 let ctx = create_triggered_context();
3111
3112 let result = evm_transaction
3113 .handle_status_impl(tx, Some(ctx))
3114 .await
3115 .unwrap();
3116
3117 assert_eq!(result.status, TransactionStatus::Failed);
3118 assert!(result.status_reason.is_some());
3119 assert!(result.status_reason.unwrap().contains("consecutive errors"));
3120 }
3121
3122 #[tokio::test]
3123 async fn test_circuit_breaker_sent_marks_as_failed() {
3124 let mut mocks = default_test_mocks();
3125 let relayer = create_test_relayer();
3126 let tx = make_test_transaction(TransactionStatus::Sent);
3127
3128 mocks
3130 .tx_repo
3131 .expect_partial_update()
3132 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3133 .returning(|_, update| {
3134 let mut updated_tx = make_test_transaction(TransactionStatus::Sent);
3135 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3136 updated_tx.status_reason = update.status_reason.clone();
3137 Ok(updated_tx)
3138 });
3139
3140 mocks
3142 .job_producer
3143 .expect_produce_send_notification_job()
3144 .returning(|_, _| Box::pin(async { Ok(()) }));
3145
3146 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3147 let ctx = create_triggered_context();
3148
3149 let result = evm_transaction
3150 .handle_status_impl(tx, Some(ctx))
3151 .await
3152 .unwrap();
3153
3154 assert_eq!(result.status, TransactionStatus::Failed);
3155 }
3156
3157 #[tokio::test]
3158 async fn test_circuit_breaker_submitted_triggers_noop() {
3159 let mut mocks = default_test_mocks();
3160 let relayer = create_test_relayer();
3161 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3162 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3163
3164 mocks
3166 .network_repo
3167 .expect_get_by_chain_id()
3168 .returning(|_, _| Ok(Some(create_test_network_model())));
3169
3170 mocks
3172 .tx_repo
3173 .expect_partial_update()
3174 .returning(|_, update| {
3175 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3176 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3177 updated_tx.status_reason = update.status_reason.clone();
3178 updated_tx.noop_count = update.noop_count;
3179 Ok(updated_tx)
3180 });
3181
3182 mocks
3184 .job_producer
3185 .expect_produce_submit_transaction_job()
3186 .returning(|_, _| Box::pin(async { Ok(()) }));
3187
3188 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3189 let ctx = create_triggered_context();
3190
3191 let result = evm_transaction
3192 .handle_status_impl(tx, Some(ctx))
3193 .await
3194 .unwrap();
3195
3196 assert!(result.noop_count.is_some());
3198 }
3199
3200 #[tokio::test]
3201 async fn test_circuit_breaker_noop_tx_excluded() {
3202 let mut mocks = default_test_mocks();
3203 let relayer = create_test_relayer();
3204
3205 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3207 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3208 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3209 evm_data.to = Some(evm_data.from.clone()); evm_data.value = U256::from(0);
3211 evm_data.data = Some("0x".to_string());
3212 evm_data.hash = Some("0xFakeHash".to_string());
3213 }
3214
3215 mocks
3218 .provider
3219 .expect_get_transaction_receipt()
3220 .returning(|_| Box::pin(async { Ok(None) }));
3221
3222 mocks
3223 .network_repo
3224 .expect_get_by_chain_id()
3225 .returning(|_, _| Ok(Some(create_test_network_model())));
3226
3227 mocks
3228 .job_producer
3229 .expect_produce_check_transaction_status_job()
3230 .returning(|_, _| Box::pin(async { Ok(()) }));
3231
3232 mocks
3234 .job_producer
3235 .expect_produce_submit_transaction_job()
3236 .returning(|_, _| Box::pin(async { Ok(()) }));
3237
3238 mocks
3239 .tx_repo
3240 .expect_partial_update()
3241 .returning(|_, update| {
3242 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3243 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3244 Ok(updated_tx)
3245 });
3246
3247 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3248 let ctx = create_triggered_context();
3249
3250 let result = evm_transaction
3251 .handle_status_impl(tx, Some(ctx))
3252 .await
3253 .unwrap();
3254
3255 assert_eq!(result.status, TransactionStatus::Submitted);
3257 }
3258
3259 #[tokio::test]
3260 async fn test_circuit_breaker_total_failures_triggers() {
3261 let mut mocks = default_test_mocks();
3262 let relayer = create_test_relayer();
3263 let tx = make_test_transaction(TransactionStatus::Pending);
3264
3265 mocks
3267 .tx_repo
3268 .expect_partial_update()
3269 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3270 .returning(|_, update| {
3271 let mut updated_tx = make_test_transaction(TransactionStatus::Pending);
3272 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3273 updated_tx.status_reason = update.status_reason.clone();
3274 Ok(updated_tx)
3275 });
3276
3277 mocks
3278 .job_producer
3279 .expect_produce_send_notification_job()
3280 .returning(|_, _| Box::pin(async { Ok(()) }));
3281
3282 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3283 let ctx = create_total_triggered_context();
3285
3286 let result = evm_transaction
3287 .handle_status_impl(tx, Some(ctx))
3288 .await
3289 .unwrap();
3290
3291 assert_eq!(result.status, TransactionStatus::Failed);
3292 assert!(result.status_reason.is_some());
3293 }
3294
3295 #[tokio::test]
3296 async fn test_circuit_breaker_below_threshold_continues_normally() {
3297 let mut mocks = default_test_mocks();
3298 let relayer = create_test_relayer();
3299 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3300 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3301
3302 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3303 evm_data.hash = Some("0xFakeHash".to_string());
3304 }
3305
3306 mocks
3308 .provider
3309 .expect_get_transaction_receipt()
3310 .returning(|_| Box::pin(async { Ok(None) }));
3311
3312 mocks
3313 .network_repo
3314 .expect_get_by_chain_id()
3315 .returning(|_, _| Ok(Some(create_test_network_model())));
3316
3317 mocks
3318 .job_producer
3319 .expect_produce_check_transaction_status_job()
3320 .returning(|_, _| Box::pin(async { Ok(()) }));
3321
3322 mocks
3323 .tx_repo
3324 .expect_partial_update()
3325 .returning(|_, update| {
3326 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3327 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3328 Ok(updated_tx)
3329 });
3330
3331 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3332 let ctx = create_safe_context();
3333
3334 let result = evm_transaction
3335 .handle_status_impl(tx, Some(ctx))
3336 .await
3337 .unwrap();
3338
3339 assert_eq!(result.status, TransactionStatus::Submitted);
3341 }
3342
3343 #[tokio::test]
3344 async fn test_circuit_breaker_no_context_continues_normally() {
3345 let mut mocks = default_test_mocks();
3346 let relayer = create_test_relayer();
3347 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3348 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3349
3350 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3351 evm_data.hash = Some("0xFakeHash".to_string());
3352 }
3353
3354 mocks
3356 .provider
3357 .expect_get_transaction_receipt()
3358 .returning(|_| Box::pin(async { Ok(None) }));
3359
3360 mocks
3361 .network_repo
3362 .expect_get_by_chain_id()
3363 .returning(|_, _| Ok(Some(create_test_network_model())));
3364
3365 mocks
3366 .job_producer
3367 .expect_produce_check_transaction_status_job()
3368 .returning(|_, _| Box::pin(async { Ok(()) }));
3369
3370 mocks
3371 .tx_repo
3372 .expect_partial_update()
3373 .returning(|_, update| {
3374 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3375 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3376 Ok(updated_tx)
3377 });
3378
3379 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3380
3381 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3383
3384 assert_eq!(result.status, TransactionStatus::Submitted);
3386 }
3387
3388 #[tokio::test]
3389 async fn test_circuit_breaker_final_state_early_return() {
3390 let mocks = default_test_mocks();
3391 let relayer = create_test_relayer();
3392 let tx = make_test_transaction(TransactionStatus::Confirmed);
3394
3395 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3396 let ctx = create_triggered_context();
3397
3398 let result = evm_transaction
3400 .handle_status_impl(tx, Some(ctx))
3401 .await
3402 .unwrap();
3403
3404 assert_eq!(result.status, TransactionStatus::Confirmed);
3405 }
3406 }
3407
3408 mod hash_recovery_tests {
3410 use super::*;
3411
3412 #[tokio::test]
3413 async fn test_should_try_hash_recovery_not_submitted() {
3414 let mocks = default_test_mocks();
3415 let relayer = create_test_relayer();
3416
3417 let mut tx = make_test_transaction(TransactionStatus::Sent);
3418 tx.hashes = vec![
3419 "0xHash1".to_string(),
3420 "0xHash2".to_string(),
3421 "0xHash3".to_string(),
3422 ];
3423
3424 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3425 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3426
3427 assert!(
3428 !result,
3429 "Should not attempt recovery for non-Submitted transactions"
3430 );
3431 }
3432
3433 #[tokio::test]
3434 async fn test_should_try_hash_recovery_not_enough_hashes() {
3435 let mocks = default_test_mocks();
3436 let relayer = create_test_relayer();
3437
3438 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3439 tx.hashes = vec!["0xHash1".to_string()]; tx.sent_at = Some((Utc::now() - Duration::minutes(3)).to_rfc3339());
3441
3442 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3443 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3444
3445 assert!(
3446 !result,
3447 "Should not attempt recovery with insufficient hashes"
3448 );
3449 }
3450
3451 #[tokio::test]
3452 async fn test_should_try_hash_recovery_too_recent() {
3453 let mocks = default_test_mocks();
3454 let relayer = create_test_relayer();
3455
3456 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3457 tx.hashes = vec![
3458 "0xHash1".to_string(),
3459 "0xHash2".to_string(),
3460 "0xHash3".to_string(),
3461 ];
3462 tx.sent_at = Some(Utc::now().to_rfc3339()); let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3465 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3466
3467 assert!(
3468 !result,
3469 "Should not attempt recovery for recently sent transactions"
3470 );
3471 }
3472
3473 #[tokio::test]
3474 async fn test_should_try_hash_recovery_success() {
3475 let mocks = default_test_mocks();
3476 let relayer = create_test_relayer();
3477
3478 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3479 tx.hashes = vec![
3480 "0xHash1".to_string(),
3481 "0xHash2".to_string(),
3482 "0xHash3".to_string(),
3483 ];
3484 tx.sent_at = Some((Utc::now() - Duration::minutes(3)).to_rfc3339());
3485
3486 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3487 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3488
3489 assert!(
3490 result,
3491 "Should attempt recovery for stuck transactions with multiple hashes"
3492 );
3493 }
3494
3495 #[tokio::test]
3496 async fn test_try_recover_no_historical_hash_found() {
3497 let mut mocks = default_test_mocks();
3498 let relayer = create_test_relayer();
3499
3500 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3501 tx.hashes = vec![
3502 "0xHash1".to_string(),
3503 "0xHash2".to_string(),
3504 "0xHash3".to_string(),
3505 ];
3506
3507 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3508 evm_data.hash = Some("0xHash3".to_string());
3509 }
3510
3511 mocks
3513 .provider
3514 .expect_get_transaction_receipt()
3515 .returning(|_| Box::pin(async { Ok(None) }));
3516
3517 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3518 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3519 let result = evm_transaction
3520 .try_recover_with_historical_hashes(&tx, &evm_data)
3521 .await
3522 .unwrap();
3523
3524 assert!(
3525 result.is_none(),
3526 "Should return None when no historical hash is found"
3527 );
3528 }
3529
3530 #[tokio::test]
3531 async fn test_try_recover_finds_mined_historical_hash() {
3532 let mut mocks = default_test_mocks();
3533 let relayer = create_test_relayer();
3534
3535 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3536 tx.hashes = vec![
3537 "0xHash1".to_string(),
3538 "0xHash2".to_string(), "0xHash3".to_string(),
3540 ];
3541
3542 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3543 evm_data.hash = Some("0xHash3".to_string()); }
3545
3546 mocks
3548 .provider
3549 .expect_get_transaction_receipt()
3550 .returning(|hash| {
3551 if hash == "0xHash2" {
3552 Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) })
3553 } else {
3554 Box::pin(async { Ok(None) })
3555 }
3556 });
3557
3558 let tx_clone = tx.clone();
3560 mocks
3561 .tx_repo
3562 .expect_partial_update()
3563 .returning(move |_, update| {
3564 let mut updated_tx = tx_clone.clone();
3565 if let Some(status) = update.status {
3566 updated_tx.status = status;
3567 }
3568 if let Some(NetworkTransactionData::Evm(ref evm_data)) = update.network_data {
3569 if let NetworkTransactionData::Evm(ref mut updated_evm) =
3570 updated_tx.network_data
3571 {
3572 updated_evm.hash = evm_data.hash.clone();
3573 }
3574 }
3575 Ok(updated_tx)
3576 });
3577
3578 mocks
3580 .job_producer
3581 .expect_produce_send_notification_job()
3582 .returning(|_, _| Box::pin(async { Ok(()) }));
3583
3584 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3585 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3586 let result = evm_transaction
3587 .try_recover_with_historical_hashes(&tx, &evm_data)
3588 .await
3589 .unwrap();
3590
3591 assert!(result.is_some(), "Should recover the transaction");
3592 let recovered_tx = result.unwrap();
3593 assert_eq!(recovered_tx.status, TransactionStatus::Mined);
3594 }
3595
3596 #[tokio::test]
3597 async fn test_try_recover_network_error_continues() {
3598 let mut mocks = default_test_mocks();
3599 let relayer = create_test_relayer();
3600
3601 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3602 tx.hashes = vec![
3603 "0xHash1".to_string(),
3604 "0xHash2".to_string(), "0xHash3".to_string(), ];
3607
3608 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3609 evm_data.hash = Some("0xHash1".to_string());
3610 }
3611
3612 mocks
3614 .provider
3615 .expect_get_transaction_receipt()
3616 .returning(|hash| {
3617 if hash == "0xHash2" {
3618 Box::pin(async { Err(crate::services::provider::ProviderError::Timeout) })
3619 } else if hash == "0xHash3" {
3620 Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) })
3621 } else {
3622 Box::pin(async { Ok(None) })
3623 }
3624 });
3625
3626 let tx_clone = tx.clone();
3628 mocks
3629 .tx_repo
3630 .expect_partial_update()
3631 .returning(move |_, update| {
3632 let mut updated_tx = tx_clone.clone();
3633 if let Some(status) = update.status {
3634 updated_tx.status = status;
3635 }
3636 Ok(updated_tx)
3637 });
3638
3639 mocks
3641 .job_producer
3642 .expect_produce_send_notification_job()
3643 .returning(|_, _| Box::pin(async { Ok(()) }));
3644
3645 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3646 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3647 let result = evm_transaction
3648 .try_recover_with_historical_hashes(&tx, &evm_data)
3649 .await
3650 .unwrap();
3651
3652 assert!(
3653 result.is_some(),
3654 "Should continue checking after network error and find mined hash"
3655 );
3656 }
3657
3658 #[tokio::test]
3659 async fn test_update_transaction_with_corrected_hash() {
3660 let mut mocks = default_test_mocks();
3661 let relayer = create_test_relayer();
3662
3663 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3664 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3665 evm_data.hash = Some("0xWrongHash".to_string());
3666 }
3667
3668 mocks
3670 .tx_repo
3671 .expect_partial_update()
3672 .returning(move |_, update| {
3673 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3674 if let Some(status) = update.status {
3675 updated_tx.status = status;
3676 }
3677 if let Some(NetworkTransactionData::Evm(ref evm_data)) = update.network_data {
3678 if let NetworkTransactionData::Evm(ref mut updated_evm) =
3679 updated_tx.network_data
3680 {
3681 updated_evm.hash = evm_data.hash.clone();
3682 }
3683 }
3684 Ok(updated_tx)
3685 });
3686
3687 mocks
3689 .job_producer
3690 .expect_produce_send_notification_job()
3691 .returning(|_, _| Box::pin(async { Ok(()) }));
3692
3693 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3694 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3695 let result = evm_transaction
3696 .update_transaction_with_corrected_hash(
3697 &tx,
3698 &evm_data,
3699 "0xCorrectHash",
3700 TransactionStatus::Mined,
3701 )
3702 .await
3703 .unwrap();
3704
3705 assert_eq!(result.status, TransactionStatus::Mined);
3706 if let NetworkTransactionData::Evm(ref updated_evm) = result.network_data {
3707 assert_eq!(updated_evm.hash.as_ref().unwrap(), "0xCorrectHash");
3708 }
3709 }
3710 }
3711
3712 mod check_transaction_status_edge_cases {
3714 use super::*;
3715
3716 #[tokio::test]
3717 async fn test_missing_hash_returns_error() {
3718 let mocks = default_test_mocks();
3719 let relayer = create_test_relayer();
3720
3721 let tx = make_test_transaction(TransactionStatus::Submitted);
3722 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3725 let result = evm_transaction.check_transaction_status(&tx).await;
3726
3727 assert!(result.is_err(), "Should return error when hash is missing");
3728 }
3729
3730 #[tokio::test]
3731 async fn test_pending_status_early_return() {
3732 let mocks = default_test_mocks();
3733 let relayer = create_test_relayer();
3734
3735 let tx = make_test_transaction(TransactionStatus::Pending);
3736
3737 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3738 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
3739
3740 assert_eq!(
3741 status,
3742 TransactionStatus::Pending,
3743 "Should return Pending without querying blockchain"
3744 );
3745 }
3746
3747 #[tokio::test]
3748 async fn test_sent_status_early_return() {
3749 let mocks = default_test_mocks();
3750 let relayer = create_test_relayer();
3751
3752 let tx = make_test_transaction(TransactionStatus::Sent);
3753
3754 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3755 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
3756
3757 assert_eq!(
3758 status,
3759 TransactionStatus::Sent,
3760 "Should return Sent without querying blockchain"
3761 );
3762 }
3763
3764 #[tokio::test]
3765 async fn test_final_state_early_return() {
3766 let mocks = default_test_mocks();
3767 let relayer = create_test_relayer();
3768
3769 let tx = make_test_transaction(TransactionStatus::Confirmed);
3770
3771 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3772 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
3773
3774 assert_eq!(
3775 status,
3776 TransactionStatus::Confirmed,
3777 "Should return final state without querying blockchain"
3778 );
3779 }
3780 }
3781
3782 mod nonce_recovery_tests {
3783 use super::*;
3784 use crate::domain::transaction::evm::evm_transaction::TX_NONCE_RECONCILE_TRIGGER;
3785 use crate::jobs::StatusCheckContext;
3786
3787 #[tokio::test]
3789 async fn test_nonce_recovery_nonce_consumed_externally() {
3790 let mut mocks = default_test_mocks();
3791 let relayer = create_test_relayer();
3792
3793 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3794 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3795 nonce: Some(5),
3796 hash: Some("0xhash".to_string()),
3797 raw: Some(vec![1, 2, 3]),
3798 ..tx.network_data.get_evm_transaction_data().unwrap()
3799 });
3800 tx.sent_at = Some(Utc::now().to_rfc3339());
3801
3802 mocks
3804 .provider
3805 .expect_get_transaction_receipt()
3806 .returning(|_| Box::pin(async { Ok(None) }));
3807
3808 mocks
3810 .provider
3811 .expect_get_transaction_count()
3812 .returning(|_| Box::pin(async { Ok(10) }));
3813
3814 let tx_clone = tx.clone();
3816 mocks
3817 .tx_repo
3818 .expect_partial_update()
3819 .withf(|_, update| {
3820 update.status == Some(TransactionStatus::Failed)
3821 && update
3822 .status_reason
3823 .as_ref()
3824 .map(|r| r.contains("consumed externally"))
3825 .unwrap_or(false)
3826 })
3827 .returning(move |_, update| {
3828 let mut updated_tx = tx_clone.clone();
3829 updated_tx.status = update.status.unwrap();
3830 updated_tx.status_reason = update.status_reason.clone();
3831 Ok(updated_tx)
3832 });
3833
3834 mocks
3836 .job_producer
3837 .expect_produce_relayer_health_check_job()
3838 .withf(|job, scheduled_on| {
3839 job.relayer_id == "test-relayer-id"
3840 && job.metadata.as_ref().map_or(false, |m| {
3841 m.get("health_check_action") == Some(&"nonce_health".to_string())
3842 })
3843 && scheduled_on.is_none()
3844 })
3845 .returning(|_, _| Box::pin(async { Ok(()) }));
3846
3847 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3848 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3849
3850 assert!(result.is_ok());
3851 let recovered = result.unwrap();
3852 assert!(recovered.is_some(), "Expected Some(tx) for consumed nonce");
3853 assert_eq!(recovered.unwrap().status, TransactionStatus::Failed);
3854 }
3855
3856 #[tokio::test]
3858 async fn test_nonce_recovery_nonce_not_consumed() {
3859 let mut mocks = default_test_mocks();
3860 let relayer = create_test_relayer();
3861
3862 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3863 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3864 nonce: Some(5),
3865 hash: Some("0xhash".to_string()),
3866 raw: Some(vec![1, 2, 3]),
3867 ..tx.network_data.get_evm_transaction_data().unwrap()
3868 });
3869
3870 mocks
3872 .provider
3873 .expect_get_transaction_receipt()
3874 .returning(|_| Box::pin(async { Ok(None) }));
3875
3876 mocks
3878 .provider
3879 .expect_get_transaction_count()
3880 .returning(|_| Box::pin(async { Ok(5) }));
3881
3882 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3883 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3884
3885 assert!(result.is_ok());
3886 assert!(
3887 result.unwrap().is_none(),
3888 "Expected None when nonce not consumed"
3889 );
3890 }
3891
3892 #[tokio::test]
3894 async fn test_nonce_recovery_receipt_found() {
3895 let mut mocks = default_test_mocks();
3896 let relayer = create_test_relayer();
3897
3898 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3899 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3900 nonce: Some(5),
3901 hash: Some("0xhash".to_string()),
3902 raw: Some(vec![1, 2, 3]),
3903 ..tx.network_data.get_evm_transaction_data().unwrap()
3904 });
3905
3906 mocks
3908 .provider
3909 .expect_get_transaction_receipt()
3910 .returning(|_| {
3911 let receipt = make_mock_receipt(true, Some(100));
3912 Box::pin(async move { Ok(Some(receipt)) })
3913 });
3914
3915 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3916 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3917
3918 assert!(result.is_ok());
3919 assert!(
3920 result.unwrap().is_none(),
3921 "Expected None when receipt found — defer to normal flow"
3922 );
3923 }
3924
3925 #[tokio::test]
3928 async fn test_nonce_recovery_rpc_error_prevents_force_fail() {
3929 let mut mocks = default_test_mocks();
3930 let relayer = create_test_relayer();
3931
3932 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3933 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3934 nonce: Some(5),
3935 hash: Some("0xhash".to_string()),
3936 raw: Some(vec![1, 2, 3]),
3937 ..tx.network_data.get_evm_transaction_data().unwrap()
3938 });
3939
3940 mocks
3942 .provider
3943 .expect_get_transaction_receipt()
3944 .returning(|_| {
3945 Box::pin(async {
3946 Err(crate::services::provider::ProviderError::Other(
3947 "RPC timeout".to_string(),
3948 ))
3949 })
3950 });
3951
3952 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3956 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3957
3958 assert!(result.is_ok());
3959 assert!(
3960 result.unwrap().is_none(),
3961 "Expected None when RPC errors occurred — must not force-fail on incomplete data"
3962 );
3963 }
3964
3965 #[tokio::test]
3967 async fn test_handle_status_impl_nonce_recovery_hint() {
3968 let mut mocks = default_test_mocks();
3969 let relayer = create_test_relayer();
3970
3971 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3972 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3973 nonce: Some(5),
3974 hash: Some("0xhash".to_string()),
3975 raw: Some(vec![1, 2, 3]),
3976 ..tx.network_data.get_evm_transaction_data().unwrap()
3977 });
3978 tx.sent_at = Some(Utc::now().to_rfc3339());
3979
3980 mocks
3982 .provider
3983 .expect_get_transaction_receipt()
3984 .returning(|_| Box::pin(async { Ok(None) }));
3985
3986 mocks
3988 .provider
3989 .expect_get_transaction_count()
3990 .returning(|_| Box::pin(async { Ok(10) }));
3991
3992 let tx_clone = tx.clone();
3994 mocks
3995 .tx_repo
3996 .expect_partial_update()
3997 .returning(move |_, update| {
3998 let mut updated_tx = tx_clone.clone();
3999 if let Some(status) = update.status {
4000 updated_tx.status = status;
4001 }
4002 updated_tx.status_reason = update.status_reason.clone();
4003 Ok(updated_tx)
4004 });
4005
4006 mocks
4008 .job_producer
4009 .expect_produce_relayer_health_check_job()
4010 .returning(|_, _| Box::pin(async { Ok(()) }));
4011
4012 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4013
4014 let mut metadata = std::collections::HashMap::new();
4016 metadata.insert(
4017 TX_NONCE_RECONCILE_TRIGGER.to_string(),
4018 "NonceTooLow".to_string(),
4019 );
4020 let context = StatusCheckContext::default().with_job_metadata(Some(metadata));
4021
4022 let result = evm_transaction.handle_status_impl(tx, Some(context)).await;
4023 assert!(result.is_ok());
4024 assert_eq!(result.unwrap().status, TransactionStatus::Failed);
4025 }
4026 }
4027
4028 mod circuit_breaker_sent_state_tests {
4029 use super::*;
4030 use crate::jobs::StatusCheckContext;
4031
4032 #[tokio::test]
4034 async fn test_circuit_breaker_sent_with_nonce_issues_noop() {
4035 let mut mocks = default_test_mocks_with_network();
4036 let relayer = create_test_relayer();
4037
4038 let mut tx = make_test_transaction(TransactionStatus::Sent);
4039 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4040 nonce: Some(5),
4041 hash: Some("0xhash".to_string()),
4042 raw: Some(vec![1, 2, 3]),
4043 ..tx.network_data.get_evm_transaction_data().unwrap()
4044 });
4045 tx.sent_at = Some(Utc::now().to_rfc3339());
4046
4047 let tx_clone = tx.clone();
4049 mocks
4050 .tx_repo
4051 .expect_partial_update()
4052 .returning(move |_, update| {
4053 let mut updated_tx = tx_clone.clone();
4054 if let Some(status) = update.status {
4055 updated_tx.status = status;
4056 }
4057 updated_tx.status_reason = update.status_reason.clone();
4058 Ok(updated_tx)
4059 });
4060
4061 mocks
4063 .job_producer
4064 .expect_produce_submit_transaction_job()
4065 .times(1)
4066 .returning(|_, _| Box::pin(async { Ok(()) }));
4067
4068 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4069
4070 let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
4072
4073 let result = evm_transaction
4074 .handle_circuit_breaker_safely(tx, &ctx)
4075 .await;
4076 assert!(result.is_ok());
4077 }
4078
4079 #[tokio::test]
4081 async fn test_circuit_breaker_sent_without_nonce_marks_failed() {
4082 let mut mocks = default_test_mocks();
4083 let relayer = create_test_relayer();
4084
4085 let mut tx = make_test_transaction(TransactionStatus::Sent);
4086 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4088 nonce: None,
4089 hash: None,
4090 raw: None,
4091 ..tx.network_data.get_evm_transaction_data().unwrap()
4092 });
4093 tx.sent_at = Some(Utc::now().to_rfc3339());
4094
4095 let tx_clone = tx.clone();
4097 mocks
4098 .tx_repo
4099 .expect_partial_update()
4100 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
4101 .returning(move |_, update| {
4102 let mut updated_tx = tx_clone.clone();
4103 updated_tx.status = update.status.unwrap();
4104 Ok(updated_tx)
4105 });
4106
4107 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4108
4109 let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
4110
4111 let result = evm_transaction
4112 .handle_circuit_breaker_safely(tx, &ctx)
4113 .await;
4114 assert!(result.is_ok());
4115 assert_eq!(result.unwrap().status, TransactionStatus::Failed);
4116 }
4117
4118 #[tokio::test]
4120 async fn test_circuit_breaker_pending_marks_failed() {
4121 let mut mocks = default_test_mocks();
4122 let relayer = create_test_relayer();
4123
4124 let tx = make_test_transaction(TransactionStatus::Pending);
4125
4126 let tx_clone = tx.clone();
4128 mocks
4129 .tx_repo
4130 .expect_partial_update()
4131 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
4132 .returning(move |_, update| {
4133 let mut updated_tx = tx_clone.clone();
4134 updated_tx.status = update.status.unwrap();
4135 Ok(updated_tx)
4136 });
4137
4138 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4139
4140 let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
4141
4142 let result = evm_transaction
4143 .handle_circuit_breaker_safely(tx, &ctx)
4144 .await;
4145 assert!(result.is_ok());
4146 assert_eq!(result.unwrap().status, TransactionStatus::Failed);
4147 }
4148 }
4149}