Skip to main content

rs_dapi_client/
address_list.rs

1//! Subsystem to manage DAPI nodes.
2
3use crate::address_ban_info::AddressBanInfo;
4use crate::Uri;
5use chrono::Utc;
6use rand::{rngs::SmallRng, seq::IteratorRandom, Rng, SeedableRng};
7use std::collections::hash_map::Entry;
8use std::collections::{HashMap, HashSet};
9use std::hash::{Hash, Hasher};
10use std::mem;
11use std::str::FromStr;
12use std::sync::{Arc, RwLock};
13use std::time::Duration;
14
15const DEFAULT_BASE_BAN_PERIOD: Duration = Duration::from_secs(60);
16
17/// Longest ban window either ban path can produce. Bounds `e^ban_count`,
18/// which otherwise overflows `DateTime + Duration` arithmetic (a panic that
19/// poisons the shared lock) once `ban_count` reaches the mid-20s.
20const MAX_BAN_PERIOD: Duration = Duration::from_secs(24 * 60 * 60);
21
22/// Default number of addresses that receive traffic at a time.
23///
24/// Kept small so requests reuse warm connections instead of sampling the whole
25/// list (hundreds of nodes on mainnet), where nearly every request would land
26/// on a cold host and pay a fresh TCP + TLS handshake.
27const DEFAULT_ACTIVE_SET_SIZE: usize = 5;
28
29/// How long an address may hold an active-set slot before it is retired and a
30/// random live standby is promoted in its place. Bounding slot tenure keeps
31/// connections warm for minutes at a time while spreading traffic to standby
32/// nodes over the process lifetime, when alternatives are available.
33const SLOT_LIFETIME: Duration = Duration::from_secs(5 * 60);
34
35/// DAPI address.
36#[derive(Debug, Clone, Eq)]
37#[cfg_attr(feature = "mocks", derive(serde::Serialize, serde::Deserialize))]
38pub struct Address(#[cfg_attr(feature = "mocks", serde(with = "http_serde::uri"))] Uri);
39
40impl FromStr for Address {
41    type Err = AddressListError;
42
43    fn from_str(s: &str) -> Result<Self, Self::Err> {
44        Uri::from_str(s)
45            .map_err(|e| AddressListError::InvalidAddressUri(e.to_string()))
46            .map(Address::try_from)?
47    }
48}
49
50impl PartialEq<Self> for Address {
51    fn eq(&self, other: &Self) -> bool {
52        self.0 == other.0
53    }
54}
55
56impl PartialEq<Uri> for Address {
57    fn eq(&self, other: &Uri) -> bool {
58        self.0 == *other
59    }
60}
61
62impl Hash for Address {
63    fn hash<H: Hasher>(&self, state: &mut H) {
64        self.0.hash(state);
65    }
66}
67
68impl TryFrom<Uri> for Address {
69    type Error = AddressListError;
70
71    fn try_from(value: Uri) -> Result<Self, Self::Error> {
72        if value.host().is_none() {
73            return Err(AddressListError::InvalidAddressUri(
74                "uri must contain host".to_string(),
75            ));
76        }
77
78        Ok(Address(value))
79    }
80}
81
82impl Address {
83    /// Get [Uri] of a node.
84    pub fn uri(&self) -> &Uri {
85        &self.0
86    }
87}
88
89/// Address status
90/// Contains information about the number of bans and the time until the next ban is lifted.
91#[derive(Debug, Default, Clone)]
92pub struct AddressStatus {
93    ban_count: usize,
94    banned_until: Option<chrono::DateTime<Utc>>,
95    /// Human-readable reason for the most recent ban, if any. Cleared
96    /// on [`AddressStatus::unban`]. Sourced from the error that caused
97    /// the ban (see `update_address_ban_status`).
98    ban_reason: Option<String>,
99}
100
101impl AddressStatus {
102    /// Ban the [Address] so it won't be available through [AddressList::get_live_address] for some time.
103    ///
104    /// Back-compat shim for [`AddressStatus::ban_with_reason`] with no reason.
105    pub fn ban(&mut self, base_ban_period: &Duration) {
106        self.ban_with_reason(base_ban_period, None);
107    }
108
109    /// Ban the [Address] and record the `reason` for the ban.
110    ///
111    /// Applies exponential backoff: the ban window is `base × e^ban_count`
112    /// (where `ban_count` is the value *before* this call), capped at 24 hours
113    /// even when `base` is larger, and `banned_until`
114    /// is always re-based to `now + window` unconditionally, regardless of any
115    /// existing active ban.  Concretely, a health failure on a node that already
116    /// holds a longer rate-limit window (set via [`AddressStatus::ban_for`]) will
117    /// re-base `banned_until` to the exponential value, which may be shorter.
118    /// This is intentional: the exponential health-ban ladder owns the window for
119    /// genuinely-unhealthy nodes; the no-shorten guarantee is deliberately scoped
120    /// to `ban_for → ban_for` sequences only.
121    ///
122    /// `ban_count` is incremented and `ban_reason` is updated unconditionally.
123    /// The counter resets to 0 on [`AddressStatus::unban`].
124    pub fn ban_with_reason(&mut self, base_ban_period: &Duration, reason: Option<String>) {
125        let coefficient = (self.ban_count as f64).exp();
126        let ban_secs = base_ban_period.as_secs_f64() * coefficient;
127        // NaN/inf compare false, so any overflowing window falls to the cap.
128        let ban_period = if ban_secs < MAX_BAN_PERIOD.as_secs_f64() {
129            Duration::from_secs_f64(ban_secs)
130        } else {
131            MAX_BAN_PERIOD
132        };
133
134        self.banned_until = Some(chrono::Utc::now() + ban_period);
135        self.ban_count += 1;
136        self.ban_reason = reason;
137    }
138
139    /// Ban the address for an exact `period` (server-advertised), bypassing the
140    /// exponential ladder used by [`AddressStatus::ban_with_reason`].
141    ///
142    /// The ban window is flat (not exponential) and capped at 24 hours.
143    /// `banned_until` is advanced to
144    /// `now + period` only when that timestamp is **later** than the current
145    /// `banned_until`, so a short-reset call never shortens a longer active ban
146    /// (health ban or a prior longer rate-limit ban).  `ban_reason` is updated
147    /// only when the window is extended.  `ban_count` is raised to
148    /// `max(ban_count, 1)` unconditionally so that `is_banned()` and
149    /// `ban_info()` correctly report the node as banned.  Side-effect: a
150    /// previously-clean node (ban_count 0) enters the ladder at floor 1,
151    /// meaning its *next* genuine health failure via
152    /// [`AddressStatus::ban_with_reason`] uses `60 s × e¹ ≈ 163 s` rather
153    /// than the first-rung `60 s × e⁰ = 60 s`.  The counter resets to 0 on
154    /// [`AddressStatus::unban`].
155    ///
156    /// Note: the no-shorten guard applies only to `ban_for → ban_for` call
157    /// sequences.  [`AddressStatus::ban_with_reason`] re-bases `banned_until`
158    /// unconditionally — see its docs for the intentional cross-method semantics.
159    pub fn ban_for(&mut self, period: Duration, reason: Option<String>) {
160        // A server-advertised window is clamped like the ladder: a hostile or
161        // buggy period must not overflow `DateTime + Duration`.
162        let period = period.min(MAX_BAN_PERIOD);
163        let advertised_until = chrono::Utc::now() + period;
164        if self
165            .banned_until
166            .map(|current| current < advertised_until)
167            .unwrap_or(true)
168        {
169            self.banned_until = Some(advertised_until);
170            self.ban_reason = reason;
171        }
172        self.ban_count = self.ban_count.max(1);
173    }
174
175    /// Check if [Address] is banned.
176    pub fn is_banned(&self) -> bool {
177        self.ban_count > 0
178    }
179
180    /// Check if [Address] is live at `now`: never banned, or its ban period has
181    /// already expired.
182    fn is_live(&self, now: chrono::DateTime<Utc>) -> bool {
183        self.banned_until
184            .map(|banned_until| banned_until < now)
185            .unwrap_or(true)
186    }
187
188    /// Clears ban record.
189    pub fn unban(&mut self) {
190        self.ban_count = 0;
191        self.banned_until = None;
192        self.ban_reason = None;
193    }
194}
195
196/// [AddressList] errors
197#[derive(Debug, thiserror::Error, Clone)]
198#[cfg_attr(feature = "mocks", derive(serde::Serialize, serde::Deserialize))]
199pub enum AddressListError {
200    /// A valid uri is required to create an Address
201    #[error("unable parse address: {0}")]
202    #[cfg_attr(feature = "mocks", serde(skip))]
203    InvalidAddressUri(String),
204}
205
206/// One member of the sticky active set: the address plus the moment its slot
207/// expires and a random standby replaces it.
208#[derive(Debug)]
209struct ActiveMember {
210    address: Address,
211    slot_expires_at: chrono::DateTime<Utc>,
212}
213
214/// Sticky rotation state: the addresses currently receiving traffic, the most
215/// recently served one that round-robin selection advances from, and the
216/// configured active-set size. Shared (behind one lock) by every clone of an
217/// [AddressList] so all clones drive the same rotation.
218#[derive(Debug)]
219struct Rotation {
220    active: Vec<ActiveMember>,
221    /// Evicted members to deprioritize during the next refill. Only active
222    /// members are recorded, bounding this set even after repeated errors.
223    evicted: HashSet<Address>,
224    last_served: Option<Address>,
225    active_set_size: usize,
226}
227
228impl Default for Rotation {
229    fn default() -> Self {
230        Rotation {
231            active: Vec::new(),
232            evicted: HashSet::new(),
233            last_served: None,
234            active_set_size: DEFAULT_ACTIVE_SET_SIZE,
235        }
236    }
237}
238
239/// A structure to manage DAPI addresses to select from
240/// for [DapiRequest](crate::DapiRequest) execution.
241///
242/// Address selection is sticky: requests rotate over a small active set of
243/// addresses (5 by default, see [AddressList::with_active_set_size]) and the
244/// rest of the list serves as failover standby (see
245/// [AddressList::get_live_address]).
246#[derive(Debug, Clone)]
247pub struct AddressList {
248    addresses: Arc<RwLock<HashMap<Address, AddressStatus>>>,
249    rotation: Arc<RwLock<Rotation>>,
250    base_ban_period: Duration,
251}
252
253impl Default for AddressList {
254    fn default() -> Self {
255        Self::new()
256    }
257}
258
259impl std::fmt::Display for Address {
260    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
261        self.0.fmt(f)
262    }
263}
264
265impl AddressList {
266    /// Creates an empty [AddressList] with default base ban time.
267    pub fn new() -> Self {
268        AddressList::with_settings(DEFAULT_BASE_BAN_PERIOD)
269    }
270
271    /// Creates an empty [AddressList] with adjustable base ban time.
272    pub fn with_settings(base_ban_period: Duration) -> Self {
273        AddressList {
274            addresses: Arc::new(RwLock::new(HashMap::new())),
275            rotation: Arc::new(RwLock::new(Rotation::default())),
276            base_ban_period,
277        }
278    }
279
280    /// Set how many addresses receive traffic at a time.
281    ///
282    /// Defaults to 5. `0` is clamped to 1. Smaller values maximize connection
283    /// reuse, larger values spread load over more nodes. The effective size is
284    /// additionally capped by the number of live addresses, so a very large
285    /// value (e.g. `usize::MAX`) disables stickiness and round-robins over the
286    /// whole list.
287    ///
288    /// The size lives in the rotation state shared by every clone of this
289    /// list, so it applies to all clones and takes effect on the next
290    /// selection (shrinking drops the excess members).
291    pub fn with_active_set_size(self, size: usize) -> Self {
292        self.rotation.write().unwrap().active_set_size = size.max(1);
293        self
294    }
295
296    /// Bans address
297    /// Returns false if the address is not in the list.
298    ///
299    /// Back-compat shim for [`AddressList::ban_with_reason`] with no reason.
300    pub fn ban(&self, address: &Address) -> bool {
301        self.ban_with_reason(address, None)
302    }
303
304    /// Bans address, recording the `reason` for the ban.
305    /// Returns false if the address is not in the list.
306    pub fn ban_with_reason(&self, address: &Address, reason: Option<String>) -> bool {
307        let mut guard = self.addresses.write().unwrap();
308
309        let Some(status) = guard.get_mut(address) else {
310            return false;
311        };
312
313        status.ban_with_reason(&self.base_ban_period, reason);
314
315        true
316    }
317
318    /// Ban the address for an exact `period` (server-advertised); delegates to
319    /// [`AddressStatus::ban_for`] — see that method for the full contract
320    /// including the `ban_count` floor and ladder side-effect.
321    ///
322    /// Returns `false` if the address is not in the list.
323    pub fn ban_for(&self, address: &Address, period: Duration, reason: Option<String>) -> bool {
324        let mut guard = self.addresses.write().unwrap();
325
326        let Some(status) = guard.get_mut(address) else {
327            return false;
328        };
329
330        status.ban_for(period, reason);
331
332        true
333    }
334
335    /// Clears address' ban record
336    /// Returns false if the address is not in the list.
337    pub fn unban(&self, address: &Address) -> bool {
338        let mut guard = self.addresses.write().unwrap();
339
340        let Some(status) = guard.get_mut(address) else {
341            return false;
342        };
343
344        status.unban();
345
346        true
347    }
348
349    /// Check if the address is banned.
350    pub fn is_banned(&self, address: &Address) -> bool {
351        let guard = self.addresses.read().unwrap();
352
353        guard
354            .get(address)
355            .map(|status| status.is_banned())
356            .unwrap_or(false)
357    }
358
359    /// Adds a node [Address] to [AddressList]
360    /// Returns false if the address is already in the list.
361    pub fn add(&mut self, address: Address) -> bool {
362        let mut guard = self.addresses.write().unwrap();
363
364        match guard.entry(address) {
365            Entry::Occupied(_) => false,
366            Entry::Vacant(e) => {
367                e.insert(AddressStatus::default());
368
369                true
370            }
371        }
372    }
373
374    /// Remove address from the list
375    /// Returns [AddressStatus] if the address was in the list.
376    pub fn remove(&mut self, address: &Address) -> Option<AddressStatus> {
377        let mut guard = self.addresses.write().unwrap();
378
379        guard.remove(address)
380    }
381
382    #[deprecated]
383    // TODO: Remove in favor of add
384    /// Add a node [Address] to [AddressList] by [Uri].
385    /// Returns false if the address is already in the list.
386    pub fn add_uri(&mut self, uri: Uri) -> bool {
387        self.add(Address::try_from(uri).expect("valid uri"))
388    }
389
390    /// Select a not-banned address to send the next request to.
391    ///
392    /// Not a pure getter: every call advances the shared rotation cursor and
393    /// may promote or retire active-set members, steering traffic for all
394    /// clones of this list.
395    ///
396    /// Selection is sticky: requests rotate round-robin over a small active
397    /// set of addresses (see [AddressList::with_active_set_size]) instead of
398    /// sampling the whole list, so connections to those hosts stay warm. An
399    /// active address that got banned, removed or evicted on failover (see
400    /// [AddressList::evict_from_rotation]) is dropped from the set here and a
401    /// random live standby address is promoted in its place. Each slot also
402    /// expires after a jittered lifetime (5–7.5 minutes). Evicted and expired
403    /// members are used to refill slots only after all live standbys have
404    /// been selected, preserving availability when alternatives are scarce.
405    ///
406    /// An address is considered live when it has never been banned or when its
407    /// ban period has already expired.
408    pub fn get_live_address(&self) -> Option<Address> {
409        // TODO(low): module-wide `.read()/.write().unwrap()` panics on a
410        // poisoned lock; adopt poison-tolerant locking consistently.
411        let guard = self.addresses.read().unwrap();
412
413        let now = chrono::Utc::now();
414
415        // Seeded outside the critical section: `from_entropy` panics if the OS
416        // entropy source fails, and a panic while holding the write lock would
417        // poison it and permanently disable address selection.
418        let mut rng = SmallRng::from_entropy();
419
420        // Lock ordering: `addresses` before `rotation`; this is the only place
421        // both locks are held at once.
422        let mut rotation = self.rotation.write().unwrap();
423        let mut retired = mem::take(&mut rotation.evicted);
424
425        // Drop active addresses that are banned, no longer in the list, or
426        // whose slot lifetime expired.
427        rotation.active.retain(|member| {
428            if now >= member.slot_expires_at {
429                retired.insert(member.address.clone());
430                false
431            } else {
432                guard
433                    .get(&member.address)
434                    .map(|status| status.is_live(now))
435                    .unwrap_or(false)
436            }
437        });
438
439        // Honor a shrunken size (the rotation state is shared, so it may have
440        // been reconfigured through any clone).
441        let size = rotation.active_set_size;
442        rotation.active.truncate(size);
443
444        // Refill vacancies with random live standby addresses. Bounded by the
445        // remaining list length so whole-list mode skips refilling once full
446        // and an oversized configured value cannot over-allocate.
447        let vacancies = size.min(guard.len()).saturating_sub(rotation.active.len());
448        if vacancies > 0 {
449            let candidates = guard.iter().filter(|&(address, status)| {
450                status.is_live(now)
451                    && !rotation
452                        .active
453                        .iter()
454                        .any(|member| member.address == *address)
455            });
456            let mut promoted = candidates
457                .clone()
458                .filter(|(address, _)| !retired.contains(*address))
459                .choose_multiple(&mut rng, vacancies);
460            // Prefer genuine standbys, but do not lose usable capacity when
461            // fewer standbys are live than there are vacancies.
462            let remaining = vacancies - promoted.len();
463            if remaining > 0 && !retired.is_empty() {
464                promoted.extend(
465                    candidates
466                        .filter(|(address, _)| retired.contains(*address))
467                        .choose_multiple(&mut rng, remaining),
468                );
469            }
470
471            rotation
472                .active
473                .extend(promoted.into_iter().map(|(address, _)| ActiveMember {
474                    address: address.clone(),
475                    // Jitter staggers expiries so slots retire one at a time
476                    // instead of the whole set at once.
477                    slot_expires_at: now + SLOT_LIFETIME.mul_f64(rng.gen_range(1.0..1.5)),
478                }));
479        }
480
481        if rotation.active.is_empty() {
482            return None;
483        }
484
485        // Advance from the last-served address: eviction re-orders the active
486        // set, so a positional cursor would not survive churn. Start from the
487        // head when the last-served address is gone (or nothing has been
488        // served yet).
489        let last_position = rotation.last_served.as_ref().and_then(|last| {
490            rotation
491                .active
492                .iter()
493                .position(|member| member.address == *last)
494        });
495
496        let index = last_position.map_or(0, |position| (position + 1) % rotation.active.len());
497        let address = rotation.active[index].address.clone();
498        rotation.last_served = Some(address.clone());
499        Some(address)
500    }
501
502    /// Drop `address` from the sticky active set, leaving its ban state
503    /// untouched; the next selection prefers a random live standby in its
504    /// place, falling back to evicted members if there are too few standbys.
505    ///
506    /// This is the failover path for callers that disable banning
507    /// ([RequestSettings::ban_failed_address](crate::RequestSettings)): a
508    /// failing node yields its slot's traffic to an available standby even
509    /// when it is never banned.
510    pub fn evict_from_rotation(&self, address: &Address) {
511        let mut rotation = self.rotation.write().unwrap();
512        if let Some(index) = rotation
513            .active
514            .iter()
515            .position(|member| member.address == *address)
516        {
517            let member = rotation.active.remove(index);
518            rotation.evicted.insert(member.address);
519        }
520    }
521
522    /// Get all not banned addresses.
523    ///
524    /// Returns a vector of addresses that are not currently banned or whose ban period has expired.
525    /// The returned addresses use the same filtering logic as [`Self::get_live_address`], checking if the
526    /// ban period has expired based on the current time.
527    ///
528    /// # Examples
529    ///
530    /// ```
531    /// use rs_dapi_client::{AddressList, Address};
532    ///
533    /// let mut list = AddressList::new();
534    /// list.add("http://127.0.0.1:3000".parse().unwrap());
535    /// list.add("http://127.0.0.1:3001".parse().unwrap());
536    ///
537    /// // Get all non-banned addresses
538    /// let live_addresses = list.get_live_addresses();
539    /// assert_eq!(live_addresses.len(), 2);
540    /// ```
541    pub fn get_live_addresses(&self) -> Vec<Address> {
542        let guard = self.addresses.read().unwrap();
543
544        let now = chrono::Utc::now();
545
546        guard
547            .iter()
548            .filter(|(_, status)| status.is_live(now))
549            .map(|(addr, _)| addr.clone())
550            .collect()
551    }
552
553    /// Get an owned snapshot of every address' ban state.
554    ///
555    /// Clones the current state into an owned `Vec<AddressBanInfo>` so
556    /// it can be inspected without holding the internal lock. The
557    /// `banned` flag reflects the *currently effectively banned*
558    /// semantics used by [`AddressList::get_live_address`]: the address
559    /// has been banned at least once (`ban_count > 0`) and its ban
560    /// period has not yet expired (`banned_until` is in the future).
561    pub fn ban_info(&self) -> Vec<AddressBanInfo> {
562        let guard = self.addresses.read().unwrap();
563
564        let now = chrono::Utc::now();
565
566        guard
567            .iter()
568            .map(|(addr, status)| {
569                let banned = status.ban_count > 0 && !status.is_live(now);
570                AddressBanInfo {
571                    uri: addr.to_string(),
572                    banned,
573                    ban_count: status.ban_count,
574                    banned_until: status.banned_until,
575                    reason: status.ban_reason.clone(),
576                }
577            })
578            .collect()
579    }
580
581    /// Get number of all addresses, both banned and not banned.
582    pub fn len(&self) -> usize {
583        self.addresses.read().unwrap().len()
584    }
585
586    /// Check if the list is empty.
587    /// Returns true if there are no addresses in the list.
588    /// Returns false if there is at least one address in the list.
589    /// Banned addresses are also counted.
590    pub fn is_empty(&self) -> bool {
591        self.addresses.read().unwrap().is_empty()
592    }
593}
594
595impl IntoIterator for AddressList {
596    type Item = (Address, AddressStatus);
597    type IntoIter = std::collections::hash_map::IntoIter<Address, AddressStatus>;
598
599    fn into_iter(self) -> Self::IntoIter {
600        let mut guard = self.addresses.write().unwrap();
601
602        let addresses_map = mem::take(&mut *guard);
603
604        addresses_map.into_iter()
605    }
606}
607
608impl FromStr for AddressList {
609    type Err = AddressListError;
610
611    fn from_str(s: &str) -> Result<Self, Self::Err> {
612        let uri_list: Vec<Address> = s
613            .split(',')
614            .map(Address::from_str)
615            .collect::<Result<_, _>>()?;
616
617        Ok(Self::from_iter(uri_list))
618    }
619}
620
621impl FromIterator<Address> for AddressList {
622    fn from_iter<T: IntoIterator<Item = Address>>(iter: T) -> Self {
623        let mut address_list = Self::new();
624        for uri in iter {
625            address_list.add(uri);
626        }
627
628        address_list
629    }
630}
631
632#[cfg(test)]
633mod tests {
634    use super::*;
635
636    fn list_with_addresses(count: usize) -> AddressList {
637        (0..count)
638            .map(|i| format!("http://127.0.0.1:{}", 3000 + i).parse().unwrap())
639            .collect()
640    }
641
642    #[test]
643    fn test_get_live_addresses_empty_list() {
644        let list = AddressList::new();
645        let live_addresses = list.get_live_addresses();
646        assert_eq!(live_addresses.len(), 0);
647    }
648
649    #[test]
650    fn test_get_live_addresses_all_unbanned() {
651        let mut list = AddressList::new();
652        list.add("http://127.0.0.1:3000".parse().unwrap());
653        list.add("http://127.0.0.1:3001".parse().unwrap());
654        list.add("http://127.0.0.1:3002".parse().unwrap());
655
656        let live_addresses = list.get_live_addresses();
657        assert_eq!(live_addresses.len(), 3);
658    }
659
660    #[test]
661    fn test_get_live_addresses_some_banned() {
662        let mut list = AddressList::new();
663        let addr1: Address = "http://127.0.0.1:3000".parse().unwrap();
664        let addr2: Address = "http://127.0.0.1:3001".parse().unwrap();
665        let addr3: Address = "http://127.0.0.1:3002".parse().unwrap();
666
667        list.add(addr1.clone());
668        list.add(addr2.clone());
669        list.add(addr3.clone());
670
671        // Ban addr2
672        list.ban(&addr2);
673
674        let live_addresses = list.get_live_addresses();
675        assert_eq!(live_addresses.len(), 2);
676        assert!(live_addresses.contains(&addr1));
677        assert!(live_addresses.contains(&addr3));
678        assert!(!live_addresses.contains(&addr2));
679    }
680
681    #[test]
682    fn test_get_live_addresses_all_banned() {
683        let mut list = AddressList::new();
684        let addr1: Address = "http://127.0.0.1:3000".parse().unwrap();
685        let addr2: Address = "http://127.0.0.1:3001".parse().unwrap();
686
687        list.add(addr1.clone());
688        list.add(addr2.clone());
689
690        // Ban all addresses
691        list.ban(&addr1);
692        list.ban(&addr2);
693
694        let live_addresses = list.get_live_addresses();
695        assert_eq!(live_addresses.len(), 0);
696    }
697
698    #[test]
699    fn test_get_live_addresses_unbanned_after_ban() {
700        let mut list = AddressList::new();
701        let addr1: Address = "http://127.0.0.1:3000".parse().unwrap();
702
703        list.add(addr1.clone());
704
705        // Ban and then unban
706        list.ban(&addr1);
707        list.unban(&addr1);
708
709        let live_addresses = list.get_live_addresses();
710        assert_eq!(live_addresses.len(), 1);
711        assert!(live_addresses.contains(&addr1));
712    }
713
714    #[test]
715    fn test_address_try_from_uri_without_host() {
716        let uri: Uri = Uri::from_str("/path/only").unwrap();
717        let result = Address::try_from(uri);
718        assert!(result.is_err());
719        let err = result.unwrap_err();
720        assert!(matches!(err, AddressListError::InvalidAddressUri(_)));
721    }
722
723    #[test]
724    fn test_address_from_str_invalid_uri() {
725        // Use a string with invalid URI characters that http::Uri rejects
726        let result = Address::from_str("not a valid uri\x00");
727        assert!(result.is_err());
728    }
729
730    #[test]
731    fn test_address_uri_accessor() {
732        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
733        let uri = addr.uri();
734        assert_eq!(uri.host(), Some("127.0.0.1"));
735    }
736
737    #[test]
738    fn test_address_partial_eq_with_uri() {
739        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
740        let uri = Uri::from_str("http://127.0.0.1:3000").unwrap();
741        assert!(addr == uri);
742
743        let other_uri = Uri::from_str("http://127.0.0.1:4000").unwrap();
744        assert!(addr != other_uri);
745    }
746
747    #[test]
748    fn test_address_display() {
749        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
750        let display = format!("{}", addr);
751        assert!(display.contains("127.0.0.1"));
752    }
753
754    #[test]
755    fn test_address_status_is_banned() {
756        let mut status = AddressStatus::default();
757        assert!(!status.is_banned());
758
759        status.ban(&Duration::from_secs(60));
760        assert!(status.is_banned());
761
762        status.unban();
763        assert!(!status.is_banned());
764    }
765
766    #[test]
767    fn test_address_status_exponential_ban() {
768        let mut status = AddressStatus::default();
769        let base_period = Duration::from_secs(1);
770
771        // First ban: coefficient = exp(0) = 1, period = 1s
772        status.ban(&base_period);
773        assert_eq!(status.ban_count, 1);
774        assert!(status.banned_until.is_some());
775
776        // Second ban: coefficient = exp(1) ~= 2.718, period ~= 2.718s
777        status.ban(&base_period);
778        assert_eq!(status.ban_count, 2);
779    }
780
781    #[test]
782    fn test_address_list_is_empty() {
783        let list = AddressList::new();
784        assert!(list.is_empty());
785
786        let mut list = AddressList::new();
787        list.add("http://127.0.0.1:3000".parse().unwrap());
788        assert!(!list.is_empty());
789    }
790
791    #[test]
792    fn test_address_list_len() {
793        let mut list = AddressList::new();
794        assert_eq!(list.len(), 0);
795
796        list.add("http://127.0.0.1:3000".parse().unwrap());
797        assert_eq!(list.len(), 1);
798
799        list.add("http://127.0.0.1:3001".parse().unwrap());
800        assert_eq!(list.len(), 2);
801    }
802
803    #[test]
804    fn test_address_list_add_duplicate() {
805        let mut list = AddressList::new();
806        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
807
808        assert!(list.add(addr.clone()));
809        assert!(!list.add(addr)); // duplicate returns false
810        assert_eq!(list.len(), 1);
811    }
812
813    #[test]
814    fn test_address_list_remove() {
815        let mut list = AddressList::new();
816        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
817
818        list.add(addr.clone());
819        assert_eq!(list.len(), 1);
820
821        let removed = list.remove(&addr);
822        assert!(removed.is_some());
823        assert_eq!(list.len(), 0);
824
825        // Removing non-existent address returns None
826        let removed = list.remove(&addr);
827        assert!(removed.is_none());
828    }
829
830    #[test]
831    fn test_address_list_ban_nonexistent() {
832        let list = AddressList::new();
833        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
834        assert!(!list.ban(&addr));
835    }
836
837    #[test]
838    fn test_address_list_unban_nonexistent() {
839        let list = AddressList::new();
840        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
841        assert!(!list.unban(&addr));
842    }
843
844    #[test]
845    fn test_address_list_is_banned() {
846        let mut list = AddressList::new();
847        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
848        let unknown: Address = "http://127.0.0.1:9999".parse().unwrap();
849
850        list.add(addr.clone());
851
852        assert!(!list.is_banned(&addr));
853        assert!(!list.is_banned(&unknown)); // unknown returns false
854
855        list.ban(&addr);
856        assert!(list.is_banned(&addr));
857    }
858
859    #[test]
860    fn test_address_list_from_str() {
861        let list: AddressList = "http://127.0.0.1:3000,http://127.0.0.1:3001"
862            .parse()
863            .unwrap();
864        assert_eq!(list.len(), 2);
865    }
866
867    #[test]
868    fn test_address_list_from_str_single() {
869        let list: AddressList = "http://127.0.0.1:3000".parse().unwrap();
870        assert_eq!(list.len(), 1);
871    }
872
873    #[test]
874    fn test_address_list_from_str_invalid() {
875        let result: Result<AddressList, _> = "not a valid uri\x00".parse();
876        assert!(result.is_err());
877    }
878
879    #[test]
880    fn test_address_list_get_live_address_returns_none_when_empty() {
881        let list = AddressList::new();
882        assert!(list.get_live_address().is_none());
883    }
884
885    #[test]
886    fn test_get_live_address_sticks_to_small_active_set() {
887        let list = list_with_addresses(50);
888
889        let distinct: std::collections::HashSet<String> = (0..200)
890            .map(|_| list.get_live_address().unwrap().to_string())
891            .collect();
892
893        assert_eq!(
894            distinct.len(),
895            DEFAULT_ACTIVE_SET_SIZE,
896            "all traffic must rotate over exactly the active set"
897        );
898    }
899
900    #[test]
901    fn test_get_live_address_round_robins_over_active_set() {
902        let list = list_with_addresses(5).with_active_set_size(2);
903
904        let picks: Vec<String> = (0..6)
905            .map(|_| list.get_live_address().unwrap().to_string())
906            .collect();
907
908        // Strict alternation between the two active members.
909        assert_ne!(picks[0], picks[1]);
910        assert_eq!(picks[0], picks[2]);
911        assert_eq!(picks[1], picks[3]);
912        assert_eq!(picks[0], picks[4]);
913        assert_eq!(picks[1], picks[5]);
914    }
915
916    #[test]
917    fn test_get_live_address_ban_evicts_active_and_promotes_standby() {
918        let list = list_with_addresses(3).with_active_set_size(1);
919
920        let first = list.get_live_address().unwrap();
921        for _ in 0..5 {
922            assert_eq!(
923                list.get_live_address().unwrap(),
924                first,
925                "selection must be sticky until the active address fails"
926            );
927        }
928
929        list.ban(&first);
930
931        let second = list.get_live_address().unwrap();
932        assert_ne!(second, first, "banned address must leave the active set");
933        for _ in 0..5 {
934            assert_eq!(
935                list.get_live_address().unwrap(),
936                second,
937                "selection must stick to the promoted standby"
938            );
939        }
940    }
941
942    #[test]
943    fn test_get_live_address_no_immediate_repeat_after_other_member_evicted() {
944        let list = list_with_addresses(3).with_active_set_size(2);
945
946        let first = list.get_live_address().unwrap();
947        let second = list.get_live_address().unwrap();
948        let third = list.get_live_address().unwrap();
949        assert_eq!(first, third, "two-member set alternates");
950
951        // Ban the member that was NOT just served; a standby gets promoted.
952        list.ban(&second);
953
954        let next = list.get_live_address().unwrap();
955        assert_ne!(
956            next, third,
957            "must not serve the same address twice in a row after eviction"
958        );
959        assert_ne!(next, second, "banned address must not be served");
960    }
961
962    #[test]
963    fn test_get_live_address_removed_address_pruned_from_active_set() {
964        let mut list = AddressList::new().with_active_set_size(1);
965        list.add("http://127.0.0.1:3000".parse().unwrap());
966        list.add("http://127.0.0.1:3001".parse().unwrap());
967
968        let first = list.get_live_address().unwrap();
969        list.remove(&first);
970
971        let second = list.get_live_address().unwrap();
972        assert_ne!(second, first);
973    }
974
975    #[test]
976    fn test_get_live_address_with_fewer_live_addresses_than_active_set() {
977        let mut list = AddressList::new(); // default active set size 5
978        list.add("http://127.0.0.1:3000".parse().unwrap());
979        list.add("http://127.0.0.1:3001".parse().unwrap());
980
981        let distinct: std::collections::HashSet<String> = (0..10)
982            .map(|_| list.get_live_address().unwrap().to_string())
983            .collect();
984        assert_eq!(distinct.len(), 2, "both live addresses rotate");
985    }
986
987    #[test]
988    fn test_get_live_address_all_banned_returns_none() {
989        let mut list = AddressList::new().with_active_set_size(1);
990        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
991        list.add(addr.clone());
992
993        assert!(list.get_live_address().is_some());
994        list.ban(&addr);
995        assert!(list.get_live_address().is_none());
996    }
997
998    #[test]
999    fn test_get_live_address_expired_slot_is_recycled() {
1000        let list = list_with_addresses(10);
1001
1002        // Populate the active set, then back-date every slot's expiry.
1003        list.get_live_address().unwrap();
1004        let retired = {
1005            let mut rotation = list.rotation.write().unwrap();
1006            assert_eq!(rotation.active.len(), DEFAULT_ACTIVE_SET_SIZE);
1007            for member in rotation.active.iter_mut() {
1008                member.slot_expires_at = chrono::Utc::now() - Duration::from_secs(1);
1009            }
1010            rotation
1011                .active
1012                .iter()
1013                .map(|member| member.address.clone())
1014                .collect::<Vec<_>>()
1015        };
1016
1017        let before = chrono::Utc::now();
1018        assert!(
1019            list.get_live_address().is_some(),
1020            "selection must survive whole-set expiry"
1021        );
1022
1023        let rotation = list.rotation.read().unwrap();
1024        assert_eq!(rotation.active.len(), DEFAULT_ACTIVE_SET_SIZE);
1025        assert!(
1026            rotation
1027                .active
1028                .iter()
1029                .all(|member| !retired.contains(&member.address)),
1030            "live standbys must replace expired members when enough are available"
1031        );
1032        assert!(
1033            rotation
1034                .active
1035                .iter()
1036                .all(|member| member.slot_expires_at > before),
1037            "expired slots must be retired and re-issued with fresh expiries"
1038        );
1039    }
1040
1041    #[test]
1042    fn test_get_live_address_sole_address_survives_slot_expiry() {
1043        let mut list = AddressList::new().with_active_set_size(1);
1044        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
1045        list.add(addr.clone());
1046
1047        assert_eq!(list.get_live_address().unwrap(), addr);
1048        list.rotation.write().unwrap().active[0].slot_expires_at =
1049            chrono::Utc::now() - Duration::from_secs(1);
1050
1051        assert_eq!(
1052            list.get_live_address().unwrap(),
1053            addr,
1054            "the only live address must be re-promoted after its slot expires"
1055        );
1056    }
1057
1058    #[test]
1059    fn test_evict_from_rotation_removes_member_without_ban() {
1060        let list = list_with_addresses(3).with_active_set_size(2);
1061
1062        let served = list.get_live_address().unwrap();
1063        list.evict_from_rotation(&served);
1064
1065        assert!(
1066            !list
1067                .rotation
1068                .read()
1069                .unwrap()
1070                .active
1071                .iter()
1072                .any(|member| member.address == served),
1073            "evicted address must leave the active set"
1074        );
1075        assert!(
1076            !list.is_banned(&served),
1077            "eviction must not touch ban state"
1078        );
1079        assert!(list.get_live_address().is_some());
1080        assert!(
1081            list.rotation
1082                .read()
1083                .unwrap()
1084                .active
1085                .iter()
1086                .all(|member| member.address != served),
1087            "the available standby must replace the evicted address"
1088        );
1089    }
1090
1091    #[test]
1092    fn should_keep_sole_live_address_available_after_eviction() {
1093        let mut list = AddressList::new().with_active_set_size(1);
1094        let address: Address = "http://127.0.0.1:3000".parse().unwrap();
1095        let banned: Address = "http://127.0.0.1:3001".parse().unwrap();
1096        list.add(address.clone());
1097        list.add(banned.clone());
1098        list.ban(&banned);
1099        assert_eq!(list.get_live_address(), Some(address.clone()));
1100        list.evict_from_rotation(&address);
1101        assert_eq!(list.get_live_address(), Some(address.clone()));
1102        assert!(!list.is_banned(&address));
1103    }
1104
1105    #[test]
1106    fn should_use_all_live_standbys_before_recycling_expired_members() {
1107        let list = list_with_addresses(4).with_active_set_size(3);
1108        list.get_live_address().unwrap();
1109        let standby = list
1110            .get_live_addresses()
1111            .into_iter()
1112            .find(|address| {
1113                !list
1114                    .rotation
1115                    .read()
1116                    .unwrap()
1117                    .active
1118                    .iter()
1119                    .any(|member| member.address == *address)
1120            })
1121            .unwrap();
1122        for member in &mut list.rotation.write().unwrap().active {
1123            member.slot_expires_at = chrono::Utc::now() - Duration::from_secs(1);
1124        }
1125        list.get_live_address().unwrap();
1126        let rotation = list.rotation.read().unwrap();
1127        assert_eq!(
1128            rotation.active.len(),
1129            3,
1130            "fallback must preserve the effective active-set size"
1131        );
1132        assert!(
1133            rotation
1134                .active
1135                .iter()
1136                .any(|member| member.address == standby),
1137            "the one available standby must be promoted before recycling expired members"
1138        );
1139    }
1140
1141    #[test]
1142    fn test_update_address_ban_status_evicts_when_banning_disabled() {
1143        use crate::{transport::TransportError, ExecutionError, ExecutionResult, RequestSettings};
1144        use dapi_grpc::tonic::Status;
1145
1146        let list = list_with_addresses(2).with_active_set_size(1);
1147        let failed = list.get_live_address().unwrap();
1148
1149        let result: ExecutionResult<i32, TransportError> = Err(ExecutionError {
1150            inner: TransportError::Grpc(Status::unavailable("node down")),
1151            retries: 0,
1152            address: Some(failed.clone()),
1153        });
1154        let settings = RequestSettings {
1155            ban_failed_address: Some(false),
1156            ..RequestSettings::default()
1157        }
1158        .finalize();
1159        crate::update_address_ban_status(&list, &result, &settings);
1160
1161        assert!(
1162            !list
1163                .rotation
1164                .read()
1165                .unwrap()
1166                .active
1167                .iter()
1168                .any(|member| member.address == failed),
1169            "failed address must leave the rotation even when banning is disabled"
1170        );
1171        assert!(!list.is_banned(&failed), "banning stays disabled");
1172        let next = list.get_live_address().unwrap();
1173        assert_ne!(
1174            next, failed,
1175            "the next retry must select the available standby"
1176        );
1177        for _ in 0..5 {
1178            assert_eq!(
1179                list.get_live_address().unwrap(),
1180                next,
1181                "the replacement must remain sticky"
1182            );
1183        }
1184    }
1185
1186    #[test]
1187    fn test_with_active_set_size_is_shared_across_clones_and_shrinks() {
1188        let list = list_with_addresses(10);
1189
1190        // Fill the default-sized active set through the original handle.
1191        list.get_live_address().unwrap();
1192        assert_eq!(
1193            list.rotation.read().unwrap().active.len(),
1194            DEFAULT_ACTIVE_SET_SIZE
1195        );
1196
1197        // Reconfiguring through a clone applies to the shared rotation.
1198        let _clone = list.clone().with_active_set_size(2);
1199
1200        let distinct: std::collections::HashSet<String> = (0..20)
1201            .map(|_| list.get_live_address().unwrap().to_string())
1202            .collect();
1203        assert_eq!(
1204            distinct.len(),
1205            2,
1206            "shrunken size set through a clone must constrain every handle"
1207        );
1208    }
1209
1210    #[test]
1211    fn test_with_active_set_size_max_uses_whole_list() {
1212        let list = list_with_addresses(3).with_active_set_size(usize::MAX);
1213
1214        // Must not over-allocate or panic; effectively disables stickiness.
1215        let distinct: std::collections::HashSet<String> = (0..9)
1216            .map(|_| list.get_live_address().unwrap().to_string())
1217            .collect();
1218        assert_eq!(distinct.len(), 3, "all live addresses rotate");
1219    }
1220
1221    #[test]
1222    fn test_ban_ladder_window_is_capped() {
1223        let mut status = AddressStatus::default();
1224        let base = Duration::from_secs(60);
1225
1226        // Pre-cap, ban #27 overflowed `DateTime + Duration` and panicked.
1227        for _ in 0..40 {
1228            status.ban(&base);
1229        }
1230
1231        let until = status.banned_until.expect("banned_until set");
1232        let window = until - chrono::Utc::now();
1233        assert!(
1234            window <= chrono::TimeDelta::from_std(MAX_BAN_PERIOD).unwrap(),
1235            "ban window must be capped at MAX_BAN_PERIOD"
1236        );
1237    }
1238
1239    #[test]
1240    fn test_ban_for_huge_period_is_clamped() {
1241        let mut status = AddressStatus::default();
1242
1243        // Pre-clamp this overflowed `DateTime + Duration` and panicked.
1244        status.ban_for(Duration::from_secs(u64::MAX), None);
1245
1246        let until = status.banned_until.expect("banned_until set");
1247        let window = until - chrono::Utc::now();
1248        assert!(
1249            window <= chrono::TimeDelta::from_std(MAX_BAN_PERIOD).unwrap(),
1250            "advertised ban window must be clamped to MAX_BAN_PERIOD"
1251        );
1252    }
1253
1254    #[test]
1255    fn should_cap_oversized_base_bans_without_poisoning_the_list() {
1256        for base in [Duration::from_secs(48 * 60 * 60), Duration::MAX] {
1257            let mut list = AddressList::with_settings(base);
1258            let address: Address = "http://127.0.0.1:3000".parse().unwrap();
1259            list.add(address.clone());
1260            list.ban(&address);
1261            let until = list.addresses.read().unwrap()[&address]
1262                .banned_until
1263                .unwrap();
1264            assert!(
1265                until - chrono::Utc::now() <= chrono::TimeDelta::from_std(MAX_BAN_PERIOD).unwrap()
1266            );
1267            assert!(list.get_live_address().is_none());
1268            list.unban(&address);
1269            assert_eq!(list.get_live_address(), Some(address));
1270        }
1271    }
1272
1273    #[test]
1274    fn test_address_list_get_live_address_returns_some_when_available() {
1275        let mut list = AddressList::new();
1276        list.add("http://127.0.0.1:3000".parse().unwrap());
1277        assert!(list.get_live_address().is_some());
1278    }
1279
1280    #[test]
1281    fn test_address_list_into_iter() {
1282        let mut list = AddressList::new();
1283        list.add("http://127.0.0.1:3000".parse().unwrap());
1284        list.add("http://127.0.0.1:3001".parse().unwrap());
1285
1286        let items: Vec<_> = list.into_iter().collect();
1287        assert_eq!(items.len(), 2);
1288    }
1289
1290    #[test]
1291    fn test_address_list_with_settings() {
1292        let list = AddressList::with_settings(Duration::from_secs(120));
1293        assert!(list.is_empty());
1294    }
1295
1296    #[test]
1297    fn test_address_list_default() {
1298        let list = AddressList::default();
1299        assert!(list.is_empty());
1300    }
1301
1302    #[test]
1303    fn test_address_status_ban_with_reason_stores_reason() {
1304        let mut status = AddressStatus::default();
1305        assert!(status.ban_reason.is_none());
1306
1307        status.ban_with_reason(
1308            &Duration::from_secs(60),
1309            Some("transport error".to_string()),
1310        );
1311        assert_eq!(status.ban_reason.as_deref(), Some("transport error"));
1312        assert!(status.is_banned());
1313    }
1314
1315    #[test]
1316    fn test_address_status_ban_without_reason_is_none() {
1317        let mut status = AddressStatus::default();
1318        status.ban(&Duration::from_secs(60));
1319        assert!(status.ban_reason.is_none());
1320        assert!(status.is_banned());
1321    }
1322
1323    #[test]
1324    fn test_address_status_unban_clears_reason() {
1325        let mut status = AddressStatus::default();
1326        status.ban_with_reason(&Duration::from_secs(60), Some("boom".to_string()));
1327        assert_eq!(status.ban_reason.as_deref(), Some("boom"));
1328
1329        status.unban();
1330        assert!(status.ban_reason.is_none());
1331        assert!(!status.is_banned());
1332    }
1333
1334    #[test]
1335    fn test_address_list_ban_with_reason_records_reason() {
1336        let mut list = AddressList::new();
1337        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
1338        list.add(addr.clone());
1339
1340        assert!(list.ban_with_reason(&addr, Some("node down".to_string())));
1341
1342        let info = list.ban_info();
1343        assert_eq!(info.len(), 1);
1344        let entry = &info[0];
1345        assert_eq!(entry.reason.as_deref(), Some("node down"));
1346        assert!(entry.banned);
1347        assert_eq!(entry.ban_count, 1);
1348        assert!(entry.banned_until.is_some());
1349    }
1350
1351    #[test]
1352    fn test_address_list_ban_without_reason_records_none() {
1353        let mut list = AddressList::new();
1354        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
1355        list.add(addr.clone());
1356
1357        assert!(list.ban(&addr));
1358
1359        let info = list.ban_info();
1360        assert_eq!(info.len(), 1);
1361        assert!(info[0].reason.is_none());
1362        assert!(info[0].banned);
1363    }
1364
1365    #[test]
1366    fn test_address_list_unban_clears_reason_in_ban_info() {
1367        let mut list = AddressList::new();
1368        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
1369        list.add(addr.clone());
1370
1371        list.ban_with_reason(&addr, Some("oops".to_string()));
1372        assert!(list.unban(&addr));
1373
1374        let info = list.ban_info();
1375        assert_eq!(info.len(), 1);
1376        let entry = &info[0];
1377        assert!(entry.reason.is_none());
1378        assert!(!entry.banned);
1379        assert_eq!(entry.ban_count, 0);
1380        assert!(entry.banned_until.is_none());
1381    }
1382
1383    #[test]
1384    fn test_ban_info_reflects_unbanned_address() {
1385        let mut list = AddressList::new();
1386        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
1387        list.add(addr.clone());
1388
1389        // Never banned: banned == false, no reason, ban_count == 0.
1390        let info = list.ban_info();
1391        assert_eq!(info.len(), 1);
1392        let entry = &info[0];
1393        assert!(!entry.banned);
1394        assert_eq!(entry.ban_count, 0);
1395        assert!(entry.banned_until.is_none());
1396        assert!(entry.reason.is_none());
1397        assert!(entry.uri.contains("127.0.0.1"));
1398    }
1399
1400    #[test]
1401    fn test_ban_info_empty_list() {
1402        let list = AddressList::new();
1403        assert!(list.ban_info().is_empty());
1404    }
1405
1406    #[test]
1407    fn test_address_status_ban_for_sets_exact_window_and_min_ban_count() {
1408        let mut status = AddressStatus::default();
1409        assert_eq!(status.ban_count, 0);
1410        assert!(status.banned_until.is_none());
1411
1412        let before = chrono::Utc::now();
1413        status.ban_for(Duration::from_secs(45), Some("rate limited".into()));
1414        let after = chrono::Utc::now();
1415
1416        // ban_count must be at least 1 so is_banned() / ban_info().banned are consistent.
1417        assert_eq!(status.ban_count, 1, "ban_for sets ban_count to max(0,1)=1");
1418
1419        // banned_until should be roughly now + 45 s.
1420        let until = status.banned_until.expect("banned_until must be set");
1421        let lower = (until - before).num_milliseconds() as f64 / 1000.0;
1422        let upper = (until - after).num_milliseconds() as f64 / 1000.0;
1423        assert!(
1424            lower >= 44.9,
1425            "banned_until lower bound too short: {lower}s"
1426        );
1427        assert!(upper <= 45.1, "banned_until upper bound too long: {upper}s");
1428        assert_eq!(status.ban_reason.as_deref(), Some("rate limited"));
1429    }
1430
1431    /// `ban_for` on a fresh node (ban_count = 0) raises ban_count to 1 (the
1432    /// ladder floor).  That means the *next* genuine health ban will escalate
1433    /// from position 1 (~163 s) instead of position 0 (~60 s).  This pins the
1434    /// documented side-effect so regressions are caught.
1435    #[test]
1436    fn test_ban_for_raises_fresh_node_to_ladder_floor() {
1437        let mut status = AddressStatus::default();
1438        assert_eq!(status.ban_count, 0, "starts clean");
1439
1440        // Rate-limit ban on a never-before-banned node.
1441        status.ban_for(Duration::from_secs(10), Some("rl".into()));
1442        assert_eq!(
1443            status.ban_count, 1,
1444            "ban_for must raise ban_count 0 → 1 (ladder floor)"
1445        );
1446
1447        // Subsequent genuine health failure must escalate from the floor (1),
1448        // yielding ~60 s × e^1 ≈ 163 s, NOT the first-rung ~60 s × e^0 = 60 s.
1449        let base = Duration::from_secs(60);
1450        let before = chrono::Utc::now();
1451        status.ban_with_reason(&base, None); // ban_count 1 → 2; window = 60s × e^1
1452        let after = chrono::Utc::now();
1453        assert_eq!(status.ban_count, 2);
1454
1455        let until = status.banned_until.expect("banned_until set");
1456        let lo = (until - before).num_milliseconds() as f64 / 1000.0;
1457        let hi = (until - after).num_milliseconds() as f64 / 1000.0;
1458        let expected = 60.0_f64 * std::f64::consts::E; // ≈ 163 s
1459        assert!(
1460            lo >= expected - 0.5,
1461            "window lower {lo:.1}s < expected {expected:.1}s (should escalate from floor 1)"
1462        );
1463        assert!(
1464            hi <= expected + 0.5,
1465            "window upper {hi:.1}s > expected {expected:.1}s"
1466        );
1467    }
1468
1469    #[test]
1470    fn test_address_status_ban_for_does_not_inflate_existing_ban_count() {
1471        // A node already health-banned (ban_count = 3) gets rate-limited.
1472        // ban_count must stay at 3, not grow to 4.
1473        let mut status = AddressStatus::default();
1474        let base = Duration::from_secs(60);
1475        status.ban_with_reason(&base, None); // → 1
1476        status.ban_with_reason(&base, None); // → 2
1477        status.ban_with_reason(&base, None); // → 3
1478        status.ban_for(Duration::from_secs(30), Some("rl".into()));
1479        assert_eq!(
1480            status.ban_count, 3,
1481            "ban_for must not inflate ban_count above its existing value"
1482        );
1483    }
1484
1485    #[test]
1486    fn test_address_list_ban_for_returns_false_for_unknown() {
1487        let list = AddressList::new();
1488        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
1489        assert!(!list.ban_for(&addr, Duration::from_secs(5), None));
1490    }
1491
1492    #[test]
1493    fn test_address_list_ban_for_bans_known_address() {
1494        let mut list = AddressList::new();
1495        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
1496        list.add(addr.clone());
1497
1498        assert!(list.ban_for(&addr, Duration::from_secs(60), Some("rl".into())));
1499        // The address must now be hidden from get_live_address.
1500        assert!(list.get_live_address().is_none());
1501        // ban_count is 1 (ban_for sets max(0,1)).
1502        let info = list.ban_info();
1503        assert_eq!(info.len(), 1);
1504        assert!(info[0].banned);
1505        assert_eq!(info[0].ban_count, 1);
1506    }
1507
1508    /// After `ban_for`'s window expires the address re-enters rotation via
1509    /// `get_live_address`.  We verify both directions: the node is hidden during
1510    /// an active window, and becomes live once the window has passed.
1511    ///
1512    /// Window-expiry reinstatement is orthogonal to `unban()`: `get_live_address`
1513    /// reinstates a node purely on `banned_until < now` regardless of `ban_count`,
1514    /// so after expiry the node is live again while `is_banned()` (ban_count > 0)
1515    /// is still true.  This is a different path from `unban()`, which also zeroes
1516    /// `ban_count`.
1517    #[test]
1518    fn test_ban_for_address_re_enters_rotation_after_window_expires() {
1519        let mut list = AddressList::new();
1520        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
1521        list.add(addr.clone());
1522
1523        // Active 300-second window → node hidden.
1524        assert!(list.ban_for(&addr, Duration::from_secs(300), Some("rl".into())));
1525        assert!(
1526            list.get_live_address().is_none(),
1527            "node must be hidden during active ban window"
1528        );
1529
1530        // Simulate window expiry by back-dating banned_until — do NOT touch ban_count.
1531        {
1532            let mut guard = list.addresses.write().unwrap();
1533            let status = guard.get_mut(&addr).expect("addr must be in list");
1534            status.banned_until = Some(chrono::Utc::now() - Duration::from_secs(1));
1535        }
1536
1537        // After window expiry the node must re-enter rotation …
1538        assert!(
1539            list.get_live_address().is_some(),
1540            "address must re-enter rotation after ban window expires"
1541        );
1542        // … but ban_count is still > 0, so is_banned() remains true.
1543        // This distinguishes window-expiry from an explicit unban().
1544        assert!(
1545            list.is_banned(&addr),
1546            "is_banned() must still be true after window expiry (ban_count not reset)"
1547        );
1548    }
1549
1550    /// Invariant 1 at the ladder source: the exponential ban window is
1551    /// `base × e^ban_count`, `ban_count` incrementing on each ban. This pins the
1552    /// exact formula independently of the `update_address_ban_status` entrypoint.
1553    #[test]
1554    fn test_ban_ladder_windows_match_exponential_formula() {
1555        let mut status = AddressStatus::default();
1556        let base_secs = 60.0_f64;
1557        let base = Duration::from_secs(60);
1558
1559        for n in 0..3usize {
1560            // coefficient uses ban_count BEFORE this ban (== n here).
1561            let before = chrono::Utc::now();
1562            status.ban(&base);
1563            let after = chrono::Utc::now();
1564
1565            assert_eq!(status.ban_count, n + 1, "ban_count must increment");
1566            let period = base_secs * (n as f64).exp();
1567            let banned_until = status.banned_until.expect("banned_until is set");
1568            let lower = (banned_until - before).num_milliseconds() as f64 / 1000.0;
1569            let upper = (banned_until - after).num_milliseconds() as f64 / 1000.0;
1570            assert!(
1571                lower >= period - 0.05,
1572                "ban #{} window lower bound {lower}s < expected {period}s",
1573                n + 1
1574            );
1575            assert!(
1576                upper <= period + 0.05,
1577                "ban #{} window upper bound {upper}s > expected {period}s",
1578                n + 1
1579            );
1580        }
1581    }
1582}