1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
//! An implementation of the "decorrelated jitter" algorithm for scheduling retries.
//!
//! See [`RetryDelay`] for more information.
use std::time::Duration;
use rand::Rng;
/// An implementation for retrying a remote operation based on a [decorrelated
/// jitter] schedule.
///
/// The algorithm used here has several desirable properties:
/// * It is randomized, so that multiple timeouts don't have a danger of
/// getting synchronized with each other and hammering the same servers all
/// at once.
/// * It tends on average to wait longer and longer over time, so that if the
/// server is down, it won't get pummeled by a zillion failing clients
/// when it comes back up.
/// * It has a chance of retrying promptly, which results in better client
/// performance on average.
///
/// For a more full specification, see [`dir-spec.txt`].
///
/// [decorrelated jitter]:
/// https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
/// [`dir-spec.txt`]: https://spec.torproject.org/dir-spec
#[derive(Clone, Debug)]
pub struct RetryDelay {
/// The last delay that this retry delay returned (in msec), or 0
/// if this never returned a delay.
last_delay_ms: u32,
/// The lowest allowable delay (in msec).
low_bound_ms: u32,
}
/// Lowest possible lower bound, in milliseconds.
// We're doing this in MS, and Tor does it in seconds, so I'm
// multiplying the minimum by 1000 here.
const MIN_LOW_BOUND: u32 = 1000;
/// Largest possible lower bound, in milliseconds.
const MAX_LOW_BOUND: u32 = std::u32::MAX - 1;
/// Maximum amount to multiply the previous delay by.
const MAX_DELAY_MULT: u32 = 3;
impl RetryDelay {
/// Construct a new RetryDelay from a given base delay in
/// milliseconds.
///
/// The base delay defines the lowest possible interval that can
/// be returned.
///
/// # Limitations
///
/// If the base delay is less than 1000, a base delay of 1000 is
/// used instead, since that's what the C tor implementation does.
pub fn from_msec(base_delay_msec: u32) -> Self {
let low_bound_ms = base_delay_msec.clamp(MIN_LOW_BOUND, MAX_LOW_BOUND);
RetryDelay {
last_delay_ms: 0,
low_bound_ms,
}
}
/// Construct a new RetryDelay from a given base delay.
///
/// See from_msec for more information.
pub fn from_duration(d: Duration) -> Self {
let msec = d.as_millis();
let msec = std::cmp::min(msec, u128::from(MAX_LOW_BOUND)) as u32;
RetryDelay::from_msec(msec)
}
/// Helper: Return a lower and upper bound for the next delay to
/// be yielded.
///
/// Values are in milliseconds.
fn delay_bounds(&self) -> (u32, u32) {
let low = self.low_bound_ms;
let high = std::cmp::max(
// We don't need a saturating_add here, since low is always
// <= MAX_LOW_BOUND, so low cannot be equal to u32::MAX.
low + 1,
self.last_delay_ms.saturating_mul(MAX_DELAY_MULT),
);
(low, high)
}
/// Return the next delay to be used (in milliseconds), according
/// to a given random number generator.
fn next_delay_msec<R: Rng>(&mut self, rng: &mut R) -> u32 {
let (low, high) = self.delay_bounds();
assert!(low < high);
let val = rng.gen_range(low..high);
self.last_delay_ms = val;
val
}
/// Return the next delay to be used (as a [`Duration`]),
/// according to a given random number generator.
pub fn next_delay<R: Rng>(&mut self, rng: &mut R) -> Duration {
Duration::from_millis(u64::from(self.next_delay_msec(rng)))
}
/// Return this [`RetryDelay`] to its original state.
pub fn reset(&mut self) {
self.last_delay_ms = 0;
}
}
impl Default for RetryDelay {
fn default() -> Self {
RetryDelay::from_msec(0)
}
}
#[cfg(test)]
mod test {
use super::*;
use crate::test_rng::testing_rng;
#[test]
fn init() {
let rd = RetryDelay::from_msec(2000);
assert_eq!(rd.last_delay_ms, 0);
assert_eq!(rd.low_bound_ms, 2000);
let rd = RetryDelay::from_msec(0);
assert_eq!(rd.last_delay_ms, 0);
assert_eq!(rd.low_bound_ms, 1000);
let rd = RetryDelay::from_duration(Duration::new(1, 500_000_000));
assert_eq!(rd.last_delay_ms, 0);
assert_eq!(rd.low_bound_ms, 1500);
}
#[test]
fn bounds() {
let mut rd = RetryDelay::from_msec(1000);
assert_eq!(rd.delay_bounds(), (1000, 1001));
rd.last_delay_ms = 1500;
assert_eq!(rd.delay_bounds(), (1000, 4500));
rd.last_delay_ms = 3_000_000_000;
assert_eq!(rd.delay_bounds(), (1000, std::u32::MAX));
rd.reset();
assert_eq!(rd.delay_bounds(), (1000, 1001));
}
#[test]
fn rng() {
let mut rd = RetryDelay::from_msec(50);
let real_low_bound = std::cmp::max(50, MIN_LOW_BOUND);
let mut rng = testing_rng();
for _ in 1..100 {
let (b_lo, b_hi) = rd.delay_bounds();
assert!(b_lo == real_low_bound);
assert!(b_hi > b_lo);
let delay = rd.next_delay(&mut rng).as_millis() as u32;
assert_eq!(delay, rd.last_delay_ms);
assert!(delay >= b_lo);
assert!(delay < b_hi);
}
}
}