1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0
 */

//! Classifiers for determining if a retry is necessary and related code.
//!
//! When a request fails, a retry strategy should inspect the result with retry
//! classifiers to understand if and how the request should be retried.
//!
//! Because multiple classifiers are often used, and because some are more
//! specific than others in what they identify as retryable, classifiers are
//! run in a sequence that is determined by their priority.
//!
//! Classifiers that are higher priority are run **after** classifiers
//! with a lower priority. The intention is that:
//!
//! 1. Generic classifiers that look at things like the HTTP error code run
//!     first.
//! 2. More specific classifiers such as ones that check for certain error
//!     messages are run **after** the generic classifiers. This gives them the
//!     ability to override the actions set by the generic retry classifiers.
//!
//! Put another way:
//!
//! | large nets target common failures with basic behavior | run before            | small nets target specific failures with special behavior|
//! |-------------------------------------------------------|-----------------------|----------------------------------------------------------|
//! | low priority classifiers                              | results overridden by | high priority classifiers                                |

use crate::box_error::BoxError;
use crate::client::interceptors::context::InterceptorContext;
use crate::client::runtime_components::sealed::ValidateConfig;
use crate::client::runtime_components::RuntimeComponents;
use crate::impl_shared_conversions;
use aws_smithy_types::config_bag::ConfigBag;
use aws_smithy_types::retry::ErrorKind;
use std::fmt;
use std::sync::Arc;
use std::time::Duration;

/// The result of running a [`ClassifyRetry`] on a [`InterceptorContext`].
#[non_exhaustive]
#[derive(Clone, Eq, PartialEq, Debug, Default)]
pub enum RetryAction {
    /// When a classifier can't run or has no opinion, this action is returned.
    ///
    /// For example, if a classifier requires a parsed response and response parsing failed,
    /// this action is returned. If all classifiers return this action, no retry should be
    /// attempted.
    #[default]
    NoActionIndicated,
    /// When a classifier runs and thinks a response should be retried, this action is returned.
    RetryIndicated(RetryReason),
    /// When a classifier runs and decides a response must not be retried, this action is returned.
    ///
    /// This action stops retry classification immediately, skipping any following classifiers.
    RetryForbidden,
}

impl fmt::Display for RetryAction {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::NoActionIndicated => write!(f, "no action indicated"),
            Self::RetryForbidden => write!(f, "retry forbidden"),
            Self::RetryIndicated(reason) => write!(f, "retry {reason}"),
        }
    }
}

impl RetryAction {
    /// Create a new `RetryAction` indicating that a retry is necessary.
    pub fn retryable_error(kind: ErrorKind) -> Self {
        Self::RetryIndicated(RetryReason::RetryableError {
            kind,
            retry_after: None,
        })
    }

    /// Create a new `RetryAction` indicating that a retry is necessary after an explicit delay.
    pub fn retryable_error_with_explicit_delay(kind: ErrorKind, retry_after: Duration) -> Self {
        Self::RetryIndicated(RetryReason::RetryableError {
            kind,
            retry_after: Some(retry_after),
        })
    }

    /// Create a new `RetryAction` indicating that a retry is necessary because of a transient error.
    pub fn transient_error() -> Self {
        Self::retryable_error(ErrorKind::TransientError)
    }

    /// Create a new `RetryAction` indicating that a retry is necessary because of a throttling error.
    pub fn throttling_error() -> Self {
        Self::retryable_error(ErrorKind::ThrottlingError)
    }

    /// Create a new `RetryAction` indicating that a retry is necessary because of a server error.
    pub fn server_error() -> Self {
        Self::retryable_error(ErrorKind::ServerError)
    }

    /// Create a new `RetryAction` indicating that a retry is necessary because of a client error.
    pub fn client_error() -> Self {
        Self::retryable_error(ErrorKind::ClientError)
    }

    /// Check if a retry is indicated.
    pub fn should_retry(&self) -> bool {
        match self {
            Self::NoActionIndicated | Self::RetryForbidden => false,
            Self::RetryIndicated(_) => true,
        }
    }
}

/// The reason for a retry.
#[non_exhaustive]
#[derive(Clone, Eq, PartialEq, Debug)]
pub enum RetryReason {
    /// When an error is received that should be retried, this reason is returned.
    RetryableError {
        /// The kind of error.
        kind: ErrorKind,
        /// A server may tell us to retry only after a specific time has elapsed.
        retry_after: Option<Duration>,
    },
}

impl fmt::Display for RetryReason {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::RetryableError { kind, retry_after } => {
                let after = retry_after
                    .map(|d| format!(" after {d:?}"))
                    .unwrap_or_default();
                write!(f, "{kind} error{after}")
            }
        }
    }
}

/// The priority of a retry classifier. Classifiers with a higher priority will
/// run **after** classifiers with a lower priority and may override their
/// result. Classifiers with equal priorities make no guarantees about which
/// will run first.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RetryClassifierPriority {
    inner: Inner,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Inner {
    /// The default priority for the `HttpStatusCodeClassifier`.
    HttpStatusCodeClassifier,
    /// The default priority for the `ModeledAsRetryableClassifier`.
    ModeledAsRetryableClassifier,
    /// The default priority for the `TransientErrorClassifier`.
    TransientErrorClassifier,
    /// The priority of some other classifier.
    Other(i8),
}

impl PartialOrd for RetryClassifierPriority {
    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
        Some(self.as_i8().cmp(&other.as_i8()))
    }
}

impl Ord for RetryClassifierPriority {
    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
        self.as_i8().cmp(&other.as_i8())
    }
}

impl RetryClassifierPriority {
    /// Create a new `RetryClassifierPriority` with the default priority for the `HttpStatusCodeClassifier`.
    pub fn http_status_code_classifier() -> Self {
        Self {
            inner: Inner::HttpStatusCodeClassifier,
        }
    }

    /// Create a new `RetryClassifierPriority` with the default priority for the `ModeledAsRetryableClassifier`.
    pub fn modeled_as_retryable_classifier() -> Self {
        Self {
            inner: Inner::ModeledAsRetryableClassifier,
        }
    }

    /// Create a new `RetryClassifierPriority` with the default priority for the `TransientErrorClassifier`.
    pub fn transient_error_classifier() -> Self {
        Self {
            inner: Inner::TransientErrorClassifier,
        }
    }

    #[deprecated = "use the less-confusingly-named `RetryClassifierPriority::run_before` instead"]
    /// Create a new `RetryClassifierPriority` with lower priority than the given priority.
    pub fn with_lower_priority_than(other: Self) -> Self {
        Self::run_before(other)
    }

    /// Create a new `RetryClassifierPriority` that can be overridden by the given priority.
    ///
    /// Retry classifiers are run in order from lowest to highest priority. A classifier that
    /// runs later can override a decision from a classifier that runs earlier.
    pub fn run_before(other: Self) -> Self {
        Self {
            inner: Inner::Other(other.as_i8() - 1),
        }
    }

    #[deprecated = "use the less-confusingly-named `RetryClassifierPriority::run_after` instead"]
    /// Create a new `RetryClassifierPriority` with higher priority than the given priority.
    pub fn with_higher_priority_than(other: Self) -> Self {
        Self::run_after(other)
    }

    /// Create a new `RetryClassifierPriority` that can override the given priority.
    ///
    /// Retry classifiers are run in order from lowest to highest priority. A classifier that
    /// runs later can override a decision from a classifier that runs earlier.
    pub fn run_after(other: Self) -> Self {
        Self {
            inner: Inner::Other(other.as_i8() + 1),
        }
    }

    fn as_i8(&self) -> i8 {
        match self.inner {
            Inner::HttpStatusCodeClassifier => 0,
            Inner::ModeledAsRetryableClassifier => 10,
            Inner::TransientErrorClassifier => 20,
            Inner::Other(i) => i,
        }
    }
}

impl Default for RetryClassifierPriority {
    fn default() -> Self {
        Self {
            inner: Inner::Other(0),
        }
    }
}

/// Classifies what kind of retry is needed for a given [`InterceptorContext`].
pub trait ClassifyRetry: Send + Sync + fmt::Debug {
    /// Run this classifier on the [`InterceptorContext`] to determine if the previous request
    /// should be retried. Returns a [`RetryAction`].
    fn classify_retry(&self, ctx: &InterceptorContext) -> RetryAction;

    /// The name of this retry classifier.
    ///
    /// Used for debugging purposes.
    fn name(&self) -> &'static str;

    /// The priority of this retry classifier.
    ///
    /// Classifiers with a higher priority will override the
    /// results of classifiers with a lower priority. Classifiers with equal priorities make no
    /// guarantees about which will override the other.
    ///
    /// Retry classifiers are run in order of increasing priority. Any decision
    /// (return value other than `NoActionIndicated`) from a higher priority
    /// classifier will override the decision of a lower priority classifier with one exception:
    /// [`RetryAction::RetryForbidden`] is treated differently: If ANY classifier returns `RetryForbidden`,
    /// this request will not be retried.
    fn priority(&self) -> RetryClassifierPriority {
        RetryClassifierPriority::default()
    }
}

impl_shared_conversions!(convert SharedRetryClassifier from ClassifyRetry using SharedRetryClassifier::new);

#[derive(Debug, Clone)]
/// Retry classifier used by the retry strategy to classify responses as retryable or not.
pub struct SharedRetryClassifier(Arc<dyn ClassifyRetry>);

impl SharedRetryClassifier {
    /// Given a [`ClassifyRetry`] trait object, create a new `SharedRetryClassifier`.
    pub fn new(retry_classifier: impl ClassifyRetry + 'static) -> Self {
        Self(Arc::new(retry_classifier))
    }
}

impl ClassifyRetry for SharedRetryClassifier {
    fn classify_retry(&self, ctx: &InterceptorContext) -> RetryAction {
        self.0.classify_retry(ctx)
    }

    fn name(&self) -> &'static str {
        self.0.name()
    }

    fn priority(&self) -> RetryClassifierPriority {
        self.0.priority()
    }
}

impl ValidateConfig for SharedRetryClassifier {
    fn validate_final_config(
        &self,
        _runtime_components: &RuntimeComponents,
        _cfg: &ConfigBag,
    ) -> Result<(), BoxError> {
        #[cfg(debug_assertions)]
        {
            // Because this is validating that the implementation is correct rather
            // than validating user input, we only want to run this in debug builds.
            let retry_classifiers = _runtime_components.retry_classifiers_slice();
            let out_of_order: Vec<_> = retry_classifiers
                .windows(2)
                .filter(|&w| w[0].value().priority() > w[1].value().priority())
                .collect();

            if !out_of_order.is_empty() {
                return Err("retry classifiers are mis-ordered; this is a bug".into());
            }
        }
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::{ClassifyRetry, RetryAction, RetryClassifierPriority, SharedRetryClassifier};
    use crate::client::interceptors::context::InterceptorContext;

    #[test]
    fn test_preset_priorities() {
        let before_modeled_as_retryable = RetryClassifierPriority::run_before(
            RetryClassifierPriority::modeled_as_retryable_classifier(),
        );
        let mut list = vec![
            RetryClassifierPriority::modeled_as_retryable_classifier(),
            RetryClassifierPriority::http_status_code_classifier(),
            RetryClassifierPriority::transient_error_classifier(),
            before_modeled_as_retryable,
        ];
        list.sort();

        assert_eq!(
            vec![
                RetryClassifierPriority::http_status_code_classifier(),
                before_modeled_as_retryable,
                RetryClassifierPriority::modeled_as_retryable_classifier(),
                RetryClassifierPriority::transient_error_classifier(),
            ],
            list
        );
    }

    #[test]
    fn test_classifier_run_before() {
        // Ensure low-priority classifiers run *before* high-priority classifiers.
        let high_priority_classifier = RetryClassifierPriority::default();
        let mid_priority_classifier = RetryClassifierPriority::run_before(high_priority_classifier);
        let low_priority_classifier = RetryClassifierPriority::run_before(mid_priority_classifier);

        let mut list = vec![
            mid_priority_classifier,
            high_priority_classifier,
            low_priority_classifier,
        ];
        list.sort();

        assert_eq!(
            vec![
                low_priority_classifier,
                mid_priority_classifier,
                high_priority_classifier
            ],
            list
        );
    }

    #[test]
    fn test_classifier_run_after() {
        // Ensure high-priority classifiers run *after* low-priority classifiers.
        let low_priority_classifier = RetryClassifierPriority::default();
        let mid_priority_classifier = RetryClassifierPriority::run_after(low_priority_classifier);
        let high_priority_classifier = RetryClassifierPriority::run_after(mid_priority_classifier);

        let mut list = vec![
            mid_priority_classifier,
            low_priority_classifier,
            high_priority_classifier,
        ];
        list.sort();

        assert_eq!(
            vec![
                low_priority_classifier,
                mid_priority_classifier,
                high_priority_classifier
            ],
            list
        );
    }

    #[derive(Debug)]
    struct ClassifierStub {
        name: &'static str,
        priority: RetryClassifierPriority,
    }

    impl ClassifyRetry for ClassifierStub {
        fn classify_retry(&self, _ctx: &InterceptorContext) -> RetryAction {
            todo!()
        }

        fn name(&self) -> &'static str {
            self.name
        }

        fn priority(&self) -> RetryClassifierPriority {
            self.priority
        }
    }

    fn wrap(name: &'static str, priority: RetryClassifierPriority) -> SharedRetryClassifier {
        SharedRetryClassifier::new(ClassifierStub { name, priority })
    }

    #[test]
    fn test_shared_classifier_run_before() {
        // Ensure low-priority classifiers run *before* high-priority classifiers,
        // even after wrapping.
        let high_priority_classifier = RetryClassifierPriority::default();
        let mid_priority_classifier = RetryClassifierPriority::run_before(high_priority_classifier);
        let low_priority_classifier = RetryClassifierPriority::run_before(mid_priority_classifier);

        let mut list = vec![
            wrap("mid", mid_priority_classifier),
            wrap("high", high_priority_classifier),
            wrap("low", low_priority_classifier),
        ];
        list.sort_by_key(|rc| rc.priority());

        let actual: Vec<_> = list.iter().map(|it| it.name()).collect();
        assert_eq!(vec!["low", "mid", "high"], actual);
    }

    #[test]
    fn test_shared_classifier_run_after() {
        // Ensure high-priority classifiers run *after* low-priority classifiers,
        // even after wrapping.
        let low_priority_classifier = RetryClassifierPriority::default();
        let mid_priority_classifier = RetryClassifierPriority::run_after(low_priority_classifier);
        let high_priority_classifier = RetryClassifierPriority::run_after(mid_priority_classifier);

        let mut list = vec![
            wrap("mid", mid_priority_classifier),
            wrap("high", high_priority_classifier),
            wrap("low", low_priority_classifier),
        ];
        list.sort_by_key(|rc| rc.priority());

        let actual: Vec<_> = list.iter().map(|it| it.name()).collect();
        assert_eq!(vec!["low", "mid", "high"], actual);
    }

    #[test]
    fn test_shared_preset_priorities() {
        let before_modeled_as_retryable = RetryClassifierPriority::run_before(
            RetryClassifierPriority::modeled_as_retryable_classifier(),
        );
        let mut list = vec![
            wrap(
                "modeled as retryable",
                RetryClassifierPriority::modeled_as_retryable_classifier(),
            ),
            wrap(
                "http status code",
                RetryClassifierPriority::http_status_code_classifier(),
            ),
            wrap(
                "transient error",
                RetryClassifierPriority::transient_error_classifier(),
            ),
            wrap("before 'modeled as retryable'", before_modeled_as_retryable),
        ];
        list.sort_by_key(|rc| rc.priority());

        let actual: Vec<_> = list.iter().map(|it| it.name()).collect();
        assert_eq!(
            vec![
                "http status code",
                "before 'modeled as retryable'",
                "modeled as retryable",
                "transient error"
            ],
            actual
        );
    }
}