kube_runtime/finalizer.rs
1//! Finalizer helper for [`Controller`](crate::Controller) reconcilers
2use crate::controller::Action;
3use futures::{TryFuture, TryFutureExt};
4use json_patch::{AddOperation, PatchOperation, RemoveOperation, TestOperation, jsonptr::PointerBuf};
5use kube_client::{
6 Api, Resource, ResourceExt,
7 api::{Patch, PatchParams},
8};
9
10use serde::{Serialize, de::DeserializeOwned};
11use std::{error::Error as StdError, fmt::Debug, str::FromStr, sync::Arc};
12use thiserror::Error;
13
14/// Errors from the finalizer helper
15///
16/// Note that this error takes the user's error as a type parameter.
17/// This makes its usage within loosely typed errors (such as anyhow) difficult.
18///
19/// Using proper error enums such as ones generated by e.g. thiserror is the easiest.
20/// See <https://github.com/kube-rs/kube/discussions/982>
21#[derive(Debug, Error)]
22pub enum Error<ReconcileErr>
23where
24 ReconcileErr: StdError + 'static,
25{
26 /// User side `Event::Apply` branch failed
27 #[error("failed to apply object: {0}")]
28 ApplyFailed(#[source] ReconcileErr),
29
30 /// User side `Event::Cleanup` branch failed
31 #[error("failed to clean up object: {0}")]
32 CleanupFailed(#[source] ReconcileErr),
33
34 /// The patch call adding a finalizer failed
35 #[error("failed to add finalizer: {0}")]
36 AddFinalizer(#[source] kube_client::Error),
37
38 /// The patch call removing a finalizer failed
39 #[error("failed to remove finalizer: {0}")]
40 RemoveFinalizer(#[source] kube_client::Error),
41
42 /// The object requested to be patched has no name
43 ///
44 /// This should not happen unless you are hand crafting the object without a name
45 #[error("object has no name")]
46 UnnamedObject,
47
48 /// The finalizer path was invalid
49 ///
50 /// This should not happen, the path is constructed internally and should be
51 /// valid as far as `json_patch` is concerned.
52 #[error("invalid finalizer")]
53 InvalidFinalizer,
54}
55
56struct FinalizerState {
57 finalizer_index: Option<usize>,
58 is_deleting: bool,
59}
60
61impl FinalizerState {
62 fn for_object<K: Resource>(obj: &K, finalizer_name: &str) -> Self {
63 Self {
64 finalizer_index: obj
65 .finalizers()
66 .iter()
67 .enumerate()
68 .find(|(_, fin)| *fin == finalizer_name)
69 .map(|(i, _)| i),
70 is_deleting: obj.meta().deletion_timestamp.is_some(),
71 }
72 }
73}
74
75/// Reconcile an object in a way that requires cleanup before an object can be deleted.
76///
77/// It does this by managing a [`ObjectMeta::finalizers`] entry,
78/// which prevents the object from being deleted before the cleanup is done.
79///
80/// In typical usage, if you use `finalizer` then it should be the only top-level "action"
81/// in your [`applier`](crate::applier)/[`Controller`](crate::Controller)'s `reconcile` function.
82///
83/// # Expected Flow
84///
85/// 1. User creates object
86/// 2. Reconciler sees object
87/// 3. `finalizer` adds `finalizer_name` to [`ObjectMeta::finalizers`]
88/// 4. Reconciler sees updated object
89/// 5. `finalizer` runs [`Event::Apply`]
90/// 6. User updates object
91/// 7. Reconciler sees updated object
92/// 8. `finalizer` runs [`Event::Apply`]
93/// 9. User deletes object
94/// 10. Reconciler sees deleting object
95/// 11. `finalizer` runs [`Event::Cleanup`]
96/// 12. `finalizer` removes `finalizer_name` from [`ObjectMeta::finalizers`]
97/// 13. Kubernetes sees that all [`ObjectMeta::finalizers`] are gone and finally deletes the object
98///
99/// # Guarantees
100///
101/// If [`Event::Apply`] is ever started then [`Event::Cleanup`] must succeed before the Kubernetes object deletion completes.
102///
103/// # Assumptions
104///
105/// `finalizer_name` must be unique among the controllers interacting with the object
106///
107/// [`Event::Apply`] and [`Event::Cleanup`] must both be idempotent, and tolerate being executed several times (even if previously cancelled).
108///
109/// [`Event::Cleanup`] must tolerate [`Event::Apply`] never having ran at all, or never having succeeded. Keep in mind that
110/// even infallible `.await`s are cancellation points.
111///
112/// # Caveats
113///
114/// Object deletes will get stuck while the controller is not running, or if `cleanup` fails for some reason.
115///
116/// If the controller's input stream uses [predicate filtering](crate::WatchStreamExt::predicate_filter),
117/// it must allow updates to `metadata.deletionTimestamp` through so cleanup can begin.
118/// A generation-only predicate can suppress the update that adds the finalizer in the expected flow above,
119/// since adding a finalizer leaves generation unchanged. The initial deletion marking does bump generation.
120/// See [predicate deletion handling](crate::predicates) for guidance.
121///
122/// `reconcile` should take the object that the [`Event`] contains, rather than trying to reuse `obj`, since it may have been updated.
123///
124/// # Errors
125///
126/// [`Event::Apply`] and [`Event::Cleanup`] are both fallible, their errors are passed through as [`Error::ApplyFailed`]
127/// and [`Error::CleanupFailed`], respectively.
128///
129/// In addition, adding and removing the finalizer itself may fail. In particular, this may be because of
130/// network errors, lacking permissions, or because another `finalizer` was updated in the meantime on the same object.
131///
132/// [`ObjectMeta::finalizers`]: kube_client::api::ObjectMeta#structfield.finalizers
133pub async fn finalizer<K, ReconcileFut>(
134 api: &Api<K>,
135 finalizer_name: &str,
136 obj: Arc<K>,
137 reconcile: impl FnOnce(Event<K>) -> ReconcileFut,
138) -> Result<Action, Error<ReconcileFut::Error>>
139where
140 K: Resource + Clone + DeserializeOwned + Serialize + Debug,
141 ReconcileFut: TryFuture<Ok = Action>,
142 ReconcileFut::Error: StdError + 'static,
143{
144 match FinalizerState::for_object(&*obj, finalizer_name) {
145 FinalizerState {
146 finalizer_index: Some(_),
147 is_deleting: false,
148 } => TryFutureExt::into_future(reconcile(Event::Apply(obj)))
149 .await
150 .map_err(Error::ApplyFailed),
151 FinalizerState {
152 finalizer_index: Some(finalizer_i),
153 is_deleting: true,
154 } => {
155 // Cleanup reconciliation must succeed before it's safe to remove the finalizer
156 let name = obj.meta().name.clone().ok_or(Error::UnnamedObject)?;
157 let action = TryFutureExt::into_future(reconcile(Event::Cleanup(obj)))
158 .await
159 // Short-circuit, so that we keep the finalizer if cleanup fails
160 .map_err(Error::CleanupFailed)?;
161 // Cleanup was successful, remove the finalizer so that deletion can continue
162 let finalizer_path = format!("/metadata/finalizers/{finalizer_i}");
163 api.patch::<K>(
164 &name,
165 &PatchParams::default(),
166 &Patch::Json(json_patch::Patch(vec![
167 // All finalizers run concurrently and we use an integer index
168 // `Test` ensures that we fail instead of deleting someone else's finalizer
169 // (in which case a new `Cleanup` event will be sent)
170 PatchOperation::Test(TestOperation {
171 path: PointerBuf::from_str(finalizer_path.as_str())
172 .map_err(|_err| Error::InvalidFinalizer)?,
173 value: finalizer_name.into(),
174 }),
175 PatchOperation::Remove(RemoveOperation {
176 path: PointerBuf::from_str(finalizer_path.as_str())
177 .map_err(|_err| Error::InvalidFinalizer)?,
178 }),
179 ])),
180 )
181 .await
182 .map_err(Error::RemoveFinalizer)?;
183 Ok(action)
184 }
185 FinalizerState {
186 finalizer_index: None,
187 is_deleting: false,
188 } => {
189 // Finalizer must be added before it's safe to run an `Apply` reconciliation
190 let patch = json_patch::Patch(if obj.finalizers().is_empty() {
191 vec![
192 PatchOperation::Test(TestOperation {
193 path: PointerBuf::from_str("/metadata/finalizers")
194 .map_err(|_err| Error::InvalidFinalizer)?,
195 value: serde_json::Value::Null,
196 }),
197 PatchOperation::Add(AddOperation {
198 path: PointerBuf::from_str("/metadata/finalizers")
199 .map_err(|_err| Error::InvalidFinalizer)?,
200 value: vec![finalizer_name].into(),
201 }),
202 ]
203 } else {
204 vec![
205 // Kubernetes doesn't automatically deduplicate finalizers (see
206 // https://github.com/kube-rs/kube/issues/964#issuecomment-1197311254),
207 // so we need to fail and retry if anyone else has added the finalizer in the meantime
208 PatchOperation::Test(TestOperation {
209 path: PointerBuf::from_str("/metadata/finalizers")
210 .map_err(|_err| Error::InvalidFinalizer)?,
211 value: obj.finalizers().into(),
212 }),
213 PatchOperation::Add(AddOperation {
214 path: PointerBuf::from_str("/metadata/finalizers/-")
215 .map_err(|_err| Error::InvalidFinalizer)?,
216 value: finalizer_name.into(),
217 }),
218 ]
219 });
220 api.patch::<K>(
221 obj.meta().name.as_deref().ok_or(Error::UnnamedObject)?,
222 &PatchParams::default(),
223 &Patch::Json(patch),
224 )
225 .await
226 .map_err(Error::AddFinalizer)?;
227 // No point applying here, since the patch will cause a new reconciliation
228 Ok(Action::await_change())
229 }
230 FinalizerState {
231 finalizer_index: None,
232 is_deleting: true,
233 } => {
234 // Our work here is done
235 Ok(Action::await_change())
236 }
237 }
238}
239
240/// A representation of an action that should be taken by a reconciler.
241pub enum Event<K> {
242 /// The reconciler should ensure that the actual state matches the state desired in the object.
243 ///
244 /// This must be idempotent, since it may be recalled if, for example (this list is non-exhaustive):
245 ///
246 /// - The controller is restarted
247 /// - The object is updated
248 /// - The reconciliation fails
249 /// - The grinch attacks
250 Apply(Arc<K>),
251 /// The object is being deleted, and the reconciler should remove all resources that it owns.
252 ///
253 /// This must be idempotent, since it may be recalled if, for example (this list is non-exhaustive):
254 ///
255 /// - The controller is restarted while the deletion is in progress
256 /// - The reconciliation fails
257 /// - Another finalizer was removed in the meantime
258 /// - The grinch's heart grows a size or two
259 Cleanup(Arc<K>),
260}