Skip to main content

kaniop_radius/reconcile/
apply.rs

1//! Ownership-checked Kubernetes writes for the RADIUS controller.
2//!
3//! Every generated object is checked with [`ensure_not_foreign`] before the
4//! controller touches it, so a same-named Deployment or Service created by
5//! someone else is reported as a conflict and never adopted, even when its
6//! labels match. Generated objects are written with server-side apply under
7//! [`FIELD_MANAGER`] with `force`, so a conflict with another field manager
8//! (for example `kubectl scale`) cannot wedge the reconcile with a 409. Config
9//! Secrets are create-only and immutable; deletes carry UID and
10//! resourceVersion preconditions so a recreated object is never removed.
11//!
12//! Server-side apply cannot be made conditional on non-existence, so a
13//! foreign object created between the read and the patch would be merged
14//! into. The patch response is therefore checked again and a foreign result
15//! is reported as a conflict; the window is a single API round trip.
16
17use std::fmt::Debug;
18
19use k8s_openapi::api::core::v1::Secret;
20use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ObjectMeta, OwnerReference};
21use kube::api::{DeleteParams, Patch, PatchParams, PostParams, Preconditions, PropagationPolicy};
22use kube::{Api, Resource, ResourceExt};
23use serde::Serialize;
24use serde::de::DeserializeOwned;
25
26use crate::reconcile::error::{ExecError, ExecResult, kube_error};
27use crate::reconcile::meta::{ANNOTATION_ROTATION_REQUEST, ANNOTATION_TOKEN_ID};
28
29/// Server-side apply field manager.
30pub const FIELD_MANAGER: &str = "kanidmradiusservers.kaniop.rs";
31
32/// Is `meta` controlled by `owner`? The controller reference (at most one per
33/// object) must name the owner's UID and kind. Extra non-controller
34/// references are allowed; a missing, different or non-controller reference
35/// is not ours.
36pub fn is_controlled_by(meta: &ObjectMeta, owner: &OwnerReference) -> bool {
37    meta.owner_references
38        .as_deref()
39        .unwrap_or(&[])
40        .iter()
41        .any(|r| r.controller == Some(true) && r.uid == owner.uid && r.kind == owner.kind)
42}
43
44fn foreign<K: Resource<DynamicType = ()>>(object: &K) -> ExecError {
45    ExecError::ForeignObject {
46        kind: K::kind(&()).into_owned(),
47        namespace: object.namespace().unwrap_or_default(),
48        name: object.name_any(),
49    }
50}
51
52/// Reject an existing object that is not controlled by `owner`.
53pub fn ensure_not_foreign<K: Resource<DynamicType = ()>>(
54    object: &K,
55    owner: &OwnerReference,
56) -> ExecResult<()> {
57    if is_controlled_by(object.meta(), owner) {
58        Ok(())
59    } else {
60        Err(foreign(object))
61    }
62}
63
64/// Parameters for every server-side apply: our field manager, with `force`.
65pub fn apply_params() -> PatchParams {
66    PatchParams::apply(FIELD_MANAGER).force()
67}
68
69fn identity<K: Resource<DynamicType = ()>>(object: &K) -> ExecResult<(String, String)> {
70    let name = object
71        .meta()
72        .name
73        .clone()
74        .ok_or(ExecError::MissingMetadata {
75            kind: "object",
76            field: "name",
77        })?;
78    Ok((object.namespace().unwrap_or_default(), name))
79}
80
81async fn get_optional<K>(api: &Api<K>, namespace: &str, name: &str) -> ExecResult<Option<K>>
82where
83    K: Resource<DynamicType = ()> + Clone + DeserializeOwned + Debug,
84{
85    match api.get(name).await {
86        Ok(object) => Ok(Some(object)),
87        Err(kube::Error::Api(status)) if status.code == 404 => Ok(None),
88        Err(error) => Err(kube_error("get", &K::kind(&()), namespace, name, error)),
89    }
90}
91
92/// Read the object if present and fail on a foreign one, without writing.
93/// Use it to refuse early (for example before minting a token).
94pub async fn check_not_foreign<K>(
95    api: &Api<K>,
96    namespace: &str,
97    name: &str,
98    owner: &OwnerReference,
99) -> ExecResult<Option<K>>
100where
101    K: Resource<DynamicType = ()> + Clone + DeserializeOwned + Debug,
102{
103    let existing = get_optional(api, namespace, name).await?;
104    if let Some(existing) = &existing {
105        ensure_not_foreign(existing, owner)?;
106    }
107    Ok(existing)
108}
109
110/// Server-side apply `desired` unless a foreign object holds the name.
111pub async fn apply_owned<K>(api: &Api<K>, desired: &K, owner: &OwnerReference) -> ExecResult<K>
112where
113    K: Resource<DynamicType = ()> + Clone + DeserializeOwned + Serialize + Debug,
114{
115    let (namespace, name) = identity(desired)?;
116    check_not_foreign(api, &namespace, &name, owner).await?;
117    let applied = api
118        .patch(&name, &apply_params(), &Patch::Apply(desired))
119        .await
120        .map_err(|e| kube_error("apply", &K::kind(&()), &namespace, &name, e))?;
121    ensure_not_foreign(&applied, owner)?;
122    Ok(applied)
123}
124
125/// What [`ensure_config_secret`] did.
126#[derive(Clone, Copy, Debug, PartialEq, Eq)]
127pub enum SecretOutcome {
128    Created,
129    /// An identical Secret of ours was already there.
130    Existing,
131}
132
133/// The existing Secret must be ours and byte-for-byte what we would create:
134/// same type, immutability, data, token id and rotation request. The name is
135/// content-addressed, so a difference means a conflicting writer, not a
136/// stale read.
137pub fn verify_existing_secret(
138    existing: &Secret,
139    desired: &Secret,
140    owner: &OwnerReference,
141) -> ExecResult<()> {
142    ensure_not_foreign(existing, owner)?;
143    let annotation = |secret: &Secret, key: &str| {
144        secret
145            .metadata
146            .annotations
147            .as_ref()
148            .and_then(|a| a.get(key))
149            .cloned()
150    };
151    let identical = existing.immutable == desired.immutable
152        && existing.type_ == desired.type_
153        && existing.data == desired.data
154        && annotation(existing, ANNOTATION_TOKEN_ID) == annotation(desired, ANNOTATION_TOKEN_ID)
155        && annotation(existing, ANNOTATION_ROTATION_REQUEST)
156            == annotation(desired, ANNOTATION_ROTATION_REQUEST);
157    if identical {
158        Ok(())
159    } else {
160        Err(ExecError::SecretConflict {
161            namespace: existing.namespace().unwrap_or_default(),
162            name: existing.name_any(),
163        })
164    }
165}
166
167/// Create the immutable config Secret if absent. Never patches or
168/// overwrites: an existing Secret (found by GET, or by a 409 from a racing
169/// reconcile) must be ours and identical, otherwise it is a conflict.
170pub async fn ensure_config_secret(
171    api: &Api<Secret>,
172    desired: &Secret,
173    owner: &OwnerReference,
174) -> ExecResult<SecretOutcome> {
175    let (namespace, name) = identity(desired)?;
176    if let Some(existing) = get_optional(api, &namespace, &name).await? {
177        verify_existing_secret(&existing, desired, owner)?;
178        return Ok(SecretOutcome::Existing);
179    }
180    let params = PostParams {
181        field_manager: Some(FIELD_MANAGER.to_string()),
182        ..Default::default()
183    };
184    match api.create(&params, desired).await {
185        Ok(_) => Ok(SecretOutcome::Created),
186        Err(kube::Error::Api(status)) if status.code == 409 => {
187            let existing = get_optional(api, &namespace, &name).await?.ok_or_else(|| {
188                // Created and deleted again inside one reconcile.
189                ExecError::ConcurrentMutation {
190                    kind: "Secret".into(),
191                    namespace: namespace.clone(),
192                    name: name.clone(),
193                }
194            })?;
195            verify_existing_secret(&existing, desired, owner)?;
196            Ok(SecretOutcome::Existing)
197        }
198        Err(error) => Err(kube_error("create", "Secret", &namespace, &name, error)),
199    }
200}
201
202/// How an object is deleted.
203#[derive(Clone, Copy, Debug, PartialEq, Eq)]
204pub enum DeleteMode {
205    /// Plain delete (background propagation by default).
206    Default,
207    /// The object stays until its dependents (ReplicaSets, pods) are gone.
208    Foreground,
209}
210
211/// Bind a delete to the exact object that was checked: a name can be deleted
212/// and recreated between GET and DELETE, and without both preconditions the
213/// replacement would be removed.
214pub fn delete_params(meta: &ObjectMeta, mode: DeleteMode) -> ExecResult<DeleteParams> {
215    let nonempty = |value: &Option<String>, field: &'static str| {
216        value
217            .as_ref()
218            .filter(|v| !v.is_empty())
219            .cloned()
220            .ok_or(ExecError::MissingMetadata {
221                kind: "object",
222                field,
223            })
224    };
225    let preconditions = Preconditions {
226        uid: Some(nonempty(&meta.uid, "uid")?),
227        resource_version: Some(nonempty(&meta.resource_version, "resourceVersion")?),
228    };
229    let params = match mode {
230        DeleteMode::Default => DeleteParams::default(),
231        DeleteMode::Foreground => DeleteParams {
232            propagation_policy: Some(PropagationPolicy::Foreground),
233            ..Default::default()
234        },
235    };
236    Ok(params.preconditions(preconditions))
237}
238
239/// What [`delete_owned`] did.
240#[derive(Clone, Copy, Debug, PartialEq, Eq)]
241pub enum DeleteOutcome {
242    Deleted,
243    AlreadyAbsent,
244}
245
246/// Delete an object we control. Absent is success; a foreign object is an
247/// error and is never touched; failed preconditions are
248/// [`ExecError::ConcurrentMutation`] (re-observe and retry).
249pub async fn delete_owned<K>(
250    api: &Api<K>,
251    namespace: &str,
252    name: &str,
253    owner: &OwnerReference,
254    mode: DeleteMode,
255) -> ExecResult<DeleteOutcome>
256where
257    K: Resource<DynamicType = ()> + Clone + DeserializeOwned + Debug,
258{
259    let Some(existing) = check_not_foreign(api, namespace, name, owner).await? else {
260        return Ok(DeleteOutcome::AlreadyAbsent);
261    };
262    let params = delete_params(existing.meta(), mode)?;
263    match api.delete(name, &params).await {
264        Ok(_) => Ok(DeleteOutcome::Deleted),
265        Err(kube::Error::Api(status)) if status.code == 404 => Ok(DeleteOutcome::AlreadyAbsent),
266        Err(kube::Error::Api(status)) if status.code == 409 => Err(ExecError::ConcurrentMutation {
267            kind: K::kind(&()).into_owned(),
268            namespace: namespace.to_string(),
269            name: name.to_string(),
270        }),
271        Err(error) => Err(kube_error("delete", &K::kind(&()), namespace, name, error)),
272    }
273}
274
275#[cfg(test)]
276mod tests {
277    use std::collections::BTreeMap;
278
279    use k8s_openapi::ByteString;
280    use k8s_openapi::api::apps::v1::Deployment;
281    use serde_json::json;
282
283    use super::*;
284    use crate::reconcile::kube_mock::{api_status, mock_client, to_json};
285
286    const UID: &str = "d10066c3-4599-4487-972d-26bb42587a32";
287    const NS: &str = "network";
288
289    fn owner() -> OwnerReference {
290        OwnerReference {
291            api_version: "kaniop.rs/v1beta1".into(),
292            kind: "KanidmRadiusServer".into(),
293            name: "campus".into(),
294            uid: UID.into(),
295            controller: Some(true),
296            block_owner_deletion: Some(true),
297        }
298    }
299
300    fn foreign_owner() -> OwnerReference {
301        OwnerReference {
302            api_version: "apps/v1".into(),
303            kind: "StatefulSet".into(),
304            name: "someone-else".into(),
305            uid: "other-uid".into(),
306            controller: Some(true),
307            block_owner_deletion: None,
308        }
309    }
310
311    fn deployment(owners: Option<Vec<OwnerReference>>) -> Deployment {
312        Deployment {
313            metadata: ObjectMeta {
314                name: Some("campus".into()),
315                namespace: Some(NS.into()),
316                uid: Some("dep-uid".into()),
317                resource_version: Some("42".into()),
318                owner_references: owners,
319                ..Default::default()
320            },
321            ..Default::default()
322        }
323    }
324
325    fn secret(data: &[u8], owners: Vec<OwnerReference>) -> Secret {
326        Secret {
327            metadata: ObjectMeta {
328                name: Some("campus-0123456789".into()),
329                namespace: Some(NS.into()),
330                annotations: Some(BTreeMap::from([(
331                    ANNOTATION_TOKEN_ID.to_string(),
332                    "9ad74e68-147f-49e8-96db-dc21c920ff3b".to_string(),
333                )])),
334                owner_references: Some(owners),
335                ..Default::default()
336            },
337            data: Some(BTreeMap::from([(
338                "auth_token".to_string(),
339                ByteString(data.to_vec()),
340            )])),
341            immutable: Some(true),
342            type_: Some("Opaque".into()),
343            ..Default::default()
344        }
345    }
346
347    #[test]
348    fn controller_reference_decides_ownership() {
349        let ours = deployment(Some(vec![owner()]));
350        assert!(ensure_not_foreign(&ours, &owner()).is_ok());
351
352        let ownerless = deployment(None);
353        assert!(matches!(
354            ensure_not_foreign(&ownerless, &owner()),
355            Err(ExecError::ForeignObject { .. })
356        ));
357        let other = deployment(Some(vec![foreign_owner()]));
358        assert!(matches!(
359            ensure_not_foreign(&other, &owner()),
360            Err(ExecError::ForeignObject { .. })
361        ));
362        // Same UID but not the controller reference is not ours.
363        let mut weak = owner();
364        weak.controller = Some(false);
365        assert!(ensure_not_foreign(&deployment(Some(vec![weak])), &owner()).is_err());
366        // Same name and kind, different UID (a recreated CR) is not ours.
367        let mut recreated = owner();
368        recreated.uid = "new-uid".into();
369        assert!(ensure_not_foreign(&deployment(Some(vec![recreated])), &owner()).is_err());
370        // An extra non-controller reference does not disown it.
371        let mut extra = foreign_owner();
372        extra.controller = None;
373        assert!(ensure_not_foreign(&deployment(Some(vec![extra, owner()])), &owner()).is_ok());
374    }
375
376    #[test]
377    fn foreign_error_names_object_without_content() {
378        let error = ensure_not_foreign(&deployment(None), &owner()).unwrap_err();
379        assert_eq!(
380            error.to_string(),
381            "Deployment network/campus exists but is not controlled by this resource"
382        );
383    }
384
385    #[test]
386    fn apply_params_force_and_manager() {
387        let params = apply_params();
388        assert!(params.force, "SSA without force wedges on field conflicts");
389        assert_eq!(params.field_manager.as_deref(), Some(FIELD_MANAGER));
390        assert_eq!(FIELD_MANAGER, "kanidmradiusservers.kaniop.rs");
391    }
392
393    #[test]
394    fn delete_params_need_uid_and_resource_version() {
395        let mut meta = ObjectMeta::default();
396        assert!(delete_params(&meta, DeleteMode::Default).is_err());
397        meta.uid = Some("uid-1".into());
398        assert!(delete_params(&meta, DeleteMode::Default).is_err());
399        meta.resource_version = Some("".into());
400        assert!(delete_params(&meta, DeleteMode::Default).is_err());
401        meta.resource_version = Some("77".into());
402        let params = delete_params(&meta, DeleteMode::Foreground).unwrap();
403        assert!(matches!(
404            params.propagation_policy,
405            Some(PropagationPolicy::Foreground)
406        ));
407        let pre = params.preconditions.unwrap();
408        assert_eq!(pre.uid.as_deref(), Some("uid-1"));
409        assert_eq!(pre.resource_version.as_deref(), Some("77"));
410        assert!(
411            delete_params(&meta, DeleteMode::Default)
412                .unwrap()
413                .propagation_policy
414                .is_none()
415        );
416    }
417
418    #[tokio::test]
419    async fn apply_creates_absent_object_with_forced_server_side_apply() {
420        let desired = deployment(Some(vec![owner()]));
421        let (client, recorder) =
422            mock_client(vec![api_status(404, "NotFound"), (200, to_json(&desired))]);
423        let api: Api<Deployment> = Api::namespaced(client, NS);
424        apply_owned(&api, &desired, &owner()).await.unwrap();
425        drop(api);
426        let requests = recorder.await.unwrap();
427        assert_eq!(requests.len(), 2);
428        assert_eq!(requests[0].method, "GET");
429        let patch = &requests[1];
430        assert_eq!(patch.method, "PATCH");
431        assert!(patch.uri.contains("force=true"), "uri: {}", patch.uri);
432        assert!(
433            patch.uri.contains(&format!("fieldManager={FIELD_MANAGER}")),
434            "uri: {}",
435            patch.uri
436        );
437        assert_eq!(
438            patch.content_type.as_deref(),
439            Some("application/apply-patch+yaml")
440        );
441        assert_eq!(patch.json()["metadata"]["name"], "campus");
442    }
443
444    #[tokio::test]
445    async fn apply_refuses_foreign_object_without_writing() {
446        let desired = deployment(Some(vec![owner()]));
447        let theirs = deployment(Some(vec![foreign_owner()]));
448        let (client, recorder) = mock_client(vec![(200, to_json(&theirs))]);
449        let api: Api<Deployment> = Api::namespaced(client, NS);
450        let error = apply_owned(&api, &desired, &owner()).await.unwrap_err();
451        assert!(matches!(error, ExecError::ForeignObject { .. }));
452        drop(api);
453        let requests = recorder.await.unwrap();
454        assert_eq!(requests.len(), 1, "only the read, never a patch");
455        assert_eq!(requests[0].method, "GET");
456    }
457
458    #[tokio::test]
459    async fn apply_reports_foreign_object_that_appeared_after_the_read() {
460        let desired = deployment(Some(vec![owner()]));
461        let theirs = deployment(Some(vec![foreign_owner()]));
462        let (client, recorder) =
463            mock_client(vec![api_status(404, "NotFound"), (200, to_json(&theirs))]);
464        let api: Api<Deployment> = Api::namespaced(client, NS);
465        let error = apply_owned(&api, &desired, &owner()).await.unwrap_err();
466        assert!(matches!(error, ExecError::ForeignObject { .. }));
467        drop(api);
468        recorder.await.unwrap();
469    }
470
471    #[tokio::test]
472    async fn api_failure_keeps_its_source() {
473        let desired = deployment(Some(vec![owner()]));
474        let (client, recorder) = mock_client(vec![api_status(503, "ServiceUnavailable")]);
475        let api: Api<Deployment> = Api::namespaced(client, NS);
476        let error = apply_owned(&api, &desired, &owner()).await.unwrap_err();
477        assert!(matches!(error, ExecError::Api(_)));
478        drop(api);
479        recorder.await.unwrap();
480    }
481
482    #[tokio::test]
483    async fn secret_is_created_when_absent() {
484        let desired = secret(b"bearer-one", vec![owner()]);
485        let (client, recorder) =
486            mock_client(vec![api_status(404, "NotFound"), (201, to_json(&desired))]);
487        let api: Api<Secret> = Api::namespaced(client, NS);
488        assert_eq!(
489            ensure_config_secret(&api, &desired, &owner())
490                .await
491                .unwrap(),
492            SecretOutcome::Created
493        );
494        drop(api);
495        let requests = recorder.await.unwrap();
496        assert_eq!(requests[1].method, "POST");
497        assert!(
498            requests[1]
499                .uri
500                .contains(&format!("fieldManager={FIELD_MANAGER}"))
501        );
502    }
503
504    #[tokio::test]
505    async fn secret_race_409_accepts_identical_content() {
506        let desired = secret(b"bearer-one", vec![owner()]);
507        let (client, recorder) = mock_client(vec![
508            api_status(404, "NotFound"),
509            api_status(409, "AlreadyExists"),
510            (200, to_json(&desired)),
511        ]);
512        let api: Api<Secret> = Api::namespaced(client, NS);
513        assert_eq!(
514            ensure_config_secret(&api, &desired, &owner())
515                .await
516                .unwrap(),
517            SecretOutcome::Existing
518        );
519        drop(api);
520        let methods: Vec<_> = recorder
521            .await
522            .unwrap()
523            .into_iter()
524            .map(|r| r.method)
525            .collect();
526        assert_eq!(methods, ["GET", "POST", "GET"], "never patched or replaced");
527    }
528
529    #[tokio::test]
530    async fn secret_race_409_rejects_different_content_and_foreign_owner() {
531        let desired = secret(b"bearer-one", vec![owner()]);
532        let different = secret(b"another-bearer", vec![owner()]);
533        let (client, recorder) = mock_client(vec![
534            api_status(404, "NotFound"),
535            api_status(409, "AlreadyExists"),
536            (200, to_json(&different)),
537        ]);
538        let api: Api<Secret> = Api::namespaced(client, NS);
539        let error = ensure_config_secret(&api, &desired, &owner())
540            .await
541            .unwrap_err();
542        assert!(matches!(error, ExecError::SecretConflict { .. }));
543        assert!(!error.to_string().contains("another-bearer"));
544        drop(api);
545        recorder.await.unwrap();
546
547        let theirs = secret(b"bearer-one", vec![foreign_owner()]);
548        let (client, recorder) = mock_client(vec![(200, to_json(&theirs))]);
549        let api: Api<Secret> = Api::namespaced(client, NS);
550        assert!(matches!(
551            ensure_config_secret(&api, &desired, &owner()).await,
552            Err(ExecError::ForeignObject { .. })
553        ));
554        drop(api);
555        recorder.await.unwrap();
556    }
557
558    #[test]
559    fn existing_secret_with_other_token_id_is_a_conflict() {
560        let desired = secret(b"bearer-one", vec![owner()]);
561        let mut other = desired.clone();
562        other
563            .metadata
564            .annotations
565            .as_mut()
566            .unwrap()
567            .insert(ANNOTATION_TOKEN_ID.into(), "someone-else".into());
568        assert!(matches!(
569            verify_existing_secret(&other, &desired, &owner()),
570            Err(ExecError::SecretConflict { .. })
571        ));
572        let mut mutable = desired.clone();
573        mutable.immutable = None;
574        assert!(verify_existing_secret(&mutable, &desired, &owner()).is_err());
575    }
576
577    #[tokio::test]
578    async fn delete_binds_uid_and_resource_version() {
579        let existing = deployment(Some(vec![owner()]));
580        let (client, recorder) = mock_client(vec![
581            (200, to_json(&existing)),
582            (
583                200,
584                json!({"kind": "Status", "apiVersion": "v1", "status": "Success"}),
585            ),
586        ]);
587        let api: Api<Deployment> = Api::namespaced(client, NS);
588        let outcome = delete_owned(&api, NS, "campus", &owner(), DeleteMode::Foreground)
589            .await
590            .unwrap();
591        assert_eq!(outcome, DeleteOutcome::Deleted);
592        drop(api);
593        let requests = recorder.await.unwrap();
594        assert_eq!(requests[1].method, "DELETE");
595        let body = requests[1].json();
596        assert_eq!(body["preconditions"]["uid"], "dep-uid");
597        assert_eq!(body["preconditions"]["resourceVersion"], "42");
598        assert_eq!(body["propagationPolicy"], "Foreground");
599    }
600
601    #[tokio::test]
602    async fn delete_absent_is_success_and_foreign_is_untouched() {
603        let (client, recorder) = mock_client(vec![api_status(404, "NotFound")]);
604        let api: Api<Deployment> = Api::namespaced(client, NS);
605        assert_eq!(
606            delete_owned(&api, NS, "campus", &owner(), DeleteMode::Default)
607                .await
608                .unwrap(),
609            DeleteOutcome::AlreadyAbsent
610        );
611        drop(api);
612        recorder.await.unwrap();
613
614        let theirs = deployment(Some(vec![foreign_owner()]));
615        let (client, recorder) = mock_client(vec![(200, to_json(&theirs))]);
616        let api: Api<Deployment> = Api::namespaced(client, NS);
617        assert!(matches!(
618            delete_owned(&api, NS, "campus", &owner(), DeleteMode::Default).await,
619            Err(ExecError::ForeignObject { .. })
620        ));
621        drop(api);
622        assert_eq!(recorder.await.unwrap().len(), 1, "no DELETE issued");
623    }
624
625    #[tokio::test]
626    async fn delete_precondition_failure_is_concurrent_mutation() {
627        let existing = deployment(Some(vec![owner()]));
628        let (client, recorder) =
629            mock_client(vec![(200, to_json(&existing)), api_status(409, "Conflict")]);
630        let api: Api<Deployment> = Api::namespaced(client, NS);
631        let error = delete_owned(&api, NS, "campus", &owner(), DeleteMode::Default)
632            .await
633            .unwrap_err();
634        assert!(matches!(error, ExecError::ConcurrentMutation { .. }));
635        drop(api);
636        recorder.await.unwrap();
637    }
638}