Skip to main content

kaniop_radius/reconcile/
mod.rs

1//! World-derived reconciliation of a `KanidmRadiusServer`.
2//!
3//! Every pass reads Kanidm and Kubernetes and lets the pure token planner
4//! decide; CR status is written as a report and never read back for a
5//! decision. Order:
6//!
7//! 1. Restore guard, referenced Kanidm, namespace authorization (an
8//!    unauthorized CR gets a `ResourceNotWatched` event and no finalizer).
9//! 2. Finalizer, then read-only input validation before any mutation: an
10//!    invalid input leaves the running Deployment and token untouched. An
11//!    unresolvable image blocks only a server with no Deployment yet; a
12//!    running one keeps its current image so rotation goes on.
13//! 3. Ensure the UID-bound Kanidm account and its membership edge.
14//! 4. Observe tokens, config Secrets, the Deployment template, ReplicaSets
15//!    and pods; plan.
16//! 5. Mint a candidate and verify it with `whoami`, only if the plan says so.
17//! 6. Create the immutable config Secret, then apply the Deployment and the
18//!    Service (a Service failure is reported after step 7, not before).
19//! 7. Re-observe (if anything was written), revoke and garbage-collect.
20//! 8. Write status, on every path; transient errors then drive backoff.
21
22/// Ownership-checked server-side apply, create-only Secret and delete.
23pub mod apply;
24/// Finalizer steps.
25pub mod cleanup;
26/// Validated config rendering for the upstream Kanidm RADIUS service.
27pub mod config;
28/// Typed errors of the executors and of a reconcile pass.
29pub mod error;
30/// Kanidm account naming, ownership-checked ensure and delete.
31pub mod identity;
32/// Image inference and the verified runtime identity of the RADIUS image.
33pub mod image;
34/// Fail-closed input validation and public-material digest.
35pub mod inputs;
36/// Kanidm token mint, verify and revoke executors.
37pub mod kanidm_token;
38/// One module for every label, annotation and Secret key the operator uses.
39pub mod meta;
40/// Builds the planner input from Kanidm and Kubernetes lists.
41pub mod observe;
42/// Kanidm SDK call metrics wrapper.
43mod sdk;
44/// Conditions and status, applied with server-side apply.
45pub mod status;
46/// EAP TLS certificate and private key validation.
47pub mod tls;
48/// Pure token planner: which token to run, revoke and garbage-collect.
49pub mod token;
50/// Deployment, Service and content-addressed config Secret builders.
51pub mod workload;
52
53/// Fake Kanidm HTTP server for executor tests.
54#[cfg(test)]
55pub(crate) mod kanidm_mock;
56/// Mock Kubernetes API server for executor tests.
57#[cfg(test)]
58pub(crate) mod kube_mock;
59
60use std::future::Future;
61use std::sync::Arc;
62use std::time::{Duration, Instant};
63
64use k8s_openapi::api::apps::v1::Deployment;
65use k8s_openapi::api::core::v1::{Secret, Service};
66use k8s_openapi::apimachinery::pkg::apis::meta::v1::{OwnerReference, Time};
67use k8s_openapi::jiff::Timestamp;
68use kanidm_client::KanidmClient;
69use kaniop_k8s_util::error::{Error, Result};
70use kaniop_operator::controller::context::IdmClientContext;
71use kaniop_operator::controller::idm_reconcile_interval;
72use kaniop_operator::controller::kanidm::{KanidmResource, is_resource_watched};
73use kaniop_operator::kanidm::crd::Kanidm;
74use kaniop_operator::metrics::ControllerMetrics;
75use kaniop_operator::telemetry;
76use kube::api::{Api, Patch, PatchParams};
77use kube::runtime::controller::Action;
78use kube::runtime::events::{Event, EventType};
79use kube::runtime::reflector::ObjectRef;
80use kube::{Client, Resource, ResourceExt};
81use serde_json::json;
82use time::OffsetDateTime;
83use tracing::{Span, debug, field, info, instrument, warn};
84use uuid::Uuid;
85
86use crate::controller::Context;
87use crate::crd::{KanidmRadiusServer, KanidmRadiusServerStatus};
88use crate::reconcile::apply::{
89    DeleteMode, DeleteOutcome, SecretOutcome, apply_owned, delete_owned, ensure_config_secret,
90    is_controlled_by,
91};
92use crate::reconcile::cleanup::{
93    CleanupStep, CleanupView, DRAIN_WARNING_REASON, DeploymentPhase, Finish, SkipReason,
94    drain_note, identity_cleanup, next_step,
95};
96use crate::reconcile::error::{ExecError, Failure, RadiusBlock};
97use crate::reconcile::identity::{RADIUS_GROUP, RadiusIdentity, delete_identity, ensure_identity};
98use crate::reconcile::image::resolve_radius_image;
99use crate::reconcile::inputs::{RadiusPreflightError, prepare_radius_inputs};
100use crate::reconcile::kanidm_token::{
101    RevokeOutcome, VerifyRetry, mint_token, revoke_token, verify_token,
102};
103use crate::reconcile::meta::COMPONENT;
104use crate::reconcile::observe::{observe, observe_drain};
105use crate::reconcile::status::{
106    ConditionKind, StatusReport, TokenFacts, Truth, Verdict, available_verdict, block_verdict,
107    build_status, kube_time, patch_status,
108};
109use crate::reconcile::token::{
110    Consumer, CurrentToken, MintReason, ObservedSecret, TokenPlan, TokenPolicy, TokenWorld,
111    expires_within_warning, plan_tokens,
112};
113use crate::reconcile::workload::{RadiusServiceExposure, build_config_secret, build_workload};
114
115pub const RADIUS_FINALIZER: &str = "kanidmradiusservers.kaniop.rs/finalizer";
116/// Requeue while a Kanidm restore pauses identity writes.
117const RESTORE_REQUEUE: Duration = Duration::from_secs(5);
118/// Requeue while a rollout, drain, revocation or garbage collection is
119/// pending (pods terminating are not watched).
120const CONVERGING_REQUEUE: Duration = Duration::from_secs(15);
121/// Requeue while the finalizer waits for the workload to drain.
122const DRAIN_REQUEUE: Duration = Duration::from_secs(5);
123/// Event reason when a blocked pass leaves the running token near expiry.
124const TOKEN_NEAR_EXPIRY_REASON: &str = "TokenNearExpiry";
125
126/// The live objects a pass reports on and whether it changed anything.
127#[derive(Debug, Default)]
128struct PassState {
129    deployment: Option<Deployment>,
130    service: Option<Service>,
131    changed: bool,
132    /// The pass got past the Deployment apply, so the planner's mint or keep
133    /// reached the pods: a later block does not stop rotation.
134    rotation_applied: bool,
135}
136
137/// Kanidm URL the RADIUS runtime uses: the Kanidm origin, else its domain.
138/// The RADIUS spec has no field for it.
139pub fn kanidm_uri(kanidm: &Kanidm) -> String {
140    kanidm
141        .spec
142        .origin
143        .clone()
144        .unwrap_or_else(|| format!("https://{}", kanidm.spec.domain))
145}
146
147fn has_finalizer(radius: &KanidmRadiusServer) -> bool {
148    radius.finalizers().iter().any(|f| f == RADIUS_FINALIZER)
149}
150
151#[instrument(skip(radius, ctx), fields(trace_id))]
152pub async fn reconcile_radius(
153    radius: Arc<KanidmRadiusServer>,
154    ctx: Arc<Context>,
155) -> Result<(Action, bool)> {
156    let trace_id = telemetry::get_trace_id();
157    Span::current().record("trace_id", field::display(&trace_id));
158    let _timer = ctx
159        .kaniop_ctx
160        .metrics
161        .reconcile_count_and_measure(&trace_id);
162
163    if !ctx.kaniop_ctx.kanidm_write_allowed(&radius) {
164        debug!("Kanidm restore in progress, pausing RADIUS reconciliation");
165        return Ok((Action::requeue(RESTORE_REQUEUE), false));
166    }
167    if radius.metadata.deletion_timestamp.is_some() {
168        if !has_finalizer(&radius) {
169            return Ok((Action::await_change(), false));
170        }
171        return finalize(&radius, &ctx).await;
172    }
173
174    let Some(kanidm) = ctx.kaniop_ctx.get_kanidm(&radius) else {
175        let block = RadiusBlock::KanidmNotFound {
176            kanidm: radius.kanidm_ref(),
177        };
178        return report_block(&radius, &ctx, block).await;
179    };
180    if !is_resource_watched(
181        radius.as_ref(),
182        &kanidm,
183        &ctx.kaniop_ctx.namespace_store,
184        &ctx.kaniop_ctx.client,
185    )
186    .await
187    {
188        publish_warning(
189            &radius,
190            &ctx,
191            "ResourceNotWatched",
192            "configure `radiusServerNamespaceSelector` on the Kanidm resource to watch this namespace",
193        )
194        .await?;
195        let block = RadiusBlock::NamespaceNotWatched {
196            kanidm: radius.kanidm_ref(),
197        };
198        return report_block(&radius, &ctx, block).await;
199    }
200    if !has_finalizer(&radius) {
201        add_finalizer(&radius, ctx.kaniop_ctx.client.clone()).await?;
202    }
203
204    info!("reconciling RADIUS server");
205    let mut state = initial_state(&radius, &ctx);
206    let mut status_report = StatusReport::default();
207    let idm_ctx = ctx.clone();
208    let idm_radius = radius.clone();
209    let result = run_pass(
210        &radius,
211        &kanidm,
212        ctx.kaniop_ctx.client.clone(),
213        &ctx.kaniop_ctx.metrics,
214        move || async move { idm_ctx.get_idm_client(&idm_radius).await },
215        &mut status_report,
216        &mut state,
217    )
218    .await;
219    report_pass(&radius, &ctx, state, status_report, result).await
220}
221
222/// Deployment and Service from the cache, for a pass that stops before it
223/// reads or writes them. Objects we do not control are ignored.
224fn initial_state(radius: &KanidmRadiusServer, ctx: &Context) -> PassState {
225    let Some(owner) = radius.controller_owner_ref(&()) else {
226        return PassState::default();
227    };
228    let namespace = radius.namespace().unwrap_or_default();
229    let name = radius.name_any();
230    let deployment = ctx
231        .deployment_store
232        .get(&ObjectRef::new(&name).within(&namespace))
233        .filter(|d| is_controlled_by(&d.metadata, &owner))
234        .map(|d| d.as_ref().clone());
235    let service = ctx
236        .service_store
237        .get(&ObjectRef::new(&name).within(&namespace))
238        .filter(|s| is_controlled_by(&s.metadata, &owner))
239        .map(|s| s.as_ref().clone());
240    PassState {
241        deployment,
242        service,
243        ..Default::default()
244    }
245}
246
247/// Write status for a pass blocked before it started.
248async fn report_block(
249    radius: &KanidmRadiusServer,
250    ctx: &Context,
251    block: RadiusBlock,
252) -> Result<(Action, bool)> {
253    let state = initial_state(radius, ctx);
254    report_pass(
255        radius,
256        ctx,
257        state,
258        StatusReport::default(),
259        Err(block.into()),
260    )
261    .await
262}
263
264/// Write the status first, then return the requeue or the transient error.
265async fn report_pass(
266    radius: &KanidmRadiusServer,
267    ctx: &Context,
268    state: PassState,
269    mut status_report: StatusReport,
270    result: std::result::Result<Duration, Failure>,
271) -> Result<(Action, bool)> {
272    if let Err(failure) = &result {
273        status_report.record_failure(failure);
274    }
275    let status = build_status(
276        radius.metadata.generation,
277        radius.status.as_ref(),
278        &status_report,
279        state.deployment.as_ref(),
280        state.service.as_ref(),
281        &Time(Timestamp::now()),
282    );
283    let written = patch_status(ctx.kaniop_ctx.client.clone(), radius, status).await;
284    if let Err(error) = &written {
285        ctx.kaniop_ctx.metrics.status_update_errors_inc();
286        warn!(%error, "failed to write RADIUS status");
287    }
288    match result {
289        Ok(requeue) => {
290            written?;
291            Ok((Action::requeue(requeue), state.changed))
292        }
293        Err(Failure::Block(block)) => {
294            written?;
295            debug!(reason = block.reason(), %block, "RADIUS reconciliation blocked");
296            // Only a block that stopped the pass before the token reached
297            // the pods stops rotation; with no Deployment there are no pods.
298            if state.deployment.is_some() && !state.rotation_applied {
299                warn_token_near_expiry(radius, ctx, &block, &status_report).await;
300            }
301            let requeue = block.retry_after().unwrap_or_else(idm_reconcile_interval);
302            Ok((Action::requeue(requeue), state.changed))
303        }
304        // The source error wins over a status write failure.
305        Err(Failure::Transient { error, .. }) => Err(error),
306    }
307}
308
309/// Warn when a blocked pass leaves the running token close to its expiry.
310/// Best effort: a failed event does not change the pass result.
311async fn warn_token_near_expiry(
312    radius: &KanidmRadiusServer,
313    ctx: &Context,
314    block: &RadiusBlock,
315    report: &StatusReport,
316) {
317    let policy = TokenPolicy::from_spec(Some(&radius.spec.token_rotation));
318    let running = running_token_expiry(report, radius.status.as_ref(), &policy);
319    let Some(note) = token_near_expiry_note(block, running, OffsetDateTime::now_utc()) else {
320        return;
321    };
322    warn!(
323        reason = block.reason(),
324        "RADIUS token near expiry while blocked"
325    );
326    if let Err(error) = publish_warning(radius, ctx, TOKEN_NEAR_EXPIRY_REASON, &note).await {
327        warn!(%error, "failed to publish RADIUS token expiry warning");
328    }
329}
330
331/// One reconcile pass after authorization and finalizer. Returns the
332/// requeue delay. `report` and `state` carry what the pass established even
333/// when it fails, so the caller can write status first.
334async fn run_pass<F, Fut>(
335    radius: &KanidmRadiusServer,
336    kanidm: &Kanidm,
337    kube: Client,
338    metrics: &ControllerMetrics,
339    idm_client: F,
340    report: &mut StatusReport,
341    state: &mut PassState,
342) -> std::result::Result<Duration, Failure>
343where
344    F: FnOnce() -> Fut,
345    Fut: Future<Output = Result<Arc<KanidmClient>>>,
346{
347    // Read-only validation of every input before any mutation.
348    let inputs = prepare_radius_inputs(kube.clone(), radius, &kanidm_uri(kanidm))
349        .await
350        .map_err(Failure::inputs)?;
351    // An unresolved image blocks only a server that is not running yet. A
352    // running one keeps its current image so token rotation goes on, with
353    // the block still reported on `ConfigurationReady`.
354    let (image, image_block) =
355        match resolve_radius_image(&kanidm.spec.image, radius.spec.image.as_deref()) {
356            Ok(image) => (image, None),
357            Err(error) => match state.deployment.as_ref().and_then(deployed_image) {
358                Some(image) => {
359                    debug!(%error, %image, "RADIUS image unresolved, keeping the running image");
360                    (image, Some(RadiusBlock::from(error)))
361                }
362                None => return Err(RadiusBlock::from(error).into()),
363            },
364        };
365    RadiusServiceExposure::from_spec(
366        &radius.spec.service.type_,
367        radius.spec.service.external_traffic_policy.clone(),
368    )
369    .map_err(RadiusBlock::from)?;
370    let identity = RadiusIdentity::for_resource(radius).map_err(RadiusBlock::from)?;
371    let namespace = radius
372        .namespace()
373        .ok_or(RadiusBlock::Inputs(RadiusPreflightError::MissingNamespace))?;
374    let owner = radius.controller_owner_ref(&()).ok_or(RadiusBlock::Exec {
375        error: ExecError::MissingMetadata {
376            kind: "KanidmRadiusServer",
377            field: "uid",
378        },
379        stage: ConditionKind::ConfigurationReady,
380    })?;
381    report.configuration = match &image_block {
382        Some(block) => block_verdict(block),
383        None => Verdict::yes(
384            "ConfigurationValid",
385            "referenced Secrets, CA and TLS material are valid",
386        ),
387    };
388
389    // Identity: UID-bound account, additive membership, never adopted.
390    let idm = idm_client()
391        .await
392        .map_err(Failure::transient(ConditionKind::IdentityReady))?;
393    let outcome = ensure_identity(&idm, metrics, &identity)
394        .await
395        .map_err(Failure::exec(ConditionKind::IdentityReady))?;
396    state.changed |= outcome.account_created || outcome.membership_added;
397    report.identity = Verdict::yes(
398        "AccountReady",
399        format!(
400            "Kanidm service account {} is a member of {RADIUS_GROUP}",
401            identity.account_name
402        ),
403    );
404
405    // Observe and plan.
406    let policy = TokenPolicy::from_spec(Some(&radius.spec.token_rotation));
407    let observation = observe(&idm, metrics, kube.clone(), radius, &identity)
408        .await
409        .map_err(Failure::exec(ConditionKind::Rotating))?;
410    state.deployment = observation.deployment.clone();
411    let plan = plan_tokens(&observation.world, &policy, OffsetDateTime::now_utc());
412    debug!(?plan, "token plan");
413
414    let account = identity.account_name.as_str();
415    let (token_id, bearer, rotation_request, minted) = match plan.current {
416        CurrentToken::Keep(id) => {
417            let bearer = observation
418                .template_bearer
419                .clone()
420                .ok_or(RadiusBlock::BearerUnreadable)?;
421            // Carry the recorded request forward: a config-only change must
422            // not turn a past rotation request into a new one.
423            let request = template_secret(&observation.world)
424                .and_then(|secret| secret.rotation_request.clone());
425            (id, bearer, request, None)
426        }
427        CurrentToken::Mint(reason) => {
428            let expiry = policy.new_token_expiry(OffsetDateTime::now_utc());
429            let candidate = mint_token(&idm, metrics, account, expiry)
430                .await
431                .map_err(Failure::exec(ConditionKind::Rotating))?;
432            state.changed = true;
433            info!(token_id = %candidate.token_id, reason = reason.reason(), "minted RADIUS token");
434            // Promotion gate: a candidate Kanidm does not accept never reaches
435            // a Secret or a pod; it is revoked as an orphan after the grace.
436            verify_token(
437                &idm,
438                metrics,
439                account,
440                &candidate.bearer,
441                VerifyRetry::default(),
442            )
443            .await
444            .map_err(Failure::exec(ConditionKind::Rotating))?;
445            let request = observation
446                .world
447                .rotate_request
448                .clone()
449                .filter(|value| !value.is_empty());
450            (candidate.token_id, candidate.bearer, request, Some(reason))
451        }
452        CurrentToken::Blocked => {
453            // Nothing was written since the observation, so its plan may
454            // revoke and collect; that is what clears the block.
455            report.tokens = Some(token_facts(&observation.world, &plan));
456            collect(
457                &idm, metrics, kube, &namespace, &owner, account, &plan, state,
458            )
459            .await?;
460            return Err(RadiusBlock::TooManyTokens.into());
461        }
462    };
463
464    // Immutable, content-addressed config Secret.
465    let config = inputs
466        .config_with_token(bearer.expose().to_string())
467        .map_err(|error| RadiusBlock::Inputs(error.into()))?;
468    let secret = build_config_secret(
469        radius,
470        &token_id.to_string(),
471        rotation_request.as_deref(),
472        &config,
473    )
474    .map_err(RadiusBlock::from)?;
475    let secret_name = secret.name_any();
476    let secrets: Api<Secret> = Api::namespaced(kube.clone(), &namespace);
477    let secret_created = ensure_config_secret(&secrets, &secret, &owner)
478        .await
479        .map_err(Failure::exec(ConditionKind::ConfigurationReady))?
480        == SecretOutcome::Created;
481    if secret_created {
482        info!(secret = %secret_name, "created RADIUS config Secret");
483        state.changed = true;
484    }
485
486    // Deployment and Service.
487    let workload = build_workload(
488        radius,
489        image,
490        &secret_name,
491        &token_id.to_string(),
492        inputs.inputs_digest(),
493    )
494    .map_err(RadiusBlock::from)?;
495    let deployments: Api<Deployment> = Api::namespaced(kube.clone(), &namespace);
496    let desired = workload.deployment().map_err(RadiusBlock::from)?;
497    let before = observation
498        .deployment
499        .as_ref()
500        .and_then(|d| d.metadata.resource_version.clone());
501    let applied = apply_owned(&deployments, &desired, &owner)
502        .await
503        .map_err(Failure::exec(ConditionKind::Available))?;
504    let template_changed = applied.metadata.resource_version != before;
505    state.changed |= template_changed;
506    state.deployment = Some(applied);
507    state.rotation_applied = true;
508    // The Service feeds neither the planner nor the pods' token: a failure
509    // to apply it is recorded now and returned only after revoke and garbage
510    // collection, so a persistent one (a foreign Service) cannot hold retired
511    // tokens forever.
512    let services: Api<Service> = Api::namespaced(kube.clone(), &namespace);
513    let service_before = state
514        .service
515        .as_ref()
516        .and_then(|s| s.metadata.resource_version.clone());
517    let exposed = match workload.service() {
518        Ok(desired) => apply_owned(&services, &desired, &owner)
519            .await
520            .map_err(Failure::exec(ConditionKind::Exposed)),
521        Err(error) => Err(RadiusBlock::from(error).into()),
522    };
523    let exposed = match exposed {
524        Ok(applied) => {
525            state.changed |= applied.metadata.resource_version != service_before;
526            state.service = Some(applied);
527            Ok(())
528        }
529        Err(failure) => {
530            report.record_failure(&failure);
531            Err(failure)
532        }
533    };
534
535    // Revoke and collect only from an observation made after the writes. If
536    // nothing that feeds the planner was written, the first one still holds.
537    let wrote = minted.is_some() || secret_created || template_changed;
538    let (world, plan) = if wrote {
539        let fresh = observe(&idm, metrics, kube.clone(), radius, &identity)
540            .await
541            .map_err(Failure::exec(ConditionKind::Rotating))?;
542        let plan = plan_tokens(&fresh.world, &policy, OffsetDateTime::now_utc());
543        (fresh.world, plan)
544    } else {
545        (observation.world, plan)
546    };
547    report.tokens = Some(token_facts(&world, &plan));
548    report.rotating = rotating_verdict(&world, &plan, minted);
549    collect(
550        &idm, metrics, kube, &namespace, &owner, account, &plan, state,
551    )
552    .await?;
553    exposed?;
554
555    let rollout_done = available_verdict(state.deployment.as_ref()).truth == Truth::True;
556    let converging = !plan.retiring.is_empty()
557        || !plan.unresolved_consumers.is_empty()
558        || world.secrets.len() > plan.gc_secrets.len() + 1
559        || !rollout_done;
560    Ok(if converging {
561        CONVERGING_REQUEUE
562    } else {
563        idm_reconcile_interval()
564    })
565}
566
567/// The token the pods run and its expiry, for a blocked pass: as observed in
568/// this pass, else estimated from the last reported issue time and the
569/// current rotation policy.
570///
571/// Status records no expiry, so the estimate misjudges a token minted under
572/// another rotation policy; it is used for a warning only, never a decision.
573fn running_token_expiry(
574    report: &StatusReport,
575    previous: Option<&KanidmRadiusServerStatus>,
576    policy: &TokenPolicy,
577) -> Option<(String, OffsetDateTime)> {
578    if let Some(facts) = &report.tokens {
579        let (id, _) = facts.current?;
580        return Some((id.to_string(), facts.current_expiry?));
581    }
582    let previous = previous?;
583    let issued = previous.token_issued_at.as_ref()?;
584    let issued = OffsetDateTime::from_unix_timestamp_nanos(issued.0.as_nanosecond()).ok()?;
585    Some((previous.token_id.clone()?, policy.new_token_expiry(issued)?))
586}
587
588/// Warning note when the running token expires within twice the safety
589/// margin while `block` stops rotation. Stable for an unchanged block and
590/// token, so the event recorder folds repeats into one event series.
591fn token_near_expiry_note(
592    block: &RadiusBlock,
593    running: Option<(String, OffsetDateTime)>,
594    now: OffsetDateTime,
595) -> Option<String> {
596    let (id, expiry) = running?;
597    if !expires_within_warning(expiry, now) {
598        return None;
599    }
600    let at = kube_time(expiry).map_or_else(|| expiry.to_string(), |t| t.0.to_string());
601    Some(format!(
602        "token {id} used by the RADIUS pods expires at {at} and cannot be rotated while \
603         reconciliation is blocked ({}); clear the block before the token lapses",
604        block.reason()
605    ))
606}
607
608/// The image of the RADIUS container in a Deployment's pod template.
609fn deployed_image(deployment: &Deployment) -> Option<String> {
610    deployment
611        .spec
612        .as_ref()?
613        .template
614        .spec
615        .as_ref()?
616        .containers
617        .iter()
618        .find(|c| c.name == COMPONENT)?
619        .image
620        .clone()
621        .filter(|image| !image.trim().is_empty())
622}
623
624/// The config Secret the template mounts, as observed.
625fn template_secret(world: &TokenWorld) -> Option<&ObservedSecret> {
626    let name = world.template.as_ref()?.config_secret.as_deref()?;
627    world.secrets.iter().find(|s| s.name == name)
628}
629
630/// Token facts for status: the token the template's config Secret was
631/// rendered with (what pods run), and the other tokens still held.
632fn token_facts(world: &TokenWorld, plan: &TokenPlan) -> TokenFacts {
633    let secret = template_secret(world);
634    let current = secret
635        .and_then(|s| s.token_id.as_deref())
636        .and_then(|raw| Uuid::parse_str(raw).ok())
637        .and_then(|id| world.tokens.iter().find(|t| t.id == id));
638    let current_id = current.map(|t| t.id);
639    TokenFacts {
640        current: current.map(|t| (t.id, t.issued_at)),
641        current_expiry: current.and_then(|t| t.expiry),
642        config_secret_name: secret.map(|s| s.name.clone()),
643        retiring: plan
644            .retiring
645            .iter()
646            .copied()
647            .filter(|id| Some(*id) != current_id)
648            .collect(),
649    }
650}
651
652fn rotating_verdict(world: &TokenWorld, plan: &TokenPlan, minted: Option<MintReason>) -> Verdict {
653    let facts = token_facts(world, plan);
654    if !plan.unresolved_consumers.is_empty() {
655        return Verdict::yes(
656            "ConsumerUnresolved",
657            format!(
658                "revocation is held: the token used by {} cannot be resolved",
659                plan.unresolved_consumers.join(", ")
660            ),
661        );
662    }
663    if !facts.retiring.is_empty() {
664        let ids: Vec<String> = facts.retiring.iter().map(Uuid::to_string).collect();
665        let reason = minted.map_or("RetiringTokens", MintReason::reason);
666        return Verdict::yes(
667            reason,
668            format!(
669                "tokens kept until no pod, ReplicaSet or template uses them: {}",
670                ids.join(", ")
671            ),
672        );
673    }
674    Verdict::no(
675        "TokenCurrent",
676        facts.current.map_or_else(
677            || "no other token is held".to_string(),
678            |(id, _)| format!("token {id} is current; no other token is held"),
679        ),
680    )
681}
682
683/// Revoke and garbage-collect per `plan`. Revocation goes first: a Secret
684/// whose token is still on the account is collected on a later pass, while a
685/// token whose Secret is gone is an orphan revoked after the grace period.
686#[allow(clippy::too_many_arguments)]
687async fn collect(
688    idm: &KanidmClient,
689    metrics: &ControllerMetrics,
690    kube: Client,
691    namespace: &str,
692    owner: &OwnerReference,
693    account: &str,
694    plan: &TokenPlan,
695    state: &mut PassState,
696) -> std::result::Result<(), Failure> {
697    for token_id in &plan.revoke {
698        let outcome = revoke_token(idm, metrics, account, *token_id)
699            .await
700            .map_err(Failure::exec(ConditionKind::Rotating))?;
701        if outcome == RevokeOutcome::Revoked {
702            info!(%token_id, "revoked unused RADIUS token");
703            state.changed = true;
704        }
705    }
706    let secrets: Api<Secret> = Api::namespaced(kube, namespace);
707    for name in &plan.gc_secrets {
708        let outcome = delete_owned(&secrets, namespace, name, owner, DeleteMode::Default)
709            .await
710            .map_err(Failure::exec(ConditionKind::Rotating))?;
711        if outcome == DeleteOutcome::Deleted {
712            info!(secret = %name, "deleted unused RADIUS config Secret");
713            state.changed = true;
714        }
715    }
716    Ok(())
717}
718
719/// One finalizer step per pass; see [`cleanup`].
720/// Whether the referenced Kanidm CR exists. The cache answers yes; a no is
721/// confirmed live, because skipping the identity cleanup on a cache that has
722/// not synced yet (operator start) would leak the account.
723async fn kanidm_exists(radius: &KanidmRadiusServer, ctx: &Context) -> Result<bool> {
724    if ctx.kaniop_ctx.get_kanidm(radius).is_some() {
725        return Ok(true);
726    }
727    let namespace = radius.kanidm_namespace();
728    let name = radius.kanidm_name();
729    Api::<Kanidm>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace)
730        .get_opt(&name)
731        .await
732        .map(|kanidm| kanidm.is_some())
733        .map_err(|e| Error::kube_error("get", "Kanidm", namespace, name, e))
734}
735
736async fn finalize(radius: &KanidmRadiusServer, ctx: &Context) -> Result<(Action, bool)> {
737    let client = ctx.kaniop_ctx.client.clone();
738    let drain = observe_drain(client.clone(), radius)
739        .await
740        .map_err(finalizer_error)?;
741    let view = CleanupView {
742        deployment: drain.deployment.as_ref().map(|d| {
743            if d.metadata.deletion_timestamp.is_some() {
744                DeploymentPhase::Deleting
745            } else {
746                DeploymentPhase::Live
747            }
748        }),
749        consumers: drain.consumers.iter().map(Consumer::label).collect(),
750        kanidm_present: kanidm_exists(radius, ctx).await?,
751    };
752    let finish = match next_step(&view) {
753        CleanupStep::DeleteDeployment => {
754            let namespace = radius.namespace().unwrap_or_default();
755            let owner = radius
756                .controller_owner_ref(&())
757                .ok_or_else(|| Error::MissingData("RADIUS server has no UID".into()))?;
758            let deployments: Api<Deployment> = Api::namespaced(client, &namespace);
759            delete_owned(
760                &deployments,
761                &namespace,
762                &radius.name_any(),
763                &owner,
764                DeleteMode::Foreground,
765            )
766            .await
767            .map_err(finalizer_error)?;
768            info!("deleting RADIUS Deployment before identity cleanup");
769            return Ok((Action::requeue(DRAIN_REQUEUE), true));
770        }
771        CleanupStep::AwaitDrain { consumers } => {
772            debug!(?consumers, "waiting for RADIUS pods to drain");
773            let uid = radius.uid().unwrap_or_default();
774            if ctx
775                .drain_warnings
776                .should_warn(&uid, &consumers, Instant::now())
777            {
778                // Best effort: a failed event must not turn the wait into an
779                // error with backoff.
780                let note = drain_note(&consumers);
781                if let Err(error) = publish_warning(radius, ctx, DRAIN_WARNING_REASON, &note).await
782                {
783                    warn!(%error, "failed to publish RADIUS drain warning");
784                }
785            }
786            return Ok((Action::requeue(DRAIN_REQUEUE), false));
787        }
788        CleanupStep::Skip(reason) => Finish::SkipAndRemove(reason),
789        CleanupStep::DeleteIdentity => {
790            let identity = RadiusIdentity::for_resource(radius)
791                .map_err(|e| Error::MissingData(format!("RADIUS identity: {e}")))?;
792            let idm = ctx.get_idm_client(radius).await?;
793            let result = delete_identity(&idm, &ctx.kaniop_ctx.metrics, &identity).await;
794            match identity_cleanup(result) {
795                Ok(finish) => finish,
796                Err(Failure::Transient { error, .. }) => return Err(error),
797                Err(Failure::Block(block)) => {
798                    warn!(reason = block.reason(), %block, "RADIUS identity cleanup blocked");
799                    let requeue = block.retry_after().unwrap_or(DRAIN_REQUEUE);
800                    return Ok((Action::requeue(requeue), false));
801                }
802            }
803        }
804    };
805    if let Finish::SkipAndRemove(reason) = finish {
806        warn_skip(radius, ctx, reason).await?;
807    }
808    remove_finalizer(radius, ctx.kaniop_ctx.client.clone()).await?;
809    ctx.drain_warnings.forget(&radius.uid().unwrap_or_default());
810    info!("RADIUS server cleaned up");
811    Ok((Action::await_change(), true))
812}
813
814async fn warn_skip(radius: &KanidmRadiusServer, ctx: &Context, reason: SkipReason) -> Result<()> {
815    warn!(reason = reason.reason(), "skipping RADIUS Kanidm cleanup");
816    publish_warning(radius, ctx, reason.reason(), reason.note()).await
817}
818
819/// Executor errors during deletion: transient ones keep their source; a
820/// changed object retries on the next pass through the backoff.
821fn finalizer_error(error: ExecError) -> Error {
822    match error {
823        ExecError::Api(error) => error,
824        other => Error::MissingData(format!("RADIUS finalizer: {other}")),
825    }
826}
827
828async fn publish_warning(
829    radius: &KanidmRadiusServer,
830    ctx: &Context,
831    reason: &str,
832    note: &str,
833) -> Result<()> {
834    ctx.kaniop_ctx
835        .recorder
836        .publish(
837            &Event {
838                type_: EventType::Warning,
839                reason: reason.to_string(),
840                note: Some(note.to_string()),
841                action: "Reconcile".to_string(),
842                secondary: None,
843            },
844            &radius.object_ref(&()),
845        )
846        .await
847        .map_err(|e| {
848            Error::kube_error(
849                "publish",
850                "event",
851                radius.namespace().unwrap_or_default(),
852                radius.name_any(),
853                e,
854            )
855        })
856}
857
858/// JSON patch that adds the finalizer only if the list is unchanged since
859/// it was read (same approach as kube-runtime's finalizer helper).
860fn add_finalizer_patch(radius: &KanidmRadiusServer) -> serde_json::Value {
861    match radius.metadata.finalizers.as_deref() {
862        None | Some([]) => json!([
863            {"op": "test", "path": "/metadata/finalizers", "value": radius.metadata.finalizers},
864            {"op": "add", "path": "/metadata/finalizers", "value": [RADIUS_FINALIZER]},
865        ]),
866        Some(current) => json!([
867            {"op": "test", "path": "/metadata/finalizers", "value": current},
868            {"op": "add", "path": "/metadata/finalizers/-", "value": RADIUS_FINALIZER},
869        ]),
870    }
871}
872
873/// JSON patch that removes our finalizer at its observed index, guarded by
874/// a test so a concurrent change of the list fails instead of removing the
875/// wrong entry.
876fn remove_finalizer_patch(radius: &KanidmRadiusServer) -> Option<serde_json::Value> {
877    let index = radius
878        .finalizers()
879        .iter()
880        .position(|f| f == RADIUS_FINALIZER)?;
881    let path = format!("/metadata/finalizers/{index}");
882    Some(json!([
883        {"op": "test", "path": path, "value": RADIUS_FINALIZER},
884        {"op": "remove", "path": path},
885    ]))
886}
887
888async fn patch_finalizers(
889    radius: &KanidmRadiusServer,
890    client: Client,
891    patch: serde_json::Value,
892) -> Result<()> {
893    let namespace = radius.namespace().unwrap_or_default();
894    let name = radius.name_any();
895    let patch = serde_json::from_value(patch)
896        .map_err(|e| Error::SerializationError("RADIUS finalizer patch".into(), e))?;
897    match Api::<KanidmRadiusServer>::namespaced(client, &namespace)
898        .patch(&name, &PatchParams::default(), &Patch::<()>::Json(patch))
899        .await
900    {
901        Ok(_) => Ok(()),
902        // Already gone: nothing left to finalize.
903        Err(kube::Error::Api(status)) if status.code == 404 => Ok(()),
904        Err(e) => Err(Error::kube_error(
905            "patch finalizers of",
906            "KanidmRadiusServer",
907            namespace,
908            name,
909            e,
910        )),
911    }
912}
913
914async fn add_finalizer(radius: &KanidmRadiusServer, client: Client) -> Result<()> {
915    patch_finalizers(radius, client, add_finalizer_patch(radius)).await
916}
917
918async fn remove_finalizer(radius: &KanidmRadiusServer, client: Client) -> Result<()> {
919    match remove_finalizer_patch(radius) {
920        Some(patch) => patch_finalizers(radius, client, patch).await,
921        None => Ok(()),
922    }
923}
924
925#[cfg(test)]
926mod tests {
927    use k8s_openapi::ByteString;
928    use opentelemetry::metrics::MeterProvider;
929    use opentelemetry_sdk::metrics::SdkMeterProvider;
930    use serde_json::json;
931
932    use kaniop_operator::kanidm::crd::KanidmSpec;
933
934    use openssl::asn1::Asn1Time;
935    use openssl::hash::MessageDigest;
936    use openssl::pkey::PKey;
937    use openssl::rsa::Rsa;
938    use openssl::x509::{X509, X509Name};
939    use serde_json::Value;
940
941    use super::*;
942    use crate::reconcile::kanidm_mock::{self, MockKanidm, Request};
943    use crate::reconcile::kube_mock::{Recorded, api_status, mock_client, to_json};
944    use crate::reconcile::status::{TYPE_AVAILABLE, TYPE_CONFIGURATION_READY};
945    use crate::reconcile::token::SAFETY_MARGIN;
946
947    const UID: &str = "d10066c3-4599-4487-972d-26bb42587a32";
948    const ACCOUNT: &str = "kaniop_radius_d10066c345994487972d26bb42587a32";
949    /// Current token: mounted by the Deployment template through `SECRET_A`.
950    const TOKEN_A: &str = "9ad74e68-147f-49e8-96db-dc21c920ff3b";
951    /// Retired token: nothing uses it, its Secret is `SECRET_B`.
952    const TOKEN_B: &str = "31b30b65-037a-418f-b7ca-0c376da2db28";
953    const SECRET_A: &str = "campus-0a1b2c3d4e";
954    const SECRET_B: &str = "campus-f0e1d2c3b4";
955    const RUNNING_IMAGE: &str = "registry.example.com/kanidm/radius:1.11.2";
956
957    fn radius() -> KanidmRadiusServer {
958        serde_json::from_value(json!({
959            "apiVersion": "kaniop.rs/v1beta1",
960            "kind": "KanidmRadiusServer",
961            "metadata": {"name": "campus", "namespace": "network", "uid": UID, "generation": 2},
962            "spec": {
963                "kanidmRef": {"name": "idm"},
964                "authorization": {"requiredGroups": ["allowed@idm.example.com"]},
965                "clients": [{"name": "nas", "ipaddr": "192.0.2.1",
966                             "sharedSecretRef": {"name": "nas-secret", "key": "nas"}}],
967                "radiusTls": {"secretRef": {"name": "eap-tls"}}
968            }
969        }))
970        .unwrap()
971    }
972
973    fn kanidm() -> Kanidm {
974        Kanidm {
975            metadata: kube::api::ObjectMeta {
976                name: Some("idm".into()),
977                namespace: Some("network".into()),
978                ..Default::default()
979            },
980            spec: KanidmSpec {
981                domain: "idm.example.com".into(),
982                image: "kanidm/server:1.11.2".into(),
983                ..Default::default()
984            },
985            ..Default::default()
986        }
987    }
988
989    fn metrics() -> (ControllerMetrics, SdkMeterProvider) {
990        let provider = SdkMeterProvider::builder().build();
991        (
992            ControllerMetrics::new("radius-test", &provider.meter("test")),
993            provider,
994        )
995    }
996
997    #[tokio::test]
998    async fn invalid_input_touches_nothing_and_keeps_available() {
999        // The NAS Secret exists but its key was deleted.
1000        let nas_secret = Secret {
1001            metadata: kube::api::ObjectMeta {
1002                name: Some("nas-secret".into()),
1003                namespace: Some("network".into()),
1004                ..Default::default()
1005            },
1006            data: Some([("other".to_string(), ByteString(b"x".to_vec()))].into()),
1007            ..Default::default()
1008        };
1009        let (client, recorder) = mock_client(vec![(200, to_json(&nas_secret))]);
1010        let (metrics, _provider) = metrics();
1011        let running = Deployment {
1012            metadata: kube::api::ObjectMeta {
1013                name: Some("campus".into()),
1014                generation: Some(1),
1015                ..Default::default()
1016            },
1017            spec: Some(k8s_openapi::api::apps::v1::DeploymentSpec {
1018                replicas: Some(2),
1019                ..Default::default()
1020            }),
1021            status: Some(k8s_openapi::api::apps::v1::DeploymentStatus {
1022                observed_generation: Some(1),
1023                replicas: Some(2),
1024                updated_replicas: Some(2),
1025                available_replicas: Some(2),
1026                ready_replicas: Some(2),
1027                ..Default::default()
1028            }),
1029        };
1030        let mut state = PassState {
1031            deployment: Some(running),
1032            ..Default::default()
1033        };
1034        let mut report = StatusReport::default();
1035        let result = run_pass(
1036            &radius(),
1037            &kanidm(),
1038            client,
1039            &metrics,
1040            || async { panic!("Kanidm must not be contacted for an invalid input") },
1041            &mut report,
1042            &mut state,
1043        )
1044        .await;
1045        let failure = result.unwrap_err();
1046        assert!(
1047            matches!(
1048                failure,
1049                Failure::Block(RadiusBlock::Inputs(
1050                    RadiusPreflightError::NasSecretKeyUnavailable
1051                ))
1052            ),
1053            "{failure:?}"
1054        );
1055        assert!(!state.changed);
1056        let requests = recorder_requests(recorder).await;
1057        assert_eq!(
1058            requests,
1059            ["GET /api/v1/namespaces/network/secrets/nas-secret"],
1060            "only the input read may happen"
1061        );
1062
1063        report.record_failure(&failure);
1064        let status = build_status(
1065            Some(2),
1066            None,
1067            &report,
1068            state.deployment.as_ref(),
1069            None,
1070            &Time(Timestamp::now()),
1071        );
1072        let conditions = status.conditions.unwrap();
1073        let get = |t: &str| conditions.iter().find(|c| c.type_ == t).unwrap().clone();
1074        assert_eq!(get(TYPE_CONFIGURATION_READY).status, "False");
1075        assert_eq!(get(TYPE_AVAILABLE).status, "True");
1076        assert!(!status.ready);
1077    }
1078
1079    fn owner_refs() -> Value {
1080        json!([{"apiVersion": "kaniop.rs/v1beta1", "kind": "KanidmRadiusServer",
1081                "name": "campus", "uid": UID, "controller": true}])
1082    }
1083
1084    fn input_secrets() -> [(u16, Value); 2] {
1085        let key = PKey::from_rsa(Rsa::generate(2048).unwrap()).unwrap();
1086        let mut subject = X509Name::builder().unwrap();
1087        subject
1088            .append_entry_by_text("CN", "radius.example.com")
1089            .unwrap();
1090        let subject = subject.build();
1091        let mut cert = X509::builder().unwrap();
1092        cert.set_version(2).unwrap();
1093        cert.set_subject_name(&subject).unwrap();
1094        cert.set_issuer_name(&subject).unwrap();
1095        cert.set_pubkey(&key).unwrap();
1096        cert.set_not_before(&Asn1Time::days_from_now(0).unwrap())
1097            .unwrap();
1098        cert.set_not_after(&Asn1Time::days_from_now(30).unwrap())
1099            .unwrap();
1100        cert.sign(&key, MessageDigest::sha256()).unwrap();
1101        let secret = |name: &str, data: Vec<(&str, Vec<u8>)>| Secret {
1102            metadata: kube::api::ObjectMeta {
1103                name: Some(name.into()),
1104                namespace: Some("network".into()),
1105                ..Default::default()
1106            },
1107            data: Some(
1108                data.into_iter()
1109                    .map(|(k, v)| (k.to_string(), ByteString(v)))
1110                    .collect(),
1111            ),
1112            ..Default::default()
1113        };
1114        [
1115            (
1116                200,
1117                to_json(&secret(
1118                    "nas-secret",
1119                    vec![("nas", b"a-long-enough-nas-shared-secret".to_vec())],
1120                )),
1121            ),
1122            (
1123                200,
1124                to_json(&secret(
1125                    "eap-tls",
1126                    vec![
1127                        ("tls.crt", cert.build().to_pem().unwrap()),
1128                        ("tls.key", key.private_key_to_pem_pkcs8().unwrap()),
1129                    ],
1130                )),
1131            ),
1132        ]
1133    }
1134
1135    /// Our Deployment, running `RUNNING_IMAGE` with token A from `SECRET_A`.
1136    fn running_deployment(resource_version: &str) -> Deployment {
1137        serde_json::from_value(json!({
1138            "apiVersion": "apps/v1", "kind": "Deployment",
1139            "metadata": {"name": "campus", "namespace": "network", "uid": "dep-uid",
1140                         "resourceVersion": resource_version, "generation": 1,
1141                         "ownerReferences": owner_refs()},
1142            "spec": {
1143                "replicas": 1,
1144                "selector": {"matchLabels": {"kaniop.rs/radius-uid": UID}},
1145                "template": {
1146                    "metadata": {"annotations": {"kaniop.rs/radius-token-id": TOKEN_A}},
1147                    "spec": {
1148                        "containers": [{"name": COMPONENT, "image": RUNNING_IMAGE}],
1149                        "volumes": [{"name": "radius-data", "projected": {"sources": [
1150                            {"secret": {"name": SECRET_A,
1151                                        "items": [{"key": "radius.toml", "path": "radius.toml"}]}}
1152                        ]}}]
1153                    }
1154                }
1155            }
1156        }))
1157        .unwrap()
1158    }
1159
1160    fn config_secret(name: &str, token: &str) -> Value {
1161        json!({
1162            "apiVersion": "v1", "kind": "Secret",
1163            "metadata": {"name": name, "namespace": "network", "uid": format!("uid-{name}"),
1164                         "resourceVersion": "1",
1165                         "labels": {"kaniop.rs/radius-uid": UID},
1166                         "annotations": {"kaniop.rs/radius-token-id": token},
1167                         "ownerReferences": owner_refs()},
1168            "data": {"auth_token": "YmVhcmVy"}
1169        })
1170    }
1171
1172    fn list(kind: &str, items: Vec<Value>) -> (u16, Value) {
1173        (
1174            200,
1175            json!({"apiVersion": "v1", "kind": kind, "metadata": {}, "items": items}),
1176        )
1177    }
1178
1179    /// Replies to one `observe`: the Deployment, then the namespace lists.
1180    fn observation_replies() -> Vec<(u16, Value)> {
1181        vec![
1182            (200, to_json(&running_deployment("1"))),
1183            list(
1184                "SecretList",
1185                vec![
1186                    config_secret(SECRET_A, TOKEN_A),
1187                    config_secret(SECRET_B, TOKEN_B),
1188                ],
1189            ),
1190            list("ReplicaSetList", vec![]),
1191            list("PodList", vec![]),
1192        ]
1193    }
1194
1195    /// Kanidm with our account, its membership and tokens A and B, both
1196    /// days old; token deletes succeed.
1197    async fn kanidm_with_two_tokens() -> MockKanidm {
1198        let identity = RadiusIdentity::for_resource(&radius()).unwrap();
1199        let entry = json!({"attrs": {"displayname": [identity.display_name]}}).to_string();
1200        let issued = OffsetDateTime::now_utc().unix_timestamp() - 3 * 86_400;
1201        let token = |id: &str| {
1202            json!({"account_id": "00000000-0000-0000-0000-000000000001", "token_id": id,
1203                   "label": "kaniop-radius-x", "expiry": null, "issued_at": issued,
1204                   "purpose": "readonly"})
1205        };
1206        let tokens = json!([token(TOKEN_A), token(TOKEN_B)]).to_string();
1207        let account_path = format!("/v1/service_account/{ACCOUNT}");
1208        let tokens_path = format!("{account_path}/_api_token");
1209        kanidm_mock::start(
1210            move |req: &Request| match (req.method.as_str(), req.path.as_str()) {
1211                ("GET", path) if path == account_path => (200, entry.clone()),
1212                ("GET", "/v1/group/idm_radius_servers/_attr/member") => {
1213                    (200, json!([ACCOUNT]).to_string())
1214                }
1215                ("GET", path) if path == tokens_path => (200, tokens.clone()),
1216                ("DELETE", path) if path.starts_with(&tokens_path) => (200, "null".into()),
1217                _ => (500, "{}".into()),
1218            },
1219        )
1220        .await
1221    }
1222
1223    fn uris(requests: &[Recorded]) -> Vec<String> {
1224        requests
1225            .iter()
1226            .map(|r| format!("{} {}", r.method, r.uri.split('?').next().unwrap()))
1227            .collect()
1228    }
1229
1230    /// The Deployment is applied, then the Service apply fails. Revoking the
1231    /// unused token and collecting its Secret do not depend on the Service,
1232    /// so they still run; the Service failure is reported afterwards.
1233    #[tokio::test]
1234    async fn service_apply_failure_still_revokes_and_collects() {
1235        let idm = kanidm_with_two_tokens().await;
1236        let mut replies = input_secrets().to_vec();
1237        replies.extend(observation_replies());
1238        replies.extend([
1239            // New config Secret (the rendered content differs from SECRET_A).
1240            api_status(404, "NotFound"),
1241            (201, config_secret("campus-new", TOKEN_A)),
1242            // Deployment apply.
1243            (200, to_json(&running_deployment("1"))),
1244            (200, to_json(&running_deployment("2"))),
1245            // Service apply fails.
1246            api_status(404, "NotFound"),
1247            api_status(500, "InternalError"),
1248        ]);
1249        replies.extend(observation_replies());
1250        replies.extend([
1251            (200, config_secret(SECRET_B, TOKEN_B)),
1252            (200, config_secret(SECRET_B, TOKEN_B)),
1253        ]);
1254        let (client, recorder) = mock_client(replies);
1255        let idm_client = idm.client.clone();
1256        let mut state = PassState::default();
1257        let mut report = StatusReport::default();
1258        let result = run_pass(
1259            &radius(),
1260            &kanidm(),
1261            client,
1262            &idm.metrics,
1263            move || async move { Ok(idm_client) },
1264            &mut report,
1265            &mut state,
1266        )
1267        .await;
1268
1269        match result {
1270            Err(Failure::Transient { condition, .. }) => {
1271                assert_eq!(condition, ConditionKind::Exposed)
1272            }
1273            other => panic!("expected the Service failure, got {other:?}"),
1274        }
1275        let revoked: Vec<String> = idm
1276            .writes()
1277            .into_iter()
1278            .map(|r| format!("{} {}", r.method, r.path))
1279            .collect();
1280        assert_eq!(
1281            revoked,
1282            [format!(
1283                "DELETE /v1/service_account/{ACCOUNT}/_api_token/{TOKEN_B}"
1284            )],
1285            "only the unused token is revoked"
1286        );
1287        let requests = uris(&recorder.await.unwrap());
1288        assert_eq!(
1289            requests.last().map(String::as_str),
1290            Some(format!("DELETE /api/v1/namespaces/network/secrets/{SECRET_B}").as_str()),
1291            "{requests:#?}"
1292        );
1293        assert!(state.changed);
1294        assert!(
1295            state.rotation_applied,
1296            "a block after the apply must not raise TokenNearExpiry"
1297        );
1298        assert_eq!(
1299            state
1300                .deployment
1301                .and_then(|d| d.metadata.resource_version)
1302                .as_deref(),
1303            Some("2")
1304        );
1305        assert_eq!(
1306            report.exposed.map(|v| v.reason).as_deref(),
1307            Some("TransientError")
1308        );
1309        assert_eq!(report.tokens.map(|t| t.retiring), Some(vec![]));
1310    }
1311
1312    /// A Kanidm image the RADIUS image cannot be derived from.
1313    fn kanidm_unresolvable() -> Kanidm {
1314        let mut kanidm = kanidm();
1315        kanidm.spec.image = "kanidm/server:latest".into();
1316        kanidm
1317    }
1318
1319    /// Image unresolved but a Deployment runs: the pass keeps that image and
1320    /// goes on (observe, plan, apply, revoke), reporting the block on
1321    /// `ConfigurationReady` so it stays visible.
1322    #[tokio::test]
1323    async fn unresolved_image_reuses_the_running_image_and_keeps_rotating() {
1324        let idm = kanidm_with_two_tokens().await;
1325        let mut replies = input_secrets().to_vec();
1326        replies.extend(observation_replies());
1327        let service: Service = serde_json::from_value(json!({
1328            "apiVersion": "v1", "kind": "Service",
1329            "metadata": {"name": "campus", "namespace": "network", "resourceVersion": "1",
1330                         "ownerReferences": owner_refs()},
1331            "spec": {"type": "ClusterIP", "clusterIP": "10.0.0.10"}
1332        }))
1333        .unwrap();
1334        replies.extend([
1335            api_status(404, "NotFound"),
1336            (201, config_secret("campus-new", TOKEN_A)),
1337            (200, to_json(&running_deployment("1"))),
1338            (200, to_json(&running_deployment("2"))),
1339            api_status(404, "NotFound"),
1340            (200, to_json(&service)),
1341        ]);
1342        replies.extend(observation_replies());
1343        replies.extend([
1344            (200, config_secret(SECRET_B, TOKEN_B)),
1345            (200, config_secret(SECRET_B, TOKEN_B)),
1346        ]);
1347        let (client, recorder) = mock_client(replies);
1348        let idm_client = idm.client.clone();
1349        let mut state = PassState {
1350            deployment: Some(running_deployment("1")),
1351            ..Default::default()
1352        };
1353        let mut report = StatusReport::default();
1354        let result = run_pass(
1355            &radius(),
1356            &kanidm_unresolvable(),
1357            client,
1358            &idm.metrics,
1359            move || async move { Ok(idm_client) },
1360            &mut report,
1361            &mut state,
1362        )
1363        .await;
1364
1365        assert!(result.is_ok(), "{result:?}");
1366        assert_eq!(report.configuration.truth, Truth::False);
1367        assert_eq!(report.configuration.reason, "ImageUnresolved");
1368        assert_eq!(report.identity.truth, Truth::True);
1369        let requests = recorder.await.unwrap();
1370        let apply = requests
1371            .iter()
1372            .find(|r| r.method == "PATCH" && r.uri.contains("/deployments/campus"))
1373            .expect("Deployment applied");
1374        assert_eq!(
1375            apply.json()["spec"]["template"]["spec"]["containers"][0]["image"],
1376            RUNNING_IMAGE
1377        );
1378        assert_eq!(
1379            idm.writes().len(),
1380            1,
1381            "the unused token is still revoked: {:?}",
1382            idm.writes()
1383        );
1384    }
1385
1386    /// Image unresolved and nothing runs yet: blocked before any mutation,
1387    /// as before.
1388    #[tokio::test]
1389    async fn unresolved_image_without_a_deployment_blocks_before_kanidm() {
1390        let (client, recorder) = mock_client(input_secrets().to_vec());
1391        let (metrics, _provider) = metrics();
1392        let mut state = PassState::default();
1393        let mut report = StatusReport::default();
1394        let result = run_pass(
1395            &radius(),
1396            &kanidm_unresolvable(),
1397            client,
1398            &metrics,
1399            || async { panic!("Kanidm must not be contacted without an image") },
1400            &mut report,
1401            &mut state,
1402        )
1403        .await;
1404        assert!(
1405            matches!(result, Err(Failure::Block(RadiusBlock::Image(_)))),
1406            "{result:?}"
1407        );
1408        assert!(!state.changed);
1409        assert!(!state.rotation_applied);
1410        assert_eq!(recorder_requests(recorder).await.len(), 2);
1411    }
1412
1413    fn at(seconds: i64) -> OffsetDateTime {
1414        OffsetDateTime::from_unix_timestamp(seconds).unwrap()
1415    }
1416
1417    fn observed_facts(expiry: Option<OffsetDateTime>) -> StatusReport {
1418        StatusReport {
1419            tokens: Some(TokenFacts {
1420                current: Some((Uuid::parse_str(TOKEN_A).unwrap(), at(0))),
1421                current_expiry: expiry,
1422                ..Default::default()
1423            }),
1424            ..Default::default()
1425        }
1426    }
1427
1428    #[test]
1429    fn running_token_expiry_prefers_the_observed_token() {
1430        let rotation = TokenPolicy::from_spec(Some(&radius().spec.token_rotation));
1431        let observed = observed_facts(Some(at(1_000_000)));
1432        assert_eq!(
1433            running_token_expiry(&observed, None, &rotation),
1434            Some((TOKEN_A.to_string(), at(1_000_000)))
1435        );
1436        // Observed without expiry: it never lapses.
1437        assert_eq!(
1438            running_token_expiry(&observed_facts(None), None, &rotation),
1439            None
1440        );
1441    }
1442
1443    #[test]
1444    fn running_token_expiry_estimates_from_status_when_not_observed() {
1445        let previous = KanidmRadiusServerStatus {
1446            token_id: Some(TOKEN_A.into()),
1447            token_issued_at: kube_time(at(1_000)),
1448            ..Default::default()
1449        };
1450        let enabled = TokenPolicy::from_spec(Some(&kaniop_operator::crd::SecretRotation {
1451            enabled: true,
1452            period_days: 30,
1453        }));
1454        assert_eq!(
1455            running_token_expiry(&StatusReport::default(), Some(&previous), &enabled),
1456            Some((TOKEN_A.to_string(), at(1_000) + time::Duration::days(60)))
1457        );
1458        // Rotation disabled: minted without expiry.
1459        let disabled = TokenPolicy::from_spec(None);
1460        assert_eq!(
1461            running_token_expiry(&StatusReport::default(), Some(&previous), &disabled),
1462            None
1463        );
1464        assert_eq!(
1465            running_token_expiry(&StatusReport::default(), None, &enabled),
1466            None
1467        );
1468    }
1469
1470    #[test]
1471    fn near_expiry_note_within_twice_the_safety_margin() {
1472        let block = RadiusBlock::NamespaceNotWatched {
1473            kanidm: "identity/idm".into(),
1474        };
1475        let expiry = at(10_000_000);
1476        let window = SAFETY_MARGIN * 2;
1477        let running = || Some((TOKEN_A.to_string(), expiry));
1478        assert_eq!(
1479            token_near_expiry_note(
1480                &block,
1481                running(),
1482                expiry - window - time::Duration::seconds(1)
1483            ),
1484            None
1485        );
1486        let note = token_near_expiry_note(&block, running(), expiry - window).unwrap();
1487        assert!(note.contains(TOKEN_A), "{note}");
1488        assert!(note.contains("NamespaceNotWatched"), "{note}");
1489        assert!(
1490            note.contains(&kube_time(expiry).unwrap().0.to_string()),
1491            "{note}"
1492        );
1493        // Already expired still warns; nothing running never does.
1494        assert!(token_near_expiry_note(&block, running(), expiry + window).is_some());
1495        assert_eq!(token_near_expiry_note(&block, None, expiry), None);
1496    }
1497
1498    async fn recorder_requests(
1499        recorder: tokio::task::JoinHandle<Vec<crate::reconcile::kube_mock::Recorded>>,
1500    ) -> Vec<String> {
1501        // The mock ends when the client (moved into run_pass) is dropped.
1502        recorder
1503            .await
1504            .unwrap()
1505            .into_iter()
1506            .map(|r| format!("{} {}", r.method, r.uri.split('?').next().unwrap()))
1507            .collect()
1508    }
1509
1510    #[test]
1511    fn kanidm_uri_prefers_origin_over_domain() {
1512        let mut kanidm = kanidm();
1513        assert_eq!(kanidm_uri(&kanidm), "https://idm.example.com");
1514        kanidm.spec.origin = Some("https://idm.example.com:8443".into());
1515        assert_eq!(kanidm_uri(&kanidm), "https://idm.example.com:8443");
1516    }
1517
1518    #[test]
1519    fn finalizer_patches_are_guarded() {
1520        let mut radius = radius();
1521        assert_eq!(
1522            add_finalizer_patch(&radius),
1523            json!([
1524                {"op": "test", "path": "/metadata/finalizers", "value": null},
1525                {"op": "add", "path": "/metadata/finalizers", "value": [RADIUS_FINALIZER]},
1526            ])
1527        );
1528        assert_eq!(remove_finalizer_patch(&radius), None);
1529        radius.metadata.finalizers = Some(vec!["other/finalizer".into(), RADIUS_FINALIZER.into()]);
1530        assert_eq!(
1531            remove_finalizer_patch(&radius),
1532            Some(json!([
1533                {"op": "test", "path": "/metadata/finalizers/1", "value": RADIUS_FINALIZER},
1534                {"op": "remove", "path": "/metadata/finalizers/1"},
1535            ]))
1536        );
1537        radius.metadata.finalizers = Some(vec!["other/finalizer".into()]);
1538        assert_eq!(
1539            add_finalizer_patch(&radius)[1],
1540            json!({"op": "add", "path": "/metadata/finalizers/-", "value": RADIUS_FINALIZER})
1541        );
1542    }
1543}