Skip to main content

kaniop_operator/controller/
mod.rs

1pub mod context;
2pub mod kanidm;
3
4use self::{context::Context, kanidm::KanidmClients};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics;
8use crate::prometheus_exporter;
9
10use kaniop_k8s_util::error::{Error, Result};
11
12use kaniop_k8s_util::types::short_type_name;
13
14use std::fmt::Debug;
15use std::sync::Arc;
16
17use futures::channel::mpsc;
18use futures::future::BoxFuture;
19use futures::{FutureExt, StreamExt};
20use k8s_openapi::api::core::v1::Namespace;
21use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
22use kube::Resource;
23use kube::api::{Api, ListParams, PartialObjectMeta, ResourceExt};
24use kube::client::Client;
25use kube::runtime::controller::Action;
26use kube::runtime::events::Recorder;
27use kube::runtime::reflector::store::Writer;
28use kube::runtime::reflector::{self, Lookup, ReflectHandle, Store};
29use kube::runtime::{WatchStreamExt, metadata_watcher, watcher};
30use serde::de::DeserializeOwned;
31use tokio::sync::RwLock;
32use tokio::time::Duration;
33use tracing::{debug, error, info, trace};
34
35use std::sync::OnceLock;
36
37const DEFAULT_IDM_RECONCILE_INTERVAL_SECS: u64 = 60;
38const DEFAULT_CLUSTER_DOMAIN: &str = "cluster.local";
39const DEFAULT_BACKUP_DISCOVERY_SCAN_INTERVAL_SECS: u64 = 300;
40const DEFAULT_BACKUP_DISCOVERY_STALE_SECS: u64 = 300;
41
42static IDM_RECONCILE_INTERVAL: OnceLock<Duration> = OnceLock::new();
43static CLUSTER_DOMAIN: OnceLock<String> = OnceLock::new();
44static BACKUP_DISCOVERY_SCAN_INTERVAL: OnceLock<Duration> = OnceLock::new();
45static BACKUP_DISCOVERY_STALE_THRESHOLD: OnceLock<Duration> = OnceLock::new();
46static BACKUP_JOB_VOLUME_SIZE: OnceLock<Quantity> = OnceLock::new();
47
48pub fn set_idm_reconcile_interval(duration: Duration) {
49    let _ = IDM_RECONCILE_INTERVAL.set(duration);
50}
51
52pub fn idm_reconcile_interval() -> Duration {
53    *IDM_RECONCILE_INTERVAL.get_or_init(|| Duration::from_secs(DEFAULT_IDM_RECONCILE_INTERVAL_SECS))
54}
55
56pub fn set_cluster_domain(domain: String) {
57    let _ = CLUSTER_DOMAIN.set(domain);
58}
59
60pub fn cluster_domain() -> &'static str {
61    CLUSTER_DOMAIN.get_or_init(|| DEFAULT_CLUSTER_DOMAIN.to_string())
62}
63
64pub fn set_backup_discovery_scan_interval(duration: Duration) {
65    let _ = BACKUP_DISCOVERY_SCAN_INTERVAL.set(duration);
66}
67
68pub fn backup_discovery_scan_interval() -> Duration {
69    *BACKUP_DISCOVERY_SCAN_INTERVAL
70        .get_or_init(|| Duration::from_secs(DEFAULT_BACKUP_DISCOVERY_SCAN_INTERVAL_SECS))
71}
72
73pub fn set_backup_discovery_stale_threshold(duration: Duration) {
74    let _ = BACKUP_DISCOVERY_STALE_THRESHOLD.set(duration);
75}
76
77pub fn backup_discovery_stale_threshold() -> Duration {
78    *BACKUP_DISCOVERY_STALE_THRESHOLD
79        .get_or_init(|| Duration::from_secs(DEFAULT_BACKUP_DISCOVERY_STALE_SECS))
80}
81
82pub fn set_backup_job_volume_size(quantity: Quantity) {
83    let _ = BACKUP_JOB_VOLUME_SIZE.set(quantity);
84}
85
86pub fn backup_job_volume_size() -> Quantity {
87    BACKUP_JOB_VOLUME_SIZE
88        .get_or_init(|| Quantity("10Gi".to_string()))
89        .clone()
90}
91
92pub const SUBSCRIBE_BUFFER_SIZE: usize = 256;
93pub const RELOAD_BUFFER_SIZE: usize = 16;
94pub const NAME_LABEL: &str = "app.kubernetes.io/name";
95pub const INSTANCE_LABEL: &str = "app.kubernetes.io/instance";
96pub const MANAGED_BY_LABEL: &str = "app.kubernetes.io/managed-by";
97
98pub type ControllerId = &'static str;
99
100/// State shared between the controller and the web server
101// Kanidm defined as a generic because it causes a cycle dependency with the kaniop_kanidm crate
102#[derive(Clone)]
103pub struct State {
104    /// Metrics
105    metrics: Arc<metrics::Metrics>,
106    /// Shared Kanidm cache clients with the ability to manage users and their groups
107    idm_clients: Arc<RwLock<KanidmClients>>,
108    /// Shared Kanidm cache clients with the ability to manage the operation of Kanidm as a
109    /// database and service
110    system_clients: Arc<RwLock<KanidmClients>>,
111    /// Cache for Namespace resources
112    pub namespace_store: Store<Namespace>,
113    /// Cache for Kanidm resources
114    pub kanidm_store: Store<Kanidm>,
115    /// Kubernetes client
116    pub client: Option<Client>,
117}
118
119/// Shared state for a resource stream
120pub struct ResourceReflector<K>
121where
122    K: Resource + Lookup + Clone + 'static,
123    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
124{
125    pub store: Store<K>,
126    pub writer: Writer<K>,
127    pub subscriber: ReflectHandle<K>,
128}
129
130/// State wrapper around the controller outputs for the web server
131impl State {
132    pub fn new(
133        metrics: metrics::Metrics,
134        namespace_store: Store<Namespace>,
135        kanidm_store: Store<Kanidm>,
136        client: Option<Client>,
137    ) -> Self {
138        Self {
139            metrics: Arc::new(metrics),
140            idm_clients: Arc::default(),
141            system_clients: Arc::default(),
142            namespace_store,
143            kanidm_store,
144            client,
145        }
146    }
147
148    /// Metrics getter
149    pub fn metrics(&self) -> Result<String> {
150        prometheus_exporter::format_prometheus_metrics("kaniop").map_err(|e| {
151            Error::FormattingError(format!("failed to export metrics: {}", e), std::fmt::Error)
152        })
153    }
154
155    /// Create a Controller Context that can update State
156    pub fn to_context<K>(&self, client: Client, controller_id: ControllerId) -> Context<K>
157    where
158        K: Resource + Lookup + Clone + 'static,
159        <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
160    {
161        Context::new(
162            controller_id,
163            client.clone(),
164            self.metrics
165                .controllers
166                .get(controller_id)
167                .expect("all CONTROLLER_IDs have to be registered")
168                .clone(),
169            Recorder::new(client.clone(), controller_id.into()),
170            self.idm_clients.clone(),
171            self.system_clients.clone(),
172            self.namespace_store.clone(),
173            self.kanidm_store.clone(),
174        )
175    }
176}
177
178pub async fn check_api_queryable<K>(client: Client) -> Api<K>
179where
180    K: Resource + Clone + DeserializeOwned + Debug,
181    <K as Resource>::DynamicType: Default,
182{
183    let api = Api::<K>::all(client.clone());
184    if let Err(e) = api.list(&ListParams::default().limit(1)).await {
185        error!(
186            "{} is not queryable; {e:?}. Check controller permissions",
187            short_type_name::<K>().unwrap_or("Unknown resource"),
188        );
189        std::process::exit(1);
190    }
191    api
192}
193
194pub async fn check_api_queryable_optional<K>(client: Client) -> Option<Api<K>>
195where
196    K: Resource + Clone + DeserializeOwned + Debug,
197    <K as Resource>::DynamicType: Default,
198{
199    let api = Api::<K>::all(client.clone());
200    if let Err(e) = api.list(&ListParams::default().limit(1)).await {
201        info!(
202            "{} is not queryable (optional resource); {e:?}. Skipping optional resource support.",
203            short_type_name::<K>().unwrap_or("Unknown resource"),
204        );
205        None
206    } else {
207        Some(api)
208    }
209}
210
211pub fn create_subscriber<K>(buffer_size: usize) -> ResourceReflector<K>
212where
213    K: Resource + Lookup + Clone + 'static,
214    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
215    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
216{
217    let (store, writer) = reflector::store_shared::<K>(buffer_size);
218    let subscriber = writer
219        .subscribe()
220        .expect("subscribers can only be created from shared stores");
221
222    ResourceReflector {
223        store,
224        writer,
225        subscriber,
226    }
227}
228
229fn create_generic_watcher<K, W, T, S, StreamT>(
230    api: Api<K>,
231    writer: Writer<W>,
232    reload_tx: mpsc::Sender<()>,
233    controller_id: ControllerId,
234    ctx: Arc<Context<T>>,
235    stream_fn: S,
236) -> BoxFuture<'static, ()>
237where
238    K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
239    W: Resource + ResourceExt + Lookup + Clone + Debug + Send + Sync + 'static,
240    <W as Resource>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
241    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
242    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
243    <W as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
244    T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
245    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
246    S: Fn(Api<K>, watcher::Config) -> StreamT + 'static,
247    StreamT: futures::Stream<Item = Result<watcher::Event<W>, kube::runtime::watcher::Error>>
248        + Send
249        + 'static,
250{
251    let resource_name = short_type_name::<K>().unwrap_or("Unknown");
252
253    stream_fn(
254        api,
255        watcher::Config::default().labels(&format!("{MANAGED_BY_LABEL}=kaniop-{controller_id}")),
256    )
257    .default_backoff()
258    .reflect_shared(writer)
259    .for_each(move |res| {
260        let mut reload_tx_clone = reload_tx.clone();
261        let ctx = ctx.clone();
262        async move {
263            match res {
264                Ok(event) => {
265                    trace!(?event, "watched event");
266                    match event {
267                        watcher::Event::Delete(d) => {
268                            debug!(
269                                namespace = ResourceExt::namespace(&d).unwrap(),
270                                name = d.name_any(),
271                                "delete event for {resource_name} trigger reconcile"
272                            );
273
274                            // TODO: remove for each trigger on delete logic when
275                            // (dispatch delete events issue)[https://github.com/kube-rs/kube/issues/1590]
276                            // is solved
277                            let _ignore_errors = reload_tx_clone
278                                .try_send(())
279                                .map_err(|e| error!(%e, "failed to trigger reconcile on delete"));
280                            ctx.metrics
281                                .triggered_inc(metrics::Action::Delete, resource_name);
282                        }
283                        watcher::Event::Apply(d) => {
284                            debug!(
285                                namespace = ResourceExt::namespace(&d).unwrap(),
286                                name = d.name_any(),
287                                "apply event for {resource_name} trigger reconcile"
288                            );
289                            ctx.metrics
290                                .triggered_inc(metrics::Action::Apply, resource_name);
291                        }
292                        _ => {}
293                    }
294                }
295                Err(e) => {
296                    error!(%e, "unexpected error when watching {resource_name}");
297                    ctx.metrics.watch_operations_failed_inc();
298                }
299            }
300        }
301    })
302    .boxed()
303}
304
305pub fn create_watcher<K, T>(
306    api: Api<K>,
307    writer: Writer<K>,
308    reload_tx: mpsc::Sender<()>,
309    controller_id: ControllerId,
310    ctx: Arc<Context<T>>,
311) -> BoxFuture<'static, ()>
312where
313    K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
314    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
315    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
316    T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
317    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
318{
319    create_generic_watcher::<K, K, T, _, _>(api, writer, reload_tx, controller_id, ctx, watcher)
320}
321
322pub fn create_metadata_watcher<K, T>(
323    api: Api<K>,
324    writer: Writer<PartialObjectMeta<K>>,
325    reload_tx: mpsc::Sender<()>,
326    controller_id: ControllerId,
327    ctx: Arc<Context<T>>,
328) -> BoxFuture<'static, ()>
329where
330    K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
331    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
332    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
333    T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
334    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
335{
336    create_generic_watcher::<K, PartialObjectMeta<K>, T, _, _>(
337        api,
338        writer,
339        reload_tx,
340        controller_id,
341        ctx,
342        metadata_watcher,
343    )
344}
345
346pub fn error_policy<K>(_obj: Arc<K>, _error: &Error, _ctx: Arc<Context<K>>) -> Action
347where
348    K: Resource + Lookup + Clone + 'static,
349    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
350{
351    tracing::warn!(
352        "error_policy called unexpectedly - all controllers should use backoff_reconciler! macro"
353    );
354    Action::requeue(Duration::from_secs(300))
355}
356
357#[macro_export]
358macro_rules! backoff_reconciler {
359    ($inner_reconciler:ident) => {
360        |obj, ctx| async move {
361            use $crate::controller::context::BackoffContext;
362            let _active_guard = ctx.metrics().active_reconcile_inc();
363            match $inner_reconciler(obj.clone(), ctx.clone()).await {
364                Ok((action, changed)) => {
365                    ctx.reset_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
366                        .await;
367                    if changed {
368                        ctx.metrics().reconcile_outcome_record($crate::metrics::KANIDM_OUTCOME_CHANGED);
369                    } else {
370                        ctx.metrics().reconcile_outcome_record($crate::metrics::KANIDM_OUTCOME_UNCHANGED);
371                    }
372                    Ok(action)
373                }
374                Err(error) => {
375                    // safe unwrap: all resources in the operator are namespace scoped resources
376                    let namespace = kube::ResourceExt::namespace(obj.as_ref()).unwrap();
377                    let name = kube::ResourceExt::name_any(obj.as_ref());
378                    tracing::error!(%namespace, %name, %error, "failed reconciliation");
379                    ctx.metrics().reconcile_failure_inc();
380                    ctx.metrics().reconcile_outcome_record($crate::metrics::KANIDM_OUTCOME_ERROR);
381                    let backoff_duration = ctx
382                        .get_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
383                        .await;
384                    tracing::trace!(%namespace, %name, "backoff duration: {backoff_duration:?}");
385                    Ok(kube::runtime::controller::Action::requeue(backoff_duration))
386                }
387            }
388        }
389    };
390}