kaniop_operator/controller/
mod.rs

1pub mod context;
2pub mod kanidm;
3
4use self::{context::Context, kanidm::KanidmClients};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics;
8use kaniop_k8s_util::error::{Error, Result};
9
10use kaniop_k8s_util::types::short_type_name;
11
12use std::fmt::Debug;
13use std::sync::Arc;
14
15use futures::channel::mpsc;
16use futures::future::BoxFuture;
17use futures::{FutureExt, StreamExt};
18use k8s_openapi::api::core::v1::Namespace;
19use kube::Resource;
20use kube::api::{Api, ListParams, PartialObjectMeta, ResourceExt};
21use kube::client::Client;
22use kube::runtime::controller::Action;
23use kube::runtime::events::Recorder;
24use kube::runtime::reflector::store::Writer;
25use kube::runtime::reflector::{self, Lookup, ReflectHandle, Store};
26use kube::runtime::{WatchStreamExt, metadata_watcher, watcher};
27use prometheus_client::registry::Registry;
28use serde::de::DeserializeOwned;
29use tokio::sync::RwLock;
30use tokio::time::Duration;
31use tracing::{debug, error, trace};
32
33pub const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(60);
34pub const SUBSCRIBE_BUFFER_SIZE: usize = 256;
35pub const RELOAD_BUFFER_SIZE: usize = 16;
36pub const NAME_LABEL: &str = "app.kubernetes.io/name";
37pub const INSTANCE_LABEL: &str = "app.kubernetes.io/instance";
38pub const MANAGED_BY_LABEL: &str = "app.kubernetes.io/managed-by";
39
40pub type ControllerId = &'static str;
41
42/// State shared between the controller and the web server
43// Kanidm defined as a generic because it causes a cycle dependency with the kaniop_kanidm crate
44#[derive(Clone)]
45pub struct State {
46    /// Metrics
47    metrics: Arc<metrics::Metrics>,
48    /// Shared Kanidm cache clients with the ability to manage users and their groups
49    idm_clients: Arc<RwLock<KanidmClients>>,
50    /// Shared Kanidm cache clients with the ability to manage the operation of Kanidm as a
51    /// database and service
52    system_clients: Arc<RwLock<KanidmClients>>,
53    /// Cache for Namespace resources
54    pub namespace_store: Store<Namespace>,
55    /// Cache for Kanidm resources
56    pub kanidm_store: Store<Kanidm>,
57}
58
59/// Shared state for a resource stream
60pub struct ResourceReflector<K>
61where
62    K: Resource + Lookup + Clone + 'static,
63    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
64{
65    pub store: Store<K>,
66    pub writer: Writer<K>,
67    pub subscriber: ReflectHandle<K>,
68}
69
70/// State wrapper around the controller outputs for the web server
71impl State {
72    pub fn new(
73        registry: Registry,
74        controller_names: &[&'static str],
75        namespace_store: Store<Namespace>,
76        kanidm_store: Store<Kanidm>,
77    ) -> Self {
78        Self {
79            metrics: Arc::new(metrics::Metrics::new(registry, controller_names)),
80            idm_clients: Arc::default(),
81            system_clients: Arc::default(),
82            namespace_store,
83            kanidm_store,
84        }
85    }
86
87    /// Metrics getter
88    pub fn metrics(&self) -> Result<String> {
89        let mut buffer = String::new();
90        let registry = &*self.metrics.registry;
91        prometheus_client::encoding::text::encode(&mut buffer, registry)
92            .map_err(|e| Error::FormattingError("failed to encode metrics".to_string(), e))?;
93        Ok(buffer)
94    }
95
96    /// Create a Controller Context that can update State
97    pub fn to_context<K>(&self, client: Client, controller_id: ControllerId) -> Context<K>
98    where
99        K: Resource + Lookup + Clone + 'static,
100        <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
101    {
102        Context::new(
103            controller_id,
104            client.clone(),
105            self.metrics
106                .controllers
107                .get(controller_id)
108                .expect("all CONTROLLER_IDs have to be registered")
109                .clone(),
110            Recorder::new(client.clone(), controller_id.into()),
111            self.idm_clients.clone(),
112            self.system_clients.clone(),
113            self.namespace_store.clone(),
114            self.kanidm_store.clone(),
115        )
116    }
117}
118
119pub async fn check_api_queryable<K>(client: Client) -> Api<K>
120where
121    K: Resource + Clone + DeserializeOwned + Debug,
122    <K as Resource>::DynamicType: Default,
123{
124    let api = Api::<K>::all(client.clone());
125    if let Err(e) = api.list(&ListParams::default().limit(1)).await {
126        error!(
127            "{} is not queryable; {e:?}. Check controller permissions",
128            short_type_name::<K>().unwrap_or("Unknown resource"),
129        );
130        std::process::exit(1);
131    }
132    api
133}
134
135pub fn create_subscriber<K>(buffer_size: usize) -> ResourceReflector<K>
136where
137    K: Resource + Lookup + Clone + 'static,
138    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
139    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
140{
141    let (store, writer) = reflector::store_shared::<K>(buffer_size);
142    let subscriber = writer
143        .subscribe()
144        .expect("subscribers can only be created from shared stores");
145
146    ResourceReflector {
147        store,
148        writer,
149        subscriber,
150    }
151}
152
153fn create_generic_watcher<K, W, T, S, StreamT>(
154    api: Api<K>,
155    writer: Writer<W>,
156    reload_tx: mpsc::Sender<()>,
157    controller_id: ControllerId,
158    ctx: Arc<Context<T>>,
159    stream_fn: S,
160) -> BoxFuture<'static, ()>
161where
162    K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
163    W: Resource + ResourceExt + Lookup + Clone + Debug + Send + Sync + 'static,
164    <W as Resource>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
165    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
166    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
167    <W as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
168    T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
169    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
170    S: Fn(Api<K>, watcher::Config) -> StreamT + 'static,
171    StreamT: futures::Stream<Item = Result<watcher::Event<W>, kube::runtime::watcher::Error>>
172        + Send
173        + 'static,
174{
175    let resource_name = short_type_name::<K>().unwrap_or("Unknown");
176
177    stream_fn(
178        api,
179        watcher::Config::default().labels(&format!("{MANAGED_BY_LABEL}=kaniop-{controller_id}")),
180    )
181    .default_backoff()
182    .reflect_shared(writer)
183    .for_each(move |res| {
184        let mut reload_tx_clone = reload_tx.clone();
185        let ctx = ctx.clone();
186        async move {
187            match res {
188                Ok(event) => {
189                    trace!(msg = "watched event", ?event);
190                    match event {
191                        watcher::Event::Delete(d) => {
192                            debug!(
193                                msg = format!("delete event for {resource_name} trigger reconcile"),
194                                namespace = ResourceExt::namespace(&d).unwrap(),
195                                name = d.name_any()
196                            );
197
198                            // TODO: remove for each trigger on delete logic when
199                            // (dispatch delete events issue)[https://github.com/kube-rs/kube/issues/1590]
200                            // is solved
201                            let _ignore_errors = reload_tx_clone.try_send(()).map_err(
202                                |e| error!(msg = "failed to trigger reconcile on delete", %e),
203                            );
204                            ctx.metrics
205                                .triggered_inc(metrics::Action::Delete, resource_name);
206                        }
207                        watcher::Event::Apply(d) => {
208                            debug!(
209                                msg = format!("apply event for {resource_name} trigger reconcile"),
210                                namespace = ResourceExt::namespace(&d).unwrap(),
211                                name = d.name_any()
212                            );
213                            ctx.metrics
214                                .triggered_inc(metrics::Action::Apply, resource_name);
215                        }
216                        _ => {}
217                    }
218                }
219                Err(e) => {
220                    error!(msg = format!("unexpected error when watching {resource_name}"), %e);
221                    ctx.metrics.watch_operations_failed_inc();
222                }
223            }
224        }
225    })
226    .boxed()
227}
228
229pub fn create_watcher<K, T>(
230    api: Api<K>,
231    writer: Writer<K>,
232    reload_tx: mpsc::Sender<()>,
233    controller_id: ControllerId,
234    ctx: Arc<Context<T>>,
235) -> BoxFuture<'static, ()>
236where
237    K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
238    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
239    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
240    T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
241    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
242{
243    create_generic_watcher::<K, K, T, _, _>(api, writer, reload_tx, controller_id, ctx, watcher)
244}
245
246pub fn create_metadata_watcher<K, T>(
247    api: Api<K>,
248    writer: Writer<PartialObjectMeta<K>>,
249    reload_tx: mpsc::Sender<()>,
250    controller_id: ControllerId,
251    ctx: Arc<Context<T>>,
252) -> BoxFuture<'static, ()>
253where
254    K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
255    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
256    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
257    T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
258    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
259{
260    create_generic_watcher::<K, PartialObjectMeta<K>, T, _, _>(
261        api,
262        writer,
263        reload_tx,
264        controller_id,
265        ctx,
266        metadata_watcher,
267    )
268}
269
270pub fn error_policy<K>(_obj: Arc<K>, _error: &Error, _ctx: Arc<Context<K>>) -> Action
271where
272    K: Resource + Lookup + Clone + 'static,
273    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
274{
275    unreachable!("Handle in backoff_reconciler macro")
276}
277
278#[macro_export]
279macro_rules! backoff_reconciler {
280    ($inner_reconciler:ident) => {
281        |obj, ctx| async move {
282            use $crate::controller::context::BackoffContext;
283            match $inner_reconciler(obj.clone(), ctx.clone()).await {
284                Ok(action) => {
285                    ctx.reset_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
286                        .await;
287                    Ok(action)
288                }
289                Err(error) => {
290                    // safe unwrap: all resources in the operator are namespace scoped resources
291                    let namespace = kube::ResourceExt::namespace(obj.as_ref()).unwrap();
292                    let name = kube::ResourceExt::name_any(obj.as_ref());
293                    tracing::error!(msg = "failed reconciliation", %namespace, %name, %error);
294                    ctx.metrics().reconcile_failure_inc();
295                    let backoff_duration = ctx
296                        .get_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
297                        .await;
298                    tracing::trace!(
299                        msg = format!("backoff duration: {backoff_duration:?}"),
300                        %namespace,
301                        %name,
302                    );
303                    Ok(kube::runtime::controller::Action::requeue(backoff_duration))
304                }
305            }
306        }
307    };
308}