kaniop_operator/controller/
mod.rs1pub mod context;
2pub mod kanidm;
3
4use self::{context::Context, kanidm::KanidmClients};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics;
8use crate::prometheus_exporter;
9
10use kaniop_k8s_util::error::{Error, Result};
11
12use kaniop_k8s_util::types::short_type_name;
13
14use std::fmt::Debug;
15use std::sync::Arc;
16
17use futures::channel::mpsc;
18use futures::future::BoxFuture;
19use futures::{FutureExt, StreamExt};
20use k8s_openapi::api::core::v1::Namespace;
21use kube::Resource;
22use kube::api::{Api, ListParams, PartialObjectMeta, ResourceExt};
23use kube::client::Client;
24use kube::runtime::controller::Action;
25use kube::runtime::events::Recorder;
26use kube::runtime::reflector::store::Writer;
27use kube::runtime::reflector::{self, Lookup, ReflectHandle, Store};
28use kube::runtime::{WatchStreamExt, metadata_watcher, watcher};
29use serde::de::DeserializeOwned;
30use tokio::sync::RwLock;
31use tokio::time::Duration;
32use tracing::{debug, error, trace};
33
34use std::sync::OnceLock;
35
36const DEFAULT_IDM_RECONCILE_INTERVAL_SECS: u64 = 60;
37
38static IDM_RECONCILE_INTERVAL: OnceLock<Duration> = OnceLock::new();
39
40pub fn set_idm_reconcile_interval(duration: Duration) {
41 let _ = IDM_RECONCILE_INTERVAL.set(duration);
42}
43
44pub fn idm_reconcile_interval() -> Duration {
45 *IDM_RECONCILE_INTERVAL.get_or_init(|| Duration::from_secs(DEFAULT_IDM_RECONCILE_INTERVAL_SECS))
46}
47pub const SUBSCRIBE_BUFFER_SIZE: usize = 256;
48pub const RELOAD_BUFFER_SIZE: usize = 16;
49pub const NAME_LABEL: &str = "app.kubernetes.io/name";
50pub const INSTANCE_LABEL: &str = "app.kubernetes.io/instance";
51pub const MANAGED_BY_LABEL: &str = "app.kubernetes.io/managed-by";
52
53pub type ControllerId = &'static str;
54
55#[derive(Clone)]
58pub struct State {
59 metrics: Arc<metrics::Metrics>,
61 idm_clients: Arc<RwLock<KanidmClients>>,
63 system_clients: Arc<RwLock<KanidmClients>>,
66 pub namespace_store: Store<Namespace>,
68 pub kanidm_store: Store<Kanidm>,
70 pub client: Option<Client>,
72}
73
74pub struct ResourceReflector<K>
76where
77 K: Resource + Lookup + Clone + 'static,
78 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
79{
80 pub store: Store<K>,
81 pub writer: Writer<K>,
82 pub subscriber: ReflectHandle<K>,
83}
84
85impl State {
87 pub fn new(
88 metrics: metrics::Metrics,
89 namespace_store: Store<Namespace>,
90 kanidm_store: Store<Kanidm>,
91 client: Option<Client>,
92 ) -> Self {
93 Self {
94 metrics: Arc::new(metrics),
95 idm_clients: Arc::default(),
96 system_clients: Arc::default(),
97 namespace_store,
98 kanidm_store,
99 client,
100 }
101 }
102
103 pub fn metrics(&self) -> Result<String> {
105 prometheus_exporter::format_prometheus_metrics("kaniop").map_err(|e| {
106 Error::FormattingError(format!("failed to export metrics: {}", e), std::fmt::Error)
107 })
108 }
109
110 pub fn to_context<K>(&self, client: Client, controller_id: ControllerId) -> Context<K>
112 where
113 K: Resource + Lookup + Clone + 'static,
114 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
115 {
116 Context::new(
117 controller_id,
118 client.clone(),
119 self.metrics
120 .controllers
121 .get(controller_id)
122 .expect("all CONTROLLER_IDs have to be registered")
123 .clone(),
124 Recorder::new(client.clone(), controller_id.into()),
125 self.idm_clients.clone(),
126 self.system_clients.clone(),
127 self.namespace_store.clone(),
128 self.kanidm_store.clone(),
129 )
130 }
131}
132
133pub async fn check_api_queryable<K>(client: Client) -> Api<K>
134where
135 K: Resource + Clone + DeserializeOwned + Debug,
136 <K as Resource>::DynamicType: Default,
137{
138 let api = Api::<K>::all(client.clone());
139 if let Err(e) = api.list(&ListParams::default().limit(1)).await {
140 error!(
141 "{} is not queryable; {e:?}. Check controller permissions",
142 short_type_name::<K>().unwrap_or("Unknown resource"),
143 );
144 std::process::exit(1);
145 }
146 api
147}
148
149pub fn create_subscriber<K>(buffer_size: usize) -> ResourceReflector<K>
150where
151 K: Resource + Lookup + Clone + 'static,
152 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
153 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
154{
155 let (store, writer) = reflector::store_shared::<K>(buffer_size);
156 let subscriber = writer
157 .subscribe()
158 .expect("subscribers can only be created from shared stores");
159
160 ResourceReflector {
161 store,
162 writer,
163 subscriber,
164 }
165}
166
167fn create_generic_watcher<K, W, T, S, StreamT>(
168 api: Api<K>,
169 writer: Writer<W>,
170 reload_tx: mpsc::Sender<()>,
171 controller_id: ControllerId,
172 ctx: Arc<Context<T>>,
173 stream_fn: S,
174) -> BoxFuture<'static, ()>
175where
176 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
177 W: Resource + ResourceExt + Lookup + Clone + Debug + Send + Sync + 'static,
178 <W as Resource>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
179 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
180 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
181 <W as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
182 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
183 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
184 S: Fn(Api<K>, watcher::Config) -> StreamT + 'static,
185 StreamT: futures::Stream<Item = Result<watcher::Event<W>, kube::runtime::watcher::Error>>
186 + Send
187 + 'static,
188{
189 let resource_name = short_type_name::<K>().unwrap_or("Unknown");
190
191 stream_fn(
192 api,
193 watcher::Config::default().labels(&format!("{MANAGED_BY_LABEL}=kaniop-{controller_id}")),
194 )
195 .default_backoff()
196 .reflect_shared(writer)
197 .for_each(move |res| {
198 let mut reload_tx_clone = reload_tx.clone();
199 let ctx = ctx.clone();
200 async move {
201 match res {
202 Ok(event) => {
203 trace!(msg = "watched event", ?event);
204 match event {
205 watcher::Event::Delete(d) => {
206 debug!(
207 msg = format!("delete event for {resource_name} trigger reconcile"),
208 namespace = ResourceExt::namespace(&d).unwrap(),
209 name = d.name_any()
210 );
211
212 let _ignore_errors = reload_tx_clone.try_send(()).map_err(
216 |e| error!(msg = "failed to trigger reconcile on delete", %e),
217 );
218 ctx.metrics
219 .triggered_inc(metrics::Action::Delete, resource_name);
220 }
221 watcher::Event::Apply(d) => {
222 debug!(
223 msg = format!("apply event for {resource_name} trigger reconcile"),
224 namespace = ResourceExt::namespace(&d).unwrap(),
225 name = d.name_any()
226 );
227 ctx.metrics
228 .triggered_inc(metrics::Action::Apply, resource_name);
229 }
230 _ => {}
231 }
232 }
233 Err(e) => {
234 error!(msg = format!("unexpected error when watching {resource_name}"), %e);
235 ctx.metrics.watch_operations_failed_inc();
236 }
237 }
238 }
239 })
240 .boxed()
241}
242
243pub fn create_watcher<K, T>(
244 api: Api<K>,
245 writer: Writer<K>,
246 reload_tx: mpsc::Sender<()>,
247 controller_id: ControllerId,
248 ctx: Arc<Context<T>>,
249) -> BoxFuture<'static, ()>
250where
251 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
252 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
253 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
254 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
255 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
256{
257 create_generic_watcher::<K, K, T, _, _>(api, writer, reload_tx, controller_id, ctx, watcher)
258}
259
260pub fn create_metadata_watcher<K, T>(
261 api: Api<K>,
262 writer: Writer<PartialObjectMeta<K>>,
263 reload_tx: mpsc::Sender<()>,
264 controller_id: ControllerId,
265 ctx: Arc<Context<T>>,
266) -> BoxFuture<'static, ()>
267where
268 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
269 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
270 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
271 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
272 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
273{
274 create_generic_watcher::<K, PartialObjectMeta<K>, T, _, _>(
275 api,
276 writer,
277 reload_tx,
278 controller_id,
279 ctx,
280 metadata_watcher,
281 )
282}
283
284pub fn error_policy<K>(_obj: Arc<K>, _error: &Error, _ctx: Arc<Context<K>>) -> Action
285where
286 K: Resource + Lookup + Clone + 'static,
287 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
288{
289 unreachable!("Handle in backoff_reconciler macro")
290}
291
292#[macro_export]
293macro_rules! backoff_reconciler {
294 ($inner_reconciler:ident) => {
295 |obj, ctx| async move {
296 use $crate::controller::context::BackoffContext;
297 match $inner_reconciler(obj.clone(), ctx.clone()).await {
298 Ok(action) => {
299 ctx.reset_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
300 .await;
301 Ok(action)
302 }
303 Err(error) => {
304 let namespace = kube::ResourceExt::namespace(obj.as_ref()).unwrap();
306 let name = kube::ResourceExt::name_any(obj.as_ref());
307 tracing::error!(msg = "failed reconciliation", %namespace, %name, %error);
308 ctx.metrics().reconcile_failure_inc();
309 let backoff_duration = ctx
310 .get_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
311 .await;
312 tracing::trace!(
313 msg = format!("backoff duration: {backoff_duration:?}"),
314 %namespace,
315 %name,
316 );
317 Ok(kube::runtime::controller::Action::requeue(backoff_duration))
318 }
319 }
320 }
321 };
322}