Skip to main content

kaniop_operator/
leader_election.rs

1use k8s_openapi::api::coordination::v1::Lease;
2use k8s_openapi::apimachinery::pkg::apis::meta::v1::MicroTime;
3use k8s_openapi::jiff::Timestamp;
4use kube::Client;
5use kube::api::{Api, Patch, PatchParams, PostParams};
6use std::time::Duration;
7use tokio::time::sleep;
8use tracing::{debug, info, warn};
9
10#[derive(Clone)]
11pub struct LeaseLock {
12    client: Client,
13    namespace: String,
14    params: LeaseLockParams,
15}
16
17#[derive(Clone)]
18pub struct LeaseLockParams {
19    pub holder_id: String,
20    pub lease_name: String,
21    pub lease_ttl: Duration,
22}
23
24pub struct LeaseLockResult {
25    pub acquired_lease: bool,
26}
27
28impl LeaseLock {
29    pub fn new(client: Client, namespace: &str, params: LeaseLockParams) -> Self {
30        Self {
31            client,
32            namespace: namespace.to_string(),
33            params,
34        }
35    }
36
37    pub async fn try_acquire_or_renew(&self) -> anyhow::Result<LeaseLockResult> {
38        let leases: Api<Lease> = Api::namespaced(self.client.clone(), &self.namespace);
39        let ttl_seconds = self.params.lease_ttl.as_secs() as i32;
40
41        let now = Timestamp::now();
42        let now_micros = MicroTime(now);
43
44        let lease = leases.get(&self.params.lease_name).await;
45
46        match lease {
47            Ok(existing) => {
48                let holder = existing
49                    .spec
50                    .as_ref()
51                    .and_then(|s| s.holder_identity.as_ref());
52                let lease_ttl = existing
53                    .spec
54                    .as_ref()
55                    .and_then(|s| s.lease_duration_seconds);
56
57                let is_current_holder =
58                    holder.map(|h| h == &self.params.holder_id).unwrap_or(false);
59                let lease_expired = lease_ttl
60                    .map(|ttl| {
61                        let renew_time = existing.spec.as_ref().and_then(|s| s.renew_time.as_ref());
62                        if let Some(renew) = renew_time {
63                            let renew_ts = renew.0;
64                            let elapsed = now - renew_ts;
65                            elapsed
66                                .total(k8s_openapi::jiff::Unit::Second)
67                                .unwrap_or(0.0)
68                                > (ttl as f64)
69                        } else {
70                            true
71                        }
72                    })
73                    .unwrap_or(true);
74
75                if is_current_holder || lease_expired || holder.is_none() {
76                    debug!(
77                        lease_name = %self.params.lease_name,
78                        holder_id = %self.params.holder_id,
79                        is_current_holder = is_current_holder,
80                        lease_expired = lease_expired,
81                        "attempting to acquire/renew lease"
82                    );
83
84                    let acquire_time = if is_current_holder {
85                        existing.spec.as_ref().and_then(|s| s.acquire_time.clone())
86                    } else {
87                        Some(now_micros.clone())
88                    };
89
90                    let patch = Lease {
91                        metadata: kube::api::ObjectMeta {
92                            name: Some(self.params.lease_name.clone()),
93                            namespace: Some(self.namespace.clone()),
94                            ..Default::default()
95                        },
96                        spec: Some(k8s_openapi::api::coordination::v1::LeaseSpec {
97                            holder_identity: Some(self.params.holder_id.clone()),
98                            lease_duration_seconds: Some(ttl_seconds),
99                            acquire_time,
100                            renew_time: Some(now_micros),
101                            ..Default::default()
102                        }),
103                    };
104
105                    let params = PatchParams::default();
106                    leases
107                        .patch(&self.params.lease_name, &params, &Patch::Merge(&patch))
108                        .await?;
109
110                    info!(
111                        lease_name = %self.params.lease_name,
112                        holder_id = %self.params.holder_id,
113                        "acquired/renewed lease"
114                    );
115
116                    Ok(LeaseLockResult {
117                        acquired_lease: true,
118                    })
119                } else {
120                    debug!(
121                        lease_name = %self.params.lease_name,
122                        current_holder = ?holder,
123                        "lease held by another instance"
124                    );
125                    Ok(LeaseLockResult {
126                        acquired_lease: false,
127                    })
128                }
129            }
130            Err(e) => {
131                if let kube::Error::Api(api_err) = &e {
132                    if api_err.code == 404 {
133                        debug!(
134                            lease_name = %self.params.lease_name,
135                            "lease does not exist, creating"
136                        );
137
138                        let new_lease = Lease {
139                            metadata: kube::api::ObjectMeta {
140                                name: Some(self.params.lease_name.clone()),
141                                namespace: Some(self.namespace.clone()),
142                                ..Default::default()
143                            },
144                            spec: Some(k8s_openapi::api::coordination::v1::LeaseSpec {
145                                holder_identity: Some(self.params.holder_id.clone()),
146                                lease_duration_seconds: Some(ttl_seconds),
147                                acquire_time: Some(now_micros.clone()),
148                                renew_time: Some(now_micros),
149                                ..Default::default()
150                            }),
151                        };
152
153                        let params = PostParams::default();
154                        leases.create(&params, &new_lease).await?;
155
156                        info!(
157                            lease_name = %self.params.lease_name,
158                            holder_id = %self.params.holder_id,
159                            "created and acquired lease"
160                        );
161
162                        return Ok(LeaseLockResult {
163                            acquired_lease: true,
164                        });
165                    }
166                }
167                warn!(
168                    lease_name = %self.params.lease_name,
169                    error = %e,
170                    "failed to get lease"
171                );
172                Err(e.into())
173            }
174        }
175    }
176
177    pub async fn step_down(&self) -> anyhow::Result<()> {
178        let leases: Api<Lease> = Api::namespaced(self.client.clone(), &self.namespace);
179
180        let lease = leases.get(&self.params.lease_name).await?;
181
182        let holder = lease.spec.as_ref().and_then(|s| s.holder_identity.as_ref());
183
184        if holder.map(|h| h == &self.params.holder_id).unwrap_or(false) {
185            debug!(
186                lease_name = %self.params.lease_name,
187                holder_id = %self.params.holder_id,
188                "stepping down from leadership"
189            );
190
191            let patch = Lease {
192                metadata: kube::api::ObjectMeta {
193                    name: Some(self.params.lease_name.clone()),
194                    namespace: Some(self.namespace.clone()),
195                    ..Default::default()
196                },
197                spec: Some(k8s_openapi::api::coordination::v1::LeaseSpec {
198                    lease_duration_seconds: Some(1),
199                    holder_identity: None,
200                    ..Default::default()
201                }),
202            };
203
204            let params = PatchParams::default();
205            leases
206                .patch(&self.params.lease_name, &params, &Patch::Merge(&patch))
207                .await?;
208
209            info!(
210                lease_name = %self.params.lease_name,
211                "stepped down from leadership"
212            );
213        }
214
215        Ok(())
216    }
217}
218
219pub async fn acquire_lease_with_retry(
220    lease_lock: &LeaseLock,
221    retry_interval: Duration,
222    max_retries: Option<u32>,
223) -> anyhow::Result<()> {
224    let mut retry_count = 0u32;
225
226    loop {
227        if let Some(max) = max_retries {
228            if retry_count >= max {
229                return Err(anyhow::anyhow!(
230                    "failed to acquire lease after {} retries",
231                    max
232                ));
233            }
234        }
235
236        let result = lease_lock.try_acquire_or_renew().await?;
237
238        if result.acquired_lease {
239            return Ok(());
240        }
241
242        retry_count += 1;
243        sleep(retry_interval).await;
244    }
245}