Skip to main content

kaniop_operator/
leader_election.rs

1use jiff::Timestamp;
2use k8s_openapi::api::coordination::v1::Lease;
3use k8s_openapi::apimachinery::pkg::apis::meta::v1::MicroTime;
4use kube::Client;
5use kube::api::{Api, Patch, PatchParams, PostParams};
6use std::time::Duration;
7use tokio::time::sleep;
8use tracing::{debug, info, warn};
9
10#[derive(Clone)]
11pub struct LeaseLock {
12    client: Client,
13    namespace: String,
14    params: LeaseLockParams,
15}
16
17#[derive(Clone)]
18pub struct LeaseLockParams {
19    pub holder_id: String,
20    pub lease_name: String,
21    pub lease_ttl: Duration,
22}
23
24pub struct LeaseLockResult {
25    pub acquired_lease: bool,
26}
27
28impl LeaseLock {
29    pub fn new(client: Client, namespace: &str, params: LeaseLockParams) -> Self {
30        Self {
31            client,
32            namespace: namespace.to_string(),
33            params,
34        }
35    }
36
37    pub async fn try_acquire_or_renew(&self) -> anyhow::Result<LeaseLockResult> {
38        let leases: Api<Lease> = Api::namespaced(self.client.clone(), &self.namespace);
39        let ttl_seconds = self.params.lease_ttl.as_secs() as i32;
40
41        let now = Timestamp::now();
42        let now_micros = MicroTime(now);
43
44        let lease = leases.get(&self.params.lease_name).await;
45
46        match lease {
47            Ok(existing) => {
48                let holder = existing
49                    .spec
50                    .as_ref()
51                    .and_then(|s| s.holder_identity.as_ref());
52                let lease_ttl = existing
53                    .spec
54                    .as_ref()
55                    .and_then(|s| s.lease_duration_seconds);
56
57                let is_current_holder =
58                    holder.map(|h| h == &self.params.holder_id).unwrap_or(false);
59                let lease_expired = lease_ttl
60                    .map(|ttl| {
61                        let renew_time = existing.spec.as_ref().and_then(|s| s.renew_time.as_ref());
62                        if let Some(renew) = renew_time {
63                            let renew_ts = renew.0;
64                            let elapsed = now - renew_ts;
65                            elapsed.total(jiff::Unit::Second).unwrap_or(0.0) > (ttl as f64)
66                        } else {
67                            true
68                        }
69                    })
70                    .unwrap_or(true);
71
72                if is_current_holder || lease_expired || holder.is_none() {
73                    debug!(
74                        lease_name = %self.params.lease_name,
75                        holder_id = %self.params.holder_id,
76                        is_current_holder = is_current_holder,
77                        lease_expired = lease_expired,
78                        "attempting to acquire/renew lease"
79                    );
80
81                    let acquire_time = if is_current_holder {
82                        existing.spec.as_ref().and_then(|s| s.acquire_time.clone())
83                    } else {
84                        Some(now_micros.clone())
85                    };
86
87                    let patch = Lease {
88                        metadata: kube::api::ObjectMeta {
89                            name: Some(self.params.lease_name.clone()),
90                            namespace: Some(self.namespace.clone()),
91                            ..Default::default()
92                        },
93                        spec: Some(k8s_openapi::api::coordination::v1::LeaseSpec {
94                            holder_identity: Some(self.params.holder_id.clone()),
95                            lease_duration_seconds: Some(ttl_seconds),
96                            acquire_time,
97                            renew_time: Some(now_micros),
98                            ..Default::default()
99                        }),
100                    };
101
102                    let params = PatchParams::default();
103                    leases
104                        .patch(&self.params.lease_name, &params, &Patch::Merge(&patch))
105                        .await?;
106
107                    info!(
108                        lease_name = %self.params.lease_name,
109                        holder_id = %self.params.holder_id,
110                        "acquired/renewed lease"
111                    );
112
113                    Ok(LeaseLockResult {
114                        acquired_lease: true,
115                    })
116                } else {
117                    debug!(
118                        lease_name = %self.params.lease_name,
119                        current_holder = ?holder,
120                        "lease held by another instance"
121                    );
122                    Ok(LeaseLockResult {
123                        acquired_lease: false,
124                    })
125                }
126            }
127            Err(e) => {
128                if let kube::Error::Api(api_err) = &e {
129                    if api_err.code == 404 {
130                        debug!(
131                            lease_name = %self.params.lease_name,
132                            "lease does not exist, creating"
133                        );
134
135                        let new_lease = Lease {
136                            metadata: kube::api::ObjectMeta {
137                                name: Some(self.params.lease_name.clone()),
138                                namespace: Some(self.namespace.clone()),
139                                ..Default::default()
140                            },
141                            spec: Some(k8s_openapi::api::coordination::v1::LeaseSpec {
142                                holder_identity: Some(self.params.holder_id.clone()),
143                                lease_duration_seconds: Some(ttl_seconds),
144                                acquire_time: Some(now_micros.clone()),
145                                renew_time: Some(now_micros),
146                                ..Default::default()
147                            }),
148                        };
149
150                        let params = PostParams::default();
151                        leases.create(&params, &new_lease).await?;
152
153                        info!(
154                            lease_name = %self.params.lease_name,
155                            holder_id = %self.params.holder_id,
156                            "created and acquired lease"
157                        );
158
159                        return Ok(LeaseLockResult {
160                            acquired_lease: true,
161                        });
162                    }
163                }
164                warn!(
165                    lease_name = %self.params.lease_name,
166                    error = %e,
167                    "failed to get lease"
168                );
169                Err(e.into())
170            }
171        }
172    }
173
174    pub async fn step_down(&self) -> anyhow::Result<()> {
175        let leases: Api<Lease> = Api::namespaced(self.client.clone(), &self.namespace);
176
177        let lease = leases.get(&self.params.lease_name).await?;
178
179        let holder = lease.spec.as_ref().and_then(|s| s.holder_identity.as_ref());
180
181        if holder.map(|h| h == &self.params.holder_id).unwrap_or(false) {
182            debug!(
183                lease_name = %self.params.lease_name,
184                holder_id = %self.params.holder_id,
185                "stepping down from leadership"
186            );
187
188            let patch = Lease {
189                metadata: kube::api::ObjectMeta {
190                    name: Some(self.params.lease_name.clone()),
191                    namespace: Some(self.namespace.clone()),
192                    ..Default::default()
193                },
194                spec: Some(k8s_openapi::api::coordination::v1::LeaseSpec {
195                    lease_duration_seconds: Some(1),
196                    holder_identity: None,
197                    ..Default::default()
198                }),
199            };
200
201            let params = PatchParams::default();
202            leases
203                .patch(&self.params.lease_name, &params, &Patch::Merge(&patch))
204                .await?;
205
206            info!(
207                lease_name = %self.params.lease_name,
208                "stepped down from leadership"
209            );
210        }
211
212        Ok(())
213    }
214}
215
216pub async fn acquire_lease_with_retry(
217    lease_lock: &LeaseLock,
218    retry_interval: Duration,
219    max_retries: Option<u32>,
220) -> anyhow::Result<()> {
221    let mut retry_count = 0u32;
222
223    loop {
224        if let Some(max) = max_retries {
225            if retry_count >= max {
226                return Err(anyhow::anyhow!(
227                    "failed to acquire lease after {} retries",
228                    max
229                ));
230            }
231        }
232
233        let result = lease_lock.try_acquire_or_renew().await?;
234
235        if result.acquired_lease {
236            return Ok(());
237        }
238
239        retry_count += 1;
240        sleep(retry_interval).await;
241    }
242}