diff --git a/src/mas/devops/tekton.py b/src/mas/devops/tekton.py index 7204c5d6..2ccfef2b 100644 --- a/src/mas/devops/tekton.py +++ b/src/mas/devops/tekton.py @@ -579,6 +579,89 @@ def preparePipelinesNamespace( # Create config PVC if requested if createConfigPVC: + try: + existingConfigPVC = pvcAPI.get(name="config-pvc", namespace=namespace) + existingStorageClass = existingConfigPVC.spec.storageClassName + + if existingStorageClass == storageClass: + # Storage class matches — PVC is correct, skip delete and recreate + logger.info(f"config-pvc already exists with correct storageClassName='{storageClass}', skipping recreate.") + else: + # Storage class differs — delete and recreate with the correct one. + # storageClassName is immutable in Kubernetes so the PVC must be deleted first. + logger.info( + f"config-pvc exists with storageClassName='{existingStorageClass}' but upgrade requires " f"'{storageClass}'. Deleting and recreating." + ) + pvName = existingConfigPVC.spec.volumeName + pvAPI = dynClient.resources.get(api_version="v1", kind="PersistentVolume") + + # Force-delete the config-pvc regardless of its current state (Bound, Lost, Terminating). + # Each iteration applies every known unblocking step, then checks if the PVC is gone. + for attempt in range(30): + logger.debug(f"config-pvc force-delete attempt {attempt + 1}/30") + + # Step 1: clear claimRef on the backing PV so the PVC moves from Bound → Lost. + # Using empty strings (not null) — null is ignored when PVC has a deletionTimestamp. + if pvName: + try: + pvAPI.patch( + name=pvName, + body={"spec": {"claimRef": {"name": "", "namespace": "", "uid": "", "resourceVersion": ""}}}, + content_type="application/merge-patch+json", + ) + except NotFoundError: + pvName = None # PV already gone, skip PV steps + + # Step 2: clear PV finalizers in case the PV itself is stuck Terminating + if pvName: + try: + pvAPI.patch( + name=pvName, + body={"metadata": {"finalizers": []}}, + content_type="application/merge-patch+json", + ) + except NotFoundError: + pvName = None + + # Step 3: clear PVC finalizer and issue delete + try: + pvcAPI.patch( + name="config-pvc", + namespace=namespace, + body={"metadata": {"finalizers": []}}, + content_type="application/merge-patch+json", + ) + pvcAPI.delete(name="config-pvc", namespace=namespace) + except NotFoundError: + logger.info("config-pvc is gone.") + break + + sleep(3) + + # Check if gone + try: + pvcAPI.get(name="config-pvc", namespace=namespace) + logger.debug("config-pvc still present, retrying...") + except NotFoundError: + logger.info("config-pvc deletion confirmed.") + break + + # Clean up the orphaned PV so it does not accumulate across upgrades + if pvName: + try: + pvAPI.patch( + name=pvName, + body={"metadata": {"finalizers": []}}, + content_type="application/merge-patch+json", + ) + pvAPI.delete(name=pvName) + logger.info(f"Deleted orphaned PV '{pvName}'.") + except NotFoundError: + pass # already gone + + except NotFoundError: + pass # PVC does not exist yet, will be created fresh below + logger.info("Creating config PVC") template = env.get_template("pipelines-pvc.yml.j2") renderedTemplate = template.render(