From ae48a6bb367a1a120e2faa6f61592e913814b018 Mon Sep 17 00:00:00 2001 From: Jeel Oza Date: Thu, 6 Aug 2026 20:37:41 +0530 Subject: [PATCH 01/11] [patch] delete config-pvc on storageClass mismatch before recreating --- src/mas/devops/tekton.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/mas/devops/tekton.py b/src/mas/devops/tekton.py index 7204c5d6..0c8dc18a 100644 --- a/src/mas/devops/tekton.py +++ b/src/mas/devops/tekton.py @@ -579,6 +579,28 @@ def preparePipelinesNamespace( # Create config PVC if requested if createConfigPVC: + # If config-pvc already exists with a different storageClass, delete it first. + # Kubernetes does not allow changing storageClassName on an existing PVC (immutable + # field), so patching would fail with a conflict error. Deleting and recreating + # is safe here because config-pvc only holds transient pipeline workspace data — + # it is not a source of truth for any persistent application state. + try: + existingConfigPVC = pvcAPI.get(name="config-pvc", namespace=namespace) + existingStorageClass = existingConfigPVC.spec.storageClassName + if existingStorageClass != storageClass: + logger.info( + f"config-pvc already exists with storageClassName='{existingStorageClass}' " + f"which differs from requested storageClassName='{storageClass}'. " + f"Deleting existing config-pvc so it can be recreated with the correct storageClass." + ) + pvcAPI.delete(name="config-pvc", namespace=namespace) + else: + logger.info( + f"config-pvc already exists with matching storageClassName='{existingStorageClass}', skipping delete." + ) + except NotFoundError: + pass # PVC does not exist yet — will be created below + logger.info("Creating config PVC") template = env.get_template("pipelines-pvc.yml.j2") renderedTemplate = template.render( From f927197848f969ea19f43fecae7fdc2a30c59123 Mon Sep 17 00:00:00 2001 From: Jeel Oza Date: Thu, 13 Aug 2026 01:30:14 +0530 Subject: [PATCH 02/11] [patch] fix black formatting and added wait for config-pvc deletion to complete before recreating --- src/mas/devops/tekton.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/mas/devops/tekton.py b/src/mas/devops/tekton.py index 0c8dc18a..40a901e7 100644 --- a/src/mas/devops/tekton.py +++ b/src/mas/devops/tekton.py @@ -594,10 +594,21 @@ def preparePipelinesNamespace( f"Deleting existing config-pvc so it can be recreated with the correct storageClass." ) pvcAPI.delete(name="config-pvc", namespace=namespace) + # Wait for deletion to complete before recreating. + # Kubernetes delete is asynchronous — the PVC enters "Terminating" state + # and applyResource() would still see it and try to patch it (causing the + # same immutable field conflict) if we proceed immediately. + logger.info("Waiting for config-pvc deletion to complete...") + while True: + try: + pvcAPI.get(name="config-pvc", namespace=namespace) + logger.debug("config-pvc still terminating, waiting 5s...") + sleep(5) + except NotFoundError: + logger.info("config-pvc deletion confirmed.") + break else: - logger.info( - f"config-pvc already exists with matching storageClassName='{existingStorageClass}', skipping delete." - ) + logger.info(f"config-pvc already exists with matching storageClassName='{existingStorageClass}', skipping delete.") except NotFoundError: pass # PVC does not exist yet — will be created below From 03c7be0519a7c557be4543059868362ad7503c53 Mon Sep 17 00:00:00 2001 From: Jeel Oza Date: Fri, 14 Aug 2026 09:50:55 +0530 Subject: [PATCH 03/11] [patch] Reuse existing config-pvc if Bound, recreate only if Lost or Pending --- src/mas/devops/tekton.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/src/mas/devops/tekton.py b/src/mas/devops/tekton.py index 40a901e7..ad0b93e1 100644 --- a/src/mas/devops/tekton.py +++ b/src/mas/devops/tekton.py @@ -587,17 +587,36 @@ def preparePipelinesNamespace( try: existingConfigPVC = pvcAPI.get(name="config-pvc", namespace=namespace) existingStorageClass = existingConfigPVC.spec.storageClassName - if existingStorageClass != storageClass: + existingPhase = existingConfigPVC.status.phase + if existingPhase == "Bound": + # config-pvc already exists and is healthy — storageClassName is immutable + # in Kubernetes so we must not try to patch it with a new storageClass + # (which would cause a 422 Unprocessable Entity error). The pipeline only + # needs the PVC to exist and be mountable — it does not care which + # storageClass backs it. logger.info( - f"config-pvc already exists with storageClassName='{existingStorageClass}' " - f"which differs from requested storageClassName='{storageClass}'. " - f"Deleting existing config-pvc so it can be recreated with the correct storageClass." + f"config-pvc already exists and is Bound with storageClassName='{existingStorageClass}', " + f"reusing existing PVC as-is (skipping recreate)." + ) + return + else: + # PVC exists but is not Bound (e.g. Lost or Pending because the backing + # storageClass was removed). Delete it so it can be recreated below. + # We must remove the pvc-protection finalizer first — otherwise Kubernetes + # will block deletion indefinitely while any pod (e.g. mas-cli deployment) + # still has the PVC mounted. + logger.info( + f"config-pvc exists but is in '{existingPhase}' state " + f"(storageClassName='{existingStorageClass}'). " + f"Removing finalizer and deleting so it can be recreated with storageClassName='{storageClass}'." + ) + pvcAPI.patch( + name="config-pvc", + namespace=namespace, + body={"metadata": {"finalizers": []}}, + content_type="application/merge-patch+json", ) pvcAPI.delete(name="config-pvc", namespace=namespace) - # Wait for deletion to complete before recreating. - # Kubernetes delete is asynchronous — the PVC enters "Terminating" state - # and applyResource() would still see it and try to patch it (causing the - # same immutable field conflict) if we proceed immediately. logger.info("Waiting for config-pvc deletion to complete...") while True: try: @@ -607,8 +626,6 @@ def preparePipelinesNamespace( except NotFoundError: logger.info("config-pvc deletion confirmed.") break - else: - logger.info(f"config-pvc already exists with matching storageClassName='{existingStorageClass}', skipping delete.") except NotFoundError: pass # PVC does not exist yet — will be created below From 4dfe2ec4adaa916f6dbbb0116115f4f8c4f6db5f Mon Sep 17 00:00:00 2001 From: Jeel Oza Date: Fri, 14 Aug 2026 18:12:02 +0530 Subject: [PATCH 04/11] [patch] remove pvc-protection finalizer before deleting config-pvc --- src/mas/devops/tekton.py | 65 +++++++++++++--------------------------- 1 file changed, 21 insertions(+), 44 deletions(-) diff --git a/src/mas/devops/tekton.py b/src/mas/devops/tekton.py index ad0b93e1..4c2b41c2 100644 --- a/src/mas/devops/tekton.py +++ b/src/mas/devops/tekton.py @@ -579,55 +579,32 @@ def preparePipelinesNamespace( # Create config PVC if requested if createConfigPVC: - # If config-pvc already exists with a different storageClass, delete it first. - # Kubernetes does not allow changing storageClassName on an existing PVC (immutable - # field), so patching would fail with a conflict error. Deleting and recreating - # is safe here because config-pvc only holds transient pipeline workspace data — - # it is not a source of truth for any persistent application state. + # If config-pvc already exists, remove its pvc-protection finalizer and delete it + # so it can be recreated with the correct storageClass. storageClassName is immutable + # in Kubernetes — patching it causes a 422 error. try: existingConfigPVC = pvcAPI.get(name="config-pvc", namespace=namespace) existingStorageClass = existingConfigPVC.spec.storageClassName existingPhase = existingConfigPVC.status.phase - if existingPhase == "Bound": - # config-pvc already exists and is healthy — storageClassName is immutable - # in Kubernetes so we must not try to patch it with a new storageClass - # (which would cause a 422 Unprocessable Entity error). The pipeline only - # needs the PVC to exist and be mountable — it does not care which - # storageClass backs it. - logger.info( - f"config-pvc already exists and is Bound with storageClassName='{existingStorageClass}', " - f"reusing existing PVC as-is (skipping recreate)." - ) - return - else: - # PVC exists but is not Bound (e.g. Lost or Pending because the backing - # storageClass was removed). Delete it so it can be recreated below. - # We must remove the pvc-protection finalizer first — otherwise Kubernetes - # will block deletion indefinitely while any pod (e.g. mas-cli deployment) - # still has the PVC mounted. - logger.info( - f"config-pvc exists but is in '{existingPhase}' state " - f"(storageClassName='{existingStorageClass}'). " - f"Removing finalizer and deleting so it can be recreated with storageClassName='{storageClass}'." - ) - pvcAPI.patch( - name="config-pvc", - namespace=namespace, - body={"metadata": {"finalizers": []}}, - content_type="application/merge-patch+json", - ) - pvcAPI.delete(name="config-pvc", namespace=namespace) - logger.info("Waiting for config-pvc deletion to complete...") - while True: - try: - pvcAPI.get(name="config-pvc", namespace=namespace) - logger.debug("config-pvc still terminating, waiting 5s...") - sleep(5) - except NotFoundError: - logger.info("config-pvc deletion confirmed.") - break + logger.info(f"config-pvc already exists (storageClassName='{existingStorageClass}', phase='{existingPhase}'). Removing finalizer and deleting to recreate with storageClassName='{storageClass}'.") + pvcAPI.patch( + name="config-pvc", + namespace=namespace, + body={"metadata": {"finalizers": []}}, + content_type="application/merge-patch+json", + ) + pvcAPI.delete(name="config-pvc", namespace=namespace) + logger.info("Waiting for config-pvc deletion to complete...") + while True: + try: + pvcAPI.get(name="config-pvc", namespace=namespace) + logger.debug("config-pvc still terminating, waiting 5s...") + sleep(5) + except NotFoundError: + logger.info("config-pvc deletion confirmed.") + break except NotFoundError: - pass # PVC does not exist yet — will be created below + pass # PVC does not exist yet, will be created below logger.info("Creating config PVC") template = env.get_template("pipelines-pvc.yml.j2") From 23a1c251bb6548d079ae87a5fc02af6aab9e9b79 Mon Sep 17 00:00:00 2001 From: Jeel Oza Date: Fri, 14 Aug 2026 18:21:30 +0530 Subject: [PATCH 05/11] [patch]fix black formatting --- src/mas/devops/tekton.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mas/devops/tekton.py b/src/mas/devops/tekton.py index 4c2b41c2..2b0fef79 100644 --- a/src/mas/devops/tekton.py +++ b/src/mas/devops/tekton.py @@ -586,7 +586,9 @@ def preparePipelinesNamespace( existingConfigPVC = pvcAPI.get(name="config-pvc", namespace=namespace) existingStorageClass = existingConfigPVC.spec.storageClassName existingPhase = existingConfigPVC.status.phase - logger.info(f"config-pvc already exists (storageClassName='{existingStorageClass}', phase='{existingPhase}'). Removing finalizer and deleting to recreate with storageClassName='{storageClass}'.") + logger.info( + f"config-pvc already exists (storageClassName='{existingStorageClass}', phase='{existingPhase}'). Removing finalizer and deleting to recreate with storageClassName='{storageClass}'." + ) pvcAPI.patch( name="config-pvc", namespace=namespace, From aecf3858f2f94810eeeef8306640599e7dda132a Mon Sep 17 00:00:00 2001 From: Jeel Oza Date: Tue, 18 Aug 2026 15:07:14 +0530 Subject: [PATCH 06/11] [patch] force-delete config-pvc to avoid Terminating deadlock --- src/mas/devops/tekton.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/mas/devops/tekton.py b/src/mas/devops/tekton.py index 2b0fef79..004a9b39 100644 --- a/src/mas/devops/tekton.py +++ b/src/mas/devops/tekton.py @@ -579,23 +579,26 @@ def preparePipelinesNamespace( # Create config PVC if requested if createConfigPVC: - # If config-pvc already exists, remove its pvc-protection finalizer and delete it - # so it can be recreated with the correct storageClass. storageClassName is immutable - # in Kubernetes — patching it causes a 422 error. try: existingConfigPVC = pvcAPI.get(name="config-pvc", namespace=namespace) existingStorageClass = existingConfigPVC.spec.storageClassName existingPhase = existingConfigPVC.status.phase logger.info( - f"config-pvc already exists (storageClassName='{existingStorageClass}', phase='{existingPhase}'). Removing finalizer and deleting to recreate with storageClassName='{storageClass}'." + f"config-pvc already exists (storageClassName='{existingStorageClass}', phase='{existingPhase}'). " + f"Deleting to recreate with storageClassName='{storageClass}'." ) + + # Remove the pvc-protection finalizer, then force-delete (grace_period_seconds=0). + # These two together are equivalent to: kubectl delete pvc --grace-period=0 --force + # Without both, the PVC gets stuck in Terminating — Kubernetes re-adds the finalizer + # as long as the PVC is Bound, and grace_period=0 tells the API server not to wait for it. pvcAPI.patch( name="config-pvc", namespace=namespace, body={"metadata": {"finalizers": []}}, content_type="application/merge-patch+json", ) - pvcAPI.delete(name="config-pvc", namespace=namespace) + pvcAPI.delete(name="config-pvc", namespace=namespace, grace_period_seconds=0) logger.info("Waiting for config-pvc deletion to complete...") while True: try: @@ -606,7 +609,7 @@ def preparePipelinesNamespace( logger.info("config-pvc deletion confirmed.") break except NotFoundError: - pass # PVC does not exist yet, will be created below + pass # PVC does not exist yet, will be created fresh below logger.info("Creating config PVC") template = env.get_template("pipelines-pvc.yml.j2") From c53da86d5326218fc857ae5e57197f93cf45e9e8 Mon Sep 17 00:00:00 2001 From: Jeel Oza Date: Wed, 19 Aug 2026 21:58:50 +0530 Subject: [PATCH 07/11] [patch] unbind PV before deleting config-pvc to avoid Terminating deadlock --- src/mas/devops/tekton.py | 45 ++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/src/mas/devops/tekton.py b/src/mas/devops/tekton.py index 004a9b39..68bb003b 100644 --- a/src/mas/devops/tekton.py +++ b/src/mas/devops/tekton.py @@ -588,17 +588,30 @@ def preparePipelinesNamespace( f"Deleting to recreate with storageClassName='{storageClass}'." ) - # Remove the pvc-protection finalizer, then force-delete (grace_period_seconds=0). - # These two together are equivalent to: kubectl delete pvc --grace-period=0 --force - # Without both, the PVC gets stuck in Terminating — Kubernetes re-adds the finalizer - # as long as the PVC is Bound, and grace_period=0 tells the API server not to wait for it. - pvcAPI.patch( - name="config-pvc", - namespace=namespace, - body={"metadata": {"finalizers": []}}, - content_type="application/merge-patch+json", - ) - pvcAPI.delete(name="config-pvc", namespace=namespace, grace_period_seconds=0) + # Unbind the PVC by removing claimRef from its backing PV. + # While the PVC is Bound, PVCProtectionController keeps re-adding the pvc-protection + # finalizer, blocking deletion. Removing claimRef moves the PVC to Lost/Released, + # causing the controller to drop the finalizer itself — then a normal delete works cleanly. + pvName = existingConfigPVC.spec.volumeName + if pvName: + logger.info(f"Unbinding config-pvc from PV '{pvName}' to allow clean deletion.") + pvAPI = dynClient.resources.get(api_version="v1", kind="PersistentVolume") + pvAPI.patch( + name=pvName, + body={"spec": {"claimRef": None}}, + content_type="application/merge-patch+json", + ) + # Wait for the PVC to leave Bound state — once unbound, PVCProtectionController + # removes the finalizer itself so the subsequent delete goes through cleanly. + for _ in range(30): + current = pvcAPI.get(name="config-pvc", namespace=namespace) + if current.status.phase != "Bound": + logger.info(f"config-pvc is now '{current.status.phase}', finalizer released.") + break + logger.debug("config-pvc still Bound, waiting 2s...") + sleep(2) + + pvcAPI.delete(name="config-pvc", namespace=namespace) logger.info("Waiting for config-pvc deletion to complete...") while True: try: @@ -608,6 +621,16 @@ def preparePipelinesNamespace( except NotFoundError: logger.info("config-pvc deletion confirmed.") break + + # Delete the now-orphaned PV so it does not accumulate across upgrades. + # The new config-pvc will get a fresh PV provisioned automatically. + if pvName: + try: + pvAPI.delete(name=pvName) + logger.info(f"Deleted orphaned PV '{pvName}'.") + except NotFoundError: + pass # already gone (e.g. reclaimPolicy: Delete handled it) + except NotFoundError: pass # PVC does not exist yet, will be created fresh below From c42959e6586583dd6c955fe91828a90b77ebaad2 Mon Sep 17 00:00:00 2001 From: Jeel Oza Date: Thu, 20 Aug 2026 08:54:15 +0530 Subject: [PATCH 08/11] [patch]unbind PV before deleting config-pvc to avoid Terminating deadlock --- src/mas/devops/tekton.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/mas/devops/tekton.py b/src/mas/devops/tekton.py index 68bb003b..db70197d 100644 --- a/src/mas/devops/tekton.py +++ b/src/mas/devops/tekton.py @@ -588,29 +588,36 @@ def preparePipelinesNamespace( f"Deleting to recreate with storageClassName='{storageClass}'." ) - # Unbind the PVC by removing claimRef from its backing PV. - # While the PVC is Bound, PVCProtectionController keeps re-adding the pvc-protection - # finalizer, blocking deletion. Removing claimRef moves the PVC to Lost/Released, - # causing the controller to drop the finalizer itself — then a normal delete works cleanly. + # Unbind the PVC by clearing the claimRef on its backing PV, then remove the finalizer. + # PVCProtectionController keeps re-adding pvc-protection on Bound PVCs so we must + # move it to Lost state first before the finalizer patch and delete will stick. pvName = existingConfigPVC.spec.volumeName + pvAPI = dynClient.resources.get(api_version="v1", kind="PersistentVolume") if pvName: logger.info(f"Unbinding config-pvc from PV '{pvName}' to allow clean deletion.") - pvAPI = dynClient.resources.get(api_version="v1", kind="PersistentVolume") + # Clear all claimRef fields — setting name/namespace/uid to empty strings forces + # Lost state even when the PVC already has a deletionTimestamp (Terminating). pvAPI.patch( name=pvName, - body={"spec": {"claimRef": None}}, + body={"spec": {"claimRef": {"name": "", "namespace": "", "uid": "", "resourceVersion": ""}}}, content_type="application/merge-patch+json", ) - # Wait for the PVC to leave Bound state — once unbound, PVCProtectionController - # removes the finalizer itself so the subsequent delete goes through cleanly. + # Wait for the PVC to leave Bound state for _ in range(30): current = pvcAPI.get(name="config-pvc", namespace=namespace) if current.status.phase != "Bound": - logger.info(f"config-pvc is now '{current.status.phase}', finalizer released.") + logger.info(f"config-pvc is now '{current.status.phase}'.") break logger.debug("config-pvc still Bound, waiting 2s...") sleep(2) + # Remove the finalizer directly — safe now that the PVC is no longer Bound + pvcAPI.patch( + name="config-pvc", + namespace=namespace, + body={"metadata": {"finalizers": []}}, + content_type="application/merge-patch+json", + ) pvcAPI.delete(name="config-pvc", namespace=namespace) logger.info("Waiting for config-pvc deletion to complete...") while True: From cc641c10ebbfd0168d0e45995d2a73ba62135f88 Mon Sep 17 00:00:00 2001 From: Jeel Oza Date: Thu, 20 Aug 2026 11:25:02 +0530 Subject: [PATCH 09/11] [patch] force-delete config-pvc by clearing PV claimRef and finalizers in a retry loop --- src/mas/devops/tekton.py | 90 +++++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/src/mas/devops/tekton.py b/src/mas/devops/tekton.py index db70197d..f3d2aa81 100644 --- a/src/mas/devops/tekton.py +++ b/src/mas/devops/tekton.py @@ -587,56 +587,72 @@ def preparePipelinesNamespace( f"config-pvc already exists (storageClassName='{existingStorageClass}', phase='{existingPhase}'). " f"Deleting to recreate with storageClassName='{storageClass}'." ) - - # Unbind the PVC by clearing the claimRef on its backing PV, then remove the finalizer. - # PVCProtectionController keeps re-adding pvc-protection on Bound PVCs so we must - # move it to Lost state first before the finalizer patch and delete will stick. pvName = existingConfigPVC.spec.volumeName pvAPI = dynClient.resources.get(api_version="v1", kind="PersistentVolume") - if pvName: - logger.info(f"Unbinding config-pvc from PV '{pvName}' to allow clean deletion.") - # Clear all claimRef fields — setting name/namespace/uid to empty strings forces - # Lost state even when the PVC already has a deletionTimestamp (Terminating). - pvAPI.patch( - name=pvName, - body={"spec": {"claimRef": {"name": "", "namespace": "", "uid": "", "resourceVersion": ""}}}, - content_type="application/merge-patch+json", - ) - # Wait for the PVC to leave Bound state - for _ in range(30): - current = pvcAPI.get(name="config-pvc", namespace=namespace) - if current.status.phase != "Bound": - logger.info(f"config-pvc is now '{current.status.phase}'.") - break - logger.debug("config-pvc still Bound, waiting 2s...") - sleep(2) - - # Remove the finalizer directly — safe now that the PVC is no longer Bound - pvcAPI.patch( - name="config-pvc", - namespace=namespace, - body={"metadata": {"finalizers": []}}, - content_type="application/merge-patch+json", - ) - pvcAPI.delete(name="config-pvc", namespace=namespace) - logger.info("Waiting for config-pvc deletion to complete...") - while True: + + # Force-delete the config-pvc regardless of its current state (Bound, Lost, Terminating). + # Each iteration applies every known unblocking step, then checks if the PVC is gone. + for attempt in range(30): + logger.debug(f"config-pvc force-delete attempt {attempt + 1}/30") + + # Step 1: clear claimRef on the backing PV so the PVC moves from Bound → Lost. + # Using empty strings (not null) — null is ignored when PVC has a deletionTimestamp. + if pvName: + try: + pvAPI.patch( + name=pvName, + body={"spec": {"claimRef": {"name": "", "namespace": "", "uid": "", "resourceVersion": ""}}}, + content_type="application/merge-patch+json", + ) + except NotFoundError: + pvName = None # PV already gone, skip PV steps + + # Step 2: clear PV finalizers in case the PV itself is stuck Terminating + if pvName: + try: + pvAPI.patch( + name=pvName, + body={"metadata": {"finalizers": []}}, + content_type="application/merge-patch+json", + ) + except NotFoundError: + pvName = None + + # Step 3: clear PVC finalizer and issue delete + try: + pvcAPI.patch( + name="config-pvc", + namespace=namespace, + body={"metadata": {"finalizers": []}}, + content_type="application/merge-patch+json", + ) + pvcAPI.delete(name="config-pvc", namespace=namespace) + except NotFoundError: + logger.info("config-pvc is gone.") + break + + sleep(3) + + # Check if gone try: pvcAPI.get(name="config-pvc", namespace=namespace) - logger.debug("config-pvc still terminating, waiting 5s...") - sleep(5) + logger.debug("config-pvc still present, retrying...") except NotFoundError: logger.info("config-pvc deletion confirmed.") break - # Delete the now-orphaned PV so it does not accumulate across upgrades. - # The new config-pvc will get a fresh PV provisioned automatically. + # Clean up the orphaned PV if still around if pvName: try: + pvAPI.patch( + name=pvName, + body={"metadata": {"finalizers": []}}, + content_type="application/merge-patch+json", + ) pvAPI.delete(name=pvName) logger.info(f"Deleted orphaned PV '{pvName}'.") except NotFoundError: - pass # already gone (e.g. reclaimPolicy: Delete handled it) + pass # already gone except NotFoundError: pass # PVC does not exist yet, will be created fresh below From d85b4abde595d1584f53b894d13b1c5a150f0741 Mon Sep 17 00:00:00 2001 From: Jeel Oza Date: Thu, 20 Aug 2026 18:15:09 +0530 Subject: [PATCH 10/11] [patch] delete and recreate config-pvc only when storage class changes on upgrade --- src/mas/devops/tekton.py | 120 ++++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 57 deletions(-) diff --git a/src/mas/devops/tekton.py b/src/mas/devops/tekton.py index f3d2aa81..1f578242 100644 --- a/src/mas/devops/tekton.py +++ b/src/mas/devops/tekton.py @@ -582,32 +582,72 @@ def preparePipelinesNamespace( try: existingConfigPVC = pvcAPI.get(name="config-pvc", namespace=namespace) existingStorageClass = existingConfigPVC.spec.storageClassName - existingPhase = existingConfigPVC.status.phase - logger.info( - f"config-pvc already exists (storageClassName='{existingStorageClass}', phase='{existingPhase}'). " - f"Deleting to recreate with storageClassName='{storageClass}'." - ) - pvName = existingConfigPVC.spec.volumeName - pvAPI = dynClient.resources.get(api_version="v1", kind="PersistentVolume") - - # Force-delete the config-pvc regardless of its current state (Bound, Lost, Terminating). - # Each iteration applies every known unblocking step, then checks if the PVC is gone. - for attempt in range(30): - logger.debug(f"config-pvc force-delete attempt {attempt + 1}/30") - # Step 1: clear claimRef on the backing PV so the PVC moves from Bound → Lost. - # Using empty strings (not null) — null is ignored when PVC has a deletionTimestamp. - if pvName: + if existingStorageClass == storageClass: + # Storage class matches — PVC is correct, skip delete and recreate + logger.info(f"config-pvc already exists with correct storageClassName='{storageClass}', skipping recreate.") + else: + # Storage class differs — delete and recreate with the correct one. + # storageClassName is immutable in Kubernetes so the PVC must be deleted first. + logger.info( + f"config-pvc exists with storageClassName='{existingStorageClass}' but upgrade requires " + f"'{storageClass}'. Deleting and recreating." + ) + pvName = existingConfigPVC.spec.volumeName + pvAPI = dynClient.resources.get(api_version="v1", kind="PersistentVolume") + + # Force-delete the config-pvc regardless of its current state (Bound, Lost, Terminating). + # Each iteration applies every known unblocking step, then checks if the PVC is gone. + for attempt in range(30): + logger.debug(f"config-pvc force-delete attempt {attempt + 1}/30") + + # Step 1: clear claimRef on the backing PV so the PVC moves from Bound → Lost. + # Using empty strings (not null) — null is ignored when PVC has a deletionTimestamp. + if pvName: + try: + pvAPI.patch( + name=pvName, + body={"spec": {"claimRef": {"name": "", "namespace": "", "uid": "", "resourceVersion": ""}}}, + content_type="application/merge-patch+json", + ) + except NotFoundError: + pvName = None # PV already gone, skip PV steps + + # Step 2: clear PV finalizers in case the PV itself is stuck Terminating + if pvName: + try: + pvAPI.patch( + name=pvName, + body={"metadata": {"finalizers": []}}, + content_type="application/merge-patch+json", + ) + except NotFoundError: + pvName = None + + # Step 3: clear PVC finalizer and issue delete try: - pvAPI.patch( - name=pvName, - body={"spec": {"claimRef": {"name": "", "namespace": "", "uid": "", "resourceVersion": ""}}}, + pvcAPI.patch( + name="config-pvc", + namespace=namespace, + body={"metadata": {"finalizers": []}}, content_type="application/merge-patch+json", ) + pvcAPI.delete(name="config-pvc", namespace=namespace) except NotFoundError: - pvName = None # PV already gone, skip PV steps + logger.info("config-pvc is gone.") + break + + sleep(3) - # Step 2: clear PV finalizers in case the PV itself is stuck Terminating + # Check if gone + try: + pvcAPI.get(name="config-pvc", namespace=namespace) + logger.debug("config-pvc still present, retrying...") + except NotFoundError: + logger.info("config-pvc deletion confirmed.") + break + + # Clean up the orphaned PV so it does not accumulate across upgrades if pvName: try: pvAPI.patch( @@ -615,44 +655,10 @@ def preparePipelinesNamespace( body={"metadata": {"finalizers": []}}, content_type="application/merge-patch+json", ) + pvAPI.delete(name=pvName) + logger.info(f"Deleted orphaned PV '{pvName}'.") except NotFoundError: - pvName = None - - # Step 3: clear PVC finalizer and issue delete - try: - pvcAPI.patch( - name="config-pvc", - namespace=namespace, - body={"metadata": {"finalizers": []}}, - content_type="application/merge-patch+json", - ) - pvcAPI.delete(name="config-pvc", namespace=namespace) - except NotFoundError: - logger.info("config-pvc is gone.") - break - - sleep(3) - - # Check if gone - try: - pvcAPI.get(name="config-pvc", namespace=namespace) - logger.debug("config-pvc still present, retrying...") - except NotFoundError: - logger.info("config-pvc deletion confirmed.") - break - - # Clean up the orphaned PV if still around - if pvName: - try: - pvAPI.patch( - name=pvName, - body={"metadata": {"finalizers": []}}, - content_type="application/merge-patch+json", - ) - pvAPI.delete(name=pvName) - logger.info(f"Deleted orphaned PV '{pvName}'.") - except NotFoundError: - pass # already gone + pass # already gone except NotFoundError: pass # PVC does not exist yet, will be created fresh below From f1e4d6bc4c5eff2a42270f082593320b2df55e28 Mon Sep 17 00:00:00 2001 From: Jeel Oza Date: Thu, 20 Aug 2026 19:03:22 +0530 Subject: [PATCH 11/11] [patch] fix black formatting in preparePipelinesNamespace --- src/mas/devops/tekton.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mas/devops/tekton.py b/src/mas/devops/tekton.py index 1f578242..2ccfef2b 100644 --- a/src/mas/devops/tekton.py +++ b/src/mas/devops/tekton.py @@ -590,8 +590,7 @@ def preparePipelinesNamespace( # Storage class differs — delete and recreate with the correct one. # storageClassName is immutable in Kubernetes so the PVC must be deleted first. logger.info( - f"config-pvc exists with storageClassName='{existingStorageClass}' but upgrade requires " - f"'{storageClass}'. Deleting and recreating." + f"config-pvc exists with storageClassName='{existingStorageClass}' but upgrade requires " f"'{storageClass}'. Deleting and recreating." ) pvName = existingConfigPVC.spec.volumeName pvAPI = dynClient.resources.get(api_version="v1", kind="PersistentVolume")