From 1c55b5ce4693b3660ec3de5a1ad1c09131dc2bd5 Mon Sep 17 00:00:00 2001
From: Afolabi Omotoso <128552490+gitafolabi@users.noreply.github.com>
Date: Sun, 21 Jun 2026 15:12:18 +0200
Subject: [PATCH 01/14] kv for new tenant

---
 .github/workflows/azure-ci.yml        | 33 +++++++++-
 gitops/azure-cluster-secret-store.yml |  4 +-
 seed-boutique-secrets.sh              | 88 +++++++++++++++++++++++++++
 3 files changed, 122 insertions(+), 3 deletions(-)
 create mode 100755 seed-boutique-secrets.sh

diff --git a/.github/workflows/azure-ci.yml b/.github/workflows/azure-ci.yml
index a1b0286..5d594c5 100644
--- a/.github/workflows/azure-ci.yml
+++ b/.github/workflows/azure-ci.yml
@@ -64,9 +64,40 @@ jobs:
           echo "Pushing $IMAGE_URI"
           docker push $IMAGE_URI
 
+  seed-secrets:
+    runs-on: ubuntu-latest
+    environment: production  # gate here if you want manual approval before seeding
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Azure CLI login
+        uses: azure/login@v2
+        with:
+          creds: |
+            {
+              "clientId":       "${{ secrets.AZURE_CLIENT_ID }}",
+              "clientSecret":   "${{ secrets.AZURE_CLIENT_SECRET }}",
+              "tenantId":       "${{ secrets.AZURE_TENANT_ID }}",
+              "subscriptionId": "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
+            }
+
+      - name: Seed boutique secrets into Key Vault
+        env:
+          KEYVAULT:  ${{ vars.KEYVAULT_NAME }}
+          SMTP_HOST: ${{ secrets.SMTP_HOST }}
+          SMTP_PORT: ${{ secrets.SMTP_PORT }}
+          SMTP_USER: ${{ secrets.SMTP_USER }}
+          SMTP_PASS: ${{ secrets.SMTP_PASS }}
+          EMAIL_FROM: ${{ secrets.EMAIL_FROM }}
+        run: |
+          chmod +x seed-boutique-secrets.sh
+          ./seed-boutique-secrets.sh
+
   update-manifests:
     runs-on: ubuntu-latest
-    needs: build-and-push
+    needs: [build-and-push, seed-secrets]
     permissions:
       contents: write
 
diff --git a/gitops/azure-cluster-secret-store.yml b/gitops/azure-cluster-secret-store.yml
index 473dd86..8a9b568 100644
--- a/gitops/azure-cluster-secret-store.yml
+++ b/gitops/azure-cluster-secret-store.yml
@@ -5,9 +5,9 @@ metadata:
 spec:
   provider:
     azurekv:
-      vaultUrl: "https://crud-kv.vault.azure.net/"
+      vaultUrl: "https://crud-kv-24cc.vault.azure.net/"
       authType: ServicePrincipal
-      tenantId: "f09536f4-5988-40db-808d-eabbdfbb5e63"
+      tenantId: "dcbf7382-8381-4638-9a5b-17998c16d54a"
       authSecretRef:
         clientId:
           name: azure-sp-credentials
diff --git a/seed-boutique-secrets.sh b/seed-boutique-secrets.sh
new file mode 100755
index 0000000..2c36aa4
--- /dev/null
+++ b/seed-boutique-secrets.sh
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# Seed boutique app secrets into Azure Key Vault.
+# All values come from environment variables — no interactive prompts.
+# Safe to re-run: skips secrets that already exist.
+#
+# Required env vars (set as pipeline secret variables):
+#   KEYVAULT          - Key Vault name
+#   SMTP_HOST, SMTP_PORT, SMTP_USER, SMTP_PASS, EMAIL_FROM
+#
+# Generated automatically if not set:
+#   POSTGRES_PASSWORD, RABBITMQ_PASSWORD
+
+set -euo pipefail
+
+KEYVAULT="${KEYVAULT:-crud-kv-24cc}"
+
+# ── Helper ────────────────────────────────────────────────────────────────────
+
+kv_set() {
+  local name="$1" value="$2"
+  if az keyvault secret show --vault-name "$KEYVAULT" --name "$name" &>/dev/null; then
+    echo "    $name already exists, skipping."
+  else
+    az keyvault secret set --vault-name "$KEYVAULT" --name "$name" --value "$value" --output none
+    echo "    $name set."
+  fi
+}
+
+gen_password() {
+  openssl rand -base64 24 | tr -d '/+=' | head -c 32
+}
+
+# ── Validate required SMTP vars ───────────────────────────────────────────────
+
+REQUIRED_SMTP=(SMTP_HOST SMTP_PORT SMTP_USER SMTP_PASS EMAIL_FROM)
+MISSING=()
+for var in "${REQUIRED_SMTP[@]}"; do
+  [[ -z "${!var:-}" ]] && MISSING+=("$var")
+done
+
+if [[ ${#MISSING[@]} -gt 0 ]]; then
+  echo "WARNING: The following SMTP variables are not set — those secrets will be skipped:"
+  printf '  %s\n' "${MISSING[@]}"
+  echo "  Set them as secret pipeline variables and re-run to populate."
+  echo ""
+fi
+
+# ── Generate infrastructure credentials ──────────────────────────────────────
+
+POSTGRES_USER="boutique"
+POSTGRES_DB="boutique"
+POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-$(gen_password)}"
+
+RABBITMQ_USER="boutique"
+RABBITMQ_PASSWORD="${RABBITMQ_PASSWORD:-$(gen_password)}"
+
+PG_HOST="postgres.boutique.svc.cluster.local"
+
+AUTH_DB_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${PG_HOST}:5432/auth_db"
+PRODUCTS_DB_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${PG_HOST}:5432/products_db"
+ORDERS_DB_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${PG_HOST}:5432/orders_db"
+USERS_DB_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${PG_HOST}:5432/users_db"
+
+RABBIT_URL="amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq.boutique.svc.cluster.local:5672"
+
+# ── Push to Key Vault ─────────────────────────────────────────────────────────
+
+echo "==> Writing secrets to Key Vault '$KEYVAULT'..."
+
+kv_set "boutique-POSTGRES-DB"       "$POSTGRES_DB"
+kv_set "boutique-POSTGRES-USER"     "$POSTGRES_USER"
+kv_set "boutique-POSTGRES-PASSWORD" "$POSTGRES_PASSWORD"
+kv_set "boutique-AUTH-DB-URL"       "$AUTH_DB_URL"
+kv_set "boutique-PRODUCTS-DB-URL"   "$PRODUCTS_DB_URL"
+kv_set "boutique-ORDERS-DB-URL"     "$ORDERS_DB_URL"
+kv_set "boutique-USERS-DB-URL"      "$USERS_DB_URL"
+kv_set "boutique-RABBITMQ-USER"     "$RABBITMQ_USER"
+kv_set "boutique-RABBITMQ-PASSWORD" "$RABBITMQ_PASSWORD"
+kv_set "boutique-RABBIT-URL"        "$RABBIT_URL"
+
+[[ -n "${SMTP_HOST:-}"  ]] && kv_set "boutique-SMTP-HOST"  "$SMTP_HOST"
+[[ -n "${SMTP_PORT:-}"  ]] && kv_set "boutique-SMTP-PORT"  "$SMTP_PORT"
+[[ -n "${SMTP_USER:-}"  ]] && kv_set "boutique-SMTP-USER"  "$SMTP_USER"
+[[ -n "${SMTP_PASS:-}"  ]] && kv_set "boutique-SMTP-PASS"  "$SMTP_PASS"
+[[ -n "${EMAIL_FROM:-}" ]] && kv_set "boutique-EMAIL-FROM" "$EMAIL_FROM"
+
+echo ""
+echo "Done. All available secrets written to '$KEYVAULT'."

From dc77bc29f343ec7bd8778cc7e085b0059dd75a79 Mon Sep 17 00:00:00 2001
From: Afolabi Omotoso <128552490+gitafolabi@users.noreply.github.com>
Date: Sun, 21 Jun 2026 15:36:49 +0200
Subject: [PATCH 02/14] fix: correct KV URL, tenant ID, secret key names and
 disable service-monitor

---
 gitops/azure-external-secret.yml | 6 +++---
 gitops/kustomization.yml         | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/gitops/azure-external-secret.yml b/gitops/azure-external-secret.yml
index 1a495c0..a21bfa1 100644
--- a/gitops/azure-external-secret.yml
+++ b/gitops/azure-external-secret.yml
@@ -14,13 +14,13 @@ spec:
   data:
     - secretKey: POSTGRES_DB
       remoteRef:
-        key: boutique-POSTGRES-DB
+        key: db-name
     - secretKey: POSTGRES_USER
       remoteRef:
-        key: boutique-POSTGRES-USER
+        key: db-user
     - secretKey: POSTGRES_PASSWORD
       remoteRef:
-        key: boutique-POSTGRES-PASSWORD
+        key: db-password
     - secretKey: AUTH_DB_URL
       remoteRef:
         key: boutique-AUTH-DB-URL
diff --git a/gitops/kustomization.yml b/gitops/kustomization.yml
index 0fa8ccc..a1dfa87 100644
--- a/gitops/kustomization.yml
+++ b/gitops/kustomization.yml
@@ -31,7 +31,7 @@ resources:
   - k8s/pdb.yml
   - k8s/grafana-dashboard.yml
   - k8s/loki-datasource.yml
-  - k8s/backend/service-monitor.yml
+  # - k8s/backend/service-monitor.yml  # re-enable after Prometheus Operator CRDs are installed
   - k8s/ingress.yml
   - k8s/monitoring-ingress.yml
   - k8s/tools-ingress.yml

From b615ba11c02e99295490d672589649d228314176 Mon Sep 17 00:00:00 2001
From: Afolabi Omotoso <128552490+gitafolabi@users.noreply.github.com>
Date: Sun, 21 Jun 2026 17:05:51 +0200
Subject: [PATCH 03/14] feat: enable Grafana, Prometheus and Alertmanager in
 kube-prometheus-stack

---
 gitops/prometheus-crds-app.yml | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/gitops/prometheus-crds-app.yml b/gitops/prometheus-crds-app.yml
index fba26f7..590f62e 100644
--- a/gitops/prometheus-crds-app.yml
+++ b/gitops/prometheus-crds-app.yml
@@ -12,18 +12,21 @@ spec:
     helm:
       values: |
         prometheus:
-          enabled: false
+          enabled: true
         grafana:
-          enabled: false
+          enabled: true
+          adminPassword: admin
+          service:
+            type: ClusterIP
         alertmanager:
-          enabled: false
+          enabled: true
         prometheusOperator:
           enabled: true
           manageCrds: true
         nodeExporter:
-          enabled: false
+          enabled: true
         kubeStateMetrics:
-          enabled: false
+          enabled: true
   destination:
     server: 'https://kubernetes.default.svc'
     namespace: monitoring

From a4ecd1c0a5d97afc9f540d9da8d0ac5ac3995c56 Mon Sep 17 00:00:00 2001
From: Afolabi Omotoso <128552490+gitafolabi@users.noreply.github.com>
Date: Sun, 21 Jun 2026 17:11:57 +0200
Subject: [PATCH 04/14] fix eso for grafana

---
 gitops/grafana-external-secret.yml | 20 ++++++++++++++++++++
 gitops/kustomization.yml           |  1 +
 gitops/prometheus-crds-app.yml     |  5 ++++-
 3 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 gitops/grafana-external-secret.yml

diff --git a/gitops/grafana-external-secret.yml b/gitops/grafana-external-secret.yml
new file mode 100644
index 0000000..ff4a31e
--- /dev/null
+++ b/gitops/grafana-external-secret.yml
@@ -0,0 +1,20 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: grafana-secrets
+  namespace: monitoring
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: azure-kv
+    kind: ClusterSecretStore
+  target:
+    name: grafana-admin-credentials
+    creationPolicy: Owner
+  data:
+    - secretKey: admin-user
+      remoteRef:
+        key: grafana-admin-user
+    - secretKey: admin-password
+      remoteRef:
+        key: grafana-admin-password
diff --git a/gitops/kustomization.yml b/gitops/kustomization.yml
index a1dfa87..4e7eefe 100644
--- a/gitops/kustomization.yml
+++ b/gitops/kustomization.yml
@@ -40,6 +40,7 @@ resources:
   - ingres-argo.yml
   - cert-manager-clusterissuer.yml
   - prometheus-crds-app.yml
+  - grafana-external-secret.yml
   - cert-manager-argo.yml
   - loki.yml
   # ELK stack — commented out to avoid resource exhaustion; uncomment to deploy:
diff --git a/gitops/prometheus-crds-app.yml b/gitops/prometheus-crds-app.yml
index 590f62e..394a5d6 100644
--- a/gitops/prometheus-crds-app.yml
+++ b/gitops/prometheus-crds-app.yml
@@ -15,7 +15,10 @@ spec:
           enabled: true
         grafana:
           enabled: true
-          adminPassword: admin
+          admin:
+            existingSecret: grafana-admin-credentials
+            userKey: admin-user
+            passwordKey: admin-password
           service:
             type: ClusterIP
         alertmanager:

From d84db8d47ae0a27f41f18b5fa64f6da2c3b2d7d7 Mon Sep 17 00:00:00 2001
From: Afolabi Omotoso <128552490+gitafolabi@users.noreply.github.com>
Date: Sun, 21 Jun 2026 17:17:58 +0200
Subject: [PATCH 05/14] fix: move Loki datasource to Helm values, disable
 sidecar to prevent duplicate default datasource

---
 gitops/kustomization.yml       |  2 +-
 gitops/prometheus-crds-app.yml | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/gitops/kustomization.yml b/gitops/kustomization.yml
index 4e7eefe..ebeb8f9 100644
--- a/gitops/kustomization.yml
+++ b/gitops/kustomization.yml
@@ -30,7 +30,7 @@ resources:
   - k8s/hpa.yml
   - k8s/pdb.yml
   - k8s/grafana-dashboard.yml
-  - k8s/loki-datasource.yml
+  # - k8s/loki-datasource.yml  # Loki datasource moved to Helm values in prometheus-crds-app.yml
   # - k8s/backend/service-monitor.yml  # re-enable after Prometheus Operator CRDs are installed
   - k8s/ingress.yml
   - k8s/monitoring-ingress.yml
diff --git a/gitops/prometheus-crds-app.yml b/gitops/prometheus-crds-app.yml
index 394a5d6..210dcf2 100644
--- a/gitops/prometheus-crds-app.yml
+++ b/gitops/prometheus-crds-app.yml
@@ -21,6 +21,17 @@ spec:
             passwordKey: admin-password
           service:
             type: ClusterIP
+          additionalDataSources:
+            - name: Loki
+              type: loki
+              access: proxy
+              url: http://loki-stack.monitoring:3100
+              isDefault: false
+              jsonData:
+                maxLines: 1000
+          sidecar:
+            datasources:
+              enabled: false
         alertmanager:
           enabled: true
         prometheusOperator:

From e3fb05d358fdc4179d40f55bd3b01809ed88161e Mon Sep 17 00:00:00 2001
From: Afolabi Omotoso <128552490+gitafolabi@users.noreply.github.com>
Date: Sun, 21 Jun 2026 17:23:36 +0200
Subject: [PATCH 06/14] fix: disable default datasource provisioning, define
 Prometheus and Loki explicitly

---
 gitops/prometheus-crds-app.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/gitops/prometheus-crds-app.yml b/gitops/prometheus-crds-app.yml
index 210dcf2..80fb65b 100644
--- a/gitops/prometheus-crds-app.yml
+++ b/gitops/prometheus-crds-app.yml
@@ -21,7 +21,16 @@ spec:
             passwordKey: admin-password
           service:
             type: ClusterIP
+          sidecar:
+            datasources:
+              enabled: false
+              defaultDatasourceEnabled: false
           additionalDataSources:
+            - name: Prometheus
+              type: prometheus
+              access: proxy
+              url: http://prometheus-operated.monitoring:9090
+              isDefault: true
             - name: Loki
               type: loki
               access: proxy
@@ -29,9 +38,6 @@ spec:
               isDefault: false
               jsonData:
                 maxLines: 1000
-          sidecar:
-            datasources:
-              enabled: false
         alertmanager:
           enabled: true
         prometheusOperator:

From aab8be89d2141740f59793f6dbf3b75bb4f76af8 Mon Sep 17 00:00:00 2001
From: Afolabi Omotoso <128552490+gitafolabi@users.noreply.github.com>
Date: Sun, 21 Jun 2026 17:34:14 +0200
Subject: [PATCH 07/14] fix: set loki datasource isDefault false to resolve
 Grafana dual-default conflict

---
 gitops/loki.yml                |  3 +++
 gitops/prometheus-crds-app.yml | 16 +---------------
 2 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/gitops/loki.yml b/gitops/loki.yml
index 31c5f62..55f61c0 100644
--- a/gitops/loki.yml
+++ b/gitops/loki.yml
@@ -39,6 +39,9 @@ spec:
               memory: 64Mi 
         grafana:
           enabled: false
+          sidecar:
+            datasources:
+              isDefaultDatasource: false
         prometheus:
           enabled: false
         fluent-bit:
diff --git a/gitops/prometheus-crds-app.yml b/gitops/prometheus-crds-app.yml
index 80fb65b..27368fc 100644
--- a/gitops/prometheus-crds-app.yml
+++ b/gitops/prometheus-crds-app.yml
@@ -23,21 +23,7 @@ spec:
             type: ClusterIP
           sidecar:
             datasources:
-              enabled: false
-              defaultDatasourceEnabled: false
-          additionalDataSources:
-            - name: Prometheus
-              type: prometheus
-              access: proxy
-              url: http://prometheus-operated.monitoring:9090
-              isDefault: true
-            - name: Loki
-              type: loki
-              access: proxy
-              url: http://loki-stack.monitoring:3100
-              isDefault: false
-              jsonData:
-                maxLines: 1000
+              enabled: true
         alertmanager:
           enabled: true
         prometheusOperator:

From 1bbef4d92b65718851736c6ba5437584fa9aefa6 Mon Sep 17 00:00:00 2001
From: Afolabi Omotoso <128552490+gitafolabi@users.noreply.github.com>
Date: Sun, 21 Jun 2026 18:01:52 +0200
Subject: [PATCH 08/14] fix: disable grafana sidecar to prevent loki-stack
 ConfigMap conflict

---
 gitops/loki.yml                |  3 ---
 gitops/prometheus-crds-app.yml | 10 +++++++++-
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/gitops/loki.yml b/gitops/loki.yml
index 55f61c0..31c5f62 100644
--- a/gitops/loki.yml
+++ b/gitops/loki.yml
@@ -39,9 +39,6 @@ spec:
               memory: 64Mi 
         grafana:
           enabled: false
-          sidecar:
-            datasources:
-              isDefaultDatasource: false
         prometheus:
           enabled: false
         fluent-bit:
diff --git a/gitops/prometheus-crds-app.yml b/gitops/prometheus-crds-app.yml
index 27368fc..6d6a895 100644
--- a/gitops/prometheus-crds-app.yml
+++ b/gitops/prometheus-crds-app.yml
@@ -23,7 +23,15 @@ spec:
             type: ClusterIP
           sidecar:
             datasources:
-              enabled: true
+              enabled: false
+          additionalDataSources:
+            - name: Loki
+              type: loki
+              access: proxy
+              url: http://loki-stack.monitoring:3100
+              isDefault: false
+              jsonData:
+                maxLines: 1000
         alertmanager:
           enabled: true
         prometheusOperator:

From 017b3e88134139905e776f1861417df4dd5b2a44 Mon Sep 17 00:00:00 2001
From: Afolabi Omotoso <128552490+gitafolabi@users.noreply.github.com>
Date: Sun, 21 Jun 2026 18:09:48 +0200
Subject: [PATCH 09/14] fix: ignore loki-stack datasource ConfigMap diff to
 allow isDefault override

---
 gitops/loki.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/gitops/loki.yml b/gitops/loki.yml
index 31c5f62..120f55b 100644
--- a/gitops/loki.yml
+++ b/gitops/loki.yml
@@ -46,6 +46,13 @@ spec:
   destination:
     server: 'https://kubernetes.default.svc'
     namespace: monitoring
+  ignoreDifferences:
+    - group: ""
+      kind: ConfigMap
+      name: loki-stack
+      namespace: monitoring
+      jqPathExpressions:
+        - '.data["loki-stack-datasource.yaml"]'
   syncPolicy:
     automated:
       prune: true

From cba1c9b3de9c6d9083884bcb40356d1a5de35358 Mon Sep 17 00:00:00 2001
From: Afolabi Omotoso <128552490+gitafolabi@users.noreply.github.com>
Date: Sun, 21 Jun 2026 18:29:06 +0200
Subject: [PATCH 10/14] fix: update monitoring ingress service names to match
 prometheus-operator-crds release

---
 gitops/k8s/monitoring-ingress.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gitops/k8s/monitoring-ingress.yml b/gitops/k8s/monitoring-ingress.yml
index bb720d9..2bf513f 100644
--- a/gitops/k8s/monitoring-ingress.yml
+++ b/gitops/k8s/monitoring-ingress.yml
@@ -22,7 +22,7 @@ spec:
         pathType: Prefix
         backend:
           service:
-            name: kube-prometheus-stack-grafana
+            name: prometheus-operator-crds-grafana
             port:
               number: 80
   - host: alerts.test.chellrach.com
@@ -32,6 +32,6 @@ spec:
         pathType: Prefix
         backend:
           service:
-            name: kube-prometheus-stack-alertmanager
+            name: prometheus-operator-crds-k-alertmanager
             port:
               number: 9093
\ No newline at end of file

From eb34366fd7aad7b4aaef2bc6abbd0b2a00771e60 Mon Sep 17 00:00:00 2001
From: Afolabi Omotoso <128552490+gitafolabi@users.noreply.github.com>
Date: Sun, 21 Jun 2026 18:36:54 +0200
Subject: [PATCH 11/14] fix: add all datasources via additionalDataSources
 since sidecar is disabled

---
 gitops/prometheus-crds-app.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/gitops/prometheus-crds-app.yml b/gitops/prometheus-crds-app.yml
index 6d6a895..4c7181b 100644
--- a/gitops/prometheus-crds-app.yml
+++ b/gitops/prometheus-crds-app.yml
@@ -25,6 +25,22 @@ spec:
             datasources:
               enabled: false
           additionalDataSources:
+            - name: Prometheus
+              type: prometheus
+              access: proxy
+              url: http://prometheus-operator-crds-k-prometheus.monitoring:9090
+              isDefault: true
+              jsonData:
+                httpMethod: POST
+                timeInterval: 30s
+            - name: Alertmanager
+              type: alertmanager
+              access: proxy
+              url: http://prometheus-operator-crds-k-alertmanager.monitoring:9093
+              isDefault: false
+              jsonData:
+                handleGrafanaManagedAlerts: false
+                implementation: prometheus
             - name: Loki
               type: loki
               access: proxy

From 6148979cd123b6b4d6f2fbab42d485aedb3bad84 Mon Sep 17 00:00:00 2001
From: Afolabi Omotoso <128552490+gitafolabi@users.noreply.github.com>
Date: Sun, 21 Jun 2026 18:51:23 +0200
Subject: [PATCH 12/14] fix: disable prometheus operator admission webhooks to
 resolve TLS cert mismatch

---
 gitops/prometheus-crds-app.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/gitops/prometheus-crds-app.yml b/gitops/prometheus-crds-app.yml
index 4c7181b..473fa15 100644
--- a/gitops/prometheus-crds-app.yml
+++ b/gitops/prometheus-crds-app.yml
@@ -53,6 +53,8 @@ spec:
         prometheusOperator:
           enabled: true
           manageCrds: true
+          admissionWebhooks:
+            enabled: false
         nodeExporter:
           enabled: true
         kubeStateMetrics:

From 1d72fc34d99299875d0831c9811a59686d8d5f7f Mon Sep 17 00:00:00 2001
From: Afolabi Omotoso <128552490+gitafolabi@users.noreply.github.com>
Date: Sun, 21 Jun 2026 19:06:05 +0200
Subject: [PATCH 13/14] ix: use cert-manager for prometheus operator admission
 webhook TLS

---
 gitops/prometheus-crds-app.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/gitops/prometheus-crds-app.yml b/gitops/prometheus-crds-app.yml
index 473fa15..cd87daa 100644
--- a/gitops/prometheus-crds-app.yml
+++ b/gitops/prometheus-crds-app.yml
@@ -54,7 +54,9 @@ spec:
           enabled: true
           manageCrds: true
           admissionWebhooks:
-            enabled: false
+            enabled: true
+            certManager:
+              enabled: true
         nodeExporter:
           enabled: true
         kubeStateMetrics:

From a1fe6304e1f8e45605221cfbee027a625e10e4e9 Mon Sep 17 00:00:00 2001
From: Afolabi Omotoso <128552490+gitafolabi@users.noreply.github.com>
Date: Sun, 21 Jun 2026 19:23:21 +0200
Subject: [PATCH 14/14] feat: upgrade loki to 3.x (chart 6.7.4) and split
 promtail into separate ArgoCD app

---
 gitops/kustomization.yml |  1 +
 gitops/loki.yml          | 78 +++++++++++++++++++++++-----------------
 gitops/promtail.yml      | 34 ++++++++++++++++++
 3 files changed, 81 insertions(+), 32 deletions(-)
 create mode 100644 gitops/promtail.yml

diff --git a/gitops/kustomization.yml b/gitops/kustomization.yml
index ebeb8f9..6694f00 100644
--- a/gitops/kustomization.yml
+++ b/gitops/kustomization.yml
@@ -43,6 +43,7 @@ resources:
   - grafana-external-secret.yml
   - cert-manager-argo.yml
   - loki.yml
+  - promtail.yml
   # ELK stack — commented out to avoid resource exhaustion; uncomment to deploy:
   # - elasticsearch-argo.yml
   # - logstash-argo.yml
diff --git a/gitops/loki.yml b/gitops/loki.yml
index 120f55b..1e6fcef 100644
--- a/gitops/loki.yml
+++ b/gitops/loki.yml
@@ -9,50 +9,64 @@ spec:
   project: default
   source:
     repoURL: 'https://grafana.github.io/helm-charts'
-    chart: loki-stack
-    targetRevision: 2.10.2
+    chart: loki
+    targetRevision: 6.7.4
     helm:
       values: |
+        deploymentMode: SingleBinary
+
         loki:
-          enabled: true
+          auth_enabled: false
+          commonConfig:
+            replication_factor: 1
+          storage:
+            type: filesystem
+          schemaConfig:
+            configs:
+              - from: "2024-04-01"
+                store: tsdb
+                object_store: filesystem
+                schema: v13
+                index:
+                  prefix: loki_index_
+                  period: 24h
+
+        singleBinary:
+          replicas: 1
           resources:
             requests:
-              cpu: 50m     
-              memory: 64Mi  
+              cpu: 100m
+              memory: 128Mi
             limits:
-              cpu: 100m     
-              memory: 128Mi 
-          persistence:
-            enabled: false  # prod: true, storageClassName: default, size: 10Gi
-          config:
-            table_manager:
-              retention_deletes_enabled: true
-              retention_period: 168h  # 7 days
-        promtail:
-          enabled: true
-          resources:
-            requests:
-              cpu: 25m      
-              memory: 32Mi  
-            limits:
-              cpu: 50m      
-              memory: 64Mi 
-        grafana:
+              cpu: 200m
+              memory: 256Mi
+
+        read:
+          replicas: 0
+        write:
+          replicas: 0
+        backend:
+          replicas: 0
+
+        chunksCache:
+          enabled: false
+        resultsCache:
+          enabled: false
+
+        minio:
+          enabled: false
+
+        gateway:
           enabled: false
-        prometheus:
+
+        test:
           enabled: false
-        fluent-bit:
+
+        lokiCanary:
           enabled: false
   destination:
     server: 'https://kubernetes.default.svc'
     namespace: monitoring
-  ignoreDifferences:
-    - group: ""
-      kind: ConfigMap
-      name: loki-stack
-      namespace: monitoring
-      jqPathExpressions:
-        - '.data["loki-stack-datasource.yaml"]'
   syncPolicy:
     automated:
       prune: true
diff --git a/gitops/promtail.yml b/gitops/promtail.yml
new file mode 100644
index 0000000..4986a21
--- /dev/null
+++ b/gitops/promtail.yml
@@ -0,0 +1,34 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: promtail
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
+spec:
+  project: default
+  source:
+    repoURL: 'https://grafana.github.io/helm-charts'
+    chart: promtail
+    targetRevision: 6.16.6
+    helm:
+      values: |
+        config:
+          clients:
+            - url: http://loki-stack:3100/loki/api/v1/push
+        resources:
+          requests:
+            cpu: 25m
+            memory: 32Mi
+          limits:
+            cpu: 50m
+            memory: 64Mi
+  destination:
+    server: 'https://kubernetes.default.svc'
+    namespace: monitoring
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true