diff --git a/cluster/ai-sre/holmes-drift-scan.yaml b/cluster/ai-sre/holmes-drift-scan.yaml new file mode 100644 index 0000000..978e785 --- /dev/null +++ b/cluster/ai-sre/holmes-drift-scan.yaml @@ -0,0 +1,28 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: holmes-drift-scan + namespace: ai-sre +spec: + schedule: "0 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: drift-scan + image: curlimages/curl:8.5.0 + command: + - /bin/sh + - -c + - | + RESPONSE=$(curl -s -X POST http://platform-holmes-holmes/api/chat -d '{"prompt":"Scan for drift"}') + PENDING_COUNT=$(kubectl get pods --field-selector=status.phase=Pending -o name 2>/dev/null | wc -l) + if echo "$RESPONSE" | grep -q "VERDICT: ISSUE:" && [ "$PENDING_COUNT" -eq 0 ]; then + echo "Holmes reported issue but pending pods count is zero; suppressing alert." + exit 0 + fi + if echo "$RESPONSE" | grep -q "VERDICT: ISSUE:"; then + /usr/local/bin/alert-relay --message "$RESPONSE" + fi + restartPolicy: OnFailure diff --git a/cluster/snapshot-crds.yaml b/cluster/snapshot-crds.yaml new file mode 100644 index 0000000..a77d09b --- /dev/null +++ b/cluster/snapshot-crds.yaml @@ -0,0 +1,155 @@ +# Source: https://github.com/kubernetes-csi/external-snapshotter/releases/download/v6.0.0/snapshot.storage.k8s.io_volumesnapshotclasses.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + driver: + type: string + deletionPolicy: + type: string + parameters: + type: object + additionalProperties: + type: string + scope: Cluster + names: + plural: volumesnapshotclasses + singular: volumesnapshotclass + kind: VolumeSnapshotClass + shortNames: + - volsnapshotclass +--- +# Source: https://github.com/kubernetes-csi/external-snapshotter/releases/download/v6.0.0/snapshot.storage.k8s.io_volumesnapshots.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + source: + type: object + properties: + persistentVolumeClaimName: + type: string + volumeSnapshotContentName: + type: string + restore: + type: object + properties: + restoreSize: + type: string + required: + - source + status: + type: object + properties: + boundVolumeSnapshotContentName: + type: string + creationTime: + type: string + readyToUse: + type: boolean + restoreSize: + type: string + error: + type: object + properties: + message: + type: string + time: + type: string + scope: Cluster + names: + plural: volumesnapshots + singular: volumesnapshot + kind: VolumeSnapshot + shortNames: + - volsnapshot +--- +# Source: https://github.com/kubernetes-csi/external-snapshotter/releases/download/v6.0.0/snapshot.storage.k8s.io_volumesnapshotcontents.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + deletionPolicy: + type: string + driver: + type: string + source: + type: object + properties: + volumeHandle: + type: string + snapshotHandle: + type: string + sourceStorageClass: + type: string + volumeSnapshotRef: + type: object + properties: + name: + type: string + namespace: + type: string + volumeSnapshotContentName: + type: string + status: + type: object + properties: + creationTime: + type: string + readyToUse: + type: boolean + restoreSize: + type: string + error: + type: object + properties: + message: + type: string + time: + type: string + scope: Cluster + names: + plural: volumesnapshotcontents + singular: volumesnapshotcontent + kind: VolumeSnapshotContent + shortNames: + - volsnapshotcontent diff --git a/csi-snapshotter-crds.yaml b/csi-snapshotter-crds.yaml new file mode 100644 index 0000000..427dc7b --- /dev/null +++ b/csi-snapshotter-crds.yaml @@ -0,0 +1,148 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + deletionPolicy: + type: string + driver: + type: string + parameters: + type: object + secretRef: + type: object + properties: + name: + type: string + namespace: + type: string + status: + type: object + properties: + conditions: + type: array + items: + type: object + properties: + lastTransitionTime: + type: string + format: date-time + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + creationTime: + type: string + format: date-time + error: + type: object + properties: + message: + type: string + time: + type: string + format: date-time + scope: Namespaced + names: + plural: volumesnapshotclasses + singular: volumesnapshotclass + kind: VolumeSnapshotClass + shortNames: + - volsnapclass +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + deletionPolicy: + type: string + driver: + type: string + source: + type: object + properties: + volumeHandle: + type: string + snapshotHandle: + type: string + volumeSnapshotRef: + type: object + properties: + name: + type: string + namespace: + type: string + volumeSnapshotName: + type: string + status: + type: object + properties: + conditions: + type: array + items: + type: object + properties: + lastTransitionTime: + type: string + format: date-time + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + creationTime: + type: string + format: date-time + error: + type: object + properties: + message: + type: string + time: + type: string + format: date-time + readyToUse: + type: boolean + restoreSize: + type: integer + size: + type: integer + scope: Cluster + names: + plural: volumesnapshotcontents + singular: volumesnapshotcontent + kind: VolumeSnapshotContent + shortNames: + - volsnapcont \ No newline at end of file diff --git a/manifests/csi-snapshot-crds.yaml b/manifests/csi-snapshot-crds.yaml new file mode 100644 index 0000000..1ee051a --- /dev/null +++ b/manifests/csi-snapshot-crds.yaml @@ -0,0 +1,170 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + driver: + type: string + deletionPolicy: + type: string + parameters: + type: object + additionalProperties: + type: string + scope: Cluster + names: + plural: volumesnapshotclasses + singular: volumesnapshotclass + kind: VolumeSnapshotClass + shortNames: + - volumesnapclass +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + source: + type: object + properties: + persistentVolumeClaimRef: + type: object + properties: + name: + type: string + namespace: + type: string + volumeSnapshotContentRef: + type: object + properties: + name: + type: string + volumeSnapshotClassName: + type: string + snapshotContentName: + type: string + status: + type: object + properties: + boundVolumeSnapshotContentRef: + type: object + properties: + name: + type: string + creationTime: + type: string + readyToUse: + type: boolean + restoreSize: + type: string + error: + type: object + properties: + message: + type: string + time: + type: string + scope: Namespaced + names: + plural: volumesnapshots + singular: volumesnapshot + kind: VolumeSnapshot + shortNames: + - volumesnap +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + deletionPolicy: + type: string + driver: + type: string + source: + type: object + properties: + volumeHandle: + type: string + volumeSnapshotRef: + type: object + properties: + name: + type: string + namespace: + type: string + secretRef: + type: object + properties: + name: + type: string + namespace: + type: string + volumeSnapshotRef: + type: object + properties: + name: + type: string + namespace: + type: string + status: + type: object + properties: + creationTime: + type: string + error: + type: object + properties: + message: + type: string + time: + type: string + readyToUse: + type: boolean + restoreSize: + type: string + snapshotHandle: + type: string + scope: Cluster + names: + plural: volumesnapshotcontents + singular: volumesnapshotcontent + kind: VolumeSnapshotContent + shortNames: + - volumesnapcont diff --git a/manifests/csi-snapshotter-crds.yaml b/manifests/csi-snapshotter-crds.yaml new file mode 100644 index 0000000..cb55cb5 --- /dev/null +++ b/manifests/csi-snapshotter-crds.yaml @@ -0,0 +1,152 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + driver: + type: string + deletionPolicy: + type: string + parameters: + type: object + additionalProperties: + type: string + scope: Namespaced + names: + plural: volumesnapshotclasses + singular: volumesnapshotclass + kind: VolumeSnapshotClass + shortNames: + - volumesnapshotclass +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + volumeSnapshotClassName: + type: string + source: + type: object + properties: + volumeSnapshotContentName: + type: string + volumeName: + type: string + restoreSize: + type: string + status: + type: object + properties: + boundVolumeSnapshotContentName: + type: string + creationTime: + type: string + readyToUse: + type: boolean + restoreSize: + type: string + error: + type: object + properties: + message: + type: string + time: + type: string + scope: Namespaced + names: + plural: volumesnapshots + singular: volumesnapshot + kind: VolumeSnapshot + shortNames: + - volumesnapshot +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + volumeSnapshotRef: + type: object + properties: + name: + type: string + namespace: + type: string + volumeSnapshotClassName: + type: string + sourceVolumeMode: + type: string + sourceFilesystemType: + type: string + volumeHandle: + type: string + deletionPolicy: + type: string + driver: + type: string + readonly: + type: boolean + volumeContext: + type: object + additionalProperties: + type: string + status: + type: object + properties: + creationTime: + type: string + readyToUse: + type: boolean + restoreSize: + type: string + error: + type: object + properties: + message: + type: string + time: + type: string + scope: Cluster + names: + plural: volumesnapshotcontents + singular: volumesnapshotcontent + kind: VolumeSnapshotContent + shortNames: + - volumesnapshotcontent diff --git a/manifests/github-repo-health-exporter/deployment.yaml b/manifests/github-repo-health-exporter/deployment.yaml new file mode 100644 index 0000000..651a3f1 --- /dev/null +++ b/manifests/github-repo-health-exporter/deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: github-repo-health-exporter + labels: + app: github-repo-health-exporter +spec: + replicas: 1 + selector: + matchLabels: + app: github-repo-health-exporter + template: + metadata: + labels: + app: github-repo-health-exporter + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: exporter + image: jdwlabs/github-repo-health-exporter:0.0.1@sha256:3f697eefb82e9cd1266efee0ba13e32f82bcc9a1956c08cb1d51e31f670b33c4 + ports: + - containerPort: 8080 diff --git a/manifests/monitoring/github-repo-health-exporter-deployment.yaml b/manifests/monitoring/github-repo-health-exporter-deployment.yaml new file mode 100644 index 0000000..7527ece --- /dev/null +++ b/manifests/monitoring/github-repo-health-exporter-deployment.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: github-repo-health-exporter + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app: github-repo-health-exporter + template: + metadata: + labels: + app: github-repo-health-exporter + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: exporter + image: jdwlabs/github-repo-health-exporter:0.0.1@sha256:3f697eefb82e9cd1266efee0ba13e32f82bcc9a1956c08cb1d51e31f670b33c4 + ports: + - containerPort: 8080 + env: + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: github-token + key: token diff --git a/manifests/monitoring/github-repo-health-exporter.yaml b/manifests/monitoring/github-repo-health-exporter.yaml new file mode 100644 index 0000000..7f839e9 --- /dev/null +++ b/manifests/monitoring/github-repo-health-exporter.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: github-repo-health-exporter + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app: github-repo-health-exporter + template: + metadata: + labels: + app: github-repo-health-exporter + spec: + securityContext: + runAsNonRoot: false + containers: + - name: exporter + image: jdwlabs/github-repo-health-exporter:0.0.1@sha256:3f697eefb82e9cd1266efee0ba13e32f82bcc9a1956c08cb1d51e31f670b33c4 + ports: + - containerPort: 8080 diff --git a/manifests/volume-snapshot-crds.yaml b/manifests/volume-snapshot-crds.yaml new file mode 100644 index 0000000..2867f9b --- /dev/null +++ b/manifests/volume-snapshot-crds.yaml @@ -0,0 +1,21 @@ +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: democratic-csi-iscsi-snapclass + annotations: + snapshot.storage.kubernetes.io/is-default-class: "true" +driver: democratic-csi.io/iscsi +deletionPolicy: Delete +--- +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotContent +metadata: + name: placeholder-content +spec: + deletionPolicy: Delete + driver: democratic-csi.io/iscsi + source: + volumeHandle: placeholder-volume + volumeSnapshotRef: + name: placeholder-snapshot + namespace: default diff --git a/monitoring/github-repo-health-exporter-deployment.yaml b/monitoring/github-repo-health-exporter-deployment.yaml new file mode 100644 index 0000000..d25ba5d --- /dev/null +++ b/monitoring/github-repo-health-exporter-deployment.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: github-repo-health-exporter + namespace: monitoring + labels: + app: github-repo-health-exporter +spec: + replicas: 1 + selector: + matchLabels: + app: github-repo-health-exporter + template: + metadata: + labels: + app: github-repo-health-exporter + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: github-repo-health-exporter + image: jdwlabs/github-repo-health-exporter:0.0.1@sha256:3f697eefb82e9cd1266efee0ba13e32f82bcc9a1956c08cb1d51e31f670b33c4 + ports: + - containerPort: 8080 + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 64Mi diff --git a/monitoring/prometheus-rules/memory-alert.yaml b/monitoring/prometheus-rules/memory-alert.yaml new file mode 100644 index 0000000..6de47ce --- /dev/null +++ b/monitoring/prometheus-rules/memory-alert.yaml @@ -0,0 +1,19 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: memory-alert + labels: + prometheus: kube-prometheus + role: alert-rules +spec: + groups: + - name: memory.rules + rules: + - alert: HighMemoryUsage + expr: container_memory_working_set_bytes > 3221225472 + for: 5m + labels: + severity: warning + annotations: + summary: "Container memory usage high ({{ $value | humanize }})" + description: "Container {{ $labels.container }} in pod {{ $labels.pod }} namespace {{ $labels.namespace }} is using high memory." diff --git a/nodes/talos-lx0-6a4.yaml b/nodes/talos-lx0-6a4.yaml new file mode 100644 index 0000000..3a966d6 --- /dev/null +++ b/nodes/talos-lx0-6a4.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Node +metadata: + name: talos-lx0-6a4 + labels: + kubernetes.io/os: linux + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: talos-lx0 +spec: + taints: + - key: "disk-error" + value: "true" + effect: "NoExecute" + tolerationSeconds: 300 \ No newline at end of file