From 1a4807e60939d3063db1b03da860e6ee9b868419 Mon Sep 17 00:00:00 2001 From: apetruni Date: Tue, 23 Jun 2026 15:31:48 +0300 Subject: [PATCH 1/3] feat: add fault [Exhausted Etcd Database Storage] (SPAI-414) Add new fault type and scenario derived from real Konflux production incidents (Web RCA data). - Fault 32: Exhausted Etcd Database Storage (6 real incidents) - Scenario 116: Applied to OTel Demo etcd service - Includes molecule test for fault injection validation Co-Authored-By: Claude Opus 4.6 (1M context) --- documentation/library/faults/README.md | 1 + .../faults/exhausted-etcd-database-storage.md | 27 ++++++ documentation/library/scenarios/README.md | 1 + documentation/library/scenarios/sre/116.md | 38 ++++++++ documentation/library/scenarios/statistics.md | 6 +- scenarios/sre/library/indexes/faults/32.json | 91 +++++++++++++++++++ .../sre/library/indexes/scenarios/116.json | 64 +++++++++++++ .../converge.yml | 20 ++++ .../create.yml | 48 ++++++++++ .../destroy.yml | 15 +++ .../molecule.yml | 24 +++++ .../verify.yml | 21 +++++ ...nject_exhausted_etcd_database_storage.yaml | 36 ++++++++ .../files/scenario_116/groundtruth.yaml | 21 +++++ .../files/scenario_116/groundtruth_v1.yaml | 60 ++++++++++++ .../files/scenario_116/scenario.yaml | 22 +++++ .../exhausted-etcd-database-storage.json | 46 ++++++++++ schemas/json/library/index/scenario.json | 17 ++++ 18 files changed, 555 insertions(+), 3 deletions(-) create mode 100644 documentation/library/faults/exhausted-etcd-database-storage.md create mode 100644 documentation/library/scenarios/sre/116.md create mode 100644 scenarios/sre/library/indexes/faults/32.json create mode 100644 scenarios/sre/library/indexes/scenarios/116.json create mode 100644 scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/converge.yml create mode 100644 scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/create.yml create mode 100644 scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/destroy.yml create mode 100644 scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/molecule.yml create mode 100644 scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/verify.yml create mode 100644 scenarios/sre/project/roles/faults/tasks/inject_exhausted_etcd_database_storage.yaml create mode 100644 scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth.yaml create mode 100644 scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth_v1.yaml create mode 100644 scenarios/sre/project/roles/scenarios/files/scenario_116/scenario.yaml create mode 100644 schemas/json/faults/exhausted-etcd-database-storage.json diff --git a/documentation/library/faults/README.md b/documentation/library/faults/README.md index aff1c1be6..431d5627b 100644 --- a/documentation/library/faults/README.md +++ b/documentation/library/faults/README.md @@ -11,6 +11,7 @@ A fault is a solvable issue injected into an environment to create an incident. | [Crashing Kubernetes Workload Init Container](./crashing-kubernetes-workload-init-container.md) | Kubernetes | | [Deleted Kubernetes Service](./deleted-kubernetes-service.md) | Kubernetes | | [Disabled Istio Ambient Mode Kubernetes Namespace](./disabled-istio-ambient-mode-kubernetes-namespace.md) | Kubernetes | +| [Exhausted Etcd Database Storage](./exhausted-etcd-database-storage.md) | Kubernetes | | [Failing Name Resolution Kubernetes Workload DNS Policy](./failing-name-resolution-kubernetes-workload-dns-policy.md) | Kubernetes | | [Hanging Kubernetes Workload Init Container](./hanging-kubernetes-workload-init-container.md) | Kubernetes | | [Ingress Port Blocking Network Policy](./ingress-port-blocking-network-policy.md) | Kubernetes | diff --git a/documentation/library/faults/exhausted-etcd-database-storage.md b/documentation/library/faults/exhausted-etcd-database-storage.md new file mode 100644 index 000000000..eb6928a34 --- /dev/null +++ b/documentation/library/faults/exhausted-etcd-database-storage.md @@ -0,0 +1,27 @@ +>[!NOTE] +>This file is autogenerated. Please do not edit. + +# Exhausted Etcd Database Storage + +| [Source](../../../../scenarios/sre/library/indexes/faults/exhausted-etcd-database-storage.json) | [Schema](../../../../schemas/json/faults/exhausted-etcd-database-storage.json) | [Implementation](../../../../scenarios/sre/project/roles/faults/tasks/inject_exhausted_etcd_database_storage.yaml) | Kubernetes | Performance | + +This fault fills the etcd database storage by creating a large number of ConfigMaps, pushing etcd usage past its quota threshold. This simulates real-world incidents where etcd storage exhaustion causes the API server to reject writes, leading to cluster-wide degradation or complete outage. + +## Expectation + +The etcd database will approach or exceed its storage quota. The Kubernetes API server will begin rejecting mutating requests. + +### Alerts + +The following alerts are expected to fire when this fault is injected. + +#### Application Alerts + +- KubePodNotReady + +## Resources + +- https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/ +- https://etcd.io/docs/v3.5/op-guide/maintenance/ +- https://kubernetes.io/docs/concepts/overview/components/#etcd +- https://kubernetes.io/docs/tasks/debug/debug-cluster/ diff --git a/documentation/library/scenarios/README.md b/documentation/library/scenarios/README.md index 6b8f87e95..455316189 100644 --- a/documentation/library/scenarios/README.md +++ b/documentation/library/scenarios/README.md @@ -73,3 +73,4 @@ _For statistics on the distrubution spread of scenarios, please see [here](./sta | [102](./sre/102.md) | sre | medium | | [105](./sre/105.md) | sre | medium | | [114](./sre/114.md) | sre | low | +| [116](./sre/116.md) | sre | high | diff --git a/documentation/library/scenarios/sre/116.md b/documentation/library/scenarios/sre/116.md new file mode 100644 index 000000000..315e4885b --- /dev/null +++ b/documentation/library/scenarios/sre/116.md @@ -0,0 +1,38 @@ +> [!NOTE] +> This file is autogenerated. Please do not edit. + +# Scenario 116 + +| [Source](../../../../scenarios/sre/library/indexes/scenarios/116.json) | High Complexity | Kubernetes | Performance | + +## Description + +This scenario simulates etcd storage exhaustion by filling the cluster's etcd database with large ConfigMaps, causing the API server to reject writes and degrading cluster-wide operations including the OpenTelemetry Demo application. + +## Applications + +- [OpenTelemetry Demo](../../applications/opentelemetry-demo.md) + +## Faults + +- [Exhausted Etcd Database Storage](../../../../scenarios/sre/docs/faults.md#Exhausted-Etcd-Database-Storage) + +## Solution Guides + +Below is a noncomprehensive list of step by step guides which can lead to remediating the one or more problems in the scenario. + +A **solution** represents the process of fixing an issue created by a fault. To remediate the issues in the scenarios, follow the solutions in order of appearance. + +A solution can have multiple **variants**. A variant is a different process of deriving the solution. If multiple variants are presented, pick only one to solve the problem. + +### Solution 1 + +#### Variant 1 + +1. Compact and defragment the etcd database to reclaim storage space. + +2. Remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage. + +#### Variant 2 + +1. Increase the etcd storage quota if the current limit is too low for the cluster workload. diff --git a/documentation/library/scenarios/statistics.md b/documentation/library/scenarios/statistics.md index 4f8df3a29..25ad06a9f 100644 --- a/documentation/library/scenarios/statistics.md +++ b/documentation/library/scenarios/statistics.md @@ -7,16 +7,16 @@ | BookInfo | OpenTelemetry Demo | | --- | --- | -| 5 | 61 | +| 5 | 36 | ## Category Distribution | FinOps | SRE | | --- | --- | -| 2 | 64 | +| 2 | 39 | ## Complexity Distribution | Low | Medium | High | | --- | --- | --- | -| 17 | 46 | 3 | +| 17 | 22 | 2 | diff --git a/scenarios/sre/library/indexes/faults/32.json b/scenarios/sre/library/indexes/faults/32.json new file mode 100644 index 000000000..c1e1bbc6e --- /dev/null +++ b/scenarios/sre/library/indexes/faults/32.json @@ -0,0 +1,91 @@ +{ + "$schema": "https://raw.githubusercontent.com/itbench-hub/ITBench/refs/heads/main/schemas/library/index/fault.json", + "alerts": { + "application": [ + "KubePodNotReady" + ] + }, + "arguments": { + "jsonSchema": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + }, + "description": "This fault fills the etcd database storage by creating a large number of ConfigMaps, pushing etcd usage past its quota threshold. This simulates real-world incidents where etcd storage exhaustion causes the API server to reject writes, leading to cluster-wide degradation or complete outage.", + "expectation": "The etcd database will approach or exceed its storage quota. The Kubernetes API server will begin rejecting mutating requests.", + "id": "exhausted-etcd-database-storage", + "index": 32, + "name": "Exhausted Etcd Database Storage", + "platform": "Kubernetes", + "resources": [ + "https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/", + "https://etcd.io/docs/v3.5/op-guide/maintenance/", + "https://kubernetes.io/docs/concepts/overview/components/#etcd", + "https://kubernetes.io/docs/tasks/debug/debug-cluster/" + ], + "solutions": { + "templates": [ + { + "steps": [ + { + "text": "Compact and defragment the etcd database to reclaim storage space." + }, + { + "text": "Remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage." + } + ] + }, + { + "steps": [ + { + "text": "Increase the etcd storage quota if the current limit is too low for the cluster workload." + } + ] + } + ] + }, + "tags": [ + "Performance" + ] +} diff --git a/scenarios/sre/library/indexes/scenarios/116.json b/scenarios/sre/library/indexes/scenarios/116.json new file mode 100644 index 000000000..31bf7ccc0 --- /dev/null +++ b/scenarios/sre/library/indexes/scenarios/116.json @@ -0,0 +1,64 @@ +{ + "$schema": "https://raw.githubusercontent.com/itbench-hub/ITBench/refs/heads/main/schemas/library/index/scenario.json", + "alerts": [ + "KubePodNotReady" + ], + "category": "sre", + "complexity": "high", + "description": "This scenario simulates etcd storage exhaustion by filling the cluster's etcd database with large ConfigMaps, causing the API server to reject writes and degrading cluster-wide operations including the OpenTelemetry Demo application.", + "disruptions": [ + { + "injections": [ + { + "args": { + "kubernetesObject": { + "apiVersion": "apps/v1", + "kind": "StatefulSet", + "metadata": { + "name": "etcd", + "namespace": "otel-demo" + } + } + }, + "id": "exhausted-etcd-database-storage" + } + ] + } + ], + "environment": { + "applications": [ + { + "id": "opentelemetry-demo" + } + ] + }, + "id": 116, + "index": 116, + "platforms": [ + "Kubernetes" + ], + "solutions": [ + [ + { + "steps": [ + { + "text": "Compact and defragment the etcd database to reclaim storage space." + }, + { + "text": "Remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage." + } + ] + }, + { + "steps": [ + { + "text": "Increase the etcd storage quota if the current limit is too low for the cluster workload." + } + ] + } + ] + ], + "tags": [ + "Performance" + ] +} diff --git a/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/converge.yml b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/converge.yml new file mode 100644 index 000000000..8b56dd00f --- /dev/null +++ b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/converge.yml @@ -0,0 +1,20 @@ +--- +- name: Run fault injection test + hosts: + - localhost + tasks: + - name: Import faults role + ansible.builtin.import_role: + name: faults + vars: + faults_cluster: + kubeconfig: "{{ cluster.kubeconfig }}" + faults_injection_tasks: + - id: exhausted-etcd-database-storage + args: +kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: test-workload + namespace: exhausted-etcd-database-storage-test diff --git a/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/create.yml b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/create.yml new file mode 100644 index 000000000..390c75d5f --- /dev/null +++ b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/create.yml @@ -0,0 +1,48 @@ +--- +- name: Create test resources + hosts: + - localhost + tasks: + - name: Create test namespace + kubernetes.core.k8s: + kubeconfig: "{{ cluster.kubeconfig }}" + resource_definition: + apiVersion: v1 + kind: Namespace + metadata: + name: exhausted-etcd-database-storage-test + state: present + + - name: Create test deployment + kubernetes.core.k8s: + kubeconfig: "{{ cluster.kubeconfig }}" + resource_definition: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: test-workload + namespace: exhausted-etcd-database-storage-test + spec: + replicas: 1 + selector: + matchLabels: + app: test-workload + template: + metadata: + labels: + app: test-workload + spec: + containers: + - name: server + image: registry.access.redhat.com/ubi10-minimal:10.2-1781509346 + command: + - /bin/sh + args: + - -c + - "sleep infinity" + ports: + - containerPort: 8080 + name: http + state: present + wait: true + wait_timeout: 120 diff --git a/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/destroy.yml b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/destroy.yml new file mode 100644 index 000000000..0d385392a --- /dev/null +++ b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/destroy.yml @@ -0,0 +1,15 @@ +--- +- name: Destroy test resources + hosts: + - localhost + tasks: + - name: Delete test namespace + kubernetes.core.k8s: + kubeconfig: "{{ cluster.kubeconfig }}" + resource_definition: + apiVersion: v1 + kind: Namespace + metadata: + name: exhausted-etcd-database-storage-test + state: absent + wait: true diff --git a/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/molecule.yml b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/molecule.yml new file mode 100644 index 000000000..490bb79ea --- /dev/null +++ b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/molecule.yml @@ -0,0 +1,24 @@ +dependency: + name: galaxy +ansible: + env: + ANSIBLE_ROLES_PATH: ../../.. + executor: + backend: ansible-playbook + args: + ansible_playbook: + - --inventory=${MOLECULE_PROJECT_DIRECTORY}/../../../inventory + - --tags=inject_faults,untagged + playbooks: + create: create.yml + converge: converge.yml + destroy: destroy.yml + verify: verify.yml +scenario: + name: inject_exhausted_etcd_database_storage + test_sequence: + - syntax + - create + - converge + - verify + - destroy diff --git a/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/verify.yml b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/verify.yml new file mode 100644 index 000000000..a690bcb09 --- /dev/null +++ b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/verify.yml @@ -0,0 +1,21 @@ +--- +- name: Verify that fault injection succeeded + hosts: + - localhost + tasks: + - name: Retrieve ConfigMaps + kubernetes.core.k8s_info: + kubeconfig: "{{ cluster.kubeconfig }}" + api_version: v1 + kind: ConfigMap + namespace: exhausted-etcd-database-storage-test + label_selectors: + - app=etcd-storage-fill + register: faults_configmaps + + - name: Validate ConfigMaps were created + ansible.builtin.assert: + that: + - faults_configmaps.resources | length > 0 + fail_msg: No fill ConfigMaps found. Fault injection may have failed. + success_msg: ConfigMaps created successfully for etcd storage pressure. diff --git a/scenarios/sre/project/roles/faults/tasks/inject_exhausted_etcd_database_storage.yaml b/scenarios/sre/project/roles/faults/tasks/inject_exhausted_etcd_database_storage.yaml new file mode 100644 index 000000000..ff9b8516b --- /dev/null +++ b/scenarios/sre/project/roles/faults/tasks/inject_exhausted_etcd_database_storage.yaml @@ -0,0 +1,36 @@ +--- +- name: Include fault argument validation tasks + ansible.builtin.include_tasks: + file: validate_fault_arguments.yaml + +- name: Create ConfigMaps to fill etcd storage + kubernetes.core.k8s: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + resource_definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: "etcd-fill-{{ item }}" + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + labels: + app: etcd-storage-fill + data: + payload: "{{ lookup('pipe', 'head -c 900000 /dev/urandom | base64 | head -c 900000') }}" + state: present + loop: "{{ range(1, 201) | list }}" + loop_control: + label: "etcd-fill-{{ item }}" + +- name: Verify etcd storage pressure + kubernetes.core.k8s_info: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + api_version: v1 + kind: ConfigMap + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + label_selectors: + - app=etcd-storage-fill + register: faults_etcd_fill_result + until: + - faults_etcd_fill_result.resources | length >= 200 + retries: 5 + delay: 10 diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth.yaml new file mode 100644 index 000000000..79261568e --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: itbench.io/v2 +kind: GroundTruth +metadata: + name: scenario-116 +spec: + alerts: + - labels: {} + name: KubePodNotReady + entities: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: etcd + namespace: otel-demo + solutions: + - - steps: + - text: Compact and defragment the etcd database to reclaim storage space. + - text: Remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage. + - steps: + - text: Increase the etcd storage quota if the current limit is too low for the cluster workload. diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth_v1.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth_v1.yaml new file mode 100644 index 000000000..3c2fdf914 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth_v1.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: itbench.io/v1 +kind: GroundTruth +metadata: + name: scenario-116 +spec: + alerts: + - group_id: etcd-statefulset-1 + id: KubePodNotReady + metadata: + description: etcd pod in namespace otel-demo is experiencing storage pressure due to excessive ConfigMap creation + groups: + - id: etcd-statefulset-1 + kind: StatefulSet + namespace: otel-demo + filter: + - etcd\b + root_cause: true + - id: etcd-pod-1 + kind: Pod + namespace: otel-demo + filter: + - etcd-.* + - id: configmap-fill-1 + kind: ConfigMap + namespace: otel-demo + filter: + - etcd-fill-.* + aliases: + - - etcd-statefulset-1 + - etcd-pod-1 + propagations: + - source: configmap-fill-1 + target: etcd-statefulset-1 + condition: Large number of ConfigMaps created consuming etcd storage beyond quota threshold + effect: etcd database approaches storage quota limit, API server begins rejecting mutating requests + - source: etcd-statefulset-1 + target: etcd-pod-1 + condition: etcd storage quota exceeded or nearly exceeded + effect: etcd pods report high storage usage, cluster operations degrade as writes are rejected + fault: + - category: Fill + condition: etcd database filled with large ConfigMaps pushing storage past quota threshold + entity: + group_id: configmap-fill-1 + kind: ConfigMap + name: etcd-fill + fault_mechanism: Storage exhaustion via bulk ConfigMap creation simulating real-world etcd quota incidents + recommendedActions: + - solution: + actions: + - Delete the etcd-fill ConfigMaps using kubectl delete configmap -l app=etcd-storage-fill -n otel-demo + - Compact and defragment etcd to reclaim space + - Monitor etcd storage usage returning to normal levels + id: cleanup-configmaps + - solution: + actions: + - Increase etcd storage quota if current limit is insufficient + - Review resource creation patterns to prevent future exhaustion + id: increase-quota diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_116/scenario.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_116/scenario.yaml new file mode 100644 index 000000000..a7bd1a05b --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_116/scenario.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: itbench.io/v2 +kind: Scenario +metadata: + name: scenario-116 +spec: + applications: + opentelemetryDemo: + enabled: true + faults: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: etcd + namespace: otel-demo + id: exhausted-etcd-database-storage + tools: + sre: + enabled: true diff --git a/schemas/json/faults/exhausted-etcd-database-storage.json b/schemas/json/faults/exhausted-etcd-database-storage.json new file mode 100644 index 000000000..29c1419f2 --- /dev/null +++ b/schemas/json/faults/exhausted-etcd-database-storage.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" +} diff --git a/schemas/json/library/index/scenario.json b/schemas/json/library/index/scenario.json index 2566fce46..870656fed 100644 --- a/schemas/json/library/index/scenario.json +++ b/schemas/json/library/index/scenario.json @@ -294,6 +294,22 @@ } } }, + { + "if": { + "properties": { + "id": { + "const": "exhausted-etcd-database-storage" + } + } + }, + "then": { + "properties": { + "args": { + "$ref": "../../faults/exhausted-etcd-database-storage.json" + } + } + } + }, { "if": { "properties": { @@ -556,6 +572,7 @@ "crashing-kubernetes-workload-init-container", "valkey-workload-out-of-memory", "misconfigured-kubernetes-workload-container-readiness-probe", + "exhausted-etcd-database-storage", "unassigned-kubernetes-workload-container-resource-limits", "unschedulable-kubernetes-workload-pod-anti-affinity-rule", "modified-kubernetes-workload-container-environment-variable", From 55dd57999923785a01146991eb3c6c3229b5f48d Mon Sep 17 00:00:00 2001 From: apetruni Date: Tue, 4 Aug 2026 14:06:05 +0300 Subject: [PATCH 2/3] fix: rework etcd fault from full exhaustion to degradation Instead of fully exhausting etcd (which locks out kubectl and prevents cleanup), create storage pressure that causes latency degradation while keeping the cluster usable: - Reduce from 200x900KB to 50x100KB ConfigMaps - Change alert from KubePodNotReady to HighRequestLatency - Add kubectl cleanup commands to solutions - Allow Deployment kind (not just StatefulSet) - Target frontend deployment instead of etcd StatefulSet - Register fault in argument_specs and task_files - Add fault and scenario Jinja2 templates - Use releases[] variable in scenario template Co-Authored-By: Claude Opus 4.6 (1M context) --- .../faults/exhausted-etcd-database-storage.md | 8 +-- documentation/library/scenarios/sre/116.md | 13 +++-- scenarios/sre/library/indexes/faults/32.json | 17 ++++-- .../sre/library/indexes/scenarios/116.json | 16 ++++-- .../roles/faults/meta/argument_specs.yaml | 1 + .../verify.yml | 4 +- ...nject_exhausted_etcd_database_storage.yaml | 10 ++-- .../roles/faults/vars/main/task_files.yaml | 2 + .../files/scenario_116/groundtruth.yaml | 10 ++-- .../files/scenario_116/groundtruth_v1.yaml | 57 +++++++++++-------- .../files/scenario_116/scenario.yaml | 4 +- .../library/indexes/faults/32.yaml.j2 | 57 +++++++++++++++++++ .../library/indexes/scenarios/116.yaml.j2 | 22 +++++++ .../exhausted-etcd-database-storage.json | 1 + 14 files changed, 166 insertions(+), 56 deletions(-) create mode 100644 scenarios/sre/templates/library/indexes/faults/32.yaml.j2 create mode 100644 scenarios/sre/templates/library/indexes/scenarios/116.yaml.j2 diff --git a/documentation/library/faults/exhausted-etcd-database-storage.md b/documentation/library/faults/exhausted-etcd-database-storage.md index eb6928a34..30ac17d77 100644 --- a/documentation/library/faults/exhausted-etcd-database-storage.md +++ b/documentation/library/faults/exhausted-etcd-database-storage.md @@ -5,19 +5,19 @@ | [Source](../../../../scenarios/sre/library/indexes/faults/exhausted-etcd-database-storage.json) | [Schema](../../../../schemas/json/faults/exhausted-etcd-database-storage.json) | [Implementation](../../../../scenarios/sre/project/roles/faults/tasks/inject_exhausted_etcd_database_storage.yaml) | Kubernetes | Performance | -This fault fills the etcd database storage by creating a large number of ConfigMaps, pushing etcd usage past its quota threshold. This simulates real-world incidents where etcd storage exhaustion causes the API server to reject writes, leading to cluster-wide degradation or complete outage. +This fault creates etcd storage pressure by writing a large number of ConfigMaps, causing increased API server latency and degraded cluster performance. This simulates real-world incidents where etcd storage growth causes progressive degradation of cluster operations. ## Expectation -The etcd database will approach or exceed its storage quota. The Kubernetes API server will begin rejecting mutating requests. +The etcd database will experience increased storage pressure. The Kubernetes API server will exhibit higher latency for read and write operations. ### Alerts The following alerts are expected to fire when this fault is injected. -#### Application Alerts +#### Golden Signal Alerts -- KubePodNotReady +- HighRequestLatency ## Resources diff --git a/documentation/library/scenarios/sre/116.md b/documentation/library/scenarios/sre/116.md index 315e4885b..c9508bb87 100644 --- a/documentation/library/scenarios/sre/116.md +++ b/documentation/library/scenarios/sre/116.md @@ -7,7 +7,7 @@ ## Description -This scenario simulates etcd storage exhaustion by filling the cluster's etcd database with large ConfigMaps, causing the API server to reject writes and degrading cluster-wide operations including the OpenTelemetry Demo application. +This scenario simulates etcd storage pressure by filling the cluster's etcd database with ConfigMaps, causing increased API server latency and degrading cluster-wide operations including the OpenTelemetry Demo application. ## Applications @@ -29,10 +29,15 @@ A solution can have multiple **variants**. A variant is a different process of d #### Variant 1 -1. Compact and defragment the etcd database to reclaim storage space. +1. Remove the ConfigMaps that are consuming etcd storage. +```shell +kubectl -n otel-demo delete configmap -l app=etcd-storage-fill +``` -2. Remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage. +2. Compact and defragment the etcd database to reclaim storage space. #### Variant 2 -1. Increase the etcd storage quota if the current limit is too low for the cluster workload. +1. Identify and remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage. + +2. Increase the etcd storage quota if the current limit is too low for the cluster workload. diff --git a/scenarios/sre/library/indexes/faults/32.json b/scenarios/sre/library/indexes/faults/32.json index c1e1bbc6e..f188d7046 100644 --- a/scenarios/sre/library/indexes/faults/32.json +++ b/scenarios/sre/library/indexes/faults/32.json @@ -1,8 +1,8 @@ { "$schema": "https://raw.githubusercontent.com/itbench-hub/ITBench/refs/heads/main/schemas/library/index/fault.json", "alerts": { - "application": [ - "KubePodNotReady" + "goldenSignal": [ + "HighRequestLatency" ] }, "arguments": { @@ -18,6 +18,7 @@ }, "kind": { "enum": [ + "Deployment", "StatefulSet" ], "type": "string" @@ -52,8 +53,8 @@ "type": "object" } }, - "description": "This fault fills the etcd database storage by creating a large number of ConfigMaps, pushing etcd usage past its quota threshold. This simulates real-world incidents where etcd storage exhaustion causes the API server to reject writes, leading to cluster-wide degradation or complete outage.", - "expectation": "The etcd database will approach or exceed its storage quota. The Kubernetes API server will begin rejecting mutating requests.", + "description": "This fault creates etcd storage pressure by writing a large number of ConfigMaps, causing increased API server latency and degraded cluster performance. This simulates real-world incidents where etcd storage growth causes progressive degradation of cluster operations.", + "expectation": "The etcd database will experience increased storage pressure. The Kubernetes API server will exhibit higher latency for read and write operations.", "id": "exhausted-etcd-database-storage", "index": 32, "name": "Exhausted Etcd Database Storage", @@ -69,15 +70,19 @@ { "steps": [ { - "text": "Compact and defragment the etcd database to reclaim storage space." + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} delete configmap -l app=etcd-storage-fill", + "text": "Remove the ConfigMaps that are consuming etcd storage." }, { - "text": "Remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage." + "text": "Compact and defragment the etcd database to reclaim storage space." } ] }, { "steps": [ + { + "text": "Identify and remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage." + }, { "text": "Increase the etcd storage quota if the current limit is too low for the cluster workload." } diff --git a/scenarios/sre/library/indexes/scenarios/116.json b/scenarios/sre/library/indexes/scenarios/116.json index 31bf7ccc0..b3b45adce 100644 --- a/scenarios/sre/library/indexes/scenarios/116.json +++ b/scenarios/sre/library/indexes/scenarios/116.json @@ -1,11 +1,11 @@ { "$schema": "https://raw.githubusercontent.com/itbench-hub/ITBench/refs/heads/main/schemas/library/index/scenario.json", "alerts": [ - "KubePodNotReady" + "HighRequestLatency" ], "category": "sre", "complexity": "high", - "description": "This scenario simulates etcd storage exhaustion by filling the cluster's etcd database with large ConfigMaps, causing the API server to reject writes and degrading cluster-wide operations including the OpenTelemetry Demo application.", + "description": "This scenario simulates etcd storage pressure by filling the cluster's etcd database with ConfigMaps, causing increased API server latency and degrading cluster-wide operations including the OpenTelemetry Demo application.", "disruptions": [ { "injections": [ @@ -13,9 +13,9 @@ "args": { "kubernetesObject": { "apiVersion": "apps/v1", - "kind": "StatefulSet", + "kind": "Deployment", "metadata": { - "name": "etcd", + "name": "frontend", "namespace": "otel-demo" } } @@ -42,15 +42,19 @@ { "steps": [ { - "text": "Compact and defragment the etcd database to reclaim storage space." + "command": "kubectl -n otel-demo delete configmap -l app=etcd-storage-fill", + "text": "Remove the ConfigMaps that are consuming etcd storage." }, { - "text": "Remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage." + "text": "Compact and defragment the etcd database to reclaim storage space." } ] }, { "steps": [ + { + "text": "Identify and remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage." + }, { "text": "Increase the etcd storage quota if the current limit is too low for the cluster workload." } diff --git a/scenarios/sre/project/roles/faults/meta/argument_specs.yaml b/scenarios/sre/project/roles/faults/meta/argument_specs.yaml index 8911e1793..547bc1304 100644 --- a/scenarios/sre/project/roles/faults/meta/argument_specs.yaml +++ b/scenarios/sre/project/roles/faults/meta/argument_specs.yaml @@ -24,6 +24,7 @@ argument_specs: - crashing-kubernetes-workload-init-container - deleted-kubernetes-service - disabled-istio-ambient-mode-kubernetes-namespace + - exhausted-etcd-database-storage - failing-name-resolution-kubernetes-workload-dns-policy - hanging-kubernetes-workload-init-container - ingress-port-blocking-network-policy diff --git a/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/verify.yml b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/verify.yml index a690bcb09..ca5452820 100644 --- a/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/verify.yml +++ b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/verify.yml @@ -16,6 +16,6 @@ - name: Validate ConfigMaps were created ansible.builtin.assert: that: - - faults_configmaps.resources | length > 0 - fail_msg: No fill ConfigMaps found. Fault injection may have failed. + - faults_configmaps.resources | length >= 50 + fail_msg: Expected at least 50 fill ConfigMaps. Fault injection may have failed. success_msg: ConfigMaps created successfully for etcd storage pressure. diff --git a/scenarios/sre/project/roles/faults/tasks/inject_exhausted_etcd_database_storage.yaml b/scenarios/sre/project/roles/faults/tasks/inject_exhausted_etcd_database_storage.yaml index ff9b8516b..426c3a8f5 100644 --- a/scenarios/sre/project/roles/faults/tasks/inject_exhausted_etcd_database_storage.yaml +++ b/scenarios/sre/project/roles/faults/tasks/inject_exhausted_etcd_database_storage.yaml @@ -3,7 +3,7 @@ ansible.builtin.include_tasks: file: validate_fault_arguments.yaml -- name: Create ConfigMaps to fill etcd storage +- name: Create ConfigMaps to pressure etcd storage kubernetes.core.k8s: kubeconfig: "{{ faults_cluster.kubeconfig }}" resource_definition: @@ -15,13 +15,13 @@ labels: app: etcd-storage-fill data: - payload: "{{ lookup('pipe', 'head -c 900000 /dev/urandom | base64 | head -c 900000') }}" + payload: "{{ lookup('ansible.builtin.password', '/dev/null chars=ascii_lowercase,digits length=102400') }}" state: present - loop: "{{ range(1, 201) | list }}" + loop: "{{ range(1, 51) | list }}" loop_control: label: "etcd-fill-{{ item }}" -- name: Verify etcd storage pressure +- name: Verify ConfigMaps were created kubernetes.core.k8s_info: kubeconfig: "{{ faults_cluster.kubeconfig }}" api_version: v1 @@ -31,6 +31,6 @@ - app=etcd-storage-fill register: faults_etcd_fill_result until: - - faults_etcd_fill_result.resources | length >= 200 + - faults_etcd_fill_result.resources | length >= 50 retries: 5 delay: 10 diff --git a/scenarios/sre/project/roles/faults/vars/main/task_files.yaml b/scenarios/sre/project/roles/faults/vars/main/task_files.yaml index 70e94336d..e7d927b83 100644 --- a/scenarios/sre/project/roles/faults/vars/main/task_files.yaml +++ b/scenarios/sre/project/roles/faults/vars/main/task_files.yaml @@ -8,6 +8,8 @@ faults_task_files: injection: inject_deleted_kubernetes_service.yaml disabled-istio-ambient-mode-kubernetes-namespace: injection: inject_disabled_istio_ambient_mode_kubernetes_namespace.yaml + exhausted-etcd-database-storage: + injection: inject_exhausted_etcd_database_storage.yaml failing-name-resolution-kubernetes-workload-dns-policy: injection: inject_failing_name_resolution_kubernetes_workload_dns_policy.yaml hanging-kubernetes-workload-init-container: diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth.yaml index 79261568e..67d6e764d 100644 --- a/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth.yaml +++ b/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth.yaml @@ -6,16 +6,18 @@ metadata: spec: alerts: - labels: {} - name: KubePodNotReady + name: HighRequestLatency entities: - apiVersion: apps/v1 - kind: StatefulSet + kind: Deployment metadata: - name: etcd + name: frontend namespace: otel-demo solutions: - - steps: + - command: kubectl -n otel-demo delete configmap -l app=etcd-storage-fill + text: Remove the ConfigMaps that are consuming etcd storage. - text: Compact and defragment the etcd database to reclaim storage space. - - text: Remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage. - steps: + - text: Identify and remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage. - text: Increase the etcd storage quota if the current limit is too low for the cluster workload. diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth_v1.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth_v1.yaml index 3c2fdf914..91371e9e5 100644 --- a/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth_v1.yaml +++ b/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth_v1.yaml @@ -5,47 +5,58 @@ metadata: name: scenario-116 spec: alerts: - - group_id: etcd-statefulset-1 - id: KubePodNotReady + - group_id: frontend-service-1 + id: HighRequestLatency metadata: - description: etcd pod in namespace otel-demo is experiencing storage pressure due to excessive ConfigMap creation + description: Increased API server latency in namespace otel-demo due to etcd storage pressure from excessive ConfigMap creation groups: - - id: etcd-statefulset-1 - kind: StatefulSet + - id: configmap-fill-1 + kind: ConfigMap namespace: otel-demo filter: - - etcd\b + - etcd-fill-.* root_cause: true - - id: etcd-pod-1 + - id: frontend-deployment-1 + kind: Deployment + namespace: otel-demo + filter: + - frontend\b + - id: frontend-pod-1 kind: Pod namespace: otel-demo filter: - - etcd-.* - - id: configmap-fill-1 - kind: ConfigMap + - frontend-.* + - id: frontend-service-1 + kind: Service namespace: otel-demo filter: - - etcd-fill-.* + - frontend\b aliases: - - - etcd-statefulset-1 - - etcd-pod-1 + - - frontend-deployment-1 + - frontend-pod-1 + - - frontend-deployment-1 + - frontend-service-1 propagations: - source: configmap-fill-1 - target: etcd-statefulset-1 - condition: Large number of ConfigMaps created consuming etcd storage beyond quota threshold - effect: etcd database approaches storage quota limit, API server begins rejecting mutating requests - - source: etcd-statefulset-1 - target: etcd-pod-1 - condition: etcd storage quota exceeded or nearly exceeded - effect: etcd pods report high storage usage, cluster operations degrade as writes are rejected + target: frontend-deployment-1 + condition: Large number of ConfigMaps created consuming etcd storage + effect: etcd database experiences storage pressure, API server latency increases for all operations in the namespace + - source: frontend-deployment-1 + target: frontend-pod-1 + condition: API server latency increased due to etcd storage pressure + effect: Frontend pods experience slower responses from Kubernetes API, degrading application performance + - source: frontend-pod-1 + target: frontend-service-1 + condition: Frontend pods responding slowly due to cluster-wide API latency + effect: Frontend service exhibits higher request latency visible to end users fault: - category: Fill - condition: etcd database filled with large ConfigMaps pushing storage past quota threshold + condition: etcd database filled with ConfigMaps creating storage pressure entity: group_id: configmap-fill-1 kind: ConfigMap name: etcd-fill - fault_mechanism: Storage exhaustion via bulk ConfigMap creation simulating real-world etcd quota incidents + fault_mechanism: Storage pressure via bulk ConfigMap creation simulating real-world etcd storage growth incidents recommendedActions: - solution: actions: @@ -56,5 +67,5 @@ spec: - solution: actions: - Increase etcd storage quota if current limit is insufficient - - Review resource creation patterns to prevent future exhaustion + - Review resource creation patterns to prevent future growth id: increase-quota diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_116/scenario.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_116/scenario.yaml index a7bd1a05b..dc24fc633 100644 --- a/scenarios/sre/project/roles/scenarios/files/scenario_116/scenario.yaml +++ b/scenarios/sre/project/roles/scenarios/files/scenario_116/scenario.yaml @@ -12,9 +12,9 @@ spec: - args: kubernetesObject: apiVersion: apps/v1 - kind: StatefulSet + kind: Deployment metadata: - name: etcd + name: frontend namespace: otel-demo id: exhausted-etcd-database-storage tools: diff --git a/scenarios/sre/templates/library/indexes/faults/32.yaml.j2 b/scenarios/sre/templates/library/indexes/faults/32.yaml.j2 new file mode 100644 index 000000000..06c9a0676 --- /dev/null +++ b/scenarios/sre/templates/library/indexes/faults/32.yaml.j2 @@ -0,0 +1,57 @@ +# yaml-language-server: $schema=../../../../../../schemas/json/library/index/fault.json +--- +alerts: + goldenSignal: + - HighRequestLatency +arguments: + jsonSchema: + properties: + kubernetesObject: + properties: + apiVersion: + enum: + - apps/v1 + type: string + kind: + enum: + - Deployment + - StatefulSet + type: string + metadata: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + required: + - apiVersion + - kind + - metadata + type: object + required: + - kubernetesObject + type: object +description: This fault creates etcd storage pressure by writing a large number of ConfigMaps, causing increased API server latency and degraded cluster performance. This simulates real-world incidents where etcd storage growth causes progressive degradation of cluster operations. +expectation: The etcd database will experience increased storage pressure. The Kubernetes API server will exhibit higher latency for read and write operations. +name: Exhausted Etcd Database Storage +platform: Kubernetes +resources: +- https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/ +- https://etcd.io/docs/v3.5/op-guide/maintenance/ +- https://kubernetes.io/docs/concepts/overview/components/#etcd +- https://kubernetes.io/docs/tasks/debug/debug-cluster/ +solutions: + templates: + - steps: + - command: kubectl -n {{ args.kubernetesObject.metadata.namespace }} delete configmap -l app=etcd-storage-fill + text: Remove the ConfigMaps that are consuming etcd storage. + - text: Compact and defragment the etcd database to reclaim storage space. + - steps: + - text: Identify and remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage. + - text: Increase the etcd storage quota if the current limit is too low for the cluster workload. +tags: +- Performance diff --git a/scenarios/sre/templates/library/indexes/scenarios/116.yaml.j2 b/scenarios/sre/templates/library/indexes/scenarios/116.yaml.j2 new file mode 100644 index 000000000..76fe1e30d --- /dev/null +++ b/scenarios/sre/templates/library/indexes/scenarios/116.yaml.j2 @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=../../../../../../schemas/json/library/index/scenario.json +--- +category: sre +complexity: high +description: This scenario simulates etcd storage pressure by filling the cluster's etcd database with ConfigMaps, causing increased API server latency and degrading cluster-wide operations including the OpenTelemetry Demo application. +disruptions: +- injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: frontend + namespace: {{ releases["applications"]["opentelemetry_demo"]["kubernetes"]["namespace"] }} + id: exhausted-etcd-database-storage +environment: + applications: + - id: opentelemetry-demo +id: 116 +solutionTemplates: +- disruptionIndex: 0 + injectionIndex: 0 diff --git a/schemas/json/faults/exhausted-etcd-database-storage.json b/schemas/json/faults/exhausted-etcd-database-storage.json index 29c1419f2..e19e2a60d 100644 --- a/schemas/json/faults/exhausted-etcd-database-storage.json +++ b/schemas/json/faults/exhausted-etcd-database-storage.json @@ -11,6 +11,7 @@ }, "kind": { "enum": [ + "Deployment", "StatefulSet" ], "type": "string" From 68709ecb2258a25bcac3748bb83a9c8ee46aa1c6 Mon Sep 17 00:00:00 2001 From: apetruni Date: Wed, 5 Aug 2026 14:59:01 +0300 Subject: [PATCH 3/3] fix: lint errors in converge.yml and molecule.yml - Fix converge.yml kubernetesObject indentation under args - Fix molecule.yml indentation to match project conventions Co-Authored-By: Claude Opus 4.6 (1M context) --- .../converge.yml | 12 ++++++------ .../molecule.yml | 17 ++++++++++------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/converge.yml b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/converge.yml index 8b56dd00f..fd8cca45a 100644 --- a/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/converge.yml +++ b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/converge.yml @@ -12,9 +12,9 @@ faults_injection_tasks: - id: exhausted-etcd-database-storage args: -kubernetesObject: - apiVersion: apps/v1 - kind: Deployment - metadata: - name: test-workload - namespace: exhausted-etcd-database-storage-test + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: test-workload + namespace: exhausted-etcd-database-storage-test diff --git a/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/molecule.yml b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/molecule.yml index 490bb79ea..d95dab303 100644 --- a/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/molecule.yml +++ b/scenarios/sre/project/roles/faults/molecule/inject_exhausted_etcd_database_storage/molecule.yml @@ -1,5 +1,7 @@ +--- dependency: name: galaxy + ansible: env: ANSIBLE_ROLES_PATH: ../../.. @@ -7,18 +9,19 @@ ansible: backend: ansible-playbook args: ansible_playbook: - - --inventory=${MOLECULE_PROJECT_DIRECTORY}/../../../inventory - - --tags=inject_faults,untagged + - --inventory=${MOLECULE_PROJECT_DIRECTORY}/../../../inventory + - --tags=inject_faults,untagged playbooks: create: create.yml converge: converge.yml destroy: destroy.yml verify: verify.yml + scenario: name: inject_exhausted_etcd_database_storage test_sequence: - - syntax - - create - - converge - - verify - - destroy + - syntax + - create + - converge + - verify + - destroy