diff --git a/.gitignore b/.gitignore index 039f5bc40..528d561ac 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ .DS_Store +# Exported observability information (topology, traces, metrics, alerts) +exports/ + # Files associated with group variables (v1) scenarios/sre/group_vars/*/*.yaml scenarios/sre/dev/remote_cluster/group_vars/*/*.yaml diff --git a/.secrets.baseline b/.secrets.baseline index e869b94ef..22f78e512 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1671,6 +1671,15 @@ "verified_result": null } ], + "scenarios/sre/project/roles/awx/files/awx/credential_types/datadog_injectors.json": [ + { + "hashed_secret": "d2e2ab0f407e4ee3cf2ab87d61c31b25a74085e5", + "is_verified": false, + "line_number": 4, + "type": "Secret Keyword", + "verified_result": null + } + ], "scenarios/sre/project/roles/awx/tasks/configure_projects.yaml": [ { "hashed_secret": "d2e2ab0f407e4ee3cf2ab87d61c31b25a74085e5", @@ -1684,7 +1693,7 @@ { "hashed_secret": "28ed3a797da3c48c309a4ef792147f3c56cfec40", "is_verified": false, - "line_number": 13, + "line_number": 17, "type": "Secret Keyword", "verified_result": null } diff --git a/scenarios/sre/.gitignore b/scenarios/sre/.gitignore new file mode 100644 index 000000000..c58f1f702 --- /dev/null +++ b/scenarios/sre/.gitignore @@ -0,0 +1,8 @@ +# Observability vendor onboarding artifacts (token-bearing or environment-specific). +# e.g. secrets/dynatrace/dynakube.yaml, secrets/datadog/datadog-agent.yaml +secrets/ + +# AWX runner vendor configuration contains secret inputs (manifest/dynakube +# contents, API/app keys). Must be Ansible-Vault encrypted; never commit plaintext. +inventory/group_vars/runner/vendors.yaml +group_vars/runner/vendors.yaml diff --git a/scenarios/sre/docs/observability-vendors.md b/scenarios/sre/docs/observability-vendors.md new file mode 100644 index 000000000..984cc9b04 --- /dev/null +++ b/scenarios/sre/docs/observability-vendors.md @@ -0,0 +1,543 @@ +# Multi-Vendor Observability Support (Layer A) + +Design and implementation plan for deploying third-party SaaS observability +agents into ITBench SRE clusters, for both local and AWX-driven deployments. + +Vendors in scope: **Datadog, Dynatrace**. + +- **Layer A (this document):** deploy each vendor's agent/collector into the + Kubernetes/OpenShift cluster so telemetry flows to the vendor's SaaS tenant. +- **Layer B (future):** expose each vendor's tenant URL + a scoped read token to + the diagnosing SRE agent via the agent bundle so it can query telemetry during + incidents. Out of scope here. + +All backends are **SaaS**. This is the key architectural difference from the +existing in-cluster tools (Prometheus, ClickHouse, Jaeger), whose endpoints and +credentials are *discovered at runtime* from cluster Services/Secrets. SaaS +vendors invert this: **credentials are inputs** supplied by the operator, and +there is **no in-cluster endpoint to discover** — the "endpoint" is the vendor's +tenant URL. + +--- + +## Existing conventions (how tools are wired today) + +The SRE tooling lives under `scenarios/sre/`. There is no OO interface; it is an +Ansible convention-over-configuration pattern: + +| Concern | Mechanism | +|---------|-----------| +| Tool registry | `tools_managers` dict in `project/roles/tools/defaults/main/managers.yaml` | +| Install/uninstall "interface" | Task-file naming: `install_.yaml`, `uninstall_.yaml` | +| Endpoint discovery | `set_internal_endpoints_.yaml` / `set_external_endpoints_.yaml` | +| Credentials | `set_credentials_.yaml` (reads K8s Secrets) | +| Orchestration | `project/roles/tools/tasks/install.yaml` imports each installer, flag-gated | +| Enablement | Boolean flags in `group_vars` mapped via `project/manage_tools.yaml` | +| Helm values | `templates/helm//values.j2` | +| K8s resources | `templates/kubernetes//.j2` | +| AWX | `project/roles/awx/` job templates + custom credential types | + +Reference install to copy for Helm-based tools: `install_opencost.yaml`. + +--- + +## Status summary + +- **Dynatrace: implemented and validated on OpenShift (CRC).** Full lifecycle + exercised against a live OpenShift cluster + real Dynatrace tenant: operator + install (chart `1.9.0`, `platform=openshift`), multi-DynaKube manifest + (`v1beta6`) validation, namespace ownership, collision check, apply, DynaKube + reconcile, and staged credential-independent uninstall with namespace + preservation. Validation surfaced and fixed four issues: relative manifest path + resolution, the collision-check helper's `no_log` interaction, uninstall + discovering the DynaKube API version from the CRD, and allowing multiple + DynaKube documents. Not yet validated on plain (non-OpenShift) Kubernetes. +- **Datadog: implemented, unvalidated.** All install/uninstall/AWX/validation + code exists and passes YAML/schema checks, but has **not** been run against a + live cluster or SaaS tenant, and remains blocked on OpenShift. + +Molecule coverage is **required** before a vendor is treated as validated: +- `molecule/vendor_contract/` — credential-free, cluster-free contract tests + (manifest validation, Secret-document rejection, secret/key-reference + matching, and the non-secret projection allowlist). Always runs. +- `molecule/deployment_vendors/` — opt-in live deploy that self-skips when vendor + secrets are absent. + +### OpenShift support + +**Dynatrace: supported and validated on OpenShift** (CRC / OpenShift, cluster +`1.35.5`, operator chart `1.9.0`). The `dynatrace-operator` Helm chart natively +handles OpenShift: setting `platform=openshift` makes the operator create and +manage the `SecurityContextConstraints` its agents (OneAgent, ActiveGate, CSI +driver) need. We therefore do **not** hand-craft SCCs for Dynatrace — the +install task sets the `platform` Helm value on OpenShift and the operator owns +its SCCs (Helm uninstall removes them). Validated end-to-end: operator + webhook ++ CSI driver + ActiveGate + OTel collector reached Running; DynaKube resources +reconciled; uninstall staged cleanly (DynaKube → operator → workload drain) and +preserved the namespace when not exclusively ITBench-owned. + +##### KSPM AppArmor host path on RHEL CoreOS (edit your `dynakube.yaml`) + +If the DynaKube enables **KSPM** (Kubernetes Security Posture Management), the +operator generates a `node-config-collector` DaemonSet that mounts every +`spec.kspm.mappedHostPaths` entry as a `hostPath` with `type: Directory`. The +onboarding UI commonly includes `/sys/kernel/security/apparmor` in that list. + +RHEL CoreOS — the OpenShift node OS — uses **SELinux, not AppArmor**, so +`/sys/kernel/security/apparmor` does not exist on the node. The mount then fails +with `hostPath type check failed: /sys/kernel/security/apparmor is not a +directory` and the collector pod is stuck in `ContainerCreating`. + +Because ITBench treats the downloaded `dynakube.yaml` as authoritative and does +**not** rewrite its spec, the fix is made in **your file**: remove that one entry +from `spec.kspm.mappedHostPaths`. KSPM otherwise stays fully enabled. + +```yaml + kspm: + mappedHostPaths: + - /boot + - /etc + - /proc/sys/kernel + - /sys/fs + # - /sys/kernel/security/apparmor # remove on SELinux nodes (RHEL CoreOS) + - /usr/lib/systemd/system + - /var/lib +``` + +Re-run the tools playbook after editing; the operator regenerates the DaemonSet +without the AppArmor mount and the collector reaches `Running`. This caveat is +also documented inline at the top of +`project/roles/tools/tasks/install_dynatrace.yaml`. + +**Datadog: still blocked on OpenShift** (hard assertion). The Datadog operator's +OpenShift SCC story has not been validated here; the block stays until it is. + +> Note: Some older subsections below still describe an earlier namespace-`.j2` / +> SCC-annotation approach and an `allow_openshift` override. Those are superseded: +> namespaces are created via `ensure_vendor_namespace.yaml`, and Dynatrace +> OpenShift support is via the chart's `platform=openshift` value (not custom +> SCCs). + +#### Future work: how to unblock OpenShift (decision: implement later) + +The repo already has a working SCC precedent, so the intended path is settled — +it is deferred only until an OpenShift cluster + vendor tenant is available to +validate the specifics. **Chosen approach: custom least-privilege SCCs bound to +the agent service accounts (the Chaos Mesh pattern in +`install_chaos_mesh.yaml:6-57` / `uninstall_chaos_mesh.yaml:40-67`).** This was +preferred over binding to the built-in `system:openshift:scc:privileged` +(the `RoleBinding` idiom in `applications/.../install_opentelemetry_demo.yaml:52-69`) +because `privileged` grants far more than these agents need; the benchmark runs +untrusted workloads alongside fault injection, so least privilege matters. + +To implement (per vendor), inside a `when: tools_cluster.platform == 'openshift'` +block, mirroring Chaos Mesh: +1. Create a `SecurityContextConstraints` labeled + `app.kubernetes.io/managed-by: ITBench` + a per-vendor component label, + enumerating ONLY the capabilities/volumes the agents require (do not blanket + `allowPrivilegedContainer` unless proven necessary). +2. Bind it via the SCC `users:` list to the exact ServiceAccounts the operator + creates. These names are **UNVERIFIED** and must be confirmed against the + installed operator version, e.g. (indicative, verify before use): + - Datadog: `-agent`, `-cluster-agent`, plus the + operator SA `datadog-operator`. + - Dynatrace: OneAgent, ActiveGate, CSI driver, webhook, and operator SAs + under the `dynatrace` namespace. +3. Remove the OpenShift-blocking assertion in the installers and add labeled SCC + cleanup to the uninstallers (query by the ITBench labels, delete — as + `uninstall_chaos_mesh.yaml` does). +4. Validate on a real OpenShift cluster: agents reach Ready, no + `CreateContainerError`/SCC-denied events, and uninstall removes the SCCs. + +Until steps 1-4 are done and validated, OpenShift stays hard-blocked. + +## Cross-cutting design decisions + +1. **New registry section.** SaaS vendors get a parallel `tools_vendors` dict in + `project/roles/tools/defaults/main/vendors.yaml` (rather than overloading + `tools_managers`), since they carry SaaS-specific attributes. +2. **Non-secret config** (site, region, tenant URL, enable flags) lives in + `inventory/group_vars/environment/observability_vendors.yaml(.example)`. +3. **Secrets are inputs, source-agnostic.** Credential tasks read from a variable + with an env fallback so the same task works locally and under AWX: + `{{ datadog_api_key | default(lookup('ansible.builtin.env','DATADOG_API_KEY'), true) }}`. + All secret-handling tasks use `no_log: true`. Secrets are never committed. +4. **Enablement** via per-vendor booleans mapped through `manage_tools.yaml` into + `tools_configuration.vendors..enabled`, gating each install. +4a. **Single role-input interface.** The tools role reads vendor config **only** + from `tools_configuration.vendors.*` (never from the global + `observability_vendors` directly), so AWX and role tests have one contract. +4b. **Vendor installs run outside the SRE/FinOps gate** so a vendor-only + deployment works. +4c. **Uninstall is unconditional, idempotent, credential-independent, and + ownership-scoped.** It runs during Undeploy-Tools regardless of the enabled + flag (so disabling a vendor still removes it), no-ops when the namespace is + absent, only deletes resources labeled `itbench.io/observability-vendor=`, + and only deletes the namespace when it is ITBench-owned and drained. No blind + `delete_all` of all CRs or unconditional namespace deletion. +4d. **Manifests are treated as untrusted + secret.** User CR files are validated + (allowed kinds/apiVersions, exactly one primary CR, namespace pinning, doc + count, null-doc rejection) before apply, and all parse/apply steps use + `no_log: true`. +4e. **CRD retention.** Helm typically leaves chart CRDs behind on uninstall; we + preserve them by default (do not force-delete vendor CRDs) to avoid breaking + any remaining vendor CRs cluster-wide. +5. **AWX trigger** reuses the existing `Deploy-Tools`/`Undeploy-Tools` job + templates (vendor installs run inside `manage_tools.yaml`, flag-gated). No new + job templates. +6. **AWX secret delivery** uses custom AWX **credential types** (precedent: the + `Kubeconfig` credential type in `configure_credentials.yaml`), injecting + secrets as `extra_vars` (or files) into the Deploy-Tools job pod. +6a. **AWX input plumbing.** `manage_awx.yaml` combines a `vendors` group_var into + `awx_configuration.vendors` (with `no_log`). Non-secret vendor config reaches + the Deploy/Undeploy-Tools job templates via `extra_vars` + (`observability_vendors`), built by a projection in `configure_jobs.yaml` that + **strips secret keys** (`manifest`, `dynakube`, `api_key`, `app_key`). Secrets + reach the job only through the injected vendor credential. +6b. **Vendor credentials are scoped to the tools nodes only.** A separate + `node_tools_credentials` list (template `node_tools_credentials.j2`) carries + the vendor credentials and is attached only to the `node-deploy-tools` / + `node-undeploy-tools` workflow nodes. All other nodes keep the vendor-free + `node_credentials` list, so fault-injection and Run-Agent jobs never receive + vendor secrets. +6c. **AWX role argument spec is generated.** `meta/argument_specs.yaml` is + rendered from `templates/meta/argument_specs.j2`; vendor specs live in the + template (the generated file is overwritten on regeneration). +6d. **Plaintext-to-AWX path.** The `vendors` group_var + (`inventory/group_vars/runner/vendors.yaml.example`) holds the secret inputs + and must be Ansible-Vault encrypted or sourced from a secret manager. AWX + stores the resulting credentials encrypted; provisioning tasks use `no_log`. + +### Secret handling under AWX — why not plain env vars + +AWX runs playbooks in ephemeral job pods (the `ITBench-Custom-EE` execution +environment). There is no shell where an operator `export`ed variables, and job +pods don't inherit host env vars, so `lookup('env', ...)` returns empty. AWX +credential types inject values into the job pod at runtime (as env/extra_vars or +files) and store them encrypted at rest. The source-agnostic variable form above +picks these up transparently. + +Precedent: the `Kubeconfig` custom credential type +(`project/roles/awx/tasks/configure_credentials.yaml`, and injectors JSON at +`project/roles/awx/files/awx/credential_types/kubeconfig_injectors.json`) +supports both `extra_vars` and `file` injection. + +> **Note (pre-existing security debt):** plaintext secrets are currently committed +> in `group_vars/runner/agent.yaml` (watsonx api_key, GitHub PAT) and +> `group_vars/runner/credentials.yaml` (AWS keys). These should be rotated and +> removed; this work deliberately avoids adding new plaintext secrets. + +--- + +## Per-vendor deploy summary + +| Vendor | Helm chart | CR applied | Required secrets | +|--------|-----------|-----------|------------------| +| Datadog | `datadog-operator` (helm.datadoghq.com) | `DatadogAgent` (from user's file) | `DATADOG_API_KEY` (+ `DATADOG_APP_KEY` when the CR references `appSecret`) | +| Dynatrace | `dynatrace-operator` (OCI) | `DynaKube` (from user's file) | tokens embedded in downloaded `dynakube.yaml` | + +Common concerns: OpenShift SCC for privileged agents (precedent +`install_chaos_mesh.yaml`), Istio ambient (ztunnel) coexistence, kind +limitations for eBPF/full-stack agents, and pinned chart versions with renovate +annotations. + +--- + +## Implementation status + +| Item | File | Status | +|------|------|--------| +| Vendor registry | `project/roles/tools/defaults/main/vendors.yaml` | Created | +| Non-secret config (+ example) | `inventory/group_vars/environment/observability_vendors.yaml(.example)` | Created | +| `.gitignore` for `secrets/` | `scenarios/sre/.gitignore` | Created | +| Datadog creds | `project/roles/tools/tasks/set_credentials_datadog.yaml` | **Done** (source-agnostic) | +| Datadog namespace | `templates/kubernetes/datadog/namespace.j2` | **Done** | +| Datadog install/uninstall | `tasks/install_datadog.yaml`, `uninstall_datadog.yaml` | **Done** | +| Datadog orchestration wiring | `tasks/install.yaml`, `uninstall.yaml`, `manage_tools.yaml` | **Done** | +| Datadog AWX credential/wiring | `project/roles/awx/*` | **Done** | +| Dynatrace install/uninstall/AWX | `tasks/install_dynatrace.yaml`, `uninstall_dynatrace.yaml`, `awx/*` | **Implemented, unvalidated** (consumes user's `dynakube.yaml`; namespace created via `ensure_vendor_namespace.yaml`; `set_credentials_dynatrace.yaml` removed) | +| Shared ownership helpers | `tasks/ensure_vendor_namespace.yaml`, `assert_vendor_resource_ownership.yaml`, `delete_vendor_namespace_if_safe.yaml` | **Done** | +| Datadog manifest validation | `tasks/validate_datadog_manifest.yaml` | **Done** | +| Molecule: live deploy (opt-in) | `molecule/deployment_vendors/` | **Done** (self-skips without vendor secrets) | +| Molecule: credential-free contract | `molecule/vendor_contract/` | **Done** (validation, secret rejection, projection allowlist) | + +--- + +## Dynatrace (implemented) — detailed plan + +> Status: built. Registry pinned to operator `1.9.0` (OCI +> `oci://public.ecr.aws/dynatrace/dynatrace-operator`), install/uninstall/namespace +> tasks created, wired into `install.yaml`/`uninstall.yaml` (flag-gated on +> `tools_configuration.vendors.dynatrace.enabled`), AWX `Dynatrace` file-injector +> credential added, and the superseded `set_credentials_dynatrace.yaml` removed. + + +Dynatrace's onboarding UI produces a self-contained `dynakube.yaml` (a `Secret` +with `apiToken` + `dataIngestToken`, plus a `DynaKube` CR carrying apiUrl, +cluster name, and config). We **consume that file as-is** rather than +reconstruct it. + +### Decisions + +- Consume the downloaded `dynakube.yaml` unchanged (apply Secret + DynaKube after + the operator is installed). +- A single variable `dynatrace_dynakube_path` drives both local and AWX paths, so + `install_dynatrace.yaml` is identical in both contexts. +- **Local:** user places file at `secrets/dynatrace/dynakube.yaml` (gitignored); + path configurable via `observability_vendors.dynatrace.dynakube_path`. +- **AWX:** custom `Dynatrace-Dynakube` credential type with a **file injector** + (Kubeconfig precedent); user pastes file contents once, AWX materializes it as + a file in the job pod and sets `dynatrace_dynakube_path` to `tower.filename.dynakube`. +- **Helm:** OCI chart `oci://public.ecr.aws/dynatrace/dynatrace-operator`, pinned + `chart_version: 1.9.0`, `create_namespace: true`, `atomic: true` (v3 form). +- Reuse `Deploy-Tools`/`Undeploy-Tools`; flag-gated by + `tools_configuration.vendors.dynatrace.enabled`. + +Because the file carries everything, Dynatrace needs **no** `set_credentials`, +**no** `values.j2`, and **no** `dynakube.j2`. (An OpenShift-only `namespace.j2` +for SCC annotations is still needed.) + +### Files + +1. **`defaults/main/vendors.yaml`** (modify dynatrace entry): + ```yaml + dynatrace: + helm: + chart: + # renovate: datasource=docker depName=public.ecr.aws/dynatrace/dynatrace-operator + reference: oci://public.ecr.aws/dynatrace/dynatrace-operator + version: 1.9.0 + release: + name: dynatrace-operator + kubernetes: + namespace: dynatrace + ``` + +2. **`inventory/group_vars/environment/observability_vendors.yaml(.example)`** + (modify dynatrace block): + ```yaml + dynatrace: + enabled: false + # Path to dynakube.yaml downloaded from the Dynatrace onboarding UI + # (contains the Secret with tokens + the DynaKube CR). Keep OUT of git. + dynakube_path: "secrets/dynatrace/dynakube.yaml" + ``` + +3. **`scenarios/sre/.gitignore`** (create): add `secrets/`. + +4. **`tasks/install_dynatrace.yaml`** (create): + 1. Resolve `dynatrace_dynakube_path` = `tower.filename.dynakube` if defined + (AWX) else `observability_vendors.dynatrace.dynakube_path`. + 2. `stat` + assert file exists / non-empty (`no_log: true`). + 3. OpenShift only: create `dynatrace` ns from + `templates/kubernetes/dynatrace/namespace.j2` (SCC annotations) before install. + 4. Helm install: + ```yaml + kubernetes.core.helm: + chart_ref: "{{ tools_vendors.dynatrace.helm.chart.reference }}" + chart_version: "{{ tools_vendors.dynatrace.helm.chart.version }}" + kubeconfig: "{{ tools_cluster.kubeconfig }}" + release_name: "{{ tools_vendors.dynatrace.helm.release.name }}" + release_namespace: "{{ tools_vendors.dynatrace.kubernetes.namespace }}" + release_state: present + create_namespace: true + atomic: true + wait: true + ``` + 5. Wait for the operator webhook Deployment to be Available (retry/until) — + DynaKube apply fails if the webhook isn't up yet. + 6. Apply the user's file: + ```yaml + kubernetes.core.k8s: + kubeconfig: "{{ tools_cluster.kubeconfig }}" + definition: "{{ lookup('ansible.builtin.file', dynatrace_dynakube_path) | ansible.builtin.from_yaml_all | list }}" + state: present + wait: true + # no_log: true + ``` + +5. **`tasks/uninstall_dynatrace.yaml`** (create): delete DynaKube CR + its Secret + (by file if present, else by name/label), helm `release_state: absent`, delete + namespace — all `wait: true`, guarded to no-op if file absent. + +6. **`templates/kubernetes/dynatrace/namespace.j2`** (create — OpenShift SCC only): + standard ITBench namespace with `{% if tools_cluster.platform == 'openshift' %}` + scc annotations. + +7. **Orchestration wiring** (modify): + - `tasks/install.yaml`: flag-gated import of `install_dynatrace.yaml` + (`when: tools_configuration.vendors.dynatrace.enabled | default(false)`). + - `tasks/uninstall.yaml`: matching reverse-order import. + - `manage_tools.yaml`: add `vendors: "{{ observability_vendors | default({}) }}"` + to `tools_configuration` (both the scenario_id and non-scenario branches); + pass `observability_vendors` into the tools role vars. + +8. **AWX credential** (file injector — Kubeconfig precedent): + - Create `project/roles/awx/files/awx/credential_types/dynatrace_dynakube_injectors.json`: + ```json + { + "extra_vars": { "dynatrace_dynakube_path": "{{ tower.filename.dynakube }}" }, + "file": { "template.dynakube": "{{ dynakube }}" } + } + ``` + - Modify `project/roles/awx/tasks/configure_credentials.yaml`: create a + `Dynatrace-Dynakube` credential type (one multiline `secret: true` field + `dynakube`, injectors from the JSON) + a credential instance whose + `inputs.dynakube` is sourced from `awx_configuration.vendors.dynatrace.dynakube`. + Gate on Dynatrace enabled. + - Modify `project/roles/awx/templates/awx/workflows/node_credentials.j2`: add + `- name: Dynatrace-Dynakube` when enabled. + - Modify `project/roles/awx/meta/argument_specs.yaml`: document + `awx_configuration.vendors.dynatrace.dynakube`. + +9. **Docs**: `.example` comments explain (a) Dynatrace UI → set cluster + name/tokens → **Download dynakube.yaml**; (b) local → place at + `secrets/dynatrace/dynakube.yaml`; (c) AWX → paste contents into the + `Dynatrace-Dynakube` credential. + +### Notes / risks + +- The v1.9.0 cloud-native chart includes the CSI driver; validate on kind. The + downloaded DynaKube may use classicFullStack/hostMonitoring — that is the + user's file, not ours to change. +- OpenShift: pre-create ns with SCC annotations before the atomic install; + OneAgent may need a privileged SCC (flag for OpenShift testing). +- `atomic: true` rolls back a failed operator install; the separate webhook wait + guards the subsequent CR apply. +- `no_log: true` on all file/CR steps; `secrets/` gitignored; AWX stores the file + encrypted. +- Chart `--version` is `1.9.0` (no `v`); the git tag is `v1.9.0`. + +### Implementation notes (Dynatrace) + +- Namespace is created and ownership-checked via `ensure_vendor_namespace.yaml` + (no namespace `.j2` template). +- Wired into `install.yaml` / `uninstall.yaml` / `manage_tools.yaml`; uninstall + is unconditional, idempotent, CSI-aware, and credential-independent. +- AWX: `dynatrace_injectors.json` (file injector) + `configure_credentials.yaml` + (reconciled present/absent by enabled flag) + `node_tools_credentials.j2` + (attached only to Deploy-Tools) + generated `argument_specs.j2`. +- The superseded `set_credentials_dynatrace.yaml` has been removed. +- Testing is **required**: `molecule/vendor_contract/` (credential-free) always + runs; `molecule/deployment_vendors/` runs a live deploy when vendor secrets are + provided and self-skips otherwise. + +### Application OTLP ingest (otel-demo → Dynatrace) + +The DynaKube deploy above is **Layer A** (OneAgent/ActiveGate infrastructure +monitoring). It does **not** send the sample application's OpenTelemetry data. +That path lives in the `applications` role and is enabled by the **same** +`observability_vendors.dynatrace.enabled` flag: + +- `install_opentelemetry_demo.yaml` — when Dynatrace is enabled, parses the same + `dynakube.yaml` (reusing the `from_yaml_all` idiom from `install_dynatrace.yaml`), + extracts the **`dataIngestToken`** and **`apiUrl`**, and projects a + `dynatrace-otlp-ingest` Secret (`DT_ENDPOINT`, `DT_API_TOKEN`) into the + `otel-demo` namespace. No new token is required. +- `templates/helm/otel_demo/values.j2` — gated on + `applications_dynatrace_ingest_enabled`, adds an `otlphttp/dynatrace` exporter to + the demo collector's **logs, metrics, and traces** pipelines. +- **Metric temporality:** a `cumulativetodelta` processor is added to the metrics + pipeline — Dynatrace accepts delta temporality only, and the demo emits + cumulative sums/histograms (otherwise rejected as `UNSUPPORTED_METRIC_TYPE_*`). +- **K8s attributes:** the chart's `kubernetesAttributes` preset already tags + pod/namespace/deployment/node. We additionally inject **`k8s.cluster.name`** via a + `resource/dynatrace` processor (default: the DynaKube `metadata.name`, overridable + via `applications_dynatrace_cluster_name`) so OTLP data correlates to the K8s + cluster entity the OneAgent sees. +- **Helm list gotcha:** the opentelemetry-collector chart deep-merges maps but + **replaces lists**, so each touched pipeline's `processors` list is restated in + full (chart defaults + the added processors). +- `uninstall_opentelemetry_demo.yaml` deletes the projected secret (gated on the + same flag) before removing the namespace. + +When Dynatrace is disabled the template renders byte-for-byte as before — the +exporter, processors, and `extraEnvsFrom` are entirely absent. + +--- + +## Datadog (implemented) — operator + user-supplied `datadog-agent.yaml` + +Datadog's onboarding UI produces a **non-secret** `datadog-agent.yaml` +(`DatadogAgent` CR) that references the API/APP keys **by Secret name** rather +than embedding them. So the split is: + +- **Config (non-secret):** the `datadog-agent.yaml` (site, `clusterName`, feature + toggles, RUM IDs). May be committed/pasted freely. +- **Secret:** `datadog-secret` with `api-key` / `app-key`, created separately. + +This differs from Dynatrace (whose downloaded file embeds the tokens). We +therefore both consume the user's CR file **and** create the Secret from +`DATADOG_API_KEY` / `DATADOG_APP_KEY`. + +### Decisions + +- Consume the user's `datadog-agent.yaml` as-is (no re-templating). +- CR file source: variable `datadog_manifest_path` (env fallback via extra_var), + default local path `secrets/datadog/datadog-agent.yaml` (gitignored). Under AWX + it is injected as a file via a custom `Datadog` credential. +- Secret created by `set_credentials_datadog.yaml` from source-agnostic vars + `datadog_api_key` / `datadog_app_key` (env fallback + `DATADOG_API_KEY`/`DATADOG_APP_KEY`); `no_log: true`. Secret name and key names + are configurable (`observability_vendors.datadog.{secret_name,api_key_name,app_key_name}`) + and **must match** the `credentials.apiSecret/appSecret` refs in the CR. +- Helm: classic repo chart `datadog/datadog-operator` from + `https://helm.datadoghq.com`, pinned `chart_version: 2.24.0`, `atomic: true`, + `wait: true`. (The onboarding command uses the operator chart, not the agent + chart.) +- Reuse `Deploy-Tools`/`Undeploy-Tools`; flag-gated by + `tools_configuration.vendors.datadog.enabled`. + +### Files (all created) + +| File | Purpose | +|------|---------| +| `defaults/main/vendors.yaml` (datadog entry) | Operator chart `datadog-operator` @ `2.24.0`, ns `datadog` | +| `inventory/group_vars/environment/observability_vendors.yaml(.example)` | `datadog.{enabled,manifest_path,secret_name,api_key_name,app_key_name}` | +| `tasks/set_credentials_datadog.yaml` | Source-agnostic keys → create `datadog-secret` (`no_log`) | +| `templates/kubernetes/datadog/namespace.j2` | Namespace (OpenShift SCC annotations) | +| `tasks/install_datadog.yaml` | Resolve+assert manifest → ns → helm operator (atomic/wait) → wait for `datadogagents.datadoghq.com` CRD Established → create secret → apply CR file | +| `tasks/uninstall_datadog.yaml` | Delete DatadogAgent CRs → helm absent → delete ns | +| `tasks/install.yaml` / `uninstall.yaml` | Flag-gated import of Datadog tasks | +| `manage_tools.yaml` | `tools_configuration.vendors` from `observability_vendors` | +| `awx/files/awx/credential_types/datadog_injectors.json` | file→`datadog_manifest_path`; extra_vars→`datadog_api_key`/`datadog_app_key` | +| `awx/tasks/configure_credentials.yaml` | `Datadog` credential type + instance (gated on enabled) | +| `awx/templates/awx/workflows/node_credentials.j2` | Attach `Datadog` credential to Deploy/Undeploy-Tools nodes | +| `awx/meta/argument_specs.yaml` | Document `awx_configuration.vendors.datadog.*` | + +### Install flow (`install_datadog.yaml`) + +1. Resolve `datadog_manifest_path` (AWX extra_var/file, else group_var default). +2. `stat` + assert the manifest exists / non-empty. +3. Create namespace from `namespace.j2`. +4. Helm install `datadog-operator` (repo chart, `atomic: true`, `wait: true`). +5. Wait for the `datadogagents.datadoghq.com` CRD to be **Established**. +6. Import `set_credentials_datadog.yaml` → create `datadog-secret`. +7. Apply the user's `datadog-agent.yaml` (`from_yaml_all`), into the `datadog` ns. + +### Usage + +**Local:** +```bash +export DATADOG_API_KEY=... # from the onboarding command # pragma: allowlist secret +export DATADOG_APP_KEY=... # optional, needed for Private Action Runner +mkdir -p scenarios/sre/secrets/datadog +cp ~/Downloads/datadog-agent.yaml scenarios/sre/secrets/datadog/datadog-agent.yaml +# set observability_vendors.datadog.enabled: true, then run Deploy-Tools +``` + +**AWX:** set `awx_configuration.vendors.datadog.{enabled: true, manifest: , api_key, app_key}` so the `Datadog` credential is created and attached +to the Deploy-Tools node; also set `observability_vendors.datadog.enabled: true` +in the committed group_vars (the flag the playbook reads). AWX injects the +manifest as a file and the keys as extra_vars into the job pod. + +### Notes / risks + +- Secret key names must match the CR's `credentials.apiSecret.keyName` / + `appSecret.keyName` (defaults `api-key` / `app-key`). +- The sample CR enables OTel collector host ports 4317/4318 and APM SSI — + validate host-port availability and privileged access on kind/OpenShift. +- On OpenShift the node agent likely needs a privileged SCC (not yet added; + flag for OpenShift testing) in addition to the namespace SCC annotations. +- App key is optional for basic monitoring but **required** for the Private + Action Runner / "Agent to take action" feature shown in onboarding. diff --git a/scenarios/sre/inventory/group_vars/environment/observability_vendors.yaml.example b/scenarios/sre/inventory/group_vars/environment/observability_vendors.yaml.example new file mode 100644 index 000000000..4fc6c2499 --- /dev/null +++ b/scenarios/sre/inventory/group_vars/environment/observability_vendors.yaml.example @@ -0,0 +1,76 @@ +--- +# Third-party SaaS observability vendor configuration (Layer A: agent deployment). +# +# NON-SECRET values only (site, region, tenant/environment URL, toggles) belong +# in this file. SECRETS (API keys, license keys, tokens) MUST NOT be placed here. +# They are read from environment variables at install time. Export the relevant +# variables before running the tools playbook, for example: +# +# # Datadog +# export DATADOG_API_KEY=... # required # pragma: allowlist secret +# export DATADOG_APP_KEY=... # optional (needed for some features) +# +# # Dynatrace agent deploy (Layer A): no env vars. Download dynakube.yaml from +# # the Dynatrace onboarding UI (contains tokens + DynaKube CR) and reference +# # it via observability_vendors.dynatrace.dynakube_path (see below). +# +# # Dynatrace Grail recorder (exports telemetry back out at teardown): its +# # PLATFORM token (prefix dt0s16., Grail read permissions) is read from a +# # gitignored file referenced by dynatrace.recorder.platform_token_path below +# # (same convention as dynakube_path) — no env export needed. This is a +# # DIFFERENT token from the agent ingest tokens in dynakube.yaml. +# +# Each vendor is disabled by default. Enable a vendor and supply its non-secret +# configuration below, then export the matching secret environment variable(s). + +observability_vendors: + datadog: + enabled: false + # Path to the datadog-agent.yaml (DatadogAgent CR) shown in the Datadog + # onboarding UI. This file is NON-SECRET (it references the api/app keys by + # Secret name) so it may be committed. It carries site, clusterName, and + # feature toggles. + manifest_path: "secrets/datadog/datadog-agent.yaml" + # Name and keys of the Kubernetes Secret created from DATADOG_API_KEY / + # DATADOG_APP_KEY. These MUST match the credentials.apiSecret/appSecret + # references inside the DatadogAgent CR above. + secret_name: datadog-secret + api_key_name: api-key + app_key_name: app-key + dynatrace: + enabled: false + # Path to the dynakube.yaml downloaded from the Dynatrace onboarding UI. + # This file is SENSITIVE: it contains a Secret (apiToken + dataIngestToken) + # AND the DynaKube CR (apiUrl, cluster name, config). Keep OUT of git. + dynakube_path: "secrets/dynatrace/dynakube.yaml" # pragma: allowlist secret + # Supported DynaKube API version. The DynaKube document in dynakube.yaml must + # use this exact apiVersion; it is also used for readiness and cleanup so all + # three stay consistent. + dynakube_api_version: "dynatrace.com/v1beta3" + # Dynatrace Grail RECORDER (Layer B-ish export). Runs once at teardown as a + # Kubernetes Job that back-queries Grail over [scenario start -> now] and + # writes JSON+CSV (metrics, spans, logs, events, k8s metrics) into the + # recorder export directory. Independent of the agent deploy above: it only + # reads telemetry, and requires a PLATFORM token exported as DT_PLATFORM_TOKEN. + recorder: + enabled: false + # Dynatrace platform (apps) URL — NOT the live/ingest URL. NON-SECRET. + # e.g. https://.apps.dynatrace.com + platform_url: "" + # Path to a file containing the platform token (prefix dt0s16.) with Grail + # read permissions. SENSITIVE: keep under secrets/ (gitignored). The file + # should contain only the token. + platform_token_path: "secrets/dynatrace/platform_token" # pragma: allowlist secret + # Kubernetes namespace whose telemetry is exported. + namespace: otel-demo + # Query end (Grail relative time or ISO-8601). The start is taken from the + # scenario start time stamped at install; fallback_from is used only if + # that marker is missing. + timeframe_to: now + fallback_from: now-1h + # Timeseries bucket size for metric/span datasets. + interval: 1m + # Row cap for the logs/events datasets. + limit: 1000 + # Output format: json | csv | both. + format: both diff --git a/scenarios/sre/inventory/group_vars/runner/vendors.yaml.example b/scenarios/sre/inventory/group_vars/runner/vendors.yaml.example new file mode 100644 index 000000000..1bb828316 --- /dev/null +++ b/scenarios/sre/inventory/group_vars/runner/vendors.yaml.example @@ -0,0 +1,30 @@ +--- +# Observability vendor configuration for the AWX runner entry point +# (manage_awx.yaml). This drives both: +# - AWX credential creation (secret inputs: manifest/dynakube contents, keys) +# - Non-secret job extra_vars (enable flags + config), projected automatically +# with secrets stripped in roles/awx/tasks/configure_jobs.yaml +# +# SECURITY: this file contains secrets. DO NOT commit it in plaintext. Encrypt it +# with Ansible Vault (ansible-vault encrypt vendors.yaml) or source the secret +# fields from a secret manager. The provisioning tasks use no_log; AWX stores the +# resulting credentials encrypted at rest. +# +# vendors: +# datadog: +# enabled: true +# # Non-secret config (also consumed by the tools role): +# manifest_path: "secrets/datadog/datadog-agent.yaml" +# secret_name: datadog-secret +# api_key_name: api-key +# app_key_name: app-key +# # Secret inputs (injected into AWX credentials, stripped from job vars): +# manifest: | +# # full contents of the datadog-agent.yaml downloaded from Datadog +# api_key: "" # pragma: allowlist secret +# app_key: "" # pragma: allowlist secret +# dynatrace: +# enabled: true +# dynakube_path: "secrets/dynatrace/dynakube.yaml" +# dynakube: | +# # full contents of the dynakube.yaml downloaded from Dynatrace diff --git a/scenarios/sre/project/manage_awx.yaml b/scenarios/sre/project/manage_awx.yaml index b708f4694..157348502 100644 --- a/scenarios/sre/project/manage_awx.yaml +++ b/scenarios/sre/project/manage_awx.yaml @@ -36,6 +36,13 @@ when: - cluster_provider != "kind" + - name: Add observability vendors variable + ansible.builtin.set_fact: + awx_configuration: "{{ awx_configuration | ansible.builtin.combine({'vendors': vendors}) }}" + when: + - vendors is ansible.builtin.defined + no_log: true + - name: Create experiments variable ansible.builtin.set_fact: awx_experiments: diff --git a/scenarios/sre/project/manage_recorders.yaml b/scenarios/sre/project/manage_recorders.yaml index 906f4a776..f763d10a7 100644 --- a/scenarios/sre/project/manage_recorders.yaml +++ b/scenarios/sre/project/manage_recorders.yaml @@ -95,3 +95,14 @@ recorders_cluster: kubeconfig: "{{ cluster.kubeconfig }}" platform: "{{ cluster_platform }}" + # Pass ONLY the recorder subset the role declares (not the whole + # observability_vendors blob, which also carries agent-deploy keys the + # recorders role's argument spec does not accept). + recorders_vendors: + dynatrace: + recorder: >- + {{ + (observability_vendors | ansible.builtin.default({})) + .get('dynatrace', {}) + .get('recorder', {'enabled': false}) + }} diff --git a/scenarios/sre/project/manage_tools.yaml b/scenarios/sre/project/manage_tools.yaml index d83d2d1f4..69548d7cf 100644 --- a/scenarios/sre/project/manage_tools.yaml +++ b/scenarios/sre/project/manage_tools.yaml @@ -32,7 +32,11 @@ - name: Create tool installation variable ansible.builtin.set_fact: - tools_configuration: "{{ scenarios_scenario.spec.tools }}" + tools_configuration: >- + {{ + scenarios_scenario.spec.tools | + ansible.builtin.combine({'vendors': observability_vendors | ansible.builtin.default({})}) + }} - name: Create role variables tags: @@ -47,6 +51,7 @@ enabled: "{{ tools.sre | ansible.builtin.default(false) }}" finops: enabled: "{{ tools.finops | ansible.builtin.default(false) }}" + vendors: "{{ observability_vendors | ansible.builtin.default({}) }}" - name: Import tools role ansible.builtin.import_role: diff --git a/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml b/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml index 4e7b57886..309e65810 100644 --- a/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml +++ b/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml @@ -68,6 +68,95 @@ name: system:openshift:scc:anyuid state: present +# --- Dynatrace OTLP ingest (optional). --------------------------------------- +# When the Dynatrace vendor is enabled, route the otel-demo collector's OTLP +# traces/metrics/logs to Dynatrace in addition to the in-cluster backends. The +# dataIngestToken and apiUrl are reused from the same dynakube.yaml consumed by +# the tools role (roles/tools/tasks/install_dynatrace.yaml), so no new token is +# needed. Everything here is gated on the vendor flag; when disabled the demo +# renders and installs exactly as before. +- name: Configure Dynatrace OTLP ingest for OpenTelemetry Demo + when: + - observability_vendors.dynatrace.enabled | ansible.builtin.default(false) + block: + - name: Resolve the Dynatrace DynaKube manifest path + ansible.builtin.set_fact: + applications_dynatrace_dynakube_path: >- + {{ + dynatrace_dynakube_path | + ansible.builtin.default(observability_vendors.dynatrace.dynakube_path, true) + }} + + # Relative paths are anchored to the SRE project root (parent of the playbook + # directory), matching install_dynatrace.yaml; absolute paths are used as-is. + - name: Anchor a relative Dynatrace manifest path to the project root + ansible.builtin.set_fact: + applications_dynatrace_dynakube_path: >- + {{ + [playbook_dir, '..', applications_dynatrace_dynakube_path] + | ansible.builtin.path_join | ansible.builtin.realpath + }} + when: + - not (applications_dynatrace_dynakube_path is ansible.builtin.abs) + + - name: Parse the Dynatrace DynaKube manifest documents + ansible.builtin.set_fact: + applications_dynatrace_documents: >- + {{ + lookup('ansible.builtin.file', applications_dynatrace_dynakube_path) + | ansible.builtin.from_yaml_all | list | ansible.builtin.reject('none') | list + }} + no_log: true + + - name: Derive Dynatrace ingest facts from the manifest + vars: + applications_dynatrace_secret: >- + {{ + applications_dynatrace_documents + | selectattr('kind', 'equalto', 'Secret') | list | first + }} + applications_dynatrace_dynakube: >- + {{ + applications_dynatrace_documents + | selectattr('kind', 'equalto', 'DynaKube') | list | first + }} + ansible.builtin.set_fact: + # dataIngestToken is base64-encoded in the Secret's data map; decode to + # the raw token for the Authorization header value. + applications_dynatrace_ingest_token: "{{ applications_dynatrace_secret.data.dataIngestToken | ansible.builtin.b64decode }}" + # OTLP endpoint is the DynaKube apiUrl (…/api) plus the OTLP v2 path. + applications_dynatrace_otlp_endpoint: "{{ applications_dynatrace_dynakube.spec.apiUrl }}/v2/otlp" + # Cluster name for entity correlation: explicit override, else DynaKube name. + applications_dynatrace_cluster_name: >- + {{ + applications_dynatrace_cluster_name | + ansible.builtin.default(applications_dynatrace_dynakube.metadata.name, true) + }} + no_log: true + + - name: Create the Dynatrace OTLP ingest Secret in the demo namespace + kubernetes.core.k8s: + kubeconfig: "{{ applications_cluster.kubeconfig }}" + resource_definition: + apiVersion: v1 + kind: Secret + metadata: + name: dynatrace-otlp-ingest + namespace: "{{ applications_releases.opentelemetry_demo.namespace }}" + labels: + app.kubernetes.io/managed-by: ITBench + itbench.io/observability-vendor: dynatrace + type: Opaque + stringData: + DT_ENDPOINT: "{{ applications_dynatrace_otlp_endpoint }}" + DT_API_TOKEN: "{{ applications_dynatrace_ingest_token }}" + state: present + no_log: true + + - name: Enable Dynatrace ingest in the OpenTelemetry Demo values + ansible.builtin.set_fact: + applications_dynatrace_ingest_enabled: true + - name: Install OpenTelemetry Demo (Astronomy Shop) kubernetes.core.helm: chart_ref: "oci://ghcr.io/open-telemetry/opentelemetry-helm-charts/opentelemetry-demo" diff --git a/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml b/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml index 820b01d09..0ac4a128a 100644 --- a/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml +++ b/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml @@ -7,6 +7,22 @@ release_state: absent wait: true +# Explicitly remove the projected Dynatrace ingest secret. The namespace +# deletion below also removes it, but this keeps teardown symmetric with the +# install and covers cases where the namespace is retained. +- name: Delete the Dynatrace OTLP ingest Secret + kubernetes.core.k8s: + kubeconfig: "{{ applications_cluster.kubeconfig }}" + resource_definition: + apiVersion: v1 + kind: Secret + metadata: + name: dynatrace-otlp-ingest + namespace: "{{ applications_releases.opentelemetry_demo.namespace }}" + state: absent + when: + - observability_vendors.dynatrace.enabled | ansible.builtin.default(false) + - name: Delete Namespace kubernetes.core.k8s: kubeconfig: "{{ applications_cluster.kubeconfig }}" diff --git a/scenarios/sre/project/roles/applications/templates/helm/otel_demo/values.j2 b/scenarios/sre/project/roles/applications/templates/helm/otel_demo/values.j2 index f67807dbb..7cc4a3ada 100644 --- a/scenarios/sre/project/roles/applications/templates/helm/otel_demo/values.j2 +++ b/scenarios/sre/project/roles/applications/templates/helm/otel_demo/values.j2 @@ -81,12 +81,32 @@ components: runAsNonRoot: true runAsUser: 999 opentelemetry-collector: +{% if applications_dynatrace_ingest_enabled | default(false) %} + # Dynatrace OTLP ingest is enabled: mount the projected credentials secret so + # the exporter can read DT_ENDPOINT / DT_API_TOKEN as environment variables. + extraEnvsFrom: + - secretRef: + name: dynatrace-otlp-ingest +{% endif %} config: connectors: spanmetrics: dimensions: - name: namespace default: {{ applications_releases.opentelemetry_demo.namespace }} +{% if applications_dynatrace_ingest_enabled | default(false) %} + processors: + # Dynatrace accepts DELTA temporality only; the demo emits cumulative + # sums/histograms, so convert them or they are rejected on ingest. + cumulativetodelta: {} + # Tag every signal with the cluster name so Dynatrace correlates the OTLP + # data to the Kubernetes cluster entity seen by the OneAgent/ActiveGate. + resource/dynatrace: + attributes: + - key: k8s.cluster.name + value: "{{ applications_dynatrace_cluster_name }}" + action: upsert +{% endif %} exporters: clickhouse: username: {{ tools_clickhouse_username }} @@ -98,22 +118,69 @@ opentelemetry-collector: endpoint: {{ tools_jaeger_endpoint.collector.grpc }} prometheus: endpoint: 0.0.0.0:8888 +{% if applications_dynatrace_ingest_enabled | default(false) %} + otlphttp/dynatrace: + endpoint: ${env:DT_ENDPOINT} + headers: + Authorization: Api-Token ${env:DT_API_TOKEN} +{% endif %} service: pipelines: logs: +{% if applications_dynatrace_ingest_enabled | default(false) %} + # Full processor list restated (Helm REPLACES lists) + resource/dynatrace. + processors: + - k8sattributes + - memory_limiter + - resourcedetection + - resource + - resource/dynatrace + - batch +{% endif %} exporters: - clickhouse - debug +{% if applications_dynatrace_ingest_enabled | default(false) %} + - otlphttp/dynatrace +{% endif %} metrics: +{% if applications_dynatrace_ingest_enabled | default(false) %} + # cumulativetodelta added before batch; resource/dynatrace for cluster tag. + processors: + - k8sattributes + - memory_limiter + - resourcedetection + - resource + - resource/dynatrace + - cumulativetodelta + - batch +{% endif %} exporters: - debug - prometheus +{% if applications_dynatrace_ingest_enabled | default(false) %} + - otlphttp/dynatrace +{% endif %} traces: +{% if applications_dynatrace_ingest_enabled | default(false) %} + # Restated defaults (incl. the chart's frontend-route transform) + resource/dynatrace. + processors: + - k8sattributes + - memory_limiter + - resourcedetection + - resource + - resource/dynatrace + - transform + - batch +{% endif %} exporters: - clickhouse - debug - otlp/jaeger - spanmetrics +{% if applications_dynatrace_ingest_enabled | default(false) %} + - otlphttp/dynatrace +{% endif %} telemetry: metrics: level: detailed diff --git a/scenarios/sre/project/roles/awx/files/awx/credential_types/datadog_injectors.json b/scenarios/sre/project/roles/awx/files/awx/credential_types/datadog_injectors.json new file mode 100644 index 000000000..d8f1bb413 --- /dev/null +++ b/scenarios/sre/project/roles/awx/files/awx/credential_types/datadog_injectors.json @@ -0,0 +1,10 @@ +{ + "extra_vars": { + "datadog_manifest_path": "{{ tower.filename.manifest }}", + "datadog_api_key": "{{ api_key }}", + "datadog_app_key": "{{ app_key }}" + }, + "file": { + "template.manifest": "{{ manifest }}" + } +} diff --git a/scenarios/sre/project/roles/awx/files/awx/credential_types/dynatrace_injectors.json b/scenarios/sre/project/roles/awx/files/awx/credential_types/dynatrace_injectors.json new file mode 100644 index 000000000..a939eb310 --- /dev/null +++ b/scenarios/sre/project/roles/awx/files/awx/credential_types/dynatrace_injectors.json @@ -0,0 +1,8 @@ +{ + "extra_vars": { + "dynatrace_dynakube_path": "{{ tower.filename.dynakube }}" + }, + "file": { + "template.dynakube": "{{ dynakube }}" + } +} diff --git a/scenarios/sre/project/roles/awx/meta/argument_specs.yaml b/scenarios/sre/project/roles/awx/meta/argument_specs.yaml index deaf9af41..11e3b9a00 100644 --- a/scenarios/sre/project/roles/awx/meta/argument_specs.yaml +++ b/scenarios/sre/project/roles/awx/meta/argument_specs.yaml @@ -85,6 +85,59 @@ argument_specs: type: dict required: false type: dict + vendors: + options: + datadog: + options: + enabled: + default: false + required: false + type: bool + manifest_path: + required: false + type: str + secret_name: + required: false + type: str + api_key_name: + required: false + type: str + app_key_name: + required: false + type: str + manifest: + required: false + type: str + api_key: + no_log: true + required: false + type: str + app_key: + no_log: true + required: false + type: str + required: false + type: dict + dynatrace: + options: + enabled: + default: false + required: false + type: bool + dynakube_path: + required: false + type: str + dynakube_api_version: + required: false + type: str + dynakube: + no_log: true + required: false + type: str + required: false + type: dict + required: false + type: dict scenarios: elements: dict options: diff --git a/scenarios/sre/project/roles/awx/tasks/configure_credentials.yaml b/scenarios/sre/project/roles/awx/tasks/configure_credentials.yaml index 0cdd3f03b..2092754f4 100644 --- a/scenarios/sre/project/roles/awx/tasks/configure_credentials.yaml +++ b/scenarios/sre/project/roles/awx/tasks/configure_credentials.yaml @@ -76,6 +76,108 @@ organization: ITBench state: present +- name: Create credentials for Datadog observability vendor + when: + - awx_configuration.vendors.datadog is ansible.builtin.defined + vars: + awx_datadog_enabled: "{{ awx_configuration.vendors.datadog.enabled | ansible.builtin.default(false) }}" + block: + - name: Create Datadog credential type + awx.awx.credential_type: + controller_host: "{{ awx_controller_host }}" + controller_password: "{{ awx_controller_password }}" # pragma: allowlist secret + controller_username: admin + description: Credentials type for the Datadog observability vendor + inputs: + fields: + - id: manifest + type: string + label: DatadogAgent manifest (datadog-agent.yaml) + secret: true + multiline: true + - id: api_key + type: string + label: Datadog API key + secret: true + - id: app_key + type: string + label: Datadog application key + secret: true + required: + - manifest + - api_key + injectors: "{{ lookup('ansible.builtin.file', 'files/awx/credential_types/datadog_injectors.json') }}" + kind: cloud + name: Datadog + state: present + when: + - awx_datadog_enabled + + # Reconcile the credential INSTANCE to match the enabled flag: present when + # enabled, absent when disabled (so disabling a vendor removes its stored + # credential rather than leaving it selectable). + - name: Reconcile Datadog credential + awx.awx.credential: + controller_host: "{{ awx_controller_host }}" + controller_password: "{{ awx_controller_password }}" # pragma: allowlist secret + controller_username: admin + credential_type: Datadog + description: Datadog agent manifest and API/application keys + inputs: "{{ awx_datadog_inputs if awx_datadog_enabled else omit }}" + name: Datadog + organization: ITBench + state: "{{ 'present' if awx_datadog_enabled else 'absent' }}" + vars: + awx_datadog_inputs: + manifest: "{{ awx_configuration.vendors.datadog.manifest | ansible.builtin.default('') }}" + api_key: "{{ awx_configuration.vendors.datadog.api_key | ansible.builtin.default('') }}" # pragma: allowlist secret + app_key: "{{ awx_configuration.vendors.datadog.app_key | ansible.builtin.default('') }}" # pragma: allowlist secret + no_log: true + +- name: Create credentials for Dynatrace observability vendor + when: + - awx_configuration.vendors.dynatrace is ansible.builtin.defined + vars: + awx_dynatrace_enabled: "{{ awx_configuration.vendors.dynatrace.enabled | ansible.builtin.default(false) }}" + block: + - name: Create Dynatrace credential type + awx.awx.credential_type: + controller_host: "{{ awx_controller_host }}" + controller_password: "{{ awx_controller_password }}" # pragma: allowlist secret + controller_username: admin + description: Credentials type for the Dynatrace observability vendor + inputs: + fields: + - id: dynakube + type: string + label: DynaKube manifest (dynakube.yaml) + secret: true + multiline: true + required: + - dynakube + injectors: "{{ lookup('ansible.builtin.file', 'files/awx/credential_types/dynatrace_injectors.json') }}" + kind: cloud + name: Dynatrace + state: present + when: + - awx_dynatrace_enabled + + - name: Reconcile Dynatrace credential + awx.awx.credential: + controller_host: "{{ awx_controller_host }}" + controller_password: "{{ awx_controller_password }}" # pragma: allowlist secret + controller_username: admin + credential_type: Dynatrace + description: Dynatrace DynaKube manifest (contains Secret and DynaKube CR) + inputs: "{{ awx_dynatrace_inputs if awx_dynatrace_enabled else omit }}" + name: Dynatrace + organization: ITBench + state: "{{ 'present' if awx_dynatrace_enabled else 'absent' }}" + vars: + awx_dynatrace_inputs: + dynakube: "{{ awx_configuration.vendors.dynatrace.dynakube | ansible.builtin.default('') }}" # pragma: allowlist secret + no_log: true + - name: Create credentials for GitHub repositories awx.awx.credential: controller_host: "{{ awx_controller_host }}" diff --git a/scenarios/sre/project/roles/awx/tasks/configure_jobs.yaml b/scenarios/sre/project/roles/awx/tasks/configure_jobs.yaml index 57433afbe..5a3aa1d53 100644 --- a/scenarios/sre/project/roles/awx/tasks/configure_jobs.yaml +++ b/scenarios/sre/project/roles/awx/tasks/configure_jobs.yaml @@ -1,4 +1,35 @@ --- +# Build a NON-SECRET projection of the vendor configuration for job extra_vars +# using an ALLOWLIST per vendor. Only explicitly-listed non-secret keys are +# forwarded; anything else (including future secret fields like token/license_key) +# is dropped by default. Secret inputs are delivered separately through vendor +# credentials attached to the tools nodes. +- name: Build non-secret observability vendor configuration for job extra vars + ansible.builtin.set_fact: + awx_observability_vendors_config: >- + {{ + dict( + (awx_configuration.vendors | ansible.builtin.default({})).keys() + | zip( + (awx_configuration.vendors | ansible.builtin.default({})).keys() + | map('extract', awx_configuration.vendors | ansible.builtin.default({})) + | map('dict2items') + | map('selectattr', 'key', 'in', awx_vendor_nonsecret_allowlist) + | map('list') | map('items2dict') + ) + ) + }} + vars: + # Allowlist of non-secret config keys that may be forwarded as job extra_vars. + awx_vendor_nonsecret_allowlist: + - enabled + - manifest_path + - secret_name + - api_key_name + - app_key_name + - dynakube_path + - dynakube_api_version + - name: Add ITBench job templates awx.awx.job_template: allow_simultaneous: true @@ -16,13 +47,20 @@ playbook: "{{ job_template.playbook }}" project: GitHub-ITBench state: present + extra_vars: "{{ job_template.extra_vars | ansible.builtin.default(omit) }}" loop: - name: Deploy-Tools playbook: scenarios/sre/project/manage_tools.yaml job_tags: install_tools + # Non-secret observability vendor configuration (enable flags, sites, + # manifest paths). Secrets are injected separately via vendor credentials. + extra_vars: + observability_vendors: "{{ awx_observability_vendors_config }}" - name: Undeploy-Tools playbook: scenarios/sre/project/manage_tools.yaml job_tags: uninstall_tools + extra_vars: + observability_vendors: "{{ awx_observability_vendors_config }}" - name: Deploy-Applications playbook: scenarios/sre/project/manage_applications.yaml job_tags: install_applications diff --git a/scenarios/sre/project/roles/awx/tasks/configure_workflows.yaml b/scenarios/sre/project/roles/awx/tasks/configure_workflows.yaml index 4b8b68b4e..a93e80474 100644 --- a/scenarios/sre/project/roles/awx/tasks/configure_workflows.yaml +++ b/scenarios/sre/project/roles/awx/tasks/configure_workflows.yaml @@ -14,6 +14,10 @@ lookup("ansible.builtin.template", "templates/awx/workflows/node_credentials.j2") | ansible.builtin.from_yaml ), + "node_tools_credentials": ( + lookup("ansible.builtin.template", "templates/awx/workflows/node_tools_credentials.j2") | + ansible.builtin.from_yaml + ), "runner_id": kubeconfig_index + 1, "scenario_id": scenario.id } diff --git a/scenarios/sre/project/roles/awx/templates/awx/workflows/node_tools_credentials.j2 b/scenarios/sre/project/roles/awx/templates/awx/workflows/node_tools_credentials.j2 new file mode 100644 index 000000000..21ba19d5a --- /dev/null +++ b/scenarios/sre/project/roles/awx/templates/awx/workflows/node_tools_credentials.j2 @@ -0,0 +1,11 @@ +--- +{% if awx_configuration.providers.aws.enabled | ansible.builtin.default(false) %} + - name: AWS +{% endif %} +{% if awx_configuration.vendors.datadog.enabled | ansible.builtin.default(false) %} + - name: Datadog +{% endif %} +{% if awx_configuration.vendors.dynatrace.enabled | ansible.builtin.default(false) %} + - name: Dynatrace +{% endif %} + - name: Cluster-{{ kubeconfig_index + 1 }}-Kubeconfig diff --git a/scenarios/sre/project/roles/awx/templates/awx/workflows/nodes/scenario_execution.j2 b/scenarios/sre/project/roles/awx/templates/awx/workflows/nodes/scenario_execution.j2 index 984e48329..f7ce4580d 100644 --- a/scenarios/sre/project/roles/awx/templates/awx/workflows/nodes/scenario_execution.j2 +++ b/scenarios/sre/project/roles/awx/templates/awx/workflows/nodes/scenario_execution.j2 @@ -2,7 +2,7 @@ {% if workflow_configuration[0].suffix != "Partial-Execution-Stop" %} - identifier: node-deploy-tools related: - credentials: {{ workflow_configuration[1].node_credentials }} + credentials: {{ workflow_configuration[1].node_tools_credentials }} {% if not workflow_configuration[0].is_leaderboard_workflow %} failure_nodes: - identifier: node-undeploy-tools diff --git a/scenarios/sre/project/roles/awx/templates/meta/argument_specs.j2 b/scenarios/sre/project/roles/awx/templates/meta/argument_specs.j2 index ce451f96f..6c1fe8d36 100644 --- a/scenarios/sre/project/roles/awx/templates/meta/argument_specs.j2 +++ b/scenarios/sre/project/roles/awx/templates/meta/argument_specs.j2 @@ -85,6 +85,59 @@ argument_specs: type: dict required: false type: dict + vendors: + options: + datadog: + options: + enabled: + default: false + required: false + type: bool + manifest_path: + required: false + type: str + secret_name: + required: false + type: str + api_key_name: + required: false + type: str + app_key_name: + required: false + type: str + manifest: + required: false + type: str + api_key: + no_log: true + required: false + type: str + app_key: + no_log: true + required: false + type: str + required: false + type: dict + dynatrace: + options: + enabled: + default: false + required: false + type: bool + dynakube_path: + required: false + type: str + dynakube_api_version: + required: false + type: str + dynakube: + no_log: true + required: false + type: str + required: false + type: dict + required: false + type: dict scenarios: elements: dict options: diff --git a/scenarios/sre/project/roles/cluster/vars/main/minimum_versions.yaml b/scenarios/sre/project/roles/cluster/vars/main/minimum_versions.yaml index a760f12e7..135381fe2 100644 --- a/scenarios/sre/project/roles/cluster/vars/main/minimum_versions.yaml +++ b/scenarios/sre/project/roles/cluster/vars/main/minimum_versions.yaml @@ -1,4 +1,4 @@ --- cluster_minimum_versions: - kubernetes: "1.34" + kubernetes: "1.32" openshift: "4.19" diff --git a/scenarios/sre/project/roles/recorders/defaults/main/managers.yaml b/scenarios/sre/project/roles/recorders/defaults/main/managers.yaml new file mode 100644 index 000000000..6f7b24646 --- /dev/null +++ b/scenarios/sre/project/roles/recorders/defaults/main/managers.yaml @@ -0,0 +1,11 @@ +--- +recorders_managers: + data_recorders: + kubernetes: + namespace: data-recorders + instances: + clickhouse: clickhouse-recorder + dynatrace: dynatrace-recorder + jaeger: jaeger-recorder + kubernetes_topology_monitor: kubernetes-topology-monitor-recorder + prometheus: prometheus-recorder diff --git a/scenarios/sre/project/roles/recorders/defaults/main/vendors.yaml b/scenarios/sre/project/roles/recorders/defaults/main/vendors.yaml new file mode 100644 index 000000000..9d5da26e4 --- /dev/null +++ b/scenarios/sre/project/roles/recorders/defaults/main/vendors.yaml @@ -0,0 +1,8 @@ +--- +# SaaS observability vendor recorder configuration, passed into the role from +# the `observability_vendors` group_var (see manage_recorders.yaml). Defaults to +# disabled so the role is a no-op unless a vendor recorder is explicitly enabled. +recorders_vendors: + dynatrace: + recorder: + enabled: true diff --git a/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/gather.py b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/gather.py new file mode 100644 index 000000000..46591c40f --- /dev/null +++ b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/gather.py @@ -0,0 +1,636 @@ +import csv +import datetime +import json +import logging +import os +import re +import sys +import time + +from datetime import datetime, timedelta, timezone + +import requests + +from requests.adapters import HTTPAdapter +from urllib3.util import Retry + +# Logging +logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + +logger = logging.getLogger(__name__) + +# NOTE: This recorder is the SaaS counterpart to the in-cluster scrapers +# (clickhouse/jaeger/prometheus). Dynatrace Grail retains history that is +# queryable by timeframe, so a single run at teardown back-queries the +# whole incident window [scenario start -> now] for every dataset. The +# topology dataset is the exception: it is captured as two point-in-time +# snapshots (init = scenario start, stop = teardown). It is +# adapted from dynatrace_scripts/pull_dql_json.py: the CLI, .env, and +# dotenv dependency are removed and all configuration comes from +# environment variables injected by the recorder Job. + +# ── Config (from environment) ──────────────────────────────────────────────── +PLATFORM_TOKEN = os.environ.get("DT_PLATFORM_TOKEN", "") +PLATFORM_URL = os.environ.get("DT_PLATFORM_URL", "").rstrip("/") + +NAMESPACE = os.environ.get("DT_K8S_NAMESPACE", "otel-demo").strip() +TIME_FROM = os.environ.get("DT_DQL_FROM", "now-1h") +TIME_TO = os.environ.get("DT_DQL_TO", "now") +INTERVAL = os.environ.get("DT_DQL_INTERVAL", "1m") +LIMIT = os.environ.get("DT_DQL_LIMIT", "1000").strip() +OUTPUT_FORMAT = os.environ.get("DT_DQL_FORMAT", "both").strip() + +# Topology snapshots are point-in-time, but a Grail entity query needs a +# non-empty timeframe: each snapshot queries a short window (this many minutes) +# anchored at its timestamp — the scenario start for "init", teardown for "stop". +TOPOLOGY_WINDOW_MIN = int(os.environ.get("DT_TOPOLOGY_WINDOW_MIN", "5")) + +# All records are written into the recorder PVC (~/records), matching the +# convention used by the jaeger/prometheus recorders. +OUTDIR = os.path.join(os.path.expanduser("~"), "records") + +# ── Kubernetes metric catalog ──────────────────────────────────────────────── +# Each entry: (metric_key, aggregation, grouping). These are queried ONE AT A +# TIME and merged into a single output file (see the "k8s-metrics" dataset). +# Querying individually — rather than one combined `timeseries {a, b, c}` — is +# deliberate: in a combined block a single metric with no data collapses the +# whole result to 0 rows. Per-metric queries keep every populated metric. +# +# grouping "pod" -> by:{k8s.pod.name, k8s.namespace.name} +# grouping "container" -> by:{k8s.pod.name, k8s.container.name, k8s.namespace.name} +K8S_METRICS = [ + # Pod network + ("dt.kubernetes.pod.network_received_data", "sum", "pod"), + ("dt.kubernetes.pod.network_received_errors", "sum", "pod"), + ("dt.kubernetes.pod.network_received_packets_dropped", "sum", "pod"), + ("dt.kubernetes.pod.network_transmitted_data", "sum", "pod"), + ("dt.kubernetes.pod.network_transmitted_errors", "sum", "pod"), + ("dt.kubernetes.pod.network_transmitted_packets_dropped", "sum", "pod"), + # Pod status + ("dt.kubernetes.pod.containers_desired", "max", "pod"), + ("dt.kubernetes.pod.restarts", "max", "pod"), + # Container resources + ("dt.kubernetes.container.requests_cpu", "max", "container"), + ("dt.kubernetes.container.limits_cpu", "max", "container"), + ("dt.kubernetes.container.requests_memory", "max", "container"), + ("dt.kubernetes.container.limits_memory", "max", "container"), + ("dt.kubernetes.container.cpu_usage", "avg", "container"), + ("dt.kubernetes.container.memory_working_set", "avg", "container"), +] +K8S_GROUP_BY = { + "pod": "k8s.pod.name, k8s.namespace.name", + "container": "k8s.pod.name, k8s.container.name, k8s.namespace.name", +} + +# ── Dataset registry ───────────────────────────────────────────────────────── +# Each dataset maps to an output file stem. A timestamp is appended at write +# time so a run produces e.g. dynatrace_logs_2026-07-30T12-00-00.000000.json. +DATASETS = { + "responsetime": {"stem": "dynatrace_responsetime"}, + "errorrate": {"stem": "dynatrace_errorrate"}, + "span-responsetime": {"stem": "dynatrace_span_responsetime"}, + "span-errorrate": {"stem": "dynatrace_span_errorrate"}, + "logs": {"stem": "dynatrace_logs"}, + "events": {"stem": "dynatrace_events"}, + "traces": {"stem": "dynatrace_traces"}, + "problems": {"stem": "dynatrace_problems"}, + # Special: runs every K8S_METRICS key and merges them into one file. + "k8s-metrics": {"stem": "dynatrace_k8s_metrics", "multi": True}, + # Special: Smartscape topology (service graph + infra edges) for the ns. + # Captured as two point-in-time snapshots (see export_topology): "init" at + # the scenario start time and "stop" at teardown. + "topology": {"stem": "dynatrace_topology", "topology": True}, +} +# The recorder always exports the full set (metrics, spans, traces, logs, +# events, problems, k8s, topology). +ALL_DATASETS = [ + "responsetime", "errorrate", "span-responsetime", "span-errorrate", + "logs", "events", "traces", "problems", "k8s-metrics", "topology", +] +# ───────────────────────────────────────────────────────────────────────────── + + +def build_session(): + retries = Retry(total=3, backoff_factor=0.3) + adapter = HTTPAdapter(max_retries=retries) + session = requests.Session() + session.mount("http://", adapter) + session.mount("https://", adapter) + return session + + +def validate_config(): + missing = [ + name + for name, val in [ + ("DT_PLATFORM_TOKEN", PLATFORM_TOKEN), + ("DT_PLATFORM_URL", PLATFORM_URL), + ] + if not val + ] + if missing: + sys.exit( + "error: missing required environment variable(s): {0}".format( + ", ".join(missing) + ) + ) + + +def build_dql(dataset, namespace, interval, limit): + """Return the DQL for a dataset, scoped to the namespace. + + logs/events fetch RAW rows (no projection) so the output keeps every field. + The metric/span datasets aggregate, which is intrinsic to the measurement. + """ + ns_filter = f'filter: {{ k8s.namespace.name == "{namespace}" }}' + if dataset == "errorrate": + return ( + "timeseries {" + " total = sum(dt.service.request.count)," + " failed = sum(dt.service.request.failure_count)" + f" }}, by:{{dt.entity.service}}, {ns_filter}, interval:{interval}" + "| fieldsAdd error_rate = if(arraySum(total) > 100," + " (failed[] / total[]) * 100, else: 0)" + ) + if dataset == "responsetime": + return ( + "timeseries responsetime = avg(dt.service.request.response_time)," + f" by:{{dt.entity.service}}, {ns_filter}, interval:{interval}" + ) + if dataset == "span-responsetime": + return ( + f'fetch spans | filter k8s.namespace.name == "{namespace}"' + " and request.is_root_span == true" + f" | makeTimeseries rt_ns = avg(duration), by:{{service.name}}," + f" interval:{interval}" + " | fieldsAdd responsetime_ms = rt_ns[] / 1000000.0" + ) + if dataset == "span-errorrate": + return ( + f'fetch spans | filter k8s.namespace.name == "{namespace}"' + " and request.is_root_span == true" + " | fieldsAdd failed_num = if(request.is_failed == true, 1, else: 0)" + f" | makeTimeseries err = avg(failed_num), by:{{service.name}}," + f" interval:{interval}" + " | fieldsAdd error_rate = err[] * 100.0" + ) + if dataset == "logs": + return ( + f'fetch logs | filter k8s.namespace.name == "{namespace}"' + f" | sort timestamp desc | limit {limit}" + ) + if dataset == "events": + return ( + f'fetch events | filter k8s.namespace.name == "{namespace}"' + f" | sort timestamp desc | limit {limit}" + ) + if dataset == "traces": + # Raw spans (distributed traces); no projection so the output keeps every + # span attribute. Sorted newest-first by span start_time. + return ( + f'fetch spans | filter k8s.namespace.name == "{namespace}"' + f" | sort start_time desc | limit {limit}" + ) + if dataset == "problems": + # Davis problems; no projection so the output keeps every field. Scoped to + # the namespace via the record's k8s.namespace.name ARRAY field (resolved + # from affected entities): in(, k8s.namespace.name). Duplicates dropped. + return ( + "fetch dt.davis.problems | filter not(dt.davis.is_duplicate)" + f' and in("{namespace}", k8s.namespace.name)' + f" | sort event.start desc | limit {limit}" + ) + raise ValueError(f"Unknown dataset '{dataset}'") + + +def build_metric_dql(metric_key, agg, grouping, namespace, interval): + """DQL for a single k8s metric timeseries, scoped to the namespace.""" + by = K8S_GROUP_BY[grouping] + return ( + f"timeseries value = {agg}({metric_key}), by:{{{by}}}," + f' filter:{{ k8s.namespace.name == "{namespace}" }}, interval:{interval}' + ) + + +def to_iso(value): + """Convert `now` / `now-` to ISO-8601 UTC (Grail rejects relative + strings in defaultTimeframeStart/End). ISO input is passed through, so a + scenario-start timestamp supplied via DT_DQL_FROM works directly.""" + value = value.strip() + now = datetime.now(timezone.utc) + if value == "now": + dt = now + else: + m = re.fullmatch(r"now-(\d+)([mhd])", value) + if not m: + return value + n, unit = int(m.group(1)), m.group(2) + delta = {"m": "minutes", "h": "hours", "d": "days"}[unit] + dt = now - timedelta(**{delta: n}) + return dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + +def _raise_with_body(resp): + """raise_for_status but surface the API error body (Grail puts the reason — + INVALID_TIMEFRAME, token/permission errors — in the JSON).""" + if resp.status_code >= 400: + try: + detail = resp.json() + except ValueError: + detail = resp.text + logger.warning("HTTP %s from %s\n%s", resp.status_code, resp.url, detail) + resp.raise_for_status() + + +def run_query(session, dql, time_from, time_to): + """Execute DQL with the platform token; poll until the query completes. + Returns the full `result` object (records + metadata).""" + headers = { + "Authorization": "Bearer {0}".format(PLATFORM_TOKEN), + "Content-Type": "application/json", + "Accept": "application/json", + } + exec_resp = session.post( + "{0}/platform/storage/query/v1/query:execute".format(PLATFORM_URL), + headers=headers, + json={"query": dql, "defaultTimeframeStart": to_iso(time_from), + "defaultTimeframeEnd": to_iso(time_to)}, + timeout=60, + ) + _raise_with_body(exec_resp) + body = exec_resp.json() + + if body.get("state") == "SUCCEEDED" and "result" in body: + return body["result"] + + request_token = body["requestToken"] + poll_url = "{0}/platform/storage/query/v1/query:poll".format(PLATFORM_URL) + while True: + poll = session.get( + poll_url, headers=headers, + params={"request-token": request_token}, timeout=60, + ) + _raise_with_body(poll) + pbody = poll.json() + state = pbody.get("state") + if state == "SUCCEEDED": + return pbody["result"] + if state in ("FAILED", "CANCELLED"): + raise RuntimeError("Query {0}: {1}".format(state, pbody)) + time.sleep(2) + + +def grail_notifications(result): + return [ + n.get("message", "") + for n in result.get("metadata", {}).get("grail", {}).get("notifications", []) + if n.get("message") + ] + + +# ── CSV flattening ─────────────────────────────────────────────────────────── +def _parse_iso(s): + """Parse Grail ISO-8601 (nanosecond precision) into a UTC datetime.""" + m = re.match(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d+))?", s) + base = datetime.strptime(m.group(1), "%Y-%m-%dT%H:%M:%S") + if m.group(2): + base = base.replace(microsecond=int(m.group(2)[:6].ljust(6, "0"))) + return base.replace(tzinfo=timezone.utc) + + +def _series_timestamps(rec, n): + """Per-bucket timestamps for a timeseries record: use explicit timestamps[] + if present, else derive from timeframe.start + interval.""" + tf = rec.get("timeframe", {}) or {} + if tf.get("timestamps"): + return tf["timestamps"] + start, interval_ns = tf.get("start"), rec.get("interval") + if start and interval_ns: + base = _parse_iso(start) + step = timedelta(microseconds=int(interval_ns) / 1000) + return [(base + i * step).strftime("%Y-%m-%dT%H:%M:%S.000Z") + for i in range(n)] + return list(range(n)) + + +def flatten_result(result, extra=None): + """Flatten a Grail result into a list of flat dict rows for CSV. + + - Flat records (logs/events): one row per record, scalar fields as-is. + - Timeseries records (value arrays + timeframe/interval): expanded to one + row per bucket with a reconstructed `timestamp` column. + `extra` adds constant columns to every row (e.g. the metric key).""" + rows = [] + extra = extra or {} + for rec in result.get("records", []): + array_fields = {k: v for k, v in rec.items() + if isinstance(v, list) and k != "timestamps"} + scalar_fields = {k: v for k, v in rec.items() + if k not in array_fields and k not in + ("timeframe", "interval")} + + if array_fields: + length = max((len(v) for v in array_fields.values()), default=0) + timestamps = _series_timestamps(rec, length) + for i in range(length): + if all(v[i] is None for v in array_fields.values() + if i < len(v)): + continue # skip buckets where every series is null + row = dict(extra) + row["timestamp"] = timestamps[i] if i < len(timestamps) else i + row.update(scalar_fields) + for k, v in array_fields.items(): + row[k] = v[i] if i < len(v) else None + rows.append(row) + else: + row = dict(extra) + row.update(scalar_fields) + rows.append(row) + return rows + + +def write_csv_rows(rows, path): + """Write flat dict rows to CSV; columns = union of keys across rows.""" + columns = [] + for r in rows: + for k in r: + if k not in columns: + columns.append(k) + with open(path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=columns) + writer.writeheader() + writer.writerows(rows) + return len(rows) + + +def out_paths(dataset, timestamp): + """(json_path, csv_path) for a dataset, with the run timestamp appended.""" + stem = "{0}_{1}".format(DATASETS[dataset]["stem"], timestamp) + base = os.path.join(OUTDIR, stem) + return base + ".json", base + ".csv" + + +def _shift_iso(iso, minutes): + """Shift an ISO-8601 UTC timestamp by a number of minutes (may be negative).""" + shifted = _parse_iso(iso) + timedelta(minutes=minutes) + return shifted.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + +def _resolve_namespace_entity(session, namespace, time_from, time_to): + """Look up the CLOUD_APPLICATION_NAMESPACE entity id for a k8s namespace + name. Smartscape scopes services by their `belongs_to` relationship to this + entity — so we need its id to filter the topology to one namespace.""" + dql = ( + "fetch dt.entity.cloud_application_namespace" + f' | filter entity.name == "{namespace}"' + " | fields id, entity.name" + ) + recs = run_query(session, dql, time_from, time_to).get("records", []) + return recs[0]["id"] if recs else None + + +def _topology_snapshot(session, ns_id, time_from, time_to): + """One point-in-time Smartscape snapshot over [time_from, time_to]. + + Fetch every service that `belongs_to` the namespace along with its + relationship columns, then derive directed edges: service->service `calls`, + and service->host / service->process_group `runs_on`. Returns + (dql, services, nodes, edges).""" + dql = ( + "fetch dt.entity.service" + " | fieldsAdd ns = belongs_to[dt.entity.cloud_application_namespace]" + f' | filter in("{ns_id}", ns)' + " | fields id, entity.name," + " calls[dt.entity.service]," + " runs_on[dt.entity.host]," + " runs_on[dt.entity.process_group]" + ) + result = run_query(session, dql, time_from, time_to) + for msg in grail_notifications(result): + logger.info("[topology] Grail notice: %s", msg) + + services = result.get("records", []) + # Set of in-namespace service ids: we keep call edges to any service, but + # tag whether the callee is inside the namespace (target_in_namespace). + ns_service_ids = {s.get("id") for s in services} + # id -> service name, so the edge list is readable without a separate join. + names = {s.get("id"): s.get("entity.name") for s in services} + + nodes, edges = [], [] + for svc in services: + sid = svc.get("id") + sname = svc.get("entity.name") + nodes.append({"id": sid, "name": sname, "type": "SERVICE"}) + # service -> service call edges + for tgt in (svc.get("calls[dt.entity.service]") or []): + edges.append({"source": sid, "source_name": sname, + "target": tgt, "target_name": names.get(tgt, ""), + "type": "CALLS", + "target_in_namespace": tgt in ns_service_ids}) + # service -> host / process_group placement edges (targets are infra + # entities outside the service set, so no name is resolved here) + for host in (svc.get("runs_on[dt.entity.host]") or []): + edges.append({"source": sid, "source_name": sname, "target": host, + "target_name": "", "type": "RUNS_ON_HOST"}) + for pg in (svc.get("runs_on[dt.entity.process_group]") or []): + edges.append({"source": sid, "source_name": sname, "target": pg, + "target_name": "", "type": "RUNS_ON_PROCESS_GROUP"}) + return dql, services, nodes, edges + + +def export_topology(session, dataset, timestamp): + """Export the Smartscape topology as TWO point-in-time snapshots: + + * init — anchored at the scenario start time (DT_DQL_FROM) + * stop — anchored at teardown (DT_DQL_TO) + + Each snapshot queries a short window around its anchor (an entity query + needs a non-empty timeframe) and is written to its own files, e.g. + dynatrace_topology_init_.json / dynatrace_topology_stop_.json. + Mirrors dynatrace_scripts/pull_dql_json.py's export_topology graph shape.""" + # Resolve each snapshot's [from, to] window around its anchor timestamp. + init_from = to_iso(TIME_FROM) + init_to = _shift_iso(init_from, TOPOLOGY_WINDOW_MIN) + stop_to = to_iso(TIME_TO) + stop_from = _shift_iso(stop_to, -TOPOLOGY_WINDOW_MIN) + snapshots = [ + ("init", init_from, init_to), + ("stop", stop_from, stop_to), + ] + + any_data = False + for phase, snap_from, snap_to in snapshots: + stem = "{0}_{1}_{2}".format(DATASETS[dataset]["stem"], phase, timestamp) + base = os.path.join(OUTDIR, stem) + json_path, csv_path = base + ".json", base + ".csv" + + try: + ns_id = _resolve_namespace_entity(session, NAMESPACE, + snap_from, snap_to) + if not ns_id: + logger.warning("[topology/%s] no namespace entity for '%s'", + phase, NAMESPACE) + continue + dql, services, nodes, edges = _topology_snapshot( + session, ns_id, snap_from, snap_to) + except Exception as exc: + logger.warning("[topology/%s] ERROR: %s", phase, exc) + continue + + written = [] + if OUTPUT_FORMAT in ("json", "both"): + payload = { + "dataset": dataset, + "phase": phase, + "namespace": NAMESPACE, + "namespace_entity_id": ns_id, + "timeframe": {"from": snap_from, "to": snap_to}, + "query": dql, + "node_count": len(nodes), + "edge_count": len(edges), + "graph": {"nodes": nodes, "edges": edges}, + "services": services, # raw records, all relationship columns + } + with open(json_path, "w") as f: + json.dump(payload, f, indent=2) + written.append(json_path) + + if OUTPUT_FORMAT in ("csv", "both"): + # Flatten to an edge list — the natural tabular view of a graph. + write_csv_rows(edges, csv_path) + written.append(csv_path) + + logger.info("[topology/%s] %d services, %d edges -> %s", + phase, len(nodes), len(edges), ", ".join(written)) + any_data = any_data or bool(nodes) + + return any_data + + +def export_k8s_metrics(session, dataset, timestamp): + """Query every K8S_METRICS key individually and merge them into ONE file — + so a single run exports many metrics together. Each metric is a separate + query (a combined `timeseries {..}` would zero out if any one metric had no + data), and each metric's full result is kept under its key.""" + json_path, csv_path = out_paths(dataset, timestamp) + metrics_out = [] + ok = 0 + for metric_key, agg, grouping in K8S_METRICS: + entry = {"metric": metric_key, "aggregation": agg, "grouping": grouping} + dql = build_metric_dql(metric_key, agg, grouping, NAMESPACE, INTERVAL) + try: + result = run_query(session, dql, TIME_FROM, TIME_TO) + except Exception as exc: + logger.warning("[%s] %s: ERROR: %s", dataset, metric_key, exc) + entry.update(query=dql, error=str(exc), record_count=0, result=None) + metrics_out.append(entry) + continue + + for msg in grail_notifications(result): + logger.info("[%s] %s: Grail notice: %s", dataset, metric_key, msg) + n = len(result.get("records", [])) + if n: + ok += 1 + logger.info("[%s] %s: %d records", dataset, metric_key, n) + entry.update(query=dql, record_count=n, result=result) + metrics_out.append(entry) + + written = [] + + if OUTPUT_FORMAT in ("json", "both"): + payload = { + "dataset": dataset, + "namespace": NAMESPACE, + "timeframe": {"from": TIME_FROM, "to": TIME_TO}, + "interval": INTERVAL, + "metric_count": len(metrics_out), + "metrics_with_data": ok, + "metrics": metrics_out, + } + with open(json_path, "w") as f: + json.dump(payload, f, indent=2) + written.append(json_path) + + if OUTPUT_FORMAT in ("csv", "both"): + # Merge every metric into one CSV, tagging each row with its metric key. + rows = [] + for entry in metrics_out: + if entry.get("result"): + rows.extend(flatten_result(entry["result"], + extra={"metric": entry["metric"]})) + write_csv_rows(rows, csv_path) + written.append(csv_path) + + logger.info("[%s] wrote %d metrics (%d with data) -> %s", + dataset, len(metrics_out), ok, ", ".join(written)) + return ok > 0 + + +def export_dataset(session, dataset, timestamp): + """Run one dataset and write its output. Returns True if it produced data. + Never raises — failures are captured so a multi-dataset run continues.""" + if DATASETS[dataset].get("multi"): + return export_k8s_metrics(session, dataset, timestamp) + if DATASETS[dataset].get("topology"): + return export_topology(session, dataset, timestamp) + + json_path, csv_path = out_paths(dataset, timestamp) + try: + dql = build_dql(dataset, NAMESPACE, INTERVAL, LIMIT) + logger.info("[%s] %s", dataset, dql) + result = run_query(session, dql, TIME_FROM, TIME_TO) + except Exception as exc: + logger.warning("[%s] ERROR: %s", dataset, exc) + return False + + for msg in grail_notifications(result): + logger.info("[%s] Grail notice: %s", dataset, msg) + + records = result.get("records", []) + written = [] + + if OUTPUT_FORMAT in ("json", "both"): + # Persist the full response verbatim: query context + records + metadata. + payload = { + "dataset": dataset, + "query": dql, + "timeframe": {"from": TIME_FROM, "to": TIME_TO}, + "record_count": len(records), + "result": result, + } + with open(json_path, "w") as f: + json.dump(payload, f, indent=2) + written.append(json_path) + + if OUTPUT_FORMAT in ("csv", "both"): + rows = flatten_result(result) + write_csv_rows(rows, csv_path) + written.append(csv_path) + + logger.info("[%s] wrote %d records -> %s", + dataset, len(records), ", ".join(written)) + return len(records) > 0 + + +def main(): + validate_config() + + if OUTPUT_FORMAT not in ("json", "csv", "both"): + sys.exit("error: DT_DQL_FORMAT must be one of json, csv, both") + + os.makedirs(OUTDIR, exist_ok=True) + session = build_session() + + timestamp = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H-%M-%S.%f') + logger.info("Datasets: %s | namespace=%s | %s -> %s | format=%s", + ", ".join(ALL_DATASETS), NAMESPACE, TIME_FROM, TIME_TO, + OUTPUT_FORMAT) + + results = [export_dataset(session, ds, timestamp) for ds in ALL_DATASETS] + + if not any(results): + logger.warning("no datasets returned any records") + + +if __name__ == "__main__": + main() diff --git a/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/requirements.txt b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/requirements.txt new file mode 100644 index 000000000..a258782f6 --- /dev/null +++ b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/requirements.txt @@ -0,0 +1 @@ +requests==2.34.2 diff --git a/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/run_local.sh b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/run_local.sh new file mode 100755 index 000000000..eb733a9fe --- /dev/null +++ b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/run_local.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# +# Run the Dynatrace Grail recorder (gather.py) locally, against the Dynatrace +# SaaS API — no Kubernetes cluster needed. This is the manual counterpart to the +# in-cluster recorder Job: identical script, output written to ./dynatrace-records +# on this machine instead of a PVC. +# +# Config resolution (first non-empty wins): +# 1. environment variables (DT_PLATFORM_URL, DT_PLATFORM_TOKEN, ...) +# 2. the recorder block in observability_vendors.yaml (platform_url, +# platform_token_path, namespace) +# 3. built-in defaults +# +# Usage: +# ./run_local.sh # window = now-1h .. now +# DT_DQL_FROM=2026-07-30T09:00:00Z ./run_local.sh +# DT_DQL_FROM=now-6h DT_DQL_FORMAT=csv ./run_local.sh +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# scripts/dynatrace -> scenarios/sre is six levels up +# (dynatrace/scripts/files/recorders/roles/project -> sre). +SRE_ROOT="$(cd "${SCRIPT_DIR}/../../../../../.." && pwd)" +VENDORS_FILE="${SRE_ROOT}/inventory/group_vars/environment/observability_vendors.yaml" + +# ── Tiny YAML value reader for observability_vendors.dynatrace.recorder. ── +# Only used as a fallback when the matching env var is unset. Uses python3 (yaml +# is available with the project's uv env) and falls back to empty on any error. +vendor_get() { + local key="$1" + python3 - "$VENDORS_FILE" "$key" <<'PY' 2>/dev/null || true +import sys, yaml +path, key = sys.argv[1], sys.argv[2] +try: + with open(path) as f: + data = yaml.safe_load(f) or {} + rec = (((data.get("observability_vendors") or {}) + .get("dynatrace") or {}) + .get("recorder") or {}) + val = rec.get(key, "") + print("" if val is None else val) +except Exception: + print("") +PY +} + +# ── Resolve config ──────────────────────────────────────────────────────────── +DT_PLATFORM_URL="${DT_PLATFORM_URL:-$(vendor_get platform_url)}" +DT_K8S_NAMESPACE="${DT_K8S_NAMESPACE:-$(vendor_get namespace)}" +DT_K8S_NAMESPACE="${DT_K8S_NAMESPACE:-otel-demo}" + +# Token: explicit env wins, else read the file referenced by platform_token_path. +if [ -z "${DT_PLATFORM_TOKEN:-}" ]; then + token_path="$(vendor_get platform_token_path)" + if [ -n "$token_path" ]; then + case "$token_path" in + /*) : ;; # absolute + *) token_path="${SRE_ROOT}/${token_path}" ;; # relative to SRE root + esac + if [ -f "$token_path" ]; then + DT_PLATFORM_TOKEN="$(tr -d '\r\n' < "$token_path")" + fi + fi +fi + +DT_DQL_FROM="${DT_DQL_FROM:-now-1h}" +DT_DQL_TO="${DT_DQL_TO:-now}" +DT_DQL_INTERVAL="${DT_DQL_INTERVAL:-1m}" +DT_DQL_LIMIT="${DT_DQL_LIMIT:-1000}" +DT_DQL_FORMAT="${DT_DQL_FORMAT:-both}" + +# Output goes to a local directory the script owns (HOME/records is what +# gather.py writes to, so point HOME at our output dir for this run). +OUTDIR="${DT_LOCAL_OUTDIR:-${SCRIPT_DIR}/dynatrace-records}" + +# ── Validate ──────────────────────────────────────────────────────────────── +missing="" +[ -z "${DT_PLATFORM_URL}" ] && missing="${missing} DT_PLATFORM_URL(or recorder.platform_url)" +[ -z "${DT_PLATFORM_TOKEN}" ] && missing="${missing} DT_PLATFORM_TOKEN(or recorder.platform_token_path)" +if [ -n "$missing" ]; then + echo "error: missing config:${missing}" >&2 + exit 1 +fi + +# ── Python env ──────────────────────────────────────────────────────────────── +VENV="${DT_LOCAL_VENV:-/tmp/dtrec}" +if [ ! -x "${VENV}/bin/python" ]; then + echo "Creating virtualenv at ${VENV} ..." + python3 -m venv "${VENV}" +fi +"${VENV}/bin/pip" install --quiet --disable-pip-version-check -r "${SCRIPT_DIR}/requirements.txt" + +# ── Run ────────────────────────────────────────────────────────────────────── +mkdir -p "${OUTDIR}/records" +echo "Dynatrace recorder (local)" +echo " URL : ${DT_PLATFORM_URL}" +echo " namespace : ${DT_K8S_NAMESPACE}" +echo " window : ${DT_DQL_FROM} -> ${DT_DQL_TO}" +echo " format : ${DT_DQL_FORMAT}" +echo " output : ${OUTDIR}/records" +echo + +# gather.py writes to \$HOME/records; point HOME at OUTDIR just for this process. +HOME="${OUTDIR}" \ +DT_PLATFORM_URL="${DT_PLATFORM_URL}" \ +DT_PLATFORM_TOKEN="${DT_PLATFORM_TOKEN}" \ +DT_K8S_NAMESPACE="${DT_K8S_NAMESPACE}" \ +DT_DQL_FROM="${DT_DQL_FROM}" \ +DT_DQL_TO="${DT_DQL_TO}" \ +DT_DQL_INTERVAL="${DT_DQL_INTERVAL}" \ +DT_DQL_LIMIT="${DT_DQL_LIMIT}" \ +DT_DQL_FORMAT="${DT_DQL_FORMAT}" \ + "${VENV}/bin/python" "${SCRIPT_DIR}/gather.py" + +echo +echo "Done. Files:" +ls -la "${OUTDIR}/records" diff --git a/scenarios/sre/project/roles/recorders/meta/argument_specs.yaml b/scenarios/sre/project/roles/recorders/meta/argument_specs.yaml index eb2b41b50..26694281b 100644 --- a/scenarios/sre/project/roles/recorders/meta/argument_specs.yaml +++ b/scenarios/sre/project/roles/recorders/meta/argument_specs.yaml @@ -48,6 +48,56 @@ argument_specs: default: false required: false type: bool + recorders_vendors: + required: false + type: dict + options: + dynatrace: + required: false + type: dict + options: + recorder: + required: false + type: dict + options: + enabled: + default: false + required: false + type: bool + platform_url: + required: false + type: str + platform_token_path: + required: false + type: str + namespace: + default: otel-demo + required: false + type: str + timeframe_to: + default: now + required: false + type: str + fallback_from: + default: now-1h + required: false + type: str + interval: + default: 1m + required: false + type: str + limit: + default: 1000 + required: false + type: int + format: + choices: + - json + - csv + - both + default: both + required: false + type: str recorders_storage: required: false type: dict diff --git a/scenarios/sre/project/roles/recorders/tasks/install.yaml b/scenarios/sre/project/roles/recorders/tasks/install.yaml index e2380bf81..8a42cdd42 100644 --- a/scenarios/sre/project/roles/recorders/tasks/install.yaml +++ b/scenarios/sre/project/roles/recorders/tasks/install.yaml @@ -48,6 +48,16 @@ when: - recorders_configuration.phase == "pre-fault-injection" + - name: Stamp scenario start time for Dynatrace Recorder + # The Dynatrace recorder itself runs at teardown (Grail is queryable by + # timeframe); here we only record the scenario start so teardown can + # back-query [scenario start -> now]. + ansible.builtin.import_tasks: + file: stamp_dynatrace_start_time.yaml + when: + - recorders_configuration.phase == "pre-fault-injection" + - recorders_vendors.dynatrace.recorder.enabled | ansible.builtin.default(false) + - name: Install SRE specific recorders when: - recorders_configuration.tools.sre.enabled | ansible.builtin.default(false) diff --git a/scenarios/sre/project/roles/recorders/tasks/install_dynatrace_recorder.yaml b/scenarios/sre/project/roles/recorders/tasks/install_dynatrace_recorder.yaml new file mode 100644 index 000000000..abbc1669e --- /dev/null +++ b/scenarios/sre/project/roles/recorders/tasks/install_dynatrace_recorder.yaml @@ -0,0 +1,131 @@ +--- +# Dynatrace Grail recorder (SaaS). Unlike the in-cluster recorders, Dynatrace +# retains history queryable by timeframe, so this runs ONCE at teardown and +# back-queries the whole incident window [scenario start -> now] for every +# dataset (metrics, spans, logs, events, k8s metrics). +# +# All configuration comes from observability_vendors.dynatrace.recorder. The +# platform token is read at teardown from a gitignored file (platform_token_path, +# under secrets/ — same convention as dynakube_path) and injected straight into +# the recorder Job as an environment variable value, mirroring how the ClickHouse +# recorder injects CLICKHOUSE_PASSWORD. No environment export and no Kubernetes +# Secret are required. + +- name: Resolve Dynatrace recorder configuration + ansible.builtin.set_fact: + recorders_dynatrace_config: "{{ recorders_vendors.dynatrace.recorder }}" + +- name: Validate that the Dynatrace platform token file is configured + ansible.builtin.assert: + that: + - recorders_dynatrace_config.platform_token_path | default("") | ansible.builtin.length > 0 + fail_msg: >- + Dynatrace recorder is enabled but observability_vendors.dynatrace.recorder.platform_token_path + is not set. Point it at a file (under secrets/, gitignored) containing a + platform token (prefix dt0s16.) with Grail read permissions. + success_msg: Dynatrace platform token path found. + quiet: true + +- name: Resolve the Dynatrace platform token path + ansible.builtin.set_fact: + recorders_dynatrace_token_path: "{{ recorders_dynatrace_config.platform_token_path }}" + +# A relative platform_token_path is resolved against the SRE project root (parent +# of the playbook directory) so it works regardless of the working directory — +# mirroring how install_datadog.yaml anchors its manifest_path. +- name: Anchor a relative Dynatrace platform token path to the project root + ansible.builtin.set_fact: + recorders_dynatrace_token_path: "{{ [playbook_dir, '..', recorders_dynatrace_token_path] | ansible.builtin.path_join | ansible.builtin.realpath }}" + when: + - not (recorders_dynatrace_token_path is ansible.builtin.abs) + +- name: Validate that the Dynatrace platform token file exists + ansible.builtin.stat: + path: "{{ recorders_dynatrace_token_path }}" + register: recorders_dynatrace_token_file + +- name: Assert the Dynatrace platform token file is present and non-empty + ansible.builtin.assert: + that: + - recorders_dynatrace_token_file.stat.exists + - recorders_dynatrace_token_file.stat.size > 0 + fail_msg: >- + Dynatrace platform token file not found or empty: + {{ recorders_dynatrace_token_path }} + success_msg: Dynatrace platform token file found. + quiet: true + +- name: Read the Dynatrace platform token + ansible.builtin.set_fact: + recorders_dynatrace_platform_token: >- + {{ lookup('ansible.builtin.file', recorders_dynatrace_token_path) | trim }} + no_log: true + +- name: Validate that the Dynatrace platform URL is set + ansible.builtin.assert: + that: + - recorders_dynatrace_config.platform_url | default("") | ansible.builtin.length > 0 + fail_msg: >- + Dynatrace recorder is enabled but observability_vendors.dynatrace.recorder.platform_url + is not set (e.g. https://.apps.dynatrace.com). + success_msg: Dynatrace platform URL found. + quiet: true + +- name: Retrieve scenario start time metadata + kubernetes.core.k8s_info: + api_version: v1 + kind: ConfigMap + kubeconfig: "{{ recorders_cluster.kubeconfig }}" + name: "{{ recorders_managers.data_recorders.instances.dynatrace }}-metadata" + namespace: "{{ recorders_managers.data_recorders.kubernetes.namespace }}" + register: recorders_dynatrace_metadata + +- name: Resolve query timeframe start + # Prefer the scenario start time stamped at install; fall back to the + # configured relative lookback if the marker is missing. + ansible.builtin.set_fact: + recorders_dynatrace_from: >- + {{ + ( + recorders_dynatrace_metadata.resources[0].data.scenario_start_time + if (recorders_dynatrace_metadata.resources | ansible.builtin.length > 0) + else recorders_dynatrace_config.fallback_from | default("now-1h") + ) + }} + +- name: Create Dynatrace Data Recorder environment variables + # The token is injected as a plain env value (same approach as the ClickHouse + # recorder's CLICKHOUSE_PASSWORD). no_log hides it from task output. + ansible.builtin.set_fact: + recorders_dynatrace_env_vars: + - name: DT_PLATFORM_URL + value: "{{ recorders_dynatrace_config.platform_url }}" + - name: DT_PLATFORM_TOKEN + value: "{{ recorders_dynatrace_platform_token }}" + - name: DT_K8S_NAMESPACE + value: "{{ recorders_dynatrace_config.namespace | default('otel-demo') }}" + - name: DT_DQL_FROM + value: "{{ recorders_dynatrace_from }}" + - name: DT_DQL_TO + value: "{{ recorders_dynatrace_config.timeframe_to | default('now') }}" + - name: DT_DQL_INTERVAL + value: "{{ recorders_dynatrace_config.interval | default('1m') }}" + - name: DT_DQL_LIMIT + value: "{{ recorders_dynatrace_config.limit | default(1000) | ansible.builtin.string }}" + - name: DT_DQL_FORMAT + value: "{{ recorders_dynatrace_config.format | default('both') }}" + no_log: true + +- name: Import recorder deployment tasks + ansible.builtin.import_tasks: + file: deploy_python_recorder.yaml + vars: + python_recorder: + job: + environment_variables: "{{ recorders_dynatrace_env_vars }}" + name: "{{ recorders_managers.data_recorders.instances.dynatrace }}" + scripts: + requirements_file_path: "files/scripts/dynatrace/requirements.txt" + scripts_file_path: "files/scripts/dynatrace/gather.py" + volume: + size: 2Gi diff --git a/scenarios/sre/project/roles/recorders/tasks/stamp_dynatrace_start_time.yaml b/scenarios/sre/project/roles/recorders/tasks/stamp_dynatrace_start_time.yaml new file mode 100644 index 000000000..f6c9c6ddf --- /dev/null +++ b/scenarios/sre/project/roles/recorders/tasks/stamp_dynatrace_start_time.yaml @@ -0,0 +1,34 @@ +--- +# Persist the scenario start time so the Dynatrace recorder (which runs once at +# teardown) can back-query Grail over exactly [scenario start -> now]. Install +# and uninstall are separate playbook runs, so in-memory facts do not survive; +# the timestamp is written into a ConfigMap in the recorders namespace, which +# lives until teardown deletes the namespace. +- name: Check for an existing Dynatrace recorder metadata ConfigMap + kubernetes.core.k8s_info: + api_version: v1 + kind: ConfigMap + kubeconfig: "{{ recorders_cluster.kubeconfig }}" + name: "{{ recorders_managers.data_recorders.instances.dynatrace }}-metadata" + namespace: "{{ recorders_managers.data_recorders.kubernetes.namespace }}" + register: recorders_dynatrace_metadata + +- name: Stamp scenario start time for Dynatrace recorder + # Only stamp once so a re-run of the install phase does not reset the start + # time; the first recorded value is the authoritative scenario start. + when: + - recorders_dynatrace_metadata.resources | ansible.builtin.length == 0 + kubernetes.core.k8s: + kubeconfig: "{{ recorders_cluster.kubeconfig }}" + resource_definition: + apiVersion: v1 + kind: ConfigMap + metadata: + labels: + "app.kubernetes.io/managed-by": ITBench + "app.kubernetes.io/name": "{{ recorders_managers.data_recorders.instances.dynatrace }}" + name: "{{ recorders_managers.data_recorders.instances.dynatrace }}-metadata" + namespace: "{{ recorders_managers.data_recorders.kubernetes.namespace }}" + data: + scenario_start_time: "{{ now(utc=True, fmt='%Y-%m-%dT%H:%M:%SZ') }}" + state: present diff --git a/scenarios/sre/project/roles/recorders/tasks/uninstall.yaml b/scenarios/sre/project/roles/recorders/tasks/uninstall.yaml index 112712a39..4e6a7e6b5 100644 --- a/scenarios/sre/project/roles/recorders/tasks/uninstall.yaml +++ b/scenarios/sre/project/roles/recorders/tasks/uninstall.yaml @@ -28,6 +28,15 @@ label: cronjob/{{ cronjob.metadata.name }} loop_var: cronjob +- name: Deploy Dynatrace Recorder + # Dynatrace Grail is queryable by timeframe, so the recorder runs ONCE here at + # teardown and back-queries [scenario start -> now]. It is deployed before the + # job wait below so its one-shot Job is awaited like the others. + ansible.builtin.import_tasks: + file: install_dynatrace_recorder.yaml + when: + - recorders_vendors.dynatrace.recorder.enabled | ansible.builtin.default(false) + - name: Retrieve Jobs kubernetes.core.k8s_info: api_version: batch/v1 @@ -96,6 +105,17 @@ path: alerts name: "{{ recorders_instances.prometheus_data_recorder.name }}" +- name: Import Dynatrace recorder export tasks + ansible.builtin.import_tasks: + file: copy_python_recorder_files.yaml + vars: + python_recorder: + export: + path: observability_information_from_dynatrace + name: "{{ recorders_managers.data_recorders.instances.dynatrace }}" + when: + - recorders_vendors.dynatrace.recorder.enabled | ansible.builtin.default(false) + - name: Delete the namespace kubernetes.core.k8s: kubeconfig: "{{ recorders_cluster.kubeconfig }}" diff --git a/scenarios/sre/project/roles/tools/defaults/main/vendors.yaml b/scenarios/sre/project/roles/tools/defaults/main/vendors.yaml new file mode 100644 index 000000000..663d5fe13 --- /dev/null +++ b/scenarios/sre/project/roles/tools/defaults/main/vendors.yaml @@ -0,0 +1,31 @@ +--- +# Registry of third-party SaaS observability vendors (Layer A: agent deployment). +# +# Unlike `tools_managers` (in-cluster backends whose endpoints/credentials are +# discovered at runtime), these vendors are SaaS: their agents run in-cluster via +# a Helm chart but ship telemetry to the vendor's cloud tenant. Non-secret tenant +# configuration (site, region, environment URL) is supplied via the +# `observability_vendors` group_vars; secrets are injected via environment +# variables at install time (see set_credentials_.yaml). +tools_vendors: + datadog: + helm: + chart: + # renovate: datasource=helm depName=datadog-operator registryUrl=https://helm.datadoghq.com + reference: datadog-operator + repository: https://helm.datadoghq.com + version: 2.24.0 + release: + name: datadog-operator + kubernetes: + namespace: datadog + dynatrace: + helm: + chart: + # renovate: datasource=docker depName=public.ecr.aws/dynatrace/dynatrace-operator + reference: oci://public.ecr.aws/dynatrace/dynatrace-operator + version: 1.9.0 + release: + name: dynatrace-operator + kubernetes: + namespace: dynatrace diff --git a/scenarios/sre/project/roles/tools/molecule/deployment_vendors/converge.yml b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/converge.yml new file mode 100644 index 000000000..5c67b7e5f --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/converge.yml @@ -0,0 +1,8 @@ +--- +- name: Skip convergence + hosts: + - localhost + tasks: + - name: Display message + ansible.builtin.debug: + msg: "Deployment completed in create phase. Proceeding to verification." diff --git a/scenarios/sre/project/roles/tools/molecule/deployment_vendors/create.yml b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/create.yml new file mode 100644 index 000000000..cd52b0eab --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/create.yml @@ -0,0 +1,56 @@ +--- +- name: Install observability vendor agents + hosts: + - localhost + tasks: + - name: Resolve vendor manifest paths from environment + ansible.builtin.set_fact: + molecule_datadog_manifest: "{{ lookup('ansible.builtin.env', 'DATADOG_MANIFEST_PATH') }}" + molecule_dynatrace_dynakube: "{{ lookup('ansible.builtin.env', 'DYNATRACE_DYNAKUBE_PATH') }}" + + # The vendor scenarios require real SaaS credentials and manifests. Skip the + # whole run cleanly when they are not supplied so the scenario is safe in CI. + - name: Skip when no vendor credentials are provided + ansible.builtin.meta: end_play + when: + - molecule_datadog_manifest | ansible.builtin.length == 0 + - molecule_dynatrace_dynakube | ansible.builtin.length == 0 + + - name: Import cluster role to set platform and provider variables + ansible.builtin.import_role: + name: cluster + vars: + cluster_files: + kubeconfig: "{{ cluster.kubeconfig }}" + + - name: Import tools role + ansible.builtin.import_role: + name: tools + vars: + tools_cluster: + kubeconfig: "{{ cluster.kubeconfig }}" + platform: "{{ cluster_platform }}" + provider: "{{ cluster_provider }}" + tools_configuration: + sre: + enabled: false + finops: + enabled: false + vendors: + datadog: >- + {{ + { + 'enabled': molecule_datadog_manifest | ansible.builtin.length > 0, + 'manifest_path': molecule_datadog_manifest, + 'secret_name': 'datadog-secret', + 'api_key_name': 'api-key', + 'app_key_name': 'app-key' + } + }} + dynatrace: >- + {{ + { + 'enabled': molecule_dynatrace_dynakube | ansible.builtin.length > 0, + 'dynakube_path': molecule_dynatrace_dynakube + } + }} diff --git a/scenarios/sre/project/roles/tools/molecule/deployment_vendors/destroy.yml b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/destroy.yml new file mode 100644 index 000000000..bea2e00bc --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/destroy.yml @@ -0,0 +1,36 @@ +--- +- name: Uninstall observability vendor agents only + hosts: + - localhost + tasks: + - name: Import cluster role to set platform and provider variables + ansible.builtin.import_role: + name: cluster + vars: + cluster_files: + kubeconfig: "{{ cluster.kubeconfig }}" + + # Invoke ONLY the vendor uninstall task files so this scenario never removes + # unrelated tools (Prometheus, Istio, cert-manager, etc.). Vendor uninstall is + # idempotent and credential-independent, so it safely no-ops when a vendor was + # never installed. + - name: Set tools role variables for vendor uninstall + ansible.builtin.set_fact: + tools_cluster: + kubeconfig: "{{ cluster.kubeconfig }}" + platform: "{{ cluster_platform }}" + provider: "{{ cluster_provider }}" + tools_configuration: + vendors: + dynatrace: "{{ observability_vendors.dynatrace | ansible.builtin.default({}) }}" + datadog: "{{ observability_vendors.datadog | ansible.builtin.default({}) }}" + + - name: Uninstall Dynatrace vendor agent + ansible.builtin.include_role: + name: tools + tasks_from: uninstall_dynatrace.yaml + + - name: Uninstall Datadog vendor agent + ansible.builtin.include_role: + name: tools + tasks_from: uninstall_datadog.yaml diff --git a/scenarios/sre/project/roles/tools/molecule/deployment_vendors/molecule.yml b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/molecule.yml new file mode 100644 index 000000000..936643347 --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/molecule.yml @@ -0,0 +1,27 @@ +--- +dependency: + name: galaxy + +ansible: + env: + ANSIBLE_ROLES_PATH: ../../.. + executor: + backend: ansible-playbook + args: + ansible_playbook: + - --inventory=${MOLECULE_PROJECT_DIRECTORY}/../../../inventory + - --tags=install_tools,install_vendors,untagged + playbooks: + converge: converge.yml + create: create.yml + destroy: destroy.yml + verify: verify.yml + +scenario: + name: deployment_vendors + test_sequence: + - syntax + - create + - converge + - verify + - destroy diff --git a/scenarios/sre/project/roles/tools/molecule/deployment_vendors/verify.yml b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/verify.yml new file mode 100644 index 000000000..7bf519705 --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/verify.yml @@ -0,0 +1,102 @@ +--- +- name: Verify observability vendor agents were installed + hosts: + - localhost + tasks: + - name: Resolve vendor manifest paths from environment + ansible.builtin.set_fact: + molecule_datadog_manifest: "{{ lookup('ansible.builtin.env', 'DATADOG_MANIFEST_PATH') }}" + molecule_dynatrace_dynakube: "{{ lookup('ansible.builtin.env', 'DYNATRACE_DYNAKUBE_PATH') }}" + + - name: Skip verification when no vendor credentials were provided + ansible.builtin.meta: end_play + when: + - molecule_datadog_manifest | ansible.builtin.length == 0 + - molecule_dynatrace_dynakube | ansible.builtin.length == 0 + + - name: Include vendor registry variables + ansible.builtin.include_vars: + file: |- + {{ + [ + playbook_dir, "..", "..", "..", "tools", "defaults", "main", "vendors.yaml" + ] | ansible.builtin.path_join + }} + + - name: Verify Datadog + when: + - molecule_datadog_manifest | ansible.builtin.length > 0 + block: + - name: Retrieve Datadog namespace + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + name: "{{ tools_vendors.datadog.kubernetes.namespace }}" + kubeconfig: "{{ cluster.kubeconfig }}" + register: molecule_datadog_ns + + - name: Validate Datadog namespace exists and is ITBench-owned + ansible.builtin.assert: + that: + - molecule_datadog_ns.resources | ansible.builtin.length == 1 + - >- + molecule_datadog_ns.resources[0].metadata.labels['app.kubernetes.io/managed-by'] | ansible.builtin.default('') + == 'ITBench' + fail_msg: Datadog namespace missing or not ITBench-owned. + success_msg: Datadog namespace present and ITBench-owned. + + - name: Retrieve ITBench-owned DatadogAgent resources + kubernetes.core.k8s_info: + api_version: datadoghq.com/v2alpha1 + kind: DatadogAgent + namespace: "{{ tools_vendors.datadog.kubernetes.namespace }}" + label_selectors: + - itbench.io/observability-vendor=datadog + kubeconfig: "{{ cluster.kubeconfig }}" + register: molecule_datadog_agents + + - name: Validate a DatadogAgent resource is present + ansible.builtin.assert: + that: + - molecule_datadog_agents.resources | ansible.builtin.length == 1 + fail_msg: DatadogAgent resource not found. Installation failed. + success_msg: DatadogAgent resource found. + + - name: Verify Dynatrace + when: + - molecule_dynatrace_dynakube | ansible.builtin.length > 0 + block: + - name: Retrieve Dynatrace namespace + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + name: "{{ tools_vendors.dynatrace.kubernetes.namespace }}" + kubeconfig: "{{ cluster.kubeconfig }}" + register: molecule_dynatrace_ns + + - name: Validate Dynatrace namespace exists and is ITBench-owned + ansible.builtin.assert: + that: + - molecule_dynatrace_ns.resources | ansible.builtin.length == 1 + - >- + molecule_dynatrace_ns.resources[0].metadata.labels['app.kubernetes.io/managed-by'] | ansible.builtin.default('') + == 'ITBench' + fail_msg: Dynatrace namespace missing or not ITBench-owned. + success_msg: Dynatrace namespace present and ITBench-owned. + + - name: Retrieve ITBench-owned DynaKube resources + kubernetes.core.k8s_info: + api_version: dynatrace.com/v1beta3 + kind: DynaKube + namespace: "{{ tools_vendors.dynatrace.kubernetes.namespace }}" + label_selectors: + - itbench.io/observability-vendor=dynatrace + kubeconfig: "{{ cluster.kubeconfig }}" + register: molecule_dynatrace_dynakubes + + - name: Validate a DynaKube resource is present + ansible.builtin.assert: + that: + - molecule_dynatrace_dynakubes.resources | ansible.builtin.length == 1 + fail_msg: DynaKube resource not found. Installation failed. + success_msg: DynaKube resource found. diff --git a/scenarios/sre/project/roles/tools/molecule/vendor_contract/converge.yml b/scenarios/sre/project/roles/tools/molecule/vendor_contract/converge.yml new file mode 100644 index 000000000..42556fa46 --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/vendor_contract/converge.yml @@ -0,0 +1,120 @@ +--- +# Credential-free, cluster-free contract tests for vendor manifest validation. +# Exercises the extracted validation task against fixture manifests and asserts +# that valid input passes and invalid input is rejected. +- name: Vendor manifest validation contract tests + hosts: + - localhost + gather_facts: false + vars: + tools_datadog_namespace: datadog + tools_datadog_secret_name: datadog-secret + tools_datadog_api_key_name: api-key + tools_datadog_app_key_name: app-key + validate_task: >- + {{ + [playbook_dir, "..", "..", "tasks", "validate_datadog_manifest.yaml"] + | ansible.builtin.path_join + }} + tasks: + - name: Valid manifest passes validation + block: + - name: Parse valid fixture + ansible.builtin.set_fact: + tools_datadog_documents: "{{ lookup('ansible.builtin.file', 'files/datadog_valid.yaml') | ansible.builtin.from_yaml_all | list }}" + + - name: Validate valid fixture + ansible.builtin.include_tasks: "{{ validate_task }}" + + - name: Assert app-key requirement detected + ansible.builtin.assert: + that: + - tools_datadog_requires_app_key + fail_msg: Valid fixture references appSecret but requirement not detected. + success_msg: Valid manifest accepted and app-key requirement detected. + + - name: Manifest containing a Secret document is rejected + block: + - name: Parse secret-bearing fixture + ansible.builtin.set_fact: + tools_datadog_documents: "{{ lookup('ansible.builtin.file', 'files/datadog_with_secret.yaml') | ansible.builtin.from_yaml_all | list }}" + + - name: Validate secret-bearing fixture (should fail) + ansible.builtin.include_tasks: "{{ validate_task }}" + register: contract_secret_result + + - name: Fail because validation did not reject the Secret document + ansible.builtin.fail: + msg: Manifest containing a Secret document was NOT rejected. + rescue: + - name: Confirm secret-bearing manifest was rejected + ansible.builtin.debug: + msg: Manifest containing a Secret document was correctly rejected. + + - name: Manifest with mismatched Secret references is rejected + block: + - name: Parse mismatched fixture + ansible.builtin.set_fact: + tools_datadog_documents: "{{ lookup('ansible.builtin.file', 'files/datadog_mismatched_secret.yaml') | ansible.builtin.from_yaml_all | list }}" + + - name: Validate mismatched fixture (should fail) + ansible.builtin.include_tasks: "{{ validate_task }}" + + - name: Fail because validation did not reject the mismatch + ansible.builtin.fail: + msg: Manifest with mismatched Secret references was NOT rejected. + rescue: + - name: Confirm mismatched manifest was rejected + ansible.builtin.debug: + msg: Manifest with mismatched Secret references was correctly rejected. + + - name: Non-secret vendor projection excludes secret keys (allowlist) + vars: + awx_configuration: + vendors: + datadog: + enabled: true + manifest_path: p + secret_name: s + api_key_name: a + app_key_name: b + manifest: SECRET + api_key: SECRET + app_key: SECRET + token: FUTURE_SECRET + awx_vendor_nonsecret_allowlist: + - enabled + - manifest_path + - secret_name + - api_key_name + - app_key_name + - dynakube_path + - dynakube_api_version + block: + - name: Build projection + ansible.builtin.set_fact: + projected: >- + {{ + dict( + awx_configuration.vendors.keys() + | zip( + awx_configuration.vendors.keys() + | map('extract', awx_configuration.vendors) + | map('dict2items') + | map('selectattr', 'key', 'in', awx_vendor_nonsecret_allowlist) + | map('list') | map('items2dict') + ) + ) + }} + + - name: Assert secrets are excluded from projection + ansible.builtin.assert: + that: + - "'manifest' not in projected.datadog" + - "'api_key' not in projected.datadog" + - "'app_key' not in projected.datadog" + - "'token' not in projected.datadog" + - projected.datadog.enabled + - projected.datadog.manifest_path == 'p' + fail_msg: Projection leaked a secret field or dropped a non-secret field. + success_msg: Non-secret projection correctly excludes all secret fields. diff --git a/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_mismatched_secret.yaml b/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_mismatched_secret.yaml new file mode 100644 index 000000000..0c2ffadd9 --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_mismatched_secret.yaml @@ -0,0 +1,13 @@ +--- +# Invalid: apiSecret reference does not match the ITBench-created Secret name/key. +kind: "DatadogAgent" +apiVersion: "datadoghq.com/v2alpha1" +metadata: + name: "datadog" + namespace: "datadog" +spec: + global: + credentials: + apiSecret: + secretName: "some-other-secret" + keyName: "wrong-key" diff --git a/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_valid.yaml b/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_valid.yaml new file mode 100644 index 000000000..076531843 --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_valid.yaml @@ -0,0 +1,20 @@ +--- +kind: "DatadogAgent" +apiVersion: "datadoghq.com/v2alpha1" +metadata: + name: "datadog" + namespace: "datadog" +spec: + global: + clusterName: "itbench" + site: "us5.datadoghq.com" + credentials: + apiSecret: + secretName: "datadog-secret" + keyName: "api-key" + appSecret: + secretName: "datadog-secret" + keyName: "app-key" + features: + logCollection: + enabled: true diff --git a/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_with_secret.yaml b/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_with_secret.yaml new file mode 100644 index 000000000..741424f57 --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_with_secret.yaml @@ -0,0 +1,21 @@ +--- +# Invalid: includes a Secret document (must be rejected - ITBench creates it). +kind: "DatadogAgent" +apiVersion: "datadoghq.com/v2alpha1" +metadata: + name: "datadog" + namespace: "datadog" +spec: + global: + credentials: + apiSecret: + secretName: "datadog-secret" + keyName: "api-key" +--- +apiVersion: v1 +kind: Secret +metadata: + name: "datadog-secret" + namespace: "datadog" +stringData: + api-key: "should-not-be-here" # pragma: allowlist secret diff --git a/scenarios/sre/project/roles/tools/molecule/vendor_contract/molecule.yml b/scenarios/sre/project/roles/tools/molecule/vendor_contract/molecule.yml new file mode 100644 index 000000000..58e1ac6f0 --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/vendor_contract/molecule.yml @@ -0,0 +1,14 @@ +--- +dependency: + name: galaxy + +ansible: + env: + ANSIBLE_ROLES_PATH: ../../.. + +scenario: + name: vendor_contract + # Credential-free and cluster-free: only runs the converge contract tests. + test_sequence: + - syntax + - converge diff --git a/scenarios/sre/project/roles/tools/tasks/assert_vendor_resource_ownership.yaml b/scenarios/sre/project/roles/tools/tasks/assert_vendor_resource_ownership.yaml new file mode 100644 index 000000000..9045ae891 --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/assert_vendor_resource_ownership.yaml @@ -0,0 +1,63 @@ +--- +# Reject name-collisions before a server-side apply claims foreign resources. +# +# Inputs: +# vendor_ownership_vendor - expected itbench.io/observability-vendor label +# vendor_ownership_namespace - namespace the documents target +# vendor_ownership_documents - list of resource documents to be applied +# +# For each document, query the live object by apiVersion/kind/name/namespace. If +# it already exists, it must already carry the matching ITBench ownership label; +# otherwise fail rather than relabel a foreign object. +- name: Reset the collision-check summary for {{ vendor_ownership_vendor }} + ansible.builtin.set_fact: + vendor_ownership_summary: [] + +- name: Query live objects for collision detection ({{ vendor_ownership_vendor }}) + kubernetes.core.k8s_info: + api_version: "{{ vendor_ownership_document.apiVersion }}" + kind: "{{ vendor_ownership_document.kind }}" + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: "{{ vendor_ownership_document.metadata.name }}" + namespace: "{{ vendor_ownership_document.metadata.namespace | ansible.builtin.default(vendor_ownership_namespace) }}" + loop: "{{ vendor_ownership_documents }}" + loop_control: + loop_var: vendor_ownership_document + label: "{{ vendor_ownership_document.kind }}/{{ vendor_ownership_document.metadata.name }}" + register: vendor_ownership_live + no_log: true + +# Build a plain list of {kind, name, owned} triples so the assertion does not +# depend on the (no_log-censored) registered loop `.item` field. +- name: Summarize collision-check results for {{ vendor_ownership_vendor }} + ansible.builtin.set_fact: + vendor_ownership_summary: >- + {{ + vendor_ownership_summary | ansible.builtin.default([]) + [{ + 'kind': vendor_ownership_pair.0.kind, + 'name': vendor_ownership_pair.0.metadata.name, + 'exists': (vendor_ownership_pair.1.resources | ansible.builtin.length > 0), + 'owned': ( + vendor_ownership_pair.1.resources | ansible.builtin.length > 0 + and (vendor_ownership_pair.1.resources[0].metadata.labels['itbench.io/observability-vendor'] | ansible.builtin.default('')) == vendor_ownership_vendor + ) + }] + }} + loop: "{{ vendor_ownership_documents | zip(vendor_ownership_live.results) | list }}" + loop_control: + loop_var: vendor_ownership_pair + label: "{{ vendor_ownership_pair.0.kind }}/{{ vendor_ownership_pair.0.metadata.name }}" + +- name: Assert no unowned {{ vendor_ownership_vendor }} resource collisions + ansible.builtin.assert: + that: + - vendor_ownership_conflicts | ansible.builtin.length == 0 + fail_msg: >- + These resources already exist and are not owned by ITBench for the + {{ vendor_ownership_vendor }} vendor: + {{ vendor_ownership_conflicts | map(attribute='kind') | zip(vendor_ownership_conflicts | map(attribute='name')) | map('join', '/') | join(', ') }}. + Refusing to claim them. Remove the conflicting resources or use different names. + success_msg: No unowned {{ vendor_ownership_vendor }} resource collisions detected. + vars: + vendor_ownership_conflicts: >- + {{ vendor_ownership_summary | ansible.builtin.default([]) | selectattr('exists') | rejectattr('owned') | list }} diff --git a/scenarios/sre/project/roles/tools/tasks/delete_vendor_namespace_if_safe.yaml b/scenarios/sre/project/roles/tools/tasks/delete_vendor_namespace_if_safe.yaml new file mode 100644 index 000000000..16918727e --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/delete_vendor_namespace_if_safe.yaml @@ -0,0 +1,84 @@ +--- +# Delete a vendor namespace ONLY when it is exclusively ITBench-owned and drained +# of meaningful resources. Preserves namespaces by default. +# +# Inputs: +# vendor_delete_namespace - namespace to consider for deletion +# vendor_delete_vendor - expected itbench.io/observability-vendor label +- name: Re-read the {{ vendor_delete_vendor }} namespace ownership + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: "{{ vendor_delete_namespace }}" + register: vendor_delete_ns_owner + +- name: Inventory remaining resources in the {{ vendor_delete_vendor }} namespace + kubernetes.core.k8s_info: + api_version: "{{ vendor_delete_kind.api_version }}" + kind: "{{ vendor_delete_kind.kind }}" + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ vendor_delete_namespace }}" + loop: + - {api_version: v1, kind: Pod} + - {api_version: v1, kind: PersistentVolumeClaim} + - {api_version: v1, kind: Service} + - {api_version: v1, kind: ConfigMap} + - {api_version: v1, kind: Secret} + - {api_version: apps/v1, kind: Deployment} + - {api_version: apps/v1, kind: DaemonSet} + - {api_version: apps/v1, kind: StatefulSet} + - {api_version: batch/v1, kind: Job} + loop_control: + loop_var: vendor_delete_kind + label: "{{ vendor_delete_kind.kind }}" + register: vendor_delete_inventory + when: + - vendor_delete_ns_owner.resources | ansible.builtin.length == 1 + +# Consider only resources NOT owned by ITBench for this vendor. Default k8s +# objects such as the `kube-root-ca.crt` ConfigMap and the default ServiceAccount +# token are ignored implicitly because we only block on foreign, non-default +# workloads/data below. +- name: Compute foreign (non-ITBench) resources remaining in the namespace + ansible.builtin.set_fact: + vendor_delete_foreign_resources: >- + {{ + vendor_delete_inventory.results + | map(attribute='resources') | sum(start=[]) + | rejectattr('metadata.labels.itbench.io/observability-vendor', 'defined') + | rejectattr('metadata.name', 'match', '^(kube-root-ca\\.crt|default-token.*|default)$') + | list + }} + when: + - vendor_delete_ns_owner.resources | ansible.builtin.length == 1 + +- name: Delete the {{ vendor_delete_vendor }} namespace when ITBench-owned and empty + kubernetes.core.k8s: + api_version: v1 + kind: Namespace + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: "{{ vendor_delete_namespace }}" + state: absent + wait: true + when: + - vendor_delete_ns_owner.resources | ansible.builtin.length == 1 + - >- + vendor_delete_ns_owner.resources[0].metadata.labels['app.kubernetes.io/managed-by'] | ansible.builtin.default('') + == 'ITBench' + - >- + vendor_delete_ns_owner.resources[0].metadata.labels['itbench.io/observability-vendor'] | ansible.builtin.default('') + == vendor_delete_vendor + - vendor_delete_foreign_resources | ansible.builtin.length == 0 + +- name: Report that the {{ vendor_delete_vendor }} namespace was preserved + ansible.builtin.debug: + msg: >- + Namespace '{{ vendor_delete_namespace }}' was preserved because it is not + exclusively ITBench-owned or still contains non-ITBench resources. + when: + - vendor_delete_ns_owner.resources | ansible.builtin.length == 1 + - >- + (vendor_delete_ns_owner.resources[0].metadata.labels['app.kubernetes.io/managed-by'] | ansible.builtin.default('') != 'ITBench') + or (vendor_delete_ns_owner.resources[0].metadata.labels['itbench.io/observability-vendor'] | ansible.builtin.default('') != vendor_delete_vendor) + or (vendor_delete_foreign_resources | ansible.builtin.default([]) | ansible.builtin.length > 0) diff --git a/scenarios/sre/project/roles/tools/tasks/ensure_vendor_namespace.yaml b/scenarios/sre/project/roles/tools/tasks/ensure_vendor_namespace.yaml new file mode 100644 index 000000000..70fdf7b30 --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/ensure_vendor_namespace.yaml @@ -0,0 +1,52 @@ +--- +# Safely ensure an ITBench-owned namespace exists. +# +# Inputs: +# vendor_namespace_name - namespace to ensure +# vendor_namespace_vendor - value for the itbench.io/observability-vendor label +# +# Behaviour: +# - If the namespace does not exist, create it with ITBench ownership labels. +# - If it exists and is already ITBench-owned for this vendor, leave it. +# - If it exists but is NOT ITBench-owned for this vendor, fail (never claim a +# pre-existing/foreign namespace). +- name: Look up the {{ vendor_namespace_vendor }} namespace + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: "{{ vendor_namespace_name }}" + register: vendor_namespace_existing + +- name: Assert an existing {{ vendor_namespace_vendor }} namespace is ITBench-owned + ansible.builtin.assert: + that: + - >- + vendor_namespace_existing.resources[0].metadata.labels['app.kubernetes.io/managed-by'] | ansible.builtin.default('') + == 'ITBench' + - >- + vendor_namespace_existing.resources[0].metadata.labels['itbench.io/observability-vendor'] | ansible.builtin.default('') + == vendor_namespace_vendor + fail_msg: >- + Namespace '{{ vendor_namespace_name }}' already exists but is not owned by + ITBench for the {{ vendor_namespace_vendor }} vendor. Refusing to claim it. + Remove or rename the existing namespace, or point the vendor at a different + namespace. + success_msg: Existing {{ vendor_namespace_vendor }} namespace is ITBench-owned. + when: + - vendor_namespace_existing.resources | ansible.builtin.length == 1 + +- name: Create the {{ vendor_namespace_vendor }} namespace when absent + kubernetes.core.k8s: + kubeconfig: "{{ tools_cluster.kubeconfig }}" + state: present + resource_definition: + apiVersion: v1 + kind: Namespace + metadata: + name: "{{ vendor_namespace_name }}" + labels: + app.kubernetes.io/managed-by: ITBench + itbench.io/observability-vendor: "{{ vendor_namespace_vendor }}" + when: + - vendor_namespace_existing.resources | ansible.builtin.length == 0 diff --git a/scenarios/sre/project/roles/tools/tasks/install.yaml b/scenarios/sre/project/roles/tools/tasks/install.yaml index f0a5a23a3..55b9472a1 100644 --- a/scenarios/sre/project/roles/tools/tasks/install.yaml +++ b/scenarios/sre/project/roles/tools/tasks/install.yaml @@ -120,3 +120,20 @@ value: "0" when: - tools_cluster.platform == "openshift" + +- name: Install third-party observability vendor agents + tags: + - install_tools + - install_vendors + block: + - name: Import Datadog installation tasks + ansible.builtin.import_tasks: + file: install_datadog.yaml + when: + - tools_configuration.vendors.datadog.enabled | ansible.builtin.default(false) + + - name: Import Dynatrace installation tasks + ansible.builtin.import_tasks: + file: install_dynatrace.yaml + when: + - tools_configuration.vendors.dynatrace.enabled | ansible.builtin.default(false) diff --git a/scenarios/sre/project/roles/tools/tasks/install_datadog.yaml b/scenarios/sre/project/roles/tools/tasks/install_datadog.yaml new file mode 100644 index 000000000..4e00f680a --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/install_datadog.yaml @@ -0,0 +1,190 @@ +--- +# Datadog agent deployment (Layer A). +# +# Consumes the user's datadog-agent.yaml (a single DatadogAgent CR) and creates +# the api/app-key Secret separately via set_credentials_datadog.yaml. All +# configuration is read from tools_configuration.vendors.datadog.* (the single +# role input interface). +- name: Set Datadog configuration facts + ansible.builtin.set_fact: + tools_datadog_config: "{{ tools_configuration.vendors.datadog }}" + tools_datadog_namespace: "{{ tools_vendors.datadog.kubernetes.namespace }}" + tools_datadog_secret_name: "{{ tools_configuration.vendors.datadog.secret_name | ansible.builtin.default('datadog-secret') }}" + tools_datadog_api_key_name: "{{ tools_configuration.vendors.datadog.api_key_name | ansible.builtin.default('api-key') }}" + tools_datadog_app_key_name: "{{ tools_configuration.vendors.datadog.app_key_name | ansible.builtin.default('app-key') }}" + +# OpenShift requires dedicated SecurityContextConstraints bound to the agent +# service accounts, which is not yet implemented. Blocked (no override) until the +# prerequisites exist. +- name: Assert Datadog is supported on this platform + ansible.builtin.assert: + that: + - tools_cluster.platform != 'openshift' + fail_msg: >- + Datadog vendor deployment on OpenShift is not supported yet: it requires + SecurityContextConstraints bound to the agent service accounts, which is + not implemented. Deploy on a Kubernetes cluster. + success_msg: Platform supported for Datadog deployment. + +- name: Resolve Datadog agent manifest path + ansible.builtin.set_fact: + tools_datadog_manifest_path: >- + {{ + datadog_manifest_path | + ansible.builtin.default(tools_datadog_config.manifest_path, true) + }} + +# A relative manifest_path is resolved against the SRE project root (parent of +# the playbook directory) so it works regardless of the working directory. +- name: Anchor a relative Datadog manifest path to the project root + ansible.builtin.set_fact: + tools_datadog_manifest_path: "{{ [playbook_dir, '..', tools_datadog_manifest_path] | ansible.builtin.path_join | ansible.builtin.realpath }}" + when: + - not (tools_datadog_manifest_path is ansible.builtin.abs) + +- name: Verify that the Datadog agent manifest exists + ansible.builtin.stat: + path: "{{ tools_datadog_manifest_path }}" + register: tools_datadog_manifest_stat + +- name: Assert that the Datadog agent manifest is present and not empty + ansible.builtin.assert: + that: + - tools_datadog_manifest_stat.stat.exists + - tools_datadog_manifest_stat.stat.size > 0 + fail_msg: >- + The Datadog agent manifest could not be found at + '{{ tools_datadog_manifest_path }}'. Download datadog-agent.yaml from the + Datadog onboarding UI and place it there, or set datadog_manifest_path. + success_msg: Datadog agent manifest found. + +# --- Manifest validation (S1/M1). Treat the file as untrusted/secret. --------- +# The manifest must contain EXACTLY one DatadogAgent and NOTHING else. Secrets +# are provisioned only by ITBench (set_credentials_datadog.yaml), never accepted +# from the user's file. +- name: Parse the Datadog agent manifest documents + ansible.builtin.set_fact: + tools_datadog_documents: >- + {{ + lookup('ansible.builtin.file', tools_datadog_manifest_path) + | ansible.builtin.from_yaml_all | list + }} + no_log: true + +- name: Validate the Datadog agent manifest (M1/M2) + ansible.builtin.include_tasks: validate_datadog_manifest.yaml + +# --- H1: ensure/claim the namespace safely. ----------------------------------- +- name: Ensure the Datadog namespace exists and is ITBench-owned + ansible.builtin.include_tasks: ensure_vendor_namespace.yaml + vars: + vendor_namespace_name: "{{ tools_datadog_namespace }}" + vendor_namespace_vendor: datadog + +- name: Install Datadog Operator + kubernetes.core.helm: + chart_ref: "{{ tools_vendors.datadog.helm.chart.reference }}" + chart_repo_url: "{{ tools_vendors.datadog.helm.chart.repository }}" + chart_version: "{{ tools_vendors.datadog.helm.chart.version }}" + kubeconfig: "{{ tools_cluster.kubeconfig }}" + release_name: "{{ tools_vendors.datadog.helm.release.name }}" + release_namespace: "{{ tools_datadog_namespace }}" + release_state: present + atomic: true + timeout: 10m0s + wait: true + +- name: Wait for the DatadogAgent custom resource definition to be established + kubernetes.core.k8s_info: + api_version: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: datadogagents.datadoghq.com + register: tools_datadog_crd + until: + - tools_datadog_crd.resources | ansible.builtin.length == 1 + - >- + tools_datadog_crd.resources[0].status.conditions | ansible.builtin.default([]) + | selectattr('type', 'equalto', 'Established') + | selectattr('status', 'equalto', 'True') + | list | ansible.builtin.length > 0 + delay: 15 + retries: 20 + +- name: Wait for the Datadog Operator deployment to be fully rolled out (M4) + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: Deployment + kubeconfig: "{{ tools_cluster.kubeconfig }}" + label_selectors: + - app.kubernetes.io/name=datadog-operator + namespace: "{{ tools_datadog_namespace }}" + register: tools_datadog_operator + until: + - tools_datadog_operator.resources | ansible.builtin.length > 0 + # A healthy rollout for every operator deployment: at least one ready + # replica, and no unavailable replicas. + - >- + tools_datadog_operator.resources + | map(attribute='status.unavailableReplicas', default=0) + | select('gt', 0) | list | ansible.builtin.length == 0 + - >- + tools_datadog_operator.resources + | map(attribute='status.readyReplicas', default=0) + | select('gt', 0) | list | ansible.builtin.length + == tools_datadog_operator.resources | ansible.builtin.length + delay: 15 + retries: 20 + +- name: Import Datadog credential variable setting tasks + ansible.builtin.import_tasks: + file: set_credentials_datadog.yaml + +# --- H3: reject collisions before applying the CR. ---------------------------- +- name: Build the ITBench-labeled DatadogAgent document + ansible.builtin.set_fact: + tools_datadog_labeled_documents: >- + {{ + (tools_datadog_documents | ansible.builtin.reject('none') | list) + | map('combine', {'metadata': {'namespace': tools_datadog_namespace, 'labels': {'app.kubernetes.io/managed-by': 'ITBench', 'itbench.io/observability-vendor': 'datadog'}}}, recursive=True) + | list + }} + no_log: true + +- name: Assert no unowned Datadog resource collisions + ansible.builtin.include_tasks: assert_vendor_resource_ownership.yaml + vars: + vendor_ownership_vendor: datadog + vendor_ownership_namespace: "{{ tools_datadog_namespace }}" + vendor_ownership_documents: "{{ tools_datadog_labeled_documents }}" + +- name: Apply the Datadog agent manifest (labelled ITBench-owned) + kubernetes.core.k8s: + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_datadog_namespace }}" + definition: "{{ tools_datadog_labeled_documents }}" + apply: true + state: present + wait: true + no_log: true + +- name: Wait for the DatadogAgent to report a healthy status (M4) + kubernetes.core.k8s_info: + api_version: datadoghq.com/v2alpha1 + kind: DatadogAgent + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_datadog_namespace }}" + label_selectors: + - itbench.io/observability-vendor=datadog + register: tools_datadog_agent_status + until: + - tools_datadog_agent_status.resources | ansible.builtin.length == 1 + # The Agent (node) component must report a Running/ready state via its + # documented status.conditions rather than merely having a status object. + - >- + tools_datadog_agent_status.resources[0].status.conditions | ansible.builtin.default([]) + | selectattr('type', 'in', ['Reconcile', 'Ready', 'AgentReconcileConditionType']) + | selectattr('status', 'equalto', 'True') + | list | ansible.builtin.length > 0 + delay: 20 + retries: 30 diff --git a/scenarios/sre/project/roles/tools/tasks/install_dynatrace.yaml b/scenarios/sre/project/roles/tools/tasks/install_dynatrace.yaml new file mode 100644 index 000000000..be2d87391 --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/install_dynatrace.yaml @@ -0,0 +1,216 @@ +--- +# Dynatrace agent deployment (Layer A). +# +# Consumes the user's dynakube.yaml (Secret with tokens + DynaKube CR). All +# configuration is read from tools_configuration.vendors.dynatrace.* (the single +# role input interface). The file is treated as secret throughout (no_log). +# +# The dynakube.yaml is downloaded, as-is, from the Dynatrace onboarding UI and +# placed at observability_vendors.dynatrace.dynakube_path (default: +# secrets/dynatrace/dynakube.yaml, gitignored) or injected via AWX. It is +# authoritative: ITBench applies it without rewriting its spec. Environment- +# specific adjustments are therefore made in that file, NOT here. +# +# Known environment caveat (edit your dynakube.yaml accordingly): +# * OpenShift / RHEL CoreOS nodes use SELinux, not AppArmor. If the DynaKube +# enables KSPM, its spec.kspm.mappedHostPaths often includes +# "/sys/kernel/security/apparmor". That path does not exist on RHCOS, so the +# operator-generated node-config-collector DaemonSet fails to mount it +# (hostPath type check failed: not a directory) and stays in +# ContainerCreating. Remove that single entry from mappedHostPaths on +# SELinux-based nodes; KSPM otherwise remains fully enabled. See +# docs/observability-vendors.md (Dynatrace on OpenShift) for details. +- name: Set Dynatrace configuration facts + ansible.builtin.set_fact: + tools_dynatrace_config: "{{ tools_configuration.vendors.dynatrace }}" + tools_dynatrace_namespace: "{{ tools_vendors.dynatrace.kubernetes.namespace }}" + tools_dynatrace_api_version: "{{ tools_configuration.vendors.dynatrace.dynakube_api_version | ansible.builtin.default('dynatrace.com/v1beta3') }}" + # The dynatrace-operator Helm chart natively handles OpenShift: setting + # platform=openshift makes the operator create and manage the + # SecurityContextConstraints its agents (OneAgent/ActiveGate/CSI) require. + # On plain Kubernetes the platform value is left at the chart default (""). + tools_dynatrace_helm_platform: "{{ 'openshift' if tools_cluster.platform == 'openshift' else '' }}" + +- name: Resolve Dynatrace DynaKube manifest path + ansible.builtin.set_fact: + tools_dynatrace_dynakube_path: >- + {{ + dynatrace_dynakube_path | + ansible.builtin.default(tools_dynatrace_config.dynakube_path, true) + }} + +# A relative dynakube_path is resolved against the SRE project root (the parent +# of the playbook directory), so it works regardless of the process working +# directory. Absolute paths (e.g. an AWX-injected file) are used as-is. +- name: Anchor a relative Dynatrace manifest path to the project root + ansible.builtin.set_fact: + tools_dynatrace_dynakube_path: "{{ [playbook_dir, '..', tools_dynatrace_dynakube_path] | ansible.builtin.path_join | ansible.builtin.realpath }}" + when: + - not (tools_dynatrace_dynakube_path is ansible.builtin.abs) + +- name: Verify that the Dynatrace DynaKube manifest exists + ansible.builtin.stat: + path: "{{ tools_dynatrace_dynakube_path }}" + register: tools_dynatrace_dynakube_stat + no_log: true + +- name: Assert that the Dynatrace DynaKube manifest is present and not empty + ansible.builtin.assert: + that: + - tools_dynatrace_dynakube_stat.stat.exists + - tools_dynatrace_dynakube_stat.stat.size > 0 + fail_msg: >- + The Dynatrace DynaKube manifest could not be found at + '{{ tools_dynatrace_dynakube_path }}'. Download dynakube.yaml from the + Dynatrace onboarding UI and place it there, or set dynatrace_dynakube_path. + success_msg: Dynatrace DynaKube manifest found. + +# --- Manifest validation (S1/M3). -------------------------------------------- +- name: Parse the Dynatrace DynaKube manifest documents + ansible.builtin.set_fact: + tools_dynatrace_documents: >- + {{ + lookup('ansible.builtin.file', tools_dynatrace_dynakube_path) + | ansible.builtin.from_yaml_all | list + }} + no_log: true + +- name: Validate the Dynatrace DynaKube manifest + vars: + tools_dynatrace_non_null: "{{ tools_dynatrace_documents | ansible.builtin.reject('none') | list }}" + ansible.builtin.assert: + that: + - tools_dynatrace_non_null | ansible.builtin.length > 0 + - tools_dynatrace_non_null | ansible.builtin.length <= 8 + - tools_dynatrace_documents | ansible.builtin.length == tools_dynatrace_non_null | ansible.builtin.length + # Only DynaKube + Secret kinds are permitted. + - >- + tools_dynatrace_non_null | map(attribute='kind') | unique + | difference(['DynaKube', 'Secret']) | ansible.builtin.length == 0 + # At least one DynaKube CR (the Dynatrace UI may emit several, e.g. a + # separate ActiveGate-only and a cloudNativeFullStack DynaKube). + - >- + tools_dynatrace_non_null | selectattr('kind', 'equalto', 'DynaKube') | list + | ansible.builtin.length >= 1 + # Every DynaKube must use the explicit supported API version (M3). + - >- + tools_dynatrace_non_null | selectattr('kind', 'equalto', 'DynaKube') + | map(attribute='apiVersion') | unique + | difference([tools_dynatrace_api_version]) | ansible.builtin.length == 0 + # No document may target another namespace. + - >- + tools_dynatrace_non_null + | map(attribute='metadata.namespace', default=tools_dynatrace_namespace) + | unique | difference([tools_dynatrace_namespace]) | ansible.builtin.length == 0 + fail_msg: >- + The Dynatrace DynaKube manifest must contain 1-8 non-null documents, only + DynaKube and Secret kinds, at least one DynaKube (all using apiVersion + '{{ tools_dynatrace_api_version }}'), and no resources targeting a namespace + other than '{{ tools_dynatrace_namespace }}'. + success_msg: Dynatrace DynaKube manifest passed validation. + no_log: true + +- name: Collect all DynaKube resource names for later reconciliation checks + ansible.builtin.set_fact: + tools_dynatrace_dynakube_names: >- + {{ + tools_dynatrace_documents | ansible.builtin.reject('none') | list + | selectattr('kind', 'equalto', 'DynaKube') + | map(attribute='metadata.name') | list + }} + no_log: true + +# --- H1: ensure/claim the namespace safely. ----------------------------------- +- name: Ensure the Dynatrace namespace exists and is ITBench-owned + ansible.builtin.include_tasks: ensure_vendor_namespace.yaml + vars: + vendor_namespace_name: "{{ tools_dynatrace_namespace }}" + vendor_namespace_vendor: dynatrace + +- name: Install Dynatrace Operator + kubernetes.core.helm: + chart_ref: "{{ tools_vendors.dynatrace.helm.chart.reference }}" + chart_version: "{{ tools_vendors.dynatrace.helm.chart.version }}" + kubeconfig: "{{ tools_cluster.kubeconfig }}" + release_name: "{{ tools_vendors.dynatrace.helm.release.name }}" + release_namespace: "{{ tools_dynatrace_namespace }}" + release_state: present + # Only set values that differ from chart defaults. On OpenShift, platform is + # set so the operator manages its own SCCs; on Kubernetes the block is empty. + values: >- + {{ + {'platform': tools_dynatrace_helm_platform} + if tools_dynatrace_helm_platform | ansible.builtin.length > 0 + else {} + }} + atomic: true + timeout: 10m0s + wait: true + +- name: Wait for the Dynatrace Operator webhook deployment to be rolled out (M4) + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: Deployment + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: dynatrace-webhook + namespace: "{{ tools_dynatrace_namespace }}" + register: tools_dynatrace_webhook + until: + - tools_dynatrace_webhook.resources | ansible.builtin.length == 1 + - tools_dynatrace_webhook.resources[0].status.unavailableReplicas | ansible.builtin.default(0) == 0 + - tools_dynatrace_webhook.resources[0].status.readyReplicas | ansible.builtin.default(0) > 0 + delay: 15 + retries: 20 + +# --- H3: reject collisions before applying the manifest. ---------------------- +- name: Build the ITBench-labeled Dynatrace documents + ansible.builtin.set_fact: + tools_dynatrace_labeled_documents: >- + {{ + (tools_dynatrace_documents | ansible.builtin.reject('none') | list) + | map('combine', {'metadata': {'namespace': tools_dynatrace_namespace, 'labels': {'app.kubernetes.io/managed-by': 'ITBench', 'itbench.io/observability-vendor': 'dynatrace'}}}, recursive=True) + | list + }} + no_log: true + +- name: Assert no unowned Dynatrace resource collisions + ansible.builtin.include_tasks: assert_vendor_resource_ownership.yaml + vars: + vendor_ownership_vendor: dynatrace + vendor_ownership_namespace: "{{ tools_dynatrace_namespace }}" + vendor_ownership_documents: "{{ tools_dynatrace_labeled_documents }}" + +- name: Apply the Dynatrace DynaKube manifest (labelled ITBench-owned) + kubernetes.core.k8s: + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_dynatrace_namespace }}" + definition: "{{ tools_dynatrace_labeled_documents }}" + apply: true + state: present + wait: true + no_log: true + +- name: Wait for each DynaKube resource to report a healthy status (M4) + kubernetes.core.k8s_info: + api_version: "{{ tools_dynatrace_api_version }}" + kind: DynaKube + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: "{{ tools_dynatrace_dynakube_item }}" + namespace: "{{ tools_dynatrace_namespace }}" + register: tools_dynatrace_dynakube_status + loop: "{{ tools_dynatrace_dynakube_names }}" + loop_control: + loop_var: tools_dynatrace_dynakube_item + label: "dynakube/{{ tools_dynatrace_dynakube_item }}" + until: + - tools_dynatrace_dynakube_status.resources | ansible.builtin.length == 1 + # DynaKube reports readiness via conditions; require at least one True + # condition and no False conditions (proves reconciliation succeeded). + - >- + tools_dynatrace_dynakube_status.resources[0].status.conditions | ansible.builtin.default([]) + | selectattr('status', 'equalto', 'True') | list | ansible.builtin.length > 0 + - >- + tools_dynatrace_dynakube_status.resources[0].status.conditions | ansible.builtin.default([]) + | selectattr('status', 'equalto', 'False') | list | ansible.builtin.length == 0 + delay: 20 + retries: 30 diff --git a/scenarios/sre/project/roles/tools/tasks/set_credentials_datadog.yaml b/scenarios/sre/project/roles/tools/tasks/set_credentials_datadog.yaml new file mode 100644 index 000000000..eafe3e5d4 --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/set_credentials_datadog.yaml @@ -0,0 +1,60 @@ +--- +# Creates the Datadog api/app-key Secret referenced by the DatadogAgent CR. +# Secret values are source-agnostic: an extra variable (e.g. injected by AWX) +# takes precedence, falling back to an environment variable for local runs. +- name: Read Datadog secrets from environment or extra variables + ansible.builtin.set_fact: + tools_datadog_api_key: "{{ datadog_api_key | ansible.builtin.default(lookup('ansible.builtin.env', 'DATADOG_API_KEY'), true) }}" # pragma: allowlist secret + tools_datadog_app_key: "{{ datadog_app_key | ansible.builtin.default(lookup('ansible.builtin.env', 'DATADOG_APP_KEY'), true) }}" # pragma: allowlist secret + no_log: true + +- name: Verify that the Datadog API key is provided + ansible.builtin.assert: + that: + - tools_datadog_api_key | ansible.builtin.default('') | ansible.builtin.length > 0 + fail_msg: The Datadog API key must be provided via the DATADOG_API_KEY environment variable or the datadog_api_key extra variable before installing the Datadog agent. + success_msg: Datadog API key found. + no_log: true + +- name: Verify that an application key is provided when the manifest requires it + ansible.builtin.assert: + that: + - tools_datadog_app_key | ansible.builtin.default('') | ansible.builtin.length > 0 + fail_msg: >- + The DatadogAgent manifest references an application key + (spec.global.credentials.appSecret) but no application key was provided. + Set DATADOG_APP_KEY or the datadog_app_key extra variable. + success_msg: Datadog application key found. + when: + - tools_datadog_requires_app_key | ansible.builtin.default(false) + no_log: true + +- name: Build Datadog secret data + ansible.builtin.set_fact: + tools_datadog_secret_data: >- + {{ + {(tools_datadog_config.api_key_name | ansible.builtin.default('api-key')): tools_datadog_api_key} + | ansible.builtin.combine( + {(tools_datadog_config.app_key_name | ansible.builtin.default('app-key')): tools_datadog_app_key} + if (tools_datadog_app_key | ansible.builtin.default('') | ansible.builtin.length > 0) + else {} + ) + }} + no_log: true + +- name: Create Datadog credentials secret + kubernetes.core.k8s: + kubeconfig: "{{ tools_cluster.kubeconfig }}" + resource_definition: + apiVersion: v1 + kind: Secret + metadata: + name: "{{ tools_datadog_secret_name }}" + namespace: "{{ tools_datadog_namespace }}" + labels: + app.kubernetes.io/managed-by: ITBench + itbench.io/observability-vendor: datadog + type: Opaque + stringData: "{{ tools_datadog_secret_data }}" # pragma: allowlist secret + state: present + no_log: true diff --git a/scenarios/sre/project/roles/tools/tasks/uninstall.yaml b/scenarios/sre/project/roles/tools/tasks/uninstall.yaml index 1962f8d6c..a185adde5 100644 --- a/scenarios/sre/project/roles/tools/tasks/uninstall.yaml +++ b/scenarios/sre/project/roles/tools/tasks/uninstall.yaml @@ -1,4 +1,21 @@ --- +# Vendor cleanup runs unconditionally and idempotently: the tasks no-op when the +# vendor namespace is absent, so disabling a vendor still removes its resources +# during Undeploy-Tools (rather than orphaning them). +# +# ORDERING NOTE: Dynatrace is uninstalled LAST (just before CRD removal), not +# here. Dynatrace's OneAgent injects a `csi.oneagent.dynatrace.com` volume into +# pods across other tool namespaces (e.g. Prometheus). If the Dynatrace operator +# / CSI DaemonSet is removed first, those injected pods can no longer unmount +# their CSI volume when their own namespace is later deleted, so they hang in +# Terminating and block namespace deletion (600s timeout). Tearing Dynatrace +# down after the injected workloads keeps the CSI driver available for a clean +# unmount. Datadog uses a DaemonSet agent (no injected CSI volumes) and is safe +# to remove early. +- name: Import Datadog uninstallation tasks + ansible.builtin.import_tasks: + file: uninstall_datadog.yaml + - name: Import Kubernetes Topology Monitor uninstallation tasks ansible.builtin.import_tasks: file: uninstall_kubernetes_topology_monitor.yaml @@ -45,6 +62,13 @@ ansible.builtin.import_tasks: file: uninstall_kubernetes_gateway.yaml +# Dynatrace is torn down here (after all other tools) so its OneAgent CSI driver +# remains available while Dynatrace-injected pods in other namespaces terminate. +# See the ORDERING NOTE at the top of this file. +- name: Import Dynatrace uninstallation tasks + ansible.builtin.import_tasks: + file: uninstall_dynatrace.yaml + - name: Import Custom Resource Definition removal tasks ansible.builtin.import_tasks: file: remove_custom_resource_definitions.yaml diff --git a/scenarios/sre/project/roles/tools/tasks/uninstall_datadog.yaml b/scenarios/sre/project/roles/tools/tasks/uninstall_datadog.yaml new file mode 100644 index 000000000..c6f9ade33 --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/uninstall_datadog.yaml @@ -0,0 +1,56 @@ +--- +# Idempotent, credential-independent Datadog cleanup. Safe to run even when the +# vendor was never installed (all lookups/deletes no-op if absent). Only removes +# resources ITBench created (label itbench.io/observability-vendor=datadog) and +# preserves the namespace unless it is exclusively ITBench-owned and drained. +- name: Set Datadog uninstall facts + ansible.builtin.set_fact: + tools_datadog_namespace: "{{ tools_vendors.datadog.kubernetes.namespace }}" + +- name: Check whether the Datadog namespace exists + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: "{{ tools_datadog_namespace }}" + register: tools_datadog_ns + +- name: Clean up Datadog resources + when: + - tools_datadog_ns.resources | ansible.builtin.length == 1 + block: + - name: Delete ITBench-owned DatadogAgent resources + kubernetes.core.k8s: + api_version: datadoghq.com/v2alpha1 + kind: DatadogAgent + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_datadog_namespace }}" + label_selectors: + - itbench.io/observability-vendor=datadog + state: absent + wait: true + + - name: Delete ITBench-owned Datadog credential secrets + kubernetes.core.k8s: + api_version: v1 + kind: Secret + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_datadog_namespace }}" + label_selectors: + - itbench.io/observability-vendor=datadog + state: absent + wait: true + + - name: Uninstall the Datadog Operator Helm release + kubernetes.core.helm: + kubeconfig: "{{ tools_cluster.kubeconfig }}" + release_name: "{{ tools_vendors.datadog.helm.release.name }}" + release_namespace: "{{ tools_datadog_namespace }}" + release_state: absent + wait: true + + - name: Delete the Datadog namespace only when exclusively owned and drained + ansible.builtin.include_tasks: delete_vendor_namespace_if_safe.yaml + vars: + vendor_delete_namespace: "{{ tools_datadog_namespace }}" + vendor_delete_vendor: datadog diff --git a/scenarios/sre/project/roles/tools/tasks/uninstall_dynatrace.yaml b/scenarios/sre/project/roles/tools/tasks/uninstall_dynatrace.yaml new file mode 100644 index 000000000..7e8a68228 --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/uninstall_dynatrace.yaml @@ -0,0 +1,140 @@ +--- +# Idempotent, credential-independent, CSI-aware Dynatrace cleanup. Does not +# depend on the original dynakube.yaml. Safe to run when never installed. +# +# Staged order (important for the CSI driver / OneAgent full-stack): +# 1. Delete DynaKube CRs while the operator is still running so it can run +# finalizers, unmount CSI volumes, and tear down injected workloads. Do NOT +# suppress failures here. +# 2. Confirm the DynaKube API reports the resources are gone (a stuck finalizer +# must stop the play before we remove the operator). +# 3. Uninstall the operator Helm release (this removes chart-owned CSI +# DaemonSets, webhook, etc.). +# 4. Verify chart-owned workloads disappeared, then remove the namespace only +# if exclusively ITBench-owned and drained. +- name: Set Dynatrace uninstall facts + ansible.builtin.set_fact: + tools_dynatrace_namespace: "{{ tools_vendors.dynatrace.kubernetes.namespace }}" + +# Discover the served DynaKube API version from the cluster CRD so uninstall is +# independent of any config/manifest (the CRD is the source of truth). Falls back +# to configured/default values only if the CRD is absent. +- name: Look up the DynaKube custom resource definition + kubernetes.core.k8s_info: + api_version: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: dynakubes.dynatrace.com + register: tools_dynatrace_crd + +- name: Determine the DynaKube API version to use for cleanup + ansible.builtin.set_fact: + tools_dynatrace_api_version: >- + {{ + ('dynatrace.com/' + ( + tools_dynatrace_crd.resources[0].spec.versions + | selectattr('served') | map(attribute='name') | list | last + )) + if (tools_dynatrace_crd.resources | ansible.builtin.length == 1) + else ((tools_configuration.vendors.dynatrace | ansible.builtin.default({})).dynakube_api_version | ansible.builtin.default('dynatrace.com/v1beta3')) + }} + +- name: Check whether the Dynatrace namespace exists + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: "{{ tools_dynatrace_namespace }}" + register: tools_dynatrace_ns + +- name: Clean up Dynatrace resources + when: + - tools_dynatrace_ns.resources | ansible.builtin.length == 1 + block: + # Stage 1: remove DynaKube CRs while the operator remains active. Failures + # here are NOT suppressed - a failed delete request must surface. + - name: Delete ITBench-owned DynaKube resources + kubernetes.core.k8s: + api_version: "{{ tools_dynatrace_api_version }}" + kind: DynaKube + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_dynatrace_namespace }}" + label_selectors: + - itbench.io/observability-vendor=dynatrace + state: absent + wait: true + wait_timeout: 300 + + # Stage 2: confirm the DynaKube resources are actually gone. If a finalizer + # is stuck, this fails and the play stops BEFORE the operator is removed + # (removing the operator would strand the finalizer permanently). + - name: Confirm all ITBench-owned DynaKube resources are removed + kubernetes.core.k8s_info: + api_version: "{{ tools_dynatrace_api_version }}" + kind: DynaKube + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_dynatrace_namespace }}" + label_selectors: + - itbench.io/observability-vendor=dynatrace + register: tools_dynatrace_remaining_dynakubes + until: + - tools_dynatrace_remaining_dynakubes.resources | ansible.builtin.length == 0 + delay: 15 + retries: 20 + + - name: Fail if DynaKube resources are still present (stuck finalizer) + ansible.builtin.assert: + that: + - tools_dynatrace_remaining_dynakubes.resources | ansible.builtin.length == 0 + fail_msg: >- + DynaKube resources still present after deletion (likely a stuck + finalizer). Stopping before Helm uninstall to avoid stranding the + finalizer. Investigate the DynaKube status/finalizers and retry. + success_msg: All ITBench-owned DynaKube resources removed. + + # Stage 3: now remove the operator itself (removes chart-owned workloads). + - name: Uninstall the Dynatrace Operator Helm release + kubernetes.core.helm: + kubeconfig: "{{ tools_cluster.kubeconfig }}" + release_name: "{{ tools_vendors.dynatrace.helm.release.name }}" + release_namespace: "{{ tools_dynatrace_namespace }}" + release_state: absent + wait: true + + # Verify chart-owned workloads (webhook, CSI DaemonSet, etc.) are gone after + # the Helm uninstall. + - name: Wait for Dynatrace workloads to drain after Helm uninstall + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: "{{ tools_dynatrace_workload_kind }}" + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_dynatrace_namespace }}" + loop: + - DaemonSet + - Deployment + - StatefulSet + loop_control: + loop_var: tools_dynatrace_workload_kind + register: tools_dynatrace_remaining_workloads + until: + - tools_dynatrace_remaining_workloads.resources | ansible.builtin.length == 0 + delay: 15 + retries: 20 + + - name: Delete any remaining ITBench-owned Dynatrace secrets + kubernetes.core.k8s: + api_version: v1 + kind: Secret + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_dynatrace_namespace }}" + label_selectors: + - itbench.io/observability-vendor=dynatrace + state: absent + wait: true + + # Stage 4: remove the namespace only if exclusively ITBench-owned and drained. + - name: Delete the Dynatrace namespace only when exclusively owned and drained + ansible.builtin.include_tasks: delete_vendor_namespace_if_safe.yaml + vars: + vendor_delete_namespace: "{{ tools_dynatrace_namespace }}" + vendor_delete_vendor: dynatrace diff --git a/scenarios/sre/project/roles/tools/tasks/validate_datadog_manifest.yaml b/scenarios/sre/project/roles/tools/tasks/validate_datadog_manifest.yaml new file mode 100644 index 000000000..5c447fe5f --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/validate_datadog_manifest.yaml @@ -0,0 +1,62 @@ +--- +# Validate a Datadog agent manifest. Credential-free and side-effect-free so it +# can be unit-tested. Sets tools_datadog_agent_cr and tools_datadog_requires_app_key. +# +# Inputs: +# tools_datadog_documents - parsed list of manifest documents +# tools_datadog_namespace - managed namespace +# tools_datadog_secret_name - ITBench-created Secret name +# tools_datadog_api_key_name - ITBench Secret key for the API key +# tools_datadog_app_key_name - ITBench Secret key for the app key +- name: Validate the Datadog agent manifest + vars: + tools_datadog_non_null: "{{ tools_datadog_documents | ansible.builtin.reject('none') | list }}" + ansible.builtin.assert: + that: + - tools_datadog_documents | ansible.builtin.length == 1 + - tools_datadog_non_null | ansible.builtin.length == 1 + - tools_datadog_non_null[0].kind == 'DatadogAgent' + - tools_datadog_non_null[0].apiVersion == 'datadoghq.com/v2alpha1' + - >- + tools_datadog_non_null[0].metadata.namespace | ansible.builtin.default(tools_datadog_namespace) + == tools_datadog_namespace + fail_msg: >- + The Datadog agent manifest must contain exactly one DatadogAgent + (datadoghq.com/v2alpha1) document, no additional documents (Secrets are + created by ITBench, not accepted from the manifest), and must not target a + namespace other than '{{ tools_datadog_namespace }}'. + success_msg: Datadog agent manifest passed validation. + +- name: Extract the DatadogAgent credential references + ansible.builtin.set_fact: + tools_datadog_agent_cr: "{{ (tools_datadog_documents | ansible.builtin.reject('none') | list)[0] }}" + +- name: Determine referenced credential secrets + ansible.builtin.set_fact: + tools_datadog_api_secret: "{{ tools_datadog_agent_cr.spec.global.credentials.apiSecret | ansible.builtin.default({}) }}" # pragma: allowlist secret + tools_datadog_app_secret: "{{ tools_datadog_agent_cr.spec.global.credentials.appSecret | ansible.builtin.default({}) }}" # pragma: allowlist secret + tools_datadog_requires_app_key: "{{ tools_datadog_agent_cr.spec.global.credentials.appSecret is ansible.builtin.defined }}" + +- name: Assert the DatadogAgent apiSecret reference matches the ITBench secret + ansible.builtin.assert: + that: + - tools_datadog_api_secret.secretName | ansible.builtin.default('') == tools_datadog_secret_name + - tools_datadog_api_secret.keyName | ansible.builtin.default('') == tools_datadog_api_key_name + fail_msg: >- + The DatadogAgent spec.global.credentials.apiSecret must reference + secretName='{{ tools_datadog_secret_name }}' keyName='{{ tools_datadog_api_key_name }}' + to match the Secret ITBench creates. + success_msg: DatadogAgent apiSecret reference matches the ITBench secret. + +- name: Assert the DatadogAgent appSecret reference matches the ITBench secret + ansible.builtin.assert: + that: + - tools_datadog_app_secret.secretName | ansible.builtin.default('') == tools_datadog_secret_name + - tools_datadog_app_secret.keyName | ansible.builtin.default('') == tools_datadog_app_key_name + fail_msg: >- + The DatadogAgent spec.global.credentials.appSecret must reference + secretName='{{ tools_datadog_secret_name }}' keyName='{{ tools_datadog_app_key_name }}' + to match the Secret ITBench creates. + success_msg: DatadogAgent appSecret reference matches the ITBench secret. + when: + - tools_datadog_requires_app_key diff --git a/scenarios/sre/project/run_dynatrace_recorder.yaml b/scenarios/sre/project/run_dynatrace_recorder.yaml new file mode 100644 index 000000000..1fb69bc21 --- /dev/null +++ b/scenarios/sre/project/run_dynatrace_recorder.yaml @@ -0,0 +1,39 @@ +--- +# Manual, standalone deploy of the Dynatrace Grail recorder Job. +# +# This runs ONLY the recorder task (install_dynatrace_recorder.yaml) against an +# already-running scenario. It does NOT export records, suspend cronjobs, or +# delete the data-recorders namespace — unlike the full uninstall_recorders flow. +# +# Prereqs: +# * The scenario is running and the `data-recorders` namespace exists (it is +# created when recorders are installed at scenario start). +# * observability_vendors.dynatrace.recorder is configured (enabled, platform_url, +# platform_token_path) and the token file exists. +# +# Run from the scenarios/sre directory so the relative platform_token_path +# resolves the same way it does in the normal playbooks: +# +# uv run ansible-playbook -i inventory project/run_dynatrace_recorder.yaml \ +# -e cluster_kubeconfig=$HOME/.kube/config -e cluster_platform=kubernetes +# +- name: Manually deploy the Dynatrace Grail recorder + hosts: localhost + gather_facts: false + vars: + cluster_kubeconfig: "{{ lookup('ansible.builtin.env', 'KUBECONFIG') | default('~/.kube/config', true) }}" + cluster_platform: kubernetes + tasks: + - name: Load observability vendor configuration + ansible.builtin.include_vars: + file: "{{ playbook_dir }}/../inventory/group_vars/environment/observability_vendors.yaml" + + - name: Deploy the Dynatrace recorder Job + ansible.builtin.import_role: + name: recorders + tasks_from: install_dynatrace_recorder + vars: + recorders_cluster: + kubeconfig: "{{ cluster_kubeconfig }}" + platform: "{{ cluster_platform }}" + recorders_vendors: "{{ observability_vendors }}"