From 9bb6216d7d09e79fd72b8eb2e127b543b7ea22d5 Mon Sep 17 00:00:00 2001 From: "Rohan R. Arora" Date: Tue, 21 Jul 2026 10:34:36 -0500 Subject: [PATCH 01/11] feat(sre): add Datadog observability vendor Signed-off-by: Rohan R. Arora --- .../roles/tools/tasks/install_datadog.yaml | 190 ++++++++++++++++++ .../tools/tasks/set_credentials_datadog.yaml | 60 ++++++ .../roles/tools/tasks/uninstall_datadog.yaml | 56 ++++++ .../tasks/validate_datadog_manifest.yaml | 62 ++++++ 4 files changed, 368 insertions(+) create mode 100644 scenarios/sre/project/roles/tools/tasks/install_datadog.yaml create mode 100644 scenarios/sre/project/roles/tools/tasks/set_credentials_datadog.yaml create mode 100644 scenarios/sre/project/roles/tools/tasks/uninstall_datadog.yaml create mode 100644 scenarios/sre/project/roles/tools/tasks/validate_datadog_manifest.yaml diff --git a/scenarios/sre/project/roles/tools/tasks/install_datadog.yaml b/scenarios/sre/project/roles/tools/tasks/install_datadog.yaml new file mode 100644 index 000000000..4e00f680a --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/install_datadog.yaml @@ -0,0 +1,190 @@ +--- +# Datadog agent deployment (Layer A). +# +# Consumes the user's datadog-agent.yaml (a single DatadogAgent CR) and creates +# the api/app-key Secret separately via set_credentials_datadog.yaml. All +# configuration is read from tools_configuration.vendors.datadog.* (the single +# role input interface). +- name: Set Datadog configuration facts + ansible.builtin.set_fact: + tools_datadog_config: "{{ tools_configuration.vendors.datadog }}" + tools_datadog_namespace: "{{ tools_vendors.datadog.kubernetes.namespace }}" + tools_datadog_secret_name: "{{ tools_configuration.vendors.datadog.secret_name | ansible.builtin.default('datadog-secret') }}" + tools_datadog_api_key_name: "{{ tools_configuration.vendors.datadog.api_key_name | ansible.builtin.default('api-key') }}" + tools_datadog_app_key_name: "{{ tools_configuration.vendors.datadog.app_key_name | ansible.builtin.default('app-key') }}" + +# OpenShift requires dedicated SecurityContextConstraints bound to the agent +# service accounts, which is not yet implemented. Blocked (no override) until the +# prerequisites exist. +- name: Assert Datadog is supported on this platform + ansible.builtin.assert: + that: + - tools_cluster.platform != 'openshift' + fail_msg: >- + Datadog vendor deployment on OpenShift is not supported yet: it requires + SecurityContextConstraints bound to the agent service accounts, which is + not implemented. Deploy on a Kubernetes cluster. + success_msg: Platform supported for Datadog deployment. + +- name: Resolve Datadog agent manifest path + ansible.builtin.set_fact: + tools_datadog_manifest_path: >- + {{ + datadog_manifest_path | + ansible.builtin.default(tools_datadog_config.manifest_path, true) + }} + +# A relative manifest_path is resolved against the SRE project root (parent of +# the playbook directory) so it works regardless of the working directory. +- name: Anchor a relative Datadog manifest path to the project root + ansible.builtin.set_fact: + tools_datadog_manifest_path: "{{ [playbook_dir, '..', tools_datadog_manifest_path] | ansible.builtin.path_join | ansible.builtin.realpath }}" + when: + - not (tools_datadog_manifest_path is ansible.builtin.abs) + +- name: Verify that the Datadog agent manifest exists + ansible.builtin.stat: + path: "{{ tools_datadog_manifest_path }}" + register: tools_datadog_manifest_stat + +- name: Assert that the Datadog agent manifest is present and not empty + ansible.builtin.assert: + that: + - tools_datadog_manifest_stat.stat.exists + - tools_datadog_manifest_stat.stat.size > 0 + fail_msg: >- + The Datadog agent manifest could not be found at + '{{ tools_datadog_manifest_path }}'. Download datadog-agent.yaml from the + Datadog onboarding UI and place it there, or set datadog_manifest_path. + success_msg: Datadog agent manifest found. + +# --- Manifest validation (S1/M1). Treat the file as untrusted/secret. --------- +# The manifest must contain EXACTLY one DatadogAgent and NOTHING else. Secrets +# are provisioned only by ITBench (set_credentials_datadog.yaml), never accepted +# from the user's file. +- name: Parse the Datadog agent manifest documents + ansible.builtin.set_fact: + tools_datadog_documents: >- + {{ + lookup('ansible.builtin.file', tools_datadog_manifest_path) + | ansible.builtin.from_yaml_all | list + }} + no_log: true + +- name: Validate the Datadog agent manifest (M1/M2) + ansible.builtin.include_tasks: validate_datadog_manifest.yaml + +# --- H1: ensure/claim the namespace safely. ----------------------------------- +- name: Ensure the Datadog namespace exists and is ITBench-owned + ansible.builtin.include_tasks: ensure_vendor_namespace.yaml + vars: + vendor_namespace_name: "{{ tools_datadog_namespace }}" + vendor_namespace_vendor: datadog + +- name: Install Datadog Operator + kubernetes.core.helm: + chart_ref: "{{ tools_vendors.datadog.helm.chart.reference }}" + chart_repo_url: "{{ tools_vendors.datadog.helm.chart.repository }}" + chart_version: "{{ tools_vendors.datadog.helm.chart.version }}" + kubeconfig: "{{ tools_cluster.kubeconfig }}" + release_name: "{{ tools_vendors.datadog.helm.release.name }}" + release_namespace: "{{ tools_datadog_namespace }}" + release_state: present + atomic: true + timeout: 10m0s + wait: true + +- name: Wait for the DatadogAgent custom resource definition to be established + kubernetes.core.k8s_info: + api_version: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: datadogagents.datadoghq.com + register: tools_datadog_crd + until: + - tools_datadog_crd.resources | ansible.builtin.length == 1 + - >- + tools_datadog_crd.resources[0].status.conditions | ansible.builtin.default([]) + | selectattr('type', 'equalto', 'Established') + | selectattr('status', 'equalto', 'True') + | list | ansible.builtin.length > 0 + delay: 15 + retries: 20 + +- name: Wait for the Datadog Operator deployment to be fully rolled out (M4) + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: Deployment + kubeconfig: "{{ tools_cluster.kubeconfig }}" + label_selectors: + - app.kubernetes.io/name=datadog-operator + namespace: "{{ tools_datadog_namespace }}" + register: tools_datadog_operator + until: + - tools_datadog_operator.resources | ansible.builtin.length > 0 + # A healthy rollout for every operator deployment: at least one ready + # replica, and no unavailable replicas. + - >- + tools_datadog_operator.resources + | map(attribute='status.unavailableReplicas', default=0) + | select('gt', 0) | list | ansible.builtin.length == 0 + - >- + tools_datadog_operator.resources + | map(attribute='status.readyReplicas', default=0) + | select('gt', 0) | list | ansible.builtin.length + == tools_datadog_operator.resources | ansible.builtin.length + delay: 15 + retries: 20 + +- name: Import Datadog credential variable setting tasks + ansible.builtin.import_tasks: + file: set_credentials_datadog.yaml + +# --- H3: reject collisions before applying the CR. ---------------------------- +- name: Build the ITBench-labeled DatadogAgent document + ansible.builtin.set_fact: + tools_datadog_labeled_documents: >- + {{ + (tools_datadog_documents | ansible.builtin.reject('none') | list) + | map('combine', {'metadata': {'namespace': tools_datadog_namespace, 'labels': {'app.kubernetes.io/managed-by': 'ITBench', 'itbench.io/observability-vendor': 'datadog'}}}, recursive=True) + | list + }} + no_log: true + +- name: Assert no unowned Datadog resource collisions + ansible.builtin.include_tasks: assert_vendor_resource_ownership.yaml + vars: + vendor_ownership_vendor: datadog + vendor_ownership_namespace: "{{ tools_datadog_namespace }}" + vendor_ownership_documents: "{{ tools_datadog_labeled_documents }}" + +- name: Apply the Datadog agent manifest (labelled ITBench-owned) + kubernetes.core.k8s: + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_datadog_namespace }}" + definition: "{{ tools_datadog_labeled_documents }}" + apply: true + state: present + wait: true + no_log: true + +- name: Wait for the DatadogAgent to report a healthy status (M4) + kubernetes.core.k8s_info: + api_version: datadoghq.com/v2alpha1 + kind: DatadogAgent + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_datadog_namespace }}" + label_selectors: + - itbench.io/observability-vendor=datadog + register: tools_datadog_agent_status + until: + - tools_datadog_agent_status.resources | ansible.builtin.length == 1 + # The Agent (node) component must report a Running/ready state via its + # documented status.conditions rather than merely having a status object. + - >- + tools_datadog_agent_status.resources[0].status.conditions | ansible.builtin.default([]) + | selectattr('type', 'in', ['Reconcile', 'Ready', 'AgentReconcileConditionType']) + | selectattr('status', 'equalto', 'True') + | list | ansible.builtin.length > 0 + delay: 20 + retries: 30 diff --git a/scenarios/sre/project/roles/tools/tasks/set_credentials_datadog.yaml b/scenarios/sre/project/roles/tools/tasks/set_credentials_datadog.yaml new file mode 100644 index 000000000..eafe3e5d4 --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/set_credentials_datadog.yaml @@ -0,0 +1,60 @@ +--- +# Creates the Datadog api/app-key Secret referenced by the DatadogAgent CR. +# Secret values are source-agnostic: an extra variable (e.g. injected by AWX) +# takes precedence, falling back to an environment variable for local runs. +- name: Read Datadog secrets from environment or extra variables + ansible.builtin.set_fact: + tools_datadog_api_key: "{{ datadog_api_key | ansible.builtin.default(lookup('ansible.builtin.env', 'DATADOG_API_KEY'), true) }}" # pragma: allowlist secret + tools_datadog_app_key: "{{ datadog_app_key | ansible.builtin.default(lookup('ansible.builtin.env', 'DATADOG_APP_KEY'), true) }}" # pragma: allowlist secret + no_log: true + +- name: Verify that the Datadog API key is provided + ansible.builtin.assert: + that: + - tools_datadog_api_key | ansible.builtin.default('') | ansible.builtin.length > 0 + fail_msg: The Datadog API key must be provided via the DATADOG_API_KEY environment variable or the datadog_api_key extra variable before installing the Datadog agent. + success_msg: Datadog API key found. + no_log: true + +- name: Verify that an application key is provided when the manifest requires it + ansible.builtin.assert: + that: + - tools_datadog_app_key | ansible.builtin.default('') | ansible.builtin.length > 0 + fail_msg: >- + The DatadogAgent manifest references an application key + (spec.global.credentials.appSecret) but no application key was provided. + Set DATADOG_APP_KEY or the datadog_app_key extra variable. + success_msg: Datadog application key found. + when: + - tools_datadog_requires_app_key | ansible.builtin.default(false) + no_log: true + +- name: Build Datadog secret data + ansible.builtin.set_fact: + tools_datadog_secret_data: >- + {{ + {(tools_datadog_config.api_key_name | ansible.builtin.default('api-key')): tools_datadog_api_key} + | ansible.builtin.combine( + {(tools_datadog_config.app_key_name | ansible.builtin.default('app-key')): tools_datadog_app_key} + if (tools_datadog_app_key | ansible.builtin.default('') | ansible.builtin.length > 0) + else {} + ) + }} + no_log: true + +- name: Create Datadog credentials secret + kubernetes.core.k8s: + kubeconfig: "{{ tools_cluster.kubeconfig }}" + resource_definition: + apiVersion: v1 + kind: Secret + metadata: + name: "{{ tools_datadog_secret_name }}" + namespace: "{{ tools_datadog_namespace }}" + labels: + app.kubernetes.io/managed-by: ITBench + itbench.io/observability-vendor: datadog + type: Opaque + stringData: "{{ tools_datadog_secret_data }}" # pragma: allowlist secret + state: present + no_log: true diff --git a/scenarios/sre/project/roles/tools/tasks/uninstall_datadog.yaml b/scenarios/sre/project/roles/tools/tasks/uninstall_datadog.yaml new file mode 100644 index 000000000..c6f9ade33 --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/uninstall_datadog.yaml @@ -0,0 +1,56 @@ +--- +# Idempotent, credential-independent Datadog cleanup. Safe to run even when the +# vendor was never installed (all lookups/deletes no-op if absent). Only removes +# resources ITBench created (label itbench.io/observability-vendor=datadog) and +# preserves the namespace unless it is exclusively ITBench-owned and drained. +- name: Set Datadog uninstall facts + ansible.builtin.set_fact: + tools_datadog_namespace: "{{ tools_vendors.datadog.kubernetes.namespace }}" + +- name: Check whether the Datadog namespace exists + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: "{{ tools_datadog_namespace }}" + register: tools_datadog_ns + +- name: Clean up Datadog resources + when: + - tools_datadog_ns.resources | ansible.builtin.length == 1 + block: + - name: Delete ITBench-owned DatadogAgent resources + kubernetes.core.k8s: + api_version: datadoghq.com/v2alpha1 + kind: DatadogAgent + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_datadog_namespace }}" + label_selectors: + - itbench.io/observability-vendor=datadog + state: absent + wait: true + + - name: Delete ITBench-owned Datadog credential secrets + kubernetes.core.k8s: + api_version: v1 + kind: Secret + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_datadog_namespace }}" + label_selectors: + - itbench.io/observability-vendor=datadog + state: absent + wait: true + + - name: Uninstall the Datadog Operator Helm release + kubernetes.core.helm: + kubeconfig: "{{ tools_cluster.kubeconfig }}" + release_name: "{{ tools_vendors.datadog.helm.release.name }}" + release_namespace: "{{ tools_datadog_namespace }}" + release_state: absent + wait: true + + - name: Delete the Datadog namespace only when exclusively owned and drained + ansible.builtin.include_tasks: delete_vendor_namespace_if_safe.yaml + vars: + vendor_delete_namespace: "{{ tools_datadog_namespace }}" + vendor_delete_vendor: datadog diff --git a/scenarios/sre/project/roles/tools/tasks/validate_datadog_manifest.yaml b/scenarios/sre/project/roles/tools/tasks/validate_datadog_manifest.yaml new file mode 100644 index 000000000..5c447fe5f --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/validate_datadog_manifest.yaml @@ -0,0 +1,62 @@ +--- +# Validate a Datadog agent manifest. Credential-free and side-effect-free so it +# can be unit-tested. Sets tools_datadog_agent_cr and tools_datadog_requires_app_key. +# +# Inputs: +# tools_datadog_documents - parsed list of manifest documents +# tools_datadog_namespace - managed namespace +# tools_datadog_secret_name - ITBench-created Secret name +# tools_datadog_api_key_name - ITBench Secret key for the API key +# tools_datadog_app_key_name - ITBench Secret key for the app key +- name: Validate the Datadog agent manifest + vars: + tools_datadog_non_null: "{{ tools_datadog_documents | ansible.builtin.reject('none') | list }}" + ansible.builtin.assert: + that: + - tools_datadog_documents | ansible.builtin.length == 1 + - tools_datadog_non_null | ansible.builtin.length == 1 + - tools_datadog_non_null[0].kind == 'DatadogAgent' + - tools_datadog_non_null[0].apiVersion == 'datadoghq.com/v2alpha1' + - >- + tools_datadog_non_null[0].metadata.namespace | ansible.builtin.default(tools_datadog_namespace) + == tools_datadog_namespace + fail_msg: >- + The Datadog agent manifest must contain exactly one DatadogAgent + (datadoghq.com/v2alpha1) document, no additional documents (Secrets are + created by ITBench, not accepted from the manifest), and must not target a + namespace other than '{{ tools_datadog_namespace }}'. + success_msg: Datadog agent manifest passed validation. + +- name: Extract the DatadogAgent credential references + ansible.builtin.set_fact: + tools_datadog_agent_cr: "{{ (tools_datadog_documents | ansible.builtin.reject('none') | list)[0] }}" + +- name: Determine referenced credential secrets + ansible.builtin.set_fact: + tools_datadog_api_secret: "{{ tools_datadog_agent_cr.spec.global.credentials.apiSecret | ansible.builtin.default({}) }}" # pragma: allowlist secret + tools_datadog_app_secret: "{{ tools_datadog_agent_cr.spec.global.credentials.appSecret | ansible.builtin.default({}) }}" # pragma: allowlist secret + tools_datadog_requires_app_key: "{{ tools_datadog_agent_cr.spec.global.credentials.appSecret is ansible.builtin.defined }}" + +- name: Assert the DatadogAgent apiSecret reference matches the ITBench secret + ansible.builtin.assert: + that: + - tools_datadog_api_secret.secretName | ansible.builtin.default('') == tools_datadog_secret_name + - tools_datadog_api_secret.keyName | ansible.builtin.default('') == tools_datadog_api_key_name + fail_msg: >- + The DatadogAgent spec.global.credentials.apiSecret must reference + secretName='{{ tools_datadog_secret_name }}' keyName='{{ tools_datadog_api_key_name }}' + to match the Secret ITBench creates. + success_msg: DatadogAgent apiSecret reference matches the ITBench secret. + +- name: Assert the DatadogAgent appSecret reference matches the ITBench secret + ansible.builtin.assert: + that: + - tools_datadog_app_secret.secretName | ansible.builtin.default('') == tools_datadog_secret_name + - tools_datadog_app_secret.keyName | ansible.builtin.default('') == tools_datadog_app_key_name + fail_msg: >- + The DatadogAgent spec.global.credentials.appSecret must reference + secretName='{{ tools_datadog_secret_name }}' keyName='{{ tools_datadog_app_key_name }}' + to match the Secret ITBench creates. + success_msg: DatadogAgent appSecret reference matches the ITBench secret. + when: + - tools_datadog_requires_app_key From f6c7e441900c6c03e5963a0789cb1ce600d7a9e8 Mon Sep 17 00:00:00 2001 From: "Rohan R. Arora" Date: Tue, 21 Jul 2026 10:36:38 -0500 Subject: [PATCH 02/11] feat(sre): wire observability vendors into tool install/uninstall Signed-off-by: Rohan R. Arora --- .../sre/project/roles/tools/tasks/install.yaml | 17 +++++++++++++++++ .../project/roles/tools/tasks/uninstall.yaml | 11 +++++++++++ 2 files changed, 28 insertions(+) diff --git a/scenarios/sre/project/roles/tools/tasks/install.yaml b/scenarios/sre/project/roles/tools/tasks/install.yaml index f0a5a23a3..55b9472a1 100644 --- a/scenarios/sre/project/roles/tools/tasks/install.yaml +++ b/scenarios/sre/project/roles/tools/tasks/install.yaml @@ -120,3 +120,20 @@ value: "0" when: - tools_cluster.platform == "openshift" + +- name: Install third-party observability vendor agents + tags: + - install_tools + - install_vendors + block: + - name: Import Datadog installation tasks + ansible.builtin.import_tasks: + file: install_datadog.yaml + when: + - tools_configuration.vendors.datadog.enabled | ansible.builtin.default(false) + + - name: Import Dynatrace installation tasks + ansible.builtin.import_tasks: + file: install_dynatrace.yaml + when: + - tools_configuration.vendors.dynatrace.enabled | ansible.builtin.default(false) diff --git a/scenarios/sre/project/roles/tools/tasks/uninstall.yaml b/scenarios/sre/project/roles/tools/tasks/uninstall.yaml index 1962f8d6c..1b191fcc7 100644 --- a/scenarios/sre/project/roles/tools/tasks/uninstall.yaml +++ b/scenarios/sre/project/roles/tools/tasks/uninstall.yaml @@ -1,4 +1,15 @@ --- +# Vendor cleanup runs unconditionally and idempotently: the tasks no-op when the +# vendor namespace is absent, so disabling a vendor still removes its resources +# during Undeploy-Tools (rather than orphaning them). +- name: Import Dynatrace uninstallation tasks + ansible.builtin.import_tasks: + file: uninstall_dynatrace.yaml + +- name: Import Datadog uninstallation tasks + ansible.builtin.import_tasks: + file: uninstall_datadog.yaml + - name: Import Kubernetes Topology Monitor uninstallation tasks ansible.builtin.import_tasks: file: uninstall_kubernetes_topology_monitor.yaml From 538f5b680507eb7e4e29f8ad02670f80a606e21c Mon Sep 17 00:00:00 2001 From: "Rohan R. Arora" Date: Tue, 21 Jul 2026 10:41:09 -0500 Subject: [PATCH 03/11] feat(sre): add observability vendor framework for the tools role Introduce shared scaffolding for deploying third-party SaaS observability vendors as SRE tools: - tools_vendors registry (defaults/main/vendors.yaml) and non-secret observability_vendors config (example), wired into tools_configuration.vendors via manage_tools.yaml. - Reusable ownership-safe helpers: ensure_vendor_namespace, assert_vendor_resource_ownership, delete_vendor_namespace_if_safe. - .gitignore for token-bearing onboarding artifacts; allowlist a Jinja-template false positive in .secrets.baseline. Signed-off-by: Rohan R. Arora --- .secrets.baseline | 13 ++- scenarios/sre/.gitignore | 8 ++ .../observability_vendors.yaml.example | 43 ++++++++++ scenarios/sre/project/manage_tools.yaml | 7 +- .../roles/tools/defaults/main/vendors.yaml | 31 +++++++ .../assert_vendor_resource_ownership.yaml | 63 ++++++++++++++ .../delete_vendor_namespace_if_safe.yaml | 84 +++++++++++++++++++ .../tools/tasks/ensure_vendor_namespace.yaml | 52 ++++++++++++ 8 files changed, 298 insertions(+), 3 deletions(-) create mode 100644 scenarios/sre/.gitignore create mode 100644 scenarios/sre/inventory/group_vars/environment/observability_vendors.yaml.example create mode 100644 scenarios/sre/project/roles/tools/defaults/main/vendors.yaml create mode 100644 scenarios/sre/project/roles/tools/tasks/assert_vendor_resource_ownership.yaml create mode 100644 scenarios/sre/project/roles/tools/tasks/delete_vendor_namespace_if_safe.yaml create mode 100644 scenarios/sre/project/roles/tools/tasks/ensure_vendor_namespace.yaml diff --git a/.secrets.baseline b/.secrets.baseline index 149290fe0..09d472973 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2026-07-02T23:28:03Z", + "generated_at": "2026-07-21T15:39:38Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -1872,6 +1872,15 @@ "verified_result": null } ], + "scenarios/sre/project/roles/awx/files/awx/credential_types/datadog_injectors.json": [ + { + "hashed_secret": "d2e2ab0f407e4ee3cf2ab87d61c31b25a74085e5", + "is_verified": false, + "line_number": 4, + "type": "Secret Keyword", + "verified_result": null + } + ], "scenarios/sre/project/roles/awx/tasks/configure_projects.yaml": [ { "hashed_secret": "d2e2ab0f407e4ee3cf2ab87d61c31b25a74085e5", @@ -1885,7 +1894,7 @@ { "hashed_secret": "28ed3a797da3c48c309a4ef792147f3c56cfec40", "is_verified": false, - "line_number": 13, + "line_number": 17, "type": "Secret Keyword", "verified_result": null } diff --git a/scenarios/sre/.gitignore b/scenarios/sre/.gitignore new file mode 100644 index 000000000..c58f1f702 --- /dev/null +++ b/scenarios/sre/.gitignore @@ -0,0 +1,8 @@ +# Observability vendor onboarding artifacts (token-bearing or environment-specific). +# e.g. secrets/dynatrace/dynakube.yaml, secrets/datadog/datadog-agent.yaml +secrets/ + +# AWX runner vendor configuration contains secret inputs (manifest/dynakube +# contents, API/app keys). Must be Ansible-Vault encrypted; never commit plaintext. +inventory/group_vars/runner/vendors.yaml +group_vars/runner/vendors.yaml diff --git a/scenarios/sre/inventory/group_vars/environment/observability_vendors.yaml.example b/scenarios/sre/inventory/group_vars/environment/observability_vendors.yaml.example new file mode 100644 index 000000000..4030ca454 --- /dev/null +++ b/scenarios/sre/inventory/group_vars/environment/observability_vendors.yaml.example @@ -0,0 +1,43 @@ +--- +# Third-party SaaS observability vendor configuration (Layer A: agent deployment). +# +# NON-SECRET values only (site, region, tenant/environment URL, toggles) belong +# in this file. SECRETS (API keys, license keys, tokens) MUST NOT be placed here. +# They are read from environment variables at install time. Export the relevant +# variables before running the tools playbook, for example: +# +# # Datadog +# export DATADOG_API_KEY=... # required # pragma: allowlist secret +# export DATADOG_APP_KEY=... # optional (needed for some features) +# +# # Dynatrace: no env vars. Download dynakube.yaml from the Dynatrace +# # onboarding UI (contains tokens + DynaKube CR) and reference it via +# # observability_vendors.dynatrace.dynakube_path (see below). +# +# Each vendor is disabled by default. Enable a vendor and supply its non-secret +# configuration below, then export the matching secret environment variable(s). + +observability_vendors: + datadog: + enabled: false + # Path to the datadog-agent.yaml (DatadogAgent CR) shown in the Datadog + # onboarding UI. This file is NON-SECRET (it references the api/app keys by + # Secret name) so it may be committed. It carries site, clusterName, and + # feature toggles. + manifest_path: "secrets/datadog/datadog-agent.yaml" + # Name and keys of the Kubernetes Secret created from DATADOG_API_KEY / + # DATADOG_APP_KEY. These MUST match the credentials.apiSecret/appSecret + # references inside the DatadogAgent CR above. + secret_name: datadog-secret + api_key_name: api-key + app_key_name: app-key + dynatrace: + enabled: false + # Path to the dynakube.yaml downloaded from the Dynatrace onboarding UI. + # This file is SENSITIVE: it contains a Secret (apiToken + dataIngestToken) + # AND the DynaKube CR (apiUrl, cluster name, config). Keep OUT of git. + dynakube_path: "secrets/dynatrace/dynakube.yaml" # pragma: allowlist secret + # Supported DynaKube API version. The DynaKube document in dynakube.yaml must + # use this exact apiVersion; it is also used for readiness and cleanup so all + # three stay consistent. + dynakube_api_version: "dynatrace.com/v1beta3" diff --git a/scenarios/sre/project/manage_tools.yaml b/scenarios/sre/project/manage_tools.yaml index d83d2d1f4..69548d7cf 100644 --- a/scenarios/sre/project/manage_tools.yaml +++ b/scenarios/sre/project/manage_tools.yaml @@ -32,7 +32,11 @@ - name: Create tool installation variable ansible.builtin.set_fact: - tools_configuration: "{{ scenarios_scenario.spec.tools }}" + tools_configuration: >- + {{ + scenarios_scenario.spec.tools | + ansible.builtin.combine({'vendors': observability_vendors | ansible.builtin.default({})}) + }} - name: Create role variables tags: @@ -47,6 +51,7 @@ enabled: "{{ tools.sre | ansible.builtin.default(false) }}" finops: enabled: "{{ tools.finops | ansible.builtin.default(false) }}" + vendors: "{{ observability_vendors | ansible.builtin.default({}) }}" - name: Import tools role ansible.builtin.import_role: diff --git a/scenarios/sre/project/roles/tools/defaults/main/vendors.yaml b/scenarios/sre/project/roles/tools/defaults/main/vendors.yaml new file mode 100644 index 000000000..663d5fe13 --- /dev/null +++ b/scenarios/sre/project/roles/tools/defaults/main/vendors.yaml @@ -0,0 +1,31 @@ +--- +# Registry of third-party SaaS observability vendors (Layer A: agent deployment). +# +# Unlike `tools_managers` (in-cluster backends whose endpoints/credentials are +# discovered at runtime), these vendors are SaaS: their agents run in-cluster via +# a Helm chart but ship telemetry to the vendor's cloud tenant. Non-secret tenant +# configuration (site, region, environment URL) is supplied via the +# `observability_vendors` group_vars; secrets are injected via environment +# variables at install time (see set_credentials_.yaml). +tools_vendors: + datadog: + helm: + chart: + # renovate: datasource=helm depName=datadog-operator registryUrl=https://helm.datadoghq.com + reference: datadog-operator + repository: https://helm.datadoghq.com + version: 2.24.0 + release: + name: datadog-operator + kubernetes: + namespace: datadog + dynatrace: + helm: + chart: + # renovate: datasource=docker depName=public.ecr.aws/dynatrace/dynatrace-operator + reference: oci://public.ecr.aws/dynatrace/dynatrace-operator + version: 1.9.0 + release: + name: dynatrace-operator + kubernetes: + namespace: dynatrace diff --git a/scenarios/sre/project/roles/tools/tasks/assert_vendor_resource_ownership.yaml b/scenarios/sre/project/roles/tools/tasks/assert_vendor_resource_ownership.yaml new file mode 100644 index 000000000..9045ae891 --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/assert_vendor_resource_ownership.yaml @@ -0,0 +1,63 @@ +--- +# Reject name-collisions before a server-side apply claims foreign resources. +# +# Inputs: +# vendor_ownership_vendor - expected itbench.io/observability-vendor label +# vendor_ownership_namespace - namespace the documents target +# vendor_ownership_documents - list of resource documents to be applied +# +# For each document, query the live object by apiVersion/kind/name/namespace. If +# it already exists, it must already carry the matching ITBench ownership label; +# otherwise fail rather than relabel a foreign object. +- name: Reset the collision-check summary for {{ vendor_ownership_vendor }} + ansible.builtin.set_fact: + vendor_ownership_summary: [] + +- name: Query live objects for collision detection ({{ vendor_ownership_vendor }}) + kubernetes.core.k8s_info: + api_version: "{{ vendor_ownership_document.apiVersion }}" + kind: "{{ vendor_ownership_document.kind }}" + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: "{{ vendor_ownership_document.metadata.name }}" + namespace: "{{ vendor_ownership_document.metadata.namespace | ansible.builtin.default(vendor_ownership_namespace) }}" + loop: "{{ vendor_ownership_documents }}" + loop_control: + loop_var: vendor_ownership_document + label: "{{ vendor_ownership_document.kind }}/{{ vendor_ownership_document.metadata.name }}" + register: vendor_ownership_live + no_log: true + +# Build a plain list of {kind, name, owned} triples so the assertion does not +# depend on the (no_log-censored) registered loop `.item` field. +- name: Summarize collision-check results for {{ vendor_ownership_vendor }} + ansible.builtin.set_fact: + vendor_ownership_summary: >- + {{ + vendor_ownership_summary | ansible.builtin.default([]) + [{ + 'kind': vendor_ownership_pair.0.kind, + 'name': vendor_ownership_pair.0.metadata.name, + 'exists': (vendor_ownership_pair.1.resources | ansible.builtin.length > 0), + 'owned': ( + vendor_ownership_pair.1.resources | ansible.builtin.length > 0 + and (vendor_ownership_pair.1.resources[0].metadata.labels['itbench.io/observability-vendor'] | ansible.builtin.default('')) == vendor_ownership_vendor + ) + }] + }} + loop: "{{ vendor_ownership_documents | zip(vendor_ownership_live.results) | list }}" + loop_control: + loop_var: vendor_ownership_pair + label: "{{ vendor_ownership_pair.0.kind }}/{{ vendor_ownership_pair.0.metadata.name }}" + +- name: Assert no unowned {{ vendor_ownership_vendor }} resource collisions + ansible.builtin.assert: + that: + - vendor_ownership_conflicts | ansible.builtin.length == 0 + fail_msg: >- + These resources already exist and are not owned by ITBench for the + {{ vendor_ownership_vendor }} vendor: + {{ vendor_ownership_conflicts | map(attribute='kind') | zip(vendor_ownership_conflicts | map(attribute='name')) | map('join', '/') | join(', ') }}. + Refusing to claim them. Remove the conflicting resources or use different names. + success_msg: No unowned {{ vendor_ownership_vendor }} resource collisions detected. + vars: + vendor_ownership_conflicts: >- + {{ vendor_ownership_summary | ansible.builtin.default([]) | selectattr('exists') | rejectattr('owned') | list }} diff --git a/scenarios/sre/project/roles/tools/tasks/delete_vendor_namespace_if_safe.yaml b/scenarios/sre/project/roles/tools/tasks/delete_vendor_namespace_if_safe.yaml new file mode 100644 index 000000000..16918727e --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/delete_vendor_namespace_if_safe.yaml @@ -0,0 +1,84 @@ +--- +# Delete a vendor namespace ONLY when it is exclusively ITBench-owned and drained +# of meaningful resources. Preserves namespaces by default. +# +# Inputs: +# vendor_delete_namespace - namespace to consider for deletion +# vendor_delete_vendor - expected itbench.io/observability-vendor label +- name: Re-read the {{ vendor_delete_vendor }} namespace ownership + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: "{{ vendor_delete_namespace }}" + register: vendor_delete_ns_owner + +- name: Inventory remaining resources in the {{ vendor_delete_vendor }} namespace + kubernetes.core.k8s_info: + api_version: "{{ vendor_delete_kind.api_version }}" + kind: "{{ vendor_delete_kind.kind }}" + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ vendor_delete_namespace }}" + loop: + - {api_version: v1, kind: Pod} + - {api_version: v1, kind: PersistentVolumeClaim} + - {api_version: v1, kind: Service} + - {api_version: v1, kind: ConfigMap} + - {api_version: v1, kind: Secret} + - {api_version: apps/v1, kind: Deployment} + - {api_version: apps/v1, kind: DaemonSet} + - {api_version: apps/v1, kind: StatefulSet} + - {api_version: batch/v1, kind: Job} + loop_control: + loop_var: vendor_delete_kind + label: "{{ vendor_delete_kind.kind }}" + register: vendor_delete_inventory + when: + - vendor_delete_ns_owner.resources | ansible.builtin.length == 1 + +# Consider only resources NOT owned by ITBench for this vendor. Default k8s +# objects such as the `kube-root-ca.crt` ConfigMap and the default ServiceAccount +# token are ignored implicitly because we only block on foreign, non-default +# workloads/data below. +- name: Compute foreign (non-ITBench) resources remaining in the namespace + ansible.builtin.set_fact: + vendor_delete_foreign_resources: >- + {{ + vendor_delete_inventory.results + | map(attribute='resources') | sum(start=[]) + | rejectattr('metadata.labels.itbench.io/observability-vendor', 'defined') + | rejectattr('metadata.name', 'match', '^(kube-root-ca\\.crt|default-token.*|default)$') + | list + }} + when: + - vendor_delete_ns_owner.resources | ansible.builtin.length == 1 + +- name: Delete the {{ vendor_delete_vendor }} namespace when ITBench-owned and empty + kubernetes.core.k8s: + api_version: v1 + kind: Namespace + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: "{{ vendor_delete_namespace }}" + state: absent + wait: true + when: + - vendor_delete_ns_owner.resources | ansible.builtin.length == 1 + - >- + vendor_delete_ns_owner.resources[0].metadata.labels['app.kubernetes.io/managed-by'] | ansible.builtin.default('') + == 'ITBench' + - >- + vendor_delete_ns_owner.resources[0].metadata.labels['itbench.io/observability-vendor'] | ansible.builtin.default('') + == vendor_delete_vendor + - vendor_delete_foreign_resources | ansible.builtin.length == 0 + +- name: Report that the {{ vendor_delete_vendor }} namespace was preserved + ansible.builtin.debug: + msg: >- + Namespace '{{ vendor_delete_namespace }}' was preserved because it is not + exclusively ITBench-owned or still contains non-ITBench resources. + when: + - vendor_delete_ns_owner.resources | ansible.builtin.length == 1 + - >- + (vendor_delete_ns_owner.resources[0].metadata.labels['app.kubernetes.io/managed-by'] | ansible.builtin.default('') != 'ITBench') + or (vendor_delete_ns_owner.resources[0].metadata.labels['itbench.io/observability-vendor'] | ansible.builtin.default('') != vendor_delete_vendor) + or (vendor_delete_foreign_resources | ansible.builtin.default([]) | ansible.builtin.length > 0) diff --git a/scenarios/sre/project/roles/tools/tasks/ensure_vendor_namespace.yaml b/scenarios/sre/project/roles/tools/tasks/ensure_vendor_namespace.yaml new file mode 100644 index 000000000..70fdf7b30 --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/ensure_vendor_namespace.yaml @@ -0,0 +1,52 @@ +--- +# Safely ensure an ITBench-owned namespace exists. +# +# Inputs: +# vendor_namespace_name - namespace to ensure +# vendor_namespace_vendor - value for the itbench.io/observability-vendor label +# +# Behaviour: +# - If the namespace does not exist, create it with ITBench ownership labels. +# - If it exists and is already ITBench-owned for this vendor, leave it. +# - If it exists but is NOT ITBench-owned for this vendor, fail (never claim a +# pre-existing/foreign namespace). +- name: Look up the {{ vendor_namespace_vendor }} namespace + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: "{{ vendor_namespace_name }}" + register: vendor_namespace_existing + +- name: Assert an existing {{ vendor_namespace_vendor }} namespace is ITBench-owned + ansible.builtin.assert: + that: + - >- + vendor_namespace_existing.resources[0].metadata.labels['app.kubernetes.io/managed-by'] | ansible.builtin.default('') + == 'ITBench' + - >- + vendor_namespace_existing.resources[0].metadata.labels['itbench.io/observability-vendor'] | ansible.builtin.default('') + == vendor_namespace_vendor + fail_msg: >- + Namespace '{{ vendor_namespace_name }}' already exists but is not owned by + ITBench for the {{ vendor_namespace_vendor }} vendor. Refusing to claim it. + Remove or rename the existing namespace, or point the vendor at a different + namespace. + success_msg: Existing {{ vendor_namespace_vendor }} namespace is ITBench-owned. + when: + - vendor_namespace_existing.resources | ansible.builtin.length == 1 + +- name: Create the {{ vendor_namespace_vendor }} namespace when absent + kubernetes.core.k8s: + kubeconfig: "{{ tools_cluster.kubeconfig }}" + state: present + resource_definition: + apiVersion: v1 + kind: Namespace + metadata: + name: "{{ vendor_namespace_name }}" + labels: + app.kubernetes.io/managed-by: ITBench + itbench.io/observability-vendor: "{{ vendor_namespace_vendor }}" + when: + - vendor_namespace_existing.resources | ansible.builtin.length == 0 From 99f104a559e4b436a24a9231dd151bec052c8637 Mon Sep 17 00:00:00 2001 From: "Rohan R. Arora" Date: Tue, 21 Jul 2026 10:41:33 -0500 Subject: [PATCH 04/11] feat(sre): add Dynatrace observability vendor (Layer A) Deploy the Dynatrace operator and consume the user-provided dynakube.yaml (Secret + DynaKube CRs) as authoritative. Includes manifest validation (kinds/apiVersion/namespace, multi-DynaKube), namespace ownership and collision checks, healthy-status readiness waits, OpenShift support via the chart's platform=openshift value, and a staged CSI-aware, credential-independent uninstall. Signed-off-by: Rohan R. Arora --- .../roles/tools/tasks/install_dynatrace.yaml | 216 ++++++++++++++++++ .../tools/tasks/uninstall_dynatrace.yaml | 140 ++++++++++++ 2 files changed, 356 insertions(+) create mode 100644 scenarios/sre/project/roles/tools/tasks/install_dynatrace.yaml create mode 100644 scenarios/sre/project/roles/tools/tasks/uninstall_dynatrace.yaml diff --git a/scenarios/sre/project/roles/tools/tasks/install_dynatrace.yaml b/scenarios/sre/project/roles/tools/tasks/install_dynatrace.yaml new file mode 100644 index 000000000..be2d87391 --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/install_dynatrace.yaml @@ -0,0 +1,216 @@ +--- +# Dynatrace agent deployment (Layer A). +# +# Consumes the user's dynakube.yaml (Secret with tokens + DynaKube CR). All +# configuration is read from tools_configuration.vendors.dynatrace.* (the single +# role input interface). The file is treated as secret throughout (no_log). +# +# The dynakube.yaml is downloaded, as-is, from the Dynatrace onboarding UI and +# placed at observability_vendors.dynatrace.dynakube_path (default: +# secrets/dynatrace/dynakube.yaml, gitignored) or injected via AWX. It is +# authoritative: ITBench applies it without rewriting its spec. Environment- +# specific adjustments are therefore made in that file, NOT here. +# +# Known environment caveat (edit your dynakube.yaml accordingly): +# * OpenShift / RHEL CoreOS nodes use SELinux, not AppArmor. If the DynaKube +# enables KSPM, its spec.kspm.mappedHostPaths often includes +# "/sys/kernel/security/apparmor". That path does not exist on RHCOS, so the +# operator-generated node-config-collector DaemonSet fails to mount it +# (hostPath type check failed: not a directory) and stays in +# ContainerCreating. Remove that single entry from mappedHostPaths on +# SELinux-based nodes; KSPM otherwise remains fully enabled. See +# docs/observability-vendors.md (Dynatrace on OpenShift) for details. +- name: Set Dynatrace configuration facts + ansible.builtin.set_fact: + tools_dynatrace_config: "{{ tools_configuration.vendors.dynatrace }}" + tools_dynatrace_namespace: "{{ tools_vendors.dynatrace.kubernetes.namespace }}" + tools_dynatrace_api_version: "{{ tools_configuration.vendors.dynatrace.dynakube_api_version | ansible.builtin.default('dynatrace.com/v1beta3') }}" + # The dynatrace-operator Helm chart natively handles OpenShift: setting + # platform=openshift makes the operator create and manage the + # SecurityContextConstraints its agents (OneAgent/ActiveGate/CSI) require. + # On plain Kubernetes the platform value is left at the chart default (""). + tools_dynatrace_helm_platform: "{{ 'openshift' if tools_cluster.platform == 'openshift' else '' }}" + +- name: Resolve Dynatrace DynaKube manifest path + ansible.builtin.set_fact: + tools_dynatrace_dynakube_path: >- + {{ + dynatrace_dynakube_path | + ansible.builtin.default(tools_dynatrace_config.dynakube_path, true) + }} + +# A relative dynakube_path is resolved against the SRE project root (the parent +# of the playbook directory), so it works regardless of the process working +# directory. Absolute paths (e.g. an AWX-injected file) are used as-is. +- name: Anchor a relative Dynatrace manifest path to the project root + ansible.builtin.set_fact: + tools_dynatrace_dynakube_path: "{{ [playbook_dir, '..', tools_dynatrace_dynakube_path] | ansible.builtin.path_join | ansible.builtin.realpath }}" + when: + - not (tools_dynatrace_dynakube_path is ansible.builtin.abs) + +- name: Verify that the Dynatrace DynaKube manifest exists + ansible.builtin.stat: + path: "{{ tools_dynatrace_dynakube_path }}" + register: tools_dynatrace_dynakube_stat + no_log: true + +- name: Assert that the Dynatrace DynaKube manifest is present and not empty + ansible.builtin.assert: + that: + - tools_dynatrace_dynakube_stat.stat.exists + - tools_dynatrace_dynakube_stat.stat.size > 0 + fail_msg: >- + The Dynatrace DynaKube manifest could not be found at + '{{ tools_dynatrace_dynakube_path }}'. Download dynakube.yaml from the + Dynatrace onboarding UI and place it there, or set dynatrace_dynakube_path. + success_msg: Dynatrace DynaKube manifest found. + +# --- Manifest validation (S1/M3). -------------------------------------------- +- name: Parse the Dynatrace DynaKube manifest documents + ansible.builtin.set_fact: + tools_dynatrace_documents: >- + {{ + lookup('ansible.builtin.file', tools_dynatrace_dynakube_path) + | ansible.builtin.from_yaml_all | list + }} + no_log: true + +- name: Validate the Dynatrace DynaKube manifest + vars: + tools_dynatrace_non_null: "{{ tools_dynatrace_documents | ansible.builtin.reject('none') | list }}" + ansible.builtin.assert: + that: + - tools_dynatrace_non_null | ansible.builtin.length > 0 + - tools_dynatrace_non_null | ansible.builtin.length <= 8 + - tools_dynatrace_documents | ansible.builtin.length == tools_dynatrace_non_null | ansible.builtin.length + # Only DynaKube + Secret kinds are permitted. + - >- + tools_dynatrace_non_null | map(attribute='kind') | unique + | difference(['DynaKube', 'Secret']) | ansible.builtin.length == 0 + # At least one DynaKube CR (the Dynatrace UI may emit several, e.g. a + # separate ActiveGate-only and a cloudNativeFullStack DynaKube). + - >- + tools_dynatrace_non_null | selectattr('kind', 'equalto', 'DynaKube') | list + | ansible.builtin.length >= 1 + # Every DynaKube must use the explicit supported API version (M3). + - >- + tools_dynatrace_non_null | selectattr('kind', 'equalto', 'DynaKube') + | map(attribute='apiVersion') | unique + | difference([tools_dynatrace_api_version]) | ansible.builtin.length == 0 + # No document may target another namespace. + - >- + tools_dynatrace_non_null + | map(attribute='metadata.namespace', default=tools_dynatrace_namespace) + | unique | difference([tools_dynatrace_namespace]) | ansible.builtin.length == 0 + fail_msg: >- + The Dynatrace DynaKube manifest must contain 1-8 non-null documents, only + DynaKube and Secret kinds, at least one DynaKube (all using apiVersion + '{{ tools_dynatrace_api_version }}'), and no resources targeting a namespace + other than '{{ tools_dynatrace_namespace }}'. + success_msg: Dynatrace DynaKube manifest passed validation. + no_log: true + +- name: Collect all DynaKube resource names for later reconciliation checks + ansible.builtin.set_fact: + tools_dynatrace_dynakube_names: >- + {{ + tools_dynatrace_documents | ansible.builtin.reject('none') | list + | selectattr('kind', 'equalto', 'DynaKube') + | map(attribute='metadata.name') | list + }} + no_log: true + +# --- H1: ensure/claim the namespace safely. ----------------------------------- +- name: Ensure the Dynatrace namespace exists and is ITBench-owned + ansible.builtin.include_tasks: ensure_vendor_namespace.yaml + vars: + vendor_namespace_name: "{{ tools_dynatrace_namespace }}" + vendor_namespace_vendor: dynatrace + +- name: Install Dynatrace Operator + kubernetes.core.helm: + chart_ref: "{{ tools_vendors.dynatrace.helm.chart.reference }}" + chart_version: "{{ tools_vendors.dynatrace.helm.chart.version }}" + kubeconfig: "{{ tools_cluster.kubeconfig }}" + release_name: "{{ tools_vendors.dynatrace.helm.release.name }}" + release_namespace: "{{ tools_dynatrace_namespace }}" + release_state: present + # Only set values that differ from chart defaults. On OpenShift, platform is + # set so the operator manages its own SCCs; on Kubernetes the block is empty. + values: >- + {{ + {'platform': tools_dynatrace_helm_platform} + if tools_dynatrace_helm_platform | ansible.builtin.length > 0 + else {} + }} + atomic: true + timeout: 10m0s + wait: true + +- name: Wait for the Dynatrace Operator webhook deployment to be rolled out (M4) + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: Deployment + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: dynatrace-webhook + namespace: "{{ tools_dynatrace_namespace }}" + register: tools_dynatrace_webhook + until: + - tools_dynatrace_webhook.resources | ansible.builtin.length == 1 + - tools_dynatrace_webhook.resources[0].status.unavailableReplicas | ansible.builtin.default(0) == 0 + - tools_dynatrace_webhook.resources[0].status.readyReplicas | ansible.builtin.default(0) > 0 + delay: 15 + retries: 20 + +# --- H3: reject collisions before applying the manifest. ---------------------- +- name: Build the ITBench-labeled Dynatrace documents + ansible.builtin.set_fact: + tools_dynatrace_labeled_documents: >- + {{ + (tools_dynatrace_documents | ansible.builtin.reject('none') | list) + | map('combine', {'metadata': {'namespace': tools_dynatrace_namespace, 'labels': {'app.kubernetes.io/managed-by': 'ITBench', 'itbench.io/observability-vendor': 'dynatrace'}}}, recursive=True) + | list + }} + no_log: true + +- name: Assert no unowned Dynatrace resource collisions + ansible.builtin.include_tasks: assert_vendor_resource_ownership.yaml + vars: + vendor_ownership_vendor: dynatrace + vendor_ownership_namespace: "{{ tools_dynatrace_namespace }}" + vendor_ownership_documents: "{{ tools_dynatrace_labeled_documents }}" + +- name: Apply the Dynatrace DynaKube manifest (labelled ITBench-owned) + kubernetes.core.k8s: + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_dynatrace_namespace }}" + definition: "{{ tools_dynatrace_labeled_documents }}" + apply: true + state: present + wait: true + no_log: true + +- name: Wait for each DynaKube resource to report a healthy status (M4) + kubernetes.core.k8s_info: + api_version: "{{ tools_dynatrace_api_version }}" + kind: DynaKube + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: "{{ tools_dynatrace_dynakube_item }}" + namespace: "{{ tools_dynatrace_namespace }}" + register: tools_dynatrace_dynakube_status + loop: "{{ tools_dynatrace_dynakube_names }}" + loop_control: + loop_var: tools_dynatrace_dynakube_item + label: "dynakube/{{ tools_dynatrace_dynakube_item }}" + until: + - tools_dynatrace_dynakube_status.resources | ansible.builtin.length == 1 + # DynaKube reports readiness via conditions; require at least one True + # condition and no False conditions (proves reconciliation succeeded). + - >- + tools_dynatrace_dynakube_status.resources[0].status.conditions | ansible.builtin.default([]) + | selectattr('status', 'equalto', 'True') | list | ansible.builtin.length > 0 + - >- + tools_dynatrace_dynakube_status.resources[0].status.conditions | ansible.builtin.default([]) + | selectattr('status', 'equalto', 'False') | list | ansible.builtin.length == 0 + delay: 20 + retries: 30 diff --git a/scenarios/sre/project/roles/tools/tasks/uninstall_dynatrace.yaml b/scenarios/sre/project/roles/tools/tasks/uninstall_dynatrace.yaml new file mode 100644 index 000000000..7e8a68228 --- /dev/null +++ b/scenarios/sre/project/roles/tools/tasks/uninstall_dynatrace.yaml @@ -0,0 +1,140 @@ +--- +# Idempotent, credential-independent, CSI-aware Dynatrace cleanup. Does not +# depend on the original dynakube.yaml. Safe to run when never installed. +# +# Staged order (important for the CSI driver / OneAgent full-stack): +# 1. Delete DynaKube CRs while the operator is still running so it can run +# finalizers, unmount CSI volumes, and tear down injected workloads. Do NOT +# suppress failures here. +# 2. Confirm the DynaKube API reports the resources are gone (a stuck finalizer +# must stop the play before we remove the operator). +# 3. Uninstall the operator Helm release (this removes chart-owned CSI +# DaemonSets, webhook, etc.). +# 4. Verify chart-owned workloads disappeared, then remove the namespace only +# if exclusively ITBench-owned and drained. +- name: Set Dynatrace uninstall facts + ansible.builtin.set_fact: + tools_dynatrace_namespace: "{{ tools_vendors.dynatrace.kubernetes.namespace }}" + +# Discover the served DynaKube API version from the cluster CRD so uninstall is +# independent of any config/manifest (the CRD is the source of truth). Falls back +# to configured/default values only if the CRD is absent. +- name: Look up the DynaKube custom resource definition + kubernetes.core.k8s_info: + api_version: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: dynakubes.dynatrace.com + register: tools_dynatrace_crd + +- name: Determine the DynaKube API version to use for cleanup + ansible.builtin.set_fact: + tools_dynatrace_api_version: >- + {{ + ('dynatrace.com/' + ( + tools_dynatrace_crd.resources[0].spec.versions + | selectattr('served') | map(attribute='name') | list | last + )) + if (tools_dynatrace_crd.resources | ansible.builtin.length == 1) + else ((tools_configuration.vendors.dynatrace | ansible.builtin.default({})).dynakube_api_version | ansible.builtin.default('dynatrace.com/v1beta3')) + }} + +- name: Check whether the Dynatrace namespace exists + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + kubeconfig: "{{ tools_cluster.kubeconfig }}" + name: "{{ tools_dynatrace_namespace }}" + register: tools_dynatrace_ns + +- name: Clean up Dynatrace resources + when: + - tools_dynatrace_ns.resources | ansible.builtin.length == 1 + block: + # Stage 1: remove DynaKube CRs while the operator remains active. Failures + # here are NOT suppressed - a failed delete request must surface. + - name: Delete ITBench-owned DynaKube resources + kubernetes.core.k8s: + api_version: "{{ tools_dynatrace_api_version }}" + kind: DynaKube + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_dynatrace_namespace }}" + label_selectors: + - itbench.io/observability-vendor=dynatrace + state: absent + wait: true + wait_timeout: 300 + + # Stage 2: confirm the DynaKube resources are actually gone. If a finalizer + # is stuck, this fails and the play stops BEFORE the operator is removed + # (removing the operator would strand the finalizer permanently). + - name: Confirm all ITBench-owned DynaKube resources are removed + kubernetes.core.k8s_info: + api_version: "{{ tools_dynatrace_api_version }}" + kind: DynaKube + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_dynatrace_namespace }}" + label_selectors: + - itbench.io/observability-vendor=dynatrace + register: tools_dynatrace_remaining_dynakubes + until: + - tools_dynatrace_remaining_dynakubes.resources | ansible.builtin.length == 0 + delay: 15 + retries: 20 + + - name: Fail if DynaKube resources are still present (stuck finalizer) + ansible.builtin.assert: + that: + - tools_dynatrace_remaining_dynakubes.resources | ansible.builtin.length == 0 + fail_msg: >- + DynaKube resources still present after deletion (likely a stuck + finalizer). Stopping before Helm uninstall to avoid stranding the + finalizer. Investigate the DynaKube status/finalizers and retry. + success_msg: All ITBench-owned DynaKube resources removed. + + # Stage 3: now remove the operator itself (removes chart-owned workloads). + - name: Uninstall the Dynatrace Operator Helm release + kubernetes.core.helm: + kubeconfig: "{{ tools_cluster.kubeconfig }}" + release_name: "{{ tools_vendors.dynatrace.helm.release.name }}" + release_namespace: "{{ tools_dynatrace_namespace }}" + release_state: absent + wait: true + + # Verify chart-owned workloads (webhook, CSI DaemonSet, etc.) are gone after + # the Helm uninstall. + - name: Wait for Dynatrace workloads to drain after Helm uninstall + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: "{{ tools_dynatrace_workload_kind }}" + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_dynatrace_namespace }}" + loop: + - DaemonSet + - Deployment + - StatefulSet + loop_control: + loop_var: tools_dynatrace_workload_kind + register: tools_dynatrace_remaining_workloads + until: + - tools_dynatrace_remaining_workloads.resources | ansible.builtin.length == 0 + delay: 15 + retries: 20 + + - name: Delete any remaining ITBench-owned Dynatrace secrets + kubernetes.core.k8s: + api_version: v1 + kind: Secret + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: "{{ tools_dynatrace_namespace }}" + label_selectors: + - itbench.io/observability-vendor=dynatrace + state: absent + wait: true + + # Stage 4: remove the namespace only if exclusively ITBench-owned and drained. + - name: Delete the Dynatrace namespace only when exclusively owned and drained + ansible.builtin.include_tasks: delete_vendor_namespace_if_safe.yaml + vars: + vendor_delete_namespace: "{{ tools_dynatrace_namespace }}" + vendor_delete_vendor: dynatrace From 1ff659bab492917610c2d13749707667ea3739bd Mon Sep 17 00:00:00 2001 From: "Rohan R. Arora" Date: Tue, 21 Jul 2026 10:41:55 -0500 Subject: [PATCH 05/11] feat(sre): AWX support, tests, and docs for observability vendors - AWX credential types (Datadog, Dynatrace) with file/extra-var injectors, reconciled present/absent by the enabled flag; non-secret config forwarded to Deploy/Undeploy-Tools via an allowlist projection; vendor credentials scoped to the tools workflow nodes only (node_tools_credentials). - argument_specs updated in the generator template and the generated file. - Molecule coverage: credential-free vendor_contract (validation, secret rejection, projection allowlist) and opt-in deployment_vendors live test. - docs/observability-vendors.md design and status. Signed-off-by: Rohan R. Arora --- scenarios/sre/docs/observability-vendors.md | 511 ++++++++++++++++++ .../group_vars/runner/vendors.yaml.example | 30 + scenarios/sre/project/manage_awx.yaml | 7 + .../credential_types/datadog_injectors.json | 10 + .../credential_types/dynatrace_injectors.json | 8 + .../roles/awx/meta/argument_specs.yaml | 53 ++ .../awx/tasks/configure_credentials.yaml | 102 ++++ .../roles/awx/tasks/configure_jobs.yaml | 38 ++ .../roles/awx/tasks/configure_workflows.yaml | 4 + .../awx/workflows/node_tools_credentials.j2 | 11 + .../awx/workflows/nodes/scenario_execution.j2 | 2 +- .../awx/templates/meta/argument_specs.j2 | 53 ++ .../molecule/deployment_vendors/converge.yml | 8 + .../molecule/deployment_vendors/create.yml | 56 ++ .../molecule/deployment_vendors/destroy.yml | 36 ++ .../molecule/deployment_vendors/molecule.yml | 27 + .../molecule/deployment_vendors/verify.yml | 102 ++++ .../molecule/vendor_contract/converge.yml | 120 ++++ .../files/datadog_mismatched_secret.yaml | 13 + .../vendor_contract/files/datadog_valid.yaml | 20 + .../files/datadog_with_secret.yaml | 21 + .../molecule/vendor_contract/molecule.yml | 14 + 22 files changed, 1245 insertions(+), 1 deletion(-) create mode 100644 scenarios/sre/docs/observability-vendors.md create mode 100644 scenarios/sre/inventory/group_vars/runner/vendors.yaml.example create mode 100644 scenarios/sre/project/roles/awx/files/awx/credential_types/datadog_injectors.json create mode 100644 scenarios/sre/project/roles/awx/files/awx/credential_types/dynatrace_injectors.json create mode 100644 scenarios/sre/project/roles/awx/templates/awx/workflows/node_tools_credentials.j2 create mode 100644 scenarios/sre/project/roles/tools/molecule/deployment_vendors/converge.yml create mode 100644 scenarios/sre/project/roles/tools/molecule/deployment_vendors/create.yml create mode 100644 scenarios/sre/project/roles/tools/molecule/deployment_vendors/destroy.yml create mode 100644 scenarios/sre/project/roles/tools/molecule/deployment_vendors/molecule.yml create mode 100644 scenarios/sre/project/roles/tools/molecule/deployment_vendors/verify.yml create mode 100644 scenarios/sre/project/roles/tools/molecule/vendor_contract/converge.yml create mode 100644 scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_mismatched_secret.yaml create mode 100644 scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_valid.yaml create mode 100644 scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_with_secret.yaml create mode 100644 scenarios/sre/project/roles/tools/molecule/vendor_contract/molecule.yml diff --git a/scenarios/sre/docs/observability-vendors.md b/scenarios/sre/docs/observability-vendors.md new file mode 100644 index 000000000..8b2d33a37 --- /dev/null +++ b/scenarios/sre/docs/observability-vendors.md @@ -0,0 +1,511 @@ +# Multi-Vendor Observability Support (Layer A) + +Design and implementation plan for deploying third-party SaaS observability +agents into ITBench SRE clusters, for both local and AWX-driven deployments. + +Vendors in scope: **Datadog, Dynatrace**. + +- **Layer A (this document):** deploy each vendor's agent/collector into the + Kubernetes/OpenShift cluster so telemetry flows to the vendor's SaaS tenant. +- **Layer B (future):** expose each vendor's tenant URL + a scoped read token to + the diagnosing SRE agent via the agent bundle so it can query telemetry during + incidents. Out of scope here. + +All backends are **SaaS**. This is the key architectural difference from the +existing in-cluster tools (Prometheus, ClickHouse, Jaeger), whose endpoints and +credentials are *discovered at runtime* from cluster Services/Secrets. SaaS +vendors invert this: **credentials are inputs** supplied by the operator, and +there is **no in-cluster endpoint to discover** — the "endpoint" is the vendor's +tenant URL. + +--- + +## Existing conventions (how tools are wired today) + +The SRE tooling lives under `scenarios/sre/`. There is no OO interface; it is an +Ansible convention-over-configuration pattern: + +| Concern | Mechanism | +|---------|-----------| +| Tool registry | `tools_managers` dict in `project/roles/tools/defaults/main/managers.yaml` | +| Install/uninstall "interface" | Task-file naming: `install_.yaml`, `uninstall_.yaml` | +| Endpoint discovery | `set_internal_endpoints_.yaml` / `set_external_endpoints_.yaml` | +| Credentials | `set_credentials_.yaml` (reads K8s Secrets) | +| Orchestration | `project/roles/tools/tasks/install.yaml` imports each installer, flag-gated | +| Enablement | Boolean flags in `group_vars` mapped via `project/manage_tools.yaml` | +| Helm values | `templates/helm//values.j2` | +| K8s resources | `templates/kubernetes//.j2` | +| AWX | `project/roles/awx/` job templates + custom credential types | + +Reference install to copy for Helm-based tools: `install_opencost.yaml`. + +--- + +## Status summary + +- **Dynatrace: implemented and validated on OpenShift (CRC).** Full lifecycle + exercised against a live OpenShift cluster + real Dynatrace tenant: operator + install (chart `1.9.0`, `platform=openshift`), multi-DynaKube manifest + (`v1beta6`) validation, namespace ownership, collision check, apply, DynaKube + reconcile, and staged credential-independent uninstall with namespace + preservation. Validation surfaced and fixed four issues: relative manifest path + resolution, the collision-check helper's `no_log` interaction, uninstall + discovering the DynaKube API version from the CRD, and allowing multiple + DynaKube documents. Not yet validated on plain (non-OpenShift) Kubernetes. +- **Datadog: implemented, unvalidated.** All install/uninstall/AWX/validation + code exists and passes YAML/schema checks, but has **not** been run against a + live cluster or SaaS tenant, and remains blocked on OpenShift. + +Molecule coverage is **required** before a vendor is treated as validated: +- `molecule/vendor_contract/` — credential-free, cluster-free contract tests + (manifest validation, Secret-document rejection, secret/key-reference + matching, and the non-secret projection allowlist). Always runs. +- `molecule/deployment_vendors/` — opt-in live deploy that self-skips when vendor + secrets are absent. + +### OpenShift support + +**Dynatrace: supported and validated on OpenShift** (CRC / OpenShift, cluster +`1.35.5`, operator chart `1.9.0`). The `dynatrace-operator` Helm chart natively +handles OpenShift: setting `platform=openshift` makes the operator create and +manage the `SecurityContextConstraints` its agents (OneAgent, ActiveGate, CSI +driver) need. We therefore do **not** hand-craft SCCs for Dynatrace — the +install task sets the `platform` Helm value on OpenShift and the operator owns +its SCCs (Helm uninstall removes them). Validated end-to-end: operator + webhook ++ CSI driver + ActiveGate + OTel collector reached Running; DynaKube resources +reconciled; uninstall staged cleanly (DynaKube → operator → workload drain) and +preserved the namespace when not exclusively ITBench-owned. + +##### KSPM AppArmor host path on RHEL CoreOS (edit your `dynakube.yaml`) + +If the DynaKube enables **KSPM** (Kubernetes Security Posture Management), the +operator generates a `node-config-collector` DaemonSet that mounts every +`spec.kspm.mappedHostPaths` entry as a `hostPath` with `type: Directory`. The +onboarding UI commonly includes `/sys/kernel/security/apparmor` in that list. + +RHEL CoreOS — the OpenShift node OS — uses **SELinux, not AppArmor**, so +`/sys/kernel/security/apparmor` does not exist on the node. The mount then fails +with `hostPath type check failed: /sys/kernel/security/apparmor is not a +directory` and the collector pod is stuck in `ContainerCreating`. + +Because ITBench treats the downloaded `dynakube.yaml` as authoritative and does +**not** rewrite its spec, the fix is made in **your file**: remove that one entry +from `spec.kspm.mappedHostPaths`. KSPM otherwise stays fully enabled. + +```yaml + kspm: + mappedHostPaths: + - /boot + - /etc + - /proc/sys/kernel + - /sys/fs + # - /sys/kernel/security/apparmor # remove on SELinux nodes (RHEL CoreOS) + - /usr/lib/systemd/system + - /var/lib +``` + +Re-run the tools playbook after editing; the operator regenerates the DaemonSet +without the AppArmor mount and the collector reaches `Running`. This caveat is +also documented inline at the top of +`project/roles/tools/tasks/install_dynatrace.yaml`. + +**Datadog: still blocked on OpenShift** (hard assertion). The Datadog operator's +OpenShift SCC story has not been validated here; the block stays until it is. + +> Note: Some older subsections below still describe an earlier namespace-`.j2` / +> SCC-annotation approach and an `allow_openshift` override. Those are superseded: +> namespaces are created via `ensure_vendor_namespace.yaml`, and Dynatrace +> OpenShift support is via the chart's `platform=openshift` value (not custom +> SCCs). + +#### Future work: how to unblock OpenShift (decision: implement later) + +The repo already has a working SCC precedent, so the intended path is settled — +it is deferred only until an OpenShift cluster + vendor tenant is available to +validate the specifics. **Chosen approach: custom least-privilege SCCs bound to +the agent service accounts (the Chaos Mesh pattern in +`install_chaos_mesh.yaml:6-57` / `uninstall_chaos_mesh.yaml:40-67`).** This was +preferred over binding to the built-in `system:openshift:scc:privileged` +(the `RoleBinding` idiom in `applications/.../install_opentelemetry_demo.yaml:52-69`) +because `privileged` grants far more than these agents need; the benchmark runs +untrusted workloads alongside fault injection, so least privilege matters. + +To implement (per vendor), inside a `when: tools_cluster.platform == 'openshift'` +block, mirroring Chaos Mesh: +1. Create a `SecurityContextConstraints` labeled + `app.kubernetes.io/managed-by: ITBench` + a per-vendor component label, + enumerating ONLY the capabilities/volumes the agents require (do not blanket + `allowPrivilegedContainer` unless proven necessary). +2. Bind it via the SCC `users:` list to the exact ServiceAccounts the operator + creates. These names are **UNVERIFIED** and must be confirmed against the + installed operator version, e.g. (indicative, verify before use): + - Datadog: `-agent`, `-cluster-agent`, plus the + operator SA `datadog-operator`. + - Dynatrace: OneAgent, ActiveGate, CSI driver, webhook, and operator SAs + under the `dynatrace` namespace. +3. Remove the OpenShift-blocking assertion in the installers and add labeled SCC + cleanup to the uninstallers (query by the ITBench labels, delete — as + `uninstall_chaos_mesh.yaml` does). +4. Validate on a real OpenShift cluster: agents reach Ready, no + `CreateContainerError`/SCC-denied events, and uninstall removes the SCCs. + +Until steps 1-4 are done and validated, OpenShift stays hard-blocked. + +## Cross-cutting design decisions + +1. **New registry section.** SaaS vendors get a parallel `tools_vendors` dict in + `project/roles/tools/defaults/main/vendors.yaml` (rather than overloading + `tools_managers`), since they carry SaaS-specific attributes. +2. **Non-secret config** (site, region, tenant URL, enable flags) lives in + `inventory/group_vars/environment/observability_vendors.yaml(.example)`. +3. **Secrets are inputs, source-agnostic.** Credential tasks read from a variable + with an env fallback so the same task works locally and under AWX: + `{{ datadog_api_key | default(lookup('ansible.builtin.env','DATADOG_API_KEY'), true) }}`. + All secret-handling tasks use `no_log: true`. Secrets are never committed. +4. **Enablement** via per-vendor booleans mapped through `manage_tools.yaml` into + `tools_configuration.vendors..enabled`, gating each install. +4a. **Single role-input interface.** The tools role reads vendor config **only** + from `tools_configuration.vendors.*` (never from the global + `observability_vendors` directly), so AWX and role tests have one contract. +4b. **Vendor installs run outside the SRE/FinOps gate** so a vendor-only + deployment works. +4c. **Uninstall is unconditional, idempotent, credential-independent, and + ownership-scoped.** It runs during Undeploy-Tools regardless of the enabled + flag (so disabling a vendor still removes it), no-ops when the namespace is + absent, only deletes resources labeled `itbench.io/observability-vendor=`, + and only deletes the namespace when it is ITBench-owned and drained. No blind + `delete_all` of all CRs or unconditional namespace deletion. +4d. **Manifests are treated as untrusted + secret.** User CR files are validated + (allowed kinds/apiVersions, exactly one primary CR, namespace pinning, doc + count, null-doc rejection) before apply, and all parse/apply steps use + `no_log: true`. +4e. **CRD retention.** Helm typically leaves chart CRDs behind on uninstall; we + preserve them by default (do not force-delete vendor CRDs) to avoid breaking + any remaining vendor CRs cluster-wide. +5. **AWX trigger** reuses the existing `Deploy-Tools`/`Undeploy-Tools` job + templates (vendor installs run inside `manage_tools.yaml`, flag-gated). No new + job templates. +6. **AWX secret delivery** uses custom AWX **credential types** (precedent: the + `Kubeconfig` credential type in `configure_credentials.yaml`), injecting + secrets as `extra_vars` (or files) into the Deploy-Tools job pod. +6a. **AWX input plumbing.** `manage_awx.yaml` combines a `vendors` group_var into + `awx_configuration.vendors` (with `no_log`). Non-secret vendor config reaches + the Deploy/Undeploy-Tools job templates via `extra_vars` + (`observability_vendors`), built by a projection in `configure_jobs.yaml` that + **strips secret keys** (`manifest`, `dynakube`, `api_key`, `app_key`). Secrets + reach the job only through the injected vendor credential. +6b. **Vendor credentials are scoped to the tools nodes only.** A separate + `node_tools_credentials` list (template `node_tools_credentials.j2`) carries + the vendor credentials and is attached only to the `node-deploy-tools` / + `node-undeploy-tools` workflow nodes. All other nodes keep the vendor-free + `node_credentials` list, so fault-injection and Run-Agent jobs never receive + vendor secrets. +6c. **AWX role argument spec is generated.** `meta/argument_specs.yaml` is + rendered from `templates/meta/argument_specs.j2`; vendor specs live in the + template (the generated file is overwritten on regeneration). +6d. **Plaintext-to-AWX path.** The `vendors` group_var + (`inventory/group_vars/runner/vendors.yaml.example`) holds the secret inputs + and must be Ansible-Vault encrypted or sourced from a secret manager. AWX + stores the resulting credentials encrypted; provisioning tasks use `no_log`. + +### Secret handling under AWX — why not plain env vars + +AWX runs playbooks in ephemeral job pods (the `ITBench-Custom-EE` execution +environment). There is no shell where an operator `export`ed variables, and job +pods don't inherit host env vars, so `lookup('env', ...)` returns empty. AWX +credential types inject values into the job pod at runtime (as env/extra_vars or +files) and store them encrypted at rest. The source-agnostic variable form above +picks these up transparently. + +Precedent: the `Kubeconfig` custom credential type +(`project/roles/awx/tasks/configure_credentials.yaml`, and injectors JSON at +`project/roles/awx/files/awx/credential_types/kubeconfig_injectors.json`) +supports both `extra_vars` and `file` injection. + +> **Note (pre-existing security debt):** plaintext secrets are currently committed +> in `group_vars/runner/agent.yaml` (watsonx api_key, GitHub PAT) and +> `group_vars/runner/credentials.yaml` (AWS keys). These should be rotated and +> removed; this work deliberately avoids adding new plaintext secrets. + +--- + +## Per-vendor deploy summary + +| Vendor | Helm chart | CR applied | Required secrets | +|--------|-----------|-----------|------------------| +| Datadog | `datadog-operator` (helm.datadoghq.com) | `DatadogAgent` (from user's file) | `DATADOG_API_KEY` (+ `DATADOG_APP_KEY` when the CR references `appSecret`) | +| Dynatrace | `dynatrace-operator` (OCI) | `DynaKube` (from user's file) | tokens embedded in downloaded `dynakube.yaml` | + +Common concerns: OpenShift SCC for privileged agents (precedent +`install_chaos_mesh.yaml`), Istio ambient (ztunnel) coexistence, kind +limitations for eBPF/full-stack agents, and pinned chart versions with renovate +annotations. + +--- + +## Implementation status + +| Item | File | Status | +|------|------|--------| +| Vendor registry | `project/roles/tools/defaults/main/vendors.yaml` | Created | +| Non-secret config (+ example) | `inventory/group_vars/environment/observability_vendors.yaml(.example)` | Created | +| `.gitignore` for `secrets/` | `scenarios/sre/.gitignore` | Created | +| Datadog creds | `project/roles/tools/tasks/set_credentials_datadog.yaml` | **Done** (source-agnostic) | +| Datadog namespace | `templates/kubernetes/datadog/namespace.j2` | **Done** | +| Datadog install/uninstall | `tasks/install_datadog.yaml`, `uninstall_datadog.yaml` | **Done** | +| Datadog orchestration wiring | `tasks/install.yaml`, `uninstall.yaml`, `manage_tools.yaml` | **Done** | +| Datadog AWX credential/wiring | `project/roles/awx/*` | **Done** | +| Dynatrace install/uninstall/AWX | `tasks/install_dynatrace.yaml`, `uninstall_dynatrace.yaml`, `awx/*` | **Implemented, unvalidated** (consumes user's `dynakube.yaml`; namespace created via `ensure_vendor_namespace.yaml`; `set_credentials_dynatrace.yaml` removed) | +| Shared ownership helpers | `tasks/ensure_vendor_namespace.yaml`, `assert_vendor_resource_ownership.yaml`, `delete_vendor_namespace_if_safe.yaml` | **Done** | +| Datadog manifest validation | `tasks/validate_datadog_manifest.yaml` | **Done** | +| Molecule: live deploy (opt-in) | `molecule/deployment_vendors/` | **Done** (self-skips without vendor secrets) | +| Molecule: credential-free contract | `molecule/vendor_contract/` | **Done** (validation, secret rejection, projection allowlist) | + +--- + +## Dynatrace (implemented) — detailed plan + +> Status: built. Registry pinned to operator `1.9.0` (OCI +> `oci://public.ecr.aws/dynatrace/dynatrace-operator`), install/uninstall/namespace +> tasks created, wired into `install.yaml`/`uninstall.yaml` (flag-gated on +> `tools_configuration.vendors.dynatrace.enabled`), AWX `Dynatrace` file-injector +> credential added, and the superseded `set_credentials_dynatrace.yaml` removed. + + +Dynatrace's onboarding UI produces a self-contained `dynakube.yaml` (a `Secret` +with `apiToken` + `dataIngestToken`, plus a `DynaKube` CR carrying apiUrl, +cluster name, and config). We **consume that file as-is** rather than +reconstruct it. + +### Decisions + +- Consume the downloaded `dynakube.yaml` unchanged (apply Secret + DynaKube after + the operator is installed). +- A single variable `dynatrace_dynakube_path` drives both local and AWX paths, so + `install_dynatrace.yaml` is identical in both contexts. +- **Local:** user places file at `secrets/dynatrace/dynakube.yaml` (gitignored); + path configurable via `observability_vendors.dynatrace.dynakube_path`. +- **AWX:** custom `Dynatrace-Dynakube` credential type with a **file injector** + (Kubeconfig precedent); user pastes file contents once, AWX materializes it as + a file in the job pod and sets `dynatrace_dynakube_path` to `tower.filename.dynakube`. +- **Helm:** OCI chart `oci://public.ecr.aws/dynatrace/dynatrace-operator`, pinned + `chart_version: 1.9.0`, `create_namespace: true`, `atomic: true` (v3 form). +- Reuse `Deploy-Tools`/`Undeploy-Tools`; flag-gated by + `tools_configuration.vendors.dynatrace.enabled`. + +Because the file carries everything, Dynatrace needs **no** `set_credentials`, +**no** `values.j2`, and **no** `dynakube.j2`. (An OpenShift-only `namespace.j2` +for SCC annotations is still needed.) + +### Files + +1. **`defaults/main/vendors.yaml`** (modify dynatrace entry): + ```yaml + dynatrace: + helm: + chart: + # renovate: datasource=docker depName=public.ecr.aws/dynatrace/dynatrace-operator + reference: oci://public.ecr.aws/dynatrace/dynatrace-operator + version: 1.9.0 + release: + name: dynatrace-operator + kubernetes: + namespace: dynatrace + ``` + +2. **`inventory/group_vars/environment/observability_vendors.yaml(.example)`** + (modify dynatrace block): + ```yaml + dynatrace: + enabled: false + # Path to dynakube.yaml downloaded from the Dynatrace onboarding UI + # (contains the Secret with tokens + the DynaKube CR). Keep OUT of git. + dynakube_path: "secrets/dynatrace/dynakube.yaml" + ``` + +3. **`scenarios/sre/.gitignore`** (create): add `secrets/`. + +4. **`tasks/install_dynatrace.yaml`** (create): + 1. Resolve `dynatrace_dynakube_path` = `tower.filename.dynakube` if defined + (AWX) else `observability_vendors.dynatrace.dynakube_path`. + 2. `stat` + assert file exists / non-empty (`no_log: true`). + 3. OpenShift only: create `dynatrace` ns from + `templates/kubernetes/dynatrace/namespace.j2` (SCC annotations) before install. + 4. Helm install: + ```yaml + kubernetes.core.helm: + chart_ref: "{{ tools_vendors.dynatrace.helm.chart.reference }}" + chart_version: "{{ tools_vendors.dynatrace.helm.chart.version }}" + kubeconfig: "{{ tools_cluster.kubeconfig }}" + release_name: "{{ tools_vendors.dynatrace.helm.release.name }}" + release_namespace: "{{ tools_vendors.dynatrace.kubernetes.namespace }}" + release_state: present + create_namespace: true + atomic: true + wait: true + ``` + 5. Wait for the operator webhook Deployment to be Available (retry/until) — + DynaKube apply fails if the webhook isn't up yet. + 6. Apply the user's file: + ```yaml + kubernetes.core.k8s: + kubeconfig: "{{ tools_cluster.kubeconfig }}" + definition: "{{ lookup('ansible.builtin.file', dynatrace_dynakube_path) | ansible.builtin.from_yaml_all | list }}" + state: present + wait: true + # no_log: true + ``` + +5. **`tasks/uninstall_dynatrace.yaml`** (create): delete DynaKube CR + its Secret + (by file if present, else by name/label), helm `release_state: absent`, delete + namespace — all `wait: true`, guarded to no-op if file absent. + +6. **`templates/kubernetes/dynatrace/namespace.j2`** (create — OpenShift SCC only): + standard ITBench namespace with `{% if tools_cluster.platform == 'openshift' %}` + scc annotations. + +7. **Orchestration wiring** (modify): + - `tasks/install.yaml`: flag-gated import of `install_dynatrace.yaml` + (`when: tools_configuration.vendors.dynatrace.enabled | default(false)`). + - `tasks/uninstall.yaml`: matching reverse-order import. + - `manage_tools.yaml`: add `vendors: "{{ observability_vendors | default({}) }}"` + to `tools_configuration` (both the scenario_id and non-scenario branches); + pass `observability_vendors` into the tools role vars. + +8. **AWX credential** (file injector — Kubeconfig precedent): + - Create `project/roles/awx/files/awx/credential_types/dynatrace_dynakube_injectors.json`: + ```json + { + "extra_vars": { "dynatrace_dynakube_path": "{{ tower.filename.dynakube }}" }, + "file": { "template.dynakube": "{{ dynakube }}" } + } + ``` + - Modify `project/roles/awx/tasks/configure_credentials.yaml`: create a + `Dynatrace-Dynakube` credential type (one multiline `secret: true` field + `dynakube`, injectors from the JSON) + a credential instance whose + `inputs.dynakube` is sourced from `awx_configuration.vendors.dynatrace.dynakube`. + Gate on Dynatrace enabled. + - Modify `project/roles/awx/templates/awx/workflows/node_credentials.j2`: add + `- name: Dynatrace-Dynakube` when enabled. + - Modify `project/roles/awx/meta/argument_specs.yaml`: document + `awx_configuration.vendors.dynatrace.dynakube`. + +9. **Docs**: `.example` comments explain (a) Dynatrace UI → set cluster + name/tokens → **Download dynakube.yaml**; (b) local → place at + `secrets/dynatrace/dynakube.yaml`; (c) AWX → paste contents into the + `Dynatrace-Dynakube` credential. + +### Notes / risks + +- The v1.9.0 cloud-native chart includes the CSI driver; validate on kind. The + downloaded DynaKube may use classicFullStack/hostMonitoring — that is the + user's file, not ours to change. +- OpenShift: pre-create ns with SCC annotations before the atomic install; + OneAgent may need a privileged SCC (flag for OpenShift testing). +- `atomic: true` rolls back a failed operator install; the separate webhook wait + guards the subsequent CR apply. +- `no_log: true` on all file/CR steps; `secrets/` gitignored; AWX stores the file + encrypted. +- Chart `--version` is `1.9.0` (no `v`); the git tag is `v1.9.0`. + +### Implementation notes (Dynatrace) + +- Namespace is created and ownership-checked via `ensure_vendor_namespace.yaml` + (no namespace `.j2` template). +- Wired into `install.yaml` / `uninstall.yaml` / `manage_tools.yaml`; uninstall + is unconditional, idempotent, CSI-aware, and credential-independent. +- AWX: `dynatrace_injectors.json` (file injector) + `configure_credentials.yaml` + (reconciled present/absent by enabled flag) + `node_tools_credentials.j2` + (attached only to Deploy-Tools) + generated `argument_specs.j2`. +- The superseded `set_credentials_dynatrace.yaml` has been removed. +- Testing is **required**: `molecule/vendor_contract/` (credential-free) always + runs; `molecule/deployment_vendors/` runs a live deploy when vendor secrets are + provided and self-skips otherwise. + +--- + +## Datadog (implemented) — operator + user-supplied `datadog-agent.yaml` + +Datadog's onboarding UI produces a **non-secret** `datadog-agent.yaml` +(`DatadogAgent` CR) that references the API/APP keys **by Secret name** rather +than embedding them. So the split is: + +- **Config (non-secret):** the `datadog-agent.yaml` (site, `clusterName`, feature + toggles, RUM IDs). May be committed/pasted freely. +- **Secret:** `datadog-secret` with `api-key` / `app-key`, created separately. + +This differs from Dynatrace (whose downloaded file embeds the tokens). We +therefore both consume the user's CR file **and** create the Secret from +`DATADOG_API_KEY` / `DATADOG_APP_KEY`. + +### Decisions + +- Consume the user's `datadog-agent.yaml` as-is (no re-templating). +- CR file source: variable `datadog_manifest_path` (env fallback via extra_var), + default local path `secrets/datadog/datadog-agent.yaml` (gitignored). Under AWX + it is injected as a file via a custom `Datadog` credential. +- Secret created by `set_credentials_datadog.yaml` from source-agnostic vars + `datadog_api_key` / `datadog_app_key` (env fallback + `DATADOG_API_KEY`/`DATADOG_APP_KEY`); `no_log: true`. Secret name and key names + are configurable (`observability_vendors.datadog.{secret_name,api_key_name,app_key_name}`) + and **must match** the `credentials.apiSecret/appSecret` refs in the CR. +- Helm: classic repo chart `datadog/datadog-operator` from + `https://helm.datadoghq.com`, pinned `chart_version: 2.24.0`, `atomic: true`, + `wait: true`. (The onboarding command uses the operator chart, not the agent + chart.) +- Reuse `Deploy-Tools`/`Undeploy-Tools`; flag-gated by + `tools_configuration.vendors.datadog.enabled`. + +### Files (all created) + +| File | Purpose | +|------|---------| +| `defaults/main/vendors.yaml` (datadog entry) | Operator chart `datadog-operator` @ `2.24.0`, ns `datadog` | +| `inventory/group_vars/environment/observability_vendors.yaml(.example)` | `datadog.{enabled,manifest_path,secret_name,api_key_name,app_key_name}` | +| `tasks/set_credentials_datadog.yaml` | Source-agnostic keys → create `datadog-secret` (`no_log`) | +| `templates/kubernetes/datadog/namespace.j2` | Namespace (OpenShift SCC annotations) | +| `tasks/install_datadog.yaml` | Resolve+assert manifest → ns → helm operator (atomic/wait) → wait for `datadogagents.datadoghq.com` CRD Established → create secret → apply CR file | +| `tasks/uninstall_datadog.yaml` | Delete DatadogAgent CRs → helm absent → delete ns | +| `tasks/install.yaml` / `uninstall.yaml` | Flag-gated import of Datadog tasks | +| `manage_tools.yaml` | `tools_configuration.vendors` from `observability_vendors` | +| `awx/files/awx/credential_types/datadog_injectors.json` | file→`datadog_manifest_path`; extra_vars→`datadog_api_key`/`datadog_app_key` | +| `awx/tasks/configure_credentials.yaml` | `Datadog` credential type + instance (gated on enabled) | +| `awx/templates/awx/workflows/node_credentials.j2` | Attach `Datadog` credential to Deploy/Undeploy-Tools nodes | +| `awx/meta/argument_specs.yaml` | Document `awx_configuration.vendors.datadog.*` | + +### Install flow (`install_datadog.yaml`) + +1. Resolve `datadog_manifest_path` (AWX extra_var/file, else group_var default). +2. `stat` + assert the manifest exists / non-empty. +3. Create namespace from `namespace.j2`. +4. Helm install `datadog-operator` (repo chart, `atomic: true`, `wait: true`). +5. Wait for the `datadogagents.datadoghq.com` CRD to be **Established**. +6. Import `set_credentials_datadog.yaml` → create `datadog-secret`. +7. Apply the user's `datadog-agent.yaml` (`from_yaml_all`), into the `datadog` ns. + +### Usage + +**Local:** +```bash +export DATADOG_API_KEY=... # from the onboarding command # pragma: allowlist secret +export DATADOG_APP_KEY=... # optional, needed for Private Action Runner +mkdir -p scenarios/sre/secrets/datadog +cp ~/Downloads/datadog-agent.yaml scenarios/sre/secrets/datadog/datadog-agent.yaml +# set observability_vendors.datadog.enabled: true, then run Deploy-Tools +``` + +**AWX:** set `awx_configuration.vendors.datadog.{enabled: true, manifest: , api_key, app_key}` so the `Datadog` credential is created and attached +to the Deploy-Tools node; also set `observability_vendors.datadog.enabled: true` +in the committed group_vars (the flag the playbook reads). AWX injects the +manifest as a file and the keys as extra_vars into the job pod. + +### Notes / risks + +- Secret key names must match the CR's `credentials.apiSecret.keyName` / + `appSecret.keyName` (defaults `api-key` / `app-key`). +- The sample CR enables OTel collector host ports 4317/4318 and APM SSI — + validate host-port availability and privileged access on kind/OpenShift. +- On OpenShift the node agent likely needs a privileged SCC (not yet added; + flag for OpenShift testing) in addition to the namespace SCC annotations. +- App key is optional for basic monitoring but **required** for the Private + Action Runner / "Agent to take action" feature shown in onboarding. diff --git a/scenarios/sre/inventory/group_vars/runner/vendors.yaml.example b/scenarios/sre/inventory/group_vars/runner/vendors.yaml.example new file mode 100644 index 000000000..1bb828316 --- /dev/null +++ b/scenarios/sre/inventory/group_vars/runner/vendors.yaml.example @@ -0,0 +1,30 @@ +--- +# Observability vendor configuration for the AWX runner entry point +# (manage_awx.yaml). This drives both: +# - AWX credential creation (secret inputs: manifest/dynakube contents, keys) +# - Non-secret job extra_vars (enable flags + config), projected automatically +# with secrets stripped in roles/awx/tasks/configure_jobs.yaml +# +# SECURITY: this file contains secrets. DO NOT commit it in plaintext. Encrypt it +# with Ansible Vault (ansible-vault encrypt vendors.yaml) or source the secret +# fields from a secret manager. The provisioning tasks use no_log; AWX stores the +# resulting credentials encrypted at rest. +# +# vendors: +# datadog: +# enabled: true +# # Non-secret config (also consumed by the tools role): +# manifest_path: "secrets/datadog/datadog-agent.yaml" +# secret_name: datadog-secret +# api_key_name: api-key +# app_key_name: app-key +# # Secret inputs (injected into AWX credentials, stripped from job vars): +# manifest: | +# # full contents of the datadog-agent.yaml downloaded from Datadog +# api_key: "" # pragma: allowlist secret +# app_key: "" # pragma: allowlist secret +# dynatrace: +# enabled: true +# dynakube_path: "secrets/dynatrace/dynakube.yaml" +# dynakube: | +# # full contents of the dynakube.yaml downloaded from Dynatrace diff --git a/scenarios/sre/project/manage_awx.yaml b/scenarios/sre/project/manage_awx.yaml index b708f4694..157348502 100644 --- a/scenarios/sre/project/manage_awx.yaml +++ b/scenarios/sre/project/manage_awx.yaml @@ -36,6 +36,13 @@ when: - cluster_provider != "kind" + - name: Add observability vendors variable + ansible.builtin.set_fact: + awx_configuration: "{{ awx_configuration | ansible.builtin.combine({'vendors': vendors}) }}" + when: + - vendors is ansible.builtin.defined + no_log: true + - name: Create experiments variable ansible.builtin.set_fact: awx_experiments: diff --git a/scenarios/sre/project/roles/awx/files/awx/credential_types/datadog_injectors.json b/scenarios/sre/project/roles/awx/files/awx/credential_types/datadog_injectors.json new file mode 100644 index 000000000..d8f1bb413 --- /dev/null +++ b/scenarios/sre/project/roles/awx/files/awx/credential_types/datadog_injectors.json @@ -0,0 +1,10 @@ +{ + "extra_vars": { + "datadog_manifest_path": "{{ tower.filename.manifest }}", + "datadog_api_key": "{{ api_key }}", + "datadog_app_key": "{{ app_key }}" + }, + "file": { + "template.manifest": "{{ manifest }}" + } +} diff --git a/scenarios/sre/project/roles/awx/files/awx/credential_types/dynatrace_injectors.json b/scenarios/sre/project/roles/awx/files/awx/credential_types/dynatrace_injectors.json new file mode 100644 index 000000000..a939eb310 --- /dev/null +++ b/scenarios/sre/project/roles/awx/files/awx/credential_types/dynatrace_injectors.json @@ -0,0 +1,8 @@ +{ + "extra_vars": { + "dynatrace_dynakube_path": "{{ tower.filename.dynakube }}" + }, + "file": { + "template.dynakube": "{{ dynakube }}" + } +} diff --git a/scenarios/sre/project/roles/awx/meta/argument_specs.yaml b/scenarios/sre/project/roles/awx/meta/argument_specs.yaml index deaf9af41..11e3b9a00 100644 --- a/scenarios/sre/project/roles/awx/meta/argument_specs.yaml +++ b/scenarios/sre/project/roles/awx/meta/argument_specs.yaml @@ -85,6 +85,59 @@ argument_specs: type: dict required: false type: dict + vendors: + options: + datadog: + options: + enabled: + default: false + required: false + type: bool + manifest_path: + required: false + type: str + secret_name: + required: false + type: str + api_key_name: + required: false + type: str + app_key_name: + required: false + type: str + manifest: + required: false + type: str + api_key: + no_log: true + required: false + type: str + app_key: + no_log: true + required: false + type: str + required: false + type: dict + dynatrace: + options: + enabled: + default: false + required: false + type: bool + dynakube_path: + required: false + type: str + dynakube_api_version: + required: false + type: str + dynakube: + no_log: true + required: false + type: str + required: false + type: dict + required: false + type: dict scenarios: elements: dict options: diff --git a/scenarios/sre/project/roles/awx/tasks/configure_credentials.yaml b/scenarios/sre/project/roles/awx/tasks/configure_credentials.yaml index 0cdd3f03b..2092754f4 100644 --- a/scenarios/sre/project/roles/awx/tasks/configure_credentials.yaml +++ b/scenarios/sre/project/roles/awx/tasks/configure_credentials.yaml @@ -76,6 +76,108 @@ organization: ITBench state: present +- name: Create credentials for Datadog observability vendor + when: + - awx_configuration.vendors.datadog is ansible.builtin.defined + vars: + awx_datadog_enabled: "{{ awx_configuration.vendors.datadog.enabled | ansible.builtin.default(false) }}" + block: + - name: Create Datadog credential type + awx.awx.credential_type: + controller_host: "{{ awx_controller_host }}" + controller_password: "{{ awx_controller_password }}" # pragma: allowlist secret + controller_username: admin + description: Credentials type for the Datadog observability vendor + inputs: + fields: + - id: manifest + type: string + label: DatadogAgent manifest (datadog-agent.yaml) + secret: true + multiline: true + - id: api_key + type: string + label: Datadog API key + secret: true + - id: app_key + type: string + label: Datadog application key + secret: true + required: + - manifest + - api_key + injectors: "{{ lookup('ansible.builtin.file', 'files/awx/credential_types/datadog_injectors.json') }}" + kind: cloud + name: Datadog + state: present + when: + - awx_datadog_enabled + + # Reconcile the credential INSTANCE to match the enabled flag: present when + # enabled, absent when disabled (so disabling a vendor removes its stored + # credential rather than leaving it selectable). + - name: Reconcile Datadog credential + awx.awx.credential: + controller_host: "{{ awx_controller_host }}" + controller_password: "{{ awx_controller_password }}" # pragma: allowlist secret + controller_username: admin + credential_type: Datadog + description: Datadog agent manifest and API/application keys + inputs: "{{ awx_datadog_inputs if awx_datadog_enabled else omit }}" + name: Datadog + organization: ITBench + state: "{{ 'present' if awx_datadog_enabled else 'absent' }}" + vars: + awx_datadog_inputs: + manifest: "{{ awx_configuration.vendors.datadog.manifest | ansible.builtin.default('') }}" + api_key: "{{ awx_configuration.vendors.datadog.api_key | ansible.builtin.default('') }}" # pragma: allowlist secret + app_key: "{{ awx_configuration.vendors.datadog.app_key | ansible.builtin.default('') }}" # pragma: allowlist secret + no_log: true + +- name: Create credentials for Dynatrace observability vendor + when: + - awx_configuration.vendors.dynatrace is ansible.builtin.defined + vars: + awx_dynatrace_enabled: "{{ awx_configuration.vendors.dynatrace.enabled | ansible.builtin.default(false) }}" + block: + - name: Create Dynatrace credential type + awx.awx.credential_type: + controller_host: "{{ awx_controller_host }}" + controller_password: "{{ awx_controller_password }}" # pragma: allowlist secret + controller_username: admin + description: Credentials type for the Dynatrace observability vendor + inputs: + fields: + - id: dynakube + type: string + label: DynaKube manifest (dynakube.yaml) + secret: true + multiline: true + required: + - dynakube + injectors: "{{ lookup('ansible.builtin.file', 'files/awx/credential_types/dynatrace_injectors.json') }}" + kind: cloud + name: Dynatrace + state: present + when: + - awx_dynatrace_enabled + + - name: Reconcile Dynatrace credential + awx.awx.credential: + controller_host: "{{ awx_controller_host }}" + controller_password: "{{ awx_controller_password }}" # pragma: allowlist secret + controller_username: admin + credential_type: Dynatrace + description: Dynatrace DynaKube manifest (contains Secret and DynaKube CR) + inputs: "{{ awx_dynatrace_inputs if awx_dynatrace_enabled else omit }}" + name: Dynatrace + organization: ITBench + state: "{{ 'present' if awx_dynatrace_enabled else 'absent' }}" + vars: + awx_dynatrace_inputs: + dynakube: "{{ awx_configuration.vendors.dynatrace.dynakube | ansible.builtin.default('') }}" # pragma: allowlist secret + no_log: true + - name: Create credentials for GitHub repositories awx.awx.credential: controller_host: "{{ awx_controller_host }}" diff --git a/scenarios/sre/project/roles/awx/tasks/configure_jobs.yaml b/scenarios/sre/project/roles/awx/tasks/configure_jobs.yaml index 57433afbe..5a3aa1d53 100644 --- a/scenarios/sre/project/roles/awx/tasks/configure_jobs.yaml +++ b/scenarios/sre/project/roles/awx/tasks/configure_jobs.yaml @@ -1,4 +1,35 @@ --- +# Build a NON-SECRET projection of the vendor configuration for job extra_vars +# using an ALLOWLIST per vendor. Only explicitly-listed non-secret keys are +# forwarded; anything else (including future secret fields like token/license_key) +# is dropped by default. Secret inputs are delivered separately through vendor +# credentials attached to the tools nodes. +- name: Build non-secret observability vendor configuration for job extra vars + ansible.builtin.set_fact: + awx_observability_vendors_config: >- + {{ + dict( + (awx_configuration.vendors | ansible.builtin.default({})).keys() + | zip( + (awx_configuration.vendors | ansible.builtin.default({})).keys() + | map('extract', awx_configuration.vendors | ansible.builtin.default({})) + | map('dict2items') + | map('selectattr', 'key', 'in', awx_vendor_nonsecret_allowlist) + | map('list') | map('items2dict') + ) + ) + }} + vars: + # Allowlist of non-secret config keys that may be forwarded as job extra_vars. + awx_vendor_nonsecret_allowlist: + - enabled + - manifest_path + - secret_name + - api_key_name + - app_key_name + - dynakube_path + - dynakube_api_version + - name: Add ITBench job templates awx.awx.job_template: allow_simultaneous: true @@ -16,13 +47,20 @@ playbook: "{{ job_template.playbook }}" project: GitHub-ITBench state: present + extra_vars: "{{ job_template.extra_vars | ansible.builtin.default(omit) }}" loop: - name: Deploy-Tools playbook: scenarios/sre/project/manage_tools.yaml job_tags: install_tools + # Non-secret observability vendor configuration (enable flags, sites, + # manifest paths). Secrets are injected separately via vendor credentials. + extra_vars: + observability_vendors: "{{ awx_observability_vendors_config }}" - name: Undeploy-Tools playbook: scenarios/sre/project/manage_tools.yaml job_tags: uninstall_tools + extra_vars: + observability_vendors: "{{ awx_observability_vendors_config }}" - name: Deploy-Applications playbook: scenarios/sre/project/manage_applications.yaml job_tags: install_applications diff --git a/scenarios/sre/project/roles/awx/tasks/configure_workflows.yaml b/scenarios/sre/project/roles/awx/tasks/configure_workflows.yaml index 4b8b68b4e..a93e80474 100644 --- a/scenarios/sre/project/roles/awx/tasks/configure_workflows.yaml +++ b/scenarios/sre/project/roles/awx/tasks/configure_workflows.yaml @@ -14,6 +14,10 @@ lookup("ansible.builtin.template", "templates/awx/workflows/node_credentials.j2") | ansible.builtin.from_yaml ), + "node_tools_credentials": ( + lookup("ansible.builtin.template", "templates/awx/workflows/node_tools_credentials.j2") | + ansible.builtin.from_yaml + ), "runner_id": kubeconfig_index + 1, "scenario_id": scenario.id } diff --git a/scenarios/sre/project/roles/awx/templates/awx/workflows/node_tools_credentials.j2 b/scenarios/sre/project/roles/awx/templates/awx/workflows/node_tools_credentials.j2 new file mode 100644 index 000000000..21ba19d5a --- /dev/null +++ b/scenarios/sre/project/roles/awx/templates/awx/workflows/node_tools_credentials.j2 @@ -0,0 +1,11 @@ +--- +{% if awx_configuration.providers.aws.enabled | ansible.builtin.default(false) %} + - name: AWS +{% endif %} +{% if awx_configuration.vendors.datadog.enabled | ansible.builtin.default(false) %} + - name: Datadog +{% endif %} +{% if awx_configuration.vendors.dynatrace.enabled | ansible.builtin.default(false) %} + - name: Dynatrace +{% endif %} + - name: Cluster-{{ kubeconfig_index + 1 }}-Kubeconfig diff --git a/scenarios/sre/project/roles/awx/templates/awx/workflows/nodes/scenario_execution.j2 b/scenarios/sre/project/roles/awx/templates/awx/workflows/nodes/scenario_execution.j2 index 984e48329..f7ce4580d 100644 --- a/scenarios/sre/project/roles/awx/templates/awx/workflows/nodes/scenario_execution.j2 +++ b/scenarios/sre/project/roles/awx/templates/awx/workflows/nodes/scenario_execution.j2 @@ -2,7 +2,7 @@ {% if workflow_configuration[0].suffix != "Partial-Execution-Stop" %} - identifier: node-deploy-tools related: - credentials: {{ workflow_configuration[1].node_credentials }} + credentials: {{ workflow_configuration[1].node_tools_credentials }} {% if not workflow_configuration[0].is_leaderboard_workflow %} failure_nodes: - identifier: node-undeploy-tools diff --git a/scenarios/sre/project/roles/awx/templates/meta/argument_specs.j2 b/scenarios/sre/project/roles/awx/templates/meta/argument_specs.j2 index ce451f96f..6c1fe8d36 100644 --- a/scenarios/sre/project/roles/awx/templates/meta/argument_specs.j2 +++ b/scenarios/sre/project/roles/awx/templates/meta/argument_specs.j2 @@ -85,6 +85,59 @@ argument_specs: type: dict required: false type: dict + vendors: + options: + datadog: + options: + enabled: + default: false + required: false + type: bool + manifest_path: + required: false + type: str + secret_name: + required: false + type: str + api_key_name: + required: false + type: str + app_key_name: + required: false + type: str + manifest: + required: false + type: str + api_key: + no_log: true + required: false + type: str + app_key: + no_log: true + required: false + type: str + required: false + type: dict + dynatrace: + options: + enabled: + default: false + required: false + type: bool + dynakube_path: + required: false + type: str + dynakube_api_version: + required: false + type: str + dynakube: + no_log: true + required: false + type: str + required: false + type: dict + required: false + type: dict scenarios: elements: dict options: diff --git a/scenarios/sre/project/roles/tools/molecule/deployment_vendors/converge.yml b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/converge.yml new file mode 100644 index 000000000..5c67b7e5f --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/converge.yml @@ -0,0 +1,8 @@ +--- +- name: Skip convergence + hosts: + - localhost + tasks: + - name: Display message + ansible.builtin.debug: + msg: "Deployment completed in create phase. Proceeding to verification." diff --git a/scenarios/sre/project/roles/tools/molecule/deployment_vendors/create.yml b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/create.yml new file mode 100644 index 000000000..cd52b0eab --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/create.yml @@ -0,0 +1,56 @@ +--- +- name: Install observability vendor agents + hosts: + - localhost + tasks: + - name: Resolve vendor manifest paths from environment + ansible.builtin.set_fact: + molecule_datadog_manifest: "{{ lookup('ansible.builtin.env', 'DATADOG_MANIFEST_PATH') }}" + molecule_dynatrace_dynakube: "{{ lookup('ansible.builtin.env', 'DYNATRACE_DYNAKUBE_PATH') }}" + + # The vendor scenarios require real SaaS credentials and manifests. Skip the + # whole run cleanly when they are not supplied so the scenario is safe in CI. + - name: Skip when no vendor credentials are provided + ansible.builtin.meta: end_play + when: + - molecule_datadog_manifest | ansible.builtin.length == 0 + - molecule_dynatrace_dynakube | ansible.builtin.length == 0 + + - name: Import cluster role to set platform and provider variables + ansible.builtin.import_role: + name: cluster + vars: + cluster_files: + kubeconfig: "{{ cluster.kubeconfig }}" + + - name: Import tools role + ansible.builtin.import_role: + name: tools + vars: + tools_cluster: + kubeconfig: "{{ cluster.kubeconfig }}" + platform: "{{ cluster_platform }}" + provider: "{{ cluster_provider }}" + tools_configuration: + sre: + enabled: false + finops: + enabled: false + vendors: + datadog: >- + {{ + { + 'enabled': molecule_datadog_manifest | ansible.builtin.length > 0, + 'manifest_path': molecule_datadog_manifest, + 'secret_name': 'datadog-secret', + 'api_key_name': 'api-key', + 'app_key_name': 'app-key' + } + }} + dynatrace: >- + {{ + { + 'enabled': molecule_dynatrace_dynakube | ansible.builtin.length > 0, + 'dynakube_path': molecule_dynatrace_dynakube + } + }} diff --git a/scenarios/sre/project/roles/tools/molecule/deployment_vendors/destroy.yml b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/destroy.yml new file mode 100644 index 000000000..bea2e00bc --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/destroy.yml @@ -0,0 +1,36 @@ +--- +- name: Uninstall observability vendor agents only + hosts: + - localhost + tasks: + - name: Import cluster role to set platform and provider variables + ansible.builtin.import_role: + name: cluster + vars: + cluster_files: + kubeconfig: "{{ cluster.kubeconfig }}" + + # Invoke ONLY the vendor uninstall task files so this scenario never removes + # unrelated tools (Prometheus, Istio, cert-manager, etc.). Vendor uninstall is + # idempotent and credential-independent, so it safely no-ops when a vendor was + # never installed. + - name: Set tools role variables for vendor uninstall + ansible.builtin.set_fact: + tools_cluster: + kubeconfig: "{{ cluster.kubeconfig }}" + platform: "{{ cluster_platform }}" + provider: "{{ cluster_provider }}" + tools_configuration: + vendors: + dynatrace: "{{ observability_vendors.dynatrace | ansible.builtin.default({}) }}" + datadog: "{{ observability_vendors.datadog | ansible.builtin.default({}) }}" + + - name: Uninstall Dynatrace vendor agent + ansible.builtin.include_role: + name: tools + tasks_from: uninstall_dynatrace.yaml + + - name: Uninstall Datadog vendor agent + ansible.builtin.include_role: + name: tools + tasks_from: uninstall_datadog.yaml diff --git a/scenarios/sre/project/roles/tools/molecule/deployment_vendors/molecule.yml b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/molecule.yml new file mode 100644 index 000000000..936643347 --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/molecule.yml @@ -0,0 +1,27 @@ +--- +dependency: + name: galaxy + +ansible: + env: + ANSIBLE_ROLES_PATH: ../../.. + executor: + backend: ansible-playbook + args: + ansible_playbook: + - --inventory=${MOLECULE_PROJECT_DIRECTORY}/../../../inventory + - --tags=install_tools,install_vendors,untagged + playbooks: + converge: converge.yml + create: create.yml + destroy: destroy.yml + verify: verify.yml + +scenario: + name: deployment_vendors + test_sequence: + - syntax + - create + - converge + - verify + - destroy diff --git a/scenarios/sre/project/roles/tools/molecule/deployment_vendors/verify.yml b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/verify.yml new file mode 100644 index 000000000..7bf519705 --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/deployment_vendors/verify.yml @@ -0,0 +1,102 @@ +--- +- name: Verify observability vendor agents were installed + hosts: + - localhost + tasks: + - name: Resolve vendor manifest paths from environment + ansible.builtin.set_fact: + molecule_datadog_manifest: "{{ lookup('ansible.builtin.env', 'DATADOG_MANIFEST_PATH') }}" + molecule_dynatrace_dynakube: "{{ lookup('ansible.builtin.env', 'DYNATRACE_DYNAKUBE_PATH') }}" + + - name: Skip verification when no vendor credentials were provided + ansible.builtin.meta: end_play + when: + - molecule_datadog_manifest | ansible.builtin.length == 0 + - molecule_dynatrace_dynakube | ansible.builtin.length == 0 + + - name: Include vendor registry variables + ansible.builtin.include_vars: + file: |- + {{ + [ + playbook_dir, "..", "..", "..", "tools", "defaults", "main", "vendors.yaml" + ] | ansible.builtin.path_join + }} + + - name: Verify Datadog + when: + - molecule_datadog_manifest | ansible.builtin.length > 0 + block: + - name: Retrieve Datadog namespace + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + name: "{{ tools_vendors.datadog.kubernetes.namespace }}" + kubeconfig: "{{ cluster.kubeconfig }}" + register: molecule_datadog_ns + + - name: Validate Datadog namespace exists and is ITBench-owned + ansible.builtin.assert: + that: + - molecule_datadog_ns.resources | ansible.builtin.length == 1 + - >- + molecule_datadog_ns.resources[0].metadata.labels['app.kubernetes.io/managed-by'] | ansible.builtin.default('') + == 'ITBench' + fail_msg: Datadog namespace missing or not ITBench-owned. + success_msg: Datadog namespace present and ITBench-owned. + + - name: Retrieve ITBench-owned DatadogAgent resources + kubernetes.core.k8s_info: + api_version: datadoghq.com/v2alpha1 + kind: DatadogAgent + namespace: "{{ tools_vendors.datadog.kubernetes.namespace }}" + label_selectors: + - itbench.io/observability-vendor=datadog + kubeconfig: "{{ cluster.kubeconfig }}" + register: molecule_datadog_agents + + - name: Validate a DatadogAgent resource is present + ansible.builtin.assert: + that: + - molecule_datadog_agents.resources | ansible.builtin.length == 1 + fail_msg: DatadogAgent resource not found. Installation failed. + success_msg: DatadogAgent resource found. + + - name: Verify Dynatrace + when: + - molecule_dynatrace_dynakube | ansible.builtin.length > 0 + block: + - name: Retrieve Dynatrace namespace + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + name: "{{ tools_vendors.dynatrace.kubernetes.namespace }}" + kubeconfig: "{{ cluster.kubeconfig }}" + register: molecule_dynatrace_ns + + - name: Validate Dynatrace namespace exists and is ITBench-owned + ansible.builtin.assert: + that: + - molecule_dynatrace_ns.resources | ansible.builtin.length == 1 + - >- + molecule_dynatrace_ns.resources[0].metadata.labels['app.kubernetes.io/managed-by'] | ansible.builtin.default('') + == 'ITBench' + fail_msg: Dynatrace namespace missing or not ITBench-owned. + success_msg: Dynatrace namespace present and ITBench-owned. + + - name: Retrieve ITBench-owned DynaKube resources + kubernetes.core.k8s_info: + api_version: dynatrace.com/v1beta3 + kind: DynaKube + namespace: "{{ tools_vendors.dynatrace.kubernetes.namespace }}" + label_selectors: + - itbench.io/observability-vendor=dynatrace + kubeconfig: "{{ cluster.kubeconfig }}" + register: molecule_dynatrace_dynakubes + + - name: Validate a DynaKube resource is present + ansible.builtin.assert: + that: + - molecule_dynatrace_dynakubes.resources | ansible.builtin.length == 1 + fail_msg: DynaKube resource not found. Installation failed. + success_msg: DynaKube resource found. diff --git a/scenarios/sre/project/roles/tools/molecule/vendor_contract/converge.yml b/scenarios/sre/project/roles/tools/molecule/vendor_contract/converge.yml new file mode 100644 index 000000000..42556fa46 --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/vendor_contract/converge.yml @@ -0,0 +1,120 @@ +--- +# Credential-free, cluster-free contract tests for vendor manifest validation. +# Exercises the extracted validation task against fixture manifests and asserts +# that valid input passes and invalid input is rejected. +- name: Vendor manifest validation contract tests + hosts: + - localhost + gather_facts: false + vars: + tools_datadog_namespace: datadog + tools_datadog_secret_name: datadog-secret + tools_datadog_api_key_name: api-key + tools_datadog_app_key_name: app-key + validate_task: >- + {{ + [playbook_dir, "..", "..", "tasks", "validate_datadog_manifest.yaml"] + | ansible.builtin.path_join + }} + tasks: + - name: Valid manifest passes validation + block: + - name: Parse valid fixture + ansible.builtin.set_fact: + tools_datadog_documents: "{{ lookup('ansible.builtin.file', 'files/datadog_valid.yaml') | ansible.builtin.from_yaml_all | list }}" + + - name: Validate valid fixture + ansible.builtin.include_tasks: "{{ validate_task }}" + + - name: Assert app-key requirement detected + ansible.builtin.assert: + that: + - tools_datadog_requires_app_key + fail_msg: Valid fixture references appSecret but requirement not detected. + success_msg: Valid manifest accepted and app-key requirement detected. + + - name: Manifest containing a Secret document is rejected + block: + - name: Parse secret-bearing fixture + ansible.builtin.set_fact: + tools_datadog_documents: "{{ lookup('ansible.builtin.file', 'files/datadog_with_secret.yaml') | ansible.builtin.from_yaml_all | list }}" + + - name: Validate secret-bearing fixture (should fail) + ansible.builtin.include_tasks: "{{ validate_task }}" + register: contract_secret_result + + - name: Fail because validation did not reject the Secret document + ansible.builtin.fail: + msg: Manifest containing a Secret document was NOT rejected. + rescue: + - name: Confirm secret-bearing manifest was rejected + ansible.builtin.debug: + msg: Manifest containing a Secret document was correctly rejected. + + - name: Manifest with mismatched Secret references is rejected + block: + - name: Parse mismatched fixture + ansible.builtin.set_fact: + tools_datadog_documents: "{{ lookup('ansible.builtin.file', 'files/datadog_mismatched_secret.yaml') | ansible.builtin.from_yaml_all | list }}" + + - name: Validate mismatched fixture (should fail) + ansible.builtin.include_tasks: "{{ validate_task }}" + + - name: Fail because validation did not reject the mismatch + ansible.builtin.fail: + msg: Manifest with mismatched Secret references was NOT rejected. + rescue: + - name: Confirm mismatched manifest was rejected + ansible.builtin.debug: + msg: Manifest with mismatched Secret references was correctly rejected. + + - name: Non-secret vendor projection excludes secret keys (allowlist) + vars: + awx_configuration: + vendors: + datadog: + enabled: true + manifest_path: p + secret_name: s + api_key_name: a + app_key_name: b + manifest: SECRET + api_key: SECRET + app_key: SECRET + token: FUTURE_SECRET + awx_vendor_nonsecret_allowlist: + - enabled + - manifest_path + - secret_name + - api_key_name + - app_key_name + - dynakube_path + - dynakube_api_version + block: + - name: Build projection + ansible.builtin.set_fact: + projected: >- + {{ + dict( + awx_configuration.vendors.keys() + | zip( + awx_configuration.vendors.keys() + | map('extract', awx_configuration.vendors) + | map('dict2items') + | map('selectattr', 'key', 'in', awx_vendor_nonsecret_allowlist) + | map('list') | map('items2dict') + ) + ) + }} + + - name: Assert secrets are excluded from projection + ansible.builtin.assert: + that: + - "'manifest' not in projected.datadog" + - "'api_key' not in projected.datadog" + - "'app_key' not in projected.datadog" + - "'token' not in projected.datadog" + - projected.datadog.enabled + - projected.datadog.manifest_path == 'p' + fail_msg: Projection leaked a secret field or dropped a non-secret field. + success_msg: Non-secret projection correctly excludes all secret fields. diff --git a/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_mismatched_secret.yaml b/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_mismatched_secret.yaml new file mode 100644 index 000000000..0c2ffadd9 --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_mismatched_secret.yaml @@ -0,0 +1,13 @@ +--- +# Invalid: apiSecret reference does not match the ITBench-created Secret name/key. +kind: "DatadogAgent" +apiVersion: "datadoghq.com/v2alpha1" +metadata: + name: "datadog" + namespace: "datadog" +spec: + global: + credentials: + apiSecret: + secretName: "some-other-secret" + keyName: "wrong-key" diff --git a/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_valid.yaml b/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_valid.yaml new file mode 100644 index 000000000..076531843 --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_valid.yaml @@ -0,0 +1,20 @@ +--- +kind: "DatadogAgent" +apiVersion: "datadoghq.com/v2alpha1" +metadata: + name: "datadog" + namespace: "datadog" +spec: + global: + clusterName: "itbench" + site: "us5.datadoghq.com" + credentials: + apiSecret: + secretName: "datadog-secret" + keyName: "api-key" + appSecret: + secretName: "datadog-secret" + keyName: "app-key" + features: + logCollection: + enabled: true diff --git a/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_with_secret.yaml b/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_with_secret.yaml new file mode 100644 index 000000000..741424f57 --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/vendor_contract/files/datadog_with_secret.yaml @@ -0,0 +1,21 @@ +--- +# Invalid: includes a Secret document (must be rejected - ITBench creates it). +kind: "DatadogAgent" +apiVersion: "datadoghq.com/v2alpha1" +metadata: + name: "datadog" + namespace: "datadog" +spec: + global: + credentials: + apiSecret: + secretName: "datadog-secret" + keyName: "api-key" +--- +apiVersion: v1 +kind: Secret +metadata: + name: "datadog-secret" + namespace: "datadog" +stringData: + api-key: "should-not-be-here" # pragma: allowlist secret diff --git a/scenarios/sre/project/roles/tools/molecule/vendor_contract/molecule.yml b/scenarios/sre/project/roles/tools/molecule/vendor_contract/molecule.yml new file mode 100644 index 000000000..58e1ac6f0 --- /dev/null +++ b/scenarios/sre/project/roles/tools/molecule/vendor_contract/molecule.yml @@ -0,0 +1,14 @@ +--- +dependency: + name: galaxy + +ansible: + env: + ANSIBLE_ROLES_PATH: ../../.. + +scenario: + name: vendor_contract + # Credential-free and cluster-free: only runs the converge contract tests. + test_sequence: + - syntax + - converge From 0f1c231f54fd63947ca3868b80b05a27af024831 Mon Sep 17 00:00:00 2001 From: Bekir Turkkan Date: Mon, 3 Aug 2026 12:35:31 -0400 Subject: [PATCH 06/11] dynatrace exporter initial commit. The functionality tested separately but it needs testing with ITBench scenario running --- .../observability_vendors.yaml.example | 39 +- scenarios/sre/project/manage_recorders.yaml | 11 + .../recorders/defaults/main/managers.yaml | 1 + .../recorders/defaults/main/vendors.yaml | 8 + ...k8s_metrics_2026-07-30T16-48-44.313012.csv | 1 + ...8s_metrics_2026-07-30T16-48-44.313012.json | 139 ++++++ .../files/scripts/dynatrace/gather.py | 467 ++++++++++++++++++ .../files/scripts/dynatrace/requirements.txt | 1 + .../files/scripts/dynatrace/run_local.sh | 118 +++++ .../roles/recorders/meta/argument_specs.yaml | 50 ++ .../roles/recorders/tasks/install.yaml | 10 + .../tasks/install_dynatrace_recorder.yaml | 131 +++++ .../tasks/stamp_dynatrace_start_time.yaml | 34 ++ .../roles/recorders/tasks/uninstall.yaml | 20 + .../sre/project/run_dynatrace_recorder.yaml | 39 ++ 15 files changed, 1066 insertions(+), 3 deletions(-) create mode 100644 scenarios/sre/project/roles/recorders/defaults/main/vendors.yaml create mode 100644 scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.csv create mode 100644 scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.json create mode 100644 scenarios/sre/project/roles/recorders/files/scripts/dynatrace/gather.py create mode 100644 scenarios/sre/project/roles/recorders/files/scripts/dynatrace/requirements.txt create mode 100755 scenarios/sre/project/roles/recorders/files/scripts/dynatrace/run_local.sh create mode 100644 scenarios/sre/project/roles/recorders/tasks/install_dynatrace_recorder.yaml create mode 100644 scenarios/sre/project/roles/recorders/tasks/stamp_dynatrace_start_time.yaml create mode 100644 scenarios/sre/project/run_dynatrace_recorder.yaml diff --git a/scenarios/sre/inventory/group_vars/environment/observability_vendors.yaml.example b/scenarios/sre/inventory/group_vars/environment/observability_vendors.yaml.example index 4030ca454..4fc6c2499 100644 --- a/scenarios/sre/inventory/group_vars/environment/observability_vendors.yaml.example +++ b/scenarios/sre/inventory/group_vars/environment/observability_vendors.yaml.example @@ -10,9 +10,15 @@ # export DATADOG_API_KEY=... # required # pragma: allowlist secret # export DATADOG_APP_KEY=... # optional (needed for some features) # -# # Dynatrace: no env vars. Download dynakube.yaml from the Dynatrace -# # onboarding UI (contains tokens + DynaKube CR) and reference it via -# # observability_vendors.dynatrace.dynakube_path (see below). +# # Dynatrace agent deploy (Layer A): no env vars. Download dynakube.yaml from +# # the Dynatrace onboarding UI (contains tokens + DynaKube CR) and reference +# # it via observability_vendors.dynatrace.dynakube_path (see below). +# +# # Dynatrace Grail recorder (exports telemetry back out at teardown): its +# # PLATFORM token (prefix dt0s16., Grail read permissions) is read from a +# # gitignored file referenced by dynatrace.recorder.platform_token_path below +# # (same convention as dynakube_path) — no env export needed. This is a +# # DIFFERENT token from the agent ingest tokens in dynakube.yaml. # # Each vendor is disabled by default. Enable a vendor and supply its non-secret # configuration below, then export the matching secret environment variable(s). @@ -41,3 +47,30 @@ observability_vendors: # use this exact apiVersion; it is also used for readiness and cleanup so all # three stay consistent. dynakube_api_version: "dynatrace.com/v1beta3" + # Dynatrace Grail RECORDER (Layer B-ish export). Runs once at teardown as a + # Kubernetes Job that back-queries Grail over [scenario start -> now] and + # writes JSON+CSV (metrics, spans, logs, events, k8s metrics) into the + # recorder export directory. Independent of the agent deploy above: it only + # reads telemetry, and requires a PLATFORM token exported as DT_PLATFORM_TOKEN. + recorder: + enabled: false + # Dynatrace platform (apps) URL — NOT the live/ingest URL. NON-SECRET. + # e.g. https://.apps.dynatrace.com + platform_url: "" + # Path to a file containing the platform token (prefix dt0s16.) with Grail + # read permissions. SENSITIVE: keep under secrets/ (gitignored). The file + # should contain only the token. + platform_token_path: "secrets/dynatrace/platform_token" # pragma: allowlist secret + # Kubernetes namespace whose telemetry is exported. + namespace: otel-demo + # Query end (Grail relative time or ISO-8601). The start is taken from the + # scenario start time stamped at install; fallback_from is used only if + # that marker is missing. + timeframe_to: now + fallback_from: now-1h + # Timeseries bucket size for metric/span datasets. + interval: 1m + # Row cap for the logs/events datasets. + limit: 1000 + # Output format: json | csv | both. + format: both diff --git a/scenarios/sre/project/manage_recorders.yaml b/scenarios/sre/project/manage_recorders.yaml index 906f4a776..f763d10a7 100644 --- a/scenarios/sre/project/manage_recorders.yaml +++ b/scenarios/sre/project/manage_recorders.yaml @@ -95,3 +95,14 @@ recorders_cluster: kubeconfig: "{{ cluster.kubeconfig }}" platform: "{{ cluster_platform }}" + # Pass ONLY the recorder subset the role declares (not the whole + # observability_vendors blob, which also carries agent-deploy keys the + # recorders role's argument spec does not accept). + recorders_vendors: + dynatrace: + recorder: >- + {{ + (observability_vendors | ansible.builtin.default({})) + .get('dynatrace', {}) + .get('recorder', {'enabled': false}) + }} diff --git a/scenarios/sre/project/roles/recorders/defaults/main/managers.yaml b/scenarios/sre/project/roles/recorders/defaults/main/managers.yaml index 4ec628059..6f7b24646 100644 --- a/scenarios/sre/project/roles/recorders/defaults/main/managers.yaml +++ b/scenarios/sre/project/roles/recorders/defaults/main/managers.yaml @@ -5,6 +5,7 @@ recorders_managers: namespace: data-recorders instances: clickhouse: clickhouse-recorder + dynatrace: dynatrace-recorder jaeger: jaeger-recorder kubernetes_topology_monitor: kubernetes-topology-monitor-recorder prometheus: prometheus-recorder diff --git a/scenarios/sre/project/roles/recorders/defaults/main/vendors.yaml b/scenarios/sre/project/roles/recorders/defaults/main/vendors.yaml new file mode 100644 index 000000000..e9d76c7ed --- /dev/null +++ b/scenarios/sre/project/roles/recorders/defaults/main/vendors.yaml @@ -0,0 +1,8 @@ +--- +# SaaS observability vendor recorder configuration, passed into the role from +# the `observability_vendors` group_var (see manage_recorders.yaml). Defaults to +# disabled so the role is a no-op unless a vendor recorder is explicitly enabled. +recorders_vendors: + dynatrace: + recorder: + enabled: false diff --git a/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.csv b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.csv new file mode 100644 index 000000000..d3f5a12fa --- /dev/null +++ b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.csv @@ -0,0 +1 @@ + diff --git a/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.json b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.json new file mode 100644 index 000000000..5f7a6ae84 --- /dev/null +++ b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.json @@ -0,0 +1,139 @@ +{ + "dataset": "k8s-metrics", + "namespace": "otel-demo", + "timeframe": { + "from": "now-1h", + "to": "now" + }, + "interval": "1m", + "metric_count": 14, + "metrics_with_data": 0, + "metrics": [ + { + "metric": "dt.kubernetes.pod.network_received_data", + "aggregation": "sum", + "grouping": "pod", + "query": "timeseries value = sum(dt.kubernetes.pod.network_received_data), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", + "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", + "record_count": 0, + "result": null + }, + { + "metric": "dt.kubernetes.pod.network_received_errors", + "aggregation": "sum", + "grouping": "pod", + "query": "timeseries value = sum(dt.kubernetes.pod.network_received_errors), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", + "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", + "record_count": 0, + "result": null + }, + { + "metric": "dt.kubernetes.pod.network_received_packets_dropped", + "aggregation": "sum", + "grouping": "pod", + "query": "timeseries value = sum(dt.kubernetes.pod.network_received_packets_dropped), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", + "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", + "record_count": 0, + "result": null + }, + { + "metric": "dt.kubernetes.pod.network_transmitted_data", + "aggregation": "sum", + "grouping": "pod", + "query": "timeseries value = sum(dt.kubernetes.pod.network_transmitted_data), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", + "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", + "record_count": 0, + "result": null + }, + { + "metric": "dt.kubernetes.pod.network_transmitted_errors", + "aggregation": "sum", + "grouping": "pod", + "query": "timeseries value = sum(dt.kubernetes.pod.network_transmitted_errors), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", + "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", + "record_count": 0, + "result": null + }, + { + "metric": "dt.kubernetes.pod.network_transmitted_packets_dropped", + "aggregation": "sum", + "grouping": "pod", + "query": "timeseries value = sum(dt.kubernetes.pod.network_transmitted_packets_dropped), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", + "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", + "record_count": 0, + "result": null + }, + { + "metric": "dt.kubernetes.pod.containers_desired", + "aggregation": "max", + "grouping": "pod", + "query": "timeseries value = max(dt.kubernetes.pod.containers_desired), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", + "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", + "record_count": 0, + "result": null + }, + { + "metric": "dt.kubernetes.pod.restarts", + "aggregation": "max", + "grouping": "pod", + "query": "timeseries value = max(dt.kubernetes.pod.restarts), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", + "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", + "record_count": 0, + "result": null + }, + { + "metric": "dt.kubernetes.container.requests_cpu", + "aggregation": "max", + "grouping": "container", + "query": "timeseries value = max(dt.kubernetes.container.requests_cpu), by:{k8s.pod.name, k8s.container.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", + "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", + "record_count": 0, + "result": null + }, + { + "metric": "dt.kubernetes.container.limits_cpu", + "aggregation": "max", + "grouping": "container", + "query": "timeseries value = max(dt.kubernetes.container.limits_cpu), by:{k8s.pod.name, k8s.container.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", + "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", + "record_count": 0, + "result": null + }, + { + "metric": "dt.kubernetes.container.requests_memory", + "aggregation": "max", + "grouping": "container", + "query": "timeseries value = max(dt.kubernetes.container.requests_memory), by:{k8s.pod.name, k8s.container.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", + "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", + "record_count": 0, + "result": null + }, + { + "metric": "dt.kubernetes.container.limits_memory", + "aggregation": "max", + "grouping": "container", + "query": "timeseries value = max(dt.kubernetes.container.limits_memory), by:{k8s.pod.name, k8s.container.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", + "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", + "record_count": 0, + "result": null + }, + { + "metric": "dt.kubernetes.container.cpu_usage", + "aggregation": "avg", + "grouping": "container", + "query": "timeseries value = avg(dt.kubernetes.container.cpu_usage), by:{k8s.pod.name, k8s.container.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", + "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", + "record_count": 0, + "result": null + }, + { + "metric": "dt.kubernetes.container.memory_working_set", + "aggregation": "avg", + "grouping": "container", + "query": "timeseries value = avg(dt.kubernetes.container.memory_working_set), by:{k8s.pod.name, k8s.container.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", + "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", + "record_count": 0, + "result": null + } + ] +} \ No newline at end of file diff --git a/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/gather.py b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/gather.py new file mode 100644 index 000000000..e03d0d507 --- /dev/null +++ b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/gather.py @@ -0,0 +1,467 @@ +import csv +import datetime +import json +import logging +import os +import re +import sys +import time + +from datetime import datetime, timedelta, timezone + +import requests + +from requests.adapters import HTTPAdapter +from urllib3.util import Retry + +# Logging +logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + +logger = logging.getLogger(__name__) + +# NOTE: This recorder is the SaaS counterpart to the in-cluster scrapers +# (clickhouse/jaeger/prometheus). Dynatrace Grail retains history that is +# queryable by timeframe, so a single run at teardown back-queries the +# whole incident window [scenario start -> now] for every dataset. It is +# adapted from dynatrace_scripts/pull_dql_json.py: the CLI, .env, and +# dotenv dependency are removed and all configuration comes from +# environment variables injected by the recorder Job. + +# ── Config (from environment) ──────────────────────────────────────────────── +PLATFORM_TOKEN = os.environ.get("DT_PLATFORM_TOKEN", "") +PLATFORM_URL = os.environ.get("DT_PLATFORM_URL", "").rstrip("/") + +NAMESPACE = os.environ.get("DT_K8S_NAMESPACE", "otel-demo").strip() +TIME_FROM = os.environ.get("DT_DQL_FROM", "now-1h") +TIME_TO = os.environ.get("DT_DQL_TO", "now") +INTERVAL = os.environ.get("DT_DQL_INTERVAL", "1m") +LIMIT = os.environ.get("DT_DQL_LIMIT", "1000").strip() +OUTPUT_FORMAT = os.environ.get("DT_DQL_FORMAT", "both").strip() + +# All records are written into the recorder PVC (~/records), matching the +# convention used by the jaeger/prometheus recorders. +OUTDIR = os.path.join(os.path.expanduser("~"), "records") + +# ── Kubernetes metric catalog ──────────────────────────────────────────────── +# Each entry: (metric_key, aggregation, grouping). These are queried ONE AT A +# TIME and merged into a single output file (see the "k8s-metrics" dataset). +# Querying individually — rather than one combined `timeseries {a, b, c}` — is +# deliberate: in a combined block a single metric with no data collapses the +# whole result to 0 rows. Per-metric queries keep every populated metric. +# +# grouping "pod" -> by:{k8s.pod.name, k8s.namespace.name} +# grouping "container" -> by:{k8s.pod.name, k8s.container.name, k8s.namespace.name} +K8S_METRICS = [ + # Pod network + ("dt.kubernetes.pod.network_received_data", "sum", "pod"), + ("dt.kubernetes.pod.network_received_errors", "sum", "pod"), + ("dt.kubernetes.pod.network_received_packets_dropped", "sum", "pod"), + ("dt.kubernetes.pod.network_transmitted_data", "sum", "pod"), + ("dt.kubernetes.pod.network_transmitted_errors", "sum", "pod"), + ("dt.kubernetes.pod.network_transmitted_packets_dropped", "sum", "pod"), + # Pod status + ("dt.kubernetes.pod.containers_desired", "max", "pod"), + ("dt.kubernetes.pod.restarts", "max", "pod"), + # Container resources + ("dt.kubernetes.container.requests_cpu", "max", "container"), + ("dt.kubernetes.container.limits_cpu", "max", "container"), + ("dt.kubernetes.container.requests_memory", "max", "container"), + ("dt.kubernetes.container.limits_memory", "max", "container"), + ("dt.kubernetes.container.cpu_usage", "avg", "container"), + ("dt.kubernetes.container.memory_working_set", "avg", "container"), +] +K8S_GROUP_BY = { + "pod": "k8s.pod.name, k8s.namespace.name", + "container": "k8s.pod.name, k8s.container.name, k8s.namespace.name", +} + +# ── Dataset registry ───────────────────────────────────────────────────────── +# Each dataset maps to an output file stem. A timestamp is appended at write +# time so a run produces e.g. dynatrace_logs_2026-07-30T12-00-00.000000.json. +DATASETS = { + "responsetime": {"stem": "dynatrace_responsetime"}, + "errorrate": {"stem": "dynatrace_errorrate"}, + "span-responsetime": {"stem": "dynatrace_span_responsetime"}, + "span-errorrate": {"stem": "dynatrace_span_errorrate"}, + "logs": {"stem": "dynatrace_logs"}, + "events": {"stem": "dynatrace_events"}, + # Special: runs every K8S_METRICS key and merges them into one file. + "k8s-metrics": {"stem": "dynatrace_k8s_metrics", "multi": True}, +} +# The recorder always exports the full set (metrics, spans, logs, events, k8s). +ALL_DATASETS = [ + "responsetime", "errorrate", "span-responsetime", "span-errorrate", + "logs", "events", "k8s-metrics", +] +# ───────────────────────────────────────────────────────────────────────────── + + +def build_session(): + retries = Retry(total=3, backoff_factor=0.3) + adapter = HTTPAdapter(max_retries=retries) + session = requests.Session() + session.mount("http://", adapter) + session.mount("https://", adapter) + return session + + +def validate_config(): + missing = [ + name + for name, val in [ + ("DT_PLATFORM_TOKEN", PLATFORM_TOKEN), + ("DT_PLATFORM_URL", PLATFORM_URL), + ] + if not val + ] + if missing: + sys.exit( + "error: missing required environment variable(s): {0}".format( + ", ".join(missing) + ) + ) + + +def build_dql(dataset, namespace, interval, limit): + """Return the DQL for a dataset, scoped to the namespace. + + logs/events fetch RAW rows (no projection) so the output keeps every field. + The metric/span datasets aggregate, which is intrinsic to the measurement. + """ + ns_filter = f'filter: {{ k8s.namespace.name == "{namespace}" }}' + if dataset == "errorrate": + return ( + "timeseries {" + " total = sum(dt.service.request.count)," + " failed = sum(dt.service.request.failure_count)" + f" }}, by:{{dt.entity.service}}, {ns_filter}, interval:{interval}" + "| fieldsAdd error_rate = if(arraySum(total) > 100," + " (failed[] / total[]) * 100, else: 0)" + ) + if dataset == "responsetime": + return ( + "timeseries responsetime = avg(dt.service.request.response_time)," + f" by:{{dt.entity.service}}, {ns_filter}, interval:{interval}" + ) + if dataset == "span-responsetime": + return ( + f'fetch spans | filter k8s.namespace.name == "{namespace}"' + " and request.is_root_span == true" + f" | makeTimeseries rt_ns = avg(duration), by:{{service.name}}," + f" interval:{interval}" + " | fieldsAdd responsetime_ms = rt_ns[] / 1000000.0" + ) + if dataset == "span-errorrate": + return ( + f'fetch spans | filter k8s.namespace.name == "{namespace}"' + " and request.is_root_span == true" + " | fieldsAdd failed_num = if(request.is_failed == true, 1, else: 0)" + f" | makeTimeseries err = avg(failed_num), by:{{service.name}}," + f" interval:{interval}" + " | fieldsAdd error_rate = err[] * 100.0" + ) + if dataset == "logs": + return ( + f'fetch logs | filter k8s.namespace.name == "{namespace}"' + f" | sort timestamp desc | limit {limit}" + ) + if dataset == "events": + return ( + f'fetch events | filter k8s.namespace.name == "{namespace}"' + f" | sort timestamp desc | limit {limit}" + ) + raise ValueError(f"Unknown dataset '{dataset}'") + + +def build_metric_dql(metric_key, agg, grouping, namespace, interval): + """DQL for a single k8s metric timeseries, scoped to the namespace.""" + by = K8S_GROUP_BY[grouping] + return ( + f"timeseries value = {agg}({metric_key}), by:{{{by}}}," + f' filter:{{ k8s.namespace.name == "{namespace}" }}, interval:{interval}' + ) + + +def to_iso(value): + """Convert `now` / `now-` to ISO-8601 UTC (Grail rejects relative + strings in defaultTimeframeStart/End). ISO input is passed through, so a + scenario-start timestamp supplied via DT_DQL_FROM works directly.""" + value = value.strip() + now = datetime.now(timezone.utc) + if value == "now": + dt = now + else: + m = re.fullmatch(r"now-(\d+)([mhd])", value) + if not m: + return value + n, unit = int(m.group(1)), m.group(2) + delta = {"m": "minutes", "h": "hours", "d": "days"}[unit] + dt = now - timedelta(**{delta: n}) + return dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + +def _raise_with_body(resp): + """raise_for_status but surface the API error body (Grail puts the reason — + INVALID_TIMEFRAME, token/permission errors — in the JSON).""" + if resp.status_code >= 400: + try: + detail = resp.json() + except ValueError: + detail = resp.text + logger.warning("HTTP %s from %s\n%s", resp.status_code, resp.url, detail) + resp.raise_for_status() + + +def run_query(session, dql, time_from, time_to): + """Execute DQL with the platform token; poll until the query completes. + Returns the full `result` object (records + metadata).""" + headers = { + "Authorization": "Bearer {0}".format(PLATFORM_TOKEN), + "Content-Type": "application/json", + "Accept": "application/json", + } + exec_resp = session.post( + "{0}/platform/storage/query/v1/query:execute".format(PLATFORM_URL), + headers=headers, + json={"query": dql, "defaultTimeframeStart": to_iso(time_from), + "defaultTimeframeEnd": to_iso(time_to)}, + timeout=60, + ) + _raise_with_body(exec_resp) + body = exec_resp.json() + + if body.get("state") == "SUCCEEDED" and "result" in body: + return body["result"] + + request_token = body["requestToken"] + poll_url = "{0}/platform/storage/query/v1/query:poll".format(PLATFORM_URL) + while True: + poll = session.get( + poll_url, headers=headers, + params={"request-token": request_token}, timeout=60, + ) + _raise_with_body(poll) + pbody = poll.json() + state = pbody.get("state") + if state == "SUCCEEDED": + return pbody["result"] + if state in ("FAILED", "CANCELLED"): + raise RuntimeError("Query {0}: {1}".format(state, pbody)) + time.sleep(2) + + +def grail_notifications(result): + return [ + n.get("message", "") + for n in result.get("metadata", {}).get("grail", {}).get("notifications", []) + if n.get("message") + ] + + +# ── CSV flattening ─────────────────────────────────────────────────────────── +def _parse_iso(s): + """Parse Grail ISO-8601 (nanosecond precision) into a UTC datetime.""" + m = re.match(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d+))?", s) + base = datetime.strptime(m.group(1), "%Y-%m-%dT%H:%M:%S") + if m.group(2): + base = base.replace(microsecond=int(m.group(2)[:6].ljust(6, "0"))) + return base.replace(tzinfo=timezone.utc) + + +def _series_timestamps(rec, n): + """Per-bucket timestamps for a timeseries record: use explicit timestamps[] + if present, else derive from timeframe.start + interval.""" + tf = rec.get("timeframe", {}) or {} + if tf.get("timestamps"): + return tf["timestamps"] + start, interval_ns = tf.get("start"), rec.get("interval") + if start and interval_ns: + base = _parse_iso(start) + step = timedelta(microseconds=int(interval_ns) / 1000) + return [(base + i * step).strftime("%Y-%m-%dT%H:%M:%S.000Z") + for i in range(n)] + return list(range(n)) + + +def flatten_result(result, extra=None): + """Flatten a Grail result into a list of flat dict rows for CSV. + + - Flat records (logs/events): one row per record, scalar fields as-is. + - Timeseries records (value arrays + timeframe/interval): expanded to one + row per bucket with a reconstructed `timestamp` column. + `extra` adds constant columns to every row (e.g. the metric key).""" + rows = [] + extra = extra or {} + for rec in result.get("records", []): + array_fields = {k: v for k, v in rec.items() + if isinstance(v, list) and k != "timestamps"} + scalar_fields = {k: v for k, v in rec.items() + if k not in array_fields and k not in + ("timeframe", "interval")} + + if array_fields: + length = max((len(v) for v in array_fields.values()), default=0) + timestamps = _series_timestamps(rec, length) + for i in range(length): + if all(v[i] is None for v in array_fields.values() + if i < len(v)): + continue # skip buckets where every series is null + row = dict(extra) + row["timestamp"] = timestamps[i] if i < len(timestamps) else i + row.update(scalar_fields) + for k, v in array_fields.items(): + row[k] = v[i] if i < len(v) else None + rows.append(row) + else: + row = dict(extra) + row.update(scalar_fields) + rows.append(row) + return rows + + +def write_csv_rows(rows, path): + """Write flat dict rows to CSV; columns = union of keys across rows.""" + columns = [] + for r in rows: + for k in r: + if k not in columns: + columns.append(k) + with open(path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=columns) + writer.writeheader() + writer.writerows(rows) + return len(rows) + + +def out_paths(dataset, timestamp): + """(json_path, csv_path) for a dataset, with the run timestamp appended.""" + stem = "{0}_{1}".format(DATASETS[dataset]["stem"], timestamp) + base = os.path.join(OUTDIR, stem) + return base + ".json", base + ".csv" + + +def export_k8s_metrics(session, dataset, timestamp): + """Query every K8S_METRICS key individually and merge them into ONE file — + so a single run exports many metrics together. Each metric is a separate + query (a combined `timeseries {..}` would zero out if any one metric had no + data), and each metric's full result is kept under its key.""" + json_path, csv_path = out_paths(dataset, timestamp) + metrics_out = [] + ok = 0 + for metric_key, agg, grouping in K8S_METRICS: + entry = {"metric": metric_key, "aggregation": agg, "grouping": grouping} + dql = build_metric_dql(metric_key, agg, grouping, NAMESPACE, INTERVAL) + try: + result = run_query(session, dql, TIME_FROM, TIME_TO) + except Exception as exc: + logger.warning("[%s] %s: ERROR: %s", dataset, metric_key, exc) + entry.update(query=dql, error=str(exc), record_count=0, result=None) + metrics_out.append(entry) + continue + + for msg in grail_notifications(result): + logger.info("[%s] %s: Grail notice: %s", dataset, metric_key, msg) + n = len(result.get("records", [])) + if n: + ok += 1 + logger.info("[%s] %s: %d records", dataset, metric_key, n) + entry.update(query=dql, record_count=n, result=result) + metrics_out.append(entry) + + written = [] + + if OUTPUT_FORMAT in ("json", "both"): + payload = { + "dataset": dataset, + "namespace": NAMESPACE, + "timeframe": {"from": TIME_FROM, "to": TIME_TO}, + "interval": INTERVAL, + "metric_count": len(metrics_out), + "metrics_with_data": ok, + "metrics": metrics_out, + } + with open(json_path, "w") as f: + json.dump(payload, f, indent=2) + written.append(json_path) + + if OUTPUT_FORMAT in ("csv", "both"): + # Merge every metric into one CSV, tagging each row with its metric key. + rows = [] + for entry in metrics_out: + if entry.get("result"): + rows.extend(flatten_result(entry["result"], + extra={"metric": entry["metric"]})) + write_csv_rows(rows, csv_path) + written.append(csv_path) + + logger.info("[%s] wrote %d metrics (%d with data) -> %s", + dataset, len(metrics_out), ok, ", ".join(written)) + return ok > 0 + + +def export_dataset(session, dataset, timestamp): + """Run one dataset and write its output. Returns True if it produced data. + Never raises — failures are captured so a multi-dataset run continues.""" + if DATASETS[dataset].get("multi"): + return export_k8s_metrics(session, dataset, timestamp) + + json_path, csv_path = out_paths(dataset, timestamp) + try: + dql = build_dql(dataset, NAMESPACE, INTERVAL, LIMIT) + logger.info("[%s] %s", dataset, dql) + result = run_query(session, dql, TIME_FROM, TIME_TO) + except Exception as exc: + logger.warning("[%s] ERROR: %s", dataset, exc) + return False + + for msg in grail_notifications(result): + logger.info("[%s] Grail notice: %s", dataset, msg) + + records = result.get("records", []) + written = [] + + if OUTPUT_FORMAT in ("json", "both"): + # Persist the full response verbatim: query context + records + metadata. + payload = { + "dataset": dataset, + "query": dql, + "timeframe": {"from": TIME_FROM, "to": TIME_TO}, + "record_count": len(records), + "result": result, + } + with open(json_path, "w") as f: + json.dump(payload, f, indent=2) + written.append(json_path) + + if OUTPUT_FORMAT in ("csv", "both"): + rows = flatten_result(result) + write_csv_rows(rows, csv_path) + written.append(csv_path) + + logger.info("[%s] wrote %d records -> %s", + dataset, len(records), ", ".join(written)) + return len(records) > 0 + + +def main(): + validate_config() + + if OUTPUT_FORMAT not in ("json", "csv", "both"): + sys.exit("error: DT_DQL_FORMAT must be one of json, csv, both") + + os.makedirs(OUTDIR, exist_ok=True) + session = build_session() + + timestamp = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H-%M-%S.%f') + logger.info("Datasets: %s | namespace=%s | %s -> %s | format=%s", + ", ".join(ALL_DATASETS), NAMESPACE, TIME_FROM, TIME_TO, + OUTPUT_FORMAT) + + results = [export_dataset(session, ds, timestamp) for ds in ALL_DATASETS] + + if not any(results): + logger.warning("no datasets returned any records") + + +if __name__ == "__main__": + main() diff --git a/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/requirements.txt b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/requirements.txt new file mode 100644 index 000000000..a258782f6 --- /dev/null +++ b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/requirements.txt @@ -0,0 +1 @@ +requests==2.34.2 diff --git a/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/run_local.sh b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/run_local.sh new file mode 100755 index 000000000..eb733a9fe --- /dev/null +++ b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/run_local.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# +# Run the Dynatrace Grail recorder (gather.py) locally, against the Dynatrace +# SaaS API — no Kubernetes cluster needed. This is the manual counterpart to the +# in-cluster recorder Job: identical script, output written to ./dynatrace-records +# on this machine instead of a PVC. +# +# Config resolution (first non-empty wins): +# 1. environment variables (DT_PLATFORM_URL, DT_PLATFORM_TOKEN, ...) +# 2. the recorder block in observability_vendors.yaml (platform_url, +# platform_token_path, namespace) +# 3. built-in defaults +# +# Usage: +# ./run_local.sh # window = now-1h .. now +# DT_DQL_FROM=2026-07-30T09:00:00Z ./run_local.sh +# DT_DQL_FROM=now-6h DT_DQL_FORMAT=csv ./run_local.sh +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# scripts/dynatrace -> scenarios/sre is six levels up +# (dynatrace/scripts/files/recorders/roles/project -> sre). +SRE_ROOT="$(cd "${SCRIPT_DIR}/../../../../../.." && pwd)" +VENDORS_FILE="${SRE_ROOT}/inventory/group_vars/environment/observability_vendors.yaml" + +# ── Tiny YAML value reader for observability_vendors.dynatrace.recorder. ── +# Only used as a fallback when the matching env var is unset. Uses python3 (yaml +# is available with the project's uv env) and falls back to empty on any error. +vendor_get() { + local key="$1" + python3 - "$VENDORS_FILE" "$key" <<'PY' 2>/dev/null || true +import sys, yaml +path, key = sys.argv[1], sys.argv[2] +try: + with open(path) as f: + data = yaml.safe_load(f) or {} + rec = (((data.get("observability_vendors") or {}) + .get("dynatrace") or {}) + .get("recorder") or {}) + val = rec.get(key, "") + print("" if val is None else val) +except Exception: + print("") +PY +} + +# ── Resolve config ──────────────────────────────────────────────────────────── +DT_PLATFORM_URL="${DT_PLATFORM_URL:-$(vendor_get platform_url)}" +DT_K8S_NAMESPACE="${DT_K8S_NAMESPACE:-$(vendor_get namespace)}" +DT_K8S_NAMESPACE="${DT_K8S_NAMESPACE:-otel-demo}" + +# Token: explicit env wins, else read the file referenced by platform_token_path. +if [ -z "${DT_PLATFORM_TOKEN:-}" ]; then + token_path="$(vendor_get platform_token_path)" + if [ -n "$token_path" ]; then + case "$token_path" in + /*) : ;; # absolute + *) token_path="${SRE_ROOT}/${token_path}" ;; # relative to SRE root + esac + if [ -f "$token_path" ]; then + DT_PLATFORM_TOKEN="$(tr -d '\r\n' < "$token_path")" + fi + fi +fi + +DT_DQL_FROM="${DT_DQL_FROM:-now-1h}" +DT_DQL_TO="${DT_DQL_TO:-now}" +DT_DQL_INTERVAL="${DT_DQL_INTERVAL:-1m}" +DT_DQL_LIMIT="${DT_DQL_LIMIT:-1000}" +DT_DQL_FORMAT="${DT_DQL_FORMAT:-both}" + +# Output goes to a local directory the script owns (HOME/records is what +# gather.py writes to, so point HOME at our output dir for this run). +OUTDIR="${DT_LOCAL_OUTDIR:-${SCRIPT_DIR}/dynatrace-records}" + +# ── Validate ──────────────────────────────────────────────────────────────── +missing="" +[ -z "${DT_PLATFORM_URL}" ] && missing="${missing} DT_PLATFORM_URL(or recorder.platform_url)" +[ -z "${DT_PLATFORM_TOKEN}" ] && missing="${missing} DT_PLATFORM_TOKEN(or recorder.platform_token_path)" +if [ -n "$missing" ]; then + echo "error: missing config:${missing}" >&2 + exit 1 +fi + +# ── Python env ──────────────────────────────────────────────────────────────── +VENV="${DT_LOCAL_VENV:-/tmp/dtrec}" +if [ ! -x "${VENV}/bin/python" ]; then + echo "Creating virtualenv at ${VENV} ..." + python3 -m venv "${VENV}" +fi +"${VENV}/bin/pip" install --quiet --disable-pip-version-check -r "${SCRIPT_DIR}/requirements.txt" + +# ── Run ────────────────────────────────────────────────────────────────────── +mkdir -p "${OUTDIR}/records" +echo "Dynatrace recorder (local)" +echo " URL : ${DT_PLATFORM_URL}" +echo " namespace : ${DT_K8S_NAMESPACE}" +echo " window : ${DT_DQL_FROM} -> ${DT_DQL_TO}" +echo " format : ${DT_DQL_FORMAT}" +echo " output : ${OUTDIR}/records" +echo + +# gather.py writes to \$HOME/records; point HOME at OUTDIR just for this process. +HOME="${OUTDIR}" \ +DT_PLATFORM_URL="${DT_PLATFORM_URL}" \ +DT_PLATFORM_TOKEN="${DT_PLATFORM_TOKEN}" \ +DT_K8S_NAMESPACE="${DT_K8S_NAMESPACE}" \ +DT_DQL_FROM="${DT_DQL_FROM}" \ +DT_DQL_TO="${DT_DQL_TO}" \ +DT_DQL_INTERVAL="${DT_DQL_INTERVAL}" \ +DT_DQL_LIMIT="${DT_DQL_LIMIT}" \ +DT_DQL_FORMAT="${DT_DQL_FORMAT}" \ + "${VENV}/bin/python" "${SCRIPT_DIR}/gather.py" + +echo +echo "Done. Files:" +ls -la "${OUTDIR}/records" diff --git a/scenarios/sre/project/roles/recorders/meta/argument_specs.yaml b/scenarios/sre/project/roles/recorders/meta/argument_specs.yaml index eb2b41b50..26694281b 100644 --- a/scenarios/sre/project/roles/recorders/meta/argument_specs.yaml +++ b/scenarios/sre/project/roles/recorders/meta/argument_specs.yaml @@ -48,6 +48,56 @@ argument_specs: default: false required: false type: bool + recorders_vendors: + required: false + type: dict + options: + dynatrace: + required: false + type: dict + options: + recorder: + required: false + type: dict + options: + enabled: + default: false + required: false + type: bool + platform_url: + required: false + type: str + platform_token_path: + required: false + type: str + namespace: + default: otel-demo + required: false + type: str + timeframe_to: + default: now + required: false + type: str + fallback_from: + default: now-1h + required: false + type: str + interval: + default: 1m + required: false + type: str + limit: + default: 1000 + required: false + type: int + format: + choices: + - json + - csv + - both + default: both + required: false + type: str recorders_storage: required: false type: dict diff --git a/scenarios/sre/project/roles/recorders/tasks/install.yaml b/scenarios/sre/project/roles/recorders/tasks/install.yaml index f6bf1f497..d11736139 100644 --- a/scenarios/sre/project/roles/recorders/tasks/install.yaml +++ b/scenarios/sre/project/roles/recorders/tasks/install.yaml @@ -48,6 +48,16 @@ when: - recorders_configuration.phase == "pre-fault-injection" + - name: Stamp scenario start time for Dynatrace Recorder + # The Dynatrace recorder itself runs at teardown (Grail is queryable by + # timeframe); here we only record the scenario start so teardown can + # back-query [scenario start -> now]. + ansible.builtin.import_tasks: + file: stamp_dynatrace_start_time.yaml + when: + - recorders_configuration.phase == "pre-fault-injection" + - recorders_vendors.dynatrace.recorder.enabled | ansible.builtin.default(false) + - name: Install SRE specific recorders when: - recorders_configuration.tools.sre.enabled | ansible.builtin.default(false) diff --git a/scenarios/sre/project/roles/recorders/tasks/install_dynatrace_recorder.yaml b/scenarios/sre/project/roles/recorders/tasks/install_dynatrace_recorder.yaml new file mode 100644 index 000000000..abbc1669e --- /dev/null +++ b/scenarios/sre/project/roles/recorders/tasks/install_dynatrace_recorder.yaml @@ -0,0 +1,131 @@ +--- +# Dynatrace Grail recorder (SaaS). Unlike the in-cluster recorders, Dynatrace +# retains history queryable by timeframe, so this runs ONCE at teardown and +# back-queries the whole incident window [scenario start -> now] for every +# dataset (metrics, spans, logs, events, k8s metrics). +# +# All configuration comes from observability_vendors.dynatrace.recorder. The +# platform token is read at teardown from a gitignored file (platform_token_path, +# under secrets/ — same convention as dynakube_path) and injected straight into +# the recorder Job as an environment variable value, mirroring how the ClickHouse +# recorder injects CLICKHOUSE_PASSWORD. No environment export and no Kubernetes +# Secret are required. + +- name: Resolve Dynatrace recorder configuration + ansible.builtin.set_fact: + recorders_dynatrace_config: "{{ recorders_vendors.dynatrace.recorder }}" + +- name: Validate that the Dynatrace platform token file is configured + ansible.builtin.assert: + that: + - recorders_dynatrace_config.platform_token_path | default("") | ansible.builtin.length > 0 + fail_msg: >- + Dynatrace recorder is enabled but observability_vendors.dynatrace.recorder.platform_token_path + is not set. Point it at a file (under secrets/, gitignored) containing a + platform token (prefix dt0s16.) with Grail read permissions. + success_msg: Dynatrace platform token path found. + quiet: true + +- name: Resolve the Dynatrace platform token path + ansible.builtin.set_fact: + recorders_dynatrace_token_path: "{{ recorders_dynatrace_config.platform_token_path }}" + +# A relative platform_token_path is resolved against the SRE project root (parent +# of the playbook directory) so it works regardless of the working directory — +# mirroring how install_datadog.yaml anchors its manifest_path. +- name: Anchor a relative Dynatrace platform token path to the project root + ansible.builtin.set_fact: + recorders_dynatrace_token_path: "{{ [playbook_dir, '..', recorders_dynatrace_token_path] | ansible.builtin.path_join | ansible.builtin.realpath }}" + when: + - not (recorders_dynatrace_token_path is ansible.builtin.abs) + +- name: Validate that the Dynatrace platform token file exists + ansible.builtin.stat: + path: "{{ recorders_dynatrace_token_path }}" + register: recorders_dynatrace_token_file + +- name: Assert the Dynatrace platform token file is present and non-empty + ansible.builtin.assert: + that: + - recorders_dynatrace_token_file.stat.exists + - recorders_dynatrace_token_file.stat.size > 0 + fail_msg: >- + Dynatrace platform token file not found or empty: + {{ recorders_dynatrace_token_path }} + success_msg: Dynatrace platform token file found. + quiet: true + +- name: Read the Dynatrace platform token + ansible.builtin.set_fact: + recorders_dynatrace_platform_token: >- + {{ lookup('ansible.builtin.file', recorders_dynatrace_token_path) | trim }} + no_log: true + +- name: Validate that the Dynatrace platform URL is set + ansible.builtin.assert: + that: + - recorders_dynatrace_config.platform_url | default("") | ansible.builtin.length > 0 + fail_msg: >- + Dynatrace recorder is enabled but observability_vendors.dynatrace.recorder.platform_url + is not set (e.g. https://.apps.dynatrace.com). + success_msg: Dynatrace platform URL found. + quiet: true + +- name: Retrieve scenario start time metadata + kubernetes.core.k8s_info: + api_version: v1 + kind: ConfigMap + kubeconfig: "{{ recorders_cluster.kubeconfig }}" + name: "{{ recorders_managers.data_recorders.instances.dynatrace }}-metadata" + namespace: "{{ recorders_managers.data_recorders.kubernetes.namespace }}" + register: recorders_dynatrace_metadata + +- name: Resolve query timeframe start + # Prefer the scenario start time stamped at install; fall back to the + # configured relative lookback if the marker is missing. + ansible.builtin.set_fact: + recorders_dynatrace_from: >- + {{ + ( + recorders_dynatrace_metadata.resources[0].data.scenario_start_time + if (recorders_dynatrace_metadata.resources | ansible.builtin.length > 0) + else recorders_dynatrace_config.fallback_from | default("now-1h") + ) + }} + +- name: Create Dynatrace Data Recorder environment variables + # The token is injected as a plain env value (same approach as the ClickHouse + # recorder's CLICKHOUSE_PASSWORD). no_log hides it from task output. + ansible.builtin.set_fact: + recorders_dynatrace_env_vars: + - name: DT_PLATFORM_URL + value: "{{ recorders_dynatrace_config.platform_url }}" + - name: DT_PLATFORM_TOKEN + value: "{{ recorders_dynatrace_platform_token }}" + - name: DT_K8S_NAMESPACE + value: "{{ recorders_dynatrace_config.namespace | default('otel-demo') }}" + - name: DT_DQL_FROM + value: "{{ recorders_dynatrace_from }}" + - name: DT_DQL_TO + value: "{{ recorders_dynatrace_config.timeframe_to | default('now') }}" + - name: DT_DQL_INTERVAL + value: "{{ recorders_dynatrace_config.interval | default('1m') }}" + - name: DT_DQL_LIMIT + value: "{{ recorders_dynatrace_config.limit | default(1000) | ansible.builtin.string }}" + - name: DT_DQL_FORMAT + value: "{{ recorders_dynatrace_config.format | default('both') }}" + no_log: true + +- name: Import recorder deployment tasks + ansible.builtin.import_tasks: + file: deploy_python_recorder.yaml + vars: + python_recorder: + job: + environment_variables: "{{ recorders_dynatrace_env_vars }}" + name: "{{ recorders_managers.data_recorders.instances.dynatrace }}" + scripts: + requirements_file_path: "files/scripts/dynatrace/requirements.txt" + scripts_file_path: "files/scripts/dynatrace/gather.py" + volume: + size: 2Gi diff --git a/scenarios/sre/project/roles/recorders/tasks/stamp_dynatrace_start_time.yaml b/scenarios/sre/project/roles/recorders/tasks/stamp_dynatrace_start_time.yaml new file mode 100644 index 000000000..f6c9c6ddf --- /dev/null +++ b/scenarios/sre/project/roles/recorders/tasks/stamp_dynatrace_start_time.yaml @@ -0,0 +1,34 @@ +--- +# Persist the scenario start time so the Dynatrace recorder (which runs once at +# teardown) can back-query Grail over exactly [scenario start -> now]. Install +# and uninstall are separate playbook runs, so in-memory facts do not survive; +# the timestamp is written into a ConfigMap in the recorders namespace, which +# lives until teardown deletes the namespace. +- name: Check for an existing Dynatrace recorder metadata ConfigMap + kubernetes.core.k8s_info: + api_version: v1 + kind: ConfigMap + kubeconfig: "{{ recorders_cluster.kubeconfig }}" + name: "{{ recorders_managers.data_recorders.instances.dynatrace }}-metadata" + namespace: "{{ recorders_managers.data_recorders.kubernetes.namespace }}" + register: recorders_dynatrace_metadata + +- name: Stamp scenario start time for Dynatrace recorder + # Only stamp once so a re-run of the install phase does not reset the start + # time; the first recorded value is the authoritative scenario start. + when: + - recorders_dynatrace_metadata.resources | ansible.builtin.length == 0 + kubernetes.core.k8s: + kubeconfig: "{{ recorders_cluster.kubeconfig }}" + resource_definition: + apiVersion: v1 + kind: ConfigMap + metadata: + labels: + "app.kubernetes.io/managed-by": ITBench + "app.kubernetes.io/name": "{{ recorders_managers.data_recorders.instances.dynatrace }}" + name: "{{ recorders_managers.data_recorders.instances.dynatrace }}-metadata" + namespace: "{{ recorders_managers.data_recorders.kubernetes.namespace }}" + data: + scenario_start_time: "{{ now(utc=True, fmt='%Y-%m-%dT%H:%M:%SZ') }}" + state: present diff --git a/scenarios/sre/project/roles/recorders/tasks/uninstall.yaml b/scenarios/sre/project/roles/recorders/tasks/uninstall.yaml index 6abb8af96..7aeed2c1f 100644 --- a/scenarios/sre/project/roles/recorders/tasks/uninstall.yaml +++ b/scenarios/sre/project/roles/recorders/tasks/uninstall.yaml @@ -28,6 +28,15 @@ label: cronjob/{{ cronjob.metadata.name }} loop_var: cronjob +- name: Deploy Dynatrace Recorder + # Dynatrace Grail is queryable by timeframe, so the recorder runs ONCE here at + # teardown and back-queries [scenario start -> now]. It is deployed before the + # job wait below so its one-shot Job is awaited like the others. + ansible.builtin.import_tasks: + file: install_dynatrace_recorder.yaml + when: + - recorders_vendors.dynatrace.recorder.enabled | ansible.builtin.default(false) + - name: Retrieve Jobs kubernetes.core.k8s_info: api_version: batch/v1 @@ -96,6 +105,17 @@ path: alerts name: "{{ recorders_managers.data_recorders.instances.prometheus }}" +- name: Import Dynatrace recorder export tasks + ansible.builtin.import_tasks: + file: copy_python_recorder_files.yaml + vars: + python_recorder: + export: + path: observability_information_from_dynatrace + name: "{{ recorders_managers.data_recorders.instances.dynatrace }}" + when: + - recorders_vendors.dynatrace.recorder.enabled | ansible.builtin.default(false) + - name: Delete the namespace kubernetes.core.k8s: kubeconfig: "{{ recorders_cluster.kubeconfig }}" diff --git a/scenarios/sre/project/run_dynatrace_recorder.yaml b/scenarios/sre/project/run_dynatrace_recorder.yaml new file mode 100644 index 000000000..1fb69bc21 --- /dev/null +++ b/scenarios/sre/project/run_dynatrace_recorder.yaml @@ -0,0 +1,39 @@ +--- +# Manual, standalone deploy of the Dynatrace Grail recorder Job. +# +# This runs ONLY the recorder task (install_dynatrace_recorder.yaml) against an +# already-running scenario. It does NOT export records, suspend cronjobs, or +# delete the data-recorders namespace — unlike the full uninstall_recorders flow. +# +# Prereqs: +# * The scenario is running and the `data-recorders` namespace exists (it is +# created when recorders are installed at scenario start). +# * observability_vendors.dynatrace.recorder is configured (enabled, platform_url, +# platform_token_path) and the token file exists. +# +# Run from the scenarios/sre directory so the relative platform_token_path +# resolves the same way it does in the normal playbooks: +# +# uv run ansible-playbook -i inventory project/run_dynatrace_recorder.yaml \ +# -e cluster_kubeconfig=$HOME/.kube/config -e cluster_platform=kubernetes +# +- name: Manually deploy the Dynatrace Grail recorder + hosts: localhost + gather_facts: false + vars: + cluster_kubeconfig: "{{ lookup('ansible.builtin.env', 'KUBECONFIG') | default('~/.kube/config', true) }}" + cluster_platform: kubernetes + tasks: + - name: Load observability vendor configuration + ansible.builtin.include_vars: + file: "{{ playbook_dir }}/../inventory/group_vars/environment/observability_vendors.yaml" + + - name: Deploy the Dynatrace recorder Job + ansible.builtin.import_role: + name: recorders + tasks_from: install_dynatrace_recorder + vars: + recorders_cluster: + kubeconfig: "{{ cluster_kubeconfig }}" + platform: "{{ cluster_platform }}" + recorders_vendors: "{{ observability_vendors }}" From 66dd6c389abed30261d0ab0bb04b67b05ab345f0 Mon Sep 17 00:00:00 2001 From: Bekir Turkkan Date: Mon, 3 Aug 2026 15:27:26 -0400 Subject: [PATCH 07/11] Dynatrace integration is completed and related exporter has been implemented --- scenarios/sre/docs/observability-vendors.md | 32 +++++++ .../tasks/install_opentelemetry_demo.yaml | 89 +++++++++++++++++++ .../tasks/uninstall_opentelemetry_demo.yaml | 16 ++++ .../templates/helm/otel_demo/values.j2 | 67 ++++++++++++++ .../recorders/defaults/main/vendors.yaml | 2 +- 5 files changed, 205 insertions(+), 1 deletion(-) diff --git a/scenarios/sre/docs/observability-vendors.md b/scenarios/sre/docs/observability-vendors.md index 8b2d33a37..984cc9b04 100644 --- a/scenarios/sre/docs/observability-vendors.md +++ b/scenarios/sre/docs/observability-vendors.md @@ -422,6 +422,38 @@ for SCC annotations is still needed.) runs; `molecule/deployment_vendors/` runs a live deploy when vendor secrets are provided and self-skips otherwise. +### Application OTLP ingest (otel-demo → Dynatrace) + +The DynaKube deploy above is **Layer A** (OneAgent/ActiveGate infrastructure +monitoring). It does **not** send the sample application's OpenTelemetry data. +That path lives in the `applications` role and is enabled by the **same** +`observability_vendors.dynatrace.enabled` flag: + +- `install_opentelemetry_demo.yaml` — when Dynatrace is enabled, parses the same + `dynakube.yaml` (reusing the `from_yaml_all` idiom from `install_dynatrace.yaml`), + extracts the **`dataIngestToken`** and **`apiUrl`**, and projects a + `dynatrace-otlp-ingest` Secret (`DT_ENDPOINT`, `DT_API_TOKEN`) into the + `otel-demo` namespace. No new token is required. +- `templates/helm/otel_demo/values.j2` — gated on + `applications_dynatrace_ingest_enabled`, adds an `otlphttp/dynatrace` exporter to + the demo collector's **logs, metrics, and traces** pipelines. +- **Metric temporality:** a `cumulativetodelta` processor is added to the metrics + pipeline — Dynatrace accepts delta temporality only, and the demo emits + cumulative sums/histograms (otherwise rejected as `UNSUPPORTED_METRIC_TYPE_*`). +- **K8s attributes:** the chart's `kubernetesAttributes` preset already tags + pod/namespace/deployment/node. We additionally inject **`k8s.cluster.name`** via a + `resource/dynatrace` processor (default: the DynaKube `metadata.name`, overridable + via `applications_dynatrace_cluster_name`) so OTLP data correlates to the K8s + cluster entity the OneAgent sees. +- **Helm list gotcha:** the opentelemetry-collector chart deep-merges maps but + **replaces lists**, so each touched pipeline's `processors` list is restated in + full (chart defaults + the added processors). +- `uninstall_opentelemetry_demo.yaml` deletes the projected secret (gated on the + same flag) before removing the namespace. + +When Dynatrace is disabled the template renders byte-for-byte as before — the +exporter, processors, and `extraEnvsFrom` are entirely absent. + --- ## Datadog (implemented) — operator + user-supplied `datadog-agent.yaml` diff --git a/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml b/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml index ff5353cd6..fa38ca01d 100644 --- a/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml +++ b/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml @@ -68,6 +68,95 @@ name: system:openshift:scc:anyuid state: present +# --- Dynatrace OTLP ingest (optional). --------------------------------------- +# When the Dynatrace vendor is enabled, route the otel-demo collector's OTLP +# traces/metrics/logs to Dynatrace in addition to the in-cluster backends. The +# dataIngestToken and apiUrl are reused from the same dynakube.yaml consumed by +# the tools role (roles/tools/tasks/install_dynatrace.yaml), so no new token is +# needed. Everything here is gated on the vendor flag; when disabled the demo +# renders and installs exactly as before. +- name: Configure Dynatrace OTLP ingest for OpenTelemetry Demo + when: + - observability_vendors.dynatrace.enabled | ansible.builtin.default(false) + block: + - name: Resolve the Dynatrace DynaKube manifest path + ansible.builtin.set_fact: + applications_dynatrace_dynakube_path: >- + {{ + dynatrace_dynakube_path | + ansible.builtin.default(observability_vendors.dynatrace.dynakube_path, true) + }} + + # Relative paths are anchored to the SRE project root (parent of the playbook + # directory), matching install_dynatrace.yaml; absolute paths are used as-is. + - name: Anchor a relative Dynatrace manifest path to the project root + ansible.builtin.set_fact: + applications_dynatrace_dynakube_path: >- + {{ + [playbook_dir, '..', applications_dynatrace_dynakube_path] + | ansible.builtin.path_join | ansible.builtin.realpath + }} + when: + - not (applications_dynatrace_dynakube_path is ansible.builtin.abs) + + - name: Parse the Dynatrace DynaKube manifest documents + ansible.builtin.set_fact: + applications_dynatrace_documents: >- + {{ + lookup('ansible.builtin.file', applications_dynatrace_dynakube_path) + | ansible.builtin.from_yaml_all | list | ansible.builtin.reject('none') | list + }} + no_log: true + + - name: Derive Dynatrace ingest facts from the manifest + vars: + applications_dynatrace_secret: >- + {{ + applications_dynatrace_documents + | selectattr('kind', 'equalto', 'Secret') | list | first + }} + applications_dynatrace_dynakube: >- + {{ + applications_dynatrace_documents + | selectattr('kind', 'equalto', 'DynaKube') | list | first + }} + ansible.builtin.set_fact: + # dataIngestToken is base64-encoded in the Secret's data map; decode to + # the raw token for the Authorization header value. + applications_dynatrace_ingest_token: "{{ applications_dynatrace_secret.data.dataIngestToken | ansible.builtin.b64decode }}" + # OTLP endpoint is the DynaKube apiUrl (…/api) plus the OTLP v2 path. + applications_dynatrace_otlp_endpoint: "{{ applications_dynatrace_dynakube.spec.apiUrl }}/v2/otlp" + # Cluster name for entity correlation: explicit override, else DynaKube name. + applications_dynatrace_cluster_name: >- + {{ + applications_dynatrace_cluster_name | + ansible.builtin.default(applications_dynatrace_dynakube.metadata.name, true) + }} + no_log: true + + - name: Create the Dynatrace OTLP ingest Secret in the demo namespace + kubernetes.core.k8s: + kubeconfig: "{{ applications_cluster.kubeconfig }}" + resource_definition: + apiVersion: v1 + kind: Secret + metadata: + name: dynatrace-otlp-ingest + namespace: "{{ applications_managers.opentelemetry_demo.kubernetes.namespace }}" + labels: + app.kubernetes.io/managed-by: ITBench + itbench.io/observability-vendor: dynatrace + type: Opaque + stringData: + DT_ENDPOINT: "{{ applications_dynatrace_otlp_endpoint }}" + DT_API_TOKEN: "{{ applications_dynatrace_ingest_token }}" + state: present + no_log: true + + - name: Enable Dynatrace ingest in the OpenTelemetry Demo values + ansible.builtin.set_fact: + applications_dynatrace_ingest_enabled: true + - name: Install OpenTelemetry Demo (Astronomy Shop) kubernetes.core.helm: chart_ref: "{{ applications_managers.opentelemetry_demo.helm.chart.reference }}" diff --git a/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml b/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml index d29409f73..7dceeed8a 100644 --- a/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml +++ b/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml @@ -7,6 +7,22 @@ release_state: absent wait: true +# Explicitly remove the projected Dynatrace ingest secret. The namespace +# deletion below also removes it, but this keeps teardown symmetric with the +# install and covers cases where the namespace is retained. +- name: Delete the Dynatrace OTLP ingest Secret + kubernetes.core.k8s: + kubeconfig: "{{ applications_cluster.kubeconfig }}" + resource_definition: + apiVersion: v1 + kind: Secret + metadata: + name: dynatrace-otlp-ingest + namespace: "{{ applications_managers.opentelemetry_demo.kubernetes.namespace }}" + state: absent + when: + - observability_vendors.dynatrace.enabled | ansible.builtin.default(false) + - name: Delete Namespace kubernetes.core.k8s: kubeconfig: "{{ applications_cluster.kubeconfig }}" diff --git a/scenarios/sre/project/roles/applications/templates/helm/otel_demo/values.j2 b/scenarios/sre/project/roles/applications/templates/helm/otel_demo/values.j2 index 37b893077..2184c76f8 100644 --- a/scenarios/sre/project/roles/applications/templates/helm/otel_demo/values.j2 +++ b/scenarios/sre/project/roles/applications/templates/helm/otel_demo/values.j2 @@ -81,12 +81,32 @@ components: runAsNonRoot: true runAsUser: 999 opentelemetry-collector: +{% if applications_dynatrace_ingest_enabled | default(false) %} + # Dynatrace OTLP ingest is enabled: mount the projected credentials secret so + # the exporter can read DT_ENDPOINT / DT_API_TOKEN as environment variables. + extraEnvsFrom: + - secretRef: + name: dynatrace-otlp-ingest +{% endif %} config: connectors: spanmetrics: dimensions: - name: namespace default: {{ applications_managers.opentelemetry_demo.kubernetes.namespace }} +{% if applications_dynatrace_ingest_enabled | default(false) %} + processors: + # Dynatrace accepts DELTA temporality only; the demo emits cumulative + # sums/histograms, so convert them or they are rejected on ingest. + cumulativetodelta: {} + # Tag every signal with the cluster name so Dynatrace correlates the OTLP + # data to the Kubernetes cluster entity seen by the OneAgent/ActiveGate. + resource/dynatrace: + attributes: + - key: k8s.cluster.name + value: "{{ applications_dynatrace_cluster_name }}" + action: upsert +{% endif %} exporters: clickhouse: username: {{ tools_clickhouse_username }} @@ -98,22 +118,69 @@ opentelemetry-collector: endpoint: {{ tools_jaeger_endpoint.collector.grpc }} prometheus: endpoint: 0.0.0.0:8888 +{% if applications_dynatrace_ingest_enabled | default(false) %} + otlphttp/dynatrace: + endpoint: ${env:DT_ENDPOINT} + headers: + Authorization: Api-Token ${env:DT_API_TOKEN} +{% endif %} service: pipelines: logs: +{% if applications_dynatrace_ingest_enabled | default(false) %} + # Full processor list restated (Helm REPLACES lists) + resource/dynatrace. + processors: + - k8sattributes + - memory_limiter + - resourcedetection + - resource + - resource/dynatrace + - batch +{% endif %} exporters: - clickhouse - debug +{% if applications_dynatrace_ingest_enabled | default(false) %} + - otlphttp/dynatrace +{% endif %} metrics: +{% if applications_dynatrace_ingest_enabled | default(false) %} + # cumulativetodelta added before batch; resource/dynatrace for cluster tag. + processors: + - k8sattributes + - memory_limiter + - resourcedetection + - resource + - resource/dynatrace + - cumulativetodelta + - batch +{% endif %} exporters: - debug - prometheus +{% if applications_dynatrace_ingest_enabled | default(false) %} + - otlphttp/dynatrace +{% endif %} traces: +{% if applications_dynatrace_ingest_enabled | default(false) %} + # Restated defaults (incl. the chart's frontend-route transform) + resource/dynatrace. + processors: + - k8sattributes + - memory_limiter + - resourcedetection + - resource + - resource/dynatrace + - transform + - batch +{% endif %} exporters: - clickhouse - debug - otlp/jaeger - spanmetrics +{% if applications_dynatrace_ingest_enabled | default(false) %} + - otlphttp/dynatrace +{% endif %} telemetry: metrics: level: detailed diff --git a/scenarios/sre/project/roles/recorders/defaults/main/vendors.yaml b/scenarios/sre/project/roles/recorders/defaults/main/vendors.yaml index e9d76c7ed..9d5da26e4 100644 --- a/scenarios/sre/project/roles/recorders/defaults/main/vendors.yaml +++ b/scenarios/sre/project/roles/recorders/defaults/main/vendors.yaml @@ -5,4 +5,4 @@ recorders_vendors: dynatrace: recorder: - enabled: false + enabled: true From ffcfd571bdcec57ccb1f78db6971960cc922bee5 Mon Sep 17 00:00:00 2001 From: Bekir Turkkan Date: Mon, 3 Aug 2026 15:38:52 -0400 Subject: [PATCH 08/11] minor update --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 039f5bc40..528d561ac 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ .DS_Store +# Exported observability information (topology, traces, metrics, alerts) +exports/ + # Files associated with group variables (v1) scenarios/sre/group_vars/*/*.yaml scenarios/sre/dev/remote_cluster/group_vars/*/*.yaml From 44fadc5277b2df447ed9862e0d9dce38e7f5e1b2 Mon Sep 17 00:00:00 2001 From: Bekir Turkkan Date: Mon, 3 Aug 2026 20:04:37 -0400 Subject: [PATCH 09/11] "Dynatrace integration related updates have been added. Uninstall script has been updated to make sure dynatrace is removed last since it has injections on Prometheus and application pods for otel data collection." --- .../tasks/install_opentelemetry_demo.yaml | 2 +- .../tasks/uninstall_opentelemetry_demo.yaml | 2 +- .../project/roles/tools/tasks/uninstall.yaml | 21 +++++++++++++++---- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml b/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml index 65ad59c3e..309e65810 100644 --- a/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml +++ b/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml @@ -142,7 +142,7 @@ kind: Secret metadata: name: dynatrace-otlp-ingest - namespace: "{{ applications_managers.opentelemetry_demo.kubernetes.namespace }}" + namespace: "{{ applications_releases.opentelemetry_demo.namespace }}" labels: app.kubernetes.io/managed-by: ITBench itbench.io/observability-vendor: dynatrace diff --git a/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml b/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml index b09f91fb9..0ac4a128a 100644 --- a/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml +++ b/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml @@ -18,7 +18,7 @@ kind: Secret metadata: name: dynatrace-otlp-ingest - namespace: "{{ applications_managers.opentelemetry_demo.kubernetes.namespace }}" + namespace: "{{ applications_releases.opentelemetry_demo.namespace }}" state: absent when: - observability_vendors.dynatrace.enabled | ansible.builtin.default(false) diff --git a/scenarios/sre/project/roles/tools/tasks/uninstall.yaml b/scenarios/sre/project/roles/tools/tasks/uninstall.yaml index 1b191fcc7..a185adde5 100644 --- a/scenarios/sre/project/roles/tools/tasks/uninstall.yaml +++ b/scenarios/sre/project/roles/tools/tasks/uninstall.yaml @@ -2,10 +2,16 @@ # Vendor cleanup runs unconditionally and idempotently: the tasks no-op when the # vendor namespace is absent, so disabling a vendor still removes its resources # during Undeploy-Tools (rather than orphaning them). -- name: Import Dynatrace uninstallation tasks - ansible.builtin.import_tasks: - file: uninstall_dynatrace.yaml - +# +# ORDERING NOTE: Dynatrace is uninstalled LAST (just before CRD removal), not +# here. Dynatrace's OneAgent injects a `csi.oneagent.dynatrace.com` volume into +# pods across other tool namespaces (e.g. Prometheus). If the Dynatrace operator +# / CSI DaemonSet is removed first, those injected pods can no longer unmount +# their CSI volume when their own namespace is later deleted, so they hang in +# Terminating and block namespace deletion (600s timeout). Tearing Dynatrace +# down after the injected workloads keeps the CSI driver available for a clean +# unmount. Datadog uses a DaemonSet agent (no injected CSI volumes) and is safe +# to remove early. - name: Import Datadog uninstallation tasks ansible.builtin.import_tasks: file: uninstall_datadog.yaml @@ -56,6 +62,13 @@ ansible.builtin.import_tasks: file: uninstall_kubernetes_gateway.yaml +# Dynatrace is torn down here (after all other tools) so its OneAgent CSI driver +# remains available while Dynatrace-injected pods in other namespaces terminate. +# See the ORDERING NOTE at the top of this file. +- name: Import Dynatrace uninstallation tasks + ansible.builtin.import_tasks: + file: uninstall_dynatrace.yaml + - name: Import Custom Resource Definition removal tasks ansible.builtin.import_tasks: file: remove_custom_resource_definitions.yaml From 52cc0ce044904cbb799f0246b9fe6307ddb28e36 Mon Sep 17 00:00:00 2001 From: Bekir Turkkan Date: Tue, 4 Aug 2026 12:59:03 -0400 Subject: [PATCH 10/11] sample files have been removed from recorders --- ...k8s_metrics_2026-07-30T16-48-44.313012.csv | 1 - ...8s_metrics_2026-07-30T16-48-44.313012.json | 139 ------------------ 2 files changed, 140 deletions(-) delete mode 100644 scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.csv delete mode 100644 scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.json diff --git a/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.csv b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.csv deleted file mode 100644 index d3f5a12fa..000000000 --- a/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.csv +++ /dev/null @@ -1 +0,0 @@ - diff --git a/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.json b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.json deleted file mode 100644 index 5f7a6ae84..000000000 --- a/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/dynatrace-records/records/dynatrace_k8s_metrics_2026-07-30T16-48-44.313012.json +++ /dev/null @@ -1,139 +0,0 @@ -{ - "dataset": "k8s-metrics", - "namespace": "otel-demo", - "timeframe": { - "from": "now-1h", - "to": "now" - }, - "interval": "1m", - "metric_count": 14, - "metrics_with_data": 0, - "metrics": [ - { - "metric": "dt.kubernetes.pod.network_received_data", - "aggregation": "sum", - "grouping": "pod", - "query": "timeseries value = sum(dt.kubernetes.pod.network_received_data), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", - "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", - "record_count": 0, - "result": null - }, - { - "metric": "dt.kubernetes.pod.network_received_errors", - "aggregation": "sum", - "grouping": "pod", - "query": "timeseries value = sum(dt.kubernetes.pod.network_received_errors), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", - "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", - "record_count": 0, - "result": null - }, - { - "metric": "dt.kubernetes.pod.network_received_packets_dropped", - "aggregation": "sum", - "grouping": "pod", - "query": "timeseries value = sum(dt.kubernetes.pod.network_received_packets_dropped), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", - "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", - "record_count": 0, - "result": null - }, - { - "metric": "dt.kubernetes.pod.network_transmitted_data", - "aggregation": "sum", - "grouping": "pod", - "query": "timeseries value = sum(dt.kubernetes.pod.network_transmitted_data), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", - "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", - "record_count": 0, - "result": null - }, - { - "metric": "dt.kubernetes.pod.network_transmitted_errors", - "aggregation": "sum", - "grouping": "pod", - "query": "timeseries value = sum(dt.kubernetes.pod.network_transmitted_errors), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", - "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", - "record_count": 0, - "result": null - }, - { - "metric": "dt.kubernetes.pod.network_transmitted_packets_dropped", - "aggregation": "sum", - "grouping": "pod", - "query": "timeseries value = sum(dt.kubernetes.pod.network_transmitted_packets_dropped), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", - "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", - "record_count": 0, - "result": null - }, - { - "metric": "dt.kubernetes.pod.containers_desired", - "aggregation": "max", - "grouping": "pod", - "query": "timeseries value = max(dt.kubernetes.pod.containers_desired), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", - "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", - "record_count": 0, - "result": null - }, - { - "metric": "dt.kubernetes.pod.restarts", - "aggregation": "max", - "grouping": "pod", - "query": "timeseries value = max(dt.kubernetes.pod.restarts), by:{k8s.pod.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", - "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", - "record_count": 0, - "result": null - }, - { - "metric": "dt.kubernetes.container.requests_cpu", - "aggregation": "max", - "grouping": "container", - "query": "timeseries value = max(dt.kubernetes.container.requests_cpu), by:{k8s.pod.name, k8s.container.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", - "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", - "record_count": 0, - "result": null - }, - { - "metric": "dt.kubernetes.container.limits_cpu", - "aggregation": "max", - "grouping": "container", - "query": "timeseries value = max(dt.kubernetes.container.limits_cpu), by:{k8s.pod.name, k8s.container.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", - "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", - "record_count": 0, - "result": null - }, - { - "metric": "dt.kubernetes.container.requests_memory", - "aggregation": "max", - "grouping": "container", - "query": "timeseries value = max(dt.kubernetes.container.requests_memory), by:{k8s.pod.name, k8s.container.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", - "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", - "record_count": 0, - "result": null - }, - { - "metric": "dt.kubernetes.container.limits_memory", - "aggregation": "max", - "grouping": "container", - "query": "timeseries value = max(dt.kubernetes.container.limits_memory), by:{k8s.pod.name, k8s.container.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", - "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", - "record_count": 0, - "result": null - }, - { - "metric": "dt.kubernetes.container.cpu_usage", - "aggregation": "avg", - "grouping": "container", - "query": "timeseries value = avg(dt.kubernetes.container.cpu_usage), by:{k8s.pod.name, k8s.container.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", - "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", - "record_count": 0, - "result": null - }, - { - "metric": "dt.kubernetes.container.memory_working_set", - "aggregation": "avg", - "grouping": "container", - "query": "timeseries value = avg(dt.kubernetes.container.memory_working_set), by:{k8s.pod.name, k8s.container.name, k8s.namespace.name}, filter:{ k8s.namespace.name == \"otel-demo\" }, interval:1m", - "error": "403 Client Error: Forbidden for url: https://xqp06790.apps.dynatrace.com/platform/storage/query/v1/query:execute", - "record_count": 0, - "result": null - } - ] -} \ No newline at end of file From f53e336328cc88de7ea1c0cf2e8a05c37fb3d7e0 Mon Sep 17 00:00:00 2001 From: Bekir Turkkan Date: Thu, 6 Aug 2026 16:17:50 -0400 Subject: [PATCH 11/11] Topology, traces, and problems data have been approved --- .../cluster/vars/main/minimum_versions.yaml | 2 +- .../files/scripts/dynatrace/gather.py | 175 +++++++++++++++++- 2 files changed, 173 insertions(+), 4 deletions(-) diff --git a/scenarios/sre/project/roles/cluster/vars/main/minimum_versions.yaml b/scenarios/sre/project/roles/cluster/vars/main/minimum_versions.yaml index a760f12e7..135381fe2 100644 --- a/scenarios/sre/project/roles/cluster/vars/main/minimum_versions.yaml +++ b/scenarios/sre/project/roles/cluster/vars/main/minimum_versions.yaml @@ -1,4 +1,4 @@ --- cluster_minimum_versions: - kubernetes: "1.34" + kubernetes: "1.32" openshift: "4.19" diff --git a/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/gather.py b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/gather.py index e03d0d507..46591c40f 100644 --- a/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/gather.py +++ b/scenarios/sre/project/roles/recorders/files/scripts/dynatrace/gather.py @@ -22,7 +22,9 @@ # NOTE: This recorder is the SaaS counterpart to the in-cluster scrapers # (clickhouse/jaeger/prometheus). Dynatrace Grail retains history that is # queryable by timeframe, so a single run at teardown back-queries the -# whole incident window [scenario start -> now] for every dataset. It is +# whole incident window [scenario start -> now] for every dataset. The +# topology dataset is the exception: it is captured as two point-in-time +# snapshots (init = scenario start, stop = teardown). It is # adapted from dynatrace_scripts/pull_dql_json.py: the CLI, .env, and # dotenv dependency are removed and all configuration comes from # environment variables injected by the recorder Job. @@ -38,6 +40,11 @@ LIMIT = os.environ.get("DT_DQL_LIMIT", "1000").strip() OUTPUT_FORMAT = os.environ.get("DT_DQL_FORMAT", "both").strip() +# Topology snapshots are point-in-time, but a Grail entity query needs a +# non-empty timeframe: each snapshot queries a short window (this many minutes) +# anchored at its timestamp — the scenario start for "init", teardown for "stop". +TOPOLOGY_WINDOW_MIN = int(os.environ.get("DT_TOPOLOGY_WINDOW_MIN", "5")) + # All records are written into the recorder PVC (~/records), matching the # convention used by the jaeger/prometheus recorders. OUTDIR = os.path.join(os.path.expanduser("~"), "records") @@ -85,13 +92,20 @@ "span-errorrate": {"stem": "dynatrace_span_errorrate"}, "logs": {"stem": "dynatrace_logs"}, "events": {"stem": "dynatrace_events"}, + "traces": {"stem": "dynatrace_traces"}, + "problems": {"stem": "dynatrace_problems"}, # Special: runs every K8S_METRICS key and merges them into one file. "k8s-metrics": {"stem": "dynatrace_k8s_metrics", "multi": True}, + # Special: Smartscape topology (service graph + infra edges) for the ns. + # Captured as two point-in-time snapshots (see export_topology): "init" at + # the scenario start time and "stop" at teardown. + "topology": {"stem": "dynatrace_topology", "topology": True}, } -# The recorder always exports the full set (metrics, spans, logs, events, k8s). +# The recorder always exports the full set (metrics, spans, traces, logs, +# events, problems, k8s, topology). ALL_DATASETS = [ "responsetime", "errorrate", "span-responsetime", "span-errorrate", - "logs", "events", "k8s-metrics", + "logs", "events", "traces", "problems", "k8s-metrics", "topology", ] # ───────────────────────────────────────────────────────────────────────────── @@ -170,6 +184,22 @@ def build_dql(dataset, namespace, interval, limit): f'fetch events | filter k8s.namespace.name == "{namespace}"' f" | sort timestamp desc | limit {limit}" ) + if dataset == "traces": + # Raw spans (distributed traces); no projection so the output keeps every + # span attribute. Sorted newest-first by span start_time. + return ( + f'fetch spans | filter k8s.namespace.name == "{namespace}"' + f" | sort start_time desc | limit {limit}" + ) + if dataset == "problems": + # Davis problems; no projection so the output keeps every field. Scoped to + # the namespace via the record's k8s.namespace.name ARRAY field (resolved + # from affected entities): in(, k8s.namespace.name). Duplicates dropped. + return ( + "fetch dt.davis.problems | filter not(dt.davis.is_duplicate)" + f' and in("{namespace}", k8s.namespace.name)' + f" | sort event.start desc | limit {limit}" + ) raise ValueError(f"Unknown dataset '{dataset}'") @@ -340,6 +370,143 @@ def out_paths(dataset, timestamp): return base + ".json", base + ".csv" +def _shift_iso(iso, minutes): + """Shift an ISO-8601 UTC timestamp by a number of minutes (may be negative).""" + shifted = _parse_iso(iso) + timedelta(minutes=minutes) + return shifted.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + +def _resolve_namespace_entity(session, namespace, time_from, time_to): + """Look up the CLOUD_APPLICATION_NAMESPACE entity id for a k8s namespace + name. Smartscape scopes services by their `belongs_to` relationship to this + entity — so we need its id to filter the topology to one namespace.""" + dql = ( + "fetch dt.entity.cloud_application_namespace" + f' | filter entity.name == "{namespace}"' + " | fields id, entity.name" + ) + recs = run_query(session, dql, time_from, time_to).get("records", []) + return recs[0]["id"] if recs else None + + +def _topology_snapshot(session, ns_id, time_from, time_to): + """One point-in-time Smartscape snapshot over [time_from, time_to]. + + Fetch every service that `belongs_to` the namespace along with its + relationship columns, then derive directed edges: service->service `calls`, + and service->host / service->process_group `runs_on`. Returns + (dql, services, nodes, edges).""" + dql = ( + "fetch dt.entity.service" + " | fieldsAdd ns = belongs_to[dt.entity.cloud_application_namespace]" + f' | filter in("{ns_id}", ns)' + " | fields id, entity.name," + " calls[dt.entity.service]," + " runs_on[dt.entity.host]," + " runs_on[dt.entity.process_group]" + ) + result = run_query(session, dql, time_from, time_to) + for msg in grail_notifications(result): + logger.info("[topology] Grail notice: %s", msg) + + services = result.get("records", []) + # Set of in-namespace service ids: we keep call edges to any service, but + # tag whether the callee is inside the namespace (target_in_namespace). + ns_service_ids = {s.get("id") for s in services} + # id -> service name, so the edge list is readable without a separate join. + names = {s.get("id"): s.get("entity.name") for s in services} + + nodes, edges = [], [] + for svc in services: + sid = svc.get("id") + sname = svc.get("entity.name") + nodes.append({"id": sid, "name": sname, "type": "SERVICE"}) + # service -> service call edges + for tgt in (svc.get("calls[dt.entity.service]") or []): + edges.append({"source": sid, "source_name": sname, + "target": tgt, "target_name": names.get(tgt, ""), + "type": "CALLS", + "target_in_namespace": tgt in ns_service_ids}) + # service -> host / process_group placement edges (targets are infra + # entities outside the service set, so no name is resolved here) + for host in (svc.get("runs_on[dt.entity.host]") or []): + edges.append({"source": sid, "source_name": sname, "target": host, + "target_name": "", "type": "RUNS_ON_HOST"}) + for pg in (svc.get("runs_on[dt.entity.process_group]") or []): + edges.append({"source": sid, "source_name": sname, "target": pg, + "target_name": "", "type": "RUNS_ON_PROCESS_GROUP"}) + return dql, services, nodes, edges + + +def export_topology(session, dataset, timestamp): + """Export the Smartscape topology as TWO point-in-time snapshots: + + * init — anchored at the scenario start time (DT_DQL_FROM) + * stop — anchored at teardown (DT_DQL_TO) + + Each snapshot queries a short window around its anchor (an entity query + needs a non-empty timeframe) and is written to its own files, e.g. + dynatrace_topology_init_.json / dynatrace_topology_stop_.json. + Mirrors dynatrace_scripts/pull_dql_json.py's export_topology graph shape.""" + # Resolve each snapshot's [from, to] window around its anchor timestamp. + init_from = to_iso(TIME_FROM) + init_to = _shift_iso(init_from, TOPOLOGY_WINDOW_MIN) + stop_to = to_iso(TIME_TO) + stop_from = _shift_iso(stop_to, -TOPOLOGY_WINDOW_MIN) + snapshots = [ + ("init", init_from, init_to), + ("stop", stop_from, stop_to), + ] + + any_data = False + for phase, snap_from, snap_to in snapshots: + stem = "{0}_{1}_{2}".format(DATASETS[dataset]["stem"], phase, timestamp) + base = os.path.join(OUTDIR, stem) + json_path, csv_path = base + ".json", base + ".csv" + + try: + ns_id = _resolve_namespace_entity(session, NAMESPACE, + snap_from, snap_to) + if not ns_id: + logger.warning("[topology/%s] no namespace entity for '%s'", + phase, NAMESPACE) + continue + dql, services, nodes, edges = _topology_snapshot( + session, ns_id, snap_from, snap_to) + except Exception as exc: + logger.warning("[topology/%s] ERROR: %s", phase, exc) + continue + + written = [] + if OUTPUT_FORMAT in ("json", "both"): + payload = { + "dataset": dataset, + "phase": phase, + "namespace": NAMESPACE, + "namespace_entity_id": ns_id, + "timeframe": {"from": snap_from, "to": snap_to}, + "query": dql, + "node_count": len(nodes), + "edge_count": len(edges), + "graph": {"nodes": nodes, "edges": edges}, + "services": services, # raw records, all relationship columns + } + with open(json_path, "w") as f: + json.dump(payload, f, indent=2) + written.append(json_path) + + if OUTPUT_FORMAT in ("csv", "both"): + # Flatten to an edge list — the natural tabular view of a graph. + write_csv_rows(edges, csv_path) + written.append(csv_path) + + logger.info("[topology/%s] %d services, %d edges -> %s", + phase, len(nodes), len(edges), ", ".join(written)) + any_data = any_data or bool(nodes) + + return any_data + + def export_k8s_metrics(session, dataset, timestamp): """Query every K8S_METRICS key individually and merge them into ONE file — so a single run exports many metrics together. Each metric is a separate @@ -404,6 +571,8 @@ def export_dataset(session, dataset, timestamp): Never raises — failures are captured so a multi-dataset run continues.""" if DATASETS[dataset].get("multi"): return export_k8s_metrics(session, dataset, timestamp) + if DATASETS[dataset].get("topology"): + return export_topology(session, dataset, timestamp) json_path, csv_path = out_paths(dataset, timestamp) try: