Dynatrace Integration as observability vendors

.gitignore

@@ -2,6 +2,9 @@
 
 .DS_Store
 
+# Exported observability information (topology, traces, metrics, alerts)
+exports/
+
 # Files associated with group variables (v1)
 scenarios/sre/group_vars/*/*.yaml
 scenarios/sre/dev/remote_cluster/group_vars/*/*.yaml

scenarios/sre/.gitignore

@@ -0,0 +1,8 @@
+# Observability vendor onboarding artifacts (token-bearing or environment-specific).
+# e.g. secrets/dynatrace/dynakube.yaml, secrets/datadog/datadog-agent.yaml
+secrets/
+
+# AWX runner vendor configuration contains secret inputs (manifest/dynakube
+# contents, API/app keys). Must be Ansible-Vault encrypted; never commit plaintext.
+inventory/group_vars/runner/vendors.yaml
+group_vars/runner/vendors.yaml

scenarios/sre/inventory/group_vars/environment/observability_vendors.yaml.example

@@ -0,0 +1,76 @@
+---
+# Third-party SaaS observability vendor configuration (Layer A: agent deployment).
+#
+# NON-SECRET values only (site, region, tenant/environment URL, toggles) belong
+# in this file. SECRETS (API keys, license keys, tokens) MUST NOT be placed here.
+# They are read from environment variables at install time. Export the relevant
+# variables before running the tools playbook, for example:
+#
+#   # Datadog
+#   export DATADOG_API_KEY=...            # required  # pragma: allowlist secret
+#   export DATADOG_APP_KEY=...            # optional (needed for some features)
+#
+#   # Dynatrace agent deploy (Layer A): no env vars. Download dynakube.yaml from
+#   # the Dynatrace onboarding UI (contains tokens + DynaKube CR) and reference
+#   # it via observability_vendors.dynatrace.dynakube_path (see below).
+#
+#   # Dynatrace Grail recorder (exports telemetry back out at teardown): its
+#   # PLATFORM token (prefix dt0s16., Grail read permissions) is read from a
+#   # gitignored file referenced by dynatrace.recorder.platform_token_path below
+#   # (same convention as dynakube_path) — no env export needed. This is a
+#   # DIFFERENT token from the agent ingest tokens in dynakube.yaml.
+#
+# Each vendor is disabled by default. Enable a vendor and supply its non-secret
+# configuration below, then export the matching secret environment variable(s).
+
+observability_vendors:
+  datadog:
+    enabled: false
+    # Path to the datadog-agent.yaml (DatadogAgent CR) shown in the Datadog
+    # onboarding UI. This file is NON-SECRET (it references the api/app keys by
+    # Secret name) so it may be committed. It carries site, clusterName, and
+    # feature toggles.
+    manifest_path: "secrets/datadog/datadog-agent.yaml"
+    # Name and keys of the Kubernetes Secret created from DATADOG_API_KEY /
+    # DATADOG_APP_KEY. These MUST match the credentials.apiSecret/appSecret
+    # references inside the DatadogAgent CR above.
+    secret_name: datadog-secret
+    api_key_name: api-key
+    app_key_name: app-key
+  dynatrace:
+    enabled: false
+    # Path to the dynakube.yaml downloaded from the Dynatrace onboarding UI.
+    # This file is SENSITIVE: it contains a Secret (apiToken + dataIngestToken)
+    # AND the DynaKube CR (apiUrl, cluster name, config). Keep OUT of git.
+    dynakube_path: "secrets/dynatrace/dynakube.yaml"  # pragma: allowlist secret
+    # Supported DynaKube API version. The DynaKube document in dynakube.yaml must
+    # use this exact apiVersion; it is also used for readiness and cleanup so all
+    # three stay consistent.
+    dynakube_api_version: "dynatrace.com/v1beta3"
+    # Dynatrace Grail RECORDER (Layer B-ish export). Runs once at teardown as a
+    # Kubernetes Job that back-queries Grail over [scenario start -> now] and
+    # writes JSON+CSV (metrics, spans, logs, events, k8s metrics) into the
+    # recorder export directory. Independent of the agent deploy above: it only
+    # reads telemetry, and requires a PLATFORM token exported as DT_PLATFORM_TOKEN.
+    recorder:
+      enabled: false
+      # Dynatrace platform (apps) URL — NOT the live/ingest URL. NON-SECRET.
+      # e.g. https://<env-id>.apps.dynatrace.com
+      platform_url: ""
+      # Path to a file containing the platform token (prefix dt0s16.) with Grail
+      # read permissions. SENSITIVE: keep under secrets/ (gitignored). The file
+      # should contain only the token.
+      platform_token_path: "secrets/dynatrace/platform_token"  # pragma: allowlist secret
+      # Kubernetes namespace whose telemetry is exported.
+      namespace: otel-demo
+      # Query end (Grail relative time or ISO-8601). The start is taken from the
+      # scenario start time stamped at install; fallback_from is used only if
+      # that marker is missing.
+      timeframe_to: now
+      fallback_from: now-1h
+      # Timeseries bucket size for metric/span datasets.
+      interval: 1m
+      # Row cap for the logs/events datasets.
+      limit: 1000
+      # Output format: json | csv | both.
+      format: both

scenarios/sre/inventory/group_vars/runner/vendors.yaml.example

@@ -0,0 +1,30 @@
+---
+# Observability vendor configuration for the AWX runner entry point
+# (manage_awx.yaml). This drives both:
+#   - AWX credential creation (secret inputs: manifest/dynakube contents, keys)
+#   - Non-secret job extra_vars (enable flags + config), projected automatically
+#     with secrets stripped in roles/awx/tasks/configure_jobs.yaml
+#
+# SECURITY: this file contains secrets. DO NOT commit it in plaintext. Encrypt it
+# with Ansible Vault (ansible-vault encrypt vendors.yaml) or source the secret
+# fields from a secret manager. The provisioning tasks use no_log; AWX stores the
+# resulting credentials encrypted at rest.
+#
+# vendors:
+#   datadog:
+#     enabled: true
+#     # Non-secret config (also consumed by the tools role):
+#     manifest_path: "secrets/datadog/datadog-agent.yaml"
+#     secret_name: datadog-secret
+#     api_key_name: api-key
+#     app_key_name: app-key
+#     # Secret inputs (injected into AWX credentials, stripped from job vars):
+#     manifest: |
+#       # full contents of the datadog-agent.yaml downloaded from Datadog
+#     api_key: ""       # pragma: allowlist secret
+#     app_key: ""       # pragma: allowlist secret
+#   dynatrace:
+#     enabled: true
+#     dynakube_path: "secrets/dynatrace/dynakube.yaml"
+#     dynakube: |
+#       # full contents of the dynakube.yaml downloaded from Dynatrace

scenarios/sre/project/manage_awx.yaml

@@ -36,6 +36,13 @@
           when:
             - cluster_provider != "kind"
 
+        - name: Add observability vendors variable
+          ansible.builtin.set_fact:
+            awx_configuration: "{{ awx_configuration | ansible.builtin.combine({'vendors': vendors}) }}"
+          when:
+            - vendors is ansible.builtin.defined
+          no_log: true
+
         - name: Create experiments variable
           ansible.builtin.set_fact:
             awx_experiments:

scenarios/sre/project/manage_recorders.yaml

@@ -95,3 +95,14 @@
         recorders_cluster:
           kubeconfig: "{{ cluster.kubeconfig }}"
           platform: "{{ cluster_platform }}"
+        # Pass ONLY the recorder subset the role declares (not the whole
+        # observability_vendors blob, which also carries agent-deploy keys the
+        # recorders role's argument spec does not accept).
+        recorders_vendors:
+          dynatrace:
+            recorder: >-
+              {{
+                (observability_vendors | ansible.builtin.default({}))
+                .get('dynatrace', {})
+                .get('recorder', {'enabled': false})
+              }}

scenarios/sre/project/manage_tools.yaml

@@ -32,7 +32,11 @@
 
         - name: Create tool installation variable
           ansible.builtin.set_fact:
-            tools_configuration: "{{ scenarios_scenario.spec.tools }}"
+            tools_configuration: >-
+              {{
+                scenarios_scenario.spec.tools |
+                ansible.builtin.combine({'vendors': observability_vendors | ansible.builtin.default({})})
+              }}
 
     - name: Create role variables
       tags:
@@ -47,6 +51,7 @@
                 enabled: "{{ tools.sre | ansible.builtin.default(false) }}"
               finops:
                 enabled: "{{ tools.finops | ansible.builtin.default(false) }}"
+              vendors: "{{ observability_vendors | ansible.builtin.default({}) }}"
 
     - name: Import tools role
       ansible.builtin.import_role:

scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml

@@ -68,6 +68,95 @@
             name: system:openshift:scc:anyuid
         state: present
 
+# --- Dynatrace OTLP ingest (optional). ---------------------------------------
+# When the Dynatrace vendor is enabled, route the otel-demo collector's OTLP
+# traces/metrics/logs to Dynatrace in addition to the in-cluster backends. The
+# dataIngestToken and apiUrl are reused from the same dynakube.yaml consumed by
+# the tools role (roles/tools/tasks/install_dynatrace.yaml), so no new token is
+# needed. Everything here is gated on the vendor flag; when disabled the demo
+# renders and installs exactly as before.
+- name: Configure Dynatrace OTLP ingest for OpenTelemetry Demo
+  when:
+    - observability_vendors.dynatrace.enabled | ansible.builtin.default(false)
+  block:
+    - name: Resolve the Dynatrace DynaKube manifest path
+      ansible.builtin.set_fact:
+        applications_dynatrace_dynakube_path: >-
+          {{
+            dynatrace_dynakube_path |
+            ansible.builtin.default(observability_vendors.dynatrace.dynakube_path, true)
+          }}
+
+    # Relative paths are anchored to the SRE project root (parent of the playbook
+    # directory), matching install_dynatrace.yaml; absolute paths are used as-is.
+    - name: Anchor a relative Dynatrace manifest path to the project root
+      ansible.builtin.set_fact:
+        applications_dynatrace_dynakube_path: >-
+          {{
+            [playbook_dir, '..', applications_dynatrace_dynakube_path]
+            | ansible.builtin.path_join | ansible.builtin.realpath
+          }}
+      when:
+        - not (applications_dynatrace_dynakube_path is ansible.builtin.abs)
+
+    - name: Parse the Dynatrace DynaKube manifest documents
+      ansible.builtin.set_fact:
+        applications_dynatrace_documents: >-
+          {{
+            lookup('ansible.builtin.file', applications_dynatrace_dynakube_path)
+            | ansible.builtin.from_yaml_all | list | ansible.builtin.reject('none') | list
+          }}
+      no_log: true
+
+    - name: Derive Dynatrace ingest facts from the manifest
+      vars:
+        applications_dynatrace_secret: >-
+          {{
+            applications_dynatrace_documents
+            | selectattr('kind', 'equalto', 'Secret') | list | first
+          }}
+        applications_dynatrace_dynakube: >-
+          {{
+            applications_dynatrace_documents
+            | selectattr('kind', 'equalto', 'DynaKube') | list | first
+          }}
+      ansible.builtin.set_fact:
+        # dataIngestToken is base64-encoded in the Secret's data map; decode to
+        # the raw token for the Authorization header value.
+        applications_dynatrace_ingest_token: "{{ applications_dynatrace_secret.data.dataIngestToken | ansible.builtin.b64decode }}"
+        # OTLP endpoint is the DynaKube apiUrl (…/api) plus the OTLP v2 path.
+        applications_dynatrace_otlp_endpoint: "{{ applications_dynatrace_dynakube.spec.apiUrl }}/v2/otlp"
+        # Cluster name for entity correlation: explicit override, else DynaKube name.
+        applications_dynatrace_cluster_name: >-
+          {{
+            applications_dynatrace_cluster_name |
+            ansible.builtin.default(applications_dynatrace_dynakube.metadata.name, true)
+          }}
+      no_log: true
+
+    - name: Create the Dynatrace OTLP ingest Secret in the demo namespace
+      kubernetes.core.k8s:
+        kubeconfig: "{{ applications_cluster.kubeconfig }}"
+        resource_definition:
+          apiVersion: v1
+          kind: Secret
+          metadata:
+            name: dynatrace-otlp-ingest
+            namespace: "{{ applications_releases.opentelemetry_demo.namespace }}"
+            labels:
+              app.kubernetes.io/managed-by: ITBench
+              itbench.io/observability-vendor: dynatrace
+          type: Opaque
+          stringData:
+            DT_ENDPOINT: "{{ applications_dynatrace_otlp_endpoint }}"
+            DT_API_TOKEN: "{{ applications_dynatrace_ingest_token }}"
+        state: present
+      no_log: true
+
+    - name: Enable Dynatrace ingest in the OpenTelemetry Demo values
+      ansible.builtin.set_fact:
+        applications_dynatrace_ingest_enabled: true
+
 - name: Install OpenTelemetry Demo (Astronomy Shop)
   kubernetes.core.helm:
     chart_ref: "oci://ghcr.io/open-telemetry/opentelemetry-helm-charts/opentelemetry-demo"

scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml

@@ -7,6 +7,22 @@
     release_state: absent
     wait: true
 
+# Explicitly remove the projected Dynatrace ingest secret. The namespace
+# deletion below also removes it, but this keeps teardown symmetric with the
+# install and covers cases where the namespace is retained.
+- name: Delete the Dynatrace OTLP ingest Secret
+  kubernetes.core.k8s:
+    kubeconfig: "{{ applications_cluster.kubeconfig }}"
+    resource_definition:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: dynatrace-otlp-ingest
+        namespace: "{{ applications_releases.opentelemetry_demo.namespace }}"
+    state: absent
+  when:
+    - observability_vendors.dynatrace.enabled | ansible.builtin.default(false)
+
 - name: Delete Namespace
   kubernetes.core.k8s:
     kubeconfig: "{{ applications_cluster.kubeconfig }}"