Expand framework and platform compatibility coverage

.github/workflows/ci.yml

@@ -41,7 +41,7 @@ jobs:
         run: go mod verify
 
       - name: Install Python test dependencies
-        run: python -m pip install "ddtrace>=4.10.3" pytest
+        run: python -m pip install "ddtrace>=4.11.0" pytest
 
       - name: Format check
         run: |
@@ -78,12 +78,197 @@ jobs:
           format: go-coverprofile
           version: v5.19.0
 
+  rspec-compatibility:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - rspec: "3.12.0"
+            datadog_ci: "1.31.0"
+          - rspec: "3.13.2"
+            datadog_ci: "1.37.0"
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
+        with:
+          go-version: "1.26.5"
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1
+        with:
+          ruby-version: "3.4"
+
+      - name: Test RSpec adapter
+        run: |
+          gem install datadog-ci --version "${{ matrix.datadog_ci }}" --no-document
+          gem install rspec --version "${{ matrix.rspec }}" --no-document
+          DDTEST_RSPEC_VERSION="${{ matrix.rspec }}" \
+            DDTEST_DATADOG_CI_VERSION="${{ matrix.datadog_ci }}" \
+            go test -v ./internal/compatibility -run '^TestRSpecAdapterIntegration$'
+          DDTEST_RUBY_PLATFORM_VERSION="${{ matrix.datadog_ci }}" \
+            go test -v ./internal/compatibility -run '^TestRubyPlatformIntegration$'
+
+  minitest-compatibility:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - minitest: "5.26.0"
+            datadog_ci: "1.31.0"
+          - minitest: "6.0.6"
+            datadog_ci: "1.37.0"
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
+        with:
+          go-version: "1.26.5"
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1
+        with:
+          ruby-version: "3.4"
+
+      - name: Test Minitest adapter
+        run: |
+          gem install datadog-ci --version "${{ matrix.datadog_ci }}" --no-document
+          gem install minitest --version "${{ matrix.minitest }}" --no-document
+          gem install rake --version "13.2.1" --no-document
+          DDTEST_MINITEST_VERSION="${{ matrix.minitest }}" \
+            DDTEST_DATADOG_CI_VERSION="${{ matrix.datadog_ci }}" \
+            go test -v ./internal/compatibility -run '^TestMinitestAdapterIntegration$'
+
+  pytest-compatibility:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - pytest: "8.4.2"
+            ddtrace: "4.11.0"
+          - pytest: "9.1.1"
+            ddtrace: "4.14.0"
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
+        with:
+          go-version: "1.26.5"
+
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Test pytest adapter
+        run: |
+          python -m pip install "ddtrace==${{ matrix.ddtrace }}" "pytest==${{ matrix.pytest }}"
+          DDTEST_PYTHON_BINARY="$(command -v python)" \
+            go test -v ./internal/compatibility -run '^TestPyTestAdapterIntegration$'
+          DDTEST_PYTHON_PLATFORM_INTEGRATION=1 \
+            go test -v ./internal/compatibility -run '^TestPythonPlatformIntegration$'
+
+  javascript-platform-compatibility:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        dd_trace: ["5.111.0", "6.15.0"]
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
+        with:
+          go-version: "1.26.5"
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22"
+
+      - name: Test JavaScript platform
+        run: |
+          dd_trace_dir="${RUNNER_TEMP}/dd-trace-${{ matrix.dd_trace }}"
+          npm install --prefix "${dd_trace_dir}" "dd-trace@${{ matrix.dd_trace }}"
+          DDTEST_DD_TRACE_NODE_MODULES="${dd_trace_dir}/node_modules" \
+            go test -v ./internal/compatibility -run '^TestJavaScriptPlatformIntegration$'
+
+  jest-compatibility:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        jest: ["27.5.1", "28.1.3", "29.7.0", "30.5.1"]
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
+        with:
+          go-version: "1.26.5"
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22"
+
+      - name: Test Jest adapter
+        run: |
+          jest_dir="${RUNNER_TEMP}/jest-${{ matrix.jest }}"
+          npm install --prefix "${jest_dir}" "jest@${{ matrix.jest }}"
+          DDTEST_JEST_NODE_MODULES="${jest_dir}/node_modules" \
+            go test -v ./internal/compatibility -run '^TestJestAdapterIntegration$'
+
+  vitest-compatibility:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        vitest: ["1.6.1", "2.1.9", "3.2.7", "4.1.11", "5.0.0"]
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
+        with:
+          go-version: "1.26.5"
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22"
+
+      - name: Test Vitest adapter
+        run: |
+          vitest_dir="${RUNNER_TEMP}/vitest-${{ matrix.vitest }}"
+          npm_install_args=()
+          if [[ "${{ matrix.vitest }}" == "4.1.11" ]]; then
+            # npm 10.9.8 crashes while resolving Vitest 4's duplicated Vite dependency/peer.
+            npm_install_args+=(--legacy-peer-deps)
+          fi
+          npm install --prefix "${vitest_dir}" "${npm_install_args[@]}" "vitest@${{ matrix.vitest }}"
+          DDTEST_VITEST_NODE_MODULES="${vitest_dir}/node_modules" \
+            go test -v ./internal/compatibility -run '^TestVitestAdapterIntegration$'
+
   mocha-compatibility:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        mocha: ["8.4.0", "9.2.2", "10.8.2", "11.7.6"]
+        mocha: ["8.4.0", "9.2.2", "10.8.2", "11.7.6", "12.0.0"]
 
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -93,19 +278,24 @@ jobs:
         with:
           go-version: "1.26.5"
 
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22"
+
       - name: Test Mocha adapter
         run: |
           mocha_dir="${RUNNER_TEMP}/mocha-${{ matrix.mocha }}"
           npm install --prefix "${mocha_dir}" "mocha@${{ matrix.mocha }}"
           DDTEST_MOCHA_NODE_MODULES="${mocha_dir}/node_modules" \
-            go test -v ./internal/framework -run TestMochaAdapterIntegration
+            go test -v ./internal/compatibility -run '^TestMochaAdapter.*Integration$'
 
   cypress-compatibility:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        cypress: ["12.17.4", "15.16.0"]
+        cypress: ["12.17.4", "13.17.0", "14.5.4", "15.21.1", "16.0.0"]
 
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -115,20 +305,26 @@ jobs:
         with:
           go-version: "1.26.5"
 
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22"
+
       - name: Test Cypress adapter
         run: |
           cypress_dir="${RUNNER_TEMP}/cypress-${{ matrix.cypress }}"
           npm install --prefix "${cypress_dir}" "cypress@${{ matrix.cypress }}"
           DDTEST_CYPRESS_BINARY="${cypress_dir}/node_modules/.bin/cypress" \
             DDTEST_CYPRESS_NODE_MODULES="${cypress_dir}/node_modules" \
-            go test -v ./internal/framework -run TestCypressAdapterIntegration
+            DDTEST_CYPRESS_XVFB="$(command -v xvfb-run)" \
+            go test -v ./internal/compatibility -run '^TestCypressAdapter.*Integration$'
 
   playwright-compatibility:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        playwright: ["1.18.0", "1.31.1", "1.38.0", "1.59.1", "1.60.0", "1.62.1"]
+        playwright: ["1.18.0", "1.31.1", "1.38.0", "1.59.1", "1.60.0", "1.62.1", "1.63.0"]
 
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -138,6 +334,11 @@ jobs:
         with:
           go-version: "1.26.5"
 
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22"
+
       - name: Test Playwright adapter
         env:
           PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1"
@@ -146,14 +347,14 @@ jobs:
           npm install --prefix "${playwright_dir}" "@playwright/test@${{ matrix.playwright }}"
           DDTEST_PLAYWRIGHT_BINARY="${playwright_dir}/node_modules/.bin/playwright" \
             DDTEST_PLAYWRIGHT_NODE_MODULES="${playwright_dir}/node_modules" \
-            go test -v ./internal/framework -run TestPlaywrightAdapterIntegration
+            go test -v ./internal/compatibility -run '^TestPlaywrightAdapterIntegration$'
 
   cucumber-compatibility:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        cucumber: ["7.0.0", "10.0.0", "13.2.1"]
+        cucumber: ["7.0.0", "8.11.1", "9.6.0", "10.9.0", "11.3.0", "12.9.0", "13.2.1"]
 
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -175,7 +376,7 @@ jobs:
           DDTEST_CUCUMBER_BINARY="${cucumber_dir}/node_modules/.bin/cucumber-js" \
             DDTEST_CUCUMBER_NODE_MODULES="${cucumber_dir}/node_modules" \
             DDTEST_CUCUMBER_VERSION="${{ matrix.cucumber }}" \
-            go test -v ./internal/framework -run TestCucumberAdapterIntegration
+            go test -v ./internal/compatibility -run '^TestCucumberAdapterIntegration$'
 
   lint:
     runs-on: ubuntu-latest

internal/compatibility/cucumber_test.go

@@ -0,0 +1,80 @@
+package compatibility
+
+import (
+	"context"
+	"os"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/DataDog/ddtest/internal/discovery"
+	"github.com/DataDog/ddtest/internal/framework"
+)
+
+func TestCucumberAdapterIntegration(t *testing.T) {
+	cucumberBinary := requireEnv(t, "DDTEST_CUCUMBER_BINARY")
+	nodeModules := requireEnv(t, "DDTEST_CUCUMBER_NODE_MODULES")
+	cucumberVersion := requireEnv(t, "DDTEST_CUCUMBER_VERSION")
+	resetSettingsAfterTest(t)
+
+	root := t.TempDir()
+	t.Chdir(root)
+	if err := os.Symlink(nodeModules, "node_modules"); err != nil {
+		t.Fatal(err)
+	}
+	cucumberConfig := `module.exports = {
+  default: {
+    tags: 'not @excluded',
+    require: ['features/support/**/*.js']
+  }
+}
+`
+	if strings.HasPrefix(cucumberVersion, "7.") {
+		// Cucumber 7 profiles are CLI argument strings. Object-based profiles were
+		// introduced later and are silently treated as empty by Cucumber 7.
+		cucumberConfig = `module.exports = {
+  default: "--require 'features/support/**/*.js' --tags 'not @excluded'"
+}
+`
+	}
+	files := map[string]string{
+		"cucumber.js": cucumberConfig,
+		"features/included.feature": `Feature: included
+  Scenario: selected by the default profile
+    Given a passing step
+`,
+		"features/unassigned.feature": `Feature: unassigned
+  Scenario: must not run
+    Given a failing step
+`,
+		"features/excluded.feature": `@excluded
+Feature: excluded
+  Scenario: filtered by the default profile
+    Given a passing step
+`,
+		"features/support/steps.js": `const { Given } = require('@cucumber/cucumber')
+Given('a passing step', function () {})
+Given('a failing step', function () { throw new Error('unassigned file ran') })
+`,
+	}
+	for filename, content := range files {
+		writeFixture(t, root, filename, content)
+	}
+
+	configureFramework(shellCommand(cucumberBinary), "")
+	cucumber := framework.NewCucumber()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	discovered, err := cucumber.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: cucumber.TestPattern()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	wantFiles := []string{"features/included.feature", "features/unassigned.feature"}
+	if !slices.Equal(discovered, wantFiles) {
+		t.Fatalf("discovered = %v", discovered)
+	}
+	if err := cucumber.RunTests(ctx, []string{"features/included.feature"}, nil); err != nil {
+		t.Fatalf("selected-file run failed: %v", err)
+	}
+}