From 3dfabc7f31ab90f4308dabaf755de2c295462d3c Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 9 Sep 2026 12:57:37 +0200 Subject: [PATCH 1/4] Expand framework compatibility coverage --- .github/workflows/ci.yml | 212 +++++++++++++++++- internal/framework/compatibility_test.go | 77 +++++++ internal/framework/cucumber_test.go | 19 +- internal/framework/cypress_test.go | 56 +++++ internal/framework/jest_test.go | 43 ++++ internal/framework/minitest_test.go | 69 ++++++ internal/framework/mocha_test.go | 15 +- internal/framework/playwright_test.go | 11 +- internal/framework/pytest_test.go | 47 ++++ internal/framework/rspec_test.go | 53 +++++ internal/framework/scripts/mocha_adapter.js | 22 +- .../framework/scripts/vitest_v1_discovery.mjs | 2 +- internal/framework/vitest.go | 6 +- internal/framework/vitest_test.go | 52 +++++ internal/platform/compatibility_test.go | 102 +++++++++ internal/platform/platform_test.go | 4 +- internal/platform/python.go | 2 +- internal/platform/python_test.go | 6 +- 18 files changed, 765 insertions(+), 33 deletions(-) create mode 100644 internal/framework/compatibility_test.go create mode 100644 internal/platform/compatibility_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b32bca4..c072f94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: run: go mod verify - name: Install Python test dependencies - run: python -m pip install "ddtrace>=4.10.3" pytest + run: python -m pip install "ddtrace>=4.11.0" pytest - name: Format check run: | @@ -78,12 +78,192 @@ jobs: format: go-coverprofile version: v5.19.0 + rspec-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - rspec: "3.12.0" + datadog_ci: "1.31.0" + - rspec: "3.13.2" + datadog_ci: "1.37.0" + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version: "1.26.5" + + - name: Set up Ruby + uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1 + with: + ruby-version: "3.4" + + - name: Test RSpec adapter + run: | + gem install datadog-ci --version "${{ matrix.datadog_ci }}" --no-document + gem install rspec --version "${{ matrix.rspec }}" --no-document + DDTEST_RSPEC_VERSION="${{ matrix.rspec }}" \ + DDTEST_DATADOG_CI_VERSION="${{ matrix.datadog_ci }}" \ + go test -v ./internal/framework -run '^TestRSpecAdapterIntegration$' + DDTEST_RUBY_PLATFORM_VERSION="${{ matrix.datadog_ci }}" \ + go test -v ./internal/platform -run '^TestRubyPlatformIntegration$' + + minitest-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - minitest: "5.26.0" + datadog_ci: "1.31.0" + - minitest: "6.0.6" + datadog_ci: "1.37.0" + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version: "1.26.5" + + - name: Set up Ruby + uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1 + with: + ruby-version: "3.4" + + - name: Test Minitest adapter + run: | + gem install datadog-ci --version "${{ matrix.datadog_ci }}" --no-document + gem install minitest --version "${{ matrix.minitest }}" --no-document + gem install rake --version "13.2.1" --no-document + DDTEST_MINITEST_VERSION="${{ matrix.minitest }}" \ + DDTEST_DATADOG_CI_VERSION="${{ matrix.datadog_ci }}" \ + go test -v ./internal/framework -run '^TestMinitestAdapterIntegration$' + + pytest-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - pytest: "8.4.2" + ddtrace: "4.11.0" + - pytest: "9.1.1" + ddtrace: "4.14.0" + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version: "1.26.5" + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Test pytest adapter + run: | + python -m pip install "ddtrace==${{ matrix.ddtrace }}" "pytest==${{ matrix.pytest }}" + DDTEST_PYTHON_BINARY="$(command -v python)" \ + go test -v ./internal/framework -run '^TestPyTestAdapterIntegration$' + DDTEST_PYTHON_PLATFORM_INTEGRATION=1 \ + go test -v ./internal/platform -run '^TestPythonPlatformIntegration$' + + javascript-platform-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dd_trace: ["5.111.0", "6.15.0"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version: "1.26.5" + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + + - name: Test JavaScript platform + run: | + dd_trace_dir="${RUNNER_TEMP}/dd-trace-${{ matrix.dd_trace }}" + npm install --prefix "${dd_trace_dir}" "dd-trace@${{ matrix.dd_trace }}" + DDTEST_DD_TRACE_NODE_MODULES="${dd_trace_dir}/node_modules" \ + go test -v ./internal/platform -run '^TestJavaScriptPlatformIntegration$' + + jest-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + jest: ["27.5.1", "28.1.3", "29.7.0", "30.5.1"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version: "1.26.5" + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + + - name: Test Jest adapter + run: | + jest_dir="${RUNNER_TEMP}/jest-${{ matrix.jest }}" + npm install --prefix "${jest_dir}" "jest@${{ matrix.jest }}" + DDTEST_JEST_NODE_MODULES="${jest_dir}/node_modules" \ + go test -v ./internal/framework -run '^TestJestAdapterIntegration$' + + vitest-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + vitest: ["1.6.1", "2.1.9", "3.2.7", "4.1.11", "5.0.0"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version: "1.26.5" + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + + - name: Test Vitest adapter + run: | + vitest_dir="${RUNNER_TEMP}/vitest-${{ matrix.vitest }}" + npm install --prefix "${vitest_dir}" "vitest@${{ matrix.vitest }}" + DDTEST_VITEST_NODE_MODULES="${vitest_dir}/node_modules" \ + go test -v ./internal/framework -run '^TestVitestAdapterIntegration$' + mocha-compatibility: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - mocha: ["8.4.0", "9.2.2", "10.8.2", "11.7.6"] + mocha: ["8.4.0", "9.2.2", "10.8.2", "11.7.6", "12.0.0"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -93,19 +273,24 @@ jobs: with: go-version: "1.26.5" + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - name: Test Mocha adapter run: | mocha_dir="${RUNNER_TEMP}/mocha-${{ matrix.mocha }}" npm install --prefix "${mocha_dir}" "mocha@${{ matrix.mocha }}" DDTEST_MOCHA_NODE_MODULES="${mocha_dir}/node_modules" \ - go test -v ./internal/framework -run TestMochaAdapterIntegration + go test -v ./internal/framework -run '^TestMochaAdapter.*Integration$' cypress-compatibility: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - cypress: ["12.17.4", "15.16.0"] + cypress: ["12.17.4", "13.17.0", "14.5.4", "15.21.1", "16.0.0"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -115,20 +300,26 @@ jobs: with: go-version: "1.26.5" + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - name: Test Cypress adapter run: | cypress_dir="${RUNNER_TEMP}/cypress-${{ matrix.cypress }}" npm install --prefix "${cypress_dir}" "cypress@${{ matrix.cypress }}" DDTEST_CYPRESS_BINARY="${cypress_dir}/node_modules/.bin/cypress" \ DDTEST_CYPRESS_NODE_MODULES="${cypress_dir}/node_modules" \ - go test -v ./internal/framework -run TestCypressAdapterIntegration + DDTEST_CYPRESS_XVFB="$(command -v xvfb-run)" \ + go test -v ./internal/framework -run '^TestCypressAdapter.*Integration$' playwright-compatibility: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - playwright: ["1.18.0", "1.31.1", "1.38.0", "1.59.1", "1.60.0", "1.62.1"] + playwright: ["1.18.0", "1.31.1", "1.38.0", "1.59.1", "1.60.0", "1.62.1", "1.63.0"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -138,6 +329,11 @@ jobs: with: go-version: "1.26.5" + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - name: Test Playwright adapter env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" @@ -153,7 +349,7 @@ jobs: strategy: fail-fast: false matrix: - cucumber: ["7.0.0", "10.0.0", "13.2.1"] + cucumber: ["7.0.0", "8.11.1", "9.6.0", "10.9.0", "11.3.0", "12.9.0", "13.2.1"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -175,7 +371,7 @@ jobs: DDTEST_CUCUMBER_BINARY="${cucumber_dir}/node_modules/.bin/cucumber-js" \ DDTEST_CUCUMBER_NODE_MODULES="${cucumber_dir}/node_modules" \ DDTEST_CUCUMBER_VERSION="${{ matrix.cucumber }}" \ - go test -v ./internal/framework -run TestCucumberAdapterIntegration + go test -v ./internal/framework -run '^TestCucumberAdapterIntegration$' lint: runs-on: ubuntu-latest diff --git a/internal/framework/compatibility_test.go b/internal/framework/compatibility_test.go new file mode 100644 index 0000000..507c0b8 --- /dev/null +++ b/internal/framework/compatibility_test.go @@ -0,0 +1,77 @@ +package framework + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/DataDog/ddtest/internal/testoptimization" + "github.com/DataDog/ddtest/internal/utils" +) + +func requireCompatibilityEnv(t *testing.T, name string) string { + t.Helper() + value := os.Getenv(name) + if value == "" { + t.Skipf("%s is not set", name) + } + return value +} + +func writeCompatibilityFixture(t *testing.T, root, name, contents string) { + t.Helper() + path := filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(contents), 0644); err != nil { + t.Fatal(err) + } +} + +func requireCompatibilityFiles(t *testing.T, got, want []string) { + t.Helper() + normalized := slices.Clone(got) + for i := range normalized { + normalized[i] = utils.NormalizePath(normalized[i]) + } + slices.Sort(normalized) + + expected := slices.Clone(want) + for i := range expected { + expected[i] = utils.NormalizePath(expected[i]) + } + slices.Sort(expected) + + if !slices.Equal(normalized, expected) { + t.Fatalf("files = %v, want %v", normalized, expected) + } +} + +func requireCompatibilityTestSources(t *testing.T, tests []testoptimization.Test, want []string) { + t.Helper() + if len(tests) != len(want) { + t.Fatalf("discovered %d tests, want %d: %+v", len(tests), len(want), tests) + } + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + sources := make([]string, 0, len(tests)) + for _, test := range tests { + if strings.TrimSpace(test.Name) == "" { + t.Fatalf("discovered test has no name: %+v", test) + } + source := filepath.FromSlash(test.SuiteSourceFile) + if filepath.IsAbs(source) { + if relative, relativeErr := filepath.Rel(cwd, source); relativeErr == nil { + source = relative + } + } + sources = append(sources, utils.NormalizePath(source)) + } + requireCompatibilityFiles(t, sources, want) +} diff --git a/internal/framework/cucumber_test.go b/internal/framework/cucumber_test.go index 69e3a5a..6242f75 100644 --- a/internal/framework/cucumber_test.go +++ b/internal/framework/cucumber_test.go @@ -9,6 +9,7 @@ import ( "slices" "strings" "testing" + "time" "github.com/DataDog/ddtest/internal/discovery" "github.com/DataDog/ddtest/internal/ext" @@ -397,7 +398,6 @@ func TestCucumberAdapterIntegration(t *testing.T) { } cucumberConfig := `module.exports = { default: { - paths: ['features/**/*.feature'], tags: 'not @excluded', require: ['features/support/**/*.js'] } @@ -407,7 +407,7 @@ func TestCucumberAdapterIntegration(t *testing.T) { // Cucumber 7 profiles are CLI argument strings. Object-based profiles were // introduced later and are silently treated as empty by Cucumber 7. cucumberConfig = `module.exports = { - default: "--require 'features/support/**/*.js' --tags 'not @excluded' 'features/**/*.feature'" + default: "--require 'features/support/**/*.js' --tags 'not @excluded'" } ` } @@ -416,6 +416,10 @@ func TestCucumberAdapterIntegration(t *testing.T) { "features/included.feature": `Feature: included Scenario: selected by the default profile Given a passing step +`, + "features/unassigned.feature": `Feature: unassigned + Scenario: must not run + Given a failing step `, "features/excluded.feature": `@excluded Feature: excluded @@ -424,6 +428,7 @@ Feature: excluded `, "features/support/steps.js": `const { Given } = require('@cucumber/cucumber') Given('a passing step', function () {}) +Given('a failing step', function () { throw new Error('unassigned file ran') }) `, } for filename, content := range files { @@ -440,11 +445,17 @@ Given('a passing step', function () {}) commandOverride: []string{cucumberBinary}, platformEnv: map[string]string{}, } - discovered, err := cucumber.DiscoverTestFiles(context.Background(), discovery.TestFileSet{Pattern: cucumber.TestPattern()}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + discovered, err := cucumber.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: cucumber.TestPattern()}) if err != nil { t.Fatal(err) } - if !slices.Equal(discovered, []string{"features/included.feature"}) { + wantFiles := []string{"features/included.feature", "features/unassigned.feature"} + if !slices.Equal(discovered, wantFiles) { t.Fatalf("discovered = %v", discovered) } + if err := cucumber.RunTests(ctx, []string{"features/included.feature"}, nil); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } } diff --git a/internal/framework/cypress_test.go b/internal/framework/cypress_test.go index 58d8178..faee56e 100644 --- a/internal/framework/cypress_test.go +++ b/internal/framework/cypress_test.go @@ -448,6 +448,62 @@ func TestCypressAdapterIntegration(t *testing.T) { } } +func TestCypressAdapterExecutionIntegration(t *testing.T) { + binary := requireCompatibilityEnv(t, "DDTEST_CYPRESS_BINARY") + nodeModules := requireCompatibilityEnv(t, "DDTEST_CYPRESS_NODE_MODULES") + + root := t.TempDir() + if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { + t.Fatal(err) + } + writeCompatibilityFixture(t, root, "cypress.config.js", `module.exports = { + video: false, + viewportWidth: 777, + e2e: { + supportFile: false, + specPattern: 'cypress/e2e/**/*.cy.js', + }, +} +`) + writeCompatibilityFixture(t, root, "cypress/e2e/selected.cy.js", `describe('selected', () => { + it('runs an assigned file', () => { + expect(true).to.equal(true) + expect(Cypress.config('viewportWidth')).to.equal(777) + }) +}) +`) + writeCompatibilityFixture(t, root, "cypress/e2e/unselected.cy.js", `describe('unselected', () => { + it('must not run', () => { + throw new Error('unselected file ran') + }) +}) +`) + t.Chdir(root) + + command := []string{binary, "run"} + if xvfb := os.Getenv("DDTEST_CYPRESS_XVFB"); xvfb != "" { + command = []string{xvfb, "-a", binary, "run"} + } + cypress := &Cypress{ + executor: &ext.DefaultCommandExecutor{}, + commandOverride: command, + platformEnv: map[string]string{}, + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + files, err := cypress.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: cypress.TestPattern()}) + if err != nil { + t.Fatal(err) + } + wantFiles := []string{"cypress/e2e/selected.cy.js", "cypress/e2e/unselected.cy.js"} + requireCompatibilityFiles(t, files, wantFiles) + + if err := cypress.RunTests(ctx, []string{"cypress/e2e/selected.cy.js"}, nil); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } +} + func TestCypressDiscoverySymlinkCycle(t *testing.T) { node, err := exec.LookPath("node") if err != nil { diff --git a/internal/framework/jest_test.go b/internal/framework/jest_test.go index fe15bd1..d0bb776 100644 --- a/internal/framework/jest_test.go +++ b/internal/framework/jest_test.go @@ -8,8 +8,10 @@ import ( "slices" "strings" "testing" + "time" "github.com/DataDog/ddtest/internal/discovery" + "github.com/DataDog/ddtest/internal/ext" ) type jestCommandExecutor struct { @@ -501,3 +503,44 @@ func TestJest_RunTests_WithOverride(t *testing.T) { t.Errorf("expected args %v, got %v", expectedArgs, capturedArgs) } } + +func TestJestAdapterIntegration(t *testing.T) { + nodeModules := requireCompatibilityEnv(t, "DDTEST_JEST_NODE_MODULES") + + root := t.TempDir() + if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { + t.Fatal(err) + } + writeCompatibilityFixture(t, root, "jest.config.js", `module.exports = { + testMatch: ['/tests/**/*.test.js'], + setupFilesAfterEnv: ['/setup.js'], +} +`) + writeCompatibilityFixture(t, root, "setup.js", "globalThis.ddtestJestSetup = true\n") + writeCompatibilityFixture(t, root, "tests/selected.test.js", `test('preserves config while running an assigned file', () => { + expect(globalThis.ddtestJestSetup).toBe(true) + expect(process.env.DDTEST_JEST_WORKER).toBe('selected') +}) +`) + writeCompatibilityFixture(t, root, "tests/unselected.test.js", `test('must not run', () => { + throw new Error('unselected file ran') +}) +`) + t.Chdir(root) + + jest := &Jest{executor: &ext.DefaultCommandExecutor{}, platformEnv: map[string]string{}} + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + testFiles := discovery.TestFileSet{Pattern: jest.TestPattern()} + files, err := jest.DiscoverTestFiles(ctx, testFiles) + if err != nil { + t.Fatal(err) + } + wantFiles := []string{"tests/selected.test.js", "tests/unselected.test.js"} + requireCompatibilityFiles(t, files, wantFiles) + + if err := jest.RunTests(ctx, []string{"tests/selected.test.js"}, map[string]string{"DDTEST_JEST_WORKER": "selected"}); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } +} diff --git a/internal/framework/minitest_test.go b/internal/framework/minitest_test.go index 42201ad..1fe4765 100644 --- a/internal/framework/minitest_test.go +++ b/internal/framework/minitest_test.go @@ -3,13 +3,16 @@ package framework import ( "context" "encoding/json" + "fmt" "os" "os/exec" "path/filepath" "slices" "testing" + "time" "github.com/DataDog/ddtest/internal/discovery" + "github.com/DataDog/ddtest/internal/ext" "github.com/DataDog/ddtest/internal/settings" "github.com/DataDog/ddtest/internal/testoptimization" ) @@ -1745,3 +1748,69 @@ func TestMinitest_HasUnskippableMarker(t *testing.T) { t.Fatal("expected missing file to be treated as guarded") } } + +func TestMinitestAdapterIntegration(t *testing.T) { + minitestVersion := requireCompatibilityEnv(t, "DDTEST_MINITEST_VERSION") + datadogVersion := requireCompatibilityEnv(t, "DDTEST_DATADOG_CI_VERSION") + + root := t.TempDir() + writeCompatibilityFixture(t, root, "Gemfile", fmt.Sprintf(`source "https://rubygems.org" +gem "datadog-ci", %q +gem "minitest", %q +gem "rake", "13.2.1" +`, datadogVersion, minitestVersion)) + writeCompatibilityFixture(t, root, "Rakefile", `require "rake/testtask" + +Rake::TestTask.new(:test) do |test| + test.test_files = ENV["TEST_FILES"] ? ENV["TEST_FILES"].split : FileList["test/**/*_test.rb"] +end + +task default: :test +`) + writeCompatibilityFixture(t, root, "test/test_helper.rb", `gem "minitest", ENV.fetch("DDTEST_MINITEST_VERSION") +require "minitest/autorun" +DDTEST_MINITEST_SETUP = true +`) + writeCompatibilityFixture(t, root, "test/selected_test.rb", `require_relative "test_helper" + +class SelectedTest < Minitest::Test + def test_preserves_setup_while_running_an_assigned_file + assert DDTEST_MINITEST_SETUP + end +end +`) + writeCompatibilityFixture(t, root, "test/unselected_test.rb", `require_relative "test_helper" + +class UnselectedTest < Minitest::Test + def test_must_not_run + flunk "unselected file ran" + end +end +`) + t.Chdir(root) + + minitest := &Minitest{ + executor: &ext.DefaultCommandExecutor{}, + platformEnv: map[string]string{"RUBYOPT": "-rbundler/setup -rdatadog/ci/auto_instrument"}, + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + testFiles := discovery.TestFileSet{Pattern: minitest.TestPattern()} + files, err := minitest.DiscoverTestFiles(ctx, testFiles) + if err != nil { + t.Fatal(err) + } + wantFiles := []string{"test/selected_test.rb", "test/unselected_test.rb"} + requireCompatibilityFiles(t, files, wantFiles) + + tests, err := minitest.DiscoverTests(ctx, testFiles) + if err != nil { + t.Fatalf("full discovery failed: %v", err) + } + requireCompatibilityTestSources(t, tests, wantFiles) + + if err := minitest.RunTests(ctx, []string{"test/selected_test.rb"}, nil); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } +} diff --git a/internal/framework/mocha_test.go b/internal/framework/mocha_test.go index 7163c0f..25349d8 100644 --- a/internal/framework/mocha_test.go +++ b/internal/framework/mocha_test.go @@ -9,6 +9,7 @@ import ( "slices" "strings" "testing" + "time" "github.com/DataDog/ddtest/internal/discovery" "github.com/DataDog/ddtest/internal/ext" @@ -232,7 +233,9 @@ func TestMochaAdapterIntegration(t *testing.T) { t.Chdir(root) mocha := &Mocha{executor: &ext.DefaultCommandExecutor{}, platformEnv: make(map[string]string)} - files, err := mocha.DiscoverTestFiles(context.Background(), discovery.TestFileSet{Pattern: mocha.TestPattern()}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + files, err := mocha.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: mocha.TestPattern()}) if err != nil { t.Fatal(err) } @@ -240,14 +243,14 @@ func TestMochaAdapterIntegration(t *testing.T) { if !slices.Equal(files, want) { t.Fatalf("discovered files = %v, want %v", files, want) } - if err := mocha.RunTests(context.Background(), []string{"test/selected.spec.js"}, nil); err != nil { + if err := mocha.RunTests(ctx, []string{"test/selected.spec.js"}, nil); err != nil { t.Fatalf("selected-file run failed: %v", err) } if err := os.Remove(filepath.Join(root, ".mocharc.json")); err != nil { t.Fatal(err) } - files, err = mocha.DiscoverTestFiles(context.Background(), discovery.TestFileSet{Pattern: mocha.TestPattern()}) + files, err = mocha.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: mocha.TestPattern()}) if err != nil { t.Fatalf("default discovery failed: %v", err) } @@ -279,14 +282,16 @@ func TestMochaAdapterCustomLocationAndCommandIntegration(t *testing.T) { commandOverride: []string{wrapper, mochaCommand}, platformEnv: make(map[string]string), } - files, err := mocha.DiscoverTestFiles(context.Background(), discovery.TestFileSet{Pattern: mocha.TestPattern()}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + files, err := mocha.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: mocha.TestPattern()}) if err != nil { t.Fatal(err) } if !slices.Equal(files, []string{"spec/custom.spec.js"}) { t.Fatalf("discovered files = %v", files) } - if err := mocha.RunTests(context.Background(), files, nil); err != nil { + if err := mocha.RunTests(ctx, files, nil); err != nil { t.Fatalf("custom-command run failed: %v", err) } } diff --git a/internal/framework/playwright_test.go b/internal/framework/playwright_test.go index efdc8ea..ba0e52f 100644 --- a/internal/framework/playwright_test.go +++ b/internal/framework/playwright_test.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "testing" + "time" "github.com/DataDog/ddtest/internal/discovery" "github.com/DataDog/ddtest/internal/ext" @@ -402,7 +403,9 @@ func TestPlaywrightAdapterIntegration(t *testing.T) { commandOverride: []string{binary, "test", "--config", "apps/web/playwright.config.js"}, platformEnv: map[string]string{}, } - files, err := playwright.DiscoverTestFiles(context.Background(), discovery.TestFileSet{}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + files, err := playwright.DiscoverTestFiles(ctx, discovery.TestFileSet{}) if err != nil { t.Fatal(err) } @@ -411,7 +414,7 @@ func TestPlaywrightAdapterIntegration(t *testing.T) { t.Fatalf("files = %v, want %v", files, want) } playwright.commandOverride = []string{binary, "test", "--config", "apps/web/playwright.config.js", "--project", "one"} - if err := playwright.RunTests(context.Background(), []string{"apps/web/tests/a.spec.ts"}, nil); err != nil { + if err := playwright.RunTests(ctx, []string{"apps/web/tests/a.spec.ts"}, nil); err != nil { t.Fatalf("running one assigned file failed: %v", err) } if source, ok := playwright.SourceFileForSuite("a.spec.ts"); !ok || source != "apps/web/tests/a.spec.ts" { @@ -423,7 +426,7 @@ func TestPlaywrightAdapterIntegration(t *testing.T) { commandOverride: []string{binary, "test", "--config", "apps/web/playwright.config.js", "__ddtest_no_match__"}, platformEnv: map[string]string{}, } - if files, err := emptyPlaywright.DiscoverTestFiles(context.Background(), discovery.TestFileSet{}); err != nil || len(files) != 0 { + if files, err := emptyPlaywright.DiscoverTestFiles(ctx, discovery.TestFileSet{}); err != nil || len(files) != 0 { t.Fatalf("empty native discovery = %v, %v", files, err) } @@ -436,7 +439,7 @@ func TestPlaywrightAdapterIntegration(t *testing.T) { commandOverride: []string{binary, "test", "--config", "apps/web/playwright.config.js", "broken.spec.ts"}, platformEnv: map[string]string{}, } - if _, err := brokenPlaywright.DiscoverTestFiles(context.Background(), discovery.TestFileSet{}); err == nil { + if _, err := brokenPlaywright.DiscoverTestFiles(ctx, discovery.TestFileSet{}); err == nil { t.Fatal("collection failure was accepted as an empty discovery") } } diff --git a/internal/framework/pytest_test.go b/internal/framework/pytest_test.go index 1867903..07018e0 100644 --- a/internal/framework/pytest_test.go +++ b/internal/framework/pytest_test.go @@ -7,8 +7,10 @@ import ( "path/filepath" "slices" "testing" + "time" "github.com/DataDog/ddtest/internal/discovery" + "github.com/DataDog/ddtest/internal/ext" "github.com/DataDog/ddtest/internal/testoptimization" ) @@ -320,3 +322,48 @@ func TestPyTest_RunTests(t *testing.T) { t.Errorf("expected run env to override platform env, got %q", mockExecutor.capturedEnvMap["SHARED_VAR"]) } } + +func TestPyTestAdapterIntegration(t *testing.T) { + python := requireCompatibilityEnv(t, "DDTEST_PYTHON_BINARY") + + root := t.TempDir() + writeCompatibilityFixture(t, root, "pytest.ini", `[pytest] +testpaths = checks +python_files = check_*.py +`) + writeCompatibilityFixture(t, root, "checks/check_selected.py", `import os + +def test_preserves_worker_environment(): + assert os.environ["DDTEST_PYTEST_WORKER"] == "selected" +`) + writeCompatibilityFixture(t, root, "checks/check_unselected.py", `def test_must_not_run(): + raise AssertionError("unselected file ran") +`) + t.Chdir(root) + + pytest := &PyTest{ + executor: &ext.DefaultCommandExecutor{}, + commandOverride: []string{python, "-m", "pytest"}, + platformEnv: map[string]string{"PYTEST_ADDOPTS": "--ddtrace"}, + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + testFiles := discovery.TestFileSet{Pattern: pytest.TestPattern()} + files, err := pytest.DiscoverTestFiles(ctx, testFiles) + if err != nil { + t.Fatal(err) + } + wantFiles := []string{"checks/check_selected.py", "checks/check_unselected.py"} + requireCompatibilityFiles(t, files, wantFiles) + + tests, err := pytest.DiscoverTests(ctx, testFiles) + if err != nil { + t.Fatalf("full discovery failed: %v", err) + } + requireCompatibilityTestSources(t, tests, wantFiles) + + if err := pytest.RunTests(ctx, []string{"checks/check_selected.py"}, map[string]string{"DDTEST_PYTEST_WORKER": "selected"}); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } +} diff --git a/internal/framework/rspec_test.go b/internal/framework/rspec_test.go index 72f4156..bdc85ba 100644 --- a/internal/framework/rspec_test.go +++ b/internal/framework/rspec_test.go @@ -3,11 +3,13 @@ package framework import ( "context" "encoding/json" + "fmt" "os" "os/exec" "path/filepath" "slices" "testing" + "time" "github.com/DataDog/ddtest/internal/discovery" "github.com/DataDog/ddtest/internal/ext" @@ -1426,3 +1428,54 @@ func TestRSpec_HasUnskippableMarker(t *testing.T) { t.Fatal("expected missing file to be treated as guarded") } } + +func TestRSpecAdapterIntegration(t *testing.T) { + rspecVersion := requireCompatibilityEnv(t, "DDTEST_RSPEC_VERSION") + datadogVersion := requireCompatibilityEnv(t, "DDTEST_DATADOG_CI_VERSION") + + root := t.TempDir() + writeCompatibilityFixture(t, root, "Gemfile", fmt.Sprintf(`source "https://rubygems.org" +gem "datadog-ci", %q +gem "rspec", %q +`, datadogVersion, rspecVersion)) + writeCompatibilityFixture(t, root, ".rspec", "--require spec_helper\n") + writeCompatibilityFixture(t, root, "spec/spec_helper.rb", "DDTEST_RSPEC_SETUP = true\n") + writeCompatibilityFixture(t, root, "spec/selected_spec.rb", `RSpec.describe "selected" do + it "preserves configuration while running an assigned file" do + expect(DDTEST_RSPEC_SETUP).to eq(true) + end +end +`) + writeCompatibilityFixture(t, root, "spec/unselected_spec.rb", `RSpec.describe "unselected" do + it "must not run" do + raise "unselected file ran" + end +end +`) + t.Chdir(root) + + rspec := &RSpec{ + executor: &ext.DefaultCommandExecutor{}, + platformEnv: map[string]string{"RUBYOPT": "-rbundler/setup -rdatadog/ci/auto_instrument"}, + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + testFiles := discovery.TestFileSet{Pattern: rspec.TestPattern()} + files, err := rspec.DiscoverTestFiles(ctx, testFiles) + if err != nil { + t.Fatal(err) + } + wantFiles := []string{"spec/selected_spec.rb", "spec/unselected_spec.rb"} + requireCompatibilityFiles(t, files, wantFiles) + + tests, err := rspec.DiscoverTests(ctx, testFiles) + if err != nil { + t.Fatalf("full discovery failed: %v", err) + } + requireCompatibilityTestSources(t, tests, wantFiles) + + if err := rspec.RunTests(ctx, []string{"spec/selected_spec.rb"}, nil); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } +} diff --git a/internal/framework/scripts/mocha_adapter.js b/internal/framework/scripts/mocha_adapter.js index 1b7e357..4d0f8a4 100644 --- a/internal/framework/scripts/mocha_adapter.js +++ b/internal/framework/scripts/mocha_adapter.js @@ -33,12 +33,20 @@ function runAdapter(request, mochaEntrypoint) { throw new Error(`ddtest requires Mocha 8 or newer; found ${mochaVersion}`) } - const optionsPath = path.join(mochaRoot, "lib/cli/options.js") + // Mocha 12 converted these private CLI modules from .js to .cjs. Resolve + // both layouts so the adapter keeps working across every supported major. + const optionsPath = firstExistingPath(mochaRoot, [ + "lib/cli/options.js", + "lib/cli/options.cjs", + ]) const optionsModule = require(optionsPath) const options = optionsModule.loadOptions(request.cliArgs || []) if (request.mode === "discover") { - const collectFiles = require(path.join(mochaRoot, "lib/cli/collect-files.js")) + const collectFiles = require(firstExistingPath(mochaRoot, [ + "lib/cli/collect-files.js", + "lib/cli/collect-files.cjs", + ])) const collection = collectFiles({ ignore: options.ignore || [], extension: options.extension || [], @@ -71,3 +79,13 @@ function runAdapter(request, mochaEntrypoint) { throw new Error(`unknown ddtest Mocha adapter mode: ${request.mode}`) } } + +function firstExistingPath(root, candidates) { + for (const candidate of candidates) { + const resolved = path.join(root, candidate) + if (fs.existsSync(resolved)) { + return resolved + } + } + throw new Error(`Unable to locate Mocha CLI module: ${candidates.join(" or ")}`) +} diff --git a/internal/framework/scripts/vitest_v1_discovery.mjs b/internal/framework/scripts/vitest_v1_discovery.mjs index 04ba95e..3a5bdf0 100644 --- a/internal/framework/scripts/vitest_v1_discovery.mjs +++ b/internal/framework/scripts/vitest_v1_discovery.mjs @@ -2,7 +2,7 @@ import { createVitest, parseCLI } from 'vitest/node' const outputMarker = '__DDTEST_VITEST_FILES__' const cliArgs = JSON.parse(process.argv[1]) -const { filter, options } = parseCLI(['node', 'vitest', ...cliArgs]) +const { filter, options } = parseCLI(['vitest', ...cliArgs]) const vitest = await createVitest('test', { ...options, watch: false }) try { diff --git a/internal/framework/vitest.go b/internal/framework/vitest.go index 1ec3bf4..141c9a9 100644 --- a/internal/framework/vitest.go +++ b/internal/framework/vitest.go @@ -259,16 +259,16 @@ func vitestArgsForSubcommand(baseArgs []string, subcommand string) []string { func vitestCLIArgs(command string, baseArgs []string) []string { if isVitestExecutable(command) { - return slices.Clone(baseArgs) + return append([]string{}, baseArgs...) } for i, arg := range baseArgs { if isVitestExecutable(arg) { - return slices.Clone(baseArgs[i+1:]) + return append([]string{}, baseArgs[i+1:]...) } } - return nil + return []string{} } func isVitestExecutable(value string) bool { diff --git a/internal/framework/vitest_test.go b/internal/framework/vitest_test.go index bd356bd..ec47a1c 100644 --- a/internal/framework/vitest_test.go +++ b/internal/framework/vitest_test.go @@ -9,8 +9,10 @@ import ( "slices" "strings" "testing" + "time" "github.com/DataDog/ddtest/internal/discovery" + "github.com/DataDog/ddtest/internal/ext" "github.com/DataDog/ddtest/internal/settings" ) @@ -513,6 +515,9 @@ func TestVitestCLIArgs(t *testing.T) { } }) } + if got := vitestCLIArgs("node_modules/.bin/vitest", nil); got == nil { + t.Fatal("direct binary CLI args must encode as an empty JSON array, not null") + } } func TestStripNodeOptionsImport(t *testing.T) { @@ -522,3 +527,50 @@ func TestStripNodeOptionsImport(t *testing.T) { t.Fatalf("got %q, want %q", got, want) } } + +func TestVitestAdapterIntegration(t *testing.T) { + nodeModules := requireCompatibilityEnv(t, "DDTEST_VITEST_NODE_MODULES") + + root := t.TempDir() + if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { + t.Fatal(err) + } + writeCompatibilityFixture(t, root, "vitest.config.mjs", `export default { + test: { + include: ['checks/**/*.check.js'], + setupFiles: ['./setup.js'], + }, +} +`) + writeCompatibilityFixture(t, root, "setup.js", "globalThis.ddtestVitestSetup = true\n") + writeCompatibilityFixture(t, root, "checks/selected.check.js", `import { expect, test } from 'vitest' + +test('preserves config while running an assigned file', () => { + expect(globalThis.ddtestVitestSetup).toBe(true) + expect(process.env.DDTEST_VITEST_WORKER).toBe('selected') +}) +`) + writeCompatibilityFixture(t, root, "checks/unselected.check.js", `import { test } from 'vitest' + +test('must not run', () => { + throw new Error('unselected file ran') +}) +`) + t.Chdir(root) + + vitest := &Vitest{executor: &ext.DefaultCommandExecutor{}, platformEnv: map[string]string{}} + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + testFiles := discovery.TestFileSet{Pattern: vitest.TestPattern()} + files, err := vitest.DiscoverTestFiles(ctx, testFiles) + if err != nil { + t.Fatal(err) + } + wantFiles := []string{"checks/selected.check.js", "checks/unselected.check.js"} + requireCompatibilityFiles(t, files, wantFiles) + + if err := vitest.RunTests(ctx, []string{"checks/selected.check.js"}, map[string]string{"DDTEST_VITEST_WORKER": "selected"}); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } +} diff --git a/internal/platform/compatibility_test.go b/internal/platform/compatibility_test.go new file mode 100644 index 0000000..3a0e27a --- /dev/null +++ b/internal/platform/compatibility_test.go @@ -0,0 +1,102 @@ +package platform + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/DataDog/ddtest/internal/settings" +) + +func requirePlatformCompatibilityEnv(t *testing.T, name string) string { + t.Helper() + value := os.Getenv(name) + if value == "" { + t.Skipf("%s is not set", name) + } + return value +} + +func requireRuntimeTags(t *testing.T, tags map[string]string, language string) { + t.Helper() + want := map[string]string{ + "language": language, + "runtime.name": "", + "runtime.version": "", + "os.platform": "", + "os.architecture": "", + "os.version": "", + } + for key, exact := range want { + value, ok := tags[key] + if !ok || value == "" { + t.Fatalf("runtime tags missing %q: %v", key, tags) + } + if exact != "" && value != exact { + t.Fatalf("runtime tag %q = %q, want %q", key, value, exact) + } + } +} + +func TestRubyPlatformIntegration(t *testing.T) { + datadogVersion := requirePlatformCompatibilityEnv(t, "DDTEST_RUBY_PLATFORM_VERSION") + root := t.TempDir() + gemfile := fmt.Sprintf("source \"https://rubygems.org\"\ngem \"datadog-ci\", %q\n", datadogVersion) + if err := os.WriteFile(filepath.Join(root, "Gemfile"), []byte(gemfile), 0644); err != nil { + t.Fatal(err) + } + t.Chdir(root) + + platform := NewRuby(settings.TestSkippingLevelTest) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err := platform.SanityCheck(ctx); err != nil { + t.Fatal(err) + } + tags, err := platform.CreateTagsMap(ctx) + if err != nil { + t.Fatal(err) + } + requireRuntimeTags(t, tags, "ruby") +} + +func TestPythonPlatformIntegration(t *testing.T) { + requirePlatformCompatibilityEnv(t, "DDTEST_PYTHON_PLATFORM_INTEGRATION") + t.Chdir(t.TempDir()) + + platform := NewPython() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err := platform.SanityCheck(ctx); err != nil { + t.Fatal(err) + } + tags, err := platform.CreateTagsMap(ctx) + if err != nil { + t.Fatal(err) + } + requireRuntimeTags(t, tags, "python") +} + +func TestJavaScriptPlatformIntegration(t *testing.T) { + nodeModules := requirePlatformCompatibilityEnv(t, "DDTEST_DD_TRACE_NODE_MODULES") + root := t.TempDir() + if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { + t.Fatal(err) + } + t.Chdir(root) + + platform := NewJavaScript() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err := platform.SanityCheck(ctx); err != nil { + t.Fatal(err) + } + tags, err := platform.CreateTagsMap(ctx) + if err != nil { + t.Fatal(err) + } + requireRuntimeTags(t, tags, "javascript") +} diff --git a/internal/platform/platform_test.go b/internal/platform/platform_test.go index 9c91958..ef5592b 100644 --- a/internal/platform/platform_test.go +++ b/internal/platform/platform_test.go @@ -27,7 +27,7 @@ func TestPlatformSanityChecksPropagateContext(t *testing.T) { output []byte }{ {name: "ruby", output: []byte(" * datadog-ci (1.31.0)\n")}, - {name: "python", output: []byte("4.10.3\n")}, + {name: "python", output: []byte("4.11.0\n")}, {name: "javascript", output: []byte("v24.0.0\n")}, } @@ -78,7 +78,7 @@ func TestDetectPlatformPythonWithFakeInterpreter(t *testing.T) { binDir := t.TempDir() pythonPath := filepath.Join(binDir, "python") - if err := os.WriteFile(pythonPath, []byte("#!/bin/sh\nprintf '4.10.3\\n'\n"), 0755); err != nil { + if err := os.WriteFile(pythonPath, []byte("#!/bin/sh\nprintf '4.11.0\\n'\n"), 0755); err != nil { t.Fatal(err) } t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) diff --git a/internal/platform/python.go b/internal/platform/python.go index afe9c99..1e35073 100644 --- a/internal/platform/python.go +++ b/internal/platform/python.go @@ -33,7 +33,7 @@ var pythonEnvScript string const ( requiredPackageName = "ddtrace" - requiredPackageVersion = "4.10.3" + requiredPackageVersion = "4.11.0" pytestAddOptsEnvVar = "PYTEST_ADDOPTS" pytestDefaultAddOpts = "--ddtrace" ) diff --git a/internal/platform/python_test.go b/internal/platform/python_test.go index d69cf9c..0a58b92 100644 --- a/internal/platform/python_test.go +++ b/internal/platform/python_test.go @@ -58,7 +58,7 @@ func TestPython_SanityCheck_SuccessWithPreRelease(t *testing.T) { func TestPython_SanityCheck_Success(t *testing.T) { mockExecutor := &mockCommandExecutor{ - combinedOutput: []byte("4.10.3\n"), + combinedOutput: []byte("4.11.0\n"), onCombinedOutput: func(name string, args []string, envMap map[string]string) { if name != "python" { t.Fatalf("expected command 'python', got %q", name) @@ -99,7 +99,7 @@ func TestPython_SanityCheck_NotInstalled(t *testing.T) { func TestPython_SanityCheck_VersionTooOld(t *testing.T) { mockExecutor := &mockCommandExecutor{ - combinedOutput: []byte("4.9.0\n"), + combinedOutput: []byte("4.10.3\n"), } python := NewPython() @@ -109,7 +109,7 @@ func TestPython_SanityCheck_VersionTooOld(t *testing.T) { t.Fatal("SanityCheck() expected error for outdated ddtrace version") } - if !strings.Contains(err.Error(), "4.9.0") { + if !strings.Contains(err.Error(), "4.10.3") { t.Errorf("expected error to mention detected version, got: %v", err) } if !strings.Contains(err.Error(), requiredPackageVersion) { From a496890bbea6e68f307fc91ffa596e2017d9f1b4 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 9 Sep 2026 13:13:02 +0200 Subject: [PATCH 2/4] Stabilize Vitest 4 compatibility setup --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c072f94..9ffc49f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -254,7 +254,12 @@ jobs: - name: Test Vitest adapter run: | vitest_dir="${RUNNER_TEMP}/vitest-${{ matrix.vitest }}" - npm install --prefix "${vitest_dir}" "vitest@${{ matrix.vitest }}" + npm_install_args=() + if [[ "${{ matrix.vitest }}" == "4.1.11" ]]; then + # npm 10.9.8 crashes while resolving Vitest 4's duplicated Vite dependency/peer. + npm_install_args+=(--legacy-peer-deps) + fi + npm install --prefix "${vitest_dir}" "${npm_install_args[@]}" "vitest@${{ matrix.vitest }}" DDTEST_VITEST_NODE_MODULES="${vitest_dir}/node_modules" \ go test -v ./internal/framework -run '^TestVitestAdapterIntegration$' From 8e751731fc83893728d3e75eeb30ab0013e55f24 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 9 Sep 2026 13:48:13 +0200 Subject: [PATCH 3/4] Move compatibility tests into dedicated package --- .github/workflows/ci.yml | 24 +-- internal/compatibility/cucumber_test.go | 80 ++++++++ internal/compatibility/cypress_test.go | 176 +++++++++++++++++ .../helpers_test.go} | 42 ++++- internal/compatibility/jest_test.go | 53 ++++++ internal/compatibility/minitest_test.go | 75 ++++++++ internal/compatibility/mocha_test.go | 77 ++++++++ .../platform_test.go} | 78 ++++---- internal/compatibility/playwright_test.go | 131 +++++++++++++ internal/compatibility/pytest_test.go | 54 ++++++ internal/compatibility/rspec_test.go | 60 ++++++ internal/compatibility/vitest_test.go | 59 ++++++ internal/framework/cucumber_test.go | 85 --------- internal/framework/cypress_test.go | 177 ------------------ internal/framework/jest_test.go | 43 ----- internal/framework/minitest_test.go | 69 ------- internal/framework/mocha_test.go | 82 -------- internal/framework/playwright_test.go | 134 ------------- internal/framework/pytest_test.go | 47 ----- internal/framework/rspec_test.go | 53 ------ internal/framework/vitest_test.go | 49 ----- 21 files changed, 848 insertions(+), 800 deletions(-) create mode 100644 internal/compatibility/cucumber_test.go create mode 100644 internal/compatibility/cypress_test.go rename internal/{framework/compatibility_test.go => compatibility/helpers_test.go} (62%) create mode 100644 internal/compatibility/jest_test.go create mode 100644 internal/compatibility/minitest_test.go create mode 100644 internal/compatibility/mocha_test.go rename internal/{platform/compatibility_test.go => compatibility/platform_test.go} (68%) create mode 100644 internal/compatibility/playwright_test.go create mode 100644 internal/compatibility/pytest_test.go create mode 100644 internal/compatibility/rspec_test.go create mode 100644 internal/compatibility/vitest_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ffc49f..25ec1b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,9 +108,9 @@ jobs: gem install rspec --version "${{ matrix.rspec }}" --no-document DDTEST_RSPEC_VERSION="${{ matrix.rspec }}" \ DDTEST_DATADOG_CI_VERSION="${{ matrix.datadog_ci }}" \ - go test -v ./internal/framework -run '^TestRSpecAdapterIntegration$' + go test -v ./internal/compatibility -run '^TestRSpecAdapterIntegration$' DDTEST_RUBY_PLATFORM_VERSION="${{ matrix.datadog_ci }}" \ - go test -v ./internal/platform -run '^TestRubyPlatformIntegration$' + go test -v ./internal/compatibility -run '^TestRubyPlatformIntegration$' minitest-compatibility: runs-on: ubuntu-latest @@ -143,7 +143,7 @@ jobs: gem install rake --version "13.2.1" --no-document DDTEST_MINITEST_VERSION="${{ matrix.minitest }}" \ DDTEST_DATADOG_CI_VERSION="${{ matrix.datadog_ci }}" \ - go test -v ./internal/framework -run '^TestMinitestAdapterIntegration$' + go test -v ./internal/compatibility -run '^TestMinitestAdapterIntegration$' pytest-compatibility: runs-on: ubuntu-latest @@ -173,9 +173,9 @@ jobs: run: | python -m pip install "ddtrace==${{ matrix.ddtrace }}" "pytest==${{ matrix.pytest }}" DDTEST_PYTHON_BINARY="$(command -v python)" \ - go test -v ./internal/framework -run '^TestPyTestAdapterIntegration$' + go test -v ./internal/compatibility -run '^TestPyTestAdapterIntegration$' DDTEST_PYTHON_PLATFORM_INTEGRATION=1 \ - go test -v ./internal/platform -run '^TestPythonPlatformIntegration$' + go test -v ./internal/compatibility -run '^TestPythonPlatformIntegration$' javascript-platform-compatibility: runs-on: ubuntu-latest @@ -202,7 +202,7 @@ jobs: dd_trace_dir="${RUNNER_TEMP}/dd-trace-${{ matrix.dd_trace }}" npm install --prefix "${dd_trace_dir}" "dd-trace@${{ matrix.dd_trace }}" DDTEST_DD_TRACE_NODE_MODULES="${dd_trace_dir}/node_modules" \ - go test -v ./internal/platform -run '^TestJavaScriptPlatformIntegration$' + go test -v ./internal/compatibility -run '^TestJavaScriptPlatformIntegration$' jest-compatibility: runs-on: ubuntu-latest @@ -229,7 +229,7 @@ jobs: jest_dir="${RUNNER_TEMP}/jest-${{ matrix.jest }}" npm install --prefix "${jest_dir}" "jest@${{ matrix.jest }}" DDTEST_JEST_NODE_MODULES="${jest_dir}/node_modules" \ - go test -v ./internal/framework -run '^TestJestAdapterIntegration$' + go test -v ./internal/compatibility -run '^TestJestAdapterIntegration$' vitest-compatibility: runs-on: ubuntu-latest @@ -261,7 +261,7 @@ jobs: fi npm install --prefix "${vitest_dir}" "${npm_install_args[@]}" "vitest@${{ matrix.vitest }}" DDTEST_VITEST_NODE_MODULES="${vitest_dir}/node_modules" \ - go test -v ./internal/framework -run '^TestVitestAdapterIntegration$' + go test -v ./internal/compatibility -run '^TestVitestAdapterIntegration$' mocha-compatibility: runs-on: ubuntu-latest @@ -288,7 +288,7 @@ jobs: mocha_dir="${RUNNER_TEMP}/mocha-${{ matrix.mocha }}" npm install --prefix "${mocha_dir}" "mocha@${{ matrix.mocha }}" DDTEST_MOCHA_NODE_MODULES="${mocha_dir}/node_modules" \ - go test -v ./internal/framework -run '^TestMochaAdapter.*Integration$' + go test -v ./internal/compatibility -run '^TestMochaAdapter.*Integration$' cypress-compatibility: runs-on: ubuntu-latest @@ -317,7 +317,7 @@ jobs: DDTEST_CYPRESS_BINARY="${cypress_dir}/node_modules/.bin/cypress" \ DDTEST_CYPRESS_NODE_MODULES="${cypress_dir}/node_modules" \ DDTEST_CYPRESS_XVFB="$(command -v xvfb-run)" \ - go test -v ./internal/framework -run '^TestCypressAdapter.*Integration$' + go test -v ./internal/compatibility -run '^TestCypressAdapter.*Integration$' playwright-compatibility: runs-on: ubuntu-latest @@ -347,7 +347,7 @@ jobs: npm install --prefix "${playwright_dir}" "@playwright/test@${{ matrix.playwright }}" DDTEST_PLAYWRIGHT_BINARY="${playwright_dir}/node_modules/.bin/playwright" \ DDTEST_PLAYWRIGHT_NODE_MODULES="${playwright_dir}/node_modules" \ - go test -v ./internal/framework -run TestPlaywrightAdapterIntegration + go test -v ./internal/compatibility -run '^TestPlaywrightAdapterIntegration$' cucumber-compatibility: runs-on: ubuntu-latest @@ -376,7 +376,7 @@ jobs: DDTEST_CUCUMBER_BINARY="${cucumber_dir}/node_modules/.bin/cucumber-js" \ DDTEST_CUCUMBER_NODE_MODULES="${cucumber_dir}/node_modules" \ DDTEST_CUCUMBER_VERSION="${{ matrix.cucumber }}" \ - go test -v ./internal/framework -run '^TestCucumberAdapterIntegration$' + go test -v ./internal/compatibility -run '^TestCucumberAdapterIntegration$' lint: runs-on: ubuntu-latest diff --git a/internal/compatibility/cucumber_test.go b/internal/compatibility/cucumber_test.go new file mode 100644 index 0000000..0c696c1 --- /dev/null +++ b/internal/compatibility/cucumber_test.go @@ -0,0 +1,80 @@ +package compatibility + +import ( + "context" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/DataDog/ddtest/internal/discovery" + "github.com/DataDog/ddtest/internal/framework" +) + +func TestCucumberAdapterIntegration(t *testing.T) { + cucumberBinary := requireEnv(t, "DDTEST_CUCUMBER_BINARY") + nodeModules := requireEnv(t, "DDTEST_CUCUMBER_NODE_MODULES") + cucumberVersion := requireEnv(t, "DDTEST_CUCUMBER_VERSION") + resetSettingsAfterTest(t) + + root := t.TempDir() + t.Chdir(root) + if err := os.Symlink(nodeModules, "node_modules"); err != nil { + t.Fatal(err) + } + cucumberConfig := `module.exports = { + default: { + tags: 'not @excluded', + require: ['features/support/**/*.js'] + } +} +` + if strings.HasPrefix(cucumberVersion, "7.") { + // Cucumber 7 profiles are CLI argument strings. Object-based profiles were + // introduced later and are silently treated as empty by Cucumber 7. + cucumberConfig = `module.exports = { + default: "--require 'features/support/**/*.js' --tags 'not @excluded'" +} +` + } + files := map[string]string{ + "cucumber.js": cucumberConfig, + "features/included.feature": `Feature: included + Scenario: selected by the default profile + Given a passing step +`, + "features/unassigned.feature": `Feature: unassigned + Scenario: must not run + Given a failing step +`, + "features/excluded.feature": `@excluded +Feature: excluded + Scenario: filtered by the default profile + Given a passing step +`, + "features/support/steps.js": `const { Given } = require('@cucumber/cucumber') +Given('a passing step', function () {}) +Given('a failing step', function () { throw new Error('unassigned file ran') }) +`, + } + for filename, content := range files { + writeFixture(t, root, filename, content) + } + + configureFramework(shellCommand(cucumberBinary), "") + cucumber := framework.NewCucumber() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + discovered, err := cucumber.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: cucumber.TestPattern()}) + if err != nil { + t.Fatal(err) + } + wantFiles := []string{"features/included.feature", "features/unassigned.feature"} + if !slices.Equal(discovered, wantFiles) { + t.Fatalf("discovered = %v", discovered) + } + if err := cucumber.RunTests(ctx, []string{"features/included.feature"}, nil); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } +} diff --git a/internal/compatibility/cypress_test.go b/internal/compatibility/cypress_test.go new file mode 100644 index 0000000..9b9c305 --- /dev/null +++ b/internal/compatibility/cypress_test.go @@ -0,0 +1,176 @@ +package compatibility + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/DataDog/ddtest/internal/discovery" + "github.com/DataDog/ddtest/internal/framework" +) + +func TestCypressAdapterIntegration(t *testing.T) { + binary := requireEnv(t, "DDTEST_CYPRESS_BINARY") + nodeModules := requireEnv(t, "DDTEST_CYPRESS_NODE_MODULES") + resetSettingsAfterTest(t) + + root := t.TempDir() + t.Chdir(root) + tests := []struct { + name string + projectName string + configFilename string + configExport string + specPattern any + files []string + symlinkName string + symlinkTarget string + want []string + }{ + { + name: "minimatch extglob", + projectName: "extglob", + configFilename: "cypress.config.ts", + configExport: "export default", + specPattern: "custom/**/*.@(spec|test).cy.ts", + files: []string{ + "custom/discovered.spec.cy.ts", + "custom/discovered.test.cy.ts", + "custom/not-discovered.cy.ts", + }, + want: []string{ + "extglob/custom/discovered.spec.cy.ts", + "extglob/custom/discovered.test.cy.ts", + }, + }, + { + name: "broad pattern excludes discovery wrapper", + projectName: "broad", + configFilename: "cypress.config.js", + configExport: "module.exports =", + specPattern: "**/*.ts", + files: []string{"specs/discovered.ts", "specs/not-discovered.js"}, + want: []string{"broad/specs/discovered.ts"}, + }, + { + name: "negated spec pattern subtracts matches", + projectName: "negated", + configFilename: "cypress.config.js", + configExport: "module.exports =", + specPattern: []string{"**/*.cy.ts", "!**/slow.cy.ts"}, + files: []string{"specs/fast.cy.ts", "specs/slow.cy.ts", "specs/helper.ts"}, + want: []string{"negated/specs/fast.cy.ts"}, + }, + { + name: "symlinked spec directory", + projectName: "symlinked", + configFilename: "cypress.config.js", + configExport: "module.exports =", + specPattern: "linked/**/*.cy.ts", + files: []string{"target/discovered.cy.ts"}, + symlinkName: "linked", + symlinkTarget: "target", + want: []string{"symlinked/linked/discovered.cy.ts"}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + projectRoot := filepath.Join(root, test.projectName) + if err := os.MkdirAll(projectRoot, 0755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(nodeModules, filepath.Join(projectRoot, "node_modules")); err != nil { + t.Fatal(err) + } + specPattern, err := json.Marshal(test.specPattern) + if err != nil { + t.Fatal(err) + } + config := test.configExport + ` { + e2e: { + supportFile: false, + async setupNodeEvents(_on, config) { + return { ...config, specPattern: ` + string(specPattern) + ` } + }, + }, +} +` + writeFixture(t, projectRoot, test.configFilename, config) + for _, filename := range test.files { + writeFixture(t, projectRoot, filename, "") + } + if test.symlinkName != "" { + if err := os.Symlink(test.symlinkTarget, filepath.Join(projectRoot, test.symlinkName)); err != nil { + t.Fatal(err) + } + } + + configureFramework(shellCommand(binary, "run", "--project", test.projectName), "") + cypress := framework.NewCypress() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + files, err := cypress.DiscoverTestFiles(ctx, discovery.TestFileSet{}) + cancel() + if err != nil { + t.Fatal(err) + } + requireFiles(t, files, test.want) + }) + } +} + +func TestCypressAdapterExecutionIntegration(t *testing.T) { + binary := requireEnv(t, "DDTEST_CYPRESS_BINARY") + nodeModules := requireEnv(t, "DDTEST_CYPRESS_NODE_MODULES") + resetSettingsAfterTest(t) + + root := t.TempDir() + if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { + t.Fatal(err) + } + writeFixture(t, root, "cypress.config.js", `module.exports = { + video: false, + viewportWidth: 777, + e2e: { + supportFile: false, + specPattern: 'cypress/e2e/**/*.cy.js', + }, +} +`) + writeFixture(t, root, "cypress/e2e/selected.cy.js", `describe('selected', () => { + it('runs an assigned file', () => { + expect(true).to.equal(true) + expect(Cypress.config('viewportWidth')).to.equal(777) + }) +}) +`) + writeFixture(t, root, "cypress/e2e/unselected.cy.js", `describe('unselected', () => { + it('must not run', () => { + throw new Error('unselected file ran') + }) +}) +`) + t.Chdir(root) + + command := []string{binary, "run"} + if xvfb := os.Getenv("DDTEST_CYPRESS_XVFB"); xvfb != "" { + command = []string{xvfb, "-a", binary, "run"} + } + configureFramework(shellCommand(command...), "") + cypress := framework.NewCypress() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + files, err := cypress.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: cypress.TestPattern()}) + if err != nil { + t.Fatal(err) + } + wantFiles := []string{"cypress/e2e/selected.cy.js", "cypress/e2e/unselected.cy.js"} + requireFiles(t, files, wantFiles) + + if err := cypress.RunTests(ctx, []string{"cypress/e2e/selected.cy.js"}, nil); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } +} diff --git a/internal/framework/compatibility_test.go b/internal/compatibility/helpers_test.go similarity index 62% rename from internal/framework/compatibility_test.go rename to internal/compatibility/helpers_test.go index 507c0b8..a4d0ccb 100644 --- a/internal/framework/compatibility_test.go +++ b/internal/compatibility/helpers_test.go @@ -1,17 +1,20 @@ -package framework +package compatibility import ( "os" "path/filepath" "slices" + "strconv" "strings" "testing" + "github.com/DataDog/ddtest/internal/settings" "github.com/DataDog/ddtest/internal/testoptimization" "github.com/DataDog/ddtest/internal/utils" + "github.com/spf13/viper" ) -func requireCompatibilityEnv(t *testing.T, name string) string { +func requireEnv(t *testing.T, name string) string { t.Helper() value := os.Getenv(name) if value == "" { @@ -20,7 +23,7 @@ func requireCompatibilityEnv(t *testing.T, name string) string { return value } -func writeCompatibilityFixture(t *testing.T, root, name, contents string) { +func writeFixture(t *testing.T, root, name, contents string) { t.Helper() path := filepath.Join(root, name) if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { @@ -31,7 +34,7 @@ func writeCompatibilityFixture(t *testing.T, root, name, contents string) { } } -func requireCompatibilityFiles(t *testing.T, got, want []string) { +func requireFiles(t *testing.T, got, want []string) { t.Helper() normalized := slices.Clone(got) for i := range normalized { @@ -50,7 +53,7 @@ func requireCompatibilityFiles(t *testing.T, got, want []string) { } } -func requireCompatibilityTestSources(t *testing.T, tests []testoptimization.Test, want []string) { +func requireTestSources(t *testing.T, tests []testoptimization.Test, want []string) { t.Helper() if len(tests) != len(want) { t.Fatalf("discovered %d tests, want %d: %+v", len(tests), len(want), tests) @@ -73,5 +76,32 @@ func requireCompatibilityTestSources(t *testing.T, tests []testoptimization.Test } sources = append(sources, utils.NormalizePath(source)) } - requireCompatibilityFiles(t, sources, want) + requireFiles(t, sources, want) +} + +func resetSettingsAfterTest(t *testing.T) { + t.Helper() + t.Cleanup(func() { + viper.Reset() + settings.Init() + }) +} + +func configureFramework(command, testsLocation string) { + viper.Reset() + if command != "" { + viper.Set("command", command) + } + if testsLocation != "" { + viper.Set("tests_location", testsLocation) + } + settings.Init() +} + +func shellCommand(parts ...string) string { + quoted := make([]string, len(parts)) + for i, part := range parts { + quoted[i] = strconv.Quote(part) + } + return strings.Join(quoted, " ") } diff --git a/internal/compatibility/jest_test.go b/internal/compatibility/jest_test.go new file mode 100644 index 0000000..013a2a5 --- /dev/null +++ b/internal/compatibility/jest_test.go @@ -0,0 +1,53 @@ +package compatibility + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/DataDog/ddtest/internal/discovery" + "github.com/DataDog/ddtest/internal/framework" +) + +func TestJestAdapterIntegration(t *testing.T) { + nodeModules := requireEnv(t, "DDTEST_JEST_NODE_MODULES") + + root := t.TempDir() + if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { + t.Fatal(err) + } + writeFixture(t, root, "jest.config.js", `module.exports = { + testMatch: ['/tests/**/*.test.js'], + setupFilesAfterEnv: ['/setup.js'], +} +`) + writeFixture(t, root, "setup.js", "globalThis.ddtestJestSetup = true\n") + writeFixture(t, root, "tests/selected.test.js", `test('preserves config while running an assigned file', () => { + expect(globalThis.ddtestJestSetup).toBe(true) + expect(process.env.DDTEST_JEST_WORKER).toBe('selected') +}) +`) + writeFixture(t, root, "tests/unselected.test.js", `test('must not run', () => { + throw new Error('unselected file ran') +}) +`) + t.Chdir(root) + + jest := framework.NewJest() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + testFiles := discovery.TestFileSet{Pattern: jest.TestPattern()} + files, err := jest.DiscoverTestFiles(ctx, testFiles) + if err != nil { + t.Fatal(err) + } + wantFiles := []string{"tests/selected.test.js", "tests/unselected.test.js"} + requireFiles(t, files, wantFiles) + + if err := jest.RunTests(ctx, []string{"tests/selected.test.js"}, map[string]string{"DDTEST_JEST_WORKER": "selected"}); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } +} diff --git a/internal/compatibility/minitest_test.go b/internal/compatibility/minitest_test.go new file mode 100644 index 0000000..3c787c9 --- /dev/null +++ b/internal/compatibility/minitest_test.go @@ -0,0 +1,75 @@ +package compatibility + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/DataDog/ddtest/internal/discovery" + "github.com/DataDog/ddtest/internal/framework" +) + +func TestMinitestAdapterIntegration(t *testing.T) { + minitestVersion := requireEnv(t, "DDTEST_MINITEST_VERSION") + datadogVersion := requireEnv(t, "DDTEST_DATADOG_CI_VERSION") + + root := t.TempDir() + writeFixture(t, root, "Gemfile", fmt.Sprintf(`source "https://rubygems.org" +gem "datadog-ci", %q +gem "minitest", %q +gem "rake", "13.2.1" +`, datadogVersion, minitestVersion)) + writeFixture(t, root, "Rakefile", `require "rake/testtask" + +Rake::TestTask.new(:test) do |test| + test.test_files = ENV["TEST_FILES"] ? ENV["TEST_FILES"].split : FileList["test/**/*_test.rb"] +end + +task default: :test +`) + writeFixture(t, root, "test/test_helper.rb", `gem "minitest", ENV.fetch("DDTEST_MINITEST_VERSION") +require "minitest/autorun" +DDTEST_MINITEST_SETUP = true +`) + writeFixture(t, root, "test/selected_test.rb", `require_relative "test_helper" + +class SelectedTest < Minitest::Test + def test_preserves_setup_while_running_an_assigned_file + assert DDTEST_MINITEST_SETUP + end +end +`) + writeFixture(t, root, "test/unselected_test.rb", `require_relative "test_helper" + +class UnselectedTest < Minitest::Test + def test_must_not_run + flunk "unselected file ran" + end +end +`) + t.Chdir(root) + + minitest := framework.NewMinitest() + minitest.SetPlatformEnv(map[string]string{"RUBYOPT": "-rbundler/setup -rdatadog/ci/auto_instrument"}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + testFiles := discovery.TestFileSet{Pattern: minitest.TestPattern()} + files, err := minitest.DiscoverTestFiles(ctx, testFiles) + if err != nil { + t.Fatal(err) + } + wantFiles := []string{"test/selected_test.rb", "test/unselected_test.rb"} + requireFiles(t, files, wantFiles) + + tests, err := minitest.DiscoverTests(ctx, testFiles) + if err != nil { + t.Fatalf("full discovery failed: %v", err) + } + requireTestSources(t, tests, wantFiles) + + if err := minitest.RunTests(ctx, []string{"test/selected_test.rb"}, nil); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } +} diff --git a/internal/compatibility/mocha_test.go b/internal/compatibility/mocha_test.go new file mode 100644 index 0000000..1495d3c --- /dev/null +++ b/internal/compatibility/mocha_test.go @@ -0,0 +1,77 @@ +package compatibility + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/DataDog/ddtest/internal/discovery" + "github.com/DataDog/ddtest/internal/framework" +) + +func TestMochaAdapterIntegration(t *testing.T) { + nodeModules := requireEnv(t, "DDTEST_MOCHA_NODE_MODULES") + + root := t.TempDir() + if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { + t.Fatal(err) + } + writeFixture(t, root, ".mocharc.json", `{"spec":["test/**/*.spec.js"],"file":["setup.js"]}`) + writeFixture(t, root, "setup.js", "global.ddtestSetup = true\n") + writeFixture(t, root, "test/selected.spec.js", `const assert = require("assert"); describe("selected", () => { it("uses setup", () => assert.equal(global.ddtestSetup, true)) })`) + writeFixture(t, root, "test/unselected.spec.js", `describe("unselected", () => { it("must not run", () => { throw new Error("unselected file ran") }) })`) + t.Chdir(root) + + mocha := framework.NewMocha() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + files, err := mocha.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: mocha.TestPattern()}) + if err != nil { + t.Fatal(err) + } + want := []string{"test/selected.spec.js", "test/unselected.spec.js"} + requireFiles(t, files, want) + if err := mocha.RunTests(ctx, []string{"test/selected.spec.js"}, nil); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } + + if err := os.Remove(filepath.Join(root, ".mocharc.json")); err != nil { + t.Fatal(err) + } + files, err = mocha.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: mocha.TestPattern()}) + if err != nil { + t.Fatalf("default discovery failed: %v", err) + } + requireFiles(t, files, want) +} + +func TestMochaAdapterCustomLocationAndCommandIntegration(t *testing.T) { + nodeModules := requireEnv(t, "DDTEST_MOCHA_NODE_MODULES") + resetSettingsAfterTest(t) + + root := t.TempDir() + mochaCommand := filepath.Join(nodeModules, ".bin", "mocha") + wrapper := filepath.Join(root, "mocha-wrapper.sh") + writeFixture(t, root, "mocha-wrapper.sh", "#!/bin/sh\nexport DDTEST_MOCHA_WRAPPER=preserved\nexec \"$@\"\n") + if err := os.Chmod(wrapper, 0755); err != nil { + t.Fatal(err) + } + writeFixture(t, root, ".mocharc.json", `{"spec":["test/**/*.spec.js"]}`) + writeFixture(t, root, "spec/custom.spec.js", `const assert = require("assert"); describe("custom", () => { it("uses wrapper", () => assert.equal(process.env.DDTEST_MOCHA_WRAPPER, "preserved")) })`) + t.Chdir(root) + configureFramework(shellCommand(wrapper, mochaCommand), "spec/**/*.js") + + mocha := framework.NewMocha() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + files, err := mocha.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: mocha.TestPattern()}) + if err != nil { + t.Fatal(err) + } + requireFiles(t, files, []string{"spec/custom.spec.js"}) + if err := mocha.RunTests(ctx, files, nil); err != nil { + t.Fatalf("custom-command run failed: %v", err) + } +} diff --git a/internal/platform/compatibility_test.go b/internal/compatibility/platform_test.go similarity index 68% rename from internal/platform/compatibility_test.go rename to internal/compatibility/platform_test.go index 3a0e27a..0ba93a2 100644 --- a/internal/platform/compatibility_test.go +++ b/internal/compatibility/platform_test.go @@ -1,4 +1,4 @@ -package platform +package compatibility import ( "context" @@ -8,41 +8,12 @@ import ( "testing" "time" + "github.com/DataDog/ddtest/internal/platform" "github.com/DataDog/ddtest/internal/settings" ) -func requirePlatformCompatibilityEnv(t *testing.T, name string) string { - t.Helper() - value := os.Getenv(name) - if value == "" { - t.Skipf("%s is not set", name) - } - return value -} - -func requireRuntimeTags(t *testing.T, tags map[string]string, language string) { - t.Helper() - want := map[string]string{ - "language": language, - "runtime.name": "", - "runtime.version": "", - "os.platform": "", - "os.architecture": "", - "os.version": "", - } - for key, exact := range want { - value, ok := tags[key] - if !ok || value == "" { - t.Fatalf("runtime tags missing %q: %v", key, tags) - } - if exact != "" && value != exact { - t.Fatalf("runtime tag %q = %q, want %q", key, value, exact) - } - } -} - func TestRubyPlatformIntegration(t *testing.T) { - datadogVersion := requirePlatformCompatibilityEnv(t, "DDTEST_RUBY_PLATFORM_VERSION") + datadogVersion := requireEnv(t, "DDTEST_RUBY_PLATFORM_VERSION") root := t.TempDir() gemfile := fmt.Sprintf("source \"https://rubygems.org\"\ngem \"datadog-ci\", %q\n", datadogVersion) if err := os.WriteFile(filepath.Join(root, "Gemfile"), []byte(gemfile), 0644); err != nil { @@ -50,13 +21,13 @@ func TestRubyPlatformIntegration(t *testing.T) { } t.Chdir(root) - platform := NewRuby(settings.TestSkippingLevelTest) + ruby := platform.NewRuby(settings.TestSkippingLevelTest) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - if err := platform.SanityCheck(ctx); err != nil { + if err := ruby.SanityCheck(ctx); err != nil { t.Fatal(err) } - tags, err := platform.CreateTagsMap(ctx) + tags, err := ruby.CreateTagsMap(ctx) if err != nil { t.Fatal(err) } @@ -64,16 +35,16 @@ func TestRubyPlatformIntegration(t *testing.T) { } func TestPythonPlatformIntegration(t *testing.T) { - requirePlatformCompatibilityEnv(t, "DDTEST_PYTHON_PLATFORM_INTEGRATION") + requireEnv(t, "DDTEST_PYTHON_PLATFORM_INTEGRATION") t.Chdir(t.TempDir()) - platform := NewPython() + python := platform.NewPython() ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - if err := platform.SanityCheck(ctx); err != nil { + if err := python.SanityCheck(ctx); err != nil { t.Fatal(err) } - tags, err := platform.CreateTagsMap(ctx) + tags, err := python.CreateTagsMap(ctx) if err != nil { t.Fatal(err) } @@ -81,22 +52,43 @@ func TestPythonPlatformIntegration(t *testing.T) { } func TestJavaScriptPlatformIntegration(t *testing.T) { - nodeModules := requirePlatformCompatibilityEnv(t, "DDTEST_DD_TRACE_NODE_MODULES") + nodeModules := requireEnv(t, "DDTEST_DD_TRACE_NODE_MODULES") root := t.TempDir() if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { t.Fatal(err) } t.Chdir(root) - platform := NewJavaScript() + javascript := platform.NewJavaScript() ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - if err := platform.SanityCheck(ctx); err != nil { + if err := javascript.SanityCheck(ctx); err != nil { t.Fatal(err) } - tags, err := platform.CreateTagsMap(ctx) + tags, err := javascript.CreateTagsMap(ctx) if err != nil { t.Fatal(err) } requireRuntimeTags(t, tags, "javascript") } + +func requireRuntimeTags(t *testing.T, tags map[string]string, language string) { + t.Helper() + want := map[string]string{ + "language": language, + "runtime.name": "", + "runtime.version": "", + "os.platform": "", + "os.architecture": "", + "os.version": "", + } + for key, exact := range want { + value, ok := tags[key] + if !ok || value == "" { + t.Fatalf("runtime tags missing %q: %v", key, tags) + } + if exact != "" && value != exact { + t.Fatalf("runtime tag %q = %q, want %q", key, value, exact) + } + } +} diff --git a/internal/compatibility/playwright_test.go b/internal/compatibility/playwright_test.go new file mode 100644 index 0000000..603a568 --- /dev/null +++ b/internal/compatibility/playwright_test.go @@ -0,0 +1,131 @@ +package compatibility + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/DataDog/ddtest/internal/discovery" + "github.com/DataDog/ddtest/internal/framework" +) + +func TestPlaywrightAdapterIntegration(t *testing.T) { + binary := requireEnv(t, "DDTEST_PLAYWRIGHT_BINARY") + nodeModules := requireEnv(t, "DDTEST_PLAYWRIGHT_NODE_MODULES") + resetSettingsAfterTest(t) + + root := t.TempDir() + t.Chdir(root) + projectRoot := filepath.Join(root, "apps", "web") + if err := os.MkdirAll(filepath.Join(projectRoot, "tests"), 0755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { + t.Fatal(err) + } + projects := `[{ name: 'one' }, { name: 'two' }]` + lifecycleFiles := []string{} + if playwrightVersionAtLeast(t, binary, 1, 31) { + projects = `[ + { name: 'setup', testMatch: '**/setup.spec.ts' }, + { name: 'one', testIgnore: /(?:setup|ignored)\.spec\.ts/, dependencies: ['setup'] }, + { name: 'two', testIgnore: /(?:setup|ignored)\.spec\.ts/, dependencies: ['setup'] }, + ]` + lifecycleFiles = []string{"setup.spec.ts"} + } + // Project teardown was added after project dependencies. Older versions + // ignore the teardown property and treat the named project as a normal + // project, so only exercise teardown filtering where Playwright supports it. + if playwrightVersionAtLeast(t, binary, 1, 38) { + projects = `[ + { name: 'setup', testMatch: '**/setup.spec.ts', teardown: 'teardown' }, + { name: 'teardown', testMatch: '**/teardown.spec.ts' }, + { name: 'one', testIgnore: /(?:setup|teardown|ignored)\.spec\.ts/, dependencies: ['setup'] }, + { name: 'two', testIgnore: /(?:setup|teardown|ignored)\.spec\.ts/, dependencies: ['setup'] }, + ]` + lifecycleFiles = []string{"setup.spec.ts", "teardown.spec.ts"} + } + config := fmt.Sprintf(`module.exports = { + testDir: './tests', + testMatch: '**/*.@(spec|test).ts', + testIgnore: '**/ignored.*', + projects: %s, +} +`, projects) + writeFixture(t, projectRoot, "playwright.config.js", config) + for _, name := range []string{"a.spec.ts", "b.test.ts", "ignored.spec.ts", "not-a-test.ts"} { + content := "const { test } = require('@playwright/test'); test('works', () => {});\n" + if name == "b.test.ts" { + content = "const { test } = require('@playwright/test'); test('must not run', () => { throw new Error('unassigned file ran') });\n" + } + writeFixture(t, projectRoot, filepath.Join("tests", name), content) + } + for _, name := range lifecycleFiles { + writeFixture(t, projectRoot, filepath.Join("tests", name), "const { test } = require('@playwright/test'); test('shared lifecycle', () => {});\n") + } + + baseCommand := []string{binary, "test", "--config", "apps/web/playwright.config.js"} + configureFramework(shellCommand(baseCommand...), "") + playwright := framework.NewPlaywright() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + files, err := playwright.DiscoverTestFiles(ctx, discovery.TestFileSet{}) + if err != nil { + t.Fatal(err) + } + want := []string{"apps/web/tests/a.spec.ts", "apps/web/tests/b.test.ts"} + requireFiles(t, files, want) + + projectCommand := append(append([]string{}, baseCommand...), "--project", "one") + configureFramework(shellCommand(projectCommand...), "") + projectPlaywright := framework.NewPlaywright() + if err := projectPlaywright.RunTests(ctx, []string{"apps/web/tests/a.spec.ts"}, nil); err != nil { + t.Fatalf("running one assigned file failed: %v", err) + } + if source, ok := playwright.SourceFileForSuite("a.spec.ts"); !ok || source != "apps/web/tests/a.spec.ts" { + t.Fatalf("SourceFileForSuite() = %q, %v", source, ok) + } + + emptyCommand := append(append([]string{}, baseCommand...), "__ddtest_no_match__") + configureFramework(shellCommand(emptyCommand...), "") + emptyPlaywright := framework.NewPlaywright() + if files, err := emptyPlaywright.DiscoverTestFiles(ctx, discovery.TestFileSet{}); err != nil || len(files) != 0 { + t.Fatalf("empty native discovery = %v, %v", files, err) + } + + writeFixture(t, projectRoot, "tests/broken.spec.ts", "throw new Error('collection exploded')\n") + brokenCommand := append(append([]string{}, baseCommand...), "broken.spec.ts") + configureFramework(shellCommand(brokenCommand...), "") + brokenPlaywright := framework.NewPlaywright() + if _, err := brokenPlaywright.DiscoverTestFiles(ctx, discovery.TestFileSet{}); err == nil { + t.Fatal("collection failure was accepted as an empty discovery") + } +} + +func playwrightVersionAtLeast(t *testing.T, binary string, wantedMajor, wantedMinor int) bool { + t.Helper() + output, err := exec.Command(binary, "--version").Output() // no-dd-sa:go-security/command-injection + if err != nil { + t.Fatalf("failed to read Playwright version: %v", err) + } + fields := strings.Fields(strings.TrimSpace(string(output))) + if len(fields) == 0 { + t.Fatalf("unexpected Playwright version output: %q", output) + } + parts := strings.Split(strings.TrimPrefix(fields[len(fields)-1], "v"), ".") + if len(parts) < 2 { + t.Fatalf("unexpected Playwright version output: %q", output) + } + major, majorErr := strconv.Atoi(parts[0]) + minor, minorErr := strconv.Atoi(parts[1]) + if majorErr != nil || minorErr != nil { + t.Fatalf("unexpected Playwright version output: %q", output) + } + return major > wantedMajor || major == wantedMajor && minor >= wantedMinor +} diff --git a/internal/compatibility/pytest_test.go b/internal/compatibility/pytest_test.go new file mode 100644 index 0000000..3a7a6a3 --- /dev/null +++ b/internal/compatibility/pytest_test.go @@ -0,0 +1,54 @@ +package compatibility + +import ( + "context" + "testing" + "time" + + "github.com/DataDog/ddtest/internal/discovery" + "github.com/DataDog/ddtest/internal/framework" +) + +func TestPyTestAdapterIntegration(t *testing.T) { + python := requireEnv(t, "DDTEST_PYTHON_BINARY") + resetSettingsAfterTest(t) + + root := t.TempDir() + writeFixture(t, root, "pytest.ini", `[pytest] +testpaths = checks +python_files = check_*.py +`) + writeFixture(t, root, "checks/check_selected.py", `import os + +def test_preserves_worker_environment(): + assert os.environ["DDTEST_PYTEST_WORKER"] == "selected" +`) + writeFixture(t, root, "checks/check_unselected.py", `def test_must_not_run(): + raise AssertionError("unselected file ran") +`) + t.Chdir(root) + + configureFramework(shellCommand(python, "-m", "pytest"), "") + pytest := framework.NewPytest() + pytest.SetPlatformEnv(map[string]string{"PYTEST_ADDOPTS": "--ddtrace"}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + testFiles := discovery.TestFileSet{Pattern: pytest.TestPattern()} + files, err := pytest.DiscoverTestFiles(ctx, testFiles) + if err != nil { + t.Fatal(err) + } + wantFiles := []string{"checks/check_selected.py", "checks/check_unselected.py"} + requireFiles(t, files, wantFiles) + + tests, err := pytest.DiscoverTests(ctx, testFiles) + if err != nil { + t.Fatalf("full discovery failed: %v", err) + } + requireTestSources(t, tests, wantFiles) + + if err := pytest.RunTests(ctx, []string{"checks/check_selected.py"}, map[string]string{"DDTEST_PYTEST_WORKER": "selected"}); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } +} diff --git a/internal/compatibility/rspec_test.go b/internal/compatibility/rspec_test.go new file mode 100644 index 0000000..3fe2fc5 --- /dev/null +++ b/internal/compatibility/rspec_test.go @@ -0,0 +1,60 @@ +package compatibility + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/DataDog/ddtest/internal/discovery" + "github.com/DataDog/ddtest/internal/framework" +) + +func TestRSpecAdapterIntegration(t *testing.T) { + rspecVersion := requireEnv(t, "DDTEST_RSPEC_VERSION") + datadogVersion := requireEnv(t, "DDTEST_DATADOG_CI_VERSION") + + root := t.TempDir() + writeFixture(t, root, "Gemfile", fmt.Sprintf(`source "https://rubygems.org" +gem "datadog-ci", %q +gem "rspec", %q +`, datadogVersion, rspecVersion)) + writeFixture(t, root, ".rspec", "--require spec_helper\n") + writeFixture(t, root, "spec/spec_helper.rb", "DDTEST_RSPEC_SETUP = true\n") + writeFixture(t, root, "spec/selected_spec.rb", `RSpec.describe "selected" do + it "preserves configuration while running an assigned file" do + expect(DDTEST_RSPEC_SETUP).to eq(true) + end +end +`) + writeFixture(t, root, "spec/unselected_spec.rb", `RSpec.describe "unselected" do + it "must not run" do + raise "unselected file ran" + end +end +`) + t.Chdir(root) + + rspec := framework.NewRSpec() + rspec.SetPlatformEnv(map[string]string{"RUBYOPT": "-rbundler/setup -rdatadog/ci/auto_instrument"}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + testFiles := discovery.TestFileSet{Pattern: rspec.TestPattern()} + files, err := rspec.DiscoverTestFiles(ctx, testFiles) + if err != nil { + t.Fatal(err) + } + wantFiles := []string{"spec/selected_spec.rb", "spec/unselected_spec.rb"} + requireFiles(t, files, wantFiles) + + tests, err := rspec.DiscoverTests(ctx, testFiles) + if err != nil { + t.Fatalf("full discovery failed: %v", err) + } + requireTestSources(t, tests, wantFiles) + + if err := rspec.RunTests(ctx, []string{"spec/selected_spec.rb"}, nil); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } +} diff --git a/internal/compatibility/vitest_test.go b/internal/compatibility/vitest_test.go new file mode 100644 index 0000000..6b9e831 --- /dev/null +++ b/internal/compatibility/vitest_test.go @@ -0,0 +1,59 @@ +package compatibility + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/DataDog/ddtest/internal/discovery" + "github.com/DataDog/ddtest/internal/framework" +) + +func TestVitestAdapterIntegration(t *testing.T) { + nodeModules := requireEnv(t, "DDTEST_VITEST_NODE_MODULES") + + root := t.TempDir() + if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { + t.Fatal(err) + } + writeFixture(t, root, "vitest.config.mjs", `export default { + test: { + include: ['checks/**/*.check.js'], + setupFiles: ['./setup.js'], + }, +} +`) + writeFixture(t, root, "setup.js", "globalThis.ddtestVitestSetup = true\n") + writeFixture(t, root, "checks/selected.check.js", `import { expect, test } from 'vitest' + +test('preserves config while running an assigned file', () => { + expect(globalThis.ddtestVitestSetup).toBe(true) + expect(process.env.DDTEST_VITEST_WORKER).toBe('selected') +}) +`) + writeFixture(t, root, "checks/unselected.check.js", `import { test } from 'vitest' + +test('must not run', () => { + throw new Error('unselected file ran') +}) +`) + t.Chdir(root) + + vitest := framework.NewVitest() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + testFiles := discovery.TestFileSet{Pattern: vitest.TestPattern()} + files, err := vitest.DiscoverTestFiles(ctx, testFiles) + if err != nil { + t.Fatal(err) + } + wantFiles := []string{"checks/selected.check.js", "checks/unselected.check.js"} + requireFiles(t, files, wantFiles) + + if err := vitest.RunTests(ctx, []string{"checks/selected.check.js"}, map[string]string{"DDTEST_VITEST_WORKER": "selected"}); err != nil { + t.Fatalf("selected-file run failed: %v", err) + } +} diff --git a/internal/framework/cucumber_test.go b/internal/framework/cucumber_test.go index 6242f75..0e56cc7 100644 --- a/internal/framework/cucumber_test.go +++ b/internal/framework/cucumber_test.go @@ -9,10 +9,8 @@ import ( "slices" "strings" "testing" - "time" "github.com/DataDog/ddtest/internal/discovery" - "github.com/DataDog/ddtest/internal/ext" "github.com/DataDog/ddtest/internal/settings" "github.com/spf13/viper" ) @@ -376,86 +374,3 @@ func TestParseCucumberMessagesRejectsMalformedLine(t *testing.T) { t.Fatalf("error = %v", err) } } - -func TestCucumberAdapterIntegration(t *testing.T) { - cucumberBinary := os.Getenv("DDTEST_CUCUMBER_BINARY") - if cucumberBinary == "" { - t.Skip("DDTEST_CUCUMBER_BINARY is not set") - } - nodeModules := os.Getenv("DDTEST_CUCUMBER_NODE_MODULES") - if nodeModules == "" { - t.Fatal("DDTEST_CUCUMBER_NODE_MODULES is not set") - } - cucumberVersion := os.Getenv("DDTEST_CUCUMBER_VERSION") - if cucumberVersion == "" { - t.Fatal("DDTEST_CUCUMBER_VERSION is not set") - } - - root := t.TempDir() - t.Chdir(root) - if err := os.Symlink(nodeModules, "node_modules"); err != nil { - t.Fatal(err) - } - cucumberConfig := `module.exports = { - default: { - tags: 'not @excluded', - require: ['features/support/**/*.js'] - } -} -` - if strings.HasPrefix(cucumberVersion, "7.") { - // Cucumber 7 profiles are CLI argument strings. Object-based profiles were - // introduced later and are silently treated as empty by Cucumber 7. - cucumberConfig = `module.exports = { - default: "--require 'features/support/**/*.js' --tags 'not @excluded'" -} -` - } - files := map[string]string{ - "cucumber.js": cucumberConfig, - "features/included.feature": `Feature: included - Scenario: selected by the default profile - Given a passing step -`, - "features/unassigned.feature": `Feature: unassigned - Scenario: must not run - Given a failing step -`, - "features/excluded.feature": `@excluded -Feature: excluded - Scenario: filtered by the default profile - Given a passing step -`, - "features/support/steps.js": `const { Given } = require('@cucumber/cucumber') -Given('a passing step', function () {}) -Given('a failing step', function () { throw new Error('unassigned file ran') }) -`, - } - for filename, content := range files { - if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filename, []byte(content), 0644); err != nil { - t.Fatal(err) - } - } - - cucumber := &Cucumber{ - executor: &ext.DefaultCommandExecutor{}, - commandOverride: []string{cucumberBinary}, - platformEnv: map[string]string{}, - } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - discovered, err := cucumber.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: cucumber.TestPattern()}) - if err != nil { - t.Fatal(err) - } - wantFiles := []string{"features/included.feature", "features/unassigned.feature"} - if !slices.Equal(discovered, wantFiles) { - t.Fatalf("discovered = %v", discovered) - } - if err := cucumber.RunTests(ctx, []string{"features/included.feature"}, nil); err != nil { - t.Fatalf("selected-file run failed: %v", err) - } -} diff --git a/internal/framework/cypress_test.go b/internal/framework/cypress_test.go index faee56e..ef5a68f 100644 --- a/internal/framework/cypress_test.go +++ b/internal/framework/cypress_test.go @@ -13,7 +13,6 @@ import ( "time" "github.com/DataDog/ddtest/internal/discovery" - "github.com/DataDog/ddtest/internal/ext" ) type cypressCommandExecutor struct { @@ -328,182 +327,6 @@ func TestCypressUnskippableMarker(t *testing.T) { } } -func TestCypressAdapterIntegration(t *testing.T) { - binary := os.Getenv("DDTEST_CYPRESS_BINARY") - if binary == "" { - t.Skip("DDTEST_CYPRESS_BINARY is not set") - } - nodeModules := os.Getenv("DDTEST_CYPRESS_NODE_MODULES") - if nodeModules == "" { - t.Fatal("DDTEST_CYPRESS_NODE_MODULES must be set with DDTEST_CYPRESS_BINARY") - } - - root := t.TempDir() - t.Chdir(root) - tests := []struct { - name string - projectName string - configFilename string - configExport string - specPattern any - files []string - symlinkName string - symlinkTarget string - want []string - }{ - { - name: "minimatch extglob", - projectName: "extglob", - configFilename: "cypress.config.ts", - configExport: "export default", - specPattern: "custom/**/*.@(spec|test).cy.ts", - files: []string{ - "custom/discovered.spec.cy.ts", - "custom/discovered.test.cy.ts", - "custom/not-discovered.cy.ts", - }, - want: []string{ - "extglob/custom/discovered.spec.cy.ts", - "extglob/custom/discovered.test.cy.ts", - }, - }, - { - name: "broad pattern excludes discovery wrapper", - projectName: "broad", - configFilename: "cypress.config.js", - configExport: "module.exports =", - specPattern: "**/*.ts", - files: []string{"specs/discovered.ts", "specs/not-discovered.js"}, - want: []string{"broad/specs/discovered.ts"}, - }, - { - name: "negated spec pattern subtracts matches", - projectName: "negated", - configFilename: "cypress.config.js", - configExport: "module.exports =", - specPattern: []string{"**/*.cy.ts", "!**/slow.cy.ts"}, - files: []string{"specs/fast.cy.ts", "specs/slow.cy.ts", "specs/helper.ts"}, - want: []string{"negated/specs/fast.cy.ts"}, - }, - { - name: "symlinked spec directory", - projectName: "symlinked", - configFilename: "cypress.config.js", - configExport: "module.exports =", - specPattern: "linked/**/*.cy.ts", - files: []string{"target/discovered.cy.ts"}, - symlinkName: "linked", - symlinkTarget: "target", - want: []string{"symlinked/linked/discovered.cy.ts"}, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - projectRoot := filepath.Join(root, test.projectName) - if err := os.MkdirAll(projectRoot, 0755); err != nil { - t.Fatal(err) - } - if err := os.Symlink(nodeModules, filepath.Join(projectRoot, "node_modules")); err != nil { - t.Fatal(err) - } - specPattern, err := json.Marshal(test.specPattern) - if err != nil { - t.Fatal(err) - } - config := test.configExport + ` { - e2e: { - supportFile: false, - async setupNodeEvents(_on, config) { - return { ...config, specPattern: ` + string(specPattern) + ` } - }, - }, -} -` - if err := os.WriteFile(filepath.Join(projectRoot, test.configFilename), []byte(config), 0644); err != nil { - t.Fatal(err) - } - for _, filename := range test.files { - writeCypressFixture(t, filepath.Join(test.projectName, filename)) - } - if test.symlinkName != "" { - if err := os.Symlink(test.symlinkTarget, filepath.Join(projectRoot, test.symlinkName)); err != nil { - t.Fatal(err) - } - } - cypress := &Cypress{ - executor: &ext.DefaultCommandExecutor{}, - commandOverride: []string{binary, "run", "--project", test.projectName}, - platformEnv: make(map[string]string), - } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - files, err := cypress.DiscoverTestFiles(ctx, discovery.TestFileSet{}) - cancel() - if err != nil { - t.Fatal(err) - } - if !slices.Equal(files, test.want) { - t.Fatalf("files = %v, want %v", files, test.want) - } - }) - } -} - -func TestCypressAdapterExecutionIntegration(t *testing.T) { - binary := requireCompatibilityEnv(t, "DDTEST_CYPRESS_BINARY") - nodeModules := requireCompatibilityEnv(t, "DDTEST_CYPRESS_NODE_MODULES") - - root := t.TempDir() - if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { - t.Fatal(err) - } - writeCompatibilityFixture(t, root, "cypress.config.js", `module.exports = { - video: false, - viewportWidth: 777, - e2e: { - supportFile: false, - specPattern: 'cypress/e2e/**/*.cy.js', - }, -} -`) - writeCompatibilityFixture(t, root, "cypress/e2e/selected.cy.js", `describe('selected', () => { - it('runs an assigned file', () => { - expect(true).to.equal(true) - expect(Cypress.config('viewportWidth')).to.equal(777) - }) -}) -`) - writeCompatibilityFixture(t, root, "cypress/e2e/unselected.cy.js", `describe('unselected', () => { - it('must not run', () => { - throw new Error('unselected file ran') - }) -}) -`) - t.Chdir(root) - - command := []string{binary, "run"} - if xvfb := os.Getenv("DDTEST_CYPRESS_XVFB"); xvfb != "" { - command = []string{xvfb, "-a", binary, "run"} - } - cypress := &Cypress{ - executor: &ext.DefaultCommandExecutor{}, - commandOverride: command, - platformEnv: map[string]string{}, - } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - - files, err := cypress.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: cypress.TestPattern()}) - if err != nil { - t.Fatal(err) - } - wantFiles := []string{"cypress/e2e/selected.cy.js", "cypress/e2e/unselected.cy.js"} - requireCompatibilityFiles(t, files, wantFiles) - - if err := cypress.RunTests(ctx, []string{"cypress/e2e/selected.cy.js"}, nil); err != nil { - t.Fatalf("selected-file run failed: %v", err) - } -} - func TestCypressDiscoverySymlinkCycle(t *testing.T) { node, err := exec.LookPath("node") if err != nil { diff --git a/internal/framework/jest_test.go b/internal/framework/jest_test.go index d0bb776..fe15bd1 100644 --- a/internal/framework/jest_test.go +++ b/internal/framework/jest_test.go @@ -8,10 +8,8 @@ import ( "slices" "strings" "testing" - "time" "github.com/DataDog/ddtest/internal/discovery" - "github.com/DataDog/ddtest/internal/ext" ) type jestCommandExecutor struct { @@ -503,44 +501,3 @@ func TestJest_RunTests_WithOverride(t *testing.T) { t.Errorf("expected args %v, got %v", expectedArgs, capturedArgs) } } - -func TestJestAdapterIntegration(t *testing.T) { - nodeModules := requireCompatibilityEnv(t, "DDTEST_JEST_NODE_MODULES") - - root := t.TempDir() - if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { - t.Fatal(err) - } - writeCompatibilityFixture(t, root, "jest.config.js", `module.exports = { - testMatch: ['/tests/**/*.test.js'], - setupFilesAfterEnv: ['/setup.js'], -} -`) - writeCompatibilityFixture(t, root, "setup.js", "globalThis.ddtestJestSetup = true\n") - writeCompatibilityFixture(t, root, "tests/selected.test.js", `test('preserves config while running an assigned file', () => { - expect(globalThis.ddtestJestSetup).toBe(true) - expect(process.env.DDTEST_JEST_WORKER).toBe('selected') -}) -`) - writeCompatibilityFixture(t, root, "tests/unselected.test.js", `test('must not run', () => { - throw new Error('unselected file ran') -}) -`) - t.Chdir(root) - - jest := &Jest{executor: &ext.DefaultCommandExecutor{}, platformEnv: map[string]string{}} - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - - testFiles := discovery.TestFileSet{Pattern: jest.TestPattern()} - files, err := jest.DiscoverTestFiles(ctx, testFiles) - if err != nil { - t.Fatal(err) - } - wantFiles := []string{"tests/selected.test.js", "tests/unselected.test.js"} - requireCompatibilityFiles(t, files, wantFiles) - - if err := jest.RunTests(ctx, []string{"tests/selected.test.js"}, map[string]string{"DDTEST_JEST_WORKER": "selected"}); err != nil { - t.Fatalf("selected-file run failed: %v", err) - } -} diff --git a/internal/framework/minitest_test.go b/internal/framework/minitest_test.go index 1fe4765..42201ad 100644 --- a/internal/framework/minitest_test.go +++ b/internal/framework/minitest_test.go @@ -3,16 +3,13 @@ package framework import ( "context" "encoding/json" - "fmt" "os" "os/exec" "path/filepath" "slices" "testing" - "time" "github.com/DataDog/ddtest/internal/discovery" - "github.com/DataDog/ddtest/internal/ext" "github.com/DataDog/ddtest/internal/settings" "github.com/DataDog/ddtest/internal/testoptimization" ) @@ -1748,69 +1745,3 @@ func TestMinitest_HasUnskippableMarker(t *testing.T) { t.Fatal("expected missing file to be treated as guarded") } } - -func TestMinitestAdapterIntegration(t *testing.T) { - minitestVersion := requireCompatibilityEnv(t, "DDTEST_MINITEST_VERSION") - datadogVersion := requireCompatibilityEnv(t, "DDTEST_DATADOG_CI_VERSION") - - root := t.TempDir() - writeCompatibilityFixture(t, root, "Gemfile", fmt.Sprintf(`source "https://rubygems.org" -gem "datadog-ci", %q -gem "minitest", %q -gem "rake", "13.2.1" -`, datadogVersion, minitestVersion)) - writeCompatibilityFixture(t, root, "Rakefile", `require "rake/testtask" - -Rake::TestTask.new(:test) do |test| - test.test_files = ENV["TEST_FILES"] ? ENV["TEST_FILES"].split : FileList["test/**/*_test.rb"] -end - -task default: :test -`) - writeCompatibilityFixture(t, root, "test/test_helper.rb", `gem "minitest", ENV.fetch("DDTEST_MINITEST_VERSION") -require "minitest/autorun" -DDTEST_MINITEST_SETUP = true -`) - writeCompatibilityFixture(t, root, "test/selected_test.rb", `require_relative "test_helper" - -class SelectedTest < Minitest::Test - def test_preserves_setup_while_running_an_assigned_file - assert DDTEST_MINITEST_SETUP - end -end -`) - writeCompatibilityFixture(t, root, "test/unselected_test.rb", `require_relative "test_helper" - -class UnselectedTest < Minitest::Test - def test_must_not_run - flunk "unselected file ran" - end -end -`) - t.Chdir(root) - - minitest := &Minitest{ - executor: &ext.DefaultCommandExecutor{}, - platformEnv: map[string]string{"RUBYOPT": "-rbundler/setup -rdatadog/ci/auto_instrument"}, - } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - - testFiles := discovery.TestFileSet{Pattern: minitest.TestPattern()} - files, err := minitest.DiscoverTestFiles(ctx, testFiles) - if err != nil { - t.Fatal(err) - } - wantFiles := []string{"test/selected_test.rb", "test/unselected_test.rb"} - requireCompatibilityFiles(t, files, wantFiles) - - tests, err := minitest.DiscoverTests(ctx, testFiles) - if err != nil { - t.Fatalf("full discovery failed: %v", err) - } - requireCompatibilityTestSources(t, tests, wantFiles) - - if err := minitest.RunTests(ctx, []string{"test/selected_test.rb"}, nil); err != nil { - t.Fatalf("selected-file run failed: %v", err) - } -} diff --git a/internal/framework/mocha_test.go b/internal/framework/mocha_test.go index 25349d8..ad17422 100644 --- a/internal/framework/mocha_test.go +++ b/internal/framework/mocha_test.go @@ -9,10 +9,8 @@ import ( "slices" "strings" "testing" - "time" "github.com/DataDog/ddtest/internal/discovery" - "github.com/DataDog/ddtest/internal/ext" ) type mochaCommandExecutor struct { @@ -216,86 +214,6 @@ func TestMochaUnskippableMarker(t *testing.T) { } } -func TestMochaAdapterIntegration(t *testing.T) { - nodeModules := os.Getenv("DDTEST_MOCHA_NODE_MODULES") - if nodeModules == "" { - t.Skip("DDTEST_MOCHA_NODE_MODULES is not set") - } - - root := t.TempDir() - if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { - t.Fatal(err) - } - writeMochaFixture(t, root, ".mocharc.json", `{"spec":["test/**/*.spec.js"],"file":["setup.js"]}`) - writeMochaFixture(t, root, "setup.js", "global.ddtestSetup = true\n") - writeMochaFixture(t, root, "test/selected.spec.js", `const assert = require("assert"); describe("selected", () => { it("uses setup", () => assert.equal(global.ddtestSetup, true)) })`) - writeMochaFixture(t, root, "test/unselected.spec.js", `describe("unselected", () => { it("must not run", () => { throw new Error("unselected file ran") }) })`) - t.Chdir(root) - - mocha := &Mocha{executor: &ext.DefaultCommandExecutor{}, platformEnv: make(map[string]string)} - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - files, err := mocha.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: mocha.TestPattern()}) - if err != nil { - t.Fatal(err) - } - want := []string{"test/selected.spec.js", "test/unselected.spec.js"} - if !slices.Equal(files, want) { - t.Fatalf("discovered files = %v, want %v", files, want) - } - if err := mocha.RunTests(ctx, []string{"test/selected.spec.js"}, nil); err != nil { - t.Fatalf("selected-file run failed: %v", err) - } - - if err := os.Remove(filepath.Join(root, ".mocharc.json")); err != nil { - t.Fatal(err) - } - files, err = mocha.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: mocha.TestPattern()}) - if err != nil { - t.Fatalf("default discovery failed: %v", err) - } - if !slices.Equal(files, want) { - t.Fatalf("default discovered files = %v, want %v", files, want) - } -} - -func TestMochaAdapterCustomLocationAndCommandIntegration(t *testing.T) { - nodeModules := os.Getenv("DDTEST_MOCHA_NODE_MODULES") - if nodeModules == "" { - t.Skip("DDTEST_MOCHA_NODE_MODULES is not set") - } - - root := t.TempDir() - mochaCommand := filepath.Join(nodeModules, ".bin", "mocha") - wrapper := filepath.Join(root, "mocha-wrapper.sh") - writeMochaFixture(t, root, "mocha-wrapper.sh", "#!/bin/sh\nexport DDTEST_MOCHA_WRAPPER=preserved\nexec \"$@\"\n") - if err := os.Chmod(wrapper, 0755); err != nil { - t.Fatal(err) - } - writeMochaFixture(t, root, ".mocharc.json", `{"spec":["test/**/*.spec.js"]}`) - writeMochaFixture(t, root, "spec/custom.spec.js", `const assert = require("assert"); describe("custom", () => { it("uses wrapper", () => assert.equal(process.env.DDTEST_MOCHA_WRAPPER, "preserved")) })`) - t.Chdir(root) - setTestsLocation(t, "spec/**/*.js") - - mocha := &Mocha{ - executor: &ext.DefaultCommandExecutor{}, - commandOverride: []string{wrapper, mochaCommand}, - platformEnv: make(map[string]string), - } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - files, err := mocha.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: mocha.TestPattern()}) - if err != nil { - t.Fatal(err) - } - if !slices.Equal(files, []string{"spec/custom.spec.js"}) { - t.Fatalf("discovered files = %v", files) - } - if err := mocha.RunTests(ctx, files, nil); err != nil { - t.Fatalf("custom-command run failed: %v", err) - } -} - func writeMochaFixture(t *testing.T, root, name, contents string) { t.Helper() path := filepath.Join(root, name) diff --git a/internal/framework/playwright_test.go b/internal/framework/playwright_test.go index ba0e52f..dfd8e80 100644 --- a/internal/framework/playwright_test.go +++ b/internal/framework/playwright_test.go @@ -6,16 +6,12 @@ import ( "errors" "fmt" "os" - "os/exec" "path/filepath" "slices" - "strconv" "strings" "testing" - "time" "github.com/DataDog/ddtest/internal/discovery" - "github.com/DataDog/ddtest/internal/ext" ) type playwrightCommandExecutor struct { @@ -335,133 +331,3 @@ func TestPlaywrightSourceFileForSuiteUsesConfigDirectory(t *testing.T) { t.Fatalf("SourceFileForSuite() = %q, %v", source, ok) } } - -func TestPlaywrightAdapterIntegration(t *testing.T) { - binary := os.Getenv("DDTEST_PLAYWRIGHT_BINARY") - nodeModules := os.Getenv("DDTEST_PLAYWRIGHT_NODE_MODULES") - if binary == "" || nodeModules == "" { - t.Skip("DDTEST_PLAYWRIGHT_BINARY and DDTEST_PLAYWRIGHT_NODE_MODULES are required") - } - root := t.TempDir() - t.Chdir(root) - projectRoot := filepath.Join(root, "apps", "web") - if err := os.MkdirAll(filepath.Join(projectRoot, "tests"), 0755); err != nil { - t.Fatal(err) - } - if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { - t.Fatal(err) - } - projects := `[{ name: 'one' }, { name: 'two' }]` - lifecycleFiles := []string{} - if playwrightVersionAtLeast(t, binary, 1, 31) { - projects = `[ - { name: 'setup', testMatch: '**/setup.spec.ts' }, - { name: 'one', testIgnore: /(?:setup|ignored)\.spec\.ts/, dependencies: ['setup'] }, - { name: 'two', testIgnore: /(?:setup|ignored)\.spec\.ts/, dependencies: ['setup'] }, - ]` - lifecycleFiles = []string{"setup.spec.ts"} - } - // Project teardown was added after project dependencies. Older versions - // ignore the teardown property and treat the named project as a normal - // project, so only exercise teardown filtering where Playwright supports it. - if playwrightVersionAtLeast(t, binary, 1, 38) { - projects = `[ - { name: 'setup', testMatch: '**/setup.spec.ts', teardown: 'teardown' }, - { name: 'teardown', testMatch: '**/teardown.spec.ts' }, - { name: 'one', testIgnore: /(?:setup|teardown|ignored)\.spec\.ts/, dependencies: ['setup'] }, - { name: 'two', testIgnore: /(?:setup|teardown|ignored)\.spec\.ts/, dependencies: ['setup'] }, - ]` - lifecycleFiles = []string{"setup.spec.ts", "teardown.spec.ts"} - } - config := fmt.Sprintf(`module.exports = { - testDir: './tests', - testMatch: '**/*.@(spec|test).ts', - testIgnore: '**/ignored.*', - projects: %s, -} -`, projects) - if err := os.WriteFile(filepath.Join(projectRoot, "playwright.config.js"), []byte(config), 0644); err != nil { - t.Fatal(err) - } - for _, name := range []string{"a.spec.ts", "b.test.ts", "ignored.spec.ts", "not-a-test.ts"} { - content := "const { test } = require('@playwright/test'); test('works', () => {});\n" - if name == "b.test.ts" { - content = "const { test } = require('@playwright/test'); test('must not run', () => { throw new Error('unassigned file ran') });\n" - } - if err := os.WriteFile(filepath.Join(projectRoot, "tests", name), []byte(content), 0644); err != nil { - t.Fatal(err) - } - } - for _, name := range lifecycleFiles { - content := "const { test } = require('@playwright/test'); test('shared lifecycle', () => {});\n" - if err := os.WriteFile(filepath.Join(projectRoot, "tests", name), []byte(content), 0644); err != nil { - t.Fatal(err) - } - } - playwright := &Playwright{ - executor: &ext.DefaultCommandExecutor{}, - commandOverride: []string{binary, "test", "--config", "apps/web/playwright.config.js"}, - platformEnv: map[string]string{}, - } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - files, err := playwright.DiscoverTestFiles(ctx, discovery.TestFileSet{}) - if err != nil { - t.Fatal(err) - } - want := []string{"apps/web/tests/a.spec.ts", "apps/web/tests/b.test.ts"} - if !slices.Equal(files, want) { - t.Fatalf("files = %v, want %v", files, want) - } - playwright.commandOverride = []string{binary, "test", "--config", "apps/web/playwright.config.js", "--project", "one"} - if err := playwright.RunTests(ctx, []string{"apps/web/tests/a.spec.ts"}, nil); err != nil { - t.Fatalf("running one assigned file failed: %v", err) - } - if source, ok := playwright.SourceFileForSuite("a.spec.ts"); !ok || source != "apps/web/tests/a.spec.ts" { - t.Fatalf("SourceFileForSuite() = %q, %v", source, ok) - } - - emptyPlaywright := &Playwright{ - executor: &ext.DefaultCommandExecutor{}, - commandOverride: []string{binary, "test", "--config", "apps/web/playwright.config.js", "__ddtest_no_match__"}, - platformEnv: map[string]string{}, - } - if files, err := emptyPlaywright.DiscoverTestFiles(ctx, discovery.TestFileSet{}); err != nil || len(files) != 0 { - t.Fatalf("empty native discovery = %v, %v", files, err) - } - - brokenFile := filepath.Join(projectRoot, "tests", "broken.spec.ts") - if err := os.WriteFile(brokenFile, []byte("throw new Error('collection exploded')\n"), 0644); err != nil { - t.Fatal(err) - } - brokenPlaywright := &Playwright{ - executor: &ext.DefaultCommandExecutor{}, - commandOverride: []string{binary, "test", "--config", "apps/web/playwright.config.js", "broken.spec.ts"}, - platformEnv: map[string]string{}, - } - if _, err := brokenPlaywright.DiscoverTestFiles(ctx, discovery.TestFileSet{}); err == nil { - t.Fatal("collection failure was accepted as an empty discovery") - } -} - -func playwrightVersionAtLeast(t *testing.T, binary string, wantedMajor, wantedMinor int) bool { - t.Helper() - output, err := exec.Command(binary, "--version").Output() // no-dd-sa:go-security/command-injection - if err != nil { - t.Fatalf("failed to read Playwright version: %v", err) - } - fields := strings.Fields(strings.TrimSpace(string(output))) - if len(fields) == 0 { - t.Fatalf("unexpected Playwright version output: %q", output) - } - parts := strings.Split(strings.TrimPrefix(fields[len(fields)-1], "v"), ".") - if len(parts) < 2 { - t.Fatalf("unexpected Playwright version output: %q", output) - } - major, majorErr := strconv.Atoi(parts[0]) - minor, minorErr := strconv.Atoi(parts[1]) - if majorErr != nil || minorErr != nil { - t.Fatalf("unexpected Playwright version output: %q", output) - } - return major > wantedMajor || major == wantedMajor && minor >= wantedMinor -} diff --git a/internal/framework/pytest_test.go b/internal/framework/pytest_test.go index 07018e0..1867903 100644 --- a/internal/framework/pytest_test.go +++ b/internal/framework/pytest_test.go @@ -7,10 +7,8 @@ import ( "path/filepath" "slices" "testing" - "time" "github.com/DataDog/ddtest/internal/discovery" - "github.com/DataDog/ddtest/internal/ext" "github.com/DataDog/ddtest/internal/testoptimization" ) @@ -322,48 +320,3 @@ func TestPyTest_RunTests(t *testing.T) { t.Errorf("expected run env to override platform env, got %q", mockExecutor.capturedEnvMap["SHARED_VAR"]) } } - -func TestPyTestAdapterIntegration(t *testing.T) { - python := requireCompatibilityEnv(t, "DDTEST_PYTHON_BINARY") - - root := t.TempDir() - writeCompatibilityFixture(t, root, "pytest.ini", `[pytest] -testpaths = checks -python_files = check_*.py -`) - writeCompatibilityFixture(t, root, "checks/check_selected.py", `import os - -def test_preserves_worker_environment(): - assert os.environ["DDTEST_PYTEST_WORKER"] == "selected" -`) - writeCompatibilityFixture(t, root, "checks/check_unselected.py", `def test_must_not_run(): - raise AssertionError("unselected file ran") -`) - t.Chdir(root) - - pytest := &PyTest{ - executor: &ext.DefaultCommandExecutor{}, - commandOverride: []string{python, "-m", "pytest"}, - platformEnv: map[string]string{"PYTEST_ADDOPTS": "--ddtrace"}, - } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - - testFiles := discovery.TestFileSet{Pattern: pytest.TestPattern()} - files, err := pytest.DiscoverTestFiles(ctx, testFiles) - if err != nil { - t.Fatal(err) - } - wantFiles := []string{"checks/check_selected.py", "checks/check_unselected.py"} - requireCompatibilityFiles(t, files, wantFiles) - - tests, err := pytest.DiscoverTests(ctx, testFiles) - if err != nil { - t.Fatalf("full discovery failed: %v", err) - } - requireCompatibilityTestSources(t, tests, wantFiles) - - if err := pytest.RunTests(ctx, []string{"checks/check_selected.py"}, map[string]string{"DDTEST_PYTEST_WORKER": "selected"}); err != nil { - t.Fatalf("selected-file run failed: %v", err) - } -} diff --git a/internal/framework/rspec_test.go b/internal/framework/rspec_test.go index bdc85ba..72f4156 100644 --- a/internal/framework/rspec_test.go +++ b/internal/framework/rspec_test.go @@ -3,13 +3,11 @@ package framework import ( "context" "encoding/json" - "fmt" "os" "os/exec" "path/filepath" "slices" "testing" - "time" "github.com/DataDog/ddtest/internal/discovery" "github.com/DataDog/ddtest/internal/ext" @@ -1428,54 +1426,3 @@ func TestRSpec_HasUnskippableMarker(t *testing.T) { t.Fatal("expected missing file to be treated as guarded") } } - -func TestRSpecAdapterIntegration(t *testing.T) { - rspecVersion := requireCompatibilityEnv(t, "DDTEST_RSPEC_VERSION") - datadogVersion := requireCompatibilityEnv(t, "DDTEST_DATADOG_CI_VERSION") - - root := t.TempDir() - writeCompatibilityFixture(t, root, "Gemfile", fmt.Sprintf(`source "https://rubygems.org" -gem "datadog-ci", %q -gem "rspec", %q -`, datadogVersion, rspecVersion)) - writeCompatibilityFixture(t, root, ".rspec", "--require spec_helper\n") - writeCompatibilityFixture(t, root, "spec/spec_helper.rb", "DDTEST_RSPEC_SETUP = true\n") - writeCompatibilityFixture(t, root, "spec/selected_spec.rb", `RSpec.describe "selected" do - it "preserves configuration while running an assigned file" do - expect(DDTEST_RSPEC_SETUP).to eq(true) - end -end -`) - writeCompatibilityFixture(t, root, "spec/unselected_spec.rb", `RSpec.describe "unselected" do - it "must not run" do - raise "unselected file ran" - end -end -`) - t.Chdir(root) - - rspec := &RSpec{ - executor: &ext.DefaultCommandExecutor{}, - platformEnv: map[string]string{"RUBYOPT": "-rbundler/setup -rdatadog/ci/auto_instrument"}, - } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - - testFiles := discovery.TestFileSet{Pattern: rspec.TestPattern()} - files, err := rspec.DiscoverTestFiles(ctx, testFiles) - if err != nil { - t.Fatal(err) - } - wantFiles := []string{"spec/selected_spec.rb", "spec/unselected_spec.rb"} - requireCompatibilityFiles(t, files, wantFiles) - - tests, err := rspec.DiscoverTests(ctx, testFiles) - if err != nil { - t.Fatalf("full discovery failed: %v", err) - } - requireCompatibilityTestSources(t, tests, wantFiles) - - if err := rspec.RunTests(ctx, []string{"spec/selected_spec.rb"}, nil); err != nil { - t.Fatalf("selected-file run failed: %v", err) - } -} diff --git a/internal/framework/vitest_test.go b/internal/framework/vitest_test.go index ec47a1c..5f7d438 100644 --- a/internal/framework/vitest_test.go +++ b/internal/framework/vitest_test.go @@ -9,10 +9,8 @@ import ( "slices" "strings" "testing" - "time" "github.com/DataDog/ddtest/internal/discovery" - "github.com/DataDog/ddtest/internal/ext" "github.com/DataDog/ddtest/internal/settings" ) @@ -527,50 +525,3 @@ func TestStripNodeOptionsImport(t *testing.T) { t.Fatalf("got %q, want %q", got, want) } } - -func TestVitestAdapterIntegration(t *testing.T) { - nodeModules := requireCompatibilityEnv(t, "DDTEST_VITEST_NODE_MODULES") - - root := t.TempDir() - if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { - t.Fatal(err) - } - writeCompatibilityFixture(t, root, "vitest.config.mjs", `export default { - test: { - include: ['checks/**/*.check.js'], - setupFiles: ['./setup.js'], - }, -} -`) - writeCompatibilityFixture(t, root, "setup.js", "globalThis.ddtestVitestSetup = true\n") - writeCompatibilityFixture(t, root, "checks/selected.check.js", `import { expect, test } from 'vitest' - -test('preserves config while running an assigned file', () => { - expect(globalThis.ddtestVitestSetup).toBe(true) - expect(process.env.DDTEST_VITEST_WORKER).toBe('selected') -}) -`) - writeCompatibilityFixture(t, root, "checks/unselected.check.js", `import { test } from 'vitest' - -test('must not run', () => { - throw new Error('unselected file ran') -}) -`) - t.Chdir(root) - - vitest := &Vitest{executor: &ext.DefaultCommandExecutor{}, platformEnv: map[string]string{}} - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - - testFiles := discovery.TestFileSet{Pattern: vitest.TestPattern()} - files, err := vitest.DiscoverTestFiles(ctx, testFiles) - if err != nil { - t.Fatal(err) - } - wantFiles := []string{"checks/selected.check.js", "checks/unselected.check.js"} - requireCompatibilityFiles(t, files, wantFiles) - - if err := vitest.RunTests(ctx, []string{"checks/selected.check.js"}, map[string]string{"DDTEST_VITEST_WORKER": "selected"}); err != nil { - t.Fatalf("selected-file run failed: %v", err) - } -} From 1e81b6a01044e232f6b71edf52537e87f3d7d4cc Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 9 Sep 2026 13:54:34 +0200 Subject: [PATCH 4/4] Cover Vitest custom config compatibility --- internal/compatibility/vitest_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/compatibility/vitest_test.go b/internal/compatibility/vitest_test.go index 6b9e831..d286884 100644 --- a/internal/compatibility/vitest_test.go +++ b/internal/compatibility/vitest_test.go @@ -13,12 +13,13 @@ import ( func TestVitestAdapterIntegration(t *testing.T) { nodeModules := requireEnv(t, "DDTEST_VITEST_NODE_MODULES") + resetSettingsAfterTest(t) root := t.TempDir() if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { t.Fatal(err) } - writeFixture(t, root, "vitest.config.mjs", `export default { + writeFixture(t, root, "vitest.unit.mjs", `export default { test: { include: ['checks/**/*.check.js'], setupFiles: ['./setup.js'], @@ -40,6 +41,7 @@ test('must not run', () => { }) `) t.Chdir(root) + configureFramework(shellCommand(filepath.Join(root, "node_modules", ".bin", "vitest"), "--config", "vitest.unit.mjs"), "") vitest := framework.NewVitest() ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)