Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .github/actions/setup-nodejs/safe-chain.config.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,13 @@
"minimumPackageAgeExclusions": [
"@n8n/*",
"@n8n_io/*",
"endform",
"endform-darwin-arm64",
"endform-darwin-x86",
"endform-linux-arm64",
"endform-linux-x86",
"endform-win32-arm64",
"endform-win32-x86",
"n8n",
"n8n-containers",
"n8n-core",
Expand Down
54 changes: 43 additions & 11 deletions .github/workflows/ci-pull-requests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,19 @@ concurrency:
jobs:
install-and-build:
name: Install & Build
runs-on: ${{ vars.RUNNER_PROVIDER == 'github' && 'ubuntu-latest' || 'blacksmith-4vcpu-ubuntu-2204' }}
runs-on: ${{ (github.repository == 'endformdev/n8n' || vars.RUNNER_PROVIDER == 'github') && 'ubuntu-latest' || 'blacksmith-4vcpu-ubuntu-2204' }}
env:
NODE_OPTIONS: '--max-old-space-size=7168'
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }}
QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }}
QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }}
outputs:
ci: ${{ fromJSON(steps.ci-filter.outputs.results).ci == true }}
ci: ${{ github.repository == 'n8n-io/n8n' && fromJSON(steps.ci-filter.outputs.results).ci == true }}
runtime: ${{ fromJSON(steps.ci-filter.outputs.results).runtime == true }}
unit: ${{ fromJSON(steps.ci-filter.outputs.results).unit == true }}
unit: ${{ github.repository == 'n8n-io/n8n' && fromJSON(steps.ci-filter.outputs.results).unit == true }}
e2e: ${{ fromJSON(steps.ci-filter.outputs.results).e2e == true }}
dev_server_smoke: ${{ fromJSON(steps.ci-filter.outputs.results)['dev-server-smoke'] == true }}
dev_server_smoke: ${{ github.repository == 'n8n-io/n8n' && fromJSON(steps.ci-filter.outputs.results)['dev-server-smoke'] == true }}
workflows: ${{ fromJSON(steps.ci-filter.outputs.results).workflows == true }}
workflow_scripts: ${{ fromJSON(steps.ci-filter.outputs.results)['workflow-scripts'] == true }}
db: ${{ fromJSON(steps.ci-filter.outputs.results).db == true }}
Expand Down Expand Up @@ -253,9 +253,8 @@ jobs:
run: |
pnpm -r pack --dry-run

# Seeds the SHA-keyed Docker image cache once so that downstream e2e jobs
# (each of which invokes prepare-docker internally) short-circuit to a
# cache hit instead of racing to rebuild.
# Seeds the SHA-keyed Docker image cache once for internal container-based E2E jobs.
# The Endform fork uses a local SQLite backend and does not consume this image.
prepare-docker:
name: Prepare Docker
needs: install-and-build
Expand All @@ -270,6 +269,7 @@ jobs:
uses: ./.github/workflows/prepare-docker-reusable.yml
with:
branch: ${{ needs.install-and-build.outputs.commit_sha }}
runner: ubuntu-latest
secrets: inherit

# Internal-only 1-spec fail-fast sanity check on sqlite.
Expand All @@ -293,13 +293,41 @@ jobs:
artifact-prefix: sanity
secrets: inherit

# Gives the Endform fork an early signal while the full impacted suite is still running.
endform-smoke:
name: 'Endform: Fast Signal'
needs: install-and-build
if: >-
github.repository == 'endformdev/n8n' &&
(needs.install-and-build.outputs.runtime == 'true'
|| needs.install-and-build.outputs.e2e == 'true'
|| contains(github.event.pull_request.labels.*.name, 'force-e2e')) &&
needs.install-and-build.outputs.skip_tests != 'true' &&
github.event_name != 'merge_group'
uses: ./.github/workflows/test-e2e-reusable.yml
with:
branch: ${{ needs.install-and-build.outputs.commit_sha }}
test-mode: local
test-command: >-
pnpm --filter=n8n-playwright exec endform test --organization-id=2G1ZCj7X --project=e2e
tests/e2e/building-blocks/workflow-entry-points.spec.ts
tests/e2e/credentials/crud.spec.ts
test-runner: endform
endform-concurrency: '4'
timeout-minutes: 30
artifact-prefix: endform-smoke
secrets: inherit

# Full e2e run. Internal PRs run multi-main (postgres + redis + caddy + 2 mains + 1 worker).
# Fork PRs run sqlite-only and skip @licensed tests (no enterprise license secrets on forks).
e2e:
name: E2E
needs: [install-and-build, prepare-docker]
if: >-
needs.prepare-docker.result == 'success' &&
always() &&
(needs.prepare-docker.result == 'success'
|| github.repository == 'endformdev/n8n'
|| github.event.pull_request.head.repo.fork == true) &&
(needs.install-and-build.outputs.runtime == 'true'
|| needs.install-and-build.outputs.e2e == 'true'
|| contains(github.event.pull_request.labels.*.name, 'force-e2e')) &&
Expand All @@ -308,9 +336,13 @@ jobs:
uses: ./.github/workflows/test-e2e-reusable.yml
with:
branch: ${{ needs.install-and-build.outputs.commit_sha }}
test-mode: docker-artifact
test-command: ${{ github.event.pull_request.head.repo.fork == true && 'pnpm --filter=n8n-playwright test:container:sqlite:e2e --grep-invert=@licensed' || 'pnpm --filter=n8n-playwright test:container:multi-main:e2e' }}
test-mode: ${{ (github.repository == 'endformdev/n8n' || github.event.pull_request.head.repo.fork == true) && 'local' || 'docker-artifact' }}
test-command: ${{ (github.repository == 'endformdev/n8n' || github.event.pull_request.head.repo.fork == true) && 'pnpm --filter=n8n-playwright exec endform test --organization-id=2G1ZCj7X --project=e2e' || 'pnpm --filter=n8n-playwright exec endform test --organization-id=2G1ZCj7X --project=multi-main:e2e' }}
test-runner: endform
endform-concurrency: ${{ (github.repository == 'endformdev/n8n' || github.event.pull_request.head.repo.fork == true) && '2' || '1' }}
endform-exclusive-specs: ${{ (github.repository == 'endformdev/n8n' || github.event.pull_request.head.repo.fork == true) && '(app-config/env-feature-flags|api/webhook-isolate-skip|credentials/global|dynamic-credentials/form-trigger-submit-gate-client|mcp/mcp-(oauth|service)|settings/environments/variables|settings/workers/workers|workflows/editor/tags)\.spec\.ts$' || '' }}
workers: '1'
timeout-minutes: ${{ (github.repository == 'endformdev/n8n' || github.event.pull_request.head.repo.fork == true) && 90 || 30 }}
pre-generated-matrix: ${{ needs.install-and-build.outputs.matrix }}
artifact-prefix: e2e
secrets: inherit
Expand Down Expand Up @@ -363,7 +395,7 @@ jobs:
security-checks:
name: Security Checks
needs: install-and-build
if: needs.install-and-build.outputs.workflows == 'true'
if: needs.install-and-build.outputs.workflows == 'true' && github.repository == 'n8n-io/n8n'
uses: ./.github/workflows/sec-ci-reusable.yml
with:
ref: ${{ needs.install-and-build.outputs.commit_sha }}
Expand Down
68 changes: 66 additions & 2 deletions .github/workflows/test-e2e-reusable.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,21 @@ on:
required: false
default: 'pnpm --filter=n8n-playwright test:local'
type: string
test-runner:
description: 'Test runner: playwright or endform'
required: false
default: 'playwright'
type: string
endform-concurrency:
description: 'Maximum Endform tests sharing one backend'
required: false
default: '1'
type: string
endform-exclusive-specs:
description: 'Regex matching specs that Endform must run sequentially after the parallel lane'
required: false
default: ''
type: string
workers:
description: 'Number of parallel workers'
required: false
Expand Down Expand Up @@ -73,7 +88,7 @@ env:

jobs:
test:
runs-on: ${{ vars.RUNNER_PROVIDER == 'github' && 'ubuntu-latest' || inputs.runner }}
runs-on: ${{ (github.repository == 'endformdev/n8n' || vars.RUNNER_PROVIDER == 'github') && 'ubuntu-latest' || inputs.runner }}
timeout-minutes: ${{ inputs.timeout-minutes }}
permissions:
contents: read
Expand All @@ -98,6 +113,7 @@ jobs:
QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }}

- name: Install Browsers
if: ${{ inputs.test-runner != 'endform' }}
run: pnpm turbo run install-browsers --filter=n8n-playwright

- name: Load n8n image from cache
Expand All @@ -107,7 +123,7 @@ jobs:
build-variant: ${{ inputs.build-variant }}

- name: Pre-pull Test Container Images
if: ${{ !contains(inputs.test-command, 'test:local') }}
if: ${{ inputs.test-mode == 'docker-artifact' }}
run: pnpm --filter n8n-containers exec tsx pull-test-images.ts ${{ matrix.images }} || true

- name: Filter specs to previous-attempt failures
Expand Down Expand Up @@ -149,6 +165,44 @@ jobs:
- name: Run Tests
if: steps.shard-args.outputs.skip != 'true'
run: |
if [ "$TEST_RUNNER" = 'endform' ]; then
if [ -n "$ENDFORM_EXCLUSIVE_SPECS" ]; then
if [[ "$SHARD_ARGS" == --shard=* ]]; then
echo "Endform exclusive specs require a pre-generated spec matrix."
exit 1
fi

read -r -a shard_args <<< "$SHARD_ARGS"
parallel_args=()
exclusive_args=()
for arg in "${shard_args[@]}"; do
if [[ "$arg" =~ $ENDFORM_EXCLUSIVE_SPECS ]]; then
exclusive_args+=("$arg")
else
parallel_args+=("$arg")
fi
done

if (( ${#parallel_args[@]} > 0 )); then
echo "Running ${#parallel_args[@]} spec files with Endform concurrency $ENDFORM_CONCURRENCY."
# shellcheck disable=SC2086
$TEST_COMMAND "${parallel_args[@]}"
fi

if (( ${#exclusive_args[@]} > 0 )); then
echo "Running ${#exclusive_args[@]} shared-state spec files sequentially."
ENDFORM_CONCURRENCY=1
# shellcheck disable=SC2086
$TEST_COMMAND "${exclusive_args[@]}"
fi
exit
fi

# shellcheck disable=SC2086
$TEST_COMMAND $SHARD_ARGS
exit
fi

# --pass-with-no-tests makes Playwright exit 0 when the selected specs
# resolve to zero runnable tests. Covers the impact-map vs current-suite
# skew window: a spec the committed coverage map points at may have been
Expand All @@ -161,10 +215,14 @@ jobs:
env:
# Protect args from template injections
TEST_COMMAND: ${{ inputs.test-command }}
TEST_RUNNER: ${{ inputs.test-runner }}
ENDFORM_CONCURRENCY: ${{ inputs.endform-concurrency }}
ENDFORM_EXCLUSIVE_SPECS: ${{ inputs.endform-exclusive-specs }}
# Uses pre-distributed specs if orchestration enabled, otherwise falls back to Playwright sharding
WORKERS: ${{ env.PLAYWRIGHT_WORKERS }}
SHARD_ARGS: ${{ steps.shard-args.outputs.args }}
# Args for actual test command runner
ENDFORM_API_KEY: ${{ inputs.test-runner == 'endform' && secrets.ENDFORM_API_KEY || '' }}
CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }}
QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }}
Expand All @@ -173,6 +231,12 @@ jobs:
N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }}
N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }}
N8N_TEST_ENV: ${{ inputs.n8n-env }}
# Endform runners are remote and cannot access the job's Docker socket. For the fork's
# SQLite suite, use Playwright's existing webServer and Endform's automatic proxying.
N8N_BASE_URL: ${{ inputs.test-runner == 'endform' && github.repository == 'endformdev/n8n' && 'http://localhost:5680' || '' }}
E2E_TESTS: ${{ inputs.test-runner == 'endform' && github.repository == 'endformdev/n8n' && 'true' || '' }}
E2E_REMOTE_RUNNER: ${{ inputs.test-runner == 'endform' && 'true' || '' }}
RESET_E2E_DB: ${{ inputs.test-runner == 'endform' && github.repository == 'endformdev/n8n' && 'true' || '' }}

- name: Upload Shard Artifacts
if: always()
Expand Down
10 changes: 10 additions & 0 deletions packages/testing/playwright/endform.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
/* eslint-disable import-x/no-default-export */
import { defineEndformConfig } from 'endform';

export default defineEndformConfig({
additionalFiles: ['workflows/**/*', 'fixtures/**/*', 'expectations/**/*', 'tests/**/*'],
concurrentTestLimits: [
{ scope: 'within-suite-run', limit: Number(process.env.ENDFORM_CONCURRENCY ?? 1) },
],
environmentVariables: ['E2E_REMOTE_RUNNER', 'N8N_BASE_URL', 'RESET_E2E_DB'],
});
19 changes: 17 additions & 2 deletions packages/testing/playwright/global-setup.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { request } from '@playwright/test';
import { setTimeout as wait } from 'node:timers/promises';

import { ApiHelpers } from './services/api-helper';
import { getBackendUrl } from './utils/url-helper';
Expand All @@ -20,14 +21,28 @@ async function globalSetup() {
}

console.log(`🔄 Resetting database for ${n8nBaseUrl}...`);
// Quick hack till we find out a better health check for the database reset command!
await new Promise((resolve) => setTimeout(resolve, 3000));
// Create standalone API request context
const requestContext = await request.newContext({
baseURL: n8nBaseUrl,
});

try {
const readinessDeadline = Date.now() + 30_000;
let e2eApiReady = false;
while (Date.now() < readinessDeadline) {
const response = await requestContext.get('/rest/e2e/env-feature-flags');
// The editor fallback returns HTML with 200 before REST controllers are mounted.
if (response.ok() && response.headers()['content-type']?.includes('application/json')) {
e2eApiReady = true;
break;
}
await wait(500);
}

if (!e2eApiReady) {
throw new Error('Timed out waiting for E2E API routes');
}

const api = new ApiHelpers(requestContext);
await api.resetDatabase();
console.log('✅ Database reset completed successfully');
Expand Down
1 change: 1 addition & 0 deletions packages/testing/playwright/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
"@types/jsonwebtoken": "catalog:",
"@types/lodash": "catalog:",
"autocannon": "^8.0.0",
"endform": "0.75.3",
"eslint-plugin-playwright": "catalog:e2e",
"flatted": "catalog:",
"generate-schema": "2.6.0",
Expand Down
8 changes: 7 additions & 1 deletion packages/testing/playwright/playwright-projects.ts
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,13 @@ export function getProjects(): Project[] {
testDir: './tests/e2e',
grepInvert: ALLOW_CONTAINER_ONLY ? undefined : CONTAINER_ONLY,
fullyParallel: true,
use: { baseURL: getFrontendUrl() },
timeout: process.env.E2E_REMOTE_RUNNER === 'true' ? 120_000 : undefined,
use: {
baseURL: getFrontendUrl(),
...(process.env.E2E_REMOTE_RUNNER === 'true'
? { actionTimeout: 30_000, navigationTimeout: 30_000 }
: {}),
},
});
projects.push({
name: 'dev-server-smoke',
Expand Down
4 changes: 2 additions & 2 deletions packages/testing/playwright/playwright.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ const SKIP_WEB_SERVER = process.env.PLAYWRIGHT_SKIP_WEBSERVER === 'true';

if (BACKEND_URL && !SKIP_WEB_SERVER) {
webServer.push({
command: 'cd .. && pnpm start',
url: `${BACKEND_URL}/favicon.ico`,
command: 'pnpm --filter=n8n start',
url: `${BACKEND_URL}/rest/e2e/env-feature-flags`,
timeout: 30000,
reuseExistingServer: IS_DEV ? false : true,
env: {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ const NODES = {
};

test.describe(
'Logs',
'Logs @capability:proxy',
{
annotation: [{ type: 'owner', description: 'Catalysts' }],
},
Expand Down
Loading
Loading