From c4509b0ab957c174f9555a0cf22ade6dd890e377 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Fri, 4 Sep 2026 06:01:46 +0100 Subject: [PATCH 1/2] dev-up: bound port release by an elapsed deadline, not an iteration count Stop-LoadedStack reaped both recorded trees cleanly and then failed the whole stop because Wait-PortRelease allowed only 50 x 100 ms. On a loaded hosted runner the kernel can hold the listening socket past that, so a correctly dead stack was reported "still occupied" and the launcher exited 1. Wait-PortRelease / wait_for_port_release now poll to a configurable elapsed deadline (TASKDECK_DEV_UP_PORT_RELEASE_TIMEOUT_MS, default 30000). When the deadline passes, the port's LISTEN owners are inventoried: any live listener still fails closed with the retained PID state, and a port no live process is listening on is treated as a lingering kernel socket behind our own confirmed-dead tree and counts as released. An owner inventory that cannot be read at all also fails closed. --- scripts/ci/dev-up.test.mjs | 119 +++++++++++++++++++++++++++++++++++++ scripts/dev-up.ps1 | 93 +++++++++++++++++++++++++++-- scripts/dev-up.sh | 68 +++++++++++++++++++-- 3 files changed, 269 insertions(+), 11 deletions(-) diff --git a/scripts/ci/dev-up.test.mjs b/scripts/ci/dev-up.test.mjs index f4aa95365..a2423b311 100644 --- a/scripts/ci/dev-up.test.mjs +++ b/scripts/ci/dev-up.test.mjs @@ -1745,6 +1745,125 @@ for (const platform of platforms) { } }) + // #1898/#2378: port release is bounded by an elapsed-time deadline, not a fixed iteration count. + // Both cases stop a synthetic state whose recorded PID is already gone, so the only thing under + // test is the port-release stage. + async function writeReapedState(fixture, { apiPort, frontendPort }) { + const runId = '22222222-2222-4222-8222-222222222222' + const deadProcess = spawn(process.execPath, ['-e', 'process.exit(0)'], { + stdio: 'ignore', + windowsHide: true, + }) + const deadPid = deadProcess.pid + await new Promise((resolve) => deadProcess.once('exit', resolve)) + await mkdir(fixture.stateDir, { recursive: true }) + const prefix = join(fixture.stateDir, `dev-up-${runId}`) + await writeFile( + fixture.stateFile, + `${JSON.stringify( + { + schemaVersion: 1, + runId, + apiPort, + frontend: { url: `http://localhost:${frontendPort}/`, port: frontendPort }, + logs: { + apiStdout: `${prefix}-api.stdout.log`, + apiStderr: `${prefix}-api.stderr.log`, + frontendStdout: `${prefix}-frontend.stdout.log`, + frontendStderr: `${prefix}-frontend.stderr.log`, + }, + processes: [ + { role: 'api', pid: deadPid, name: 'node', creationToken: 'reaped-before-this-stop' }, + { role: 'frontend', pid: deadPid, name: 'node', creationToken: 'reaped-before-this-stop' }, + ], + }, + null, + 2, + )}\n`, + ) + } + + test( + `${platform.name}: a frontend port released after the old fixed budget still stops cleanly`, + { concurrency: false }, + async () => { + const fixture = await createFixture(platform) + const apiPort = await getFreePort() + let frontendPort + do frontendPort = await getFreePort() + while (frontendPort === apiPort) + // Held out-of-process because runLauncher blocks this event loop: the holder releases the + // port 12s after the launcher starts, well past the 5s fixed budget the old Wait-PortRelease + // allowed (50 x 100ms), so this case fails on the pre-fix launchers and passes on these. + const holder = spawn( + process.execPath, + [ + '-e', + "const net=require('node:net');const s=net.createServer();s.listen(Number(process.argv[1]),'127.0.0.1',()=>{console.log('ready');setTimeout(()=>process.exit(0),12000)})", + String(frontendPort), + ], + { stdio: ['ignore', 'pipe', 'ignore'], windowsHide: true }, + ) + try { + await new Promise((resolve, reject) => { + holder.stdout.once('data', resolve) + holder.once('exit', () => reject(new Error('port holder exited before it was ready'))) + }) + assert.equal(await canBind(frontendPort, '127.0.0.1'), false, 'port holder did not occupy the port') + await writeReapedState(fixture, { apiPort, frontendPort }) + const result = runLauncher(platform, fixture, { + stop: true, + timeout: 40_000, + env: { TASKDECK_DEV_UP_PORT_RELEASE_TIMEOUT_MS: '20000' }, + }) + assert.ifError(result.error) + assert.equal(result.status, 0, combinedOutput(result)) + assert.match(combinedOutput(result), /Stack stopped/) + assert.equal(await readOptional(fixture.stateFile), null) + } finally { + holder.kill('SIGKILL') + if (existsSync(fixture.stateFile)) { + runLauncher(platform, fixture, { + stop: true, + env: { TASKDECK_DEV_UP_PORT_RELEASE_TIMEOUT_MS: '1500' }, + }) + } + await removeFixture(fixture) + } + }, + ) + + test( + `${platform.name}: a live listener still holding the frontend port fails closed`, + { concurrency: false }, + async () => { + const fixture = await createFixture(platform) + const apiPort = await getFreePort() + const foreign = await listenForeign('127.0.0.1') + const frontendPort = foreign.address().port + try { + await writeReapedState(fixture, { apiPort, frontendPort }) + const result = runLauncher(platform, fixture, { + stop: true, + timeout: 30_000, + env: { TASKDECK_DEV_UP_PORT_RELEASE_TIMEOUT_MS: '1500' }, + }) + assertFailedClosed(result) + assert.match(combinedOutput(result), /Frontend port .* is still occupied/) + // Either fail-closed reason is acceptable: a readable listener inventory names the live owner, + // and a platform without one (Git Bash on Windows has neither lsof nor ss) must still refuse. + assert.match(combinedOutput(result), /held by a live|could not be inventoried/) + assert.equal(existsSync(fixture.stateFile), true, 'PID state was dropped despite a live listener') + assert.equal(foreign.listening, true, 'the live listener was disturbed') + } finally { + // No cleanup stop here: the listener is still up on purpose, so another stop would only + // burn the deadline again. removeFixture discards the whole state directory. + await new Promise((resolve) => foreign.close(resolve)) + await removeFixture(fixture) + } + }, + ) + test(`${platform.name}: high-volume stdout and stderr cannot deadlock marker acceptance`, { concurrency: false }, async () => { const fixture = await createFixture(platform) const apiPort = await getFreePort() diff --git a/scripts/dev-up.ps1 b/scripts/dev-up.ps1 index 182284fef..e3ca563fe 100644 --- a/scripts/dev-up.ps1 +++ b/scripts/dev-up.ps1 @@ -58,6 +58,8 @@ $FrontendDir = Join-Path $RepoRoot "frontend/taskdeck-web" $DataDir = Join-Path $env:LOCALAPPDATA "Taskdeck" $DevDbPath = Join-Path $DataDir "taskdeck-dev.db" $PidFile = Join-Path $DataDir "dev-up.pids" +# Hosted-safe default; override with TASKDECK_DEV_UP_PORT_RELEASE_TIMEOUT_MS. +$DefaultPortReleaseTimeoutMs = 30000 $OperationLockFile = Join-Path $DataDir "dev-up.operation.lock" $MinimumNodeVersion = [version]"24.13.1" @@ -364,13 +366,87 @@ function Test-PortBindable { return $true } -function Wait-PortRelease { +function Get-PortReleaseTimeoutMs { + $raw = $env:TASKDECK_DEV_UP_PORT_RELEASE_TIMEOUT_MS + if (-not [string]::IsNullOrWhiteSpace($raw)) { + $parsed = 0 + if ([int]::TryParse($raw.Trim(), [ref]$parsed) -and $parsed -ge 0) { return $parsed } + Write-DevWarning "Ignoring invalid TASKDECK_DEV_UP_PORT_RELEASE_TIMEOUT_MS '$raw'; using $DefaultPortReleaseTimeoutMs ms." + } + return $DefaultPortReleaseTimeoutMs +} + +# Live PIDs currently LISTENING on $Port. Determined=$false means no inventory could be read at +# all; the caller must then fail closed rather than assume the port is free of a foreign server. +function Get-LivePortListenerOwner { param([int]$Port) - for ($attempt = 0; $attempt -lt 50; $attempt++) { + $owners = New-Object System.Collections.Generic.List[int] + $determined = $false + try { + $connections = @(Get-NetTCPConnection -State Listen -LocalPort $Port -ErrorAction Stop) + $determined = $true + foreach ($connection in $connections) { $owners.Add([int]$connection.OwningProcess) } + } catch [System.Management.Automation.CommandNotFoundException] { + $determined = $false + } catch { + # Get-NetTCPConnection throws ObjectNotFound when nothing matches the port filter. + if ($_.CategoryInfo.Category -eq [System.Management.Automation.ErrorCategory]::ObjectNotFound) { + $determined = $true + } + } + if (-not $determined) { + $netstat = Join-Path $env:SystemRoot "System32/netstat.exe" + if (Test-Path -LiteralPath $netstat -PathType Leaf) { + try { + $rows = & $netstat -ano -p TCP 2>$null + if ($LASTEXITCODE -eq 0) { + $determined = $true + foreach ($row in $rows) { + $fields = ($row -split '\s+') | Where-Object { $_ -ne "" } + if ($fields.Count -lt 5) { continue } + if ($fields[0] -ne "TCP" -or $fields[3] -ne "LISTENING") { continue } + if ($fields[1] -notmatch ":(\d+)$" -or [int]$Matches[1] -ne $Port) { continue } + $ownerPid = 0 + if ([int]::TryParse($fields[4], [ref]$ownerPid)) { $owners.Add($ownerPid) } + } + } + } catch { $determined = $false } + } + } + $live = @($owners | Sort-Object -Unique | Where-Object { + $_ -gt 0 -and $null -ne (Get-Process -Id $_ -ErrorAction SilentlyContinue) + }) + return [pscustomobject]@{ Determined = $determined; Owners = $live } +} + +# Waits on an elapsed-time deadline, not a fixed iteration count: a loaded hosted runner can hold a +# listening socket well past a short fixed budget after taskkill (#1898/#2378, same family as the +# #2157 marker budget). Once the deadline passes, a port that no LIVE process is listening on is a +# lingering kernel socket behind our own confirmed-dead tree, not a foreign owner, so it counts as +# released. Any live listener - foreign or a recorded PID that outlived its reap - fails closed. +function Wait-PortRelease { + param([int]$Port, [int[]]$OwnedPids = @()) + $timeoutMs = Get-PortReleaseTimeoutMs + $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() + while ($true) { if (Test-PortBindable -Port $Port) { return $true } - Start-Sleep -Milliseconds 100 + if ($stopwatch.ElapsedMilliseconds -ge $timeoutMs) { break } + Start-Sleep -Milliseconds 250 } - return $false + $elapsed = [int]$stopwatch.ElapsedMilliseconds + $listener = Get-LivePortListenerOwner -Port $Port + if (-not $listener.Determined) { + Write-DevWarning "Port $Port stayed unbindable for $elapsed ms and its listening owner could not be inventoried." + return $false + } + if ($listener.Owners.Count -gt 0) { + $foreign = @($listener.Owners | Where-Object { $OwnedPids -notcontains $_ }) + $kind = if ($foreign.Count -gt 0) { "unrelated" } else { "recorded" } + Write-DevWarning "Port $Port is held by a live $kind listener (PID $($listener.Owners -join ', ')) after $elapsed ms." + return $false + } + Write-Info "Port $Port stayed unbindable for $elapsed ms but no live process is listening on it; treating the lingering socket as released." + return $true } function Find-SafeApiPort { @@ -425,11 +501,16 @@ function Stop-LoadedStack { $clean = $true if (-not (Stop-RecordedProcess -Record $script:State.FrontendRecord)) { $clean = $false } if (-not (Stop-RecordedProcess -Record $script:State.ApiRecord)) { $clean = $false } - if ($clean -and -not (Wait-PortRelease -Port ([int]$script:State.ApiPort))) { + $ownedPids = @( + @($script:State.ApiRecord, $script:State.FrontendRecord) | + Where-Object { $null -ne $_ } | + ForEach-Object { [int]$_.Pid } + ) + if ($clean -and -not (Wait-PortRelease -Port ([int]$script:State.ApiPort) -OwnedPids $ownedPids)) { Write-DevWarning "API port $($script:State.ApiPort) is still occupied. No foreign listener was killed; PID state is retained." $clean = $false } - if ($clean -and $null -ne $script:State.Frontend -and -not (Wait-PortRelease -Port ([int]$script:State.Frontend.Port))) { + if ($clean -and $null -ne $script:State.Frontend -and -not (Wait-PortRelease -Port ([int]$script:State.Frontend.Port) -OwnedPids $ownedPids)) { Write-DevWarning "Frontend port $($script:State.Frontend.Port) is still occupied. No foreign listener was killed; PID state is retained." $clean = $false } diff --git a/scripts/dev-up.sh b/scripts/dev-up.sh index 809d21ef0..83381d32f 100755 --- a/scripts/dev-up.sh +++ b/scripts/dev-up.sh @@ -370,10 +370,67 @@ port_is_bindable() { ' "$port" >/dev/null 2>&1 } +DEFAULT_PORT_RELEASE_TIMEOUT_MS=30000 + +port_release_timeout_ms() { + local raw="${TASKDECK_DEV_UP_PORT_RELEASE_TIMEOUT_MS:-}" + if [[ -n "$raw" ]]; then + if [[ "$raw" =~ ^[0-9]+$ ]]; then printf '%s\n' "$raw"; return 0; fi + warn "Ignoring invalid TASKDECK_DEV_UP_PORT_RELEASE_TIMEOUT_MS '$raw'; using ${DEFAULT_PORT_RELEASE_TIMEOUT_MS} ms." + fi + printf '%s\n' "$DEFAULT_PORT_RELEASE_TIMEOUT_MS" +} + +# Prints the live PIDs LISTENING on $1, one per line. Exit 2 means no inventory could be read at +# all, so the caller must fail closed instead of assuming no foreign server owns the port. +live_port_listener_owners() { + local port="$1" raw="" pid + if command -v lsof >/dev/null 2>&1; then + raw="$(lsof -nP -iTCP:"$port" -sTCP:LISTEN -t 2>/dev/null || true)" + elif command -v ss >/dev/null 2>&1; then + raw="$(ss -ltnHp "sport = :$port" 2>/dev/null | grep -oE 'pid=[0-9]+' | cut -d= -f2 || true)" + else + return 2 + fi + for pid in $(printf '%s\n' "$raw" | sort -u); do + [[ "$pid" =~ ^[1-9][0-9]*$ ]] || continue + kill -0 "$pid" 2>/dev/null && printf '%s\n' "$pid" + done + return 0 +} + +# Elapsed-time deadline, not a fixed iteration count: a loaded runner can hold a listening socket +# past a short fixed budget after the tree is reaped (#1898/#2378). After the deadline, a port with +# no LIVE listener is a lingering kernel socket behind our own confirmed-dead tree and counts as +# released; any live listener - unrelated, or a recorded PID that outlived its reap - fails closed. wait_for_port_release() { - local port="$1" - for _ in {1..50}; do port_is_bindable "$port" && return 0; sleep 0.1; done - return 1 + local port="$1" owned="${2:-}" timeout_ms deadline owners_status=0 owners kind pid + timeout_ms="$(port_release_timeout_ms)" + local started_at deadline_s elapsed + started_at="$(date +%s)" + deadline_s=$(( started_at + (timeout_ms + 999) / 1000 )) + while true; do + port_is_bindable "$port" && return 0 + (( $(date +%s) >= deadline_s )) && break + sleep 0.25 + done + elapsed=$(( $(date +%s) - started_at )) + owners="$(live_port_listener_owners "$port")" || owners_status=$? + if [[ "$owners_status" -ne 0 ]]; then + warn "Port $port stayed unbindable for ${elapsed}s and its listening owner could not be inventoried." + return 1 + fi + owners="$(printf '%s\n' "$owners" | tr '\n' ' ' | sed 's/ */ /g; s/^ //; s/ $//')" + if [[ -n "$owners" ]]; then + kind="unrelated" + for pid in $owners; do + case " $owned " in *" $pid "*) kind="recorded" ;; *) kind="unrelated"; break ;; esac + done + warn "Port $port is held by a live $kind listener (PID $owners) after ${elapsed}s." + return 1 + fi + info "Port $port stayed unbindable for ${elapsed}s but no live process is listening on it; treating the lingering socket as released." + return 0 } find_safe_api_port() { @@ -552,11 +609,12 @@ stop_loaded_stack() { local clean=1 if ! stop_recorded_process frontend "$STATE_FRONTEND_PID" "$STATE_FRONTEND_NAME" "$STATE_FRONTEND_TOKEN"; then clean=0; fi if ! stop_recorded_process api "$STATE_API_PID" "$STATE_API_NAME" "$STATE_API_TOKEN"; then clean=0; fi - if [[ "$clean" -eq 1 ]] && ! wait_for_port_release "$STATE_API_PORT"; then + local owned_pids="$STATE_API_PID $STATE_FRONTEND_PID" + if [[ "$clean" -eq 1 ]] && ! wait_for_port_release "$STATE_API_PORT" "$owned_pids"; then warn "API port $STATE_API_PORT is still occupied. No foreign listener was killed; PID state is retained." clean=0 fi - if [[ "$clean" -eq 1 && -n "$STATE_FRONTEND_PORT" ]] && ! wait_for_port_release "$STATE_FRONTEND_PORT"; then + if [[ "$clean" -eq 1 && -n "$STATE_FRONTEND_PORT" ]] && ! wait_for_port_release "$STATE_FRONTEND_PORT" "$owned_pids"; then warn "Frontend port $STATE_FRONTEND_PORT is still occupied. No foreign listener was killed; PID state is retained." clean=0 fi From d8d53002a111ee18271a456c223a5549e4cccd60 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Fri, 4 Sep 2026 06:21:20 +0100 Subject: [PATCH 2/2] dev-up: decide port release on socket existence, not owner attribution An unprivileged owner lookup cannot attribute another user's listening socket: lsof lists nothing at all, and ss -p prints the row without users:(...). The empty owner list was read as "no live listener", so a root-owned foreign listener - docker-proxy, or a systemd unit on 5000/5173 - would have been classified as a lingering socket, the PID file removed, and the stop reported clean. Get-Process on another account's PID can fail the same way on Windows. Socket existence and PID attribution are now separate. Existence comes from a source that covers every account (ss -ltn, netstat -an on BSD, the kernel TCP table via Get-NetTCPConnection or netstat -ano); attribution is best effort and only decorates the diagnostic. Release is accepted only when the port becomes bindable, or when a readable inventory positively shows no listening socket. Also: drop the "recorded" / "unrelated" survivor label, which classified by bare PID and so contradicted the PID+name+token identity the kill path requires - the warning now reports PID and command name only; bound TASKDECK_DEV_UP_PORT_RELEASE_TIMEOUT_MS to int32 in the Bash launcher to match the PowerShell parse; and report elapsed time in ms in both launchers. --- scripts/ci/dev-up.test.mjs | 63 +++++++++++++++++++++++++ scripts/dev-up.ps1 | 76 ++++++++++++++++++------------ scripts/dev-up.sh | 94 +++++++++++++++++++++++++------------- 3 files changed, 172 insertions(+), 61 deletions(-) diff --git a/scripts/ci/dev-up.test.mjs b/scripts/ci/dev-up.test.mjs index a2423b311..9caa5d391 100644 --- a/scripts/ci/dev-up.test.mjs +++ b/scripts/ci/dev-up.test.mjs @@ -1864,6 +1864,69 @@ for (const platform of platforms) { }, ) + // An unprivileged inventory can see that a socket is listening without being able to name its + // owner: `ss -p` omits `users:(...)` for another account's socket, and `lsof` cannot see it at + // all. An unattributable owner must never be read as "nothing is listening" - otherwise a + // root-owned listener (docker-proxy, a systemd unit on 5000/5173) would be reported as a clean + // stop and the PID file removed. Stubs reproduce that exact shape. + if (platform.name === 'Bash') { + test( + `${platform.name}: a listening socket with no attributable owner still fails closed`, + { concurrency: false }, + async () => { + const fixture = await createFixture(platform) + const apiPort = await getFreePort() + const foreign = await listenForeign('127.0.0.1') + const frontendPort = foreign.address().port + try { + // Prints a LISTEN row for the frontend port only - and never a `pid=` field. + await writeFile( + join(fixture.fakeBin, 'ss'), + [ + '#!/usr/bin/env bash', + 'for arg in "$@"; do', + ' case "$arg" in', + ' *":$TASKDECK_TEST_UNATTRIBUTABLE_PORT")', + ' printf \'LISTEN 0 511 0.0.0.0:%s 0.0.0.0:*\\n\' "$TASKDECK_TEST_UNATTRIBUTABLE_PORT"', + ' exit 0 ;;', + ' esac', + 'done', + 'exit 0', + '', + ].join('\n'), + ) + await chmod(join(fixture.fakeBin, 'ss'), 0o755) + // Deny the lsof attribution fallback too, so the unattributable path is deterministic + // whether or not the host has a real lsof. + await writeFile(join(fixture.fakeBin, 'lsof'), '#!/usr/bin/env bash\nexit 1\n') + await chmod(join(fixture.fakeBin, 'lsof'), 0o755) + + await writeReapedState(fixture, { apiPort, frontendPort }) + const result = runLauncher(platform, fixture, { + stop: true, + timeout: 30_000, + env: { + TASKDECK_DEV_UP_PORT_RELEASE_TIMEOUT_MS: '1500', + TASKDECK_TEST_UNATTRIBUTABLE_PORT: String(frontendPort), + }, + }) + assertFailedClosed(result) + assert.match(combinedOutput(result), /still held by a live listener/) + assert.match(combinedOutput(result), /Frontend port .* is still occupied/) + assert.equal( + existsSync(fixture.stateFile), + true, + 'PID state was dropped for a listener that could not be attributed to a PID', + ) + assert.equal(foreign.listening, true, 'the unattributable listener was disturbed') + } finally { + await new Promise((resolve) => foreign.close(resolve)) + await removeFixture(fixture) + } + }, + ) + } + test(`${platform.name}: high-volume stdout and stderr cannot deadlock marker acceptance`, { concurrency: false }, async () => { const fixture = await createFixture(platform) const apiPort = await getFreePort() diff --git a/scripts/dev-up.ps1 b/scripts/dev-up.ps1 index e3ca563fe..b0b5b2973 100644 --- a/scripts/dev-up.ps1 +++ b/scripts/dev-up.ps1 @@ -376,16 +376,25 @@ function Get-PortReleaseTimeoutMs { return $DefaultPortReleaseTimeoutMs } -# Live PIDs currently LISTENING on $Port. Determined=$false means no inventory could be read at -# all; the caller must then fail closed rather than assume the port is free of a foreign server. -function Get-LivePortListenerOwner { +# Inventory of TCP LISTEN sockets on $Port. Determined=$false means nothing could be read at all, +# and the caller must fail closed rather than assume the port is free of a foreign server. +# +# Socket EXISTENCE and PID ATTRIBUTION are deliberately separate. Existence comes from the kernel's +# TCP table, which lists every listener regardless of the owning account; attribution (a process +# name for the diagnostic) can fail for a socket owned by another user or by SYSTEM, so an +# unattributable owner must never read as "nothing is listening". +function Get-PortListenerInventory { param([int]$Port) - $owners = New-Object System.Collections.Generic.List[int] + $owners = New-Object System.Collections.Generic.List[string] $determined = $false + $listening = $false try { $connections = @(Get-NetTCPConnection -State Listen -LocalPort $Port -ErrorAction Stop) $determined = $true - foreach ($connection in $connections) { $owners.Add([int]$connection.OwningProcess) } + foreach ($connection in $connections) { + $listening = $true + $owners.Add((Format-ListenerOwner -OwnerPid ([int]$connection.OwningProcess))) + } } catch [System.Management.Automation.CommandNotFoundException] { $determined = $false } catch { @@ -406,26 +415,38 @@ function Get-LivePortListenerOwner { if ($fields.Count -lt 5) { continue } if ($fields[0] -ne "TCP" -or $fields[3] -ne "LISTENING") { continue } if ($fields[1] -notmatch ":(\d+)$" -or [int]$Matches[1] -ne $Port) { continue } + $listening = $true $ownerPid = 0 - if ([int]::TryParse($fields[4], [ref]$ownerPid)) { $owners.Add($ownerPid) } + if ([int]::TryParse($fields[4], [ref]$ownerPid)) { + $owners.Add((Format-ListenerOwner -OwnerPid $ownerPid)) + } } } } catch { $determined = $false } } } - $live = @($owners | Sort-Object -Unique | Where-Object { - $_ -gt 0 -and $null -ne (Get-Process -Id $_ -ErrorAction SilentlyContinue) - }) - return [pscustomobject]@{ Determined = $determined; Owners = $live } + return [pscustomobject]@{ + Determined = $determined + Listening = $listening + Owners = @($owners | Sort-Object -Unique) + } +} + +function Format-ListenerOwner { + param([int]$OwnerPid) + if ($OwnerPid -le 0) { return "PID $OwnerPid" } + $owner = Get-Process -Id $OwnerPid -ErrorAction SilentlyContinue + if ($null -ne $owner) { return "PID $OwnerPid ($($owner.ProcessName))" } + return "PID $OwnerPid" } # Waits on an elapsed-time deadline, not a fixed iteration count: a loaded hosted runner can hold a # listening socket well past a short fixed budget after taskkill (#1898/#2378, same family as the -# #2157 marker budget). Once the deadline passes, a port that no LIVE process is listening on is a -# lingering kernel socket behind our own confirmed-dead tree, not a foreign owner, so it counts as -# released. Any live listener - foreign or a recorded PID that outlived its reap - fails closed. +# #2157 marker budget). Once the deadline passes, a port with no listening socket at all is a +# lingering kernel socket behind our own confirmed-dead tree, so it counts as released. Any +# surviving listener - and any port whose sockets cannot be inventoried - fails closed. function Wait-PortRelease { - param([int]$Port, [int[]]$OwnedPids = @()) + param([int]$Port) $timeoutMs = Get-PortReleaseTimeoutMs $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() while ($true) { @@ -434,18 +455,20 @@ function Wait-PortRelease { Start-Sleep -Milliseconds 250 } $elapsed = [int]$stopwatch.ElapsedMilliseconds - $listener = Get-LivePortListenerOwner -Port $Port - if (-not $listener.Determined) { - Write-DevWarning "Port $Port stayed unbindable for $elapsed ms and its listening owner could not be inventoried." + $inventory = Get-PortListenerInventory -Port $Port + if (-not $inventory.Determined) { + Write-DevWarning "Port $Port stayed unbindable for $elapsed ms and its listening sockets could not be inventoried." return $false } - if ($listener.Owners.Count -gt 0) { - $foreign = @($listener.Owners | Where-Object { $OwnedPids -notcontains $_ }) - $kind = if ($foreign.Count -gt 0) { "unrelated" } else { "recorded" } - Write-DevWarning "Port $Port is held by a live $kind listener (PID $($listener.Owners -join ', ')) after $elapsed ms." + if ($inventory.Listening) { + if ($inventory.Owners.Count -gt 0) { + Write-DevWarning "Port $Port is still held by a live listener ($($inventory.Owners -join ', ')) after $elapsed ms." + } else { + Write-DevWarning "Port $Port is still held by a live listener that could not be attributed to a process after $elapsed ms." + } return $false } - Write-Info "Port $Port stayed unbindable for $elapsed ms but no live process is listening on it; treating the lingering socket as released." + Write-Info "Port $Port stayed unbindable for $elapsed ms but nothing is listening on it; treating the lingering socket as released." return $true } @@ -501,16 +524,11 @@ function Stop-LoadedStack { $clean = $true if (-not (Stop-RecordedProcess -Record $script:State.FrontendRecord)) { $clean = $false } if (-not (Stop-RecordedProcess -Record $script:State.ApiRecord)) { $clean = $false } - $ownedPids = @( - @($script:State.ApiRecord, $script:State.FrontendRecord) | - Where-Object { $null -ne $_ } | - ForEach-Object { [int]$_.Pid } - ) - if ($clean -and -not (Wait-PortRelease -Port ([int]$script:State.ApiPort) -OwnedPids $ownedPids)) { + if ($clean -and -not (Wait-PortRelease -Port ([int]$script:State.ApiPort))) { Write-DevWarning "API port $($script:State.ApiPort) is still occupied. No foreign listener was killed; PID state is retained." $clean = $false } - if ($clean -and $null -ne $script:State.Frontend -and -not (Wait-PortRelease -Port ([int]$script:State.Frontend.Port) -OwnedPids $ownedPids)) { + if ($clean -and $null -ne $script:State.Frontend -and -not (Wait-PortRelease -Port ([int]$script:State.Frontend.Port))) { Write-DevWarning "Frontend port $($script:State.Frontend.Port) is still occupied. No foreign listener was killed; PID state is retained." $clean = $false } diff --git a/scripts/dev-up.sh b/scripts/dev-up.sh index 83381d32f..39c908c37 100755 --- a/scripts/dev-up.sh +++ b/scripts/dev-up.sh @@ -375,24 +375,53 @@ DEFAULT_PORT_RELEASE_TIMEOUT_MS=30000 port_release_timeout_ms() { local raw="${TASKDECK_DEV_UP_PORT_RELEASE_TIMEOUT_MS:-}" if [[ -n "$raw" ]]; then - if [[ "$raw" =~ ^[0-9]+$ ]]; then printf '%s\n' "$raw"; return 0; fi + # Bounded to int32 so the deadline arithmetic matches the PowerShell launcher's [int] parse and + # cannot wrap on a 64-bit shell. + if [[ "$raw" =~ ^[0-9]{1,10}$ ]] && (( 10#$raw <= 2147483647 )); then + printf '%s\n' "$(( 10#$raw ))" + return 0 + fi warn "Ignoring invalid TASKDECK_DEV_UP_PORT_RELEASE_TIMEOUT_MS '$raw'; using ${DEFAULT_PORT_RELEASE_TIMEOUT_MS} ms." fi printf '%s\n' "$DEFAULT_PORT_RELEASE_TIMEOUT_MS" } -# Prints the live PIDs LISTENING on $1, one per line. Exit 2 means no inventory could be read at -# all, so the caller must fail closed instead of assuming no foreign server owns the port. -live_port_listener_owners() { - local port="$1" raw="" pid - if command -v lsof >/dev/null 2>&1; then - raw="$(lsof -nP -iTCP:"$port" -sTCP:LISTEN -t 2>/dev/null || true)" - elif command -v ss >/dev/null 2>&1; then - raw="$(ss -ltnHp "sport = :$port" 2>/dev/null | grep -oE 'pid=[0-9]+' | cut -d= -f2 || true)" +now_ms() { + local raw + raw="$(date +%s%3N 2>/dev/null || true)" + if [[ "$raw" =~ ^[0-9]+$ ]]; then printf '%s\n' "$raw"; return 0; fi + printf '%s000\n' "$(date +%s)" +} + +# Inventory of TCP LISTEN sockets on $1. First stdout line is "listening" or "free"; any further +# lines are attributable owner PIDs. Exit 2 means nothing could be read at all. +# +# Socket EXISTENCE and PID ATTRIBUTION are deliberately separate. An unprivileged `lsof -t` cannot +# see another user's socket at all, and `ss -p` prints the row but omits `users:(...)`, so an empty +# owner list is NOT evidence that the port is free - a root-owned listener (docker-proxy, a systemd +# unit on 5000/5173) would look ownerless. Existence therefore comes from a source that reads +# /proc/net/tcp for every user (`ss -ltn`, or `netstat -an` on BSD/macOS); attribution is +# best-effort and only decorates the diagnostic. +port_listener_inventory() { + local port="$1" rows="" all="" pids="" pid + if command -v ss >/dev/null 2>&1; then + rows="$(ss -ltnH "sport = :$port" 2>/dev/null)" || return 2 + elif command -v netstat >/dev/null 2>&1; then + all="$(netstat -an -p tcp 2>/dev/null)" || return 2 + rows="$(printf '%s\n' "$all" | grep -E "[:.]${port}[[:space:]]" | grep -i 'LISTEN' || true)" else return 2 fi - for pid in $(printf '%s\n' "$raw" | sort -u); do + rows="$(printf '%s' "$rows" | tr -d '[:space:]')" + if [[ -z "$rows" ]]; then printf 'free\n'; return 0; fi + printf 'listening\n' + if command -v ss >/dev/null 2>&1; then + pids="$(ss -ltnHp "sport = :$port" 2>/dev/null | grep -oE 'pid=[0-9]+' | cut -d= -f2 || true)" + fi + if [[ -z "$pids" ]] && command -v lsof >/dev/null 2>&1; then + pids="$(lsof -nP -iTCP:"$port" -sTCP:LISTEN -t 2>/dev/null || true)" + fi + for pid in $(printf '%s\n' "$pids" | sort -u); do [[ "$pid" =~ ^[1-9][0-9]*$ ]] || continue kill -0 "$pid" 2>/dev/null && printf '%s\n' "$pid" done @@ -401,35 +430,37 @@ live_port_listener_owners() { # Elapsed-time deadline, not a fixed iteration count: a loaded runner can hold a listening socket # past a short fixed budget after the tree is reaped (#1898/#2378). After the deadline, a port with -# no LIVE listener is a lingering kernel socket behind our own confirmed-dead tree and counts as -# released; any live listener - unrelated, or a recorded PID that outlived its reap - fails closed. +# no listening socket at all is a lingering kernel socket behind our own confirmed-dead tree and +# counts as released. Any surviving listener - and any port whose sockets cannot be inventoried - +# fails closed. wait_for_port_release() { - local port="$1" owned="${2:-}" timeout_ms deadline owners_status=0 owners kind pid + local port="$1" timeout_ms status=0 inventory state owners timeout_ms="$(port_release_timeout_ms)" - local started_at deadline_s elapsed - started_at="$(date +%s)" - deadline_s=$(( started_at + (timeout_ms + 999) / 1000 )) + local started_at deadline elapsed + started_at="$(now_ms)" + deadline=$(( started_at + timeout_ms )) while true; do port_is_bindable "$port" && return 0 - (( $(date +%s) >= deadline_s )) && break + (( $(now_ms) >= deadline )) && break sleep 0.25 done - elapsed=$(( $(date +%s) - started_at )) - owners="$(live_port_listener_owners "$port")" || owners_status=$? - if [[ "$owners_status" -ne 0 ]]; then - warn "Port $port stayed unbindable for ${elapsed}s and its listening owner could not be inventoried." + elapsed=$(( $(now_ms) - started_at )) + inventory="$(port_listener_inventory "$port")" || status=$? + if [[ "$status" -ne 0 ]]; then + warn "Port $port stayed unbindable for ${elapsed} ms and its listening sockets could not be inventoried." return 1 fi - owners="$(printf '%s\n' "$owners" | tr '\n' ' ' | sed 's/ */ /g; s/^ //; s/ $//')" - if [[ -n "$owners" ]]; then - kind="unrelated" - for pid in $owners; do - case " $owned " in *" $pid "*) kind="recorded" ;; *) kind="unrelated"; break ;; esac - done - warn "Port $port is held by a live $kind listener (PID $owners) after ${elapsed}s." + state="$(printf '%s\n' "$inventory" | head -n 1)" + owners="$(printf '%s\n' "$inventory" | tail -n +2 | tr '\n' ' ' | sed 's/ */ /g; s/^ //; s/ $//')" + if [[ "$state" != "free" ]]; then + if [[ -n "$owners" ]]; then + warn "Port $port is still held by a live listener (PID $owners) after ${elapsed} ms." + else + warn "Port $port is still held by a live listener that could not be attributed to a PID after ${elapsed} ms." + fi return 1 fi - info "Port $port stayed unbindable for ${elapsed}s but no live process is listening on it; treating the lingering socket as released." + info "Port $port stayed unbindable for ${elapsed} ms but nothing is listening on it; treating the lingering socket as released." return 0 } @@ -609,12 +640,11 @@ stop_loaded_stack() { local clean=1 if ! stop_recorded_process frontend "$STATE_FRONTEND_PID" "$STATE_FRONTEND_NAME" "$STATE_FRONTEND_TOKEN"; then clean=0; fi if ! stop_recorded_process api "$STATE_API_PID" "$STATE_API_NAME" "$STATE_API_TOKEN"; then clean=0; fi - local owned_pids="$STATE_API_PID $STATE_FRONTEND_PID" - if [[ "$clean" -eq 1 ]] && ! wait_for_port_release "$STATE_API_PORT" "$owned_pids"; then + if [[ "$clean" -eq 1 ]] && ! wait_for_port_release "$STATE_API_PORT"; then warn "API port $STATE_API_PORT is still occupied. No foreign listener was killed; PID state is retained." clean=0 fi - if [[ "$clean" -eq 1 && -n "$STATE_FRONTEND_PORT" ]] && ! wait_for_port_release "$STATE_FRONTEND_PORT" "$owned_pids"; then + if [[ "$clean" -eq 1 && -n "$STATE_FRONTEND_PORT" ]] && ! wait_for_port_release "$STATE_FRONTEND_PORT"; then warn "Frontend port $STATE_FRONTEND_PORT is still occupied. No foreign listener was killed; PID state is retained." clean=0 fi